BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / chain.py

chain.py

chain.py 8.2 KiB Download Raw
1"""Keten-volgen vanaf zaad-posts, in de geest van hfydl.
2
3Vanaf een startpost (wiki-eerste deel, of anders een 'Part 1'-zaad) lopen
4we de next-links af. Een kandidaat blijft alleen in de reeks als de titel
5genoeg lijkt op de tot dan toe gevolgde titels (cosine ≥ temperature).
6Expliciete 'next'-links van dezelfde auteur gaan voor op overige comment-links.
7Ontbrekende posts kunnen live worden opgehaald (Arctic Shift, niet Reddit /.json).
8We slaan geen verhaaltekst op.
9"""
10
11from __future__ import annotations
12
13import re
14from typing import Callable, Iterable
15
16from .cluster import Novel, Post, filter_story_posts, is_oneshot
17from .links import Link
18from .normalize import is_series_start, normalize_title
19from .similarity import DEFAULT_TEMPERATURE, Embedder, LexicalEmbedder, cosine, guess_title, mean_vec
20
21Fetcher = Callable[[list[str]], list[Post]]
22
23
24def _ensure(pid: str, by_id: dict[str, Post], fetcher: Fetcher | None) -> Post | None:
25 if pid in by_id:
26 return by_id[pid]
27 if not fetcher:
28 return None
29 for post in fetcher([pid]) or []:
30 if post:
31 by_id[post.id] = post
32 return by_id.get(pid)
33
34
35_START_TITLE = re.compile(
36 r"""
37 (?:^|[\s:\-–—|\[\(])
38 (?:
39 (?:part|chapter|ch|pt|episode|ep)\.?\s*(?:1|i|one|first)\b
40 |
41 \#\s*1\b
42 |
43 \(\s*1\s*(?:/\s*(?:\d+|\?))?\s*\)
44 |
45 prologue
46 )
47 """,
48 re.IGNORECASE | re.VERBOSE,
49)
50
51
52def is_seed(post: Post) -> bool:
53 if is_oneshot(post.flair, 2): # oneshot-flair, ongeacht teller
54 # Een 'first of series' telt wél als zaad, ook als de flair gek is.
55 return is_series_start(post.flair)
56 if is_series_start(post.flair):
57 return True
58 return bool(_START_TITLE.search(post.title))
59
60
61def _pick_next(
62 current: Post,
63 by_id: dict[str, Post],
64 visited: set[str],
65 embedder: Embedder,
66 title_vecs: list[list[float]],
67 temperature: float,
68 fetcher: Fetcher | None = None,
69) -> Post | None:
70 avg = mean_vec(title_vecs) if title_vecs else None
71 links = list(current.links) if current.links else [
72 Link(pid, "", "next") for pid in current.next_ids
73 ]
74 # Expliciete next eerst, daarna overige links in documentvolgorde (hfydl
75 # neemt de eerste die de drempel haalt; wij geven next alleen voorrang).
76 links = sorted(links, key=lambda L: (0 if L.kind == "next" else 1))
77 for link in links:
78 if link.kind in {"prev", "first"}:
79 continue
80 if link.id in visited:
81 continue
82 target = _ensure(link.id, by_id, fetcher)
83 if not target:
84 continue
85 if current.author.lower() != target.author.lower() and link.kind != "next":
86 continue
87 if avg is not None:
88 vec = embedder.encode([target.title])[0]
89 score = cosine(vec, avg)
90 # Expliciete next van dezelfde auteur mag iets lager scoren, maar
91 # een heel ander verhaal (ver onder de drempel) slaan we over.
92 if score < temperature and not (
93 link.kind == "next" and score >= temperature * 0.6 and current.author.lower() == target.author.lower()
94 ):
95 if link.kind != "next" or score < temperature * 0.4:
96 continue
97 if current.author.lower() != target.author.lower():
98 continue
99 return target
100 return None
101
102
103def follow_chain(
104 seed: Post,
105 by_id: dict[str, Post],
106 embedder: Embedder | None = None,
107 temperature: float = DEFAULT_TEMPERATURE,
108 max_len: int = 800,
109 fetcher: Fetcher | None = None,
110) -> list[Post]:
111 embedder = embedder or LexicalEmbedder()
112 chain: list[Post] = []
113 visited: set[str] = set()
114 current: Post | None = seed
115 title_vecs: list[list[float]] = []
116 while current and current.id not in visited and len(chain) < max_len:
117 visited.add(current.id)
118 chain.append(current)
119 title_vecs.append(embedder.encode([current.title])[0])
120 current = _pick_next(current, by_id, visited, embedder, title_vecs, temperature, fetcher=fetcher)
121 return chain
122
123
124def find_seeds(posts: Iterable[Post]) -> list[Post]:
125 items = list(posts)
126 seeds = [p for p in items if is_seed(p)]
127 seeds.sort(key=lambda p: (p.created_utc, p.id))
128 return seeds
129
130
131def cluster_from_seeds(
132 posts: Iterable[Post],
133 embedder: Embedder | None = None,
134 temperature: float = DEFAULT_TEMPERATURE,
135) -> list[Novel]:
136 """Eerst ketens vanaf zaden, daarna de rest groeperen op auteur+titel+links."""
137 from .cluster import cluster_posts, merge_novels
138
139 embedder = embedder or LexicalEmbedder()
140 items = filter_story_posts(posts)
141 by_id = {p.id: p for p in items}
142 claimed: set[str] = set()
143 novels: list[Novel] = []
144
145 for seed in find_seeds(items):
146 if seed.id in claimed:
147 continue
148 chain = follow_chain(seed, by_id, embedder=embedder, temperature=temperature)
149 if not chain:
150 continue
151 for p in chain:
152 claimed.add(p.id)
153 novels.append(_novel_from_posts(chain))
154
155 leftover = [p for p in items if p.id not in claimed]
156 if leftover:
157 # cluster_posts groepeert op auteur/titel en next-links; daarna
158 # plakken we een restgroep aan een bestaande keten als de sleutel past.
159 rest = cluster_posts(leftover)
160 by_key: dict[tuple[str, str], Novel] = {}
161 for n in novels:
162 key = (n.author.lower(), normalize_title(n.name) or n.id)
163 by_key[key] = n
164 extra: list[Novel] = []
165 for n in rest:
166 key = (n.author.lower(), normalize_title(n.name) or n.id)
167 existing = by_key.get(key)
168 if existing:
169 seen = {c.id for c in existing.chapters}
170 for ch in n.chapters:
171 if ch.id not in seen:
172 existing.chapters.append(ch)
173 seen.add(ch.id)
174 existing.chapters.sort(key=lambda p: (p.created_utc, p.id))
175 existing.name = guess_title([c.title for c in existing.chapters])
176 existing.oneshot = is_oneshot(existing.flair, len(existing.chapters))
177 existing.over_18 = existing.over_18 or n.over_18
178 else:
179 extra.append(n)
180 novels.extend(extra)
181
182 novels.sort(key=lambda n: n.updated_utc, reverse=True)
183 return novels
184
185
186def _novel_from_posts(members: list[Post]) -> Novel:
187 members = sorted(members, key=lambda p: (p.created_utc, p.id))
188 first = members[0]
189 flair = next((m.flair for m in members if m.flair), first.flair)
190 return Novel(
191 id=first.id,
192 name=guess_title([m.title for m in members]),
193 author=first.author,
194 chapters=members,
195 flair=flair,
196 over_18=any(m.over_18 for m in members),
197 oneshot=is_oneshot(flair, len(members)),
198 )
199
200
201def novel_from_wiki(
202 entry: object,
203 by_id: dict[str, Post],
204 embedder: Embedder | None = None,
205 temperature: float = DEFAULT_TEMPERATURE,
206 fetcher: Fetcher | None = None,
207) -> Novel | None:
208 """Bouw een reeks vanuit een wiki-item: TOC-volgorde, daarna next-links vanaf het laatste deel."""
209 from .wiki import WikiSeries
210
211 if not isinstance(entry, WikiSeries):
212 return None
213 embedder = embedder or LexicalEmbedder()
214 wanted = list(entry.chapter_ids)
215 if entry.first_id and entry.first_id not in wanted:
216 wanted.insert(0, entry.first_id)
217 missing = [pid for pid in wanted if pid not in by_id]
218 if missing and fetcher:
219 for post in fetcher(missing) or []:
220 if post:
221 by_id[post.id] = post
222 toc = [by_id[pid] for pid in wanted if pid in by_id]
223 seed = toc[0] if toc else None
224 if seed is None:
225 return None
226 start = toc[-1] if toc else seed
227 tail = follow_chain(start, by_id, embedder=embedder, temperature=temperature, fetcher=fetcher)
228 seen: set[str] = {p.id for p in toc}
229 members = list(toc)
230 for post in tail:
231 if post.id not in seen:
232 members.append(post)
233 seen.add(post.id)
234 if not members:
235 return None
236 novel = _novel_from_posts(members)
237 novel.name = entry.title or novel.name
238 if entry.author:
239 novel.author = entry.author
240 novel.wiki_path = entry.wiki_path
241 return novel