| 1 | """Keten-volgen vanaf zaad-posts, in de geest van hfydl. |
| 2 | |
| 3 | Vanaf een startpost (wiki-eerste deel, of anders een 'Part 1'-zaad) lopen |
| 4 | we de next-links af. Een kandidaat blijft alleen in de reeks als de titel |
| 5 | genoeg lijkt op de tot dan toe gevolgde titels (cosine ≥ temperature). |
| 6 | Expliciete 'next'-links van dezelfde auteur gaan voor op overige comment-links. |
| 7 | Ontbrekende posts kunnen live worden opgehaald (Arctic Shift, niet Reddit /.json). |
| 8 | We slaan geen verhaaltekst op. |
| 9 | """ |
| 10 | |
| 11 | from __future__ import annotations |
| 12 | |
| 13 | import re |
| 14 | from typing import Callable, Iterable |
| 15 | |
| 16 | from .cluster import Novel, Post, filter_story_posts, is_oneshot |
| 17 | from .links import Link |
| 18 | from .normalize import is_series_start, normalize_title |
| 19 | from .similarity import DEFAULT_TEMPERATURE, Embedder, LexicalEmbedder, cosine, guess_title, mean_vec |
| 20 | |
| 21 | Fetcher = Callable[[list[str]], list[Post]] |
| 22 | |
| 23 | |
| 24 | def _ensure(pid: str, by_id: dict[str, Post], fetcher: Fetcher | None) -> Post | None: |
| 25 | if pid in by_id: |
| 26 | return by_id[pid] |
| 27 | if not fetcher: |
| 28 | return None |
| 29 | for post in fetcher([pid]) or []: |
| 30 | if post: |
| 31 | by_id[post.id] = post |
| 32 | return by_id.get(pid) |
| 33 | |
| 34 | |
| 35 | _START_TITLE = re.compile( |
| 36 | r""" |
| 37 | (?:^|[\s:\-–—|\[\(]) |
| 38 | (?: |
| 39 | (?:part|chapter|ch|pt|episode|ep)\.?\s*(?:1|i|one|first)\b |
| 40 | | |
| 41 | \#\s*1\b |
| 42 | | |
| 43 | \(\s*1\s*(?:/\s*(?:\d+|\?))?\s*\) |
| 44 | | |
| 45 | prologue |
| 46 | ) |
| 47 | """, |
| 48 | re.IGNORECASE | re.VERBOSE, |
| 49 | ) |
| 50 | |
| 51 | |
| 52 | def is_seed(post: Post) -> bool: |
| 53 | if is_oneshot(post.flair, 2): # oneshot-flair, ongeacht teller |
| 54 | # Een 'first of series' telt wél als zaad, ook als de flair gek is. |
| 55 | return is_series_start(post.flair) |
| 56 | if is_series_start(post.flair): |
| 57 | return True |
| 58 | return bool(_START_TITLE.search(post.title)) |
| 59 | |
| 60 | |
| 61 | def _pick_next( |
| 62 | current: Post, |
| 63 | by_id: dict[str, Post], |
| 64 | visited: set[str], |
| 65 | embedder: Embedder, |
| 66 | title_vecs: list[list[float]], |
| 67 | temperature: float, |
| 68 | fetcher: Fetcher | None = None, |
| 69 | ) -> Post | None: |
| 70 | avg = mean_vec(title_vecs) if title_vecs else None |
| 71 | links = list(current.links) if current.links else [ |
| 72 | Link(pid, "", "next") for pid in current.next_ids |
| 73 | ] |
| 74 | # Expliciete next eerst, daarna overige links in documentvolgorde (hfydl |
| 75 | # neemt de eerste die de drempel haalt; wij geven next alleen voorrang). |
| 76 | links = sorted(links, key=lambda L: (0 if L.kind == "next" else 1)) |
| 77 | for link in links: |
| 78 | if link.kind in {"prev", "first"}: |
| 79 | continue |
| 80 | if link.id in visited: |
| 81 | continue |
| 82 | target = _ensure(link.id, by_id, fetcher) |
| 83 | if not target: |
| 84 | continue |
| 85 | if current.author.lower() != target.author.lower() and link.kind != "next": |
| 86 | continue |
| 87 | if avg is not None: |
| 88 | vec = embedder.encode([target.title])[0] |
| 89 | score = cosine(vec, avg) |
| 90 | # Expliciete next van dezelfde auteur mag iets lager scoren, maar |
| 91 | # een heel ander verhaal (ver onder de drempel) slaan we over. |
| 92 | if score < temperature and not ( |
| 93 | link.kind == "next" and score >= temperature * 0.6 and current.author.lower() == target.author.lower() |
| 94 | ): |
| 95 | if link.kind != "next" or score < temperature * 0.4: |
| 96 | continue |
| 97 | if current.author.lower() != target.author.lower(): |
| 98 | continue |
| 99 | return target |
| 100 | return None |
| 101 | |
| 102 | |
| 103 | def follow_chain( |
| 104 | seed: Post, |
| 105 | by_id: dict[str, Post], |
| 106 | embedder: Embedder | None = None, |
| 107 | temperature: float = DEFAULT_TEMPERATURE, |
| 108 | max_len: int = 800, |
| 109 | fetcher: Fetcher | None = None, |
| 110 | ) -> list[Post]: |
| 111 | embedder = embedder or LexicalEmbedder() |
| 112 | chain: list[Post] = [] |
| 113 | visited: set[str] = set() |
| 114 | current: Post | None = seed |
| 115 | title_vecs: list[list[float]] = [] |
| 116 | while current and current.id not in visited and len(chain) < max_len: |
| 117 | visited.add(current.id) |
| 118 | chain.append(current) |
| 119 | title_vecs.append(embedder.encode([current.title])[0]) |
| 120 | current = _pick_next(current, by_id, visited, embedder, title_vecs, temperature, fetcher=fetcher) |
| 121 | return chain |
| 122 | |
| 123 | |
| 124 | def find_seeds(posts: Iterable[Post]) -> list[Post]: |
| 125 | items = list(posts) |
| 126 | seeds = [p for p in items if is_seed(p)] |
| 127 | seeds.sort(key=lambda p: (p.created_utc, p.id)) |
| 128 | return seeds |
| 129 | |
| 130 | |
| 131 | def cluster_from_seeds( |
| 132 | posts: Iterable[Post], |
| 133 | embedder: Embedder | None = None, |
| 134 | temperature: float = DEFAULT_TEMPERATURE, |
| 135 | ) -> list[Novel]: |
| 136 | """Eerst ketens vanaf zaden, daarna de rest groeperen op auteur+titel+links.""" |
| 137 | from .cluster import cluster_posts, merge_novels |
| 138 | |
| 139 | embedder = embedder or LexicalEmbedder() |
| 140 | items = filter_story_posts(posts) |
| 141 | by_id = {p.id: p for p in items} |
| 142 | claimed: set[str] = set() |
| 143 | novels: list[Novel] = [] |
| 144 | |
| 145 | for seed in find_seeds(items): |
| 146 | if seed.id in claimed: |
| 147 | continue |
| 148 | chain = follow_chain(seed, by_id, embedder=embedder, temperature=temperature) |
| 149 | if not chain: |
| 150 | continue |
| 151 | for p in chain: |
| 152 | claimed.add(p.id) |
| 153 | novels.append(_novel_from_posts(chain)) |
| 154 | |
| 155 | leftover = [p for p in items if p.id not in claimed] |
| 156 | if leftover: |
| 157 | # cluster_posts groepeert op auteur/titel en next-links; daarna |
| 158 | # plakken we een restgroep aan een bestaande keten als de sleutel past. |
| 159 | rest = cluster_posts(leftover) |
| 160 | by_key: dict[tuple[str, str], Novel] = {} |
| 161 | for n in novels: |
| 162 | key = (n.author.lower(), normalize_title(n.name) or n.id) |
| 163 | by_key[key] = n |
| 164 | extra: list[Novel] = [] |
| 165 | for n in rest: |
| 166 | key = (n.author.lower(), normalize_title(n.name) or n.id) |
| 167 | existing = by_key.get(key) |
| 168 | if existing: |
| 169 | seen = {c.id for c in existing.chapters} |
| 170 | for ch in n.chapters: |
| 171 | if ch.id not in seen: |
| 172 | existing.chapters.append(ch) |
| 173 | seen.add(ch.id) |
| 174 | existing.chapters.sort(key=lambda p: (p.created_utc, p.id)) |
| 175 | existing.name = guess_title([c.title for c in existing.chapters]) |
| 176 | existing.oneshot = is_oneshot(existing.flair, len(existing.chapters)) |
| 177 | existing.over_18 = existing.over_18 or n.over_18 |
| 178 | else: |
| 179 | extra.append(n) |
| 180 | novels.extend(extra) |
| 181 | |
| 182 | novels.sort(key=lambda n: n.updated_utc, reverse=True) |
| 183 | return novels |
| 184 | |
| 185 | |
| 186 | def _novel_from_posts(members: list[Post]) -> Novel: |
| 187 | members = sorted(members, key=lambda p: (p.created_utc, p.id)) |
| 188 | first = members[0] |
| 189 | flair = next((m.flair for m in members if m.flair), first.flair) |
| 190 | return Novel( |
| 191 | id=first.id, |
| 192 | name=guess_title([m.title for m in members]), |
| 193 | author=first.author, |
| 194 | chapters=members, |
| 195 | flair=flair, |
| 196 | over_18=any(m.over_18 for m in members), |
| 197 | oneshot=is_oneshot(flair, len(members)), |
| 198 | ) |
| 199 | |
| 200 | |
| 201 | def novel_from_wiki( |
| 202 | entry: object, |
| 203 | by_id: dict[str, Post], |
| 204 | embedder: Embedder | None = None, |
| 205 | temperature: float = DEFAULT_TEMPERATURE, |
| 206 | fetcher: Fetcher | None = None, |
| 207 | ) -> Novel | None: |
| 208 | """Bouw een reeks vanuit een wiki-item: TOC-volgorde, daarna next-links vanaf het laatste deel.""" |
| 209 | from .wiki import WikiSeries |
| 210 | |
| 211 | if not isinstance(entry, WikiSeries): |
| 212 | return None |
| 213 | embedder = embedder or LexicalEmbedder() |
| 214 | wanted = list(entry.chapter_ids) |
| 215 | if entry.first_id and entry.first_id not in wanted: |
| 216 | wanted.insert(0, entry.first_id) |
| 217 | missing = [pid for pid in wanted if pid not in by_id] |
| 218 | if missing and fetcher: |
| 219 | for post in fetcher(missing) or []: |
| 220 | if post: |
| 221 | by_id[post.id] = post |
| 222 | toc = [by_id[pid] for pid in wanted if pid in by_id] |
| 223 | seed = toc[0] if toc else None |
| 224 | if seed is None: |
| 225 | return None |
| 226 | start = toc[-1] if toc else seed |
| 227 | tail = follow_chain(start, by_id, embedder=embedder, temperature=temperature, fetcher=fetcher) |
| 228 | seen: set[str] = {p.id for p in toc} |
| 229 | members = list(toc) |
| 230 | for post in tail: |
| 231 | if post.id not in seen: |
| 232 | members.append(post) |
| 233 | seen.add(post.id) |
| 234 | if not members: |
| 235 | return None |
| 236 | novel = _novel_from_posts(members) |
| 237 | novel.name = entry.title or novel.name |
| 238 | if entry.author: |
| 239 | novel.author = entry.author |
| 240 | novel.wiki_path = entry.wiki_path |
| 241 | return novel |