"""Keten-volgen vanaf zaad-posts, in de geest van hfydl. Vanaf een startpost (wiki-eerste deel, of anders een 'Part 1'-zaad) lopen we de next-links af. Een kandidaat blijft alleen in de reeks als de titel genoeg lijkt op de tot dan toe gevolgde titels (cosine ≥ temperature). Expliciete 'next'-links van dezelfde auteur gaan voor op overige comment-links. Ontbrekende posts kunnen live worden opgehaald (Arctic Shift, niet Reddit /.json). We slaan geen verhaaltekst op. """ from __future__ import annotations import re from typing import Callable, Iterable from .cluster import Novel, Post, filter_story_posts, is_oneshot from .links import Link from .normalize import is_series_start, normalize_title from .similarity import DEFAULT_TEMPERATURE, Embedder, LexicalEmbedder, cosine, guess_title, mean_vec Fetcher = Callable[[list[str]], list[Post]] def _ensure(pid: str, by_id: dict[str, Post], fetcher: Fetcher | None) -> Post | None: if pid in by_id: return by_id[pid] if not fetcher: return None for post in fetcher([pid]) or []: if post: by_id[post.id] = post return by_id.get(pid) _START_TITLE = re.compile( r""" (?:^|[\s:\-–—|\[\(]) (?: (?:part|chapter|ch|pt|episode|ep)\.?\s*(?:1|i|one|first)\b | \#\s*1\b | \(\s*1\s*(?:/\s*(?:\d+|\?))?\s*\) | prologue ) """, re.IGNORECASE | re.VERBOSE, ) def is_seed(post: Post) -> bool: if is_oneshot(post.flair, 2): # oneshot-flair, ongeacht teller # Een 'first of series' telt wél als zaad, ook als de flair gek is. return is_series_start(post.flair) if is_series_start(post.flair): return True return bool(_START_TITLE.search(post.title)) def _pick_next( current: Post, by_id: dict[str, Post], visited: set[str], embedder: Embedder, title_vecs: list[list[float]], temperature: float, fetcher: Fetcher | None = None, ) -> Post | None: avg = mean_vec(title_vecs) if title_vecs else None links = list(current.links) if current.links else [ Link(pid, "", "next") for pid in current.next_ids ] # Expliciete next eerst, daarna overige links in documentvolgorde (hfydl # neemt de eerste die de drempel haalt; wij geven next alleen voorrang). links = sorted(links, key=lambda L: (0 if L.kind == "next" else 1)) for link in links: if link.kind in {"prev", "first"}: continue if link.id in visited: continue target = _ensure(link.id, by_id, fetcher) if not target: continue if current.author.lower() != target.author.lower() and link.kind != "next": continue if avg is not None: vec = embedder.encode([target.title])[0] score = cosine(vec, avg) # Expliciete next van dezelfde auteur mag iets lager scoren, maar # een heel ander verhaal (ver onder de drempel) slaan we over. if score < temperature and not ( link.kind == "next" and score >= temperature * 0.6 and current.author.lower() == target.author.lower() ): if link.kind != "next" or score < temperature * 0.4: continue if current.author.lower() != target.author.lower(): continue return target return None def follow_chain( seed: Post, by_id: dict[str, Post], embedder: Embedder | None = None, temperature: float = DEFAULT_TEMPERATURE, max_len: int = 800, fetcher: Fetcher | None = None, ) -> list[Post]: embedder = embedder or LexicalEmbedder() chain: list[Post] = [] visited: set[str] = set() current: Post | None = seed title_vecs: list[list[float]] = [] while current and current.id not in visited and len(chain) < max_len: visited.add(current.id) chain.append(current) title_vecs.append(embedder.encode([current.title])[0]) current = _pick_next(current, by_id, visited, embedder, title_vecs, temperature, fetcher=fetcher) return chain def find_seeds(posts: Iterable[Post]) -> list[Post]: items = list(posts) seeds = [p for p in items if is_seed(p)] seeds.sort(key=lambda p: (p.created_utc, p.id)) return seeds def cluster_from_seeds( posts: Iterable[Post], embedder: Embedder | None = None, temperature: float = DEFAULT_TEMPERATURE, ) -> list[Novel]: """Eerst ketens vanaf zaden, daarna de rest groeperen op auteur+titel+links.""" from .cluster import cluster_posts, merge_novels embedder = embedder or LexicalEmbedder() items = filter_story_posts(posts) by_id = {p.id: p for p in items} claimed: set[str] = set() novels: list[Novel] = [] for seed in find_seeds(items): if seed.id in claimed: continue chain = follow_chain(seed, by_id, embedder=embedder, temperature=temperature) if not chain: continue for p in chain: claimed.add(p.id) novels.append(_novel_from_posts(chain)) leftover = [p for p in items if p.id not in claimed] if leftover: # cluster_posts groepeert op auteur/titel en next-links; daarna # plakken we een restgroep aan een bestaande keten als de sleutel past. rest = cluster_posts(leftover) by_key: dict[tuple[str, str], Novel] = {} for n in novels: key = (n.author.lower(), normalize_title(n.name) or n.id) by_key[key] = n extra: list[Novel] = [] for n in rest: key = (n.author.lower(), normalize_title(n.name) or n.id) existing = by_key.get(key) if existing: seen = {c.id for c in existing.chapters} for ch in n.chapters: if ch.id not in seen: existing.chapters.append(ch) seen.add(ch.id) existing.chapters.sort(key=lambda p: (p.created_utc, p.id)) existing.name = guess_title([c.title for c in existing.chapters]) existing.oneshot = is_oneshot(existing.flair, len(existing.chapters)) existing.over_18 = existing.over_18 or n.over_18 else: extra.append(n) novels.extend(extra) novels.sort(key=lambda n: n.updated_utc, reverse=True) return novels def _novel_from_posts(members: list[Post]) -> Novel: members = sorted(members, key=lambda p: (p.created_utc, p.id)) first = members[0] flair = next((m.flair for m in members if m.flair), first.flair) return Novel( id=first.id, name=guess_title([m.title for m in members]), author=first.author, chapters=members, flair=flair, over_18=any(m.over_18 for m in members), oneshot=is_oneshot(flair, len(members)), ) def novel_from_wiki( entry: object, by_id: dict[str, Post], embedder: Embedder | None = None, temperature: float = DEFAULT_TEMPERATURE, fetcher: Fetcher | None = None, ) -> Novel | None: """Bouw een reeks vanuit een wiki-item: TOC-volgorde, daarna next-links vanaf het laatste deel.""" from .wiki import WikiSeries if not isinstance(entry, WikiSeries): return None embedder = embedder or LexicalEmbedder() wanted = list(entry.chapter_ids) if entry.first_id and entry.first_id not in wanted: wanted.insert(0, entry.first_id) missing = [pid for pid in wanted if pid not in by_id] if missing and fetcher: for post in fetcher(missing) or []: if post: by_id[post.id] = post toc = [by_id[pid] for pid in wanted if pid in by_id] seed = toc[0] if toc else None if seed is None: return None start = toc[-1] if toc else seed tail = follow_chain(start, by_id, embedder=embedder, temperature=temperature, fetcher=fetcher) seen: set[str] = {p.id for p in toc} members = list(toc) for post in tail: if post.id not in seen: members.append(post) seen.add(post.id) if not members: return None novel = _novel_from_posts(members) novel.name = entry.title or novel.name if entry.author: novel.author = entry.author novel.wiki_path = entry.wiki_path return novel