"""Bouw novels vanuit de wiki-serie-index + hfydl-achtig keten-volgen.""" from __future__ import annotations from typing import Callable from .arctic import ArcticClient, ArcticError from .cache import PostCache from .chain import novel_from_wiki from .cluster import Novel, Post from .normalize import is_series_start, normalize_title from .similarity import DEFAULT_TEMPERATURE, Embedder, LexicalEmbedder, cosine from .wiki import WikiSeries Fetcher = Callable[[list[str]], list[Post]] def make_fetcher( client: ArcticClient | None, by_id: dict[str, Post], cache: PostCache | None = None, ) -> Fetcher: """Haal ontbrekende posts op (cache eerst, dan Arctic Shift /api/posts/ids).""" def fetch(ids: list[str]) -> list[Post]: out: list[Post] = [] missing: list[str] = [] for pid in ids: pid = (pid or "").lower() if not pid: continue if pid in by_id: out.append(by_id[pid]) continue if cache: cached = cache.get(pid) if cached: by_id[cached.id] = cached out.append(cached) continue missing.append(pid) if not missing or client is None: return out try: raw_posts = client.posts_by_ids(missing) except ArcticError as exc: # Budget op of tijdelijke fout: werk verder met wat we al hebben. if exc.status is None and "budget" in str(exc).lower(): return out raise for raw in raw_posts: post = Post.from_reddit(raw) if not post: continue by_id[post.id] = post if cache: cache.put(post) out.append(post) return out return fetch def pick_seed_from_search(entry: WikiSeries, candidates: list[Post], embedder: Embedder) -> Post | None: """Kies het waarschijnlijke eerste deel uit een titel+auteur-zoekactie.""" if not candidates: return None target = entry.title or "" target_vec = embedder.encode([target])[0] scored: list[tuple[float, int, Post]] = [] for post in candidates: if entry.author and post.author.lower() != entry.author.lower(): continue vec = embedder.encode([post.title])[0] score = cosine(vec, target_vec) # Iets extra voor een echte startpost. if is_series_start(post.flair): score += 0.05 # Genormaliseerde titel moet wel raken; "7" mag niet alles vangen. if normalize_title(target) and normalize_title(target) not in normalize_title(post.title): if score < 0.72: continue scored.append((score, post.created_utc, post)) if not scored: return None scored.sort(key=lambda row: (-row[0], row[1], row[2].id)) best_score, _utc, best = scored[0] if best_score < 0.55: return None return best def search_seed( client: ArcticClient, entry: WikiSeries, embedder: Embedder, ) -> Post | None: """Alleen title+auteur: title-zoeken op heel r/HFY geeft bij Arctic Shift HTTP 422.""" if not entry.author or not entry.title: return None try: raw = client.search_posts(author=entry.author, title=entry.title, sort="asc", limit=25) except ArcticError: return None posts = [p for p in (Post.from_reddit(r) for r in raw) if p] return pick_seed_from_search(entry, posts, embedder) def build_novels_from_wiki( entries: list[WikiSeries], by_id: dict[str, Post], fetcher: Fetcher | None = None, embedder: Embedder | None = None, temperature: float = DEFAULT_TEMPERATURE, max_series: int = 0, client: ArcticClient | None = None, cache: PostCache | None = None, on_progress: Callable[[int, int, WikiSeries, Novel | None], None] | None = None, ) -> tuple[list[Novel], dict]: """Bouw reeksen vanaf de wiki-lijst. Geeft (novels, stats) terug.""" embedder = embedder or LexicalEmbedder() fetcher = fetcher or make_fetcher(client, by_id, cache) # Eerst alle bekende wiki-hoofdstuk-IDs in batches ophalen; dat is zuiniger # dan per serie achter next-links aan te lopen. toc_ids: list[str] = [] seen_ids: set[str] = set(by_id) for entry in entries: for pid in entry.chapter_ids: if pid not in seen_ids: seen_ids.add(pid) toc_ids.append(pid) if entry.first_id and entry.first_id not in seen_ids: seen_ids.add(entry.first_id) toc_ids.append(entry.first_id) if toc_ids and fetcher: fetcher(toc_ids) novels: list[Novel] = [] unresolved: list[str] = [] used = 0 for i, entry in enumerate(entries, start=1): if max_series and used >= max_series: break if not entry.first_id and not entry.chapter_ids: seed = search_seed(client, entry, embedder) if client else None if seed: by_id[seed.id] = seed if cache: cache.put(seed) entry.first_id = seed.id entry.chapter_ids = (seed.id,) novel = novel_from_wiki( entry, by_id, embedder=embedder, temperature=temperature, fetcher=fetcher, ) if novel: novels.append(novel) used += 1 else: unresolved.append(entry.wiki_path) if on_progress: on_progress(i, len(entries), entry, novel) stats = { "wikiEntries": len(entries), "built": len(novels), "unresolved": len(unresolved), "unresolvedPaths": unresolved[:50], "pagesFound": sum(1 for e in entries if e.page_found), } return novels, stats def merge_wiki_novels(existing: list[Novel], incoming: list[Novel]) -> list[Novel]: """Nieuwe wiki-reeksen winnen; bestaande reeksen zonder overlap blijven staan.""" incoming_paths = {n.wiki_path.lower() for n in incoming if n.wiki_path} incoming_ids = {c.id for n in incoming for c in n.chapters} out = list(incoming) for novel in existing: if novel.wiki_path and novel.wiki_path.lower() in incoming_paths: continue if any(ch.id in incoming_ids for ch in novel.chapters): continue out.append(novel) out.sort(key=lambda n: n.updated_utc, reverse=True) return out