| 1 | """Bouw novels vanuit de wiki-serie-index + hfydl-achtig keten-volgen.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | from typing import Callable |
| 6 | |
| 7 | from .arctic import ArcticClient, ArcticError |
| 8 | from .cache import PostCache |
| 9 | from .chain import novel_from_wiki |
| 10 | from .cluster import Novel, Post |
| 11 | from .normalize import is_series_start, normalize_title |
| 12 | from .similarity import DEFAULT_TEMPERATURE, Embedder, LexicalEmbedder, cosine |
| 13 | from .wiki import WikiSeries |
| 14 | |
| 15 | Fetcher = Callable[[list[str]], list[Post]] |
| 16 | |
| 17 | |
| 18 | def make_fetcher( |
| 19 | client: ArcticClient | None, |
| 20 | by_id: dict[str, Post], |
| 21 | cache: PostCache | None = None, |
| 22 | ) -> Fetcher: |
| 23 | """Haal ontbrekende posts op (cache eerst, dan Arctic Shift /api/posts/ids).""" |
| 24 | |
| 25 | def fetch(ids: list[str]) -> list[Post]: |
| 26 | out: list[Post] = [] |
| 27 | missing: list[str] = [] |
| 28 | for pid in ids: |
| 29 | pid = (pid or "").lower() |
| 30 | if not pid: |
| 31 | continue |
| 32 | if pid in by_id: |
| 33 | out.append(by_id[pid]) |
| 34 | continue |
| 35 | if cache: |
| 36 | cached = cache.get(pid) |
| 37 | if cached: |
| 38 | by_id[cached.id] = cached |
| 39 | out.append(cached) |
| 40 | continue |
| 41 | missing.append(pid) |
| 42 | if not missing or client is None: |
| 43 | return out |
| 44 | try: |
| 45 | raw_posts = client.posts_by_ids(missing) |
| 46 | except ArcticError as exc: |
| 47 | # Budget op of tijdelijke fout: werk verder met wat we al hebben. |
| 48 | if exc.status is None and "budget" in str(exc).lower(): |
| 49 | return out |
| 50 | raise |
| 51 | for raw in raw_posts: |
| 52 | post = Post.from_reddit(raw) |
| 53 | if not post: |
| 54 | continue |
| 55 | by_id[post.id] = post |
| 56 | if cache: |
| 57 | cache.put(post) |
| 58 | out.append(post) |
| 59 | return out |
| 60 | |
| 61 | return fetch |
| 62 | |
| 63 | |
| 64 | def pick_seed_from_search(entry: WikiSeries, candidates: list[Post], embedder: Embedder) -> Post | None: |
| 65 | """Kies het waarschijnlijke eerste deel uit een titel+auteur-zoekactie.""" |
| 66 | if not candidates: |
| 67 | return None |
| 68 | target = entry.title or "" |
| 69 | target_vec = embedder.encode([target])[0] |
| 70 | scored: list[tuple[float, int, Post]] = [] |
| 71 | for post in candidates: |
| 72 | if entry.author and post.author.lower() != entry.author.lower(): |
| 73 | continue |
| 74 | vec = embedder.encode([post.title])[0] |
| 75 | score = cosine(vec, target_vec) |
| 76 | # Iets extra voor een echte startpost. |
| 77 | if is_series_start(post.flair): |
| 78 | score += 0.05 |
| 79 | # Genormaliseerde titel moet wel raken; "7" mag niet alles vangen. |
| 80 | if normalize_title(target) and normalize_title(target) not in normalize_title(post.title): |
| 81 | if score < 0.72: |
| 82 | continue |
| 83 | scored.append((score, post.created_utc, post)) |
| 84 | if not scored: |
| 85 | return None |
| 86 | scored.sort(key=lambda row: (-row[0], row[1], row[2].id)) |
| 87 | best_score, _utc, best = scored[0] |
| 88 | if best_score < 0.55: |
| 89 | return None |
| 90 | return best |
| 91 | |
| 92 | |
| 93 | def search_seed( |
| 94 | client: ArcticClient, |
| 95 | entry: WikiSeries, |
| 96 | embedder: Embedder, |
| 97 | ) -> Post | None: |
| 98 | """Alleen title+auteur: title-zoeken op heel r/HFY geeft bij Arctic Shift HTTP 422.""" |
| 99 | if not entry.author or not entry.title: |
| 100 | return None |
| 101 | try: |
| 102 | raw = client.search_posts(author=entry.author, title=entry.title, sort="asc", limit=25) |
| 103 | except ArcticError: |
| 104 | return None |
| 105 | posts = [p for p in (Post.from_reddit(r) for r in raw) if p] |
| 106 | return pick_seed_from_search(entry, posts, embedder) |
| 107 | |
| 108 | |
| 109 | def build_novels_from_wiki( |
| 110 | entries: list[WikiSeries], |
| 111 | by_id: dict[str, Post], |
| 112 | fetcher: Fetcher | None = None, |
| 113 | embedder: Embedder | None = None, |
| 114 | temperature: float = DEFAULT_TEMPERATURE, |
| 115 | max_series: int = 0, |
| 116 | client: ArcticClient | None = None, |
| 117 | cache: PostCache | None = None, |
| 118 | on_progress: Callable[[int, int, WikiSeries, Novel | None], None] | None = None, |
| 119 | ) -> tuple[list[Novel], dict]: |
| 120 | """Bouw reeksen vanaf de wiki-lijst. Geeft (novels, stats) terug.""" |
| 121 | embedder = embedder or LexicalEmbedder() |
| 122 | fetcher = fetcher or make_fetcher(client, by_id, cache) |
| 123 | # Eerst alle bekende wiki-hoofdstuk-IDs in batches ophalen; dat is zuiniger |
| 124 | # dan per serie achter next-links aan te lopen. |
| 125 | toc_ids: list[str] = [] |
| 126 | seen_ids: set[str] = set(by_id) |
| 127 | for entry in entries: |
| 128 | for pid in entry.chapter_ids: |
| 129 | if pid not in seen_ids: |
| 130 | seen_ids.add(pid) |
| 131 | toc_ids.append(pid) |
| 132 | if entry.first_id and entry.first_id not in seen_ids: |
| 133 | seen_ids.add(entry.first_id) |
| 134 | toc_ids.append(entry.first_id) |
| 135 | if toc_ids and fetcher: |
| 136 | fetcher(toc_ids) |
| 137 | novels: list[Novel] = [] |
| 138 | unresolved: list[str] = [] |
| 139 | used = 0 |
| 140 | for i, entry in enumerate(entries, start=1): |
| 141 | if max_series and used >= max_series: |
| 142 | break |
| 143 | if not entry.first_id and not entry.chapter_ids: |
| 144 | seed = search_seed(client, entry, embedder) if client else None |
| 145 | if seed: |
| 146 | by_id[seed.id] = seed |
| 147 | if cache: |
| 148 | cache.put(seed) |
| 149 | entry.first_id = seed.id |
| 150 | entry.chapter_ids = (seed.id,) |
| 151 | novel = novel_from_wiki( |
| 152 | entry, |
| 153 | by_id, |
| 154 | embedder=embedder, |
| 155 | temperature=temperature, |
| 156 | fetcher=fetcher, |
| 157 | ) |
| 158 | if novel: |
| 159 | novels.append(novel) |
| 160 | used += 1 |
| 161 | else: |
| 162 | unresolved.append(entry.wiki_path) |
| 163 | if on_progress: |
| 164 | on_progress(i, len(entries), entry, novel) |
| 165 | stats = { |
| 166 | "wikiEntries": len(entries), |
| 167 | "built": len(novels), |
| 168 | "unresolved": len(unresolved), |
| 169 | "unresolvedPaths": unresolved[:50], |
| 170 | "pagesFound": sum(1 for e in entries if e.page_found), |
| 171 | } |
| 172 | return novels, stats |
| 173 | |
| 174 | |
| 175 | def merge_wiki_novels(existing: list[Novel], incoming: list[Novel]) -> list[Novel]: |
| 176 | """Nieuwe wiki-reeksen winnen; bestaande reeksen zonder overlap blijven staan.""" |
| 177 | incoming_paths = {n.wiki_path.lower() for n in incoming if n.wiki_path} |
| 178 | incoming_ids = {c.id for n in incoming for c in n.chapters} |
| 179 | out = list(incoming) |
| 180 | for novel in existing: |
| 181 | if novel.wiki_path and novel.wiki_path.lower() in incoming_paths: |
| 182 | continue |
| 183 | if any(ch.id in incoming_ids for ch in novel.chapters): |
| 184 | continue |
| 185 | out.append(novel) |
| 186 | out.sort(key=lambda n: n.updated_utc, reverse=True) |
| 187 | return out |