BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / pipeline.py

pipeline.py

pipeline.py 6.4 KiB Download Raw
1"""Bouw novels vanuit de wiki-serie-index + hfydl-achtig keten-volgen."""
2
3from __future__ import annotations
4
5from typing import Callable
6
7from .arctic import ArcticClient, ArcticError
8from .cache import PostCache
9from .chain import novel_from_wiki
10from .cluster import Novel, Post
11from .normalize import is_series_start, normalize_title
12from .similarity import DEFAULT_TEMPERATURE, Embedder, LexicalEmbedder, cosine
13from .wiki import WikiSeries
14
15Fetcher = Callable[[list[str]], list[Post]]
16
17
18def make_fetcher(
19 client: ArcticClient | None,
20 by_id: dict[str, Post],
21 cache: PostCache | None = None,
22) -> Fetcher:
23 """Haal ontbrekende posts op (cache eerst, dan Arctic Shift /api/posts/ids)."""
24
25 def fetch(ids: list[str]) -> list[Post]:
26 out: list[Post] = []
27 missing: list[str] = []
28 for pid in ids:
29 pid = (pid or "").lower()
30 if not pid:
31 continue
32 if pid in by_id:
33 out.append(by_id[pid])
34 continue
35 if cache:
36 cached = cache.get(pid)
37 if cached:
38 by_id[cached.id] = cached
39 out.append(cached)
40 continue
41 missing.append(pid)
42 if not missing or client is None:
43 return out
44 try:
45 raw_posts = client.posts_by_ids(missing)
46 except ArcticError as exc:
47 # Budget op of tijdelijke fout: werk verder met wat we al hebben.
48 if exc.status is None and "budget" in str(exc).lower():
49 return out
50 raise
51 for raw in raw_posts:
52 post = Post.from_reddit(raw)
53 if not post:
54 continue
55 by_id[post.id] = post
56 if cache:
57 cache.put(post)
58 out.append(post)
59 return out
60
61 return fetch
62
63
64def pick_seed_from_search(entry: WikiSeries, candidates: list[Post], embedder: Embedder) -> Post | None:
65 """Kies het waarschijnlijke eerste deel uit een titel+auteur-zoekactie."""
66 if not candidates:
67 return None
68 target = entry.title or ""
69 target_vec = embedder.encode([target])[0]
70 scored: list[tuple[float, int, Post]] = []
71 for post in candidates:
72 if entry.author and post.author.lower() != entry.author.lower():
73 continue
74 vec = embedder.encode([post.title])[0]
75 score = cosine(vec, target_vec)
76 # Iets extra voor een echte startpost.
77 if is_series_start(post.flair):
78 score += 0.05
79 # Genormaliseerde titel moet wel raken; "7" mag niet alles vangen.
80 if normalize_title(target) and normalize_title(target) not in normalize_title(post.title):
81 if score < 0.72:
82 continue
83 scored.append((score, post.created_utc, post))
84 if not scored:
85 return None
86 scored.sort(key=lambda row: (-row[0], row[1], row[2].id))
87 best_score, _utc, best = scored[0]
88 if best_score < 0.55:
89 return None
90 return best
91
92
93def search_seed(
94 client: ArcticClient,
95 entry: WikiSeries,
96 embedder: Embedder,
97) -> Post | None:
98 """Alleen title+auteur: title-zoeken op heel r/HFY geeft bij Arctic Shift HTTP 422."""
99 if not entry.author or not entry.title:
100 return None
101 try:
102 raw = client.search_posts(author=entry.author, title=entry.title, sort="asc", limit=25)
103 except ArcticError:
104 return None
105 posts = [p for p in (Post.from_reddit(r) for r in raw) if p]
106 return pick_seed_from_search(entry, posts, embedder)
107
108
109def build_novels_from_wiki(
110 entries: list[WikiSeries],
111 by_id: dict[str, Post],
112 fetcher: Fetcher | None = None,
113 embedder: Embedder | None = None,
114 temperature: float = DEFAULT_TEMPERATURE,
115 max_series: int = 0,
116 client: ArcticClient | None = None,
117 cache: PostCache | None = None,
118 on_progress: Callable[[int, int, WikiSeries, Novel | None], None] | None = None,
119) -> tuple[list[Novel], dict]:
120 """Bouw reeksen vanaf de wiki-lijst. Geeft (novels, stats) terug."""
121 embedder = embedder or LexicalEmbedder()
122 fetcher = fetcher or make_fetcher(client, by_id, cache)
123 # Eerst alle bekende wiki-hoofdstuk-IDs in batches ophalen; dat is zuiniger
124 # dan per serie achter next-links aan te lopen.
125 toc_ids: list[str] = []
126 seen_ids: set[str] = set(by_id)
127 for entry in entries:
128 for pid in entry.chapter_ids:
129 if pid not in seen_ids:
130 seen_ids.add(pid)
131 toc_ids.append(pid)
132 if entry.first_id and entry.first_id not in seen_ids:
133 seen_ids.add(entry.first_id)
134 toc_ids.append(entry.first_id)
135 if toc_ids and fetcher:
136 fetcher(toc_ids)
137 novels: list[Novel] = []
138 unresolved: list[str] = []
139 used = 0
140 for i, entry in enumerate(entries, start=1):
141 if max_series and used >= max_series:
142 break
143 if not entry.first_id and not entry.chapter_ids:
144 seed = search_seed(client, entry, embedder) if client else None
145 if seed:
146 by_id[seed.id] = seed
147 if cache:
148 cache.put(seed)
149 entry.first_id = seed.id
150 entry.chapter_ids = (seed.id,)
151 novel = novel_from_wiki(
152 entry,
153 by_id,
154 embedder=embedder,
155 temperature=temperature,
156 fetcher=fetcher,
157 )
158 if novel:
159 novels.append(novel)
160 used += 1
161 else:
162 unresolved.append(entry.wiki_path)
163 if on_progress:
164 on_progress(i, len(entries), entry, novel)
165 stats = {
166 "wikiEntries": len(entries),
167 "built": len(novels),
168 "unresolved": len(unresolved),
169 "unresolvedPaths": unresolved[:50],
170 "pagesFound": sum(1 for e in entries if e.page_found),
171 }
172 return novels, stats
173
174
175def merge_wiki_novels(existing: list[Novel], incoming: list[Novel]) -> list[Novel]:
176 """Nieuwe wiki-reeksen winnen; bestaande reeksen zonder overlap blijven staan."""
177 incoming_paths = {n.wiki_path.lower() for n in incoming if n.wiki_path}
178 incoming_ids = {c.id for n in incoming for c in n.chapters}
179 out = list(incoming)
180 for novel in existing:
181 if novel.wiki_path and novel.wiki_path.lower() in incoming_paths:
182 continue
183 if any(ch.id in incoming_ids for ch in novel.chapters):
184 continue
185 out.append(novel)
186 out.sort(key=lambda n: n.updated_utc, reverse=True)
187 return out