BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / build_index.py

build_index.py

build_index.py 8.5 KiB Download Raw
1#!/usr/bin/env python3
2"""Bouw of ververs de statische HFY-metadata-index vanuit de r/HFY-wiki."""
3
4from __future__ import annotations
5
6import argparse
7import json
8import sys
9from datetime import datetime, timezone
10from pathlib import Path
11
12# Zodat `python3 indexer/build_index.py` werkt zonder installatie.
13sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
14
15from indexer.arctic import ArcticClient # noqa: E402
16from indexer.cache import PostCache # noqa: E402
17from indexer.cluster import Post, cluster_posts, merge_novels # noqa: E402
18from indexer.pipeline import build_novels_from_wiki, make_fetcher, merge_wiki_novels # noqa: E402
19from indexer.similarity import load_embedder # noqa: E402
20from indexer.wiki import fetch_series_catalog, load_wiki_fixture # noqa: E402
21from indexer.write_index import read_existing_posts, write_index # noqa: E402
22
23ROOT = Path(__file__).resolve().parent.parent
24DEFAULT_DEST = ROOT / "public"
25STATE_NAME = "state.json"
26WIKI_SOURCE = "r/HFY wiki-serie-index via Arctic Shift (metadata only; no story text)"
27
28
29def load_state(dest: Path) -> dict:
30 path = dest / STATE_NAME
31 if not path.exists():
32 return {}
33 return json.loads(path.read_text(encoding="utf-8"))
34
35
36def save_state(dest: Path, state: dict) -> None:
37 dest.mkdir(parents=True, exist_ok=True)
38 (dest / STATE_NAME).write_text(json.dumps(state, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
39
40
41def ingest(raw_posts: list[dict]) -> list[Post]:
42 out: list[Post] = []
43 for raw in raw_posts:
44 post = Post.from_reddit(raw)
45 if post:
46 out.append(post)
47 return out
48
49
50def _load_posts_file(path: Path) -> list[Post]:
51 payload = json.loads(path.read_text(encoding="utf-8"))
52 raw = payload.get("data", payload) if isinstance(payload, dict) else payload
53 return ingest(raw)
54
55
56def main(argv: list[str] | None = None) -> int:
57 p = argparse.ArgumentParser(description="Bouw een metadata-index van r/HFY-verhalen vanuit de wiki-serie-index.")
58 p.add_argument("--dest", type=Path, default=DEFAULT_DEST, help="Uitvoermap (standaard public/)")
59 p.add_argument("--subreddit", default="HFY")
60 p.add_argument(
61 "--max-pages",
62 type=int,
63 default=50,
64 help="Maximum aantal API-verzoeken deze run (0 = onbeperkt). Bij --full genegeerd.",
65 )
66 p.add_argument("--max-series", type=int, default=0, help="Maximum aantal reeksen om deze run te bouwen (0 = alle)")
67 p.add_argument("--full", action="store_true", help="Negeer het API-budget en bouw de wiki-catalogus opnieuw")
68 p.add_argument("--incremental", action="store_true", help="Bestaande reeksen behouden en nieuwe wiki-items toevoegen")
69 p.add_argument("--from-fixture", type=Path, help="Lees posts uit een JSON-bestand i.p.v. de API")
70 p.add_argument(
71 "--from-wiki-fixture",
72 type=Path,
73 help="Lees de serie-index uit testdata (index.md + pages/) i.p.v. Arctic Shift",
74 )
75 p.add_argument("--page-size", type=int, default=50)
76 p.add_argument("--base-url", default=None, help="Arctic Shift-basis-URL")
77 p.add_argument(
78 "--embedder",
79 default="auto",
80 help="auto (model2vec indien aanwezig, anders lexicaal), lexical, of model2vec",
81 )
82 args = p.parse_args(argv)
83
84 dest: Path = args.dest
85 existing_posts = read_existing_posts(dest)
86 state = load_state(dest)
87 cache = PostCache(dest)
88 by_id: dict[str, Post] = {p.id: p for p in existing_posts}
89
90 # --from-fixture zonder wiki-fixture: oude post-clusterpad, voor unit tests.
91 if args.from_fixture and not args.from_wiki_fixture:
92 fresh = _load_posts_file(args.from_fixture)
93 source = f"fixture:{args.from_fixture}"
94 if existing_posts and (args.incremental or not args.full):
95 novels = merge_novels(cluster_posts(existing_posts), cluster_posts(fresh))
96 else:
97 novels = cluster_posts(fresh)
98 stats = {"built": len(novels), "wikiEntries": 0, "unresolved": 0, "pagesFound": 0}
99 return _write(dest, novels, args.page_size, source, stats, fresh_count=len(fresh), state_extra={})
100
101 if args.from_wiki_fixture:
102 entries = load_wiki_fixture(args.from_wiki_fixture)
103 posts_path = args.from_fixture
104 if posts_path is None:
105 candidate = args.from_wiki_fixture / "posts.json"
106 posts_path = candidate if candidate.exists() else None
107 if posts_path:
108 for post in _load_posts_file(posts_path):
109 by_id[post.id] = post
110 client = None
111 source = f"wiki-fixture:{args.from_wiki_fixture}"
112 print(f"→ wiki-fixture: {len(entries)} series uit {args.from_wiki_fixture}", flush=True)
113 novels, stats = build_novels_from_wiki(
114 entries,
115 by_id,
116 embedder=load_embedder("lexical"),
117 max_series=args.max_series,
118 cache=cache,
119 )
120 if args.incremental and existing_posts:
121 novels = merge_wiki_novels(cluster_posts(existing_posts), novels)
122 # cluster_posts gooit wiki_path kwijt; incoming heeft die al.
123 return _write(dest, novels, args.page_size, source, stats, fresh_count=stats["built"], state_extra={})
124
125 client = ArcticClient(base=args.base_url) if args.base_url else ArcticClient()
126 if not args.full:
127 client.max_calls = max(0, args.max_pages)
128 print("→ ophalen van de r/HFY-wiki-serie-index via Arctic Shift", flush=True)
129 if client.max_calls:
130 print(f" API-budget: {client.max_calls} verzoeken", flush=True)
131 page_limit = 0
132 if args.max_series:
133 # Extra subpagina's ophalen omdat niet elke path in het archief zit.
134 page_limit = max(args.max_series * 5, args.max_series)
135 entries = fetch_series_catalog(client, subreddit=args.subreddit, page_limit=page_limit)
136 print(
137 f" {len(entries)} series in de wiki, "
138 f"{sum(1 for e in entries if e.page_found)} subpagina's in het archief, "
139 f"{sum(1 for e in entries if e.chapter_ids)} met hoofdstuklijst",
140 flush=True,
141 )
142 fetcher = make_fetcher(client, by_id, cache)
143
144 def progress(i: int, total: int, entry, novel) -> None:
145 if i % 25 == 0 or i == total:
146 name = novel.name if novel else entry.title
147 print(f" … {i}/{total} ({len(by_id)} posts in cache), laatst: {name}", flush=True)
148
149 novels, stats = build_novels_from_wiki(
150 entries,
151 by_id,
152 fetcher=fetcher,
153 embedder=load_embedder(args.embedder),
154 max_series=args.max_series,
155 client=client,
156 cache=cache,
157 on_progress=progress,
158 )
159 if args.incremental and existing_posts:
160 novels = merge_wiki_novels(cluster_posts(existing_posts), novels)
161 print(
162 f"✔️ wiki: {stats['built']} reeksen gebouwd, "
163 f"{stats['pagesFound']} subpagina's gevonden, "
164 f"{stats['unresolved']} zonder zaad, "
165 f"{client.calls} API-verzoeken",
166 flush=True,
167 )
168 extra = {
169 "apiCalls": client.calls,
170 "wikiPagesFound": stats["pagesFound"],
171 "wikiEntries": stats["wikiEntries"],
172 "unresolved": stats["unresolved"],
173 "unresolvedPaths": stats.get("unresolvedPaths") or [],
174 }
175 return _write(dest, novels, args.page_size, WIKI_SOURCE, stats, fresh_count=stats["built"], state_extra=extra)
176
177
178def _write(
179 dest: Path,
180 novels: list,
181 page_size: int,
182 source: str,
183 stats: dict,
184 fresh_count: int,
185 state_extra: dict,
186) -> int:
187 meta = write_index(novels, dest, page_size=page_size, source=source)
188 newest = max((c.created_utc for n in novels for c in n.chapters), default=0)
189 oldest = min((c.created_utc for n in novels for c in n.chapters), default=0)
190 state = {
191 "source": source,
192 "updatedAt": datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
193 "newestUtc": newest,
194 "oldestUtc": oldest,
195 "novelCount": meta["novelCount"],
196 "chapterCount": meta["chapterCount"],
197 "lastFetched": fresh_count,
198 "wikiEntries": stats.get("wikiEntries", 0),
199 "wikiPagesFound": stats.get("pagesFound", 0),
200 "unresolved": stats.get("unresolved", 0),
201 }
202 state.update(state_extra or {})
203 save_state(dest, state)
204 print(
205 f"✔️ index: {meta['novelCount']} verhalen, {meta['chapterCount']} hoofdstukken, "
206 f"{meta['pageCount']} cataloguspagina's → {dest}",
207 flush=True,
208 )
209 return 0
210
211
212if __name__ == "__main__":
213 try:
214 raise SystemExit(main())
215 except KeyboardInterrupt:
216 print("⚠️ onderbroken", file=sys.stderr)
217 raise SystemExit(130)