| 1 | #!/usr/bin/env python3 |
| 2 | """Bouw of ververs de statische HFY-metadata-index vanuit de r/HFY-wiki.""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import argparse |
| 7 | import json |
| 8 | import sys |
| 9 | from datetime import datetime, timezone |
| 10 | from pathlib import Path |
| 11 | |
| 12 | # Zodat `python3 indexer/build_index.py` werkt zonder installatie. |
| 13 | sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) |
| 14 | |
| 15 | from indexer.arctic import ArcticClient # noqa: E402 |
| 16 | from indexer.cache import PostCache # noqa: E402 |
| 17 | from indexer.cluster import Post, cluster_posts, merge_novels # noqa: E402 |
| 18 | from indexer.pipeline import build_novels_from_wiki, make_fetcher, merge_wiki_novels # noqa: E402 |
| 19 | from indexer.similarity import load_embedder # noqa: E402 |
| 20 | from indexer.wiki import fetch_series_catalog, load_wiki_fixture # noqa: E402 |
| 21 | from indexer.write_index import read_existing_posts, write_index # noqa: E402 |
| 22 | |
| 23 | ROOT = Path(__file__).resolve().parent.parent |
| 24 | DEFAULT_DEST = ROOT / "public" |
| 25 | STATE_NAME = "state.json" |
| 26 | WIKI_SOURCE = "r/HFY wiki-serie-index via Arctic Shift (metadata only; no story text)" |
| 27 | |
| 28 | |
| 29 | def load_state(dest: Path) -> dict: |
| 30 | path = dest / STATE_NAME |
| 31 | if not path.exists(): |
| 32 | return {} |
| 33 | return json.loads(path.read_text(encoding="utf-8")) |
| 34 | |
| 35 | |
| 36 | def save_state(dest: Path, state: dict) -> None: |
| 37 | dest.mkdir(parents=True, exist_ok=True) |
| 38 | (dest / STATE_NAME).write_text(json.dumps(state, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") |
| 39 | |
| 40 | |
| 41 | def ingest(raw_posts: list[dict]) -> list[Post]: |
| 42 | out: list[Post] = [] |
| 43 | for raw in raw_posts: |
| 44 | post = Post.from_reddit(raw) |
| 45 | if post: |
| 46 | out.append(post) |
| 47 | return out |
| 48 | |
| 49 | |
| 50 | def _load_posts_file(path: Path) -> list[Post]: |
| 51 | payload = json.loads(path.read_text(encoding="utf-8")) |
| 52 | raw = payload.get("data", payload) if isinstance(payload, dict) else payload |
| 53 | return ingest(raw) |
| 54 | |
| 55 | |
| 56 | def main(argv: list[str] | None = None) -> int: |
| 57 | p = argparse.ArgumentParser(description="Bouw een metadata-index van r/HFY-verhalen vanuit de wiki-serie-index.") |
| 58 | p.add_argument("--dest", type=Path, default=DEFAULT_DEST, help="Uitvoermap (standaard public/)") |
| 59 | p.add_argument("--subreddit", default="HFY") |
| 60 | p.add_argument( |
| 61 | "--max-pages", |
| 62 | type=int, |
| 63 | default=50, |
| 64 | help="Maximum aantal API-verzoeken deze run (0 = onbeperkt). Bij --full genegeerd.", |
| 65 | ) |
| 66 | p.add_argument("--max-series", type=int, default=0, help="Maximum aantal reeksen om deze run te bouwen (0 = alle)") |
| 67 | p.add_argument("--full", action="store_true", help="Negeer het API-budget en bouw de wiki-catalogus opnieuw") |
| 68 | p.add_argument("--incremental", action="store_true", help="Bestaande reeksen behouden en nieuwe wiki-items toevoegen") |
| 69 | p.add_argument("--from-fixture", type=Path, help="Lees posts uit een JSON-bestand i.p.v. de API") |
| 70 | p.add_argument( |
| 71 | "--from-wiki-fixture", |
| 72 | type=Path, |
| 73 | help="Lees de serie-index uit testdata (index.md + pages/) i.p.v. Arctic Shift", |
| 74 | ) |
| 75 | p.add_argument("--page-size", type=int, default=50) |
| 76 | p.add_argument("--base-url", default=None, help="Arctic Shift-basis-URL") |
| 77 | p.add_argument( |
| 78 | "--embedder", |
| 79 | default="auto", |
| 80 | help="auto (model2vec indien aanwezig, anders lexicaal), lexical, of model2vec", |
| 81 | ) |
| 82 | args = p.parse_args(argv) |
| 83 | |
| 84 | dest: Path = args.dest |
| 85 | existing_posts = read_existing_posts(dest) |
| 86 | state = load_state(dest) |
| 87 | cache = PostCache(dest) |
| 88 | by_id: dict[str, Post] = {p.id: p for p in existing_posts} |
| 89 | |
| 90 | # --from-fixture zonder wiki-fixture: oude post-clusterpad, voor unit tests. |
| 91 | if args.from_fixture and not args.from_wiki_fixture: |
| 92 | fresh = _load_posts_file(args.from_fixture) |
| 93 | source = f"fixture:{args.from_fixture}" |
| 94 | if existing_posts and (args.incremental or not args.full): |
| 95 | novels = merge_novels(cluster_posts(existing_posts), cluster_posts(fresh)) |
| 96 | else: |
| 97 | novels = cluster_posts(fresh) |
| 98 | stats = {"built": len(novels), "wikiEntries": 0, "unresolved": 0, "pagesFound": 0} |
| 99 | return _write(dest, novels, args.page_size, source, stats, fresh_count=len(fresh), state_extra={}) |
| 100 | |
| 101 | if args.from_wiki_fixture: |
| 102 | entries = load_wiki_fixture(args.from_wiki_fixture) |
| 103 | posts_path = args.from_fixture |
| 104 | if posts_path is None: |
| 105 | candidate = args.from_wiki_fixture / "posts.json" |
| 106 | posts_path = candidate if candidate.exists() else None |
| 107 | if posts_path: |
| 108 | for post in _load_posts_file(posts_path): |
| 109 | by_id[post.id] = post |
| 110 | client = None |
| 111 | source = f"wiki-fixture:{args.from_wiki_fixture}" |
| 112 | print(f"→ wiki-fixture: {len(entries)} series uit {args.from_wiki_fixture}", flush=True) |
| 113 | novels, stats = build_novels_from_wiki( |
| 114 | entries, |
| 115 | by_id, |
| 116 | embedder=load_embedder("lexical"), |
| 117 | max_series=args.max_series, |
| 118 | cache=cache, |
| 119 | ) |
| 120 | if args.incremental and existing_posts: |
| 121 | novels = merge_wiki_novels(cluster_posts(existing_posts), novels) |
| 122 | # cluster_posts gooit wiki_path kwijt; incoming heeft die al. |
| 123 | return _write(dest, novels, args.page_size, source, stats, fresh_count=stats["built"], state_extra={}) |
| 124 | |
| 125 | client = ArcticClient(base=args.base_url) if args.base_url else ArcticClient() |
| 126 | if not args.full: |
| 127 | client.max_calls = max(0, args.max_pages) |
| 128 | print("→ ophalen van de r/HFY-wiki-serie-index via Arctic Shift", flush=True) |
| 129 | if client.max_calls: |
| 130 | print(f" API-budget: {client.max_calls} verzoeken", flush=True) |
| 131 | page_limit = 0 |
| 132 | if args.max_series: |
| 133 | # Extra subpagina's ophalen omdat niet elke path in het archief zit. |
| 134 | page_limit = max(args.max_series * 5, args.max_series) |
| 135 | entries = fetch_series_catalog(client, subreddit=args.subreddit, page_limit=page_limit) |
| 136 | print( |
| 137 | f" {len(entries)} series in de wiki, " |
| 138 | f"{sum(1 for e in entries if e.page_found)} subpagina's in het archief, " |
| 139 | f"{sum(1 for e in entries if e.chapter_ids)} met hoofdstuklijst", |
| 140 | flush=True, |
| 141 | ) |
| 142 | fetcher = make_fetcher(client, by_id, cache) |
| 143 | |
| 144 | def progress(i: int, total: int, entry, novel) -> None: |
| 145 | if i % 25 == 0 or i == total: |
| 146 | name = novel.name if novel else entry.title |
| 147 | print(f" … {i}/{total} ({len(by_id)} posts in cache), laatst: {name}", flush=True) |
| 148 | |
| 149 | novels, stats = build_novels_from_wiki( |
| 150 | entries, |
| 151 | by_id, |
| 152 | fetcher=fetcher, |
| 153 | embedder=load_embedder(args.embedder), |
| 154 | max_series=args.max_series, |
| 155 | client=client, |
| 156 | cache=cache, |
| 157 | on_progress=progress, |
| 158 | ) |
| 159 | if args.incremental and existing_posts: |
| 160 | novels = merge_wiki_novels(cluster_posts(existing_posts), novels) |
| 161 | print( |
| 162 | f"✔️ wiki: {stats['built']} reeksen gebouwd, " |
| 163 | f"{stats['pagesFound']} subpagina's gevonden, " |
| 164 | f"{stats['unresolved']} zonder zaad, " |
| 165 | f"{client.calls} API-verzoeken", |
| 166 | flush=True, |
| 167 | ) |
| 168 | extra = { |
| 169 | "apiCalls": client.calls, |
| 170 | "wikiPagesFound": stats["pagesFound"], |
| 171 | "wikiEntries": stats["wikiEntries"], |
| 172 | "unresolved": stats["unresolved"], |
| 173 | "unresolvedPaths": stats.get("unresolvedPaths") or [], |
| 174 | } |
| 175 | return _write(dest, novels, args.page_size, WIKI_SOURCE, stats, fresh_count=stats["built"], state_extra=extra) |
| 176 | |
| 177 | |
| 178 | def _write( |
| 179 | dest: Path, |
| 180 | novels: list, |
| 181 | page_size: int, |
| 182 | source: str, |
| 183 | stats: dict, |
| 184 | fresh_count: int, |
| 185 | state_extra: dict, |
| 186 | ) -> int: |
| 187 | meta = write_index(novels, dest, page_size=page_size, source=source) |
| 188 | newest = max((c.created_utc for n in novels for c in n.chapters), default=0) |
| 189 | oldest = min((c.created_utc for n in novels for c in n.chapters), default=0) |
| 190 | state = { |
| 191 | "source": source, |
| 192 | "updatedAt": datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), |
| 193 | "newestUtc": newest, |
| 194 | "oldestUtc": oldest, |
| 195 | "novelCount": meta["novelCount"], |
| 196 | "chapterCount": meta["chapterCount"], |
| 197 | "lastFetched": fresh_count, |
| 198 | "wikiEntries": stats.get("wikiEntries", 0), |
| 199 | "wikiPagesFound": stats.get("pagesFound", 0), |
| 200 | "unresolved": stats.get("unresolved", 0), |
| 201 | } |
| 202 | state.update(state_extra or {}) |
| 203 | save_state(dest, state) |
| 204 | print( |
| 205 | f"✔️ index: {meta['novelCount']} verhalen, {meta['chapterCount']} hoofdstukken, " |
| 206 | f"{meta['pageCount']} cataloguspagina's → {dest}", |
| 207 | flush=True, |
| 208 | ) |
| 209 | return 0 |
| 210 | |
| 211 | |
| 212 | if __name__ == "__main__": |
| 213 | try: |
| 214 | raise SystemExit(main()) |
| 215 | except KeyboardInterrupt: |
| 216 | print("⚠️ onderbroken", file=sys.stderr) |
| 217 | raise SystemExit(130) |