#!/usr/bin/env python3 """Bouw of ververs de statische HFY-metadata-index vanuit de r/HFY-wiki.""" from __future__ import annotations import argparse import json import sys from datetime import datetime, timezone from pathlib import Path # Zodat `python3 indexer/build_index.py` werkt zonder installatie. sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from indexer.arctic import ArcticClient # noqa: E402 from indexer.cache import PostCache # noqa: E402 from indexer.cluster import Post, cluster_posts, merge_novels # noqa: E402 from indexer.pipeline import build_novels_from_wiki, make_fetcher, merge_wiki_novels # noqa: E402 from indexer.similarity import load_embedder # noqa: E402 from indexer.wiki import fetch_series_catalog, load_wiki_fixture # noqa: E402 from indexer.write_index import read_existing_posts, write_index # noqa: E402 ROOT = Path(__file__).resolve().parent.parent DEFAULT_DEST = ROOT / "public" STATE_NAME = "state.json" WIKI_SOURCE = "r/HFY wiki-serie-index via Arctic Shift (metadata only; no story text)" def load_state(dest: Path) -> dict: path = dest / STATE_NAME if not path.exists(): return {} return json.loads(path.read_text(encoding="utf-8")) def save_state(dest: Path, state: dict) -> None: dest.mkdir(parents=True, exist_ok=True) (dest / STATE_NAME).write_text(json.dumps(state, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") def ingest(raw_posts: list[dict]) -> list[Post]: out: list[Post] = [] for raw in raw_posts: post = Post.from_reddit(raw) if post: out.append(post) return out def _load_posts_file(path: Path) -> list[Post]: payload = json.loads(path.read_text(encoding="utf-8")) raw = payload.get("data", payload) if isinstance(payload, dict) else payload return ingest(raw) def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description="Bouw een metadata-index van r/HFY-verhalen vanuit de wiki-serie-index.") p.add_argument("--dest", type=Path, default=DEFAULT_DEST, help="Uitvoermap (standaard public/)") p.add_argument("--subreddit", default="HFY") p.add_argument( "--max-pages", type=int, default=50, help="Maximum aantal API-verzoeken deze run (0 = onbeperkt). Bij --full genegeerd.", ) p.add_argument("--max-series", type=int, default=0, help="Maximum aantal reeksen om deze run te bouwen (0 = alle)") p.add_argument("--full", action="store_true", help="Negeer het API-budget en bouw de wiki-catalogus opnieuw") p.add_argument("--incremental", action="store_true", help="Bestaande reeksen behouden en nieuwe wiki-items toevoegen") p.add_argument("--from-fixture", type=Path, help="Lees posts uit een JSON-bestand i.p.v. de API") p.add_argument( "--from-wiki-fixture", type=Path, help="Lees de serie-index uit testdata (index.md + pages/) i.p.v. Arctic Shift", ) p.add_argument("--page-size", type=int, default=50) p.add_argument("--base-url", default=None, help="Arctic Shift-basis-URL") p.add_argument( "--embedder", default="auto", help="auto (model2vec indien aanwezig, anders lexicaal), lexical, of model2vec", ) args = p.parse_args(argv) dest: Path = args.dest existing_posts = read_existing_posts(dest) state = load_state(dest) cache = PostCache(dest) by_id: dict[str, Post] = {p.id: p for p in existing_posts} # --from-fixture zonder wiki-fixture: oude post-clusterpad, voor unit tests. if args.from_fixture and not args.from_wiki_fixture: fresh = _load_posts_file(args.from_fixture) source = f"fixture:{args.from_fixture}" if existing_posts and (args.incremental or not args.full): novels = merge_novels(cluster_posts(existing_posts), cluster_posts(fresh)) else: novels = cluster_posts(fresh) stats = {"built": len(novels), "wikiEntries": 0, "unresolved": 0, "pagesFound": 0} return _write(dest, novels, args.page_size, source, stats, fresh_count=len(fresh), state_extra={}) if args.from_wiki_fixture: entries = load_wiki_fixture(args.from_wiki_fixture) posts_path = args.from_fixture if posts_path is None: candidate = args.from_wiki_fixture / "posts.json" posts_path = candidate if candidate.exists() else None if posts_path: for post in _load_posts_file(posts_path): by_id[post.id] = post client = None source = f"wiki-fixture:{args.from_wiki_fixture}" print(f"→ wiki-fixture: {len(entries)} series uit {args.from_wiki_fixture}", flush=True) novels, stats = build_novels_from_wiki( entries, by_id, embedder=load_embedder("lexical"), max_series=args.max_series, cache=cache, ) if args.incremental and existing_posts: novels = merge_wiki_novels(cluster_posts(existing_posts), novels) # cluster_posts gooit wiki_path kwijt; incoming heeft die al. return _write(dest, novels, args.page_size, source, stats, fresh_count=stats["built"], state_extra={}) client = ArcticClient(base=args.base_url) if args.base_url else ArcticClient() if not args.full: client.max_calls = max(0, args.max_pages) print("→ ophalen van de r/HFY-wiki-serie-index via Arctic Shift", flush=True) if client.max_calls: print(f" API-budget: {client.max_calls} verzoeken", flush=True) page_limit = 0 if args.max_series: # Extra subpagina's ophalen omdat niet elke path in het archief zit. page_limit = max(args.max_series * 5, args.max_series) entries = fetch_series_catalog(client, subreddit=args.subreddit, page_limit=page_limit) print( f" {len(entries)} series in de wiki, " f"{sum(1 for e in entries if e.page_found)} subpagina's in het archief, " f"{sum(1 for e in entries if e.chapter_ids)} met hoofdstuklijst", flush=True, ) fetcher = make_fetcher(client, by_id, cache) def progress(i: int, total: int, entry, novel) -> None: if i % 25 == 0 or i == total: name = novel.name if novel else entry.title print(f" … {i}/{total} ({len(by_id)} posts in cache), laatst: {name}", flush=True) novels, stats = build_novels_from_wiki( entries, by_id, fetcher=fetcher, embedder=load_embedder(args.embedder), max_series=args.max_series, client=client, cache=cache, on_progress=progress, ) if args.incremental and existing_posts: novels = merge_wiki_novels(cluster_posts(existing_posts), novels) print( f"✔️ wiki: {stats['built']} reeksen gebouwd, " f"{stats['pagesFound']} subpagina's gevonden, " f"{stats['unresolved']} zonder zaad, " f"{client.calls} API-verzoeken", flush=True, ) extra = { "apiCalls": client.calls, "wikiPagesFound": stats["pagesFound"], "wikiEntries": stats["wikiEntries"], "unresolved": stats["unresolved"], "unresolvedPaths": stats.get("unresolvedPaths") or [], } return _write(dest, novels, args.page_size, WIKI_SOURCE, stats, fresh_count=stats["built"], state_extra=extra) def _write( dest: Path, novels: list, page_size: int, source: str, stats: dict, fresh_count: int, state_extra: dict, ) -> int: meta = write_index(novels, dest, page_size=page_size, source=source) newest = max((c.created_utc for n in novels for c in n.chapters), default=0) oldest = min((c.created_utc for n in novels for c in n.chapters), default=0) state = { "source": source, "updatedAt": datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "newestUtc": newest, "oldestUtc": oldest, "novelCount": meta["novelCount"], "chapterCount": meta["chapterCount"], "lastFetched": fresh_count, "wikiEntries": stats.get("wikiEntries", 0), "wikiPagesFound": stats.get("pagesFound", 0), "unresolved": stats.get("unresolved", 0), } state.update(state_extra or {}) save_state(dest, state) print( f"✔️ index: {meta['novelCount']} verhalen, {meta['chapterCount']} hoofdstukken, " f"{meta['pageCount']} cataloguspagina's → {dest}", flush=True, ) return 0 if __name__ == "__main__": try: raise SystemExit(main()) except KeyboardInterrupt: print("⚠️ onderbroken", file=sys.stderr) raise SystemExit(130)