"""Parse en haal de officiële r/HFY-wiki-serie-index op. Bron: https://www.reddit.com/r/HFY/wiki/series/ (en gelinkte subpagina's). Ongeauthenticeerde Reddit-JSON is sinds 2026 geblokkeerd (HTTP 403), daarom lezen we de wikipagina's via Arctic Shift (`/api/subreddits/wikis`). Niet elke subpagina staat in dat archief; de hoofdpagina met de complete lijst wel. """ from __future__ import annotations import re from dataclasses import dataclass, field from pathlib import Path from .links import extract_post_id SERIES_INDEX_PATHS = ("/r/hfy/wiki/series", "/r/HFY/wiki/series") SERIES_INDEX_URL = "https://www.reddit.com/r/HFY/wiki/series/" _LIST_ITEM = re.compile(r"^\s*[\*\-]\s*\[([^\]]+)\]\(([^)]+)\)(.*)$", re.M) _AUTHOR_ITALIC = re.compile(r"\[\s*\*([^*]+)\*\s*\]") _AUTHOR_WIKI = re.compile( r"\[(?:\*\*)?([^\]*]+)(?:\*\*)?\]\((?:https?://(?:www\.)?reddit\.com)?/r/[^/]+/wiki/authors/([^)\s/]+)", re.I, ) _AUTHOR_USER = re.compile( r"(?:https?://(?:www\.)?reddit\.com)?/u(?:ser)?/([A-Za-z0-9_-]+)|(? str: return self.wiki_path.rstrip("/").rsplit("/", 1)[-1] def normalize_wiki_path(href: str) -> str | None: """Normaliseer een wiki- of comment-URL naar een pad of None.""" if not href: return None href = href.strip().split(")", 1)[0].split()[0] m = _WIKI_SERIES.search(href) if m: path = m.group(1) # Typfout in de index: /wiki/settings/series/... → /wiki/series/... path = re.sub(r"/wiki/settings/series/", "/wiki/series/", path, flags=re.I) return _canonical_wiki_path(path) return None def _canonical_wiki_path(path: str) -> str: path = path.strip() if not path.startswith("/"): path = "/" + path parts = path.split("/") # /r/{sub}/wiki/series/{slug...} if len(parts) >= 3 and parts[1].lower() == "r": parts[2] = "hfy" return "/".join(parts).rstrip("/") def _clean_heading(raw: str) -> str: text = _MD_LINK.sub(r"\1", raw) text = _MD_DECOR.sub("", text) return re.sub(r"\s+", " ", text).strip() def _author_from_rest(rest: str) -> str | None: m = _AUTHOR_ITALIC.search(rest or "") if m: name = m.group(1).strip() return name.lstrip("u/").lstrip("/u/").strip() or None return None def parse_series_index(markdown: str) -> list[WikiSeries]: """Haal titel, optionele auteur en wiki-subpagina uit de hoofdpagina.""" seen: set[str] = set() out: list[WikiSeries] = [] for m in _LIST_ITEM.finditer(markdown or ""): title = _clean_heading(m.group(1)) href = m.group(2).strip() rest = m.group(3) or "" path = normalize_wiki_path(href) if not path or "/wiki/series/" not in path.lower(): continue key = path.lower() if key in seen: continue seen.add(key) if not title: title = path.rsplit("/", 1)[-1].replace("_", " ").title() out.append(WikiSeries(title=title, wiki_path=path, author=_author_from_rest(rest))) return out def _author_from_page(markdown: str) -> str | None: m = _AUTHOR_WIKI.search(markdown) if m: return m.group(1).strip() or m.group(2).strip() m = _STORY_BY.search(markdown) if m: return m.group(1).strip() or None # Alleen in de kop (voor de eerste heading of eerste comment-link) zoeken # naar u/naam, zodat hoofdstuktitels geen auteur worden. head = markdown[:2000] m = _AUTHOR_USER.search(head) if m: return (m.group(1) or m.group(2) or "").strip() or None return None def _title_from_page(markdown: str) -> str | None: for m in _HEADING.finditer(markdown or ""): title = _clean_heading(m.group(1)) if title and title.lower() not in {"all series", "series", "one shots", "oneshots"}: return title return None def parse_series_page( markdown: str, wiki_path: str, title: str | None = None, author: str | None = None, ) -> WikiSeries: """Haal auteur, titel en hoofdstuk-IDs (in wiki-volgorde) uit een subpagina.""" text = markdown or "" page_title = _title_from_page(text) or title or wiki_path.rsplit("/", 1)[-1].replace("_", " ").title() page_author = author or _author_from_page(text) chapter_ids: list[str] = [] seen: set[str] = set() # Alleen markdown/HTML-comment-links; geen footer "All Series". from .links import extract_labeled_links for link in extract_labeled_links(text): if link.id in seen: continue seen.add(link.id) chapter_ids.append(link.id) # Bare URLs die niet als markdown-link staan, achteraan (zeldzaam). from .links import extract_ids for pid in extract_ids(text): if pid not in seen: seen.add(pid) chapter_ids.append(pid) first = chapter_ids[0] if chapter_ids else None return WikiSeries( title=title or page_title, wiki_path=_canonical_wiki_path(wiki_path), author=page_author, chapter_ids=tuple(chapter_ids), first_id=first, page_found=True, ) def merge_page(entry: WikiSeries, page: WikiSeries) -> WikiSeries: """Vul een index-item aan met gegevens van de subpagina.""" entry.page_found = entry.page_found or page.page_found if page.author and not entry.author: entry.author = page.author if page.chapter_ids: entry.chapter_ids = page.chapter_ids entry.first_id = page.first_id or entry.first_id elif page.first_id and not entry.first_id: entry.first_id = page.first_id if page.revision_date: entry.revision_date = page.revision_date return entry def load_wiki_fixture(root: Path) -> list[WikiSeries]: """Lees testdata: index.md plus pages/.md.""" root = Path(root) index = (root / "index.md").read_text(encoding="utf-8") entries = parse_series_index(index) pages_dir = root / "pages" by_slug = {e.slug.lower(): e for e in entries} if pages_dir.is_dir(): for path in sorted(pages_dir.glob("*.md")): slug = path.stem.lower() entry = by_slug.get(slug) wiki_path = entry.wiki_path if entry else f"/r/hfy/wiki/series/{path.stem}" parsed = parse_series_page( path.read_text(encoding="utf-8"), wiki_path, title=entry.title if entry else None, author=entry.author if entry else None, ) if entry: merge_page(entry, parsed) else: entries.append(parsed) by_slug[slug] = parsed return entries def fetch_series_catalog( client: object, subreddit: str = "HFY", page_limit: int = 0, ) -> list[WikiSeries]: """Haal de wiki-index plus beschikbare subpagina's op via een ArcticClient. page_limit beperkt hoeveel subpagina's we opvragen (0 = allemaal). De complete lijst komt altijd uit de hoofdpagina. """ index_md, rev = _fetch_index(client) entries = parse_series_index(index_md) if not entries: return [] paths = [e.wiki_path for e in (entries[:page_limit] if page_limit else entries)] pages = client.wiki_pages(paths) # type: ignore[attr-defined] by_path = {_canonical_wiki_path(p.get("path") or "").lower(): p for p in pages if p.get("path")} for entry in entries: raw = by_path.get(entry.wiki_path.lower()) if not raw or not raw.get("content"): continue parsed = parse_series_page( raw["content"], raw.get("path") or entry.wiki_path, title=entry.title, author=entry.author, ) parsed.revision_date = raw.get("revision_date") merge_page(entry, parsed) return entries def _fetch_index(client: object) -> tuple[str, int | None]: pages = client.wiki_pages(list(SERIES_INDEX_PATHS)) # type: ignore[attr-defined] for page in pages: content = page.get("content") or "" if content.strip(): return content, page.get("revision_date") raise RuntimeError( "Kon de r/HFY-wiki-serie-index niet ophalen via Arctic Shift " f"({', '.join(SERIES_INDEX_PATHS)})." )