| 1 | """Parse en haal de officiële r/HFY-wiki-serie-index op. |
| 2 | |
| 3 | Bron: https://www.reddit.com/r/HFY/wiki/series/ (en gelinkte subpagina's). |
| 4 | Ongeauthenticeerde Reddit-JSON is sinds 2026 geblokkeerd (HTTP 403), daarom |
| 5 | lezen we de wikipagina's via Arctic Shift (`/api/subreddits/wikis`). Niet elke |
| 6 | subpagina staat in dat archief; de hoofdpagina met de complete lijst wel. |
| 7 | """ |
| 8 | |
| 9 | from __future__ import annotations |
| 10 | |
| 11 | import re |
| 12 | from dataclasses import dataclass, field |
| 13 | from pathlib import Path |
| 14 | |
| 15 | from .links import extract_post_id |
| 16 | |
| 17 | SERIES_INDEX_PATHS = ("/r/hfy/wiki/series", "/r/HFY/wiki/series") |
| 18 | SERIES_INDEX_URL = "https://www.reddit.com/r/HFY/wiki/series/" |
| 19 | |
| 20 | _LIST_ITEM = re.compile(r"^\s*[\*\-]\s*\[([^\]]+)\]\(([^)]+)\)(.*)$", re.M) |
| 21 | _AUTHOR_ITALIC = re.compile(r"\[\s*\*([^*]+)\*\s*\]") |
| 22 | _AUTHOR_WIKI = re.compile( |
| 23 | r"\[(?:\*\*)?([^\]*]+)(?:\*\*)?\]\((?:https?://(?:www\.)?reddit\.com)?/r/[^/]+/wiki/authors/([^)\s/]+)", |
| 24 | re.I, |
| 25 | ) |
| 26 | _AUTHOR_USER = re.compile( |
| 27 | r"(?:https?://(?:www\.)?reddit\.com)?/u(?:ser)?/([A-Za-z0-9_-]+)|(?<![/\w])u/([A-Za-z0-9_-]+)", |
| 28 | re.I, |
| 29 | ) |
| 30 | _STORY_BY = re.compile( |
| 31 | r"(?:a\s+story\s+by|author)\s*[:.\-–—]?\s*\[(?:\*\*)?([^\]*]+)(?:\*\*)?\]", |
| 32 | re.I, |
| 33 | ) |
| 34 | _HEADING = re.compile(r"^#{1,4}\s*(.+)$", re.M) |
| 35 | _MD_DECOR = re.compile(r"[*_`]+") |
| 36 | _MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]+\)") |
| 37 | _WIKI_SERIES = re.compile( |
| 38 | r"(?:https?://(?:www\.)?reddit\.com)?(/r/[^/]+/wiki/(?:series|settings/series)/[^)\s#]+)", |
| 39 | re.I, |
| 40 | ) |
| 41 | |
| 42 | |
| 43 | @dataclass |
| 44 | class WikiSeries: |
| 45 | """Eén item uit de wiki-serie-index, eventueel aangevuld vanuit de subpagina.""" |
| 46 | |
| 47 | title: str |
| 48 | wiki_path: str |
| 49 | author: str | None = None |
| 50 | chapter_ids: tuple[str, ...] = () |
| 51 | first_id: str | None = None |
| 52 | revision_date: int | None = None |
| 53 | page_found: bool = False |
| 54 | notes: str = "" |
| 55 | extra: dict = field(default_factory=dict) |
| 56 | |
| 57 | @property |
| 58 | def slug(self) -> str: |
| 59 | return self.wiki_path.rstrip("/").rsplit("/", 1)[-1] |
| 60 | |
| 61 | |
| 62 | def normalize_wiki_path(href: str) -> str | None: |
| 63 | """Normaliseer een wiki- of comment-URL naar een pad of None.""" |
| 64 | if not href: |
| 65 | return None |
| 66 | href = href.strip().split(")", 1)[0].split()[0] |
| 67 | m = _WIKI_SERIES.search(href) |
| 68 | if m: |
| 69 | path = m.group(1) |
| 70 | # Typfout in de index: /wiki/settings/series/... → /wiki/series/... |
| 71 | path = re.sub(r"/wiki/settings/series/", "/wiki/series/", path, flags=re.I) |
| 72 | return _canonical_wiki_path(path) |
| 73 | return None |
| 74 | |
| 75 | |
| 76 | def _canonical_wiki_path(path: str) -> str: |
| 77 | path = path.strip() |
| 78 | if not path.startswith("/"): |
| 79 | path = "/" + path |
| 80 | parts = path.split("/") |
| 81 | # /r/{sub}/wiki/series/{slug...} |
| 82 | if len(parts) >= 3 and parts[1].lower() == "r": |
| 83 | parts[2] = "hfy" |
| 84 | return "/".join(parts).rstrip("/") |
| 85 | |
| 86 | |
| 87 | def _clean_heading(raw: str) -> str: |
| 88 | text = _MD_LINK.sub(r"\1", raw) |
| 89 | text = _MD_DECOR.sub("", text) |
| 90 | return re.sub(r"\s+", " ", text).strip() |
| 91 | |
| 92 | |
| 93 | def _author_from_rest(rest: str) -> str | None: |
| 94 | m = _AUTHOR_ITALIC.search(rest or "") |
| 95 | if m: |
| 96 | name = m.group(1).strip() |
| 97 | return name.lstrip("u/").lstrip("/u/").strip() or None |
| 98 | return None |
| 99 | |
| 100 | |
| 101 | def parse_series_index(markdown: str) -> list[WikiSeries]: |
| 102 | """Haal titel, optionele auteur en wiki-subpagina uit de hoofdpagina.""" |
| 103 | seen: set[str] = set() |
| 104 | out: list[WikiSeries] = [] |
| 105 | for m in _LIST_ITEM.finditer(markdown or ""): |
| 106 | title = _clean_heading(m.group(1)) |
| 107 | href = m.group(2).strip() |
| 108 | rest = m.group(3) or "" |
| 109 | path = normalize_wiki_path(href) |
| 110 | if not path or "/wiki/series/" not in path.lower(): |
| 111 | continue |
| 112 | key = path.lower() |
| 113 | if key in seen: |
| 114 | continue |
| 115 | seen.add(key) |
| 116 | if not title: |
| 117 | title = path.rsplit("/", 1)[-1].replace("_", " ").title() |
| 118 | out.append(WikiSeries(title=title, wiki_path=path, author=_author_from_rest(rest))) |
| 119 | return out |
| 120 | |
| 121 | |
| 122 | def _author_from_page(markdown: str) -> str | None: |
| 123 | m = _AUTHOR_WIKI.search(markdown) |
| 124 | if m: |
| 125 | return m.group(1).strip() or m.group(2).strip() |
| 126 | m = _STORY_BY.search(markdown) |
| 127 | if m: |
| 128 | return m.group(1).strip() or None |
| 129 | # Alleen in de kop (voor de eerste heading of eerste comment-link) zoeken |
| 130 | # naar u/naam, zodat hoofdstuktitels geen auteur worden. |
| 131 | head = markdown[:2000] |
| 132 | m = _AUTHOR_USER.search(head) |
| 133 | if m: |
| 134 | return (m.group(1) or m.group(2) or "").strip() or None |
| 135 | return None |
| 136 | |
| 137 | |
| 138 | def _title_from_page(markdown: str) -> str | None: |
| 139 | for m in _HEADING.finditer(markdown or ""): |
| 140 | title = _clean_heading(m.group(1)) |
| 141 | if title and title.lower() not in {"all series", "series", "one shots", "oneshots"}: |
| 142 | return title |
| 143 | return None |
| 144 | |
| 145 | |
| 146 | def parse_series_page( |
| 147 | markdown: str, |
| 148 | wiki_path: str, |
| 149 | title: str | None = None, |
| 150 | author: str | None = None, |
| 151 | ) -> WikiSeries: |
| 152 | """Haal auteur, titel en hoofdstuk-IDs (in wiki-volgorde) uit een subpagina.""" |
| 153 | text = markdown or "" |
| 154 | page_title = _title_from_page(text) or title or wiki_path.rsplit("/", 1)[-1].replace("_", " ").title() |
| 155 | page_author = author or _author_from_page(text) |
| 156 | chapter_ids: list[str] = [] |
| 157 | seen: set[str] = set() |
| 158 | # Alleen markdown/HTML-comment-links; geen footer "All Series". |
| 159 | from .links import extract_labeled_links |
| 160 | |
| 161 | for link in extract_labeled_links(text): |
| 162 | if link.id in seen: |
| 163 | continue |
| 164 | seen.add(link.id) |
| 165 | chapter_ids.append(link.id) |
| 166 | # Bare URLs die niet als markdown-link staan, achteraan (zeldzaam). |
| 167 | from .links import extract_ids |
| 168 | |
| 169 | for pid in extract_ids(text): |
| 170 | if pid not in seen: |
| 171 | seen.add(pid) |
| 172 | chapter_ids.append(pid) |
| 173 | first = chapter_ids[0] if chapter_ids else None |
| 174 | return WikiSeries( |
| 175 | title=title or page_title, |
| 176 | wiki_path=_canonical_wiki_path(wiki_path), |
| 177 | author=page_author, |
| 178 | chapter_ids=tuple(chapter_ids), |
| 179 | first_id=first, |
| 180 | page_found=True, |
| 181 | ) |
| 182 | |
| 183 | |
| 184 | def merge_page(entry: WikiSeries, page: WikiSeries) -> WikiSeries: |
| 185 | """Vul een index-item aan met gegevens van de subpagina.""" |
| 186 | entry.page_found = entry.page_found or page.page_found |
| 187 | if page.author and not entry.author: |
| 188 | entry.author = page.author |
| 189 | if page.chapter_ids: |
| 190 | entry.chapter_ids = page.chapter_ids |
| 191 | entry.first_id = page.first_id or entry.first_id |
| 192 | elif page.first_id and not entry.first_id: |
| 193 | entry.first_id = page.first_id |
| 194 | if page.revision_date: |
| 195 | entry.revision_date = page.revision_date |
| 196 | return entry |
| 197 | |
| 198 | |
| 199 | def load_wiki_fixture(root: Path) -> list[WikiSeries]: |
| 200 | """Lees testdata: index.md plus pages/<slug>.md.""" |
| 201 | root = Path(root) |
| 202 | index = (root / "index.md").read_text(encoding="utf-8") |
| 203 | entries = parse_series_index(index) |
| 204 | pages_dir = root / "pages" |
| 205 | by_slug = {e.slug.lower(): e for e in entries} |
| 206 | if pages_dir.is_dir(): |
| 207 | for path in sorted(pages_dir.glob("*.md")): |
| 208 | slug = path.stem.lower() |
| 209 | entry = by_slug.get(slug) |
| 210 | wiki_path = entry.wiki_path if entry else f"/r/hfy/wiki/series/{path.stem}" |
| 211 | parsed = parse_series_page( |
| 212 | path.read_text(encoding="utf-8"), |
| 213 | wiki_path, |
| 214 | title=entry.title if entry else None, |
| 215 | author=entry.author if entry else None, |
| 216 | ) |
| 217 | if entry: |
| 218 | merge_page(entry, parsed) |
| 219 | else: |
| 220 | entries.append(parsed) |
| 221 | by_slug[slug] = parsed |
| 222 | return entries |
| 223 | |
| 224 | |
| 225 | def fetch_series_catalog( |
| 226 | client: object, |
| 227 | subreddit: str = "HFY", |
| 228 | page_limit: int = 0, |
| 229 | ) -> list[WikiSeries]: |
| 230 | """Haal de wiki-index plus beschikbare subpagina's op via een ArcticClient. |
| 231 | |
| 232 | page_limit beperkt hoeveel subpagina's we opvragen (0 = allemaal). De |
| 233 | complete lijst komt altijd uit de hoofdpagina. |
| 234 | """ |
| 235 | index_md, rev = _fetch_index(client) |
| 236 | entries = parse_series_index(index_md) |
| 237 | if not entries: |
| 238 | return [] |
| 239 | paths = [e.wiki_path for e in (entries[:page_limit] if page_limit else entries)] |
| 240 | pages = client.wiki_pages(paths) # type: ignore[attr-defined] |
| 241 | by_path = {_canonical_wiki_path(p.get("path") or "").lower(): p for p in pages if p.get("path")} |
| 242 | for entry in entries: |
| 243 | raw = by_path.get(entry.wiki_path.lower()) |
| 244 | if not raw or not raw.get("content"): |
| 245 | continue |
| 246 | parsed = parse_series_page( |
| 247 | raw["content"], |
| 248 | raw.get("path") or entry.wiki_path, |
| 249 | title=entry.title, |
| 250 | author=entry.author, |
| 251 | ) |
| 252 | parsed.revision_date = raw.get("revision_date") |
| 253 | merge_page(entry, parsed) |
| 254 | return entries |
| 255 | |
| 256 | |
| 257 | def _fetch_index(client: object) -> tuple[str, int | None]: |
| 258 | pages = client.wiki_pages(list(SERIES_INDEX_PATHS)) # type: ignore[attr-defined] |
| 259 | for page in pages: |
| 260 | content = page.get("content") or "" |
| 261 | if content.strip(): |
| 262 | return content, page.get("revision_date") |
| 263 | raise RuntimeError( |
| 264 | "Kon de r/HFY-wiki-serie-index niet ophalen via Arctic Shift " |
| 265 | f"({', '.join(SERIES_INDEX_PATHS)})." |
| 266 | ) |