BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / wiki.py

wiki.py

wiki.py 8.9 KiB Download Raw
1"""Parse en haal de officiële r/HFY-wiki-serie-index op.
2
3Bron: https://www.reddit.com/r/HFY/wiki/series/ (en gelinkte subpagina's).
4Ongeauthenticeerde Reddit-JSON is sinds 2026 geblokkeerd (HTTP 403), daarom
5lezen we de wikipagina's via Arctic Shift (`/api/subreddits/wikis`). Niet elke
6subpagina staat in dat archief; de hoofdpagina met de complete lijst wel.
7"""
8
9from __future__ import annotations
10
11import re
12from dataclasses import dataclass, field
13from pathlib import Path
14
15from .links import extract_post_id
16
17SERIES_INDEX_PATHS = ("/r/hfy/wiki/series", "/r/HFY/wiki/series")
18SERIES_INDEX_URL = "https://www.reddit.com/r/HFY/wiki/series/"
19
20_LIST_ITEM = re.compile(r"^\s*[\*\-]\s*\[([^\]]+)\]\(([^)]+)\)(.*)$", re.M)
21_AUTHOR_ITALIC = re.compile(r"\[\s*\*([^*]+)\*\s*\]")
22_AUTHOR_WIKI = re.compile(
23 r"\[(?:\*\*)?([^\]*]+)(?:\*\*)?\]\((?:https?://(?:www\.)?reddit\.com)?/r/[^/]+/wiki/authors/([^)\s/]+)",
24 re.I,
25)
26_AUTHOR_USER = re.compile(
27 r"(?:https?://(?:www\.)?reddit\.com)?/u(?:ser)?/([A-Za-z0-9_-]+)|(?<![/\w])u/([A-Za-z0-9_-]+)",
28 re.I,
29)
30_STORY_BY = re.compile(
31 r"(?:a\s+story\s+by|author)\s*[:.\-–—]?\s*\[(?:\*\*)?([^\]*]+)(?:\*\*)?\]",
32 re.I,
33)
34_HEADING = re.compile(r"^#{1,4}\s*(.+)$", re.M)
35_MD_DECOR = re.compile(r"[*_`]+")
36_MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]+\)")
37_WIKI_SERIES = re.compile(
38 r"(?:https?://(?:www\.)?reddit\.com)?(/r/[^/]+/wiki/(?:series|settings/series)/[^)\s#]+)",
39 re.I,
40)
41
42
43@dataclass
44class WikiSeries:
45 """Eén item uit de wiki-serie-index, eventueel aangevuld vanuit de subpagina."""
46
47 title: str
48 wiki_path: str
49 author: str | None = None
50 chapter_ids: tuple[str, ...] = ()
51 first_id: str | None = None
52 revision_date: int | None = None
53 page_found: bool = False
54 notes: str = ""
55 extra: dict = field(default_factory=dict)
56
57 @property
58 def slug(self) -> str:
59 return self.wiki_path.rstrip("/").rsplit("/", 1)[-1]
60
61
62def normalize_wiki_path(href: str) -> str | None:
63 """Normaliseer een wiki- of comment-URL naar een pad of None."""
64 if not href:
65 return None
66 href = href.strip().split(")", 1)[0].split()[0]
67 m = _WIKI_SERIES.search(href)
68 if m:
69 path = m.group(1)
70 # Typfout in de index: /wiki/settings/series/... → /wiki/series/...
71 path = re.sub(r"/wiki/settings/series/", "/wiki/series/", path, flags=re.I)
72 return _canonical_wiki_path(path)
73 return None
74
75
76def _canonical_wiki_path(path: str) -> str:
77 path = path.strip()
78 if not path.startswith("/"):
79 path = "/" + path
80 parts = path.split("/")
81 # /r/{sub}/wiki/series/{slug...}
82 if len(parts) >= 3 and parts[1].lower() == "r":
83 parts[2] = "hfy"
84 return "/".join(parts).rstrip("/")
85
86
87def _clean_heading(raw: str) -> str:
88 text = _MD_LINK.sub(r"\1", raw)
89 text = _MD_DECOR.sub("", text)
90 return re.sub(r"\s+", " ", text).strip()
91
92
93def _author_from_rest(rest: str) -> str | None:
94 m = _AUTHOR_ITALIC.search(rest or "")
95 if m:
96 name = m.group(1).strip()
97 return name.lstrip("u/").lstrip("/u/").strip() or None
98 return None
99
100
101def parse_series_index(markdown: str) -> list[WikiSeries]:
102 """Haal titel, optionele auteur en wiki-subpagina uit de hoofdpagina."""
103 seen: set[str] = set()
104 out: list[WikiSeries] = []
105 for m in _LIST_ITEM.finditer(markdown or ""):
106 title = _clean_heading(m.group(1))
107 href = m.group(2).strip()
108 rest = m.group(3) or ""
109 path = normalize_wiki_path(href)
110 if not path or "/wiki/series/" not in path.lower():
111 continue
112 key = path.lower()
113 if key in seen:
114 continue
115 seen.add(key)
116 if not title:
117 title = path.rsplit("/", 1)[-1].replace("_", " ").title()
118 out.append(WikiSeries(title=title, wiki_path=path, author=_author_from_rest(rest)))
119 return out
120
121
122def _author_from_page(markdown: str) -> str | None:
123 m = _AUTHOR_WIKI.search(markdown)
124 if m:
125 return m.group(1).strip() or m.group(2).strip()
126 m = _STORY_BY.search(markdown)
127 if m:
128 return m.group(1).strip() or None
129 # Alleen in de kop (voor de eerste heading of eerste comment-link) zoeken
130 # naar u/naam, zodat hoofdstuktitels geen auteur worden.
131 head = markdown[:2000]
132 m = _AUTHOR_USER.search(head)
133 if m:
134 return (m.group(1) or m.group(2) or "").strip() or None
135 return None
136
137
138def _title_from_page(markdown: str) -> str | None:
139 for m in _HEADING.finditer(markdown or ""):
140 title = _clean_heading(m.group(1))
141 if title and title.lower() not in {"all series", "series", "one shots", "oneshots"}:
142 return title
143 return None
144
145
146def parse_series_page(
147 markdown: str,
148 wiki_path: str,
149 title: str | None = None,
150 author: str | None = None,
151) -> WikiSeries:
152 """Haal auteur, titel en hoofdstuk-IDs (in wiki-volgorde) uit een subpagina."""
153 text = markdown or ""
154 page_title = _title_from_page(text) or title or wiki_path.rsplit("/", 1)[-1].replace("_", " ").title()
155 page_author = author or _author_from_page(text)
156 chapter_ids: list[str] = []
157 seen: set[str] = set()
158 # Alleen markdown/HTML-comment-links; geen footer "All Series".
159 from .links import extract_labeled_links
160
161 for link in extract_labeled_links(text):
162 if link.id in seen:
163 continue
164 seen.add(link.id)
165 chapter_ids.append(link.id)
166 # Bare URLs die niet als markdown-link staan, achteraan (zeldzaam).
167 from .links import extract_ids
168
169 for pid in extract_ids(text):
170 if pid not in seen:
171 seen.add(pid)
172 chapter_ids.append(pid)
173 first = chapter_ids[0] if chapter_ids else None
174 return WikiSeries(
175 title=title or page_title,
176 wiki_path=_canonical_wiki_path(wiki_path),
177 author=page_author,
178 chapter_ids=tuple(chapter_ids),
179 first_id=first,
180 page_found=True,
181 )
182
183
184def merge_page(entry: WikiSeries, page: WikiSeries) -> WikiSeries:
185 """Vul een index-item aan met gegevens van de subpagina."""
186 entry.page_found = entry.page_found or page.page_found
187 if page.author and not entry.author:
188 entry.author = page.author
189 if page.chapter_ids:
190 entry.chapter_ids = page.chapter_ids
191 entry.first_id = page.first_id or entry.first_id
192 elif page.first_id and not entry.first_id:
193 entry.first_id = page.first_id
194 if page.revision_date:
195 entry.revision_date = page.revision_date
196 return entry
197
198
199def load_wiki_fixture(root: Path) -> list[WikiSeries]:
200 """Lees testdata: index.md plus pages/<slug>.md."""
201 root = Path(root)
202 index = (root / "index.md").read_text(encoding="utf-8")
203 entries = parse_series_index(index)
204 pages_dir = root / "pages"
205 by_slug = {e.slug.lower(): e for e in entries}
206 if pages_dir.is_dir():
207 for path in sorted(pages_dir.glob("*.md")):
208 slug = path.stem.lower()
209 entry = by_slug.get(slug)
210 wiki_path = entry.wiki_path if entry else f"/r/hfy/wiki/series/{path.stem}"
211 parsed = parse_series_page(
212 path.read_text(encoding="utf-8"),
213 wiki_path,
214 title=entry.title if entry else None,
215 author=entry.author if entry else None,
216 )
217 if entry:
218 merge_page(entry, parsed)
219 else:
220 entries.append(parsed)
221 by_slug[slug] = parsed
222 return entries
223
224
225def fetch_series_catalog(
226 client: object,
227 subreddit: str = "HFY",
228 page_limit: int = 0,
229) -> list[WikiSeries]:
230 """Haal de wiki-index plus beschikbare subpagina's op via een ArcticClient.
231
232 page_limit beperkt hoeveel subpagina's we opvragen (0 = allemaal). De
233 complete lijst komt altijd uit de hoofdpagina.
234 """
235 index_md, rev = _fetch_index(client)
236 entries = parse_series_index(index_md)
237 if not entries:
238 return []
239 paths = [e.wiki_path for e in (entries[:page_limit] if page_limit else entries)]
240 pages = client.wiki_pages(paths) # type: ignore[attr-defined]
241 by_path = {_canonical_wiki_path(p.get("path") or "").lower(): p for p in pages if p.get("path")}
242 for entry in entries:
243 raw = by_path.get(entry.wiki_path.lower())
244 if not raw or not raw.get("content"):
245 continue
246 parsed = parse_series_page(
247 raw["content"],
248 raw.get("path") or entry.wiki_path,
249 title=entry.title,
250 author=entry.author,
251 )
252 parsed.revision_date = raw.get("revision_date")
253 merge_page(entry, parsed)
254 return entries
255
256
257def _fetch_index(client: object) -> tuple[str, int | None]:
258 pages = client.wiki_pages(list(SERIES_INDEX_PATHS)) # type: ignore[attr-defined]
259 for page in pages:
260 content = page.get("content") or ""
261 if content.strip():
262 return content, page.get("revision_date")
263 raise RuntimeError(
264 "Kon de r/HFY-wiki-serie-index niet ophalen via Arctic Shift "
265 f"({', '.join(SERIES_INDEX_PATHS)})."
266 )