| 1 | """Dunne client voor de publieke Arctic Shift-API (geen Reddit-OAuth).""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import json |
| 6 | import time |
| 7 | import urllib.error |
| 8 | import urllib.parse |
| 9 | import urllib.request |
| 10 | from typing import Iterator |
| 11 | |
| 12 | DEFAULT_BASE = "https://arctic-shift.photon-reddit.com" |
| 13 | DEFAULT_UA = "HFY-Novels/0.1 (index builder; +https://git.timfalken.com/timfa/hfy-novels)" |
| 14 | # Velden die we nodig hebben. selftext alleen om links te vinden; die gaat niet de index in. |
| 15 | # Alleen velden die Arctic Shift als `fields=` accepteert. |
| 16 | # is_self en permalink zitten niet in die lijst; die leiden we af uit url/selftext. |
| 17 | POST_FIELDS = ",".join( |
| 18 | [ |
| 19 | "id", |
| 20 | "title", |
| 21 | "author", |
| 22 | "created_utc", |
| 23 | "selftext", |
| 24 | "link_flair_text", |
| 25 | "over_18", |
| 26 | "score", |
| 27 | "subreddit", |
| 28 | "url", |
| 29 | ] |
| 30 | ) |
| 31 | |
| 32 | |
| 33 | class ArcticError(RuntimeError): |
| 34 | def __init__(self, message: str, status: int | None = None) -> None: |
| 35 | super().__init__(message) |
| 36 | self.status = status |
| 37 | |
| 38 | |
| 39 | class ArcticClient: |
| 40 | def __init__( |
| 41 | self, |
| 42 | base: str = DEFAULT_BASE, |
| 43 | user_agent: str = DEFAULT_UA, |
| 44 | min_interval: float = 0.4, |
| 45 | timeout: int = 60, |
| 46 | retries: int = 5, |
| 47 | ) -> None: |
| 48 | self.base = base.rstrip("/") |
| 49 | self.user_agent = user_agent |
| 50 | self.min_interval = min_interval |
| 51 | self.timeout = timeout |
| 52 | self.retries = retries |
| 53 | self.max_calls = 0 |
| 54 | self.calls = 0 |
| 55 | self._last = 0.0 |
| 56 | |
| 57 | def remaining_calls(self) -> int | None: |
| 58 | if not self.max_calls: |
| 59 | return None |
| 60 | return max(0, self.max_calls - self.calls) |
| 61 | |
| 62 | def _count_call(self) -> None: |
| 63 | self.calls += 1 |
| 64 | if self.max_calls and self.calls > self.max_calls: |
| 65 | raise ArcticError(f"API-budget op ({self.max_calls} verzoeken)", status=None) |
| 66 | |
| 67 | def _sleep(self) -> None: |
| 68 | now = time.monotonic() |
| 69 | wait = self.min_interval - (now - self._last) |
| 70 | if wait > 0: |
| 71 | time.sleep(wait) |
| 72 | |
| 73 | def get_json(self, path: str, params: dict | None = None) -> dict: |
| 74 | url = self.base + path |
| 75 | if params: |
| 76 | url += "?" + urllib.parse.urlencode({k: v for k, v in params.items() if v is not None}) |
| 77 | last_err: Exception | None = None |
| 78 | for attempt in range(self.retries): |
| 79 | if self.max_calls and self.calls >= self.max_calls: |
| 80 | raise ArcticError(f"API-budget op ({self.max_calls} verzoeken)", status=None) |
| 81 | self._sleep() |
| 82 | req = urllib.request.Request( |
| 83 | url, |
| 84 | headers={ |
| 85 | "User-Agent": self.user_agent, |
| 86 | "Accept": "application/json", |
| 87 | }, |
| 88 | ) |
| 89 | try: |
| 90 | self._count_call() |
| 91 | with urllib.request.urlopen(req, timeout=self.timeout) as resp: |
| 92 | self._last = time.monotonic() |
| 93 | raw = resp.read() |
| 94 | return json.loads(raw.decode("utf-8")) |
| 95 | except urllib.error.HTTPError as exc: |
| 96 | self._last = time.monotonic() |
| 97 | body = exc.read()[:400] if exc.fp else b"" |
| 98 | if exc.code in {422, 429, 500, 502, 503, 504} and attempt + 1 < self.retries: |
| 99 | time.sleep(min(30, 2 ** attempt)) |
| 100 | last_err = ArcticError(f"HTTP {exc.code} for {path}: {body!r}", exc.code) |
| 101 | continue |
| 102 | raise ArcticError(f"HTTP {exc.code} for {path}: {body!r}", exc.code) from exc |
| 103 | except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: |
| 104 | self._last = time.monotonic() |
| 105 | last_err = exc |
| 106 | if attempt + 1 < self.retries: |
| 107 | time.sleep(min(30, 2 ** attempt)) |
| 108 | continue |
| 109 | raise ArcticError(f"request failed for {path}: {exc}") from exc |
| 110 | raise ArcticError(f"request failed for {path}: {last_err}") |
| 111 | |
| 112 | def iter_posts( |
| 113 | self, |
| 114 | subreddit: str = "HFY", |
| 115 | after: int | str | None = None, |
| 116 | before: int | str | None = None, |
| 117 | limit: str | int = "auto", |
| 118 | max_pages: int = 0, |
| 119 | newest_first: bool = True, |
| 120 | ) -> Iterator[dict]: |
| 121 | """Paginerend ophalen. newest_first=True gebruikt before=oudste van de pagina.""" |
| 122 | pages = 0 |
| 123 | cursor_before = before |
| 124 | cursor_after = after |
| 125 | while True: |
| 126 | if max_pages and pages >= max_pages: |
| 127 | return |
| 128 | params = { |
| 129 | "subreddit": subreddit, |
| 130 | "limit": str(limit), |
| 131 | "sort": "desc" if newest_first else "asc", |
| 132 | } |
| 133 | if cursor_before is not None: |
| 134 | params["before"] = str(cursor_before) |
| 135 | if cursor_after is not None and not newest_first: |
| 136 | params["after"] = str(cursor_after) |
| 137 | data = self.get_json("/api/posts/search", params) |
| 138 | posts = data.get("data") if isinstance(data, dict) else data |
| 139 | if not posts: |
| 140 | return |
| 141 | pages += 1 |
| 142 | for post in posts: |
| 143 | yield post |
| 144 | if newest_first: |
| 145 | oldest = min(int(p.get("created_utc") or 0) for p in posts) |
| 146 | # Als de API dezelfde pagina teruggeeft, stoppen. |
| 147 | if cursor_before is not None and int(cursor_before) <= oldest: |
| 148 | return |
| 149 | cursor_before = oldest |
| 150 | else: |
| 151 | newest = max(int(p.get("created_utc") or 0) for p in posts) |
| 152 | if cursor_after is not None and int(cursor_after) >= newest: |
| 153 | return |
| 154 | cursor_after = newest |
| 155 | if len(posts) < 10: |
| 156 | return |
| 157 | |
| 158 | def posts_by_ids(self, ids: list[str], md2html: bool = False) -> list[dict]: |
| 159 | out: list[dict] = [] |
| 160 | for i in range(0, len(ids), 100): |
| 161 | chunk = ids[i : i + 100] |
| 162 | params = {"ids": ",".join(chunk)} |
| 163 | if md2html: |
| 164 | params["md2html"] = "true" |
| 165 | data = self.get_json("/api/posts/ids", params) |
| 166 | posts = data.get("data") if isinstance(data, dict) else data |
| 167 | if posts: |
| 168 | out.extend(posts) |
| 169 | return out |
| 170 | |
| 171 | def search_posts( |
| 172 | self, |
| 173 | subreddit: str | None = None, |
| 174 | author: str | None = None, |
| 175 | title: str | None = None, |
| 176 | query: str | None = None, |
| 177 | after: int | str | None = None, |
| 178 | before: int | str | None = None, |
| 179 | limit: int = 25, |
| 180 | sort: str = "asc", |
| 181 | ) -> list[dict]: |
| 182 | """Zoek posts. title= op r/HFY alleen geeft 422 (te actieve subreddit).""" |
| 183 | params = { |
| 184 | "limit": str(limit), |
| 185 | "sort": sort, |
| 186 | } |
| 187 | if subreddit: |
| 188 | params["subreddit"] = subreddit |
| 189 | if author: |
| 190 | params["author"] = author |
| 191 | if title: |
| 192 | params["title"] = title |
| 193 | if query: |
| 194 | params["query"] = query |
| 195 | if after is not None: |
| 196 | params["after"] = str(after) |
| 197 | if before is not None: |
| 198 | params["before"] = str(before) |
| 199 | data = self.get_json("/api/posts/search", params) |
| 200 | posts = data.get("data") if isinstance(data, dict) else data |
| 201 | return list(posts or []) |
| 202 | |
| 203 | def wiki_list(self, subreddit: str = "HFY") -> list[str]: |
| 204 | data = self.get_json("/api/subreddits/wikis/list", {"subreddit": subreddit}) |
| 205 | paths = data.get("data") if isinstance(data, dict) else data |
| 206 | return [p for p in (paths or []) if isinstance(p, str)] |
| 207 | |
| 208 | def wiki_pages(self, paths: list[str], batch_size: int = 80) -> list[dict]: |
| 209 | """Haal wikipagina's op in batches (Arctic Shift heeft geen content voor elke path).""" |
| 210 | out: list[dict] = [] |
| 211 | # Uniek, behoud volgorde. |
| 212 | seen: set[str] = set() |
| 213 | clean: list[str] = [] |
| 214 | for path in paths: |
| 215 | key = (path or "").strip() |
| 216 | if not key or key.lower() in seen: |
| 217 | continue |
| 218 | seen.add(key.lower()) |
| 219 | clean.append(key) |
| 220 | for i in range(0, len(clean), batch_size): |
| 221 | chunk = clean[i : i + batch_size] |
| 222 | try: |
| 223 | data = self.get_json("/api/subreddits/wikis", {"paths": ",".join(chunk)}) |
| 224 | except ArcticError as exc: |
| 225 | if exc.status is None and "budget" in str(exc).lower(): |
| 226 | return out |
| 227 | raise |
| 228 | pages = data.get("data") if isinstance(data, dict) else data |
| 229 | if pages: |
| 230 | out.extend(pages) |
| 231 | return out |