"""Dunne client voor de publieke Arctic Shift-API (geen Reddit-OAuth).""" from __future__ import annotations import json import time import urllib.error import urllib.parse import urllib.request from typing import Iterator DEFAULT_BASE = "https://arctic-shift.photon-reddit.com" DEFAULT_UA = "HFY-Novels/0.1 (index builder; +https://git.timfalken.com/timfa/hfy-novels)" # Velden die we nodig hebben. selftext alleen om links te vinden; die gaat niet de index in. # Alleen velden die Arctic Shift als `fields=` accepteert. # is_self en permalink zitten niet in die lijst; die leiden we af uit url/selftext. POST_FIELDS = ",".join( [ "id", "title", "author", "created_utc", "selftext", "link_flair_text", "over_18", "score", "subreddit", "url", ] ) class ArcticError(RuntimeError): def __init__(self, message: str, status: int | None = None) -> None: super().__init__(message) self.status = status class ArcticClient: def __init__( self, base: str = DEFAULT_BASE, user_agent: str = DEFAULT_UA, min_interval: float = 0.4, timeout: int = 60, retries: int = 5, ) -> None: self.base = base.rstrip("/") self.user_agent = user_agent self.min_interval = min_interval self.timeout = timeout self.retries = retries self.max_calls = 0 self.calls = 0 self._last = 0.0 def remaining_calls(self) -> int | None: if not self.max_calls: return None return max(0, self.max_calls - self.calls) def _count_call(self) -> None: self.calls += 1 if self.max_calls and self.calls > self.max_calls: raise ArcticError(f"API-budget op ({self.max_calls} verzoeken)", status=None) def _sleep(self) -> None: now = time.monotonic() wait = self.min_interval - (now - self._last) if wait > 0: time.sleep(wait) def get_json(self, path: str, params: dict | None = None) -> dict: url = self.base + path if params: url += "?" + urllib.parse.urlencode({k: v for k, v in params.items() if v is not None}) last_err: Exception | None = None for attempt in range(self.retries): if self.max_calls and self.calls >= self.max_calls: raise ArcticError(f"API-budget op ({self.max_calls} verzoeken)", status=None) self._sleep() req = urllib.request.Request( url, headers={ "User-Agent": self.user_agent, "Accept": "application/json", }, ) try: self._count_call() with urllib.request.urlopen(req, timeout=self.timeout) as resp: self._last = time.monotonic() raw = resp.read() return json.loads(raw.decode("utf-8")) except urllib.error.HTTPError as exc: self._last = time.monotonic() body = exc.read()[:400] if exc.fp else b"" if exc.code in {422, 429, 500, 502, 503, 504} and attempt + 1 < self.retries: time.sleep(min(30, 2 ** attempt)) last_err = ArcticError(f"HTTP {exc.code} for {path}: {body!r}", exc.code) continue raise ArcticError(f"HTTP {exc.code} for {path}: {body!r}", exc.code) from exc except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: self._last = time.monotonic() last_err = exc if attempt + 1 < self.retries: time.sleep(min(30, 2 ** attempt)) continue raise ArcticError(f"request failed for {path}: {exc}") from exc raise ArcticError(f"request failed for {path}: {last_err}") def iter_posts( self, subreddit: str = "HFY", after: int | str | None = None, before: int | str | None = None, limit: str | int = "auto", max_pages: int = 0, newest_first: bool = True, ) -> Iterator[dict]: """Paginerend ophalen. newest_first=True gebruikt before=oudste van de pagina.""" pages = 0 cursor_before = before cursor_after = after while True: if max_pages and pages >= max_pages: return params = { "subreddit": subreddit, "limit": str(limit), "sort": "desc" if newest_first else "asc", } if cursor_before is not None: params["before"] = str(cursor_before) if cursor_after is not None and not newest_first: params["after"] = str(cursor_after) data = self.get_json("/api/posts/search", params) posts = data.get("data") if isinstance(data, dict) else data if not posts: return pages += 1 for post in posts: yield post if newest_first: oldest = min(int(p.get("created_utc") or 0) for p in posts) # Als de API dezelfde pagina teruggeeft, stoppen. if cursor_before is not None and int(cursor_before) <= oldest: return cursor_before = oldest else: newest = max(int(p.get("created_utc") or 0) for p in posts) if cursor_after is not None and int(cursor_after) >= newest: return cursor_after = newest if len(posts) < 10: return def posts_by_ids(self, ids: list[str], md2html: bool = False) -> list[dict]: out: list[dict] = [] for i in range(0, len(ids), 100): chunk = ids[i : i + 100] params = {"ids": ",".join(chunk)} if md2html: params["md2html"] = "true" data = self.get_json("/api/posts/ids", params) posts = data.get("data") if isinstance(data, dict) else data if posts: out.extend(posts) return out def search_posts( self, subreddit: str | None = None, author: str | None = None, title: str | None = None, query: str | None = None, after: int | str | None = None, before: int | str | None = None, limit: int = 25, sort: str = "asc", ) -> list[dict]: """Zoek posts. title= op r/HFY alleen geeft 422 (te actieve subreddit).""" params = { "limit": str(limit), "sort": sort, } if subreddit: params["subreddit"] = subreddit if author: params["author"] = author if title: params["title"] = title if query: params["query"] = query if after is not None: params["after"] = str(after) if before is not None: params["before"] = str(before) data = self.get_json("/api/posts/search", params) posts = data.get("data") if isinstance(data, dict) else data return list(posts or []) def wiki_list(self, subreddit: str = "HFY") -> list[str]: data = self.get_json("/api/subreddits/wikis/list", {"subreddit": subreddit}) paths = data.get("data") if isinstance(data, dict) else data return [p for p in (paths or []) if isinstance(p, str)] def wiki_pages(self, paths: list[str], batch_size: int = 80) -> list[dict]: """Haal wikipagina's op in batches (Arctic Shift heeft geen content voor elke path).""" out: list[dict] = [] # Uniek, behoud volgorde. seen: set[str] = set() clean: list[str] = [] for path in paths: key = (path or "").strip() if not key or key.lower() in seen: continue seen.add(key.lower()) clean.append(key) for i in range(0, len(clean), batch_size): chunk = clean[i : i + batch_size] try: data = self.get_json("/api/subreddits/wikis", {"paths": ",".join(chunk)}) except ArcticError as exc: if exc.status is None and "budget" in str(exc).lower(): return out raise pages = data.get("data") if isinstance(data, dict) else data if pages: out.extend(pages) return out