BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / arctic.py

arctic.py

arctic.py 8.4 KiB Download Raw
1"""Dunne client voor de publieke Arctic Shift-API (geen Reddit-OAuth)."""
2
3from __future__ import annotations
4
5import json
6import time
7import urllib.error
8import urllib.parse
9import urllib.request
10from typing import Iterator
11
12DEFAULT_BASE = "https://arctic-shift.photon-reddit.com"
13DEFAULT_UA = "HFY-Novels/0.1 (index builder; +https://git.timfalken.com/timfa/hfy-novels)"
14# Velden die we nodig hebben. selftext alleen om links te vinden; die gaat niet de index in.
15# Alleen velden die Arctic Shift als `fields=` accepteert.
16# is_self en permalink zitten niet in die lijst; die leiden we af uit url/selftext.
17POST_FIELDS = ",".join(
18 [
19 "id",
20 "title",
21 "author",
22 "created_utc",
23 "selftext",
24 "link_flair_text",
25 "over_18",
26 "score",
27 "subreddit",
28 "url",
29 ]
30)
31
32
33class ArcticError(RuntimeError):
34 def __init__(self, message: str, status: int | None = None) -> None:
35 super().__init__(message)
36 self.status = status
37
38
39class ArcticClient:
40 def __init__(
41 self,
42 base: str = DEFAULT_BASE,
43 user_agent: str = DEFAULT_UA,
44 min_interval: float = 0.4,
45 timeout: int = 60,
46 retries: int = 5,
47 ) -> None:
48 self.base = base.rstrip("/")
49 self.user_agent = user_agent
50 self.min_interval = min_interval
51 self.timeout = timeout
52 self.retries = retries
53 self.max_calls = 0
54 self.calls = 0
55 self._last = 0.0
56
57 def remaining_calls(self) -> int | None:
58 if not self.max_calls:
59 return None
60 return max(0, self.max_calls - self.calls)
61
62 def _count_call(self) -> None:
63 self.calls += 1
64 if self.max_calls and self.calls > self.max_calls:
65 raise ArcticError(f"API-budget op ({self.max_calls} verzoeken)", status=None)
66
67 def _sleep(self) -> None:
68 now = time.monotonic()
69 wait = self.min_interval - (now - self._last)
70 if wait > 0:
71 time.sleep(wait)
72
73 def get_json(self, path: str, params: dict | None = None) -> dict:
74 url = self.base + path
75 if params:
76 url += "?" + urllib.parse.urlencode({k: v for k, v in params.items() if v is not None})
77 last_err: Exception | None = None
78 for attempt in range(self.retries):
79 if self.max_calls and self.calls >= self.max_calls:
80 raise ArcticError(f"API-budget op ({self.max_calls} verzoeken)", status=None)
81 self._sleep()
82 req = urllib.request.Request(
83 url,
84 headers={
85 "User-Agent": self.user_agent,
86 "Accept": "application/json",
87 },
88 )
89 try:
90 self._count_call()
91 with urllib.request.urlopen(req, timeout=self.timeout) as resp:
92 self._last = time.monotonic()
93 raw = resp.read()
94 return json.loads(raw.decode("utf-8"))
95 except urllib.error.HTTPError as exc:
96 self._last = time.monotonic()
97 body = exc.read()[:400] if exc.fp else b""
98 if exc.code in {422, 429, 500, 502, 503, 504} and attempt + 1 < self.retries:
99 time.sleep(min(30, 2 ** attempt))
100 last_err = ArcticError(f"HTTP {exc.code} for {path}: {body!r}", exc.code)
101 continue
102 raise ArcticError(f"HTTP {exc.code} for {path}: {body!r}", exc.code) from exc
103 except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
104 self._last = time.monotonic()
105 last_err = exc
106 if attempt + 1 < self.retries:
107 time.sleep(min(30, 2 ** attempt))
108 continue
109 raise ArcticError(f"request failed for {path}: {exc}") from exc
110 raise ArcticError(f"request failed for {path}: {last_err}")
111
112 def iter_posts(
113 self,
114 subreddit: str = "HFY",
115 after: int | str | None = None,
116 before: int | str | None = None,
117 limit: str | int = "auto",
118 max_pages: int = 0,
119 newest_first: bool = True,
120 ) -> Iterator[dict]:
121 """Paginerend ophalen. newest_first=True gebruikt before=oudste van de pagina."""
122 pages = 0
123 cursor_before = before
124 cursor_after = after
125 while True:
126 if max_pages and pages >= max_pages:
127 return
128 params = {
129 "subreddit": subreddit,
130 "limit": str(limit),
131 "sort": "desc" if newest_first else "asc",
132 }
133 if cursor_before is not None:
134 params["before"] = str(cursor_before)
135 if cursor_after is not None and not newest_first:
136 params["after"] = str(cursor_after)
137 data = self.get_json("/api/posts/search", params)
138 posts = data.get("data") if isinstance(data, dict) else data
139 if not posts:
140 return
141 pages += 1
142 for post in posts:
143 yield post
144 if newest_first:
145 oldest = min(int(p.get("created_utc") or 0) for p in posts)
146 # Als de API dezelfde pagina teruggeeft, stoppen.
147 if cursor_before is not None and int(cursor_before) <= oldest:
148 return
149 cursor_before = oldest
150 else:
151 newest = max(int(p.get("created_utc") or 0) for p in posts)
152 if cursor_after is not None and int(cursor_after) >= newest:
153 return
154 cursor_after = newest
155 if len(posts) < 10:
156 return
157
158 def posts_by_ids(self, ids: list[str], md2html: bool = False) -> list[dict]:
159 out: list[dict] = []
160 for i in range(0, len(ids), 100):
161 chunk = ids[i : i + 100]
162 params = {"ids": ",".join(chunk)}
163 if md2html:
164 params["md2html"] = "true"
165 data = self.get_json("/api/posts/ids", params)
166 posts = data.get("data") if isinstance(data, dict) else data
167 if posts:
168 out.extend(posts)
169 return out
170
171 def search_posts(
172 self,
173 subreddit: str | None = None,
174 author: str | None = None,
175 title: str | None = None,
176 query: str | None = None,
177 after: int | str | None = None,
178 before: int | str | None = None,
179 limit: int = 25,
180 sort: str = "asc",
181 ) -> list[dict]:
182 """Zoek posts. title= op r/HFY alleen geeft 422 (te actieve subreddit)."""
183 params = {
184 "limit": str(limit),
185 "sort": sort,
186 }
187 if subreddit:
188 params["subreddit"] = subreddit
189 if author:
190 params["author"] = author
191 if title:
192 params["title"] = title
193 if query:
194 params["query"] = query
195 if after is not None:
196 params["after"] = str(after)
197 if before is not None:
198 params["before"] = str(before)
199 data = self.get_json("/api/posts/search", params)
200 posts = data.get("data") if isinstance(data, dict) else data
201 return list(posts or [])
202
203 def wiki_list(self, subreddit: str = "HFY") -> list[str]:
204 data = self.get_json("/api/subreddits/wikis/list", {"subreddit": subreddit})
205 paths = data.get("data") if isinstance(data, dict) else data
206 return [p for p in (paths or []) if isinstance(p, str)]
207
208 def wiki_pages(self, paths: list[str], batch_size: int = 80) -> list[dict]:
209 """Haal wikipagina's op in batches (Arctic Shift heeft geen content voor elke path)."""
210 out: list[dict] = []
211 # Uniek, behoud volgorde.
212 seen: set[str] = set()
213 clean: list[str] = []
214 for path in paths:
215 key = (path or "").strip()
216 if not key or key.lower() in seen:
217 continue
218 seen.add(key.lower())
219 clean.append(key)
220 for i in range(0, len(clean), batch_size):
221 chunk = clean[i : i + batch_size]
222 try:
223 data = self.get_json("/api/subreddits/wikis", {"paths": ",".join(chunk)})
224 except ArcticError as exc:
225 if exc.status is None and "budget" in str(exc).lower():
226 return out
227 raise
228 pages = data.get("data") if isinstance(data, dict) else data
229 if pages:
230 out.extend(pages)
231 return out