| 1 | """Schijfcache voor post-metadata (geen selftext) zodat een onderbroken run verder kan.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import json |
| 6 | from pathlib import Path |
| 7 | |
| 8 | from .cluster import Post |
| 9 | from .links import Link, links_from_ids |
| 10 | |
| 11 | CACHE_DIRNAME = ".cache" |
| 12 | |
| 13 | |
| 14 | class PostCache: |
| 15 | def __init__(self, dest: Path) -> None: |
| 16 | self.root = Path(dest) / CACHE_DIRNAME / "posts" |
| 17 | self.root.mkdir(parents=True, exist_ok=True) |
| 18 | |
| 19 | def _path(self, pid: str) -> Path: |
| 20 | pid = pid.lower() |
| 21 | return self.root / pid[:2] / f"{pid}.json" |
| 22 | |
| 23 | def get(self, pid: str) -> Post | None: |
| 24 | path = self._path(pid) |
| 25 | if not path.exists(): |
| 26 | return None |
| 27 | try: |
| 28 | raw = json.loads(path.read_text(encoding="utf-8")) |
| 29 | except (OSError, json.JSONDecodeError): |
| 30 | return None |
| 31 | return _post_from_cache(raw) |
| 32 | |
| 33 | def put(self, post: Post) -> None: |
| 34 | path = self._path(post.id) |
| 35 | path.parent.mkdir(parents=True, exist_ok=True) |
| 36 | payload = { |
| 37 | "id": post.id, |
| 38 | "title": post.title, |
| 39 | "author": post.author, |
| 40 | "created_utc": post.created_utc, |
| 41 | "permalink": post.permalink, |
| 42 | "flair": post.flair, |
| 43 | "over_18": post.over_18, |
| 44 | "score": post.score, |
| 45 | "is_self": post.is_self, |
| 46 | "next_ids": list(post.next_ids), |
| 47 | "prev_ids": list(post.prev_ids), |
| 48 | "first_ids": list(post.first_ids), |
| 49 | "all_linked_ids": list(post.all_linked_ids), |
| 50 | "links": [{"id": ln.id, "text": ln.text, "kind": ln.kind} for ln in post.links], |
| 51 | } |
| 52 | path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8") |
| 53 | |
| 54 | def get_many(self, ids: list[str]) -> dict[str, Post]: |
| 55 | out: dict[str, Post] = {} |
| 56 | for pid in ids: |
| 57 | post = self.get(pid) |
| 58 | if post: |
| 59 | out[post.id] = post |
| 60 | return out |
| 61 | |
| 62 | |
| 63 | def _post_from_cache(raw: dict) -> Post | None: |
| 64 | pid = (raw.get("id") or "").strip().lower() |
| 65 | if not pid: |
| 66 | return None |
| 67 | links_raw = raw.get("links") or [] |
| 68 | links = tuple( |
| 69 | Link(id=str(x["id"]).lower(), text=str(x.get("text") or ""), kind=str(x.get("kind") or "other")) |
| 70 | for x in links_raw |
| 71 | if x.get("id") |
| 72 | ) |
| 73 | if not links: |
| 74 | links = links_from_ids( |
| 75 | raw.get("next_ids") or (), |
| 76 | raw.get("prev_ids") or (), |
| 77 | raw.get("first_ids") or (), |
| 78 | raw.get("all_linked_ids") or (), |
| 79 | ) |
| 80 | return Post( |
| 81 | id=pid, |
| 82 | title=raw.get("title") or "", |
| 83 | author=raw.get("author") or "[deleted]", |
| 84 | created_utc=int(raw.get("created_utc") or 0), |
| 85 | permalink=raw.get("permalink") or f"/r/HFY/comments/{pid}/", |
| 86 | flair=raw.get("flair"), |
| 87 | over_18=bool(raw.get("over_18")), |
| 88 | score=int(raw.get("score") or 0), |
| 89 | is_self=raw.get("is_self", True), |
| 90 | next_ids=tuple(raw.get("next_ids") or ()), |
| 91 | prev_ids=tuple(raw.get("prev_ids") or ()), |
| 92 | first_ids=tuple(raw.get("first_ids") or ()), |
| 93 | all_linked_ids=tuple(raw.get("all_linked_ids") or ()), |
| 94 | links=links, |
| 95 | ) |