BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / cache.py

cache.py

cache.py 3.1 KiB Download Raw
1"""Schijfcache voor post-metadata (geen selftext) zodat een onderbroken run verder kan."""
2
3from __future__ import annotations
4
5import json
6from pathlib import Path
7
8from .cluster import Post
9from .links import Link, links_from_ids
10
11CACHE_DIRNAME = ".cache"
12
13
14class PostCache:
15 def __init__(self, dest: Path) -> None:
16 self.root = Path(dest) / CACHE_DIRNAME / "posts"
17 self.root.mkdir(parents=True, exist_ok=True)
18
19 def _path(self, pid: str) -> Path:
20 pid = pid.lower()
21 return self.root / pid[:2] / f"{pid}.json"
22
23 def get(self, pid: str) -> Post | None:
24 path = self._path(pid)
25 if not path.exists():
26 return None
27 try:
28 raw = json.loads(path.read_text(encoding="utf-8"))
29 except (OSError, json.JSONDecodeError):
30 return None
31 return _post_from_cache(raw)
32
33 def put(self, post: Post) -> None:
34 path = self._path(post.id)
35 path.parent.mkdir(parents=True, exist_ok=True)
36 payload = {
37 "id": post.id,
38 "title": post.title,
39 "author": post.author,
40 "created_utc": post.created_utc,
41 "permalink": post.permalink,
42 "flair": post.flair,
43 "over_18": post.over_18,
44 "score": post.score,
45 "is_self": post.is_self,
46 "next_ids": list(post.next_ids),
47 "prev_ids": list(post.prev_ids),
48 "first_ids": list(post.first_ids),
49 "all_linked_ids": list(post.all_linked_ids),
50 "links": [{"id": ln.id, "text": ln.text, "kind": ln.kind} for ln in post.links],
51 }
52 path.write_text(json.dumps(payload, ensure_ascii=False) + "\n", encoding="utf-8")
53
54 def get_many(self, ids: list[str]) -> dict[str, Post]:
55 out: dict[str, Post] = {}
56 for pid in ids:
57 post = self.get(pid)
58 if post:
59 out[post.id] = post
60 return out
61
62
63def _post_from_cache(raw: dict) -> Post | None:
64 pid = (raw.get("id") or "").strip().lower()
65 if not pid:
66 return None
67 links_raw = raw.get("links") or []
68 links = tuple(
69 Link(id=str(x["id"]).lower(), text=str(x.get("text") or ""), kind=str(x.get("kind") or "other"))
70 for x in links_raw
71 if x.get("id")
72 )
73 if not links:
74 links = links_from_ids(
75 raw.get("next_ids") or (),
76 raw.get("prev_ids") or (),
77 raw.get("first_ids") or (),
78 raw.get("all_linked_ids") or (),
79 )
80 return Post(
81 id=pid,
82 title=raw.get("title") or "",
83 author=raw.get("author") or "[deleted]",
84 created_utc=int(raw.get("created_utc") or 0),
85 permalink=raw.get("permalink") or f"/r/HFY/comments/{pid}/",
86 flair=raw.get("flair"),
87 over_18=bool(raw.get("over_18")),
88 score=int(raw.get("score") or 0),
89 is_self=raw.get("is_self", True),
90 next_ids=tuple(raw.get("next_ids") or ()),
91 prev_ids=tuple(raw.get("prev_ids") or ()),
92 first_ids=tuple(raw.get("first_ids") or ()),
93 all_linked_ids=tuple(raw.get("all_linked_ids") or ()),
94 links=links,
95 )