BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / write_index.py

write_index.py

write_index.py 7.1 KiB Download Raw
1"""Schrijf een statische, gesplitste metadata-index (geen verhaaltekst)."""
2
3from __future__ import annotations
4
5import json
6from datetime import datetime, timezone
7from pathlib import Path
8from typing import Iterable
9
10from .cluster import Novel
11from .covers import prune_unused_covers, write_cover
12
13PAGE_SIZE = 50
14SCHEMA_VERSION = 1
15
16
17def _iso(ts: int) -> str | None:
18 if not ts:
19 return None
20 return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
21
22
23def infer_status(novel: Novel, now: int | None = None) -> str:
24 """Grove status: 180 dagen zonder update = On Hiatus, verder Unknown."""
25 if not novel.chapters:
26 return "Unknown"
27 now = now or int(datetime.now(tz=timezone.utc).timestamp())
28 age = now - novel.updated_utc
29 if novel.oneshot:
30 return "Completed"
31 if age > 180 * 24 * 3600:
32 return "On Hiatus"
33 return "Ongoing"
34
35
36def novel_to_listing(novel: Novel, cover: str | None = None) -> dict:
37 return {
38 "id": novel.id,
39 "name": novel.name,
40 "path": novel.path,
41 "author": novel.author,
42 "cover": cover,
43 "updated": _iso(novel.updated_utc),
44 "created": _iso(novel.created_utc),
45 "chapters": len(novel.chapters),
46 "flair": novel.flair,
47 "nsfw": novel.over_18,
48 "oneshot": novel.oneshot,
49 }
50
51
52def novel_to_detail(novel: Novel, cover: str | None = None) -> dict:
53 listing = novel_to_listing(novel, cover=cover)
54 listing.update(
55 {
56 "status": infer_status(novel),
57 "genres": ", ".join(
58 x for x in ["HFY", novel.flair or ("OneShot" if novel.oneshot else "Series")] if x
59 ),
60 "wikiPath": novel.wiki_path,
61 "summary": (
62 f"{len(novel.chapters)} hoofdstuk{'en' if len(novel.chapters) != 1 else ''} "
63 f"op r/HFY door u/{novel.author}."
64 + (f" Laatste: {novel.chapters[-1].title}." if novel.chapters else "")
65 ),
66 "chapterList": [
67 {
68 "name": ch.title,
69 "path": f"chapter/{ch.id}",
70 "releaseTime": _iso(ch.created_utc),
71 "chapterNumber": i,
72 "redditId": ch.id,
73 "permalink": ch.permalink,
74 "nextIds": list(ch.next_ids),
75 "prevIds": list(ch.prev_ids),
76 "firstIds": list(ch.first_ids),
77 }
78 for i, ch in enumerate(novel.chapters, start=1)
79 ],
80 }
81 )
82 return listing
83
84
85def series_relpath(novel_id: str) -> str:
86 prefix = (novel_id[:1] or "x").lower()
87 return f"series/{prefix}/{novel_id}.json"
88
89
90def write_index(
91 novels: Iterable[Novel],
92 dest: Path,
93 page_size: int = PAGE_SIZE,
94 source: str = "r/HFY wiki-serie-index via Arctic Shift (metadata only; no story text)",
95) -> dict:
96 dest = Path(dest)
97 dest.mkdir(parents=True, exist_ok=True)
98 items = list(novels)
99 items.sort(key=lambda n: n.updated_utc, reverse=True)
100
101 covers: dict[str, str] = {}
102 for novel in items:
103 covers[novel.id] = write_cover(dest, novel.name)
104 prune_unused_covers(dest, set(covers.values()))
105
106 # Oude shards opruimen zodat verwijderde reeksen niet blijven staan.
107 series_root = dest / "series"
108 if series_root.exists():
109 for old in series_root.rglob("*.json"):
110 old.unlink()
111
112 catalog_dir = dest / "catalog"
113 catalog_dir.mkdir(parents=True, exist_ok=True)
114 for old in catalog_dir.glob("page-*.json"):
115 old.unlink()
116
117 pages = []
118 for i in range(0, max(len(items), 1), page_size):
119 chunk = items[i : i + page_size]
120 page_no = i // page_size + 1
121 payload = {
122 "schemaVersion": SCHEMA_VERSION,
123 "page": page_no,
124 "novels": [novel_to_listing(n, cover=covers.get(n.id)) for n in chunk],
125 }
126 name = f"page-{page_no:04d}.json"
127 (catalog_dir / name).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
128 pages.append({"page": page_no, "path": f"catalog/{name}", "count": len(chunk)})
129
130 for novel in chunk:
131 rel = series_relpath(novel.id)
132 path = dest / rel
133 path.parent.mkdir(parents=True, exist_ok=True)
134 path.write_text(json.dumps(novel_to_detail(novel, cover=covers.get(novel.id)), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
135
136 search = [
137 {
138 "id": n.id,
139 "name": n.name,
140 "author": n.author,
141 "path": n.path,
142 "oneshot": n.oneshot,
143 "nsfw": n.over_18,
144 }
145 for n in items
146 ]
147 (dest / "search-index.json").write_text(
148 json.dumps({"schemaVersion": SCHEMA_VERSION, "novels": search}, ensure_ascii=False) + "\n",
149 encoding="utf-8",
150 )
151
152 meta = {
153 "schemaVersion": SCHEMA_VERSION,
154 "generatedAt": datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
155 "source": source,
156 "novelCount": len(items),
157 "chapterCount": sum(len(n.chapters) for n in items),
158 "pageSize": page_size,
159 "pageCount": len(pages),
160 "pages": pages,
161 "searchIndex": "search-index.json",
162 "seriesPath": "series/{firstLetter}/{id}.json",
163 }
164 (dest / "catalog-meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
165 return meta
166
167
168def read_existing_posts(dest: Path) -> list[dict]:
169 """Lees eerder geschreven reeksen terug als platte post-dicts (zonder selftext)."""
170 from .cluster import Post
171 from .links import links_from_ids
172
173 posts: list[Post] = []
174 series_root = Path(dest) / "series"
175 if not series_root.exists():
176 return []
177 for path in series_root.rglob("*.json"):
178 data = json.loads(path.read_text(encoding="utf-8"))
179 author = data.get("author") or "Anonymous"
180 for ch in data.get("chapterList") or []:
181 pid = (ch.get("redditId") or "").strip().lower()
182 if not pid:
183 continue
184 rel = ch.get("releaseTime")
185 created = 0
186 if rel:
187 try:
188 created = int(datetime.fromisoformat(rel.replace("Z", "+00:00")).timestamp())
189 except ValueError:
190 created = 0
191 next_ids = tuple(ch.get("nextIds") or ())
192 prev_ids = tuple(ch.get("prevIds") or ())
193 first_ids = tuple(ch.get("firstIds") or ())
194 posts.append(
195 Post(
196 id=pid,
197 title=ch.get("name") or "",
198 author=author,
199 created_utc=created,
200 permalink=ch.get("permalink") or f"/r/HFY/comments/{pid}/",
201 flair=data.get("flair"),
202 over_18=bool(data.get("nsfw")),
203 is_self=True,
204 next_ids=next_ids,
205 prev_ids=prev_ids,
206 first_ids=first_ids,
207 links=links_from_ids(next_ids, prev_ids, first_ids),
208 wiki_path=data.get("wikiPath"),
209 )
210 )
211 return posts