BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / cluster.py

cluster.py

cluster.py 6.1 KiB Download Raw
1"""Groepeer posts tot reeksen (novels) op auteur, titel en next-links."""
2
3from __future__ import annotations
4
5from collections import defaultdict
6from dataclasses import dataclass, field
7from typing import Iterable
8
9from .links import Link, extract_links, links_from_ids
10from .normalize import display_series_title, is_oneshot, is_story_post, normalize_title, slug
11
12
13@dataclass
14class Post:
15 id: str
16 title: str
17 author: str
18 created_utc: int
19 permalink: str
20 flair: str | None
21 over_18: bool = False
22 score: int = 0
23 is_self: bool = True
24 next_ids: tuple[str, ...] = ()
25 prev_ids: tuple[str, ...] = ()
26 first_ids: tuple[str, ...] = ()
27 all_linked_ids: tuple[str, ...] = ()
28 links: tuple[Link, ...] = ()
29 wiki_path: str | None = None
30
31 @property
32 def norm_title(self) -> str:
33 return normalize_title(self.title)
34
35 @classmethod
36 def from_reddit(cls, raw: dict) -> Post | None:
37 pid = (raw.get("id") or "").strip().lower()
38 if not pid:
39 return None
40 author = raw.get("author") or "[deleted]"
41 title = raw.get("title") or ""
42 url = raw.get("url") or ""
43 permalink = raw.get("permalink") or ""
44 if not permalink and "/comments/" in url:
45 permalink = "/r/" + url.split("/r/", 1)[-1] if "/r/" in url else f"/r/HFY/comments/{pid}/"
46 if not permalink:
47 permalink = f"/r/HFY/comments/{pid}/"
48 if permalink.startswith("http"):
49 permalink = "/r/" + permalink.split("/r/", 1)[-1] if "/r/" in permalink else f"/r/HFY/comments/{pid}/"
50 if not permalink.startswith("/"):
51 permalink = f"/r/HFY/comments/{pid}/"
52 if "is_self" in raw:
53 is_self = bool(raw["is_self"])
54 else:
55 # Arctic Shift accepteert is_self niet als fields=; een externe url is een link-post.
56 is_self = (not url) or ("reddit.com/" in url) or url.startswith("/r/")
57 flair = raw.get("link_flair_text")
58 created = int(raw.get("created_utc") or raw.get("created") or 0)
59 links = extract_links(raw.get("selftext") or raw.get("selftext_html") or "")
60 return cls(
61 id=pid,
62 title=title,
63 author=author,
64 created_utc=created,
65 permalink=permalink,
66 flair=flair,
67 over_18=bool(raw.get("over_18")),
68 score=int(raw.get("score") or 0),
69 is_self=is_self,
70 next_ids=links.next_ids,
71 prev_ids=links.prev_ids,
72 first_ids=links.first_ids,
73 all_linked_ids=links.all_ids,
74 links=links.links or links_from_ids(links.next_ids, links.prev_ids, links.first_ids, links.all_ids),
75 )
76
77
78@dataclass
79class Novel:
80 id: str
81 name: str
82 author: str
83 chapters: list[Post] = field(default_factory=list)
84 flair: str | None = None
85 over_18: bool = False
86 oneshot: bool = False
87 wiki_path: str | None = None
88
89 @property
90 def updated_utc(self) -> int:
91 return max((c.created_utc for c in self.chapters), default=0)
92
93 @property
94 def created_utc(self) -> int:
95 return min((c.created_utc for c in self.chapters), default=0)
96
97 @property
98 def path(self) -> str:
99 return f"series/{self.id}"
100
101
102class _UnionFind:
103 def __init__(self, items: Iterable[str]) -> None:
104 self.parent = {i: i for i in items}
105
106 def find(self, x: str) -> str:
107 p = self.parent.setdefault(x, x)
108 if p != x:
109 self.parent[x] = self.find(p)
110 return self.parent[x]
111
112 def union(self, a: str, b: str) -> None:
113 ra, rb = self.find(a), self.find(b)
114 if ra != rb:
115 self.parent[rb] = ra
116
117
118def filter_story_posts(posts: Iterable[Post]) -> list[Post]:
119 return [p for p in posts if is_story_post(p.flair, p.is_self, p.title) and p.author not in {"[deleted]", "[removed]"}]
120
121
122def cluster_posts(posts: Iterable[Post]) -> list[Novel]:
123 """Groepeer op (auteur, genormaliseerde titel) en merge via next/prev van dezelfde auteur."""
124 items = filter_story_posts(posts)
125 if not items:
126 return []
127 by_id = {p.id: p for p in items}
128 uf = _UnionFind(by_id)
129
130 # 1. Zelfde auteur + dezelfde genormaliseerde titel.
131 groups: dict[tuple[str, str], list[str]] = defaultdict(list)
132 for p in items:
133 key = p.norm_title or f"__id_{p.id}"
134 groups[(p.author.lower(), key)].append(p.id)
135 for ids in groups.values():
136 head = ids[0]
137 for other in ids[1:]:
138 uf.union(head, other)
139
140 # 2. Expliciete next/prev/first-links tussen posts van dezelfde auteur.
141 def maybe_merge(src: Post, target_id: str) -> None:
142 dst = by_id.get(target_id)
143 if not dst:
144 return
145 if src.author.lower() != dst.author.lower():
146 return
147 uf.union(src.id, dst.id)
148
149 for p in items:
150 for tid in p.next_ids + p.prev_ids + p.first_ids:
151 maybe_merge(p, tid)
152
153 clustered: dict[str, list[Post]] = defaultdict(list)
154 for p in items:
155 clustered[uf.find(p.id)].append(p)
156
157 novels: list[Novel] = []
158 for members in clustered.values():
159 members.sort(key=lambda p: (p.created_utc, p.id))
160 first = members[0]
161 name = display_series_title([m.title for m in members])
162 # Stabiel id: de oudste post. Zo blijft een reeks herkenbaar als er hoofdstukken bijkomen.
163 nid = first.id
164 flair = next((m.flair for m in members if m.flair), first.flair)
165 novel = Novel(
166 id=nid,
167 name=name,
168 author=first.author,
169 chapters=members,
170 flair=flair,
171 over_18=any(m.over_18 for m in members),
172 oneshot=is_oneshot(flair, len(members)),
173 )
174 novels.append(novel)
175
176 novels.sort(key=lambda n: n.updated_utc, reverse=True)
177 return novels
178
179
180def merge_novels(existing: list[Novel], incoming: list[Novel]) -> list[Novel]:
181 """Voeg een incrementele update samen: posts met hetzelfde id winnen van de nieuwe batch."""
182 posts: dict[str, Post] = {}
183 for novel in existing + incoming:
184 for ch in novel.chapters:
185 posts[ch.id] = ch
186 return cluster_posts(posts.values())