| 1 | """Groepeer posts tot reeksen (novels) op auteur, titel en next-links.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | from collections import defaultdict |
| 6 | from dataclasses import dataclass, field |
| 7 | from typing import Iterable |
| 8 | |
| 9 | from .links import Link, extract_links, links_from_ids |
| 10 | from .normalize import display_series_title, is_oneshot, is_story_post, normalize_title, slug |
| 11 | |
| 12 | |
| 13 | @dataclass |
| 14 | class Post: |
| 15 | id: str |
| 16 | title: str |
| 17 | author: str |
| 18 | created_utc: int |
| 19 | permalink: str |
| 20 | flair: str | None |
| 21 | over_18: bool = False |
| 22 | score: int = 0 |
| 23 | is_self: bool = True |
| 24 | next_ids: tuple[str, ...] = () |
| 25 | prev_ids: tuple[str, ...] = () |
| 26 | first_ids: tuple[str, ...] = () |
| 27 | all_linked_ids: tuple[str, ...] = () |
| 28 | links: tuple[Link, ...] = () |
| 29 | wiki_path: str | None = None |
| 30 | |
| 31 | @property |
| 32 | def norm_title(self) -> str: |
| 33 | return normalize_title(self.title) |
| 34 | |
| 35 | @classmethod |
| 36 | def from_reddit(cls, raw: dict) -> Post | None: |
| 37 | pid = (raw.get("id") or "").strip().lower() |
| 38 | if not pid: |
| 39 | return None |
| 40 | author = raw.get("author") or "[deleted]" |
| 41 | title = raw.get("title") or "" |
| 42 | url = raw.get("url") or "" |
| 43 | permalink = raw.get("permalink") or "" |
| 44 | if not permalink and "/comments/" in url: |
| 45 | permalink = "/r/" + url.split("/r/", 1)[-1] if "/r/" in url else f"/r/HFY/comments/{pid}/" |
| 46 | if not permalink: |
| 47 | permalink = f"/r/HFY/comments/{pid}/" |
| 48 | if permalink.startswith("http"): |
| 49 | permalink = "/r/" + permalink.split("/r/", 1)[-1] if "/r/" in permalink else f"/r/HFY/comments/{pid}/" |
| 50 | if not permalink.startswith("/"): |
| 51 | permalink = f"/r/HFY/comments/{pid}/" |
| 52 | if "is_self" in raw: |
| 53 | is_self = bool(raw["is_self"]) |
| 54 | else: |
| 55 | # Arctic Shift accepteert is_self niet als fields=; een externe url is een link-post. |
| 56 | is_self = (not url) or ("reddit.com/" in url) or url.startswith("/r/") |
| 57 | flair = raw.get("link_flair_text") |
| 58 | created = int(raw.get("created_utc") or raw.get("created") or 0) |
| 59 | links = extract_links(raw.get("selftext") or raw.get("selftext_html") or "") |
| 60 | return cls( |
| 61 | id=pid, |
| 62 | title=title, |
| 63 | author=author, |
| 64 | created_utc=created, |
| 65 | permalink=permalink, |
| 66 | flair=flair, |
| 67 | over_18=bool(raw.get("over_18")), |
| 68 | score=int(raw.get("score") or 0), |
| 69 | is_self=is_self, |
| 70 | next_ids=links.next_ids, |
| 71 | prev_ids=links.prev_ids, |
| 72 | first_ids=links.first_ids, |
| 73 | all_linked_ids=links.all_ids, |
| 74 | links=links.links or links_from_ids(links.next_ids, links.prev_ids, links.first_ids, links.all_ids), |
| 75 | ) |
| 76 | |
| 77 | |
| 78 | @dataclass |
| 79 | class Novel: |
| 80 | id: str |
| 81 | name: str |
| 82 | author: str |
| 83 | chapters: list[Post] = field(default_factory=list) |
| 84 | flair: str | None = None |
| 85 | over_18: bool = False |
| 86 | oneshot: bool = False |
| 87 | wiki_path: str | None = None |
| 88 | |
| 89 | @property |
| 90 | def updated_utc(self) -> int: |
| 91 | return max((c.created_utc for c in self.chapters), default=0) |
| 92 | |
| 93 | @property |
| 94 | def created_utc(self) -> int: |
| 95 | return min((c.created_utc for c in self.chapters), default=0) |
| 96 | |
| 97 | @property |
| 98 | def path(self) -> str: |
| 99 | return f"series/{self.id}" |
| 100 | |
| 101 | |
| 102 | class _UnionFind: |
| 103 | def __init__(self, items: Iterable[str]) -> None: |
| 104 | self.parent = {i: i for i in items} |
| 105 | |
| 106 | def find(self, x: str) -> str: |
| 107 | p = self.parent.setdefault(x, x) |
| 108 | if p != x: |
| 109 | self.parent[x] = self.find(p) |
| 110 | return self.parent[x] |
| 111 | |
| 112 | def union(self, a: str, b: str) -> None: |
| 113 | ra, rb = self.find(a), self.find(b) |
| 114 | if ra != rb: |
| 115 | self.parent[rb] = ra |
| 116 | |
| 117 | |
| 118 | def filter_story_posts(posts: Iterable[Post]) -> list[Post]: |
| 119 | return [p for p in posts if is_story_post(p.flair, p.is_self, p.title) and p.author not in {"[deleted]", "[removed]"}] |
| 120 | |
| 121 | |
| 122 | def cluster_posts(posts: Iterable[Post]) -> list[Novel]: |
| 123 | """Groepeer op (auteur, genormaliseerde titel) en merge via next/prev van dezelfde auteur.""" |
| 124 | items = filter_story_posts(posts) |
| 125 | if not items: |
| 126 | return [] |
| 127 | by_id = {p.id: p for p in items} |
| 128 | uf = _UnionFind(by_id) |
| 129 | |
| 130 | # 1. Zelfde auteur + dezelfde genormaliseerde titel. |
| 131 | groups: dict[tuple[str, str], list[str]] = defaultdict(list) |
| 132 | for p in items: |
| 133 | key = p.norm_title or f"__id_{p.id}" |
| 134 | groups[(p.author.lower(), key)].append(p.id) |
| 135 | for ids in groups.values(): |
| 136 | head = ids[0] |
| 137 | for other in ids[1:]: |
| 138 | uf.union(head, other) |
| 139 | |
| 140 | # 2. Expliciete next/prev/first-links tussen posts van dezelfde auteur. |
| 141 | def maybe_merge(src: Post, target_id: str) -> None: |
| 142 | dst = by_id.get(target_id) |
| 143 | if not dst: |
| 144 | return |
| 145 | if src.author.lower() != dst.author.lower(): |
| 146 | return |
| 147 | uf.union(src.id, dst.id) |
| 148 | |
| 149 | for p in items: |
| 150 | for tid in p.next_ids + p.prev_ids + p.first_ids: |
| 151 | maybe_merge(p, tid) |
| 152 | |
| 153 | clustered: dict[str, list[Post]] = defaultdict(list) |
| 154 | for p in items: |
| 155 | clustered[uf.find(p.id)].append(p) |
| 156 | |
| 157 | novels: list[Novel] = [] |
| 158 | for members in clustered.values(): |
| 159 | members.sort(key=lambda p: (p.created_utc, p.id)) |
| 160 | first = members[0] |
| 161 | name = display_series_title([m.title for m in members]) |
| 162 | # Stabiel id: de oudste post. Zo blijft een reeks herkenbaar als er hoofdstukken bijkomen. |
| 163 | nid = first.id |
| 164 | flair = next((m.flair for m in members if m.flair), first.flair) |
| 165 | novel = Novel( |
| 166 | id=nid, |
| 167 | name=name, |
| 168 | author=first.author, |
| 169 | chapters=members, |
| 170 | flair=flair, |
| 171 | over_18=any(m.over_18 for m in members), |
| 172 | oneshot=is_oneshot(flair, len(members)), |
| 173 | ) |
| 174 | novels.append(novel) |
| 175 | |
| 176 | novels.sort(key=lambda n: n.updated_utc, reverse=True) |
| 177 | return novels |
| 178 | |
| 179 | |
| 180 | def merge_novels(existing: list[Novel], incoming: list[Novel]) -> list[Novel]: |
| 181 | """Voeg een incrementele update samen: posts met hetzelfde id winnen van de nieuwe batch.""" |
| 182 | posts: dict[str, Post] = {} |
| 183 | for novel in existing + incoming: |
| 184 | for ch in novel.chapters: |
| 185 | posts[ch.id] = ch |
| 186 | return cluster_posts(posts.values()) |