BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / links.py

links.py

links.py 4.4 KiB Download Raw
1"""Haal Reddit-post-IDs en next/prev-links uit markdown of HTML."""
2
3from __future__ import annotations
4
5import re
6from dataclasses import dataclass
7
8# Reddit-IDs zijn base36, typisch 5–7 tekens; we accepteren 4–10.
9_ID = r"([0-9a-z]{4,10})"
10
11_PATTERNS = [
12 re.compile(rf"https?://(?:www\.|old\.|new\.|np\.)?reddit\.com/r/[^/]+/comments/{_ID}(?:/|\b)", re.I),
13 re.compile(rf"https?://(?:www\.)?reddit\.com/comments/{_ID}(?:/|\b)", re.I),
14 re.compile(rf"https?://redd\.it/{_ID}\b", re.I),
15 re.compile(rf"/r/[^/]+/comments/{_ID}(?:/|\b)", re.I),
16 re.compile(rf"/comments/{_ID}(?:/|\b)", re.I),
17]
18
19# Markdown- of HTML-links: [tekst](url) of <a href="url">tekst</a>
20_MD_LINK = re.compile(r"\[([^\]]{0,80})\]\(([^)]+)\)")
21_HTML_LINK = re.compile(r'<a\s+[^>]*href=["\']([^"\']+)["\'][^>]*>(.*?)</a>', re.I | re.S)
22
23_NEXT = re.compile(
24 r"\b(next(?:\s*(?:chapter|part|ch|pt|episode|ep|page))?|continue(?:\s*reading)?|volgende|verder)\b",
25 re.I,
26)
27_PREV = re.compile(
28 r"\b(prev(?:ious)?(?:\s*(?:chapter|part|ch|pt))?|vorige)\b",
29 re.I,
30)
31_FIRST = re.compile(r"\b(first(?:\s*(?:chapter|part))?|start|eerste|toc|table\s*of\s*contents|index)\b", re.I)
32
33_HTML_TAG = re.compile(r"<[^>]+>")
34
35
36@dataclass(frozen=True)
37class Link:
38 """Eén gelabelde post-link, in documentvolgorde."""
39
40 id: str
41 text: str
42 kind: str # next | prev | first | other
43
44
45@dataclass(frozen=True)
46class ExtractedLinks:
47 all_ids: tuple[str, ...]
48 next_ids: tuple[str, ...]
49 prev_ids: tuple[str, ...]
50 first_ids: tuple[str, ...]
51 links: tuple[Link, ...] = ()
52
53
54def extract_post_id(url: str) -> str | None:
55 if not url:
56 return None
57 for pat in _PATTERNS:
58 m = pat.search(url)
59 if m:
60 return m.group(1).lower()
61 return None
62
63
64def extract_ids(text: str) -> list[str]:
65 if not text:
66 return []
67 seen: list[str] = []
68 found: set[str] = set()
69 for pat in _PATTERNS:
70 for m in pat.finditer(text):
71 pid = m.group(1).lower()
72 if pid not in found:
73 found.add(pid)
74 seen.append(pid)
75 return seen
76
77
78def _classify(label: str) -> str:
79 lab = _HTML_TAG.sub(" ", label).strip()
80 if _NEXT.search(lab):
81 return "next"
82 if _PREV.search(lab):
83 return "prev"
84 if _FIRST.search(lab):
85 return "first"
86 return "other"
87
88
89def _link_pairs(text: str) -> list[tuple[int, str, str]]:
90 """(offset, label, url) in documentvolgorde."""
91 items: list[tuple[int, str, str]] = []
92 for m in _MD_LINK.finditer(text):
93 items.append((m.start(), m.group(1), m.group(2)))
94 for m in _HTML_LINK.finditer(text):
95 label = _HTML_TAG.sub(" ", m.group(2))
96 items.append((m.start(), label, m.group(1)))
97 items.sort(key=lambda row: row[0])
98 return items
99
100
101def extract_labeled_links(text: str) -> list[Link]:
102 """Gelabelde comment-links in documentvolgorde (eerste keer per id)."""
103 if not text:
104 return []
105 out: list[Link] = []
106 seen: set[str] = set()
107 for _pos, label, url in _link_pairs(text):
108 pid = extract_post_id(url)
109 if not pid or pid in seen:
110 continue
111 seen.add(pid)
112 lab = _HTML_TAG.sub(" ", label).strip()
113 out.append(Link(id=pid, text=lab, kind=_classify(lab)))
114 return out
115
116
117def links_from_ids(
118 next_ids: tuple[str, ...] | list[str] = (),
119 prev_ids: tuple[str, ...] | list[str] = (),
120 first_ids: tuple[str, ...] | list[str] = (),
121 other_ids: tuple[str, ...] | list[str] = (),
122) -> tuple[Link, ...]:
123 """Synthetiseer links als we alleen IDs hebben (bijvoorbeeld uit een eerdere index)."""
124 out: list[Link] = []
125 seen: set[str] = set()
126 for kind, ids in (("next", next_ids), ("prev", prev_ids), ("first", first_ids), ("other", other_ids)):
127 for pid in ids:
128 key = (pid or "").lower()
129 if not key or key in seen:
130 continue
131 seen.add(key)
132 out.append(Link(id=key, text="", kind=kind))
133 return tuple(out)
134
135
136def extract_links(text: str) -> ExtractedLinks:
137 if not text:
138 return ExtractedLinks((), (), (), (), ())
139 labeled = extract_labeled_links(text)
140 all_ids = extract_ids(text)
141 nxt = tuple(link.id for link in labeled if link.kind == "next")
142 prev = tuple(link.id for link in labeled if link.kind == "prev")
143 first = tuple(link.id for link in labeled if link.kind == "first")
144 return ExtractedLinks(tuple(all_ids), nxt, prev, first, tuple(labeled))