"""Haal Reddit-post-IDs en next/prev-links uit markdown of HTML."""
from __future__ import annotations
import re
from dataclasses import dataclass
# Reddit-IDs zijn base36, typisch 5–7 tekens; we accepteren 4–10.
_ID = r"([0-9a-z]{4,10})"
_PATTERNS = [
re.compile(rf"https?://(?:www\.|old\.|new\.|np\.)?reddit\.com/r/[^/]+/comments/{_ID}(?:/|\b)", re.I),
re.compile(rf"https?://(?:www\.)?reddit\.com/comments/{_ID}(?:/|\b)", re.I),
re.compile(rf"https?://redd\.it/{_ID}\b", re.I),
re.compile(rf"/r/[^/]+/comments/{_ID}(?:/|\b)", re.I),
re.compile(rf"/comments/{_ID}(?:/|\b)", re.I),
]
# Markdown- of HTML-links: [tekst](url) of tekst
_MD_LINK = re.compile(r"\[([^\]]{0,80})\]\(([^)]+)\)")
_HTML_LINK = re.compile(r']*href=["\']([^"\']+)["\'][^>]*>(.*?)', re.I | re.S)
_NEXT = re.compile(
r"\b(next(?:\s*(?:chapter|part|ch|pt|episode|ep|page))?|continue(?:\s*reading)?|volgende|verder)\b",
re.I,
)
_PREV = re.compile(
r"\b(prev(?:ious)?(?:\s*(?:chapter|part|ch|pt))?|vorige)\b",
re.I,
)
_FIRST = re.compile(r"\b(first(?:\s*(?:chapter|part))?|start|eerste|toc|table\s*of\s*contents|index)\b", re.I)
_HTML_TAG = re.compile(r"<[^>]+>")
@dataclass(frozen=True)
class Link:
"""Eén gelabelde post-link, in documentvolgorde."""
id: str
text: str
kind: str # next | prev | first | other
@dataclass(frozen=True)
class ExtractedLinks:
all_ids: tuple[str, ...]
next_ids: tuple[str, ...]
prev_ids: tuple[str, ...]
first_ids: tuple[str, ...]
links: tuple[Link, ...] = ()
def extract_post_id(url: str) -> str | None:
if not url:
return None
for pat in _PATTERNS:
m = pat.search(url)
if m:
return m.group(1).lower()
return None
def extract_ids(text: str) -> list[str]:
if not text:
return []
seen: list[str] = []
found: set[str] = set()
for pat in _PATTERNS:
for m in pat.finditer(text):
pid = m.group(1).lower()
if pid not in found:
found.add(pid)
seen.append(pid)
return seen
def _classify(label: str) -> str:
lab = _HTML_TAG.sub(" ", label).strip()
if _NEXT.search(lab):
return "next"
if _PREV.search(lab):
return "prev"
if _FIRST.search(lab):
return "first"
return "other"
def _link_pairs(text: str) -> list[tuple[int, str, str]]:
"""(offset, label, url) in documentvolgorde."""
items: list[tuple[int, str, str]] = []
for m in _MD_LINK.finditer(text):
items.append((m.start(), m.group(1), m.group(2)))
for m in _HTML_LINK.finditer(text):
label = _HTML_TAG.sub(" ", m.group(2))
items.append((m.start(), label, m.group(1)))
items.sort(key=lambda row: row[0])
return items
def extract_labeled_links(text: str) -> list[Link]:
"""Gelabelde comment-links in documentvolgorde (eerste keer per id)."""
if not text:
return []
out: list[Link] = []
seen: set[str] = set()
for _pos, label, url in _link_pairs(text):
pid = extract_post_id(url)
if not pid or pid in seen:
continue
seen.add(pid)
lab = _HTML_TAG.sub(" ", label).strip()
out.append(Link(id=pid, text=lab, kind=_classify(lab)))
return out
def links_from_ids(
next_ids: tuple[str, ...] | list[str] = (),
prev_ids: tuple[str, ...] | list[str] = (),
first_ids: tuple[str, ...] | list[str] = (),
other_ids: tuple[str, ...] | list[str] = (),
) -> tuple[Link, ...]:
"""Synthetiseer links als we alleen IDs hebben (bijvoorbeeld uit een eerdere index)."""
out: list[Link] = []
seen: set[str] = set()
for kind, ids in (("next", next_ids), ("prev", prev_ids), ("first", first_ids), ("other", other_ids)):
for pid in ids:
key = (pid or "").lower()
if not key or key in seen:
continue
seen.add(key)
out.append(Link(id=key, text="", kind=kind))
return tuple(out)
def extract_links(text: str) -> ExtractedLinks:
if not text:
return ExtractedLinks((), (), (), (), ())
labeled = extract_labeled_links(text)
all_ids = extract_ids(text)
nxt = tuple(link.id for link in labeled if link.kind == "next")
prev = tuple(link.id for link in labeled if link.kind == "prev")
first = tuple(link.id for link in labeled if link.kind == "first")
return ExtractedLinks(tuple(all_ids), nxt, prev, first, tuple(labeled))