| 1 | """Titelgelijkenis en reekstitel, in de geest van hfydl (niet die code). |
| 2 | |
| 3 | hfydl gebruikt model2vec (minishlab/potion-base-8M) en cosine ≥ temperature |
| 4 | (standaard 0.5) om te beslissen of een next-link nog bij dezelfde reeks hoort. |
| 5 | De serietitel is het gemeenschappelijke woord-voorvoegsel van de titels. |
| 6 | |
| 7 | Wij doen hetzelfde principe, maar: |
| 8 | - embeddings op de post-titel (stabieler dan het label "Next"); |
| 9 | - tags en part-nummers gaan er eerst af, anders faalt het voorvoegsel op '[OC]'; |
| 10 | - zonder model2vec valt de indexer terug op een lexicale embedder, zodat tests |
| 11 | en CI zonder modeldownload werken. |
| 12 | """ |
| 13 | |
| 14 | from __future__ import annotations |
| 15 | |
| 16 | import math |
| 17 | import os |
| 18 | import re |
| 19 | from typing import Iterable, Protocol |
| 20 | |
| 21 | from .normalize import normalize_title |
| 22 | |
| 23 | DEFAULT_TEMPERATURE = 0.5 |
| 24 | MODEL_NAME = "minishlab/potion-base-8M" |
| 25 | |
| 26 | |
| 27 | class Embedder(Protocol): |
| 28 | name: str |
| 29 | |
| 30 | def encode(self, texts: list[str]) -> list[list[float]]: |
| 31 | ... |
| 32 | |
| 33 | |
| 34 | def cosine(a: list[float] | tuple[float, ...], b: list[float] | tuple[float, ...]) -> float: |
| 35 | if not a or not b or len(a) != len(b): |
| 36 | return 0.0 |
| 37 | dot = 0.0 |
| 38 | na = 0.0 |
| 39 | nb = 0.0 |
| 40 | for x, y in zip(a, b): |
| 41 | dot += x * y |
| 42 | na += x * x |
| 43 | nb += y * y |
| 44 | if na <= 0 or nb <= 0: |
| 45 | return 0.0 |
| 46 | return dot / math.sqrt(na * nb) |
| 47 | |
| 48 | |
| 49 | def mean_vec(vectors: Iterable[list[float]]) -> list[float]: |
| 50 | items = list(vectors) |
| 51 | if not items: |
| 52 | return [] |
| 53 | n = len(items) |
| 54 | dim = len(items[0]) |
| 55 | acc = [0.0] * dim |
| 56 | for v in items: |
| 57 | if len(v) != dim: |
| 58 | continue |
| 59 | for i, x in enumerate(v): |
| 60 | acc[i] += x |
| 61 | return [x / n for x in acc] |
| 62 | |
| 63 | |
| 64 | class LexicalEmbedder: |
| 65 | """Lichte, deterministische embedder op woord- en karakter-n-grams.""" |
| 66 | |
| 67 | name = "lexical" |
| 68 | |
| 69 | def encode(self, texts: list[str]) -> list[list[float]]: |
| 70 | return [self._one(t) for t in texts] |
| 71 | |
| 72 | def _one(self, text: str) -> list[float]: |
| 73 | key = normalize_title(text) or (text or "").strip().lower() |
| 74 | tokens = re.findall(r"[a-z0-9]+", key) |
| 75 | grams: dict[str, float] = {} |
| 76 | for tok in tokens: |
| 77 | grams[f"w:{tok}"] = grams.get(f"w:{tok}", 0.0) + 2.0 |
| 78 | padded = f"#{tok}#" |
| 79 | for i in range(len(padded) - 2): |
| 80 | g = padded[i : i + 3] |
| 81 | grams[f"c:{g}"] = grams.get(f"c:{g}", 0.0) + 1.0 |
| 82 | if not grams: |
| 83 | return [1.0] |
| 84 | # Vaste dimensie-volgorde: sorteer de sleutels en maak een compacte |
| 85 | # hash-bag, zodat twee titels in één gemeenschappelijke ruimte liggen. |
| 86 | dim = 256 |
| 87 | vec = [0.0] * dim |
| 88 | for k, val in grams.items(): |
| 89 | idx = (hash(k) % dim + dim) % dim |
| 90 | # hash() is per proces stabiel, niet per machine. Voor tests in |
| 91 | # één proces is dat genoeg; voor model2vec geldt die zorg niet. |
| 92 | # We gebruiken een stabiele FNV-achtige hash in plaats van hash(). |
| 93 | idx = _stable_hash(k) % dim |
| 94 | vec[idx] += val |
| 95 | return _l2(vec) |
| 96 | |
| 97 | |
| 98 | def _stable_hash(text: str) -> int: |
| 99 | h = 2166136261 |
| 100 | for ch in text.encode("utf-8"): |
| 101 | h ^= ch |
| 102 | h = (h * 16777619) & 0xFFFFFFFF |
| 103 | return h |
| 104 | |
| 105 | |
| 106 | def _l2(vec: list[float]) -> list[float]: |
| 107 | n = math.sqrt(sum(x * x for x in vec)) |
| 108 | if n <= 0: |
| 109 | return vec |
| 110 | return [x / n for x in vec] |
| 111 | |
| 112 | |
| 113 | class Model2VecEmbedder: |
| 114 | name = "model2vec" |
| 115 | |
| 116 | def __init__(self, model: object) -> None: |
| 117 | self._model = model |
| 118 | |
| 119 | def encode(self, texts: list[str]) -> list[list[float]]: |
| 120 | cleaned = [normalize_title(t) or t for t in texts] |
| 121 | raw = self._model.encode(cleaned) # type: ignore[attr-defined] |
| 122 | out: list[list[float]] = [] |
| 123 | for row in raw: |
| 124 | out.append(_l2([float(x) for x in row])) |
| 125 | return out |
| 126 | |
| 127 | |
| 128 | def load_embedder(kind: str | None = None) -> Embedder: |
| 129 | """kind: auto | lexical | model2vec. auto probeert model2vec en valt terug.""" |
| 130 | choice = (kind or os.environ.get("HFY_EMBEDDER") or "auto").strip().lower() |
| 131 | if choice in {"lexical", "lex"}: |
| 132 | return LexicalEmbedder() |
| 133 | if choice in {"auto", "model2vec", "m2v"}: |
| 134 | try: |
| 135 | from model2vec import StaticModel |
| 136 | |
| 137 | model = StaticModel.from_pretrained(MODEL_NAME) |
| 138 | return Model2VecEmbedder(model) |
| 139 | except Exception: |
| 140 | if choice in {"model2vec", "m2v"}: |
| 141 | raise |
| 142 | return LexicalEmbedder() |
| 143 | return LexicalEmbedder() |
| 144 | |
| 145 | |
| 146 | def guess_title(titles: list[str]) -> str: |
| 147 | """Gemeenschappelijk woord-voorvoegsel, ná het strippen van tags en part-nummers.""" |
| 148 | split = [normalize_title(t).split() for t in titles] |
| 149 | split = [s for s in split if s] |
| 150 | if not split: |
| 151 | return "Untitled" |
| 152 | first = split[0] |
| 153 | end = 0 |
| 154 | for i, word in enumerate(first): |
| 155 | if any(len(s) <= i or s[i] != word for s in split): |
| 156 | break |
| 157 | end = i + 1 |
| 158 | if end == 0: |
| 159 | # Geen gemeenschappelijk voorvoegsel: val terug op de langste genormaliseerde titel. |
| 160 | longest = max((normalize_title(t) for t in titles), key=len, default="") |
| 161 | return longest.title() if longest else "Untitled" |
| 162 | return " ".join(first[:end]).title() |