BlueGit

timfa hfy-novels

Files

hfy-novels / indexer / normalize.py

normalize.py

normalize.py 5.1 KiB Download Raw
1"""Titel-normalisatie om hoofdstukken van één reeks te groeperen."""
2
3from __future__ import annotations
4
5import re
6import unicodedata
7
8# Tags die auteurs vaak in de titel zetten en die niet tot de reeksnaam horen.
9_TAG = re.compile(
10 r"\[\s*(?:oc|pi|hfy|ff|fic|meta|chapter|ch|part|pt|oneshot|one[-\s]?shot)"
11 r"(?:[^\]]*)\]",
12 re.IGNORECASE,
13)
14_PREFIX_TAG = re.compile(r"^\s*(?:oc|pi|hfy|ff)\s*[:|\-–—]\s*", re.IGNORECASE)
15
16# "Part 12", "Chapter 3/40", "Ch. 7", "Ep 2", "#14"
17_PART = re.compile(
18 r"""
19 (?:^|[\s,;:\-–—|\[\(])
20 (?:
21 (?:part|chapter|chapters|ch|pt|episode|ep|update|book|vol|volume)\.?
22 \s*
23 (?:[0-9]+(?:\s*[./]\s*[0-9]+)?)
24 |
25 \#\s*[0-9]+
26 )
27 [\]\)]?
28 """,
29 re.IGNORECASE | re.VERBOSE,
30)
31
32# "Title (194/?)", "Title (194/200)", "Title (194)"
33_PAREN_NUM = re.compile(
34 r"[\s:\-–—|]*\(\s*[0-9]{1,4}\s*(?:/\s*(?:[0-9]{1,4}|\?)\s*)?\)",
35 re.IGNORECASE,
36)
37
38# "194/?" of "194/200" zonder haakjes
39_SLASH_NUM = re.compile(r"[\s:\-–—|]*[0-9]{1,4}\s*/\s*(?:[0-9]{1,4}|\?)\s*$")
40
41# "Chapter twenty-two", "Part IV" (Romeins tot 40 is overkill; we doen 1-39 en i-xxxix)
42_WORD_NUM = (
43 r"(?:twenty[-\s](?:first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|one|two|three|four|five|six|seven|eight|nine)|"
44 r"thirty[-\s](?:first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|one|two|three|four|five|six|seven|eight|nine)|"
45 r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|"
46 r"twenty|thirty|one|two|three|four|five|six|seven|eight|nine|ten|"
47 r"xxxix|xxxviii|xxxvii|xxxvi|xxxv|xxxiv|xxxiii|xxxii|xxxi|xxx|"
48 r"xxix|xxviii|xxvii|xxvi|xxv|xxiv|xxiii|xxii|xxi|xx|"
49 r"xix|xviii|xvii|xvi|xv|xiv|xiii|xii|xi|x|ix|viii|vii|vi|v|iv|iii|ii|i)"
50)
51_PART_WORD = re.compile(
52 rf"(?:^|[\s,;:\-–—|\[\(])(?:part|chapter|ch|pt|episode|ep)\.?\s*{_WORD_NUM}\b[\]\)]?",
53 re.IGNORECASE,
54)
55
56# Afsluitend kaal nummer ("Title - 12", "Title 12") ná andere opschoning.
57_TRAILING_NUM = re.compile(r"[\s:\-–—|]+(?:[0-9]{1,4})(?:\s*[./]\s*[0-9]{1,4})?\s*$")
58
59# Scheidingsteken plus een kort suffix ("Title - Puppeteer") laten we staan:
60# dat kan een ondertitel zijn. Alleen nummer-achtige suffixes gaan eraf.
61
62_SPACE = re.compile(r"\s+")
63_PUNCT = re.compile(r"[\"“”‘’`]+")
64
65
66def strip_diacritics(text: str) -> str:
67 nfkd = unicodedata.normalize("NFKD", text)
68 return "".join(ch for ch in nfkd if not unicodedata.combining(ch))
69
70
71def normalize_title(title: str) -> str:
72 """Geef een groepsleutel terug: lowercase, zonder part-nummers en tags."""
73 if not title:
74 return ""
75 text = strip_diacritics(title)
76 text = _TAG.sub(" ", text)
77 text = _PREFIX_TAG.sub(" ", text)
78 text = _PART.sub(" ", text)
79 text = _PAREN_NUM.sub(" ", text)
80 text = _PART_WORD.sub(" ", text)
81 text = _SLASH_NUM.sub(" ", text)
82 text = _PUNCT.sub("", text)
83 text = text.replace("—", "-").replace("–", "-")
84 text = _TRAILING_NUM.sub("", text)
85 text = re.sub(r"[\s:\-_|/]+$", "", text)
86 text = re.sub(r"^[\s:\-_|/]+", "", text)
87 text = _SPACE.sub(" ", text).strip().lower()
88 return text
89
90
91def display_series_title(titles: list[str]) -> str:
92 """Kies een leesbare reeksnaam uit de hoofdstuktitels."""
93 if not titles:
94 return "Untitled"
95 # De genormaliseerde vorm van de eerste titel, netjes gekapitaliseerd,
96 # tenzij die leeg is; dan de ruwe eerste titel zonder tags.
97 key = normalize_title(titles[0])
98 if key:
99 return key.title()
100 cleaned = _TAG.sub("", titles[0]).strip()
101 return cleaned or titles[0]
102
103
104def slug(text: str, max_len: int = 48) -> str:
105 text = normalize_title(text) or "untitled"
106 text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
107 return text[:max_len].strip("-") or "untitled"
108
109
110# Flares die (bijna) altijd verhalen zijn.
111STORY_FLAIRS = {
112 "oc-series",
113 "oc-oneshot",
114 "oc-firstofseries",
115 "oc",
116 "pi/ff-series",
117 "pi/ff-oneshot",
118 "pi/ff-firstofseries",
119 "pi-series",
120 "pi-oneshot",
121 "pi",
122 "fanfic-series",
123 "fanfic-oneshot",
124}
125
126# Flares die we overslaan (geen verhaal).
127SKIP_FLAIRS = {
128 "meta",
129 "mod",
130 "media",
131 "art",
132 "video",
133 "misc",
134}
135
136
137def is_story_post(flair: str | None, is_self: bool, title: str = "") -> bool:
138 if not is_self:
139 return False
140 key = (flair or "").strip().lower().replace(" ", "")
141 if key in SKIP_FLAIRS:
142 return False
143 if key in STORY_FLAIRS:
144 return True
145 # Oudere posts zonder het huidige flairsysteem: wel tekst, geen meta-achtige titel.
146 lowered = title.lower()
147 if lowered.startswith("[meta]") or lowered.startswith("[mod]"):
148 return False
149 # Zonder flair of met een onbekende flair nemen we self-posts mee,
150 # behalve als de titel duidelijk geen verhaal is.
151 if key in {"", "none"}:
152 return True
153 return True
154
155
156def is_oneshot(flair: str | None, chapter_count: int) -> bool:
157 key = (flair or "").strip().lower().replace(" ", "")
158 if "oneshot" in key or "one-shot" in key:
159 return True
160 return chapter_count <= 1
161
162
163def is_series_start(flair: str | None) -> bool:
164 key = (flair or "").strip().lower().replace(" ", "")
165 return "firstofseries" in key