BlueGit

timfa 66wiki

Files

66wiki / web / dataIndexLib.php

dataIndexLib.php

dataIndexLib.php 10.4 KiB Download Raw
1<?php
2/**
3 * Shared index logic for dataIndex.php and dataBulk.php.
4 * Defines functions only; produces no output.
5 */
6
7require_once __DIR__ . '/folderNode.php';
8
9/**
10 * Build the data index.
11 * @return array{nodes: array, indexSha256: string, files: array<string,string>}
12 * files: "data/<folder>/<file>.json" => sha256, for every file in the index
13 * (the same set and hashes that indexSha256 is computed from), sorted by path.
14 */
15function buildDataIndex(): array
16{
17 $dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';
18 $chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';
19
20 $allJson = loadAllJson($dataRoot);
21 $visited = [];
22 $baseUrl = getBaseUrl();
23
24 $chaptersTree = loadFolderTree($chaptersRoot);
25 $chapters = $chaptersTree->jsonRecursive();
26
27 usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));
28
29 foreach ($chapters as $chapterObj) {
30 $chapter = json_decode(json_encode($chapterObj), true);
31 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
32 $relPath = normalizeDataRelPath($chapterLink);
33 if ($relPath !== '')
34 $visited[$relPath] = true;
35 }
36
37 $index = [];
38 foreach ($chapters as $chapterObj) {
39 $chapter = json_decode(json_encode($chapterObj), true);
40 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
41
42 $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
43 }
44
45 $files = collectIndexFiles($index, $baseUrl);
46
47 return [
48 'nodes' => $index,
49 'indexSha256' => indexSha256FromFiles($files),
50 'files' => $files,
51 ];
52}
53
54/**
55 * sha256 (lowercase hex) of the exact bytes of a data file, given its path
56 * relative to the web root (e.g. "/data/powers/feint.json").
57 * Returns null if the file does not exist or cannot be read.
58 * Results are memoised per request and keyed by path + mtime + size.
59 */
60function fileSha256(string $webRelPath): ?string
61{
62 static $cache = [];
63
64 $webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
65 $path = __DIR__ . $webRelPath;
66 if (!is_file($path))
67 return null;
68
69 $key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
70 if (!array_key_exists($key, $cache)) {
71 $hash = hash_file('sha256', $path);
72 $cache[$key] = $hash === false ? null : $hash;
73 }
74
75 return $cache[$key];
76}
77
78/**
79 * indexSha256 definition:
80 * Collect every node in the index (recursively) that has a non-null sha256.
81 * Take the unique data files they point to, as paths relative to the web
82 * root without a leading slash (e.g. "data/powers/feint.json").
83 * Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
84 * per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
85 * indexSha256 = lowercase hex sha256 of the concatenation of those lines.
86 * Reproducible from the web/ directory with:
87 * sha256sum <those files, LC_ALL=C sorted> | sha256sum
88 */
89function computeIndexSha256(array $nodes, string $baseUrl): string
90{
91 return indexSha256FromFiles(collectIndexFiles($nodes, $baseUrl));
92}
93
94/**
95 * Unique data files referenced by the index: "data/<folder>/<file>.json" => sha256,
96 * sorted by path (byte-wise).
97 */
98function collectIndexFiles(array $nodes, string $baseUrl): array
99{
100 $files = [];
101 collectNodeHashes($nodes, $baseUrl, $files);
102 ksort($files, SORT_STRING);
103 return $files;
104}
105
106/**
107 * @param array<string,string> $files path => sha256, already sorted by path
108 */
109function indexSha256FromFiles(array $files): string
110{
111 $lines = '';
112 foreach ($files as $path => $hash)
113 $lines .= $hash . ' ' . $path . "\n";
114
115 return hash('sha256', $lines);
116}
117
118function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
119{
120 $prefix = $baseUrl . '/';
121 foreach ($nodes as $node) {
122 if (!is_array($node))
123 continue;
124
125 $link = (string) ($node['link'] ?? '');
126 if (isset($node['sha256']) && str_starts_with($link, $prefix))
127 $files[substr($link, strlen($prefix))] = $node['sha256'];
128
129 collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
130 }
131}
132
133function loadAllJson(string $dataRoot): array
134{
135 $map = [];
136
137 $it = new RecursiveIteratorIterator(
138 new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)
139 );
140
141 foreach ($it as $file) {
142 /** @var SplFileInfo $file */
143 if (strtolower($file->getExtension()) !== 'json')
144 continue;
145
146 $path = $file->getPathname();
147 $rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));
148 if ($rel === '')
149 $rel = '/' . $file->getFilename();
150 if ($rel[0] !== '/')
151 $rel = '/' . $rel;
152
153 try {
154 $raw = file_get_contents($path);
155 if ($raw === false)
156 continue;
157 $json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
158 if (!is_array($json))
159 continue;
160 } catch (Throwable $e) {
161 continue;
162 }
163
164 $json['_relPath'] = $rel;
165 $map[$rel] = $json;
166 }
167
168 return $map;
169}
170
171function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array
172{
173 $name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');
174 $fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';
175
176 $node = [
177 'name' => $name,
178 'type' => $chapter['type'] ?? 'chapter',
179 'pageType' => 'page',
180 'link' => $fullChapterLink,
181 'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
182 'children' => [],
183 ];
184
185 $folderIncludes = extractFolderIncludes($chapter, ['subheaders']);
186
187 $children = [];
188
189 foreach (($chapter['subheaders'] ?? []) as $subheader) {
190 if (!is_array($subheader))
191 continue;
192
193 $subNode = [
194 'name' => $subheader['name'] ?? 'subheader',
195 'type' => 'subheader',
196 'pageType' => 'page child element',
197 'link' => $fullChapterLink,
198 'sha256' => $node['sha256'],
199 'children' => [],
200 ];
201
202 $subIncludes = extractFolderIncludes($subheader);
203 $subNode['children'] = mergeChildren(
204 buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)
205 );
206
207 $children[] = $subNode;
208 }
209
210 $children = mergeChildren(
211 $children,
212 buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)
213 );
214
215 $node['children'] = $children;
216
217 return $node;
218}
219
220function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array
221{
222 $children = [];
223 $rels = array_keys($allJson);
224 sort($rels, SORT_NATURAL | SORT_FLAG_CASE);
225 foreach ($folders as $folder) {
226 $folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');
227 foreach ($rels as $rel) {
228 if (str_starts_with($rel, $folderRel . '/')) {
229 $children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);
230 }
231 }
232 }
233
234 return $children;
235}
236
237function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array
238{
239 $link = $baseUrl . '/data' . $relPath;
240 $json = $allJson[$relPath] ?? null;
241
242 $name = $label;
243 $type = 'linkedData';
244
245 if (is_array($json)) {
246 $type = $json['type'] ?? 'data';
247 if ($name === '') {
248 $name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));
249 }
250 }
251
252 $node = [
253 'name' => $name !== '' ? $name : basename($relPath, '.json'),
254 'type' => $type,
255 'pageType' => 'data link',
256 'link' => $link,
257 'sha256' => fileSha256('/data' . $relPath),
258 'children' => [],
259 ];
260
261 if (!is_array($json))
262 return $node;
263
264 if (isset($visited[$relPath]))
265 return $node;
266
267 $visited[$relPath] = true;
268
269 $folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);
270 $node['children'] = mergeChildren($folderChildren);
271
272 return $node;
273}
274
275function normalizeWikiPath(string $path): string
276{
277 $path = str_replace('\\', '/', $path);
278 if ($path === '')
279 return '';
280
281 if (!str_starts_with($path, '/'))
282 $path = '/' . $path;
283
284 return $path;
285}
286
287function getBaseUrl(): string
288{
289 $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
290 $host = $_SERVER['HTTP_HOST'] ?? 'localhost';
291 $scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';
292 $dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');
293 return $scheme . '://' . $host . ($dir === '' ? '' : $dir);
294}
295
296function normalizeDataRelPath(string $path): string
297{
298 $path = trim($path);
299 if ($path === '')
300 return '';
301
302 $path = str_replace('\\', '/', $path);
303 if (str_starts_with($path, '/data/'))
304 $path = substr($path, strlen('/data'));
305 if (str_starts_with($path, 'data/'))
306 $path = substr($path, strlen('data'));
307
308 if (!str_starts_with($path, '/'))
309 $path = '/' . $path;
310
311 return $path;
312}
313
314function extractFolderIncludes(mixed $value, array $skipKeys = []): array
315{
316 $folders = [];
317
318 if (is_array($value)) {
319 foreach ($value as $k => $v) {
320 if (in_array($k, $skipKeys, true))
321 continue;
322 $folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));
323 }
324 return $folders;
325 }
326
327 if (is_string($value)) {
328 return extractFolderIncludesFromString($value);
329 }
330
331 return $folders;
332}
333
334function extractFolderIncludesFromString(string $text): array
335{
336 $folders = [];
337 $pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';
338
339 if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {
340 foreach ($matches as $m) {
341 $folders[] = $m[1];
342 }
343 }
344
345 return $folders;
346}
347
348function mergeChildren(array ...$groups): array
349{
350 $merged = [];
351 $seen = [];
352
353 foreach ($groups as $children) {
354 foreach ($children as $child) {
355 if (!is_array($child))
356 continue;
357
358 $key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');
359 if (isset($seen[$key]))
360 continue;
361
362 $seen[$key] = true;
363 $merged[] = $child;
364 }
365 }
366
367 return $merged;
368}