BlueGit

timfa 66wiki

Add dataBulk.php: fetch many data files in one request

- dataIndexLib.php: index building and hash logic moved out of dataIndex.php
  (dataIndex.php output is byte-identical) so both endpoints share it
- dataBulk.php: GET = all indexed files {indexSha256, files: {path: {sha256,
  content}}}; ?files= / POST {files} to select; POST {known: {path: sha256}}
  returns only new/changed files plus deleted/unchanged; content is the raw
  file text so sha256 matches dataIndex.php; paths restricted to .json under
  web/data (no .., absolute paths or symlinks out); gzip via ob_gzhandler;
  X-Index-Sha256 header

timfa Tim Falken · pushed by timfa

Authored 2026-09-28 · 7 weeks ago

Hash
4dc6f3aaaeec30e3f1319b9f05c5a6b354bc0a08
Parent
f468d5c3
3 files changed +624 −330
web/dataBulk.php added +252 −0
@@ -0,0 +1,252 @@
1 <?php
2 /**
3 * Bulk data endpoint: fetch many data files in one request.
4 *
5 * GET dataBulk.php -> all files covered by dataIndex.php
6 * GET dataBulk.php?files=a/b.json,c/d.json -> only those files
7 * POST dataBulk.php {"files": [...]} -> only those files (may be combined with ?files=)
8 * POST dataBulk.php {"known": {"a/b.json": "<sha256>", ...}}
9 * -> only files that are new or whose sha256 differs from the given one,
10 * plus "deleted": known paths that are no longer in the index.
11 * "known" can be combined with "files" (then only the selected files are compared).
12 *
13 * Paths are relative to web/data, e.g. "powers/feint.json".
14 * Response (application/json, gzip if the client accepts it):
15 * {
16 * "indexSha256": "...", same value as dataIndex.php / X-Index-Sha256
17 * "fileCount": 451, number of files in the index
18 * "files": { "<path>": { "sha256": "...", "content": "<exact file text>" } },
19 * "missing": [...], valid-looking paths that don't exist
20 * "invalid": [...], rejected paths (traversal, absolute, not .json, ...)
21 * "deleted": [...], only with "known"
22 * "unchanged": 12 only with "known": number of files skipped
23 * }
24 * "content" is the raw file text (a JSON string), so sha256(content as UTF-8 bytes)
25 * equals "sha256" and equals the per-node sha256 in dataIndex.php.
26 */
27
28 ini_set('display_errors', '1');
29 error_reporting(E_ALL);
30
31 if (extension_loaded('zlib') && !ini_get('zlib.output_compression'))
32 ob_start('ob_gzhandler');
33
34 header('Content-Type: application/json; charset=utf-8');
35 header('Access-Control-Allow-Origin: *');
36 header('Access-Control-Allow-Methods: GET, HEAD, OPTIONS, POST');
37 header('Access-Control-Allow-Headers: *');
38 header('X-Content-Type-Options: nosniff');
39 header('X-Robots-Tag: noindex');
40 header('Cache-Control: no-store, no-cache, must-revalidate, max-age=0');
41 header('Pragma: no-cache');
42
43 set_error_handler(function ($severity, $message, $file, $line) {
44 http_response_code(500);
45 echo json_encode(['error' => 'PHP Error', 'message' => $message, 'file' => basename($file), 'line' => $line]);
46 exit;
47 });
48
49 set_exception_handler(function ($e) {
50 http_response_code(500);
51 echo json_encode(['error' => 'Uncaught Exception', 'type' => get_class($e), 'message' => $e->getMessage()]);
52 exit;
53 });
54
55 require_once __DIR__ . '/dataIndexLib.php';
56
57 $method = $_SERVER['REQUEST_METHOD'] ?? 'GET';
58 if ($method === 'OPTIONS') {
59 http_response_code(204);
60 exit;
61 }
62 if (!in_array($method, ['GET', 'HEAD', 'POST'], true))
63 bulkFail(405, 'Method not allowed');
64
65 const BULK_MAX_BODY = 4 * 1024 * 1024;
66 const BULK_MAX_PATHS = 10000;
67
68 // ---- Parse request ----
69 $requested = [];
70 $known = null;
71
72 if (isset($_GET['files']) && is_string($_GET['files']))
73 $requested = array_merge($requested, explode(',', $_GET['files']));
74
75 if ($method === 'POST') {
76 $raw = file_get_contents('php://input', false, null, 0, BULK_MAX_BODY + 1);
77 if ($raw !== false && strlen($raw) > BULK_MAX_BODY)
78 bulkFail(413, 'Request body too large');
79
80 if ($raw !== false && trim($raw) !== '') {
81 try {
82 $body = json_decode($raw, true, 16, JSON_THROW_ON_ERROR);
83 } catch (Throwable $e) {
84 bulkFail(400, 'Body is not valid JSON');
85 }
86 if (!is_array($body) || ($body !== [] && bulkIsList($body)))
87 bulkFail(400, 'Body must be a JSON object');
88
89 if (isset($body['files'])) {
90 if (!is_array($body['files']) || !bulkIsList($body['files']))
91 bulkFail(400, '"files" must be an array of paths');
92 $requested = array_merge($requested, $body['files']);
93 }
94 if (isset($body['known'])) {
95 if (!is_array($body['known']) || ($body['known'] !== [] && bulkIsList($body['known'])))
96 bulkFail(400, '"known" must be an object of path => sha256');
97 $known = $body['known'];
98 }
99 }
100 }
101
102 if (count($requested) > BULK_MAX_PATHS || ($known !== null && count($known) > BULK_MAX_PATHS))
103 bulkFail(413, 'Too many paths');
104
105 // ---- Build index (same logic and hashes as dataIndex.php) ----
106 $built = buildDataIndex();
107 $indexFiles = []; // "powers/feint.json" => sha256
108 foreach ($built['files'] as $p => $h) {
109 if (str_starts_with($p, 'data/'))
110 $indexFiles[substr($p, 5)] = $h;
111 }
112 header('X-Index-Sha256: ' . $built['indexSha256']);
113
114 $dataRoot = realpath(__DIR__ . DIRECTORY_SEPARATOR . 'data');
115 if ($dataRoot === false)
116 bulkFail(500, 'Data directory not found');
117
118 // ---- Determine scope ----
119 $missing = [];
120 $invalid = [];
121
122 if ($requested) {
123 $scope = [];
124 foreach ($requested as $p) {
125 if (!is_string($p)) {
126 $invalid[] = $p;
127 continue;
128 }
129 $p = trim($p);
130 if ($p === '')
131 continue;
132 if (!isValidDataPath($p)) {
133 $invalid[] = $p;
134 continue;
135 }
136 $scope[$p] = true;
137 }
138 $scope = array_keys($scope);
139 } else {
140 $scope = array_keys($indexFiles);
141 }
142 sort($scope, SORT_STRING);
143
144 // ---- Collect files ----
145 $files = [];
146 $unchanged = 0;
147
148 foreach ($scope as $p) {
149 $real = resolveDataFile($dataRoot, $p);
150 if ($real === null) {
151 $missing[] = $p;
152 continue;
153 }
154
155 $raw = file_get_contents($real);
156 if ($raw === false) {
157 $missing[] = $p;
158 continue;
159 }
160 $hash = hash('sha256', $raw);
161
162 if ($known !== null && isset($known[$p]) && is_string($known[$p]) && strtolower($known[$p]) === $hash) {
163 $unchanged++;
164 continue;
165 }
166
167 $entry = ['sha256' => $hash];
168 if (mb_check_encoding($raw, 'UTF-8'))
169 $entry['content'] = $raw;
170 else
171 $entry['contentBase64'] = base64_encode($raw); // should never happen (CI validates UTF-8 JSON)
172
173 $files[$p] = $entry;
174 }
175
176 $out = [
177 'indexSha256' => $built['indexSha256'],
178 'fileCount' => count($indexFiles),
179 'files' => (object) $files,
180 'missing' => array_values($missing),
181 'invalid' => array_values($invalid),
182 ];
183
184 if ($known !== null) {
185 $deleted = [];
186 foreach (array_keys($known) as $p) {
187 $p = (string) $p;
188 if ($requested && !in_array($p, $scope, true))
189 continue; // selection mode: only report selected paths
190 if (!isset($indexFiles[$p]) && !isset($files[$p]))
191 $deleted[] = $p;
192 }
193 sort($deleted, SORT_STRING);
194 $out['deleted'] = $deleted;
195 $out['unchanged'] = $unchanged;
196 }
197
198 if ($method !== 'HEAD')
199 echo json_encode($out, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR);
200
201 // ---- Helpers ----
202 // (kept compatible with PHP 8.0: no array_is_list / never)
203
204 function bulkIsList(array $a): bool
205 {
206 $i = 0;
207 foreach ($a as $k => $_) {
208 if ($k !== $i++)
209 return false;
210 }
211 return true;
212 }
213
214 function bulkFail(int $code, string $message): void
215 {
216 http_response_code($code);
217 echo json_encode(['error' => $message]);
218 exit;
219 }
220
221 /**
222 * Accept only plain relative paths like "folder/file.json":
223 * no absolute paths, no "..", no hidden segments, no backslashes, .json only.
224 */
225 function isValidDataPath(string $p): bool
226 {
227 if (strlen($p) > 255)
228 return false;
229 if (!preg_match('~^[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*\.json$~', $p))
230 return false;
231 foreach (explode('/', $p) as $seg) {
232 if ($seg === '..' || $seg === '.' || str_contains($seg, '..'))
233 return false;
234 }
235 return true;
236 }
237
238 /**
239 * Resolve a validated relative path to a real file inside web/data.
240 * Returns null if it doesn't exist or resolves (e.g. via symlink) outside web/data.
241 */
242 function resolveDataFile(string $dataRoot, string $p): ?string
243 {
244 $real = realpath($dataRoot . DIRECTORY_SEPARATOR . $p);
245 if ($real === false || !is_file($real))
246 return null;
247 if (!str_starts_with($real, $dataRoot . DIRECTORY_SEPARATOR))
248 return null;
249 if (strtolower(pathinfo($real, PATHINFO_EXTENSION)) !== 'json')
250 return null;
251 return $real;
252 }
@@ -0,0 +1,252 @@
1<?php
2/**
3 * Bulk data endpoint: fetch many data files in one request.
4 *
5 * GET dataBulk.php -> all files covered by dataIndex.php
6 * GET dataBulk.php?files=a/b.json,c/d.json -> only those files
7 * POST dataBulk.php {"files": [...]} -> only those files (may be combined with ?files=)
8 * POST dataBulk.php {"known": {"a/b.json": "<sha256>", ...}}
9 * -> only files that are new or whose sha256 differs from the given one,
10 * plus "deleted": known paths that are no longer in the index.
11 * "known" can be combined with "files" (then only the selected files are compared).
12 *
13 * Paths are relative to web/data, e.g. "powers/feint.json".
14 * Response (application/json, gzip if the client accepts it):
15 * {
16 * "indexSha256": "...", same value as dataIndex.php / X-Index-Sha256
17 * "fileCount": 451, number of files in the index
18 * "files": { "<path>": { "sha256": "...", "content": "<exact file text>" } },
19 * "missing": [...], valid-looking paths that don't exist
20 * "invalid": [...], rejected paths (traversal, absolute, not .json, ...)
21 * "deleted": [...], only with "known"
22 * "unchanged": 12 only with "known": number of files skipped
23 * }
24 * "content" is the raw file text (a JSON string), so sha256(content as UTF-8 bytes)
25 * equals "sha256" and equals the per-node sha256 in dataIndex.php.
26 */
27
28ini_set('display_errors', '1');
29error_reporting(E_ALL);
30
31if (extension_loaded('zlib') && !ini_get('zlib.output_compression'))
32 ob_start('ob_gzhandler');
33
34header('Content-Type: application/json; charset=utf-8');
35header('Access-Control-Allow-Origin: *');
36header('Access-Control-Allow-Methods: GET, HEAD, OPTIONS, POST');
37header('Access-Control-Allow-Headers: *');
38header('X-Content-Type-Options: nosniff');
39header('X-Robots-Tag: noindex');
40header('Cache-Control: no-store, no-cache, must-revalidate, max-age=0');
41header('Pragma: no-cache');
42
43set_error_handler(function ($severity, $message, $file, $line) {
44 http_response_code(500);
45 echo json_encode(['error' => 'PHP Error', 'message' => $message, 'file' => basename($file), 'line' => $line]);
46 exit;
47});
48
49set_exception_handler(function ($e) {
50 http_response_code(500);
51 echo json_encode(['error' => 'Uncaught Exception', 'type' => get_class($e), 'message' => $e->getMessage()]);
52 exit;
53});
54
55require_once __DIR__ . '/dataIndexLib.php';
56
57$method = $_SERVER['REQUEST_METHOD'] ?? 'GET';
58if ($method === 'OPTIONS') {
59 http_response_code(204);
60 exit;
61}
62if (!in_array($method, ['GET', 'HEAD', 'POST'], true))
63 bulkFail(405, 'Method not allowed');
64
65const BULK_MAX_BODY = 4 * 1024 * 1024;
66const BULK_MAX_PATHS = 10000;
67
68// ---- Parse request ----
69$requested = [];
70$known = null;
71
72if (isset($_GET['files']) && is_string($_GET['files']))
73 $requested = array_merge($requested, explode(',', $_GET['files']));
74
75if ($method === 'POST') {
76 $raw = file_get_contents('php://input', false, null, 0, BULK_MAX_BODY + 1);
77 if ($raw !== false && strlen($raw) > BULK_MAX_BODY)
78 bulkFail(413, 'Request body too large');
79
80 if ($raw !== false && trim($raw) !== '') {
81 try {
82 $body = json_decode($raw, true, 16, JSON_THROW_ON_ERROR);
83 } catch (Throwable $e) {
84 bulkFail(400, 'Body is not valid JSON');
85 }
86 if (!is_array($body) || ($body !== [] && bulkIsList($body)))
87 bulkFail(400, 'Body must be a JSON object');
88
89 if (isset($body['files'])) {
90 if (!is_array($body['files']) || !bulkIsList($body['files']))
91 bulkFail(400, '"files" must be an array of paths');
92 $requested = array_merge($requested, $body['files']);
93 }
94 if (isset($body['known'])) {
95 if (!is_array($body['known']) || ($body['known'] !== [] && bulkIsList($body['known'])))
96 bulkFail(400, '"known" must be an object of path => sha256');
97 $known = $body['known'];
98 }
99 }
100}
101
102if (count($requested) > BULK_MAX_PATHS || ($known !== null && count($known) > BULK_MAX_PATHS))
103 bulkFail(413, 'Too many paths');
104
105// ---- Build index (same logic and hashes as dataIndex.php) ----
106$built = buildDataIndex();
107$indexFiles = []; // "powers/feint.json" => sha256
108foreach ($built['files'] as $p => $h) {
109 if (str_starts_with($p, 'data/'))
110 $indexFiles[substr($p, 5)] = $h;
111}
112header('X-Index-Sha256: ' . $built['indexSha256']);
113
114$dataRoot = realpath(__DIR__ . DIRECTORY_SEPARATOR . 'data');
115if ($dataRoot === false)
116 bulkFail(500, 'Data directory not found');
117
118// ---- Determine scope ----
119$missing = [];
120$invalid = [];
121
122if ($requested) {
123 $scope = [];
124 foreach ($requested as $p) {
125 if (!is_string($p)) {
126 $invalid[] = $p;
127 continue;
128 }
129 $p = trim($p);
130 if ($p === '')
131 continue;
132 if (!isValidDataPath($p)) {
133 $invalid[] = $p;
134 continue;
135 }
136 $scope[$p] = true;
137 }
138 $scope = array_keys($scope);
139} else {
140 $scope = array_keys($indexFiles);
141}
142sort($scope, SORT_STRING);
143
144// ---- Collect files ----
145$files = [];
146$unchanged = 0;
147
148foreach ($scope as $p) {
149 $real = resolveDataFile($dataRoot, $p);
150 if ($real === null) {
151 $missing[] = $p;
152 continue;
153 }
154
155 $raw = file_get_contents($real);
156 if ($raw === false) {
157 $missing[] = $p;
158 continue;
159 }
160 $hash = hash('sha256', $raw);
161
162 if ($known !== null && isset($known[$p]) && is_string($known[$p]) && strtolower($known[$p]) === $hash) {
163 $unchanged++;
164 continue;
165 }
166
167 $entry = ['sha256' => $hash];
168 if (mb_check_encoding($raw, 'UTF-8'))
169 $entry['content'] = $raw;
170 else
171 $entry['contentBase64'] = base64_encode($raw); // should never happen (CI validates UTF-8 JSON)
172
173 $files[$p] = $entry;
174}
175
176$out = [
177 'indexSha256' => $built['indexSha256'],
178 'fileCount' => count($indexFiles),
179 'files' => (object) $files,
180 'missing' => array_values($missing),
181 'invalid' => array_values($invalid),
182];
183
184if ($known !== null) {
185 $deleted = [];
186 foreach (array_keys($known) as $p) {
187 $p = (string) $p;
188 if ($requested && !in_array($p, $scope, true))
189 continue; // selection mode: only report selected paths
190 if (!isset($indexFiles[$p]) && !isset($files[$p]))
191 $deleted[] = $p;
192 }
193 sort($deleted, SORT_STRING);
194 $out['deleted'] = $deleted;
195 $out['unchanged'] = $unchanged;
196}
197
198if ($method !== 'HEAD')
199 echo json_encode($out, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR);
200
201// ---- Helpers ----
202// (kept compatible with PHP 8.0: no array_is_list / never)
203
204function bulkIsList(array $a): bool
205{
206 $i = 0;
207 foreach ($a as $k => $_) {
208 if ($k !== $i++)
209 return false;
210 }
211 return true;
212}
213
214function bulkFail(int $code, string $message): void
215{
216 http_response_code($code);
217 echo json_encode(['error' => $message]);
218 exit;
219}
220
221/**
222 * Accept only plain relative paths like "folder/file.json":
223 * no absolute paths, no "..", no hidden segments, no backslashes, .json only.
224 */
225function isValidDataPath(string $p): bool
226{
227 if (strlen($p) > 255)
228 return false;
229 if (!preg_match('~^[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*\.json$~', $p))
230 return false;
231 foreach (explode('/', $p) as $seg) {
232 if ($seg === '..' || $seg === '.' || str_contains($seg, '..'))
233 return false;
234 }
235 return true;
236}
237
238/**
239 * Resolve a validated relative path to a real file inside web/data.
240 * Returns null if it doesn't exist or resolves (e.g. via symlink) outside web/data.
241 */
242function resolveDataFile(string $dataRoot, string $p): ?string
243{
244 $real = realpath($dataRoot . DIRECTORY_SEPARATOR . $p);
245 if ($real === false || !is_file($real))
246 return null;
247 if (!str_starts_with($real, $dataRoot . DIRECTORY_SEPARATOR))
248 return null;
249 if (strtolower(pathinfo($real, PATHINFO_EXTENSION)) !== 'json')
250 return null;
251 return $real;
252}
web/dataIndex.php +4 −330
@@ -39,42 +39,16 @@ set_exception_handler(function ($e) {
39 39 });
40 40
41 41
42 require_once __DIR__ . '/folderNode.php';
42 require_once __DIR__ . '/dataIndexLib.php';
43 43
44 44 if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'OPTIONS') {
45 45 http_response_code(204);
46 46 exit;
47 47 }
48 48
49 $dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';
50 $chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';
51
52 $allJson = loadAllJson($dataRoot);
53 $visited = [];
54 $baseUrl = getBaseUrl();
55
56 $chaptersTree = loadFolderTree($chaptersRoot);
57 $chapters = $chaptersTree->jsonRecursive();
58
59 usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));
60
61 foreach ($chapters as $chapterObj) {
62 $chapter = json_decode(json_encode($chapterObj), true);
63 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
64 $relPath = normalizeDataRelPath($chapterLink);
65 if ($relPath !== '')
66 $visited[$relPath] = true;
67 }
68
69 $index = [];
70 foreach ($chapters as $chapterObj) {
71 $chapter = json_decode(json_encode($chapterObj), true);
72 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
73
74 $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
75 }
76
77 $indexSha256 = computeIndexSha256($index, $baseUrl);
49 $built = buildDataIndex();
50 $index = $built['nodes'];
51 $indexSha256 = $built['indexSha256'];
78 52 header('X-Index-Sha256: ' . $indexSha256);
79 53
80 54 if (isset($_GET['meta'])) {
@@ -86,303 +60,3 @@ if (isset($_GET['meta'])) {
86 60 } else {
87 61 echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
88 62 }
89
90 /**
91 * sha256 (lowercase hex) of the exact bytes of a data file, given its path
92 * relative to the web root (e.g. "/data/powers/feint.json").
93 * Returns null if the file does not exist or cannot be read.
94 * Results are memoised per request and keyed by path + mtime + size.
95 */
96 function fileSha256(string $webRelPath): ?string
97 {
98 static $cache = [];
99
100 $webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
101 $path = __DIR__ . $webRelPath;
102 if (!is_file($path))
103 return null;
104
105 $key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
106 if (!array_key_exists($key, $cache)) {
107 $hash = hash_file('sha256', $path);
108 $cache[$key] = $hash === false ? null : $hash;
109 }
110
111 return $cache[$key];
112 }
113
114 /**
115 * indexSha256 definition:
116 * Collect every node in the index (recursively) that has a non-null sha256.
117 * Take the unique data files they point to, as paths relative to the web
118 * root without a leading slash (e.g. "data/powers/feint.json").
119 * Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
120 * per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
121 * indexSha256 = lowercase hex sha256 of the concatenation of those lines.
122 * Reproducible from the web/ directory with:
123 * sha256sum <those files, LC_ALL=C sorted> | sha256sum
124 */
125 function computeIndexSha256(array $nodes, string $baseUrl): string
126 {
127 $files = [];
128 collectNodeHashes($nodes, $baseUrl, $files);
129 ksort($files, SORT_STRING);
130
131 $lines = '';
132 foreach ($files as $path => $hash)
133 $lines .= $hash . ' ' . $path . "\n";
134
135 return hash('sha256', $lines);
136 }
137
138 function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
139 {
140 $prefix = $baseUrl . '/';
141 foreach ($nodes as $node) {
142 if (!is_array($node))
143 continue;
144
145 $link = (string) ($node['link'] ?? '');
146 if (isset($node['sha256']) && str_starts_with($link, $prefix))
147 $files[substr($link, strlen($prefix))] = $node['sha256'];
148
149 collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
150 }
151 }
152
153 function loadAllJson(string $dataRoot): array
154 {
155 $map = [];
156
157 $it = new RecursiveIteratorIterator(
158 new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)
159 );
160
161 foreach ($it as $file) {
162 /** @var SplFileInfo $file */
163 if (strtolower($file->getExtension()) !== 'json')
164 continue;
165
166 $path = $file->getPathname();
167 $rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));
168 if ($rel === '')
169 $rel = '/' . $file->getFilename();
170 if ($rel[0] !== '/')
171 $rel = '/' . $rel;
172
173 try {
174 $raw = file_get_contents($path);
175 if ($raw === false)
176 continue;
177 $json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
178 if (!is_array($json))
179 continue;
180 } catch (Throwable $e) {
181 continue;
182 }
183
184 $json['_relPath'] = $rel;
185 $map[$rel] = $json;
186 }
187
188 return $map;
189 }
190
191 function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array
192 {
193 $name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');
194 $fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';
195
196 $node = [
197 'name' => $name,
198 'type' => $chapter['type'] ?? 'chapter',
199 'pageType' => 'page',
200 'link' => $fullChapterLink,
201 'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
202 'children' => [],
203 ];
204
205 $folderIncludes = extractFolderIncludes($chapter, ['subheaders']);
206
207 $children = [];
208
209 foreach (($chapter['subheaders'] ?? []) as $subheader) {
210 if (!is_array($subheader))
211 continue;
212
213 $subNode = [
214 'name' => $subheader['name'] ?? 'subheader',
215 'type' => 'subheader',
216 'pageType' => 'page child element',
217 'link' => $fullChapterLink,
218 'sha256' => $node['sha256'],
219 'children' => [],
220 ];
221
222 $subIncludes = extractFolderIncludes($subheader);
223 $subNode['children'] = mergeChildren(
224 buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)
225 );
226
227 $children[] = $subNode;
228 }
229
230 $children = mergeChildren(
231 $children,
232 buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)
233 );
234
235 $node['children'] = $children;
236
237 return $node;
238 }
239
240 function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array
241 {
242 $children = [];
243 $rels = array_keys($allJson);
244 sort($rels, SORT_NATURAL | SORT_FLAG_CASE);
245 foreach ($folders as $folder) {
246 $folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');
247 foreach ($rels as $rel) {
248 if (str_starts_with($rel, $folderRel . '/')) {
249 $children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);
250 }
251 }
252 }
253
254 return $children;
255 }
256
257 function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array
258 {
259 $link = $baseUrl . '/data' . $relPath;
260 $json = $allJson[$relPath] ?? null;
261
262 $name = $label;
263 $type = 'linkedData';
264
265 if (is_array($json)) {
266 $type = $json['type'] ?? 'data';
267 if ($name === '') {
268 $name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));
269 }
270 }
271
272 $node = [
273 'name' => $name !== '' ? $name : basename($relPath, '.json'),
274 'type' => $type,
275 'pageType' => 'data link',
276 'link' => $link,
277 'sha256' => fileSha256('/data' . $relPath),
278 'children' => [],
279 ];
280
281 if (!is_array($json))
282 return $node;
283
284 if (isset($visited[$relPath]))
285 return $node;
286
287 $visited[$relPath] = true;
288
289 $folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);
290 $node['children'] = mergeChildren($folderChildren);
291
292 return $node;
293 }
294
295 function normalizeWikiPath(string $path): string
296 {
297 $path = str_replace('\\', '/', $path);
298 if ($path === '')
299 return '';
300
301 if (!str_starts_with($path, '/'))
302 $path = '/' . $path;
303
304 return $path;
305 }
306
307 function getBaseUrl(): string
308 {
309 $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
310 $host = $_SERVER['HTTP_HOST'] ?? 'localhost';
311 $scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';
312 $dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');
313 return $scheme . '://' . $host . ($dir === '' ? '' : $dir);
314 }
315
316 function normalizeDataRelPath(string $path): string
317 {
318 $path = trim($path);
319 if ($path === '')
320 return '';
321
322 $path = str_replace('\\', '/', $path);
323 if (str_starts_with($path, '/data/'))
324 $path = substr($path, strlen('/data'));
325 if (str_starts_with($path, 'data/'))
326 $path = substr($path, strlen('data'));
327
328 if (!str_starts_with($path, '/'))
329 $path = '/' . $path;
330
331 return $path;
332 }
333
334 function extractFolderIncludes(mixed $value, array $skipKeys = []): array
335 {
336 $folders = [];
337
338 if (is_array($value)) {
339 foreach ($value as $k => $v) {
340 if (in_array($k, $skipKeys, true))
341 continue;
342 $folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));
343 }
344 return $folders;
345 }
346
347 if (is_string($value)) {
348 return extractFolderIncludesFromString($value);
349 }
350
351 return $folders;
352 }
353
354 function extractFolderIncludesFromString(string $text): array
355 {
356 $folders = [];
357 $pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';
358
359 if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {
360 foreach ($matches as $m) {
361 $folders[] = $m[1];
362 }
363 }
364
365 return $folders;
366 }
367
368 function mergeChildren(array ...$groups): array
369 {
370 $merged = [];
371 $seen = [];
372
373 foreach ($groups as $children) {
374 foreach ($children as $child) {
375 if (!is_array($child))
376 continue;
377
378 $key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');
379 if (isset($seen[$key]))
380 continue;
381
382 $seen[$key] = true;
383 $merged[] = $child;
384 }
385 }
386
387 return $merged;
388 }
@@ -39,42 +39,16 @@ set_exception_handler(function ($e) {
39});
39});
40
40
41
41
42require_once __DIR__ . '/folderNode.php';
42require_once __DIR__ . '/dataIndexLib.php';
43
43
44if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'OPTIONS') {
44if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'OPTIONS') {
45 http_response_code(204);
45 http_response_code(204);
46 exit;
46 exit;
47}
47}
48
48
49$dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';
49$built = buildDataIndex();
50$chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';
50$index = $built['nodes'];
51
51$indexSha256 = $built['indexSha256'];
52$allJson = loadAllJson($dataRoot);
53$visited = [];
54$baseUrl = getBaseUrl();
55
56$chaptersTree = loadFolderTree($chaptersRoot);
57$chapters = $chaptersTree->jsonRecursive();
58
59usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));
60
61foreach ($chapters as $chapterObj) {
62 $chapter = json_decode(json_encode($chapterObj), true);
63 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
64 $relPath = normalizeDataRelPath($chapterLink);
65 if ($relPath !== '')
66 $visited[$relPath] = true;
67}
68
69$index = [];
70foreach ($chapters as $chapterObj) {
71 $chapter = json_decode(json_encode($chapterObj), true);
72 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
73
74 $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
75}
76
77$indexSha256 = computeIndexSha256($index, $baseUrl);
78header('X-Index-Sha256: ' . $indexSha256);
52header('X-Index-Sha256: ' . $indexSha256);
79
53
80if (isset($_GET['meta'])) {
54if (isset($_GET['meta'])) {
@@ -86,303 +60,3 @@ if (isset($_GET['meta'])) {
86} else {
60} else {
87 echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
61 echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
88}
62}
89
90/**
91 * sha256 (lowercase hex) of the exact bytes of a data file, given its path
92 * relative to the web root (e.g. "/data/powers/feint.json").
93 * Returns null if the file does not exist or cannot be read.
94 * Results are memoised per request and keyed by path + mtime + size.
95 */
96function fileSha256(string $webRelPath): ?string
97{
98 static $cache = [];
99
100 $webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
101 $path = __DIR__ . $webRelPath;
102 if (!is_file($path))
103 return null;
104
105 $key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
106 if (!array_key_exists($key, $cache)) {
107 $hash = hash_file('sha256', $path);
108 $cache[$key] = $hash === false ? null : $hash;
109 }
110
111 return $cache[$key];
112}
113
114/**
115 * indexSha256 definition:
116 * Collect every node in the index (recursively) that has a non-null sha256.
117 * Take the unique data files they point to, as paths relative to the web
118 * root without a leading slash (e.g. "data/powers/feint.json").
119 * Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
120 * per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
121 * indexSha256 = lowercase hex sha256 of the concatenation of those lines.
122 * Reproducible from the web/ directory with:
123 * sha256sum <those files, LC_ALL=C sorted> | sha256sum
124 */
125function computeIndexSha256(array $nodes, string $baseUrl): string
126{
127 $files = [];
128 collectNodeHashes($nodes, $baseUrl, $files);
129 ksort($files, SORT_STRING);
130
131 $lines = '';
132 foreach ($files as $path => $hash)
133 $lines .= $hash . ' ' . $path . "\n";
134
135 return hash('sha256', $lines);
136}
137
138function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
139{
140 $prefix = $baseUrl . '/';
141 foreach ($nodes as $node) {
142 if (!is_array($node))
143 continue;
144
145 $link = (string) ($node['link'] ?? '');
146 if (isset($node['sha256']) && str_starts_with($link, $prefix))
147 $files[substr($link, strlen($prefix))] = $node['sha256'];
148
149 collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
150 }
151}
152
153function loadAllJson(string $dataRoot): array
154{
155 $map = [];
156
157 $it = new RecursiveIteratorIterator(
158 new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)
159 );
160
161 foreach ($it as $file) {
162 /** @var SplFileInfo $file */
163 if (strtolower($file->getExtension()) !== 'json')
164 continue;
165
166 $path = $file->getPathname();
167 $rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));
168 if ($rel === '')
169 $rel = '/' . $file->getFilename();
170 if ($rel[0] !== '/')
171 $rel = '/' . $rel;
172
173 try {
174 $raw = file_get_contents($path);
175 if ($raw === false)
176 continue;
177 $json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
178 if (!is_array($json))
179 continue;
180 } catch (Throwable $e) {
181 continue;
182 }
183
184 $json['_relPath'] = $rel;
185 $map[$rel] = $json;
186 }
187
188 return $map;
189}
190
191function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array
192{
193 $name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');
194 $fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';
195
196 $node = [
197 'name' => $name,
198 'type' => $chapter['type'] ?? 'chapter',
199 'pageType' => 'page',
200 'link' => $fullChapterLink,
201 'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
202 'children' => [],
203 ];
204
205 $folderIncludes = extractFolderIncludes($chapter, ['subheaders']);
206
207 $children = [];
208
209 foreach (($chapter['subheaders'] ?? []) as $subheader) {
210 if (!is_array($subheader))
211 continue;
212
213 $subNode = [
214 'name' => $subheader['name'] ?? 'subheader',
215 'type' => 'subheader',
216 'pageType' => 'page child element',
217 'link' => $fullChapterLink,
218 'sha256' => $node['sha256'],
219 'children' => [],
220 ];
221
222 $subIncludes = extractFolderIncludes($subheader);
223 $subNode['children'] = mergeChildren(
224 buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)
225 );
226
227 $children[] = $subNode;
228 }
229
230 $children = mergeChildren(
231 $children,
232 buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)
233 );
234
235 $node['children'] = $children;
236
237 return $node;
238}
239
240function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array
241{
242 $children = [];
243 $rels = array_keys($allJson);
244 sort($rels, SORT_NATURAL | SORT_FLAG_CASE);
245 foreach ($folders as $folder) {
246 $folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');
247 foreach ($rels as $rel) {
248 if (str_starts_with($rel, $folderRel . '/')) {
249 $children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);
250 }
251 }
252 }
253
254 return $children;
255}
256
257function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array
258{
259 $link = $baseUrl . '/data' . $relPath;
260 $json = $allJson[$relPath] ?? null;
261
262 $name = $label;
263 $type = 'linkedData';
264
265 if (is_array($json)) {
266 $type = $json['type'] ?? 'data';
267 if ($name === '') {
268 $name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));
269 }
270 }
271
272 $node = [
273 'name' => $name !== '' ? $name : basename($relPath, '.json'),
274 'type' => $type,
275 'pageType' => 'data link',
276 'link' => $link,
277 'sha256' => fileSha256('/data' . $relPath),
278 'children' => [],
279 ];
280
281 if (!is_array($json))
282 return $node;
283
284 if (isset($visited[$relPath]))
285 return $node;
286
287 $visited[$relPath] = true;
288
289 $folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);
290 $node['children'] = mergeChildren($folderChildren);
291
292 return $node;
293}
294
295function normalizeWikiPath(string $path): string
296{
297 $path = str_replace('\\', '/', $path);
298 if ($path === '')
299 return '';
300
301 if (!str_starts_with($path, '/'))
302 $path = '/' . $path;
303
304 return $path;
305}
306
307function getBaseUrl(): string
308{
309 $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
310 $host = $_SERVER['HTTP_HOST'] ?? 'localhost';
311 $scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';
312 $dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');
313 return $scheme . '://' . $host . ($dir === '' ? '' : $dir);
314}
315
316function normalizeDataRelPath(string $path): string
317{
318 $path = trim($path);
319 if ($path === '')
320 return '';
321
322 $path = str_replace('\\', '/', $path);
323 if (str_starts_with($path, '/data/'))
324 $path = substr($path, strlen('/data'));
325 if (str_starts_with($path, 'data/'))
326 $path = substr($path, strlen('data'));
327
328 if (!str_starts_with($path, '/'))
329 $path = '/' . $path;
330
331 return $path;
332}
333
334function extractFolderIncludes(mixed $value, array $skipKeys = []): array
335{
336 $folders = [];
337
338 if (is_array($value)) {
339 foreach ($value as $k => $v) {
340 if (in_array($k, $skipKeys, true))
341 continue;
342 $folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));
343 }
344 return $folders;
345 }
346
347 if (is_string($value)) {
348 return extractFolderIncludesFromString($value);
349 }
350
351 return $folders;
352}
353
354function extractFolderIncludesFromString(string $text): array
355{
356 $folders = [];
357 $pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';
358
359 if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {
360 foreach ($matches as $m) {
361 $folders[] = $m[1];
362 }
363 }
364
365 return $folders;
366}
367
368function mergeChildren(array ...$groups): array
369{
370 $merged = [];
371 $seen = [];
372
373 foreach ($groups as $children) {
374 foreach ($children as $child) {
375 if (!is_array($child))
376 continue;
377
378 $key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');
379 if (isset($seen[$key]))
380 continue;
381
382 $seen[$key] = true;
383 $merged[] = $child;
384 }
385 }
386
387 return $merged;
388}
web/dataIndexLib.php added +368 −0
@@ -0,0 +1,368 @@
1 <?php
2 /**
3 * Shared index logic for dataIndex.php and dataBulk.php.
4 * Defines functions only; produces no output.
5 */
6
7 require_once __DIR__ . '/folderNode.php';
8
9 /**
10 * Build the data index.
11 * @return array{nodes: array, indexSha256: string, files: array<string,string>}
12 * files: "data/<folder>/<file>.json" => sha256, for every file in the index
13 * (the same set and hashes that indexSha256 is computed from), sorted by path.
14 */
15 function buildDataIndex(): array
16 {
17 $dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';
18 $chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';
19
20 $allJson = loadAllJson($dataRoot);
21 $visited = [];
22 $baseUrl = getBaseUrl();
23
24 $chaptersTree = loadFolderTree($chaptersRoot);
25 $chapters = $chaptersTree->jsonRecursive();
26
27 usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));
28
29 foreach ($chapters as $chapterObj) {
30 $chapter = json_decode(json_encode($chapterObj), true);
31 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
32 $relPath = normalizeDataRelPath($chapterLink);
33 if ($relPath !== '')
34 $visited[$relPath] = true;
35 }
36
37 $index = [];
38 foreach ($chapters as $chapterObj) {
39 $chapter = json_decode(json_encode($chapterObj), true);
40 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
41
42 $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
43 }
44
45 $files = collectIndexFiles($index, $baseUrl);
46
47 return [
48 'nodes' => $index,
49 'indexSha256' => indexSha256FromFiles($files),
50 'files' => $files,
51 ];
52 }
53
54 /**
55 * sha256 (lowercase hex) of the exact bytes of a data file, given its path
56 * relative to the web root (e.g. "/data/powers/feint.json").
57 * Returns null if the file does not exist or cannot be read.
58 * Results are memoised per request and keyed by path + mtime + size.
59 */
60 function fileSha256(string $webRelPath): ?string
61 {
62 static $cache = [];
63
64 $webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
65 $path = __DIR__ . $webRelPath;
66 if (!is_file($path))
67 return null;
68
69 $key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
70 if (!array_key_exists($key, $cache)) {
71 $hash = hash_file('sha256', $path);
72 $cache[$key] = $hash === false ? null : $hash;
73 }
74
75 return $cache[$key];
76 }
77
78 /**
79 * indexSha256 definition:
80 * Collect every node in the index (recursively) that has a non-null sha256.
81 * Take the unique data files they point to, as paths relative to the web
82 * root without a leading slash (e.g. "data/powers/feint.json").
83 * Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
84 * per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
85 * indexSha256 = lowercase hex sha256 of the concatenation of those lines.
86 * Reproducible from the web/ directory with:
87 * sha256sum <those files, LC_ALL=C sorted> | sha256sum
88 */
89 function computeIndexSha256(array $nodes, string $baseUrl): string
90 {
91 return indexSha256FromFiles(collectIndexFiles($nodes, $baseUrl));
92 }
93
94 /**
95 * Unique data files referenced by the index: "data/<folder>/<file>.json" => sha256,
96 * sorted by path (byte-wise).
97 */
98 function collectIndexFiles(array $nodes, string $baseUrl): array
99 {
100 $files = [];
101 collectNodeHashes($nodes, $baseUrl, $files);
102 ksort($files, SORT_STRING);
103 return $files;
104 }
105
106 /**
107 * @param array<string,string> $files path => sha256, already sorted by path
108 */
109 function indexSha256FromFiles(array $files): string
110 {
111 $lines = '';
112 foreach ($files as $path => $hash)
113 $lines .= $hash . ' ' . $path . "\n";
114
115 return hash('sha256', $lines);
116 }
117
118 function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
119 {
120 $prefix = $baseUrl . '/';
121 foreach ($nodes as $node) {
122 if (!is_array($node))
123 continue;
124
125 $link = (string) ($node['link'] ?? '');
126 if (isset($node['sha256']) && str_starts_with($link, $prefix))
127 $files[substr($link, strlen($prefix))] = $node['sha256'];
128
129 collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
130 }
131 }
132
133 function loadAllJson(string $dataRoot): array
134 {
135 $map = [];
136
137 $it = new RecursiveIteratorIterator(
138 new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)
139 );
140
141 foreach ($it as $file) {
142 /** @var SplFileInfo $file */
143 if (strtolower($file->getExtension()) !== 'json')
144 continue;
145
146 $path = $file->getPathname();
147 $rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));
148 if ($rel === '')
149 $rel = '/' . $file->getFilename();
150 if ($rel[0] !== '/')
151 $rel = '/' . $rel;
152
153 try {
154 $raw = file_get_contents($path);
155 if ($raw === false)
156 continue;
157 $json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
158 if (!is_array($json))
159 continue;
160 } catch (Throwable $e) {
161 continue;
162 }
163
164 $json['_relPath'] = $rel;
165 $map[$rel] = $json;
166 }
167
168 return $map;
169 }
170
171 function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array
172 {
173 $name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');
174 $fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';
175
176 $node = [
177 'name' => $name,
178 'type' => $chapter['type'] ?? 'chapter',
179 'pageType' => 'page',
180 'link' => $fullChapterLink,
181 'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
182 'children' => [],
183 ];
184
185 $folderIncludes = extractFolderIncludes($chapter, ['subheaders']);
186
187 $children = [];
188
189 foreach (($chapter['subheaders'] ?? []) as $subheader) {
190 if (!is_array($subheader))
191 continue;
192
193 $subNode = [
194 'name' => $subheader['name'] ?? 'subheader',
195 'type' => 'subheader',
196 'pageType' => 'page child element',
197 'link' => $fullChapterLink,
198 'sha256' => $node['sha256'],
199 'children' => [],
200 ];
201
202 $subIncludes = extractFolderIncludes($subheader);
203 $subNode['children'] = mergeChildren(
204 buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)
205 );
206
207 $children[] = $subNode;
208 }
209
210 $children = mergeChildren(
211 $children,
212 buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)
213 );
214
215 $node['children'] = $children;
216
217 return $node;
218 }
219
220 function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array
221 {
222 $children = [];
223 $rels = array_keys($allJson);
224 sort($rels, SORT_NATURAL | SORT_FLAG_CASE);
225 foreach ($folders as $folder) {
226 $folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');
227 foreach ($rels as $rel) {
228 if (str_starts_with($rel, $folderRel . '/')) {
229 $children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);
230 }
231 }
232 }
233
234 return $children;
235 }
236
237 function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array
238 {
239 $link = $baseUrl . '/data' . $relPath;
240 $json = $allJson[$relPath] ?? null;
241
242 $name = $label;
243 $type = 'linkedData';
244
245 if (is_array($json)) {
246 $type = $json['type'] ?? 'data';
247 if ($name === '') {
248 $name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));
249 }
250 }
251
252 $node = [
253 'name' => $name !== '' ? $name : basename($relPath, '.json'),
254 'type' => $type,
255 'pageType' => 'data link',
256 'link' => $link,
257 'sha256' => fileSha256('/data' . $relPath),
258 'children' => [],
259 ];
260
261 if (!is_array($json))
262 return $node;
263
264 if (isset($visited[$relPath]))
265 return $node;
266
267 $visited[$relPath] = true;
268
269 $folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);
270 $node['children'] = mergeChildren($folderChildren);
271
272 return $node;
273 }
274
275 function normalizeWikiPath(string $path): string
276 {
277 $path = str_replace('\\', '/', $path);
278 if ($path === '')
279 return '';
280
281 if (!str_starts_with($path, '/'))
282 $path = '/' . $path;
283
284 return $path;
285 }
286
287 function getBaseUrl(): string
288 {
289 $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
290 $host = $_SERVER['HTTP_HOST'] ?? 'localhost';
291 $scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';
292 $dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');
293 return $scheme . '://' . $host . ($dir === '' ? '' : $dir);
294 }
295
296 function normalizeDataRelPath(string $path): string
297 {
298 $path = trim($path);
299 if ($path === '')
300 return '';
301
302 $path = str_replace('\\', '/', $path);
303 if (str_starts_with($path, '/data/'))
304 $path = substr($path, strlen('/data'));
305 if (str_starts_with($path, 'data/'))
306 $path = substr($path, strlen('data'));
307
308 if (!str_starts_with($path, '/'))
309 $path = '/' . $path;
310
311 return $path;
312 }
313
314 function extractFolderIncludes(mixed $value, array $skipKeys = []): array
315 {
316 $folders = [];
317
318 if (is_array($value)) {
319 foreach ($value as $k => $v) {
320 if (in_array($k, $skipKeys, true))
321 continue;
322 $folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));
323 }
324 return $folders;
325 }
326
327 if (is_string($value)) {
328 return extractFolderIncludesFromString($value);
329 }
330
331 return $folders;
332 }
333
334 function extractFolderIncludesFromString(string $text): array
335 {
336 $folders = [];
337 $pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';
338
339 if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {
340 foreach ($matches as $m) {
341 $folders[] = $m[1];
342 }
343 }
344
345 return $folders;
346 }
347
348 function mergeChildren(array ...$groups): array
349 {
350 $merged = [];
351 $seen = [];
352
353 foreach ($groups as $children) {
354 foreach ($children as $child) {
355 if (!is_array($child))
356 continue;
357
358 $key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');
359 if (isset($seen[$key]))
360 continue;
361
362 $seen[$key] = true;
363 $merged[] = $child;
364 }
365 }
366
367 return $merged;
368 }
@@ -0,0 +1,368 @@
1<?php
2/**
3 * Shared index logic for dataIndex.php and dataBulk.php.
4 * Defines functions only; produces no output.
5 */
6
7require_once __DIR__ . '/folderNode.php';
8
9/**
10 * Build the data index.
11 * @return array{nodes: array, indexSha256: string, files: array<string,string>}
12 * files: "data/<folder>/<file>.json" => sha256, for every file in the index
13 * (the same set and hashes that indexSha256 is computed from), sorted by path.
14 */
15function buildDataIndex(): array
16{
17 $dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';
18 $chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';
19
20 $allJson = loadAllJson($dataRoot);
21 $visited = [];
22 $baseUrl = getBaseUrl();
23
24 $chaptersTree = loadFolderTree($chaptersRoot);
25 $chapters = $chaptersTree->jsonRecursive();
26
27 usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));
28
29 foreach ($chapters as $chapterObj) {
30 $chapter = json_decode(json_encode($chapterObj), true);
31 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
32 $relPath = normalizeDataRelPath($chapterLink);
33 if ($relPath !== '')
34 $visited[$relPath] = true;
35 }
36
37 $index = [];
38 foreach ($chapters as $chapterObj) {
39 $chapter = json_decode(json_encode($chapterObj), true);
40 $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
41
42 $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
43 }
44
45 $files = collectIndexFiles($index, $baseUrl);
46
47 return [
48 'nodes' => $index,
49 'indexSha256' => indexSha256FromFiles($files),
50 'files' => $files,
51 ];
52}
53
54/**
55 * sha256 (lowercase hex) of the exact bytes of a data file, given its path
56 * relative to the web root (e.g. "/data/powers/feint.json").
57 * Returns null if the file does not exist or cannot be read.
58 * Results are memoised per request and keyed by path + mtime + size.
59 */
60function fileSha256(string $webRelPath): ?string
61{
62 static $cache = [];
63
64 $webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
65 $path = __DIR__ . $webRelPath;
66 if (!is_file($path))
67 return null;
68
69 $key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
70 if (!array_key_exists($key, $cache)) {
71 $hash = hash_file('sha256', $path);
72 $cache[$key] = $hash === false ? null : $hash;
73 }
74
75 return $cache[$key];
76}
77
78/**
79 * indexSha256 definition:
80 * Collect every node in the index (recursively) that has a non-null sha256.
81 * Take the unique data files they point to, as paths relative to the web
82 * root without a leading slash (e.g. "data/powers/feint.json").
83 * Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
84 * per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
85 * indexSha256 = lowercase hex sha256 of the concatenation of those lines.
86 * Reproducible from the web/ directory with:
87 * sha256sum <those files, LC_ALL=C sorted> | sha256sum
88 */
89function computeIndexSha256(array $nodes, string $baseUrl): string
90{
91 return indexSha256FromFiles(collectIndexFiles($nodes, $baseUrl));
92}
93
94/**
95 * Unique data files referenced by the index: "data/<folder>/<file>.json" => sha256,
96 * sorted by path (byte-wise).
97 */
98function collectIndexFiles(array $nodes, string $baseUrl): array
99{
100 $files = [];
101 collectNodeHashes($nodes, $baseUrl, $files);
102 ksort($files, SORT_STRING);
103 return $files;
104}
105
106/**
107 * @param array<string,string> $files path => sha256, already sorted by path
108 */
109function indexSha256FromFiles(array $files): string
110{
111 $lines = '';
112 foreach ($files as $path => $hash)
113 $lines .= $hash . ' ' . $path . "\n";
114
115 return hash('sha256', $lines);
116}
117
118function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
119{
120 $prefix = $baseUrl . '/';
121 foreach ($nodes as $node) {
122 if (!is_array($node))
123 continue;
124
125 $link = (string) ($node['link'] ?? '');
126 if (isset($node['sha256']) && str_starts_with($link, $prefix))
127 $files[substr($link, strlen($prefix))] = $node['sha256'];
128
129 collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
130 }
131}
132
133function loadAllJson(string $dataRoot): array
134{
135 $map = [];
136
137 $it = new RecursiveIteratorIterator(
138 new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)
139 );
140
141 foreach ($it as $file) {
142 /** @var SplFileInfo $file */
143 if (strtolower($file->getExtension()) !== 'json')
144 continue;
145
146 $path = $file->getPathname();
147 $rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));
148 if ($rel === '')
149 $rel = '/' . $file->getFilename();
150 if ($rel[0] !== '/')
151 $rel = '/' . $rel;
152
153 try {
154 $raw = file_get_contents($path);
155 if ($raw === false)
156 continue;
157 $json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
158 if (!is_array($json))
159 continue;
160 } catch (Throwable $e) {
161 continue;
162 }
163
164 $json['_relPath'] = $rel;
165 $map[$rel] = $json;
166 }
167
168 return $map;
169}
170
171function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array
172{
173 $name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');
174 $fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';
175
176 $node = [
177 'name' => $name,
178 'type' => $chapter['type'] ?? 'chapter',
179 'pageType' => 'page',
180 'link' => $fullChapterLink,
181 'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
182 'children' => [],
183 ];
184
185 $folderIncludes = extractFolderIncludes($chapter, ['subheaders']);
186
187 $children = [];
188
189 foreach (($chapter['subheaders'] ?? []) as $subheader) {
190 if (!is_array($subheader))
191 continue;
192
193 $subNode = [
194 'name' => $subheader['name'] ?? 'subheader',
195 'type' => 'subheader',
196 'pageType' => 'page child element',
197 'link' => $fullChapterLink,
198 'sha256' => $node['sha256'],
199 'children' => [],
200 ];
201
202 $subIncludes = extractFolderIncludes($subheader);
203 $subNode['children'] = mergeChildren(
204 buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)
205 );
206
207 $children[] = $subNode;
208 }
209
210 $children = mergeChildren(
211 $children,
212 buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)
213 );
214
215 $node['children'] = $children;
216
217 return $node;
218}
219
220function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array
221{
222 $children = [];
223 $rels = array_keys($allJson);
224 sort($rels, SORT_NATURAL | SORT_FLAG_CASE);
225 foreach ($folders as $folder) {
226 $folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');
227 foreach ($rels as $rel) {
228 if (str_starts_with($rel, $folderRel . '/')) {
229 $children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);
230 }
231 }
232 }
233
234 return $children;
235}
236
237function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array
238{
239 $link = $baseUrl . '/data' . $relPath;
240 $json = $allJson[$relPath] ?? null;
241
242 $name = $label;
243 $type = 'linkedData';
244
245 if (is_array($json)) {
246 $type = $json['type'] ?? 'data';
247 if ($name === '') {
248 $name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));
249 }
250 }
251
252 $node = [
253 'name' => $name !== '' ? $name : basename($relPath, '.json'),
254 'type' => $type,
255 'pageType' => 'data link',
256 'link' => $link,
257 'sha256' => fileSha256('/data' . $relPath),
258 'children' => [],
259 ];
260
261 if (!is_array($json))
262 return $node;
263
264 if (isset($visited[$relPath]))
265 return $node;
266
267 $visited[$relPath] = true;
268
269 $folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);
270 $node['children'] = mergeChildren($folderChildren);
271
272 return $node;
273}
274
275function normalizeWikiPath(string $path): string
276{
277 $path = str_replace('\\', '/', $path);
278 if ($path === '')
279 return '';
280
281 if (!str_starts_with($path, '/'))
282 $path = '/' . $path;
283
284 return $path;
285}
286
287function getBaseUrl(): string
288{
289 $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
290 $host = $_SERVER['HTTP_HOST'] ?? 'localhost';
291 $scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';
292 $dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');
293 return $scheme . '://' . $host . ($dir === '' ? '' : $dir);
294}
295
296function normalizeDataRelPath(string $path): string
297{
298 $path = trim($path);
299 if ($path === '')
300 return '';
301
302 $path = str_replace('\\', '/', $path);
303 if (str_starts_with($path, '/data/'))
304 $path = substr($path, strlen('/data'));
305 if (str_starts_with($path, 'data/'))
306 $path = substr($path, strlen('data'));
307
308 if (!str_starts_with($path, '/'))
309 $path = '/' . $path;
310
311 return $path;
312}
313
314function extractFolderIncludes(mixed $value, array $skipKeys = []): array
315{
316 $folders = [];
317
318 if (is_array($value)) {
319 foreach ($value as $k => $v) {
320 if (in_array($k, $skipKeys, true))
321 continue;
322 $folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));
323 }
324 return $folders;
325 }
326
327 if (is_string($value)) {
328 return extractFolderIncludesFromString($value);
329 }
330
331 return $folders;
332}
333
334function extractFolderIncludesFromString(string $text): array
335{
336 $folders = [];
337 $pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';
338
339 if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {
340 foreach ($matches as $m) {
341 $folders[] = $m[1];
342 }
343 }
344
345 return $folders;
346}
347
348function mergeChildren(array ...$groups): array
349{
350 $merged = [];
351 $seen = [];
352
353 foreach ($groups as $children) {
354 foreach ($children as $child) {
355 if (!is_array($child))
356 continue;
357
358 $key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');
359 if (isset($seen[$key]))
360 continue;
361
362 $seen[$key] = true;
363 $merged[] = $child;
364 }
365 }
366
367 return $merged;
368}