Add dataBulk.php: fetch many data files in one request
- dataIndexLib.php: index building and hash logic moved out of dataIndex.php
(dataIndex.php output is byte-identical) so both endpoints share it
- dataBulk.php: GET = all indexed files {indexSha256, files: {path: {sha256,
content}}}; ?files= / POST {files} to select; POST {known: {path: sha256}}
returns only new/changed files plus deleted/unchanged; content is the raw
file text so sha256 matches dataIndex.php; paths restricted to .json under
web/data (no .., absolute paths or symlinks out); gzip via ob_gzhandler;
X-Index-Sha256 header
@@ -0,0 +1,252 @@
1
<?php
2
/**
3
* Bulk data endpoint: fetch many data files in one request.
4
*
5
* GET dataBulk.php -> all files covered by dataIndex.php
6
* GET dataBulk.php?files=a/b.json,c/d.json -> only those files
7
* POST dataBulk.php {"files": [...]} -> only those files (may be combined with ?files=)
8
* POST dataBulk.php {"known": {"a/b.json": "<sha256>", ...}}
9
* -> only files that are new or whose sha256 differs from the given one,
10
* plus "deleted": known paths that are no longer in the index.
11
* "known" can be combined with "files" (then only the selected files are compared).
12
*
13
* Paths are relative to web/data, e.g. "powers/feint.json".
14
* Response (application/json, gzip if the client accepts it):
15
* {
16
* "indexSha256": "...", same value as dataIndex.php / X-Index-Sha256
17
* "fileCount": 451, number of files in the index
18
* "files": { "<path>": { "sha256": "...", "content": "<exact file text>" } },
19
* "missing": [...], valid-looking paths that don't exist
20
* "invalid": [...], rejected paths (traversal, absolute, not .json, ...)
21
* "deleted": [...], only with "known"
22
* "unchanged": 12 only with "known": number of files skipped
23
* }
24
* "content" is the raw file text (a JSON string), so sha256(content as UTF-8 bytes)
25
* equals "sha256" and equals the per-node sha256 in dataIndex.php.
26
*/
27
28
ini_set('display_errors', '1');
29
error_reporting(E_ALL);
30
31
if (extension_loaded('zlib') && !ini_get('zlib.output_compression'))
32
ob_start('ob_gzhandler');
33
34
header('Content-Type: application/json; charset=utf-8');
35
header('Access-Control-Allow-Origin: *');
36
header('Access-Control-Allow-Methods: GET, HEAD, OPTIONS, POST');
37
header('Access-Control-Allow-Headers: *');
38
header('X-Content-Type-Options: nosniff');
39
header('X-Robots-Tag: noindex');
40
header('Cache-Control: no-store, no-cache, must-revalidate, max-age=0');
41
header('Pragma: no-cache');
42
43
set_error_handler(function ($severity, $message, $file, $line) {
44
http_response_code(500);
45
echo json_encode(['error' => 'PHP Error', 'message' => $message, 'file' => basename($file), 'line' => $line]);
46
exit;
47
});
48
49
set_exception_handler(function ($e) {
50
http_response_code(500);
51
echo json_encode(['error' => 'Uncaught Exception', 'type' => get_class($e), 'message' => $e->getMessage()]);
52
exit;
53
});
54
55
require_once __DIR__ . '/dataIndexLib.php';
56
57
$method = $_SERVER['REQUEST_METHOD'] ?? 'GET';
58
if ($method === 'OPTIONS') {
59
http_response_code(204);
60
exit;
61
}
62
if (!in_array($method, ['GET', 'HEAD', 'POST'], true))
63
bulkFail(405, 'Method not allowed');
64
65
const BULK_MAX_BODY = 4 * 1024 * 1024;
66
const BULK_MAX_PATHS = 10000;
67
68
// ---- Parse request ----
69
$requested = [];
70
$known = null;
71
72
if (isset($_GET['files']) && is_string($_GET['files']))
73
$requested = array_merge($requested, explode(',', $_GET['files']));
74
75
if ($method === 'POST') {
76
$raw = file_get_contents('php://input', false, null, 0, BULK_MAX_BODY + 1);
77
if ($raw !== false && strlen($raw) > BULK_MAX_BODY)
78
bulkFail(413, 'Request body too large');
79
80
if ($raw !== false && trim($raw) !== '') {
81
try {
82
$body = json_decode($raw, true, 16, JSON_THROW_ON_ERROR);
83
} catch (Throwable $e) {
84
bulkFail(400, 'Body is not valid JSON');
85
}
86
if (!is_array($body) || ($body !== [] && bulkIsList($body)))
87
bulkFail(400, 'Body must be a JSON object');
88
89
if (isset($body['files'])) {
90
if (!is_array($body['files']) || !bulkIsList($body['files']))
91
bulkFail(400, '"files" must be an array of paths');
92
$requested = array_merge($requested, $body['files']);
93
}
94
if (isset($body['known'])) {
95
if (!is_array($body['known']) || ($body['known'] !== [] && bulkIsList($body['known'])))
96
bulkFail(400, '"known" must be an object of path => sha256');
97
$known = $body['known'];
98
}
99
}
100
}
101
102
if (count($requested) > BULK_MAX_PATHS || ($known !== null && count($known) > BULK_MAX_PATHS))
103
bulkFail(413, 'Too many paths');
104
105
// ---- Build index (same logic and hashes as dataIndex.php) ----
106
$built = buildDataIndex();
107
$indexFiles = []; // "powers/feint.json" => sha256
108
foreach ($built['files'] as $p => $h) {
109
if (str_starts_with($p, 'data/'))
110
$indexFiles[substr($p, 5)] = $h;
111
}
112
header('X-Index-Sha256: ' . $built['indexSha256']);
113
114
$dataRoot = realpath(__DIR__ . DIRECTORY_SEPARATOR . 'data');
115
if ($dataRoot === false)
116
bulkFail(500, 'Data directory not found');
117
118
// ---- Determine scope ----
119
$missing = [];
120
$invalid = [];
121
122
if ($requested) {
123
$scope = [];
124
foreach ($requested as $p) {
125
if (!is_string($p)) {
126
$invalid[] = $p;
127
continue;
128
}
129
$p = trim($p);
130
if ($p === '')
131
continue;
132
if (!isValidDataPath($p)) {
133
$invalid[] = $p;
134
continue;
135
}
136
$scope[$p] = true;
137
}
138
$scope = array_keys($scope);
139
} else {
140
$scope = array_keys($indexFiles);
141
}
142
sort($scope, SORT_STRING);
143
144
// ---- Collect files ----
145
$files = [];
146
$unchanged = 0;
147
148
foreach ($scope as $p) {
149
$real = resolveDataFile($dataRoot, $p);
150
if ($real === null) {
151
$missing[] = $p;
152
continue;
153
}
154
155
$raw = file_get_contents($real);
156
if ($raw === false) {
157
$missing[] = $p;
158
continue;
159
}
160
$hash = hash('sha256', $raw);
161
162
if ($known !== null && isset($known[$p]) && is_string($known[$p]) && strtolower($known[$p]) === $hash) {
163
$unchanged++;
164
continue;
165
}
166
167
$entry = ['sha256' => $hash];
168
if (mb_check_encoding($raw, 'UTF-8'))
169
$entry['content'] = $raw;
170
else
171
$entry['contentBase64'] = base64_encode($raw); // should never happen (CI validates UTF-8 JSON)
172
173
$files[$p] = $entry;
174
}
175
176
$out = [
177
'indexSha256' => $built['indexSha256'],
178
'fileCount' => count($indexFiles),
179
'files' => (object) $files,
180
'missing' => array_values($missing),
181
'invalid' => array_values($invalid),
182
];
183
184
if ($known !== null) {
185
$deleted = [];
186
foreach (array_keys($known) as $p) {
187
$p = (string) $p;
188
if ($requested && !in_array($p, $scope, true))
189
continue; // selection mode: only report selected paths
190
if (!isset($indexFiles[$p]) && !isset($files[$p]))
191
$deleted[] = $p;
192
}
193
sort($deleted, SORT_STRING);
194
$out['deleted'] = $deleted;
195
$out['unchanged'] = $unchanged;
196
}
197
198
if ($method !== 'HEAD')
199
echo json_encode($out, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR);
200
201
// ---- Helpers ----
202
// (kept compatible with PHP 8.0: no array_is_list / never)
203
204
function bulkIsList(array $a): bool
205
{
206
$i = 0;
207
foreach ($a as $k => $_) {
208
if ($k !== $i++)
209
return false;
210
}
211
return true;
212
}
213
214
function bulkFail(int $code, string $message): void
215
{
216
http_response_code($code);
217
echo json_encode(['error' => $message]);
218
exit;
219
}
220
221
/**
222
* Accept only plain relative paths like "folder/file.json":
223
* no absolute paths, no "..", no hidden segments, no backslashes, .json only.
224
*/
225
function isValidDataPath(string $p): bool
226
{
227
if (strlen($p) > 255)
228
return false;
229
if (!preg_match('~^[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*\.json$~', $p))
230
return false;
231
foreach (explode('/', $p) as $seg) {
232
if ($seg === '..' || $seg === '.' || str_contains($seg, '..'))
233
return false;
234
}
235
return true;
236
}
237
238
/**
239
* Resolve a validated relative path to a real file inside web/data.
240
* Returns null if it doesn't exist or resolves (e.g. via symlink) outside web/data.
241
*/
242
function resolveDataFile(string $dataRoot, string $p): ?string
243
{
244
$real = realpath($dataRoot . DIRECTORY_SEPARATOR . $p);
245
if ($real === false || !is_file($real))
246
return null;
247
if (!str_starts_with($real, $dataRoot . DIRECTORY_SEPARATOR))
248
return null;
249
if (strtolower(pathinfo($real, PATHINFO_EXTENSION)) !== 'json')
250
return null;
251
return $real;
252
}
@@ -0,0 +1,252 @@1
<?php2
/**3
* Bulk data endpoint: fetch many data files in one request.4
*5
* GET dataBulk.php -> all files covered by dataIndex.php6
* GET dataBulk.php?files=a/b.json,c/d.json -> only those files7
* POST dataBulk.php {"files": [...]} -> only those files (may be combined with ?files=)8
* POST dataBulk.php {"known": {"a/b.json": "<sha256>", ...}}9
* -> only files that are new or whose sha256 differs from the given one,10
* plus "deleted": known paths that are no longer in the index.11
* "known" can be combined with "files" (then only the selected files are compared).12
*13
* Paths are relative to web/data, e.g. "powers/feint.json".14
* Response (application/json, gzip if the client accepts it):15
* {16
* "indexSha256": "...", same value as dataIndex.php / X-Index-Sha25617
* "fileCount": 451, number of files in the index18
* "files": { "<path>": { "sha256": "...", "content": "<exact file text>" } },19
* "missing": [...], valid-looking paths that don't exist20
* "invalid": [...], rejected paths (traversal, absolute, not .json, ...)21
* "deleted": [...], only with "known"22
* "unchanged": 12 only with "known": number of files skipped23
* }24
* "content" is the raw file text (a JSON string), so sha256(content as UTF-8 bytes)25
* equals "sha256" and equals the per-node sha256 in dataIndex.php.26
*/27
28
ini_set('display_errors', '1');29
error_reporting(E_ALL);30
31
if (extension_loaded('zlib') && !ini_get('zlib.output_compression'))32
ob_start('ob_gzhandler');33
34
header('Content-Type: application/json; charset=utf-8');35
header('Access-Control-Allow-Origin: *');36
header('Access-Control-Allow-Methods: GET, HEAD, OPTIONS, POST');37
header('Access-Control-Allow-Headers: *');38
header('X-Content-Type-Options: nosniff');39
header('X-Robots-Tag: noindex');40
header('Cache-Control: no-store, no-cache, must-revalidate, max-age=0');41
header('Pragma: no-cache');42
43
set_error_handler(function ($severity, $message, $file, $line) {44
http_response_code(500);45
echo json_encode(['error' => 'PHP Error', 'message' => $message, 'file' => basename($file), 'line' => $line]);46
exit;47
});48
49
set_exception_handler(function ($e) {50
http_response_code(500);51
echo json_encode(['error' => 'Uncaught Exception', 'type' => get_class($e), 'message' => $e->getMessage()]);52
exit;53
});54
55
require_once __DIR__ . '/dataIndexLib.php';56
57
$method = $_SERVER['REQUEST_METHOD'] ?? 'GET';58
if ($method === 'OPTIONS') {59
http_response_code(204);60
exit;61
}62
if (!in_array($method, ['GET', 'HEAD', 'POST'], true))63
bulkFail(405, 'Method not allowed');64
65
const BULK_MAX_BODY = 4 * 1024 * 1024;66
const BULK_MAX_PATHS = 10000;67
68
// ---- Parse request ----69
$requested = [];70
$known = null;71
72
if (isset($_GET['files']) && is_string($_GET['files']))73
$requested = array_merge($requested, explode(',', $_GET['files']));74
75
if ($method === 'POST') {76
$raw = file_get_contents('php://input', false, null, 0, BULK_MAX_BODY + 1);77
if ($raw !== false && strlen($raw) > BULK_MAX_BODY)78
bulkFail(413, 'Request body too large');79
80
if ($raw !== false && trim($raw) !== '') {81
try {82
$body = json_decode($raw, true, 16, JSON_THROW_ON_ERROR);83
} catch (Throwable $e) {84
bulkFail(400, 'Body is not valid JSON');85
}86
if (!is_array($body) || ($body !== [] && bulkIsList($body)))87
bulkFail(400, 'Body must be a JSON object');88
89
if (isset($body['files'])) {90
if (!is_array($body['files']) || !bulkIsList($body['files']))91
bulkFail(400, '"files" must be an array of paths');92
$requested = array_merge($requested, $body['files']);93
}94
if (isset($body['known'])) {95
if (!is_array($body['known']) || ($body['known'] !== [] && bulkIsList($body['known'])))96
bulkFail(400, '"known" must be an object of path => sha256');97
$known = $body['known'];98
}99
}100
}101
102
if (count($requested) > BULK_MAX_PATHS || ($known !== null && count($known) > BULK_MAX_PATHS))103
bulkFail(413, 'Too many paths');104
105
// ---- Build index (same logic and hashes as dataIndex.php) ----106
$built = buildDataIndex();107
$indexFiles = []; // "powers/feint.json" => sha256108
foreach ($built['files'] as $p => $h) {109
if (str_starts_with($p, 'data/'))110
$indexFiles[substr($p, 5)] = $h;111
}112
header('X-Index-Sha256: ' . $built['indexSha256']);113
114
$dataRoot = realpath(__DIR__ . DIRECTORY_SEPARATOR . 'data');115
if ($dataRoot === false)116
bulkFail(500, 'Data directory not found');117
118
// ---- Determine scope ----119
$missing = [];120
$invalid = [];121
122
if ($requested) {123
$scope = [];124
foreach ($requested as $p) {125
if (!is_string($p)) {126
$invalid[] = $p;127
continue;128
}129
$p = trim($p);130
if ($p === '')131
continue;132
if (!isValidDataPath($p)) {133
$invalid[] = $p;134
continue;135
}136
$scope[$p] = true;137
}138
$scope = array_keys($scope);139
} else {140
$scope = array_keys($indexFiles);141
}142
sort($scope, SORT_STRING);143
144
// ---- Collect files ----145
$files = [];146
$unchanged = 0;147
148
foreach ($scope as $p) {149
$real = resolveDataFile($dataRoot, $p);150
if ($real === null) {151
$missing[] = $p;152
continue;153
}154
155
$raw = file_get_contents($real);156
if ($raw === false) {157
$missing[] = $p;158
continue;159
}160
$hash = hash('sha256', $raw);161
162
if ($known !== null && isset($known[$p]) && is_string($known[$p]) && strtolower($known[$p]) === $hash) {163
$unchanged++;164
continue;165
}166
167
$entry = ['sha256' => $hash];168
if (mb_check_encoding($raw, 'UTF-8'))169
$entry['content'] = $raw;170
else171
$entry['contentBase64'] = base64_encode($raw); // should never happen (CI validates UTF-8 JSON)172
173
$files[$p] = $entry;174
}175
176
$out = [177
'indexSha256' => $built['indexSha256'],178
'fileCount' => count($indexFiles),179
'files' => (object) $files,180
'missing' => array_values($missing),181
'invalid' => array_values($invalid),182
];183
184
if ($known !== null) {185
$deleted = [];186
foreach (array_keys($known) as $p) {187
$p = (string) $p;188
if ($requested && !in_array($p, $scope, true))189
continue; // selection mode: only report selected paths190
if (!isset($indexFiles[$p]) && !isset($files[$p]))191
$deleted[] = $p;192
}193
sort($deleted, SORT_STRING);194
$out['deleted'] = $deleted;195
$out['unchanged'] = $unchanged;196
}197
198
if ($method !== 'HEAD')199
echo json_encode($out, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR);200
201
// ---- Helpers ----202
// (kept compatible with PHP 8.0: no array_is_list / never)203
204
function bulkIsList(array $a): bool205
{206
$i = 0;207
foreach ($a as $k => $_) {208
if ($k !== $i++)209
return false;210
}211
return true;212
}213
214
function bulkFail(int $code, string $message): void215
{216
http_response_code($code);217
echo json_encode(['error' => $message]);218
exit;219
}220
221
/**222
* Accept only plain relative paths like "folder/file.json":223
* no absolute paths, no "..", no hidden segments, no backslashes, .json only.224
*/225
function isValidDataPath(string $p): bool226
{227
if (strlen($p) > 255)228
return false;229
if (!preg_match('~^[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*\.json$~', $p))230
return false;231
foreach (explode('/', $p) as $seg) {232
if ($seg === '..' || $seg === '.' || str_contains($seg, '..'))233
return false;234
}235
return true;236
}237
238
/**239
* Resolve a validated relative path to a real file inside web/data.240
* Returns null if it doesn't exist or resolves (e.g. via symlink) outside web/data.241
*/242
function resolveDataFile(string $dataRoot, string $p): ?string243
{244
$real = realpath($dataRoot . DIRECTORY_SEPARATOR . $p);245
if ($real === false || !is_file($real))246
return null;247
if (!str_starts_with($real, $dataRoot . DIRECTORY_SEPARATOR))248
return null;249
if (strtolower(pathinfo($real, PATHINFO_EXTENSION)) !== 'json')250
return null;251
return $real;252
}@@ -39,42 +39,16 @@ set_exception_handler(function ($e) {
39
39
});
40
40
41
41
42
require_once __DIR__ . '/folderNode.php';
42
require_once __DIR__ . '/dataIndexLib.php';
43
43
44
44
if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'OPTIONS') {
45
45
http_response_code(204);
46
46
exit;
47
47
}
48
48
49
$dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';
50
$chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';
51
52
$allJson = loadAllJson($dataRoot);
53
$visited = [];
54
$baseUrl = getBaseUrl();
55
56
$chaptersTree = loadFolderTree($chaptersRoot);
57
$chapters = $chaptersTree->jsonRecursive();
58
59
usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));
60
61
foreach ($chapters as $chapterObj) {
62
$chapter = json_decode(json_encode($chapterObj), true);
63
$chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
64
$relPath = normalizeDataRelPath($chapterLink);
65
if ($relPath !== '')
66
$visited[$relPath] = true;
67
}
68
69
$index = [];
70
foreach ($chapters as $chapterObj) {
71
$chapter = json_decode(json_encode($chapterObj), true);
72
$chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
73
74
$index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
75
}
76
77
$indexSha256 = computeIndexSha256($index, $baseUrl);
49
$built = buildDataIndex();
50
$index = $built['nodes'];
51
$indexSha256 = $built['indexSha256'];
78
52
header('X-Index-Sha256: ' . $indexSha256);
79
53
80
54
if (isset($_GET['meta'])) {
@@ -86,303 +60,3 @@ if (isset($_GET['meta'])) {
86
60
} else {
87
61
echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
88
62
}
89
90
/**
91
* sha256 (lowercase hex) of the exact bytes of a data file, given its path
92
* relative to the web root (e.g. "/data/powers/feint.json").
93
* Returns null if the file does not exist or cannot be read.
94
* Results are memoised per request and keyed by path + mtime + size.
95
*/
96
function fileSha256(string $webRelPath): ?string
97
{
98
static $cache = [];
99
100
$webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
101
$path = __DIR__ . $webRelPath;
102
if (!is_file($path))
103
return null;
104
105
$key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
106
if (!array_key_exists($key, $cache)) {
107
$hash = hash_file('sha256', $path);
108
$cache[$key] = $hash === false ? null : $hash;
109
}
110
111
return $cache[$key];
112
}
113
114
/**
115
* indexSha256 definition:
116
* Collect every node in the index (recursively) that has a non-null sha256.
117
* Take the unique data files they point to, as paths relative to the web
118
* root without a leading slash (e.g. "data/powers/feint.json").
119
* Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
120
* per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
121
* indexSha256 = lowercase hex sha256 of the concatenation of those lines.
122
* Reproducible from the web/ directory with:
123
* sha256sum <those files, LC_ALL=C sorted> | sha256sum
124
*/
125
function computeIndexSha256(array $nodes, string $baseUrl): string
126
{
127
$files = [];
128
collectNodeHashes($nodes, $baseUrl, $files);
129
ksort($files, SORT_STRING);
130
131
$lines = '';
132
foreach ($files as $path => $hash)
133
$lines .= $hash . ' ' . $path . "\n";
134
135
return hash('sha256', $lines);
136
}
137
138
function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
139
{
140
$prefix = $baseUrl . '/';
141
foreach ($nodes as $node) {
142
if (!is_array($node))
143
continue;
144
145
$link = (string) ($node['link'] ?? '');
146
if (isset($node['sha256']) && str_starts_with($link, $prefix))
147
$files[substr($link, strlen($prefix))] = $node['sha256'];
148
149
collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
150
}
151
}
152
153
function loadAllJson(string $dataRoot): array
154
{
155
$map = [];
156
157
$it = new RecursiveIteratorIterator(
158
new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)
159
);
160
161
foreach ($it as $file) {
162
/** @var SplFileInfo $file */
163
if (strtolower($file->getExtension()) !== 'json')
164
continue;
165
166
$path = $file->getPathname();
167
$rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));
168
if ($rel === '')
169
$rel = '/' . $file->getFilename();
170
if ($rel[0] !== '/')
171
$rel = '/' . $rel;
172
173
try {
174
$raw = file_get_contents($path);
175
if ($raw === false)
176
continue;
177
$json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
178
if (!is_array($json))
179
continue;
180
} catch (Throwable $e) {
181
continue;
182
}
183
184
$json['_relPath'] = $rel;
185
$map[$rel] = $json;
186
}
187
188
return $map;
189
}
190
191
function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array
192
{
193
$name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');
194
$fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';
195
196
$node = [
197
'name' => $name,
198
'type' => $chapter['type'] ?? 'chapter',
199
'pageType' => 'page',
200
'link' => $fullChapterLink,
201
'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
202
'children' => [],
203
];
204
205
$folderIncludes = extractFolderIncludes($chapter, ['subheaders']);
206
207
$children = [];
208
209
foreach (($chapter['subheaders'] ?? []) as $subheader) {
210
if (!is_array($subheader))
211
continue;
212
213
$subNode = [
214
'name' => $subheader['name'] ?? 'subheader',
215
'type' => 'subheader',
216
'pageType' => 'page child element',
217
'link' => $fullChapterLink,
218
'sha256' => $node['sha256'],
219
'children' => [],
220
];
221
222
$subIncludes = extractFolderIncludes($subheader);
223
$subNode['children'] = mergeChildren(
224
buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)
225
);
226
227
$children[] = $subNode;
228
}
229
230
$children = mergeChildren(
231
$children,
232
buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)
233
);
234
235
$node['children'] = $children;
236
237
return $node;
238
}
239
240
function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array
241
{
242
$children = [];
243
$rels = array_keys($allJson);
244
sort($rels, SORT_NATURAL | SORT_FLAG_CASE);
245
foreach ($folders as $folder) {
246
$folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');
247
foreach ($rels as $rel) {
248
if (str_starts_with($rel, $folderRel . '/')) {
249
$children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);
250
}
251
}
252
}
253
254
return $children;
255
}
256
257
function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array
258
{
259
$link = $baseUrl . '/data' . $relPath;
260
$json = $allJson[$relPath] ?? null;
261
262
$name = $label;
263
$type = 'linkedData';
264
265
if (is_array($json)) {
266
$type = $json['type'] ?? 'data';
267
if ($name === '') {
268
$name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));
269
}
270
}
271
272
$node = [
273
'name' => $name !== '' ? $name : basename($relPath, '.json'),
274
'type' => $type,
275
'pageType' => 'data link',
276
'link' => $link,
277
'sha256' => fileSha256('/data' . $relPath),
278
'children' => [],
279
];
280
281
if (!is_array($json))
282
return $node;
283
284
if (isset($visited[$relPath]))
285
return $node;
286
287
$visited[$relPath] = true;
288
289
$folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);
290
$node['children'] = mergeChildren($folderChildren);
291
292
return $node;
293
}
294
295
function normalizeWikiPath(string $path): string
296
{
297
$path = str_replace('\\', '/', $path);
298
if ($path === '')
299
return '';
300
301
if (!str_starts_with($path, '/'))
302
$path = '/' . $path;
303
304
return $path;
305
}
306
307
function getBaseUrl(): string
308
{
309
$scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
310
$host = $_SERVER['HTTP_HOST'] ?? 'localhost';
311
$scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';
312
$dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');
313
return $scheme . '://' . $host . ($dir === '' ? '' : $dir);
314
}
315
316
function normalizeDataRelPath(string $path): string
317
{
318
$path = trim($path);
319
if ($path === '')
320
return '';
321
322
$path = str_replace('\\', '/', $path);
323
if (str_starts_with($path, '/data/'))
324
$path = substr($path, strlen('/data'));
325
if (str_starts_with($path, 'data/'))
326
$path = substr($path, strlen('data'));
327
328
if (!str_starts_with($path, '/'))
329
$path = '/' . $path;
330
331
return $path;
332
}
333
334
function extractFolderIncludes(mixed $value, array $skipKeys = []): array
335
{
336
$folders = [];
337
338
if (is_array($value)) {
339
foreach ($value as $k => $v) {
340
if (in_array($k, $skipKeys, true))
341
continue;
342
$folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));
343
}
344
return $folders;
345
}
346
347
if (is_string($value)) {
348
return extractFolderIncludesFromString($value);
349
}
350
351
return $folders;
352
}
353
354
function extractFolderIncludesFromString(string $text): array
355
{
356
$folders = [];
357
$pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';
358
359
if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {
360
foreach ($matches as $m) {
361
$folders[] = $m[1];
362
}
363
}
364
365
return $folders;
366
}
367
368
function mergeChildren(array ...$groups): array
369
{
370
$merged = [];
371
$seen = [];
372
373
foreach ($groups as $children) {
374
foreach ($children as $child) {
375
if (!is_array($child))
376
continue;
377
378
$key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');
379
if (isset($seen[$key]))
380
continue;
381
382
$seen[$key] = true;
383
$merged[] = $child;
384
}
385
}
386
387
return $merged;
388
}
@@ -39,42 +39,16 @@ set_exception_handler(function ($e) {39
});39
});40
40
41
41
42
require_once __DIR__ . '/folderNode.php';42
require_once __DIR__ . '/dataIndexLib.php';43
43
44
if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'OPTIONS') {44
if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'OPTIONS') {45
http_response_code(204);45
http_response_code(204);46
exit;46
exit;47
}47
}48
48
49
$dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';49
$built = buildDataIndex();50
$chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';50
$index = $built['nodes'];51
51
$indexSha256 = $built['indexSha256'];52
$allJson = loadAllJson($dataRoot);53
$visited = [];54
$baseUrl = getBaseUrl();55
56
$chaptersTree = loadFolderTree($chaptersRoot);57
$chapters = $chaptersTree->jsonRecursive();58
59
usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));60
61
foreach ($chapters as $chapterObj) {62
$chapter = json_decode(json_encode($chapterObj), true);63
$chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');64
$relPath = normalizeDataRelPath($chapterLink);65
if ($relPath !== '')66
$visited[$relPath] = true;67
}68
69
$index = [];70
foreach ($chapters as $chapterObj) {71
$chapter = json_decode(json_encode($chapterObj), true);72
$chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');73
74
$index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);75
}76
77
$indexSha256 = computeIndexSha256($index, $baseUrl);78
header('X-Index-Sha256: ' . $indexSha256);52
header('X-Index-Sha256: ' . $indexSha256);79
53
80
if (isset($_GET['meta'])) {54
if (isset($_GET['meta'])) {@@ -86,303 +60,3 @@ if (isset($_GET['meta'])) {86
} else {60
} else {87
echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);61
echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);88
}62
}89
90
/**91
* sha256 (lowercase hex) of the exact bytes of a data file, given its path92
* relative to the web root (e.g. "/data/powers/feint.json").93
* Returns null if the file does not exist or cannot be read.94
* Results are memoised per request and keyed by path + mtime + size.95
*/96
function fileSha256(string $webRelPath): ?string97
{98
static $cache = [];99
100
$webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');101
$path = __DIR__ . $webRelPath;102
if (!is_file($path))103
return null;104
105
$key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);106
if (!array_key_exists($key, $cache)) {107
$hash = hash_file('sha256', $path);108
$cache[$key] = $hash === false ? null : $hash;109
}110
111
return $cache[$key];112
}113
114
/**115
* indexSha256 definition:116
* Collect every node in the index (recursively) that has a non-null sha256.117
* Take the unique data files they point to, as paths relative to the web118
* root without a leading slash (e.g. "data/powers/feint.json").119
* Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line120
* per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).121
* indexSha256 = lowercase hex sha256 of the concatenation of those lines.122
* Reproducible from the web/ directory with:123
* sha256sum <those files, LC_ALL=C sorted> | sha256sum124
*/125
function computeIndexSha256(array $nodes, string $baseUrl): string126
{127
$files = [];128
collectNodeHashes($nodes, $baseUrl, $files);129
ksort($files, SORT_STRING);130
131
$lines = '';132
foreach ($files as $path => $hash)133
$lines .= $hash . ' ' . $path . "\n";134
135
return hash('sha256', $lines);136
}137
138
function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void139
{140
$prefix = $baseUrl . '/';141
foreach ($nodes as $node) {142
if (!is_array($node))143
continue;144
145
$link = (string) ($node['link'] ?? '');146
if (isset($node['sha256']) && str_starts_with($link, $prefix))147
$files[substr($link, strlen($prefix))] = $node['sha256'];148
149
collectNodeHashes($node['children'] ?? [], $baseUrl, $files);150
}151
}152
153
function loadAllJson(string $dataRoot): array154
{155
$map = [];156
157
$it = new RecursiveIteratorIterator(158
new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)159
);160
161
foreach ($it as $file) {162
/** @var SplFileInfo $file */163
if (strtolower($file->getExtension()) !== 'json')164
continue;165
166
$path = $file->getPathname();167
$rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));168
if ($rel === '')169
$rel = '/' . $file->getFilename();170
if ($rel[0] !== '/')171
$rel = '/' . $rel;172
173
try {174
$raw = file_get_contents($path);175
if ($raw === false)176
continue;177
$json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);178
if (!is_array($json))179
continue;180
} catch (Throwable $e) {181
continue;182
}183
184
$json['_relPath'] = $rel;185
$map[$rel] = $json;186
}187
188
return $map;189
}190
191
function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array192
{193
$name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');194
$fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';195
196
$node = [197
'name' => $name,198
'type' => $chapter['type'] ?? 'chapter',199
'pageType' => 'page',200
'link' => $fullChapterLink,201
'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,202
'children' => [],203
];204
205
$folderIncludes = extractFolderIncludes($chapter, ['subheaders']);206
207
$children = [];208
209
foreach (($chapter['subheaders'] ?? []) as $subheader) {210
if (!is_array($subheader))211
continue;212
213
$subNode = [214
'name' => $subheader['name'] ?? 'subheader',215
'type' => 'subheader',216
'pageType' => 'page child element',217
'link' => $fullChapterLink,218
'sha256' => $node['sha256'],219
'children' => [],220
];221
222
$subIncludes = extractFolderIncludes($subheader);223
$subNode['children'] = mergeChildren(224
buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)225
);226
227
$children[] = $subNode;228
}229
230
$children = mergeChildren(231
$children,232
buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)233
);234
235
$node['children'] = $children;236
237
return $node;238
}239
240
function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array241
{242
$children = [];243
$rels = array_keys($allJson);244
sort($rels, SORT_NATURAL | SORT_FLAG_CASE);245
foreach ($folders as $folder) {246
$folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');247
foreach ($rels as $rel) {248
if (str_starts_with($rel, $folderRel . '/')) {249
$children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);250
}251
}252
}253
254
return $children;255
}256
257
function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array258
{259
$link = $baseUrl . '/data' . $relPath;260
$json = $allJson[$relPath] ?? null;261
262
$name = $label;263
$type = 'linkedData';264
265
if (is_array($json)) {266
$type = $json['type'] ?? 'data';267
if ($name === '') {268
$name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));269
}270
}271
272
$node = [273
'name' => $name !== '' ? $name : basename($relPath, '.json'),274
'type' => $type,275
'pageType' => 'data link',276
'link' => $link,277
'sha256' => fileSha256('/data' . $relPath),278
'children' => [],279
];280
281
if (!is_array($json))282
return $node;283
284
if (isset($visited[$relPath]))285
return $node;286
287
$visited[$relPath] = true;288
289
$folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);290
$node['children'] = mergeChildren($folderChildren);291
292
return $node;293
}294
295
function normalizeWikiPath(string $path): string296
{297
$path = str_replace('\\', '/', $path);298
if ($path === '')299
return '';300
301
if (!str_starts_with($path, '/'))302
$path = '/' . $path;303
304
return $path;305
}306
307
function getBaseUrl(): string308
{309
$scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';310
$host = $_SERVER['HTTP_HOST'] ?? 'localhost';311
$scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';312
$dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');313
return $scheme . '://' . $host . ($dir === '' ? '' : $dir);314
}315
316
function normalizeDataRelPath(string $path): string317
{318
$path = trim($path);319
if ($path === '')320
return '';321
322
$path = str_replace('\\', '/', $path);323
if (str_starts_with($path, '/data/'))324
$path = substr($path, strlen('/data'));325
if (str_starts_with($path, 'data/'))326
$path = substr($path, strlen('data'));327
328
if (!str_starts_with($path, '/'))329
$path = '/' . $path;330
331
return $path;332
}333
334
function extractFolderIncludes(mixed $value, array $skipKeys = []): array335
{336
$folders = [];337
338
if (is_array($value)) {339
foreach ($value as $k => $v) {340
if (in_array($k, $skipKeys, true))341
continue;342
$folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));343
}344
return $folders;345
}346
347
if (is_string($value)) {348
return extractFolderIncludesFromString($value);349
}350
351
return $folders;352
}353
354
function extractFolderIncludesFromString(string $text): array355
{356
$folders = [];357
$pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';358
359
if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {360
foreach ($matches as $m) {361
$folders[] = $m[1];362
}363
}364
365
return $folders;366
}367
368
function mergeChildren(array ...$groups): array369
{370
$merged = [];371
$seen = [];372
373
foreach ($groups as $children) {374
foreach ($children as $child) {375
if (!is_array($child))376
continue;377
378
$key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');379
if (isset($seen[$key]))380
continue;381
382
$seen[$key] = true;383
$merged[] = $child;384
}385
}386
387
return $merged;388
}@@ -0,0 +1,368 @@
1
<?php
2
/**
3
* Shared index logic for dataIndex.php and dataBulk.php.
4
* Defines functions only; produces no output.
5
*/
6
7
require_once __DIR__ . '/folderNode.php';
8
9
/**
10
* Build the data index.
11
* @return array{nodes: array, indexSha256: string, files: array<string,string>}
12
* files: "data/<folder>/<file>.json" => sha256, for every file in the index
13
* (the same set and hashes that indexSha256 is computed from), sorted by path.
14
*/
15
function buildDataIndex(): array
16
{
17
$dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';
18
$chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';
19
20
$allJson = loadAllJson($dataRoot);
21
$visited = [];
22
$baseUrl = getBaseUrl();
23
24
$chaptersTree = loadFolderTree($chaptersRoot);
25
$chapters = $chaptersTree->jsonRecursive();
26
27
usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));
28
29
foreach ($chapters as $chapterObj) {
30
$chapter = json_decode(json_encode($chapterObj), true);
31
$chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
32
$relPath = normalizeDataRelPath($chapterLink);
33
if ($relPath !== '')
34
$visited[$relPath] = true;
35
}
36
37
$index = [];
38
foreach ($chapters as $chapterObj) {
39
$chapter = json_decode(json_encode($chapterObj), true);
40
$chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');
41
42
$index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
43
}
44
45
$files = collectIndexFiles($index, $baseUrl);
46
47
return [
48
'nodes' => $index,
49
'indexSha256' => indexSha256FromFiles($files),
50
'files' => $files,
51
];
52
}
53
54
/**
55
* sha256 (lowercase hex) of the exact bytes of a data file, given its path
56
* relative to the web root (e.g. "/data/powers/feint.json").
57
* Returns null if the file does not exist or cannot be read.
58
* Results are memoised per request and keyed by path + mtime + size.
59
*/
60
function fileSha256(string $webRelPath): ?string
61
{
62
static $cache = [];
63
64
$webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
65
$path = __DIR__ . $webRelPath;
66
if (!is_file($path))
67
return null;
68
69
$key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
70
if (!array_key_exists($key, $cache)) {
71
$hash = hash_file('sha256', $path);
72
$cache[$key] = $hash === false ? null : $hash;
73
}
74
75
return $cache[$key];
76
}
77
78
/**
79
* indexSha256 definition:
80
* Collect every node in the index (recursively) that has a non-null sha256.
81
* Take the unique data files they point to, as paths relative to the web
82
* root without a leading slash (e.g. "data/powers/feint.json").
83
* Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
84
* per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
85
* indexSha256 = lowercase hex sha256 of the concatenation of those lines.
86
* Reproducible from the web/ directory with:
87
* sha256sum <those files, LC_ALL=C sorted> | sha256sum
88
*/
89
function computeIndexSha256(array $nodes, string $baseUrl): string
90
{
91
return indexSha256FromFiles(collectIndexFiles($nodes, $baseUrl));
92
}
93
94
/**
95
* Unique data files referenced by the index: "data/<folder>/<file>.json" => sha256,
96
* sorted by path (byte-wise).
97
*/
98
function collectIndexFiles(array $nodes, string $baseUrl): array
99
{
100
$files = [];
101
collectNodeHashes($nodes, $baseUrl, $files);
102
ksort($files, SORT_STRING);
103
return $files;
104
}
105
106
/**
107
* @param array<string,string> $files path => sha256, already sorted by path
108
*/
109
function indexSha256FromFiles(array $files): string
110
{
111
$lines = '';
112
foreach ($files as $path => $hash)
113
$lines .= $hash . ' ' . $path . "\n";
114
115
return hash('sha256', $lines);
116
}
117
118
function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
119
{
120
$prefix = $baseUrl . '/';
121
foreach ($nodes as $node) {
122
if (!is_array($node))
123
continue;
124
125
$link = (string) ($node['link'] ?? '');
126
if (isset($node['sha256']) && str_starts_with($link, $prefix))
127
$files[substr($link, strlen($prefix))] = $node['sha256'];
128
129
collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
130
}
131
}
132
133
function loadAllJson(string $dataRoot): array
134
{
135
$map = [];
136
137
$it = new RecursiveIteratorIterator(
138
new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)
139
);
140
141
foreach ($it as $file) {
142
/** @var SplFileInfo $file */
143
if (strtolower($file->getExtension()) !== 'json')
144
continue;
145
146
$path = $file->getPathname();
147
$rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));
148
if ($rel === '')
149
$rel = '/' . $file->getFilename();
150
if ($rel[0] !== '/')
151
$rel = '/' . $rel;
152
153
try {
154
$raw = file_get_contents($path);
155
if ($raw === false)
156
continue;
157
$json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);
158
if (!is_array($json))
159
continue;
160
} catch (Throwable $e) {
161
continue;
162
}
163
164
$json['_relPath'] = $rel;
165
$map[$rel] = $json;
166
}
167
168
return $map;
169
}
170
171
function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array
172
{
173
$name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');
174
$fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';
175
176
$node = [
177
'name' => $name,
178
'type' => $chapter['type'] ?? 'chapter',
179
'pageType' => 'page',
180
'link' => $fullChapterLink,
181
'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
182
'children' => [],
183
];
184
185
$folderIncludes = extractFolderIncludes($chapter, ['subheaders']);
186
187
$children = [];
188
189
foreach (($chapter['subheaders'] ?? []) as $subheader) {
190
if (!is_array($subheader))
191
continue;
192
193
$subNode = [
194
'name' => $subheader['name'] ?? 'subheader',
195
'type' => 'subheader',
196
'pageType' => 'page child element',
197
'link' => $fullChapterLink,
198
'sha256' => $node['sha256'],
199
'children' => [],
200
];
201
202
$subIncludes = extractFolderIncludes($subheader);
203
$subNode['children'] = mergeChildren(
204
buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)
205
);
206
207
$children[] = $subNode;
208
}
209
210
$children = mergeChildren(
211
$children,
212
buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)
213
);
214
215
$node['children'] = $children;
216
217
return $node;
218
}
219
220
function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array
221
{
222
$children = [];
223
$rels = array_keys($allJson);
224
sort($rels, SORT_NATURAL | SORT_FLAG_CASE);
225
foreach ($folders as $folder) {
226
$folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');
227
foreach ($rels as $rel) {
228
if (str_starts_with($rel, $folderRel . '/')) {
229
$children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);
230
}
231
}
232
}
233
234
return $children;
235
}
236
237
function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array
238
{
239
$link = $baseUrl . '/data' . $relPath;
240
$json = $allJson[$relPath] ?? null;
241
242
$name = $label;
243
$type = 'linkedData';
244
245
if (is_array($json)) {
246
$type = $json['type'] ?? 'data';
247
if ($name === '') {
248
$name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));
249
}
250
}
251
252
$node = [
253
'name' => $name !== '' ? $name : basename($relPath, '.json'),
254
'type' => $type,
255
'pageType' => 'data link',
256
'link' => $link,
257
'sha256' => fileSha256('/data' . $relPath),
258
'children' => [],
259
];
260
261
if (!is_array($json))
262
return $node;
263
264
if (isset($visited[$relPath]))
265
return $node;
266
267
$visited[$relPath] = true;
268
269
$folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);
270
$node['children'] = mergeChildren($folderChildren);
271
272
return $node;
273
}
274
275
function normalizeWikiPath(string $path): string
276
{
277
$path = str_replace('\\', '/', $path);
278
if ($path === '')
279
return '';
280
281
if (!str_starts_with($path, '/'))
282
$path = '/' . $path;
283
284
return $path;
285
}
286
287
function getBaseUrl(): string
288
{
289
$scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
290
$host = $_SERVER['HTTP_HOST'] ?? 'localhost';
291
$scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';
292
$dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');
293
return $scheme . '://' . $host . ($dir === '' ? '' : $dir);
294
}
295
296
function normalizeDataRelPath(string $path): string
297
{
298
$path = trim($path);
299
if ($path === '')
300
return '';
301
302
$path = str_replace('\\', '/', $path);
303
if (str_starts_with($path, '/data/'))
304
$path = substr($path, strlen('/data'));
305
if (str_starts_with($path, 'data/'))
306
$path = substr($path, strlen('data'));
307
308
if (!str_starts_with($path, '/'))
309
$path = '/' . $path;
310
311
return $path;
312
}
313
314
function extractFolderIncludes(mixed $value, array $skipKeys = []): array
315
{
316
$folders = [];
317
318
if (is_array($value)) {
319
foreach ($value as $k => $v) {
320
if (in_array($k, $skipKeys, true))
321
continue;
322
$folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));
323
}
324
return $folders;
325
}
326
327
if (is_string($value)) {
328
return extractFolderIncludesFromString($value);
329
}
330
331
return $folders;
332
}
333
334
function extractFolderIncludesFromString(string $text): array
335
{
336
$folders = [];
337
$pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';
338
339
if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {
340
foreach ($matches as $m) {
341
$folders[] = $m[1];
342
}
343
}
344
345
return $folders;
346
}
347
348
function mergeChildren(array ...$groups): array
349
{
350
$merged = [];
351
$seen = [];
352
353
foreach ($groups as $children) {
354
foreach ($children as $child) {
355
if (!is_array($child))
356
continue;
357
358
$key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');
359
if (isset($seen[$key]))
360
continue;
361
362
$seen[$key] = true;
363
$merged[] = $child;
364
}
365
}
366
367
return $merged;
368
}
@@ -0,0 +1,368 @@1
<?php2
/**3
* Shared index logic for dataIndex.php and dataBulk.php.4
* Defines functions only; produces no output.5
*/6
7
require_once __DIR__ . '/folderNode.php';8
9
/**10
* Build the data index.11
* @return array{nodes: array, indexSha256: string, files: array<string,string>}12
* files: "data/<folder>/<file>.json" => sha256, for every file in the index13
* (the same set and hashes that indexSha256 is computed from), sorted by path.14
*/15
function buildDataIndex(): array16
{17
$dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data';18
$chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters';19
20
$allJson = loadAllJson($dataRoot);21
$visited = [];22
$baseUrl = getBaseUrl();23
24
$chaptersTree = loadFolderTree($chaptersRoot);25
$chapters = $chaptersTree->jsonRecursive();26
27
usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0));28
29
foreach ($chapters as $chapterObj) {30
$chapter = json_decode(json_encode($chapterObj), true);31
$chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');32
$relPath = normalizeDataRelPath($chapterLink);33
if ($relPath !== '')34
$visited[$relPath] = true;35
}36
37
$index = [];38
foreach ($chapters as $chapterObj) {39
$chapter = json_decode(json_encode($chapterObj), true);40
$chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? '');41
42
$index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);43
}44
45
$files = collectIndexFiles($index, $baseUrl);46
47
return [48
'nodes' => $index,49
'indexSha256' => indexSha256FromFiles($files),50
'files' => $files,51
];52
}53
54
/**55
* sha256 (lowercase hex) of the exact bytes of a data file, given its path56
* relative to the web root (e.g. "/data/powers/feint.json").57
* Returns null if the file does not exist or cannot be read.58
* Results are memoised per request and keyed by path + mtime + size.59
*/60
function fileSha256(string $webRelPath): ?string61
{62
static $cache = [];63
64
$webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');65
$path = __DIR__ . $webRelPath;66
if (!is_file($path))67
return null;68
69
$key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);70
if (!array_key_exists($key, $cache)) {71
$hash = hash_file('sha256', $path);72
$cache[$key] = $hash === false ? null : $hash;73
}74
75
return $cache[$key];76
}77
78
/**79
* indexSha256 definition:80
* Collect every node in the index (recursively) that has a non-null sha256.81
* Take the unique data files they point to, as paths relative to the web82
* root without a leading slash (e.g. "data/powers/feint.json").83
* Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line84
* per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).85
* indexSha256 = lowercase hex sha256 of the concatenation of those lines.86
* Reproducible from the web/ directory with:87
* sha256sum <those files, LC_ALL=C sorted> | sha256sum88
*/89
function computeIndexSha256(array $nodes, string $baseUrl): string90
{91
return indexSha256FromFiles(collectIndexFiles($nodes, $baseUrl));92
}93
94
/**95
* Unique data files referenced by the index: "data/<folder>/<file>.json" => sha256,96
* sorted by path (byte-wise).97
*/98
function collectIndexFiles(array $nodes, string $baseUrl): array99
{100
$files = [];101
collectNodeHashes($nodes, $baseUrl, $files);102
ksort($files, SORT_STRING);103
return $files;104
}105
106
/**107
* @param array<string,string> $files path => sha256, already sorted by path108
*/109
function indexSha256FromFiles(array $files): string110
{111
$lines = '';112
foreach ($files as $path => $hash)113
$lines .= $hash . ' ' . $path . "\n";114
115
return hash('sha256', $lines);116
}117
118
function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void119
{120
$prefix = $baseUrl . '/';121
foreach ($nodes as $node) {122
if (!is_array($node))123
continue;124
125
$link = (string) ($node['link'] ?? '');126
if (isset($node['sha256']) && str_starts_with($link, $prefix))127
$files[substr($link, strlen($prefix))] = $node['sha256'];128
129
collectNodeHashes($node['children'] ?? [], $baseUrl, $files);130
}131
}132
133
function loadAllJson(string $dataRoot): array134
{135
$map = [];136
137
$it = new RecursiveIteratorIterator(138
new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS)139
);140
141
foreach ($it as $file) {142
/** @var SplFileInfo $file */143
if (strtolower($file->getExtension()) !== 'json')144
continue;145
146
$path = $file->getPathname();147
$rel = str_replace('\\', '/', substr($path, strlen($dataRoot)));148
if ($rel === '')149
$rel = '/' . $file->getFilename();150
if ($rel[0] !== '/')151
$rel = '/' . $rel;152
153
try {154
$raw = file_get_contents($path);155
if ($raw === false)156
continue;157
$json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR);158
if (!is_array($json))159
continue;160
} catch (Throwable $e) {161
continue;162
}163
164
$json['_relPath'] = $rel;165
$map[$rel] = $json;166
}167
168
return $map;169
}170
171
function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array172
{173
$name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter');174
$fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : '';175
176
$node = [177
'name' => $name,178
'type' => $chapter['type'] ?? 'chapter',179
'pageType' => 'page',180
'link' => $fullChapterLink,181
'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,182
'children' => [],183
];184
185
$folderIncludes = extractFolderIncludes($chapter, ['subheaders']);186
187
$children = [];188
189
foreach (($chapter['subheaders'] ?? []) as $subheader) {190
if (!is_array($subheader))191
continue;192
193
$subNode = [194
'name' => $subheader['name'] ?? 'subheader',195
'type' => 'subheader',196
'pageType' => 'page child element',197
'link' => $fullChapterLink,198
'sha256' => $node['sha256'],199
'children' => [],200
];201
202
$subIncludes = extractFolderIncludes($subheader);203
$subNode['children'] = mergeChildren(204
buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl)205
);206
207
$children[] = $subNode;208
}209
210
$children = mergeChildren(211
$children,212
buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl)213
);214
215
$node['children'] = $children;216
217
return $node;218
}219
220
function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array221
{222
$children = [];223
$rels = array_keys($allJson);224
sort($rels, SORT_NATURAL | SORT_FLAG_CASE);225
foreach ($folders as $folder) {226
$folderRel = '/' . trim(str_replace('\\', '/', $folder), '/');227
foreach ($rels as $rel) {228
if (str_starts_with($rel, $folderRel . '/')) {229
$children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl);230
}231
}232
}233
234
return $children;235
}236
237
function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array238
{239
$link = $baseUrl . '/data' . $relPath;240
$json = $allJson[$relPath] ?? null;241
242
$name = $label;243
$type = 'linkedData';244
245
if (is_array($json)) {246
$type = $json['type'] ?? 'data';247
if ($name === '') {248
$name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json'));249
}250
}251
252
$node = [253
'name' => $name !== '' ? $name : basename($relPath, '.json'),254
'type' => $type,255
'pageType' => 'data link',256
'link' => $link,257
'sha256' => fileSha256('/data' . $relPath),258
'children' => [],259
];260
261
if (!is_array($json))262
return $node;263
264
if (isset($visited[$relPath]))265
return $node;266
267
$visited[$relPath] = true;268
269
$folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl);270
$node['children'] = mergeChildren($folderChildren);271
272
return $node;273
}274
275
function normalizeWikiPath(string $path): string276
{277
$path = str_replace('\\', '/', $path);278
if ($path === '')279
return '';280
281
if (!str_starts_with($path, '/'))282
$path = '/' . $path;283
284
return $path;285
}286
287
function getBaseUrl(): string288
{289
$scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';290
$host = $_SERVER['HTTP_HOST'] ?? 'localhost';291
$scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php';292
$dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/');293
return $scheme . '://' . $host . ($dir === '' ? '' : $dir);294
}295
296
function normalizeDataRelPath(string $path): string297
{298
$path = trim($path);299
if ($path === '')300
return '';301
302
$path = str_replace('\\', '/', $path);303
if (str_starts_with($path, '/data/'))304
$path = substr($path, strlen('/data'));305
if (str_starts_with($path, 'data/'))306
$path = substr($path, strlen('data'));307
308
if (!str_starts_with($path, '/'))309
$path = '/' . $path;310
311
return $path;312
}313
314
function extractFolderIncludes(mixed $value, array $skipKeys = []): array315
{316
$folders = [];317
318
if (is_array($value)) {319
foreach ($value as $k => $v) {320
if (in_array($k, $skipKeys, true))321
continue;322
$folders = array_merge($folders, extractFolderIncludes($v, $skipKeys));323
}324
return $folders;325
}326
327
if (is_string($value)) {328
return extractFolderIncludesFromString($value);329
}330
331
return $folders;332
}333
334
function extractFolderIncludesFromString(string $text): array335
{336
$folders = [];337
$pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/';338
339
if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) {340
foreach ($matches as $m) {341
$folders[] = $m[1];342
}343
}344
345
return $folders;346
}347
348
function mergeChildren(array ...$groups): array349
{350
$merged = [];351
$seen = [];352
353
foreach ($groups as $children) {354
foreach ($children as $child) {355
if (!is_array($child))356
continue;357
358
$key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? '');359
if (isset($seen[$key]))360
continue;361
362
$seen[$key] = true;363
$merged[] = $child;364
}365
}366
367
return $merged;368
}