| 1 | <?php |
| 2 | /** |
| 3 | * Shared index logic for dataIndex.php and dataBulk.php. |
| 4 | * Defines functions only; produces no output. |
| 5 | */ |
| 6 | |
| 7 | require_once __DIR__ . '/folderNode.php'; |
| 8 | |
| 9 | /** |
| 10 | * Build the data index. |
| 11 | * @return array{nodes: array, indexSha256: string, files: array<string,string>} |
| 12 | * files: "data/<folder>/<file>.json" => sha256, for every file in the index |
| 13 | * (the same set and hashes that indexSha256 is computed from), sorted by path. |
| 14 | */ |
| 15 | function buildDataIndex(): array |
| 16 | { |
| 17 | $dataRoot = __DIR__ . DIRECTORY_SEPARATOR . 'data'; |
| 18 | $chaptersRoot = $dataRoot . DIRECTORY_SEPARATOR . 'chapters'; |
| 19 | |
| 20 | $allJson = loadAllJson($dataRoot); |
| 21 | $visited = []; |
| 22 | $baseUrl = getBaseUrl(); |
| 23 | |
| 24 | $chaptersTree = loadFolderTree($chaptersRoot); |
| 25 | $chapters = $chaptersTree->jsonRecursive(); |
| 26 | |
| 27 | usort($chapters, fn($a, $b) => ($a->number ?? 0) <=> ($b->number ?? 0)); |
| 28 | |
| 29 | foreach ($chapters as $chapterObj) { |
| 30 | $chapter = json_decode(json_encode($chapterObj), true); |
| 31 | $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? ''); |
| 32 | $relPath = normalizeDataRelPath($chapterLink); |
| 33 | if ($relPath !== '') |
| 34 | $visited[$relPath] = true; |
| 35 | } |
| 36 | |
| 37 | $index = []; |
| 38 | foreach ($chapters as $chapterObj) { |
| 39 | $chapter = json_decode(json_encode($chapterObj), true); |
| 40 | $chapterLink = normalizeWikiPath($chapter['wikiPath'] ?? ''); |
| 41 | |
| 42 | $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl); |
| 43 | } |
| 44 | |
| 45 | $files = collectIndexFiles($index, $baseUrl); |
| 46 | |
| 47 | return [ |
| 48 | 'nodes' => $index, |
| 49 | 'indexSha256' => indexSha256FromFiles($files), |
| 50 | 'files' => $files, |
| 51 | ]; |
| 52 | } |
| 53 | |
| 54 | /** |
| 55 | * sha256 (lowercase hex) of the exact bytes of a data file, given its path |
| 56 | * relative to the web root (e.g. "/data/powers/feint.json"). |
| 57 | * Returns null if the file does not exist or cannot be read. |
| 58 | * Results are memoised per request and keyed by path + mtime + size. |
| 59 | */ |
| 60 | function fileSha256(string $webRelPath): ?string |
| 61 | { |
| 62 | static $cache = []; |
| 63 | |
| 64 | $webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/'); |
| 65 | $path = __DIR__ . $webRelPath; |
| 66 | if (!is_file($path)) |
| 67 | return null; |
| 68 | |
| 69 | $key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path); |
| 70 | if (!array_key_exists($key, $cache)) { |
| 71 | $hash = hash_file('sha256', $path); |
| 72 | $cache[$key] = $hash === false ? null : $hash; |
| 73 | } |
| 74 | |
| 75 | return $cache[$key]; |
| 76 | } |
| 77 | |
| 78 | /** |
| 79 | * indexSha256 definition: |
| 80 | * Collect every node in the index (recursively) that has a non-null sha256. |
| 81 | * Take the unique data files they point to, as paths relative to the web |
| 82 | * root without a leading slash (e.g. "data/powers/feint.json"). |
| 83 | * Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line |
| 84 | * per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces). |
| 85 | * indexSha256 = lowercase hex sha256 of the concatenation of those lines. |
| 86 | * Reproducible from the web/ directory with: |
| 87 | * sha256sum <those files, LC_ALL=C sorted> | sha256sum |
| 88 | */ |
| 89 | function computeIndexSha256(array $nodes, string $baseUrl): string |
| 90 | { |
| 91 | return indexSha256FromFiles(collectIndexFiles($nodes, $baseUrl)); |
| 92 | } |
| 93 | |
| 94 | /** |
| 95 | * Unique data files referenced by the index: "data/<folder>/<file>.json" => sha256, |
| 96 | * sorted by path (byte-wise). |
| 97 | */ |
| 98 | function collectIndexFiles(array $nodes, string $baseUrl): array |
| 99 | { |
| 100 | $files = []; |
| 101 | collectNodeHashes($nodes, $baseUrl, $files); |
| 102 | ksort($files, SORT_STRING); |
| 103 | return $files; |
| 104 | } |
| 105 | |
| 106 | /** |
| 107 | * @param array<string,string> $files path => sha256, already sorted by path |
| 108 | */ |
| 109 | function indexSha256FromFiles(array $files): string |
| 110 | { |
| 111 | $lines = ''; |
| 112 | foreach ($files as $path => $hash) |
| 113 | $lines .= $hash . ' ' . $path . "\n"; |
| 114 | |
| 115 | return hash('sha256', $lines); |
| 116 | } |
| 117 | |
| 118 | function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void |
| 119 | { |
| 120 | $prefix = $baseUrl . '/'; |
| 121 | foreach ($nodes as $node) { |
| 122 | if (!is_array($node)) |
| 123 | continue; |
| 124 | |
| 125 | $link = (string) ($node['link'] ?? ''); |
| 126 | if (isset($node['sha256']) && str_starts_with($link, $prefix)) |
| 127 | $files[substr($link, strlen($prefix))] = $node['sha256']; |
| 128 | |
| 129 | collectNodeHashes($node['children'] ?? [], $baseUrl, $files); |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | function loadAllJson(string $dataRoot): array |
| 134 | { |
| 135 | $map = []; |
| 136 | |
| 137 | $it = new RecursiveIteratorIterator( |
| 138 | new RecursiveDirectoryIterator($dataRoot, FilesystemIterator::SKIP_DOTS) |
| 139 | ); |
| 140 | |
| 141 | foreach ($it as $file) { |
| 142 | /** @var SplFileInfo $file */ |
| 143 | if (strtolower($file->getExtension()) !== 'json') |
| 144 | continue; |
| 145 | |
| 146 | $path = $file->getPathname(); |
| 147 | $rel = str_replace('\\', '/', substr($path, strlen($dataRoot))); |
| 148 | if ($rel === '') |
| 149 | $rel = '/' . $file->getFilename(); |
| 150 | if ($rel[0] !== '/') |
| 151 | $rel = '/' . $rel; |
| 152 | |
| 153 | try { |
| 154 | $raw = file_get_contents($path); |
| 155 | if ($raw === false) |
| 156 | continue; |
| 157 | $json = json_decode($raw, true, 512, JSON_THROW_ON_ERROR); |
| 158 | if (!is_array($json)) |
| 159 | continue; |
| 160 | } catch (Throwable $e) { |
| 161 | continue; |
| 162 | } |
| 163 | |
| 164 | $json['_relPath'] = $rel; |
| 165 | $map[$rel] = $json; |
| 166 | } |
| 167 | |
| 168 | return $map; |
| 169 | } |
| 170 | |
| 171 | function buildChapterNode(array $chapter, string $chapterLink, array $allJson, array &$visited, string $baseUrl): array |
| 172 | { |
| 173 | $name = $chapterLink !== '' ? basename($chapterLink, '.json') : ($chapter['name'] ?? 'chapter'); |
| 174 | $fullChapterLink = $chapterLink !== '' ? $baseUrl . $chapterLink : ''; |
| 175 | |
| 176 | $node = [ |
| 177 | 'name' => $name, |
| 178 | 'type' => $chapter['type'] ?? 'chapter', |
| 179 | 'pageType' => 'page', |
| 180 | 'link' => $fullChapterLink, |
| 181 | 'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null, |
| 182 | 'children' => [], |
| 183 | ]; |
| 184 | |
| 185 | $folderIncludes = extractFolderIncludes($chapter, ['subheaders']); |
| 186 | |
| 187 | $children = []; |
| 188 | |
| 189 | foreach (($chapter['subheaders'] ?? []) as $subheader) { |
| 190 | if (!is_array($subheader)) |
| 191 | continue; |
| 192 | |
| 193 | $subNode = [ |
| 194 | 'name' => $subheader['name'] ?? 'subheader', |
| 195 | 'type' => 'subheader', |
| 196 | 'pageType' => 'page child element', |
| 197 | 'link' => $fullChapterLink, |
| 198 | 'sha256' => $node['sha256'], |
| 199 | 'children' => [], |
| 200 | ]; |
| 201 | |
| 202 | $subIncludes = extractFolderIncludes($subheader); |
| 203 | $subNode['children'] = mergeChildren( |
| 204 | buildChildrenFromFolderIncludes($subIncludes, $allJson, $visited, $baseUrl) |
| 205 | ); |
| 206 | |
| 207 | $children[] = $subNode; |
| 208 | } |
| 209 | |
| 210 | $children = mergeChildren( |
| 211 | $children, |
| 212 | buildChildrenFromFolderIncludes($folderIncludes, $allJson, $visited, $baseUrl) |
| 213 | ); |
| 214 | |
| 215 | $node['children'] = $children; |
| 216 | |
| 217 | return $node; |
| 218 | } |
| 219 | |
| 220 | function buildChildrenFromFolderIncludes(array $folders, array $allJson, array &$visited, string $baseUrl): array |
| 221 | { |
| 222 | $children = []; |
| 223 | $rels = array_keys($allJson); |
| 224 | sort($rels, SORT_NATURAL | SORT_FLAG_CASE); |
| 225 | foreach ($folders as $folder) { |
| 226 | $folderRel = '/' . trim(str_replace('\\', '/', $folder), '/'); |
| 227 | foreach ($rels as $rel) { |
| 228 | if (str_starts_with($rel, $folderRel . '/')) { |
| 229 | $children[] = buildDataLinkNode($rel, '', $allJson, $visited, $baseUrl); |
| 230 | } |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | return $children; |
| 235 | } |
| 236 | |
| 237 | function buildDataLinkNode(string $relPath, string $label, array $allJson, array &$visited, string $baseUrl): array |
| 238 | { |
| 239 | $link = $baseUrl . '/data' . $relPath; |
| 240 | $json = $allJson[$relPath] ?? null; |
| 241 | |
| 242 | $name = $label; |
| 243 | $type = 'linkedData'; |
| 244 | |
| 245 | if (is_array($json)) { |
| 246 | $type = $json['type'] ?? 'data'; |
| 247 | if ($name === '') { |
| 248 | $name = $json['name'] ?? ($json['title'] ?? basename($relPath, '.json')); |
| 249 | } |
| 250 | } |
| 251 | |
| 252 | $node = [ |
| 253 | 'name' => $name !== '' ? $name : basename($relPath, '.json'), |
| 254 | 'type' => $type, |
| 255 | 'pageType' => 'data link', |
| 256 | 'link' => $link, |
| 257 | 'sha256' => fileSha256('/data' . $relPath), |
| 258 | 'children' => [], |
| 259 | ]; |
| 260 | |
| 261 | if (!is_array($json)) |
| 262 | return $node; |
| 263 | |
| 264 | if (isset($visited[$relPath])) |
| 265 | return $node; |
| 266 | |
| 267 | $visited[$relPath] = true; |
| 268 | |
| 269 | $folderChildren = buildChildrenFromFolderIncludes(extractFolderIncludes($json), $allJson, $visited, $baseUrl); |
| 270 | $node['children'] = mergeChildren($folderChildren); |
| 271 | |
| 272 | return $node; |
| 273 | } |
| 274 | |
| 275 | function normalizeWikiPath(string $path): string |
| 276 | { |
| 277 | $path = str_replace('\\', '/', $path); |
| 278 | if ($path === '') |
| 279 | return ''; |
| 280 | |
| 281 | if (!str_starts_with($path, '/')) |
| 282 | $path = '/' . $path; |
| 283 | |
| 284 | return $path; |
| 285 | } |
| 286 | |
| 287 | function getBaseUrl(): string |
| 288 | { |
| 289 | $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http'; |
| 290 | $host = $_SERVER['HTTP_HOST'] ?? 'localhost'; |
| 291 | $scriptName = $_SERVER['SCRIPT_NAME'] ?? '/dataIndex.php'; |
| 292 | $dir = rtrim(str_replace('\\', '/', dirname($scriptName)), '/'); |
| 293 | return $scheme . '://' . $host . ($dir === '' ? '' : $dir); |
| 294 | } |
| 295 | |
| 296 | function normalizeDataRelPath(string $path): string |
| 297 | { |
| 298 | $path = trim($path); |
| 299 | if ($path === '') |
| 300 | return ''; |
| 301 | |
| 302 | $path = str_replace('\\', '/', $path); |
| 303 | if (str_starts_with($path, '/data/')) |
| 304 | $path = substr($path, strlen('/data')); |
| 305 | if (str_starts_with($path, 'data/')) |
| 306 | $path = substr($path, strlen('data')); |
| 307 | |
| 308 | if (!str_starts_with($path, '/')) |
| 309 | $path = '/' . $path; |
| 310 | |
| 311 | return $path; |
| 312 | } |
| 313 | |
| 314 | function extractFolderIncludes(mixed $value, array $skipKeys = []): array |
| 315 | { |
| 316 | $folders = []; |
| 317 | |
| 318 | if (is_array($value)) { |
| 319 | foreach ($value as $k => $v) { |
| 320 | if (in_array($k, $skipKeys, true)) |
| 321 | continue; |
| 322 | $folders = array_merge($folders, extractFolderIncludes($v, $skipKeys)); |
| 323 | } |
| 324 | return $folders; |
| 325 | } |
| 326 | |
| 327 | if (is_string($value)) { |
| 328 | return extractFolderIncludesFromString($value); |
| 329 | } |
| 330 | |
| 331 | return $folders; |
| 332 | } |
| 333 | |
| 334 | function extractFolderIncludesFromString(string $text): array |
| 335 | { |
| 336 | $folders = []; |
| 337 | $pattern = '/\[\[\[\s*([a-zA-Z0-9_.-]+)\s*\]\]\]/'; |
| 338 | |
| 339 | if (preg_match_all($pattern, $text, $matches, PREG_SET_ORDER)) { |
| 340 | foreach ($matches as $m) { |
| 341 | $folders[] = $m[1]; |
| 342 | } |
| 343 | } |
| 344 | |
| 345 | return $folders; |
| 346 | } |
| 347 | |
| 348 | function mergeChildren(array ...$groups): array |
| 349 | { |
| 350 | $merged = []; |
| 351 | $seen = []; |
| 352 | |
| 353 | foreach ($groups as $children) { |
| 354 | foreach ($children as $child) { |
| 355 | if (!is_array($child)) |
| 356 | continue; |
| 357 | |
| 358 | $key = ($child['link'] ?? '') . '|' . ($child['name'] ?? '') . '|' . ($child['pageType'] ?? ''); |
| 359 | if (isset($seen[$key])) |
| 360 | continue; |
| 361 | |
| 362 | $seen[$key] = true; |
| 363 | $merged[] = $child; |
| 364 | } |
| 365 | } |
| 366 | |
| 367 | return $merged; |
| 368 | } |