BlueGit

timfa 66wiki

dataIndex: add per-node sha256 and index hash

- Every node gets "sha256" (right after "link"): lowercase hex sha256 of the
  exact bytes of the data file the link points to (subheaders carry their
  chapter file's hash; null if the file is missing). Memoised per request by
  path+mtime+size.
- indexSha256: sha256 over sha256sum-style lines "<hash>  <path>\n" for the
  unique files in the index, paths like data/powers/feint.json, sorted
  byte-wise. Sent as X-Index-Sha256 header; ?meta=1 returns
  {indexSha256, nodes}. Default output is still the plain node list.

timfa Tim Falken · pushed by timfa

Authored 2026-09-28 · 7 weeks ago

Hash
3e8ba3837de3a5cd21267d83cf4bdbc36758b7ee
Parent
e3e1de1e
1 file changed +78 −1
web/dataIndex.php +78 −1
@@ -74,7 +74,81 @@ foreach ($chapters as $chapterObj) {
74 74 $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
75 75 }
76 76
77 echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
77 $indexSha256 = computeIndexSha256($index, $baseUrl);
78 header('X-Index-Sha256: ' . $indexSha256);
79
80 if (isset($_GET['meta'])) {
81 // Opt-in wrapper; the default output stays the plain node list.
82 echo json_encode([
83 'indexSha256' => $indexSha256,
84 'nodes' => $index,
85 ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
86 } else {
87 echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
88 }
89
90 /**
91 * sha256 (lowercase hex) of the exact bytes of a data file, given its path
92 * relative to the web root (e.g. "/data/powers/feint.json").
93 * Returns null if the file does not exist or cannot be read.
94 * Results are memoised per request and keyed by path + mtime + size.
95 */
96 function fileSha256(string $webRelPath): ?string
97 {
98 static $cache = [];
99
100 $webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
101 $path = __DIR__ . $webRelPath;
102 if (!is_file($path))
103 return null;
104
105 $key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
106 if (!array_key_exists($key, $cache)) {
107 $hash = hash_file('sha256', $path);
108 $cache[$key] = $hash === false ? null : $hash;
109 }
110
111 return $cache[$key];
112 }
113
114 /**
115 * indexSha256 definition:
116 * Collect every node in the index (recursively) that has a non-null sha256.
117 * Take the unique data files they point to, as paths relative to the web
118 * root without a leading slash (e.g. "data/powers/feint.json").
119 * Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
120 * per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
121 * indexSha256 = lowercase hex sha256 of the concatenation of those lines.
122 * Reproducible from the web/ directory with:
123 * sha256sum <those files, LC_ALL=C sorted> | sha256sum
124 */
125 function computeIndexSha256(array $nodes, string $baseUrl): string
126 {
127 $files = [];
128 collectNodeHashes($nodes, $baseUrl, $files);
129 ksort($files, SORT_STRING);
130
131 $lines = '';
132 foreach ($files as $path => $hash)
133 $lines .= $hash . ' ' . $path . "\n";
134
135 return hash('sha256', $lines);
136 }
137
138 function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
139 {
140 $prefix = $baseUrl . '/';
141 foreach ($nodes as $node) {
142 if (!is_array($node))
143 continue;
144
145 $link = (string) ($node['link'] ?? '');
146 if (isset($node['sha256']) && str_starts_with($link, $prefix))
147 $files[substr($link, strlen($prefix))] = $node['sha256'];
148
149 collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
150 }
151 }
78 152
79 153 function loadAllJson(string $dataRoot): array
80 154 {
@@ -124,6 +198,7 @@ function buildChapterNode(array $chapter, string $chapterLink, array $allJson, a
124 198 'type' => $chapter['type'] ?? 'chapter',
125 199 'pageType' => 'page',
126 200 'link' => $fullChapterLink,
201 'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
127 202 'children' => [],
128 203 ];
129 204
@@ -140,6 +215,7 @@ function buildChapterNode(array $chapter, string $chapterLink, array $allJson, a
140 215 'type' => 'subheader',
141 216 'pageType' => 'page child element',
142 217 'link' => $fullChapterLink,
218 'sha256' => $node['sha256'],
143 219 'children' => [],
144 220 ];
145 221
@@ -198,6 +274,7 @@ function buildDataLinkNode(string $relPath, string $label, array $allJson, array
198 274 'type' => $type,
199 275 'pageType' => 'data link',
200 276 'link' => $link,
277 'sha256' => fileSha256('/data' . $relPath),
201 278 'children' => [],
202 279 ];
203 280
@@ -74,7 +74,81 @@ foreach ($chapters as $chapterObj) {
74 $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
74 $index[] = buildChapterNode($chapter, $chapterLink, $allJson, $visited, $baseUrl);
75}
75}
76
76
77echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
77$indexSha256 = computeIndexSha256($index, $baseUrl);
78header('X-Index-Sha256: ' . $indexSha256);
79
80if (isset($_GET['meta'])) {
81 // Opt-in wrapper; the default output stays the plain node list.
82 echo json_encode([
83 'indexSha256' => $indexSha256,
84 'nodes' => $index,
85 ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
86} else {
87 echo json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
88}
89
90/**
91 * sha256 (lowercase hex) of the exact bytes of a data file, given its path
92 * relative to the web root (e.g. "/data/powers/feint.json").
93 * Returns null if the file does not exist or cannot be read.
94 * Results are memoised per request and keyed by path + mtime + size.
95 */
96function fileSha256(string $webRelPath): ?string
97{
98 static $cache = [];
99
100 $webRelPath = '/' . ltrim(str_replace('\\', '/', $webRelPath), '/');
101 $path = __DIR__ . $webRelPath;
102 if (!is_file($path))
103 return null;
104
105 $key = $webRelPath . '|' . (int) @filemtime($path) . '|' . (int) @filesize($path);
106 if (!array_key_exists($key, $cache)) {
107 $hash = hash_file('sha256', $path);
108 $cache[$key] = $hash === false ? null : $hash;
109 }
110
111 return $cache[$key];
112}
113
114/**
115 * indexSha256 definition:
116 * Collect every node in the index (recursively) that has a non-null sha256.
117 * Take the unique data files they point to, as paths relative to the web
118 * root without a leading slash (e.g. "data/powers/feint.json").
119 * Sort them by path (byte-wise, strcmp / LC_ALL=C order) and build one line
120 * per file in sha256sum format: "<sha256 hex> <path>\n" (two spaces).
121 * indexSha256 = lowercase hex sha256 of the concatenation of those lines.
122 * Reproducible from the web/ directory with:
123 * sha256sum <those files, LC_ALL=C sorted> | sha256sum
124 */
125function computeIndexSha256(array $nodes, string $baseUrl): string
126{
127 $files = [];
128 collectNodeHashes($nodes, $baseUrl, $files);
129 ksort($files, SORT_STRING);
130
131 $lines = '';
132 foreach ($files as $path => $hash)
133 $lines .= $hash . ' ' . $path . "\n";
134
135 return hash('sha256', $lines);
136}
137
138function collectNodeHashes(array $nodes, string $baseUrl, array &$files): void
139{
140 $prefix = $baseUrl . '/';
141 foreach ($nodes as $node) {
142 if (!is_array($node))
143 continue;
144
145 $link = (string) ($node['link'] ?? '');
146 if (isset($node['sha256']) && str_starts_with($link, $prefix))
147 $files[substr($link, strlen($prefix))] = $node['sha256'];
148
149 collectNodeHashes($node['children'] ?? [], $baseUrl, $files);
150 }
151}
78
152
79function loadAllJson(string $dataRoot): array
153function loadAllJson(string $dataRoot): array
80{
154{
@@ -124,6 +198,7 @@ function buildChapterNode(array $chapter, string $chapterLink, array $allJson, a
124 'type' => $chapter['type'] ?? 'chapter',
198 'type' => $chapter['type'] ?? 'chapter',
125 'pageType' => 'page',
199 'pageType' => 'page',
126 'link' => $fullChapterLink,
200 'link' => $fullChapterLink,
201 'sha256' => $chapterLink !== '' ? fileSha256($chapterLink) : null,
127 'children' => [],
202 'children' => [],
128 ];
203 ];
129
204
@@ -140,6 +215,7 @@ function buildChapterNode(array $chapter, string $chapterLink, array $allJson, a
140 'type' => 'subheader',
215 'type' => 'subheader',
141 'pageType' => 'page child element',
216 'pageType' => 'page child element',
142 'link' => $fullChapterLink,
217 'link' => $fullChapterLink,
218 'sha256' => $node['sha256'],
143 'children' => [],
219 'children' => [],
144 ];
220 ];
145
221
@@ -198,6 +274,7 @@ function buildDataLinkNode(string $relPath, string $label, array $allJson, array
198 'type' => $type,
274 'type' => $type,
199 'pageType' => 'data link',
275 'pageType' => 'data link',
200 'link' => $link,
276 'link' => $link,
277 'sha256' => fileSha256('/data' . $relPath),
201 'children' => [],
278 'children' => [],
202 ];
279 ];
203
280