BlueGit

timfa 66wiki

Files

66wiki / web / dataBulk.php

dataBulk.php

dataBulk.php 8.0 KiB Download Raw
1<?php
2/**
3 * Bulk data endpoint: fetch many data files in one request.
4 *
5 * GET dataBulk.php -> all files covered by dataIndex.php
6 * GET dataBulk.php?files=a/b.json,c/d.json -> only those files
7 * POST dataBulk.php {"files": [...]} -> only those files (may be combined with ?files=)
8 * POST dataBulk.php {"known": {"a/b.json": "<sha256>", ...}}
9 * -> only files that are new or whose sha256 differs from the given one,
10 * plus "deleted": known paths that are no longer in the index.
11 * "known" can be combined with "files" (then only the selected files are compared).
12 *
13 * Paths are relative to web/data, e.g. "powers/feint.json".
14 * Response (application/json, gzip if the client accepts it):
15 * {
16 * "indexSha256": "...", same value as dataIndex.php / X-Index-Sha256
17 * "fileCount": 451, number of files in the index
18 * "files": { "<path>": { "sha256": "...", "content": "<exact file text>" } },
19 * "missing": [...], valid-looking paths that don't exist
20 * "invalid": [...], rejected paths (traversal, absolute, not .json, ...)
21 * "deleted": [...], only with "known"
22 * "unchanged": 12 only with "known": number of files skipped
23 * }
24 * "content" is the raw file text (a JSON string), so sha256(content as UTF-8 bytes)
25 * equals "sha256" and equals the per-node sha256 in dataIndex.php.
26 */
27
28ini_set('display_errors', '1');
29error_reporting(E_ALL);
30
31if (extension_loaded('zlib') && !ini_get('zlib.output_compression'))
32 ob_start('ob_gzhandler');
33
34header('Content-Type: application/json; charset=utf-8');
35header('Access-Control-Allow-Origin: *');
36header('Access-Control-Allow-Methods: GET, HEAD, OPTIONS, POST');
37header('Access-Control-Allow-Headers: *');
38header('X-Content-Type-Options: nosniff');
39header('X-Robots-Tag: noindex');
40header('Cache-Control: no-store, no-cache, must-revalidate, max-age=0');
41header('Pragma: no-cache');
42
43set_error_handler(function ($severity, $message, $file, $line) {
44 http_response_code(500);
45 echo json_encode(['error' => 'PHP Error', 'message' => $message, 'file' => basename($file), 'line' => $line]);
46 exit;
47});
48
49set_exception_handler(function ($e) {
50 http_response_code(500);
51 echo json_encode(['error' => 'Uncaught Exception', 'type' => get_class($e), 'message' => $e->getMessage()]);
52 exit;
53});
54
55require_once __DIR__ . '/dataIndexLib.php';
56
57$method = $_SERVER['REQUEST_METHOD'] ?? 'GET';
58if ($method === 'OPTIONS') {
59 http_response_code(204);
60 exit;
61}
62if (!in_array($method, ['GET', 'HEAD', 'POST'], true))
63 bulkFail(405, 'Method not allowed');
64
65const BULK_MAX_BODY = 4 * 1024 * 1024;
66const BULK_MAX_PATHS = 10000;
67
68// ---- Parse request ----
69$requested = [];
70$known = null;
71
72if (isset($_GET['files']) && is_string($_GET['files']))
73 $requested = array_merge($requested, explode(',', $_GET['files']));
74
75if ($method === 'POST') {
76 $raw = file_get_contents('php://input', false, null, 0, BULK_MAX_BODY + 1);
77 if ($raw !== false && strlen($raw) > BULK_MAX_BODY)
78 bulkFail(413, 'Request body too large');
79
80 if ($raw !== false && trim($raw) !== '') {
81 try {
82 $body = json_decode($raw, true, 16, JSON_THROW_ON_ERROR);
83 } catch (Throwable $e) {
84 bulkFail(400, 'Body is not valid JSON');
85 }
86 if (!is_array($body) || ($body !== [] && bulkIsList($body)))
87 bulkFail(400, 'Body must be a JSON object');
88
89 if (isset($body['files'])) {
90 if (!is_array($body['files']) || !bulkIsList($body['files']))
91 bulkFail(400, '"files" must be an array of paths');
92 $requested = array_merge($requested, $body['files']);
93 }
94 if (isset($body['known'])) {
95 if (!is_array($body['known']) || ($body['known'] !== [] && bulkIsList($body['known'])))
96 bulkFail(400, '"known" must be an object of path => sha256');
97 $known = $body['known'];
98 }
99 }
100}
101
102if (count($requested) > BULK_MAX_PATHS || ($known !== null && count($known) > BULK_MAX_PATHS))
103 bulkFail(413, 'Too many paths');
104
105// ---- Build index (same logic and hashes as dataIndex.php) ----
106$built = buildDataIndex();
107$indexFiles = []; // "powers/feint.json" => sha256
108foreach ($built['files'] as $p => $h) {
109 if (str_starts_with($p, 'data/'))
110 $indexFiles[substr($p, 5)] = $h;
111}
112header('X-Index-Sha256: ' . $built['indexSha256']);
113
114$dataRoot = realpath(__DIR__ . DIRECTORY_SEPARATOR . 'data');
115if ($dataRoot === false)
116 bulkFail(500, 'Data directory not found');
117
118// ---- Determine scope ----
119$missing = [];
120$invalid = [];
121
122if ($requested) {
123 $scope = [];
124 foreach ($requested as $p) {
125 if (!is_string($p)) {
126 $invalid[] = $p;
127 continue;
128 }
129 $p = trim($p);
130 if ($p === '')
131 continue;
132 if (!isValidDataPath($p)) {
133 $invalid[] = $p;
134 continue;
135 }
136 $scope[$p] = true;
137 }
138 $scope = array_keys($scope);
139} else {
140 $scope = array_keys($indexFiles);
141}
142sort($scope, SORT_STRING);
143
144// ---- Collect files ----
145$files = [];
146$unchanged = 0;
147
148foreach ($scope as $p) {
149 $real = resolveDataFile($dataRoot, $p);
150 if ($real === null) {
151 $missing[] = $p;
152 continue;
153 }
154
155 $raw = file_get_contents($real);
156 if ($raw === false) {
157 $missing[] = $p;
158 continue;
159 }
160 $hash = hash('sha256', $raw);
161
162 if ($known !== null && isset($known[$p]) && is_string($known[$p]) && strtolower($known[$p]) === $hash) {
163 $unchanged++;
164 continue;
165 }
166
167 $entry = ['sha256' => $hash];
168 if (mb_check_encoding($raw, 'UTF-8'))
169 $entry['content'] = $raw;
170 else
171 $entry['contentBase64'] = base64_encode($raw); // should never happen (CI validates UTF-8 JSON)
172
173 $files[$p] = $entry;
174}
175
176$out = [
177 'indexSha256' => $built['indexSha256'],
178 'fileCount' => count($indexFiles),
179 'files' => (object) $files,
180 'missing' => array_values($missing),
181 'invalid' => array_values($invalid),
182];
183
184if ($known !== null) {
185 $deleted = [];
186 foreach (array_keys($known) as $p) {
187 $p = (string) $p;
188 if ($requested && !in_array($p, $scope, true))
189 continue; // selection mode: only report selected paths
190 if (!isset($indexFiles[$p]) && !isset($files[$p]))
191 $deleted[] = $p;
192 }
193 sort($deleted, SORT_STRING);
194 $out['deleted'] = $deleted;
195 $out['unchanged'] = $unchanged;
196}
197
198if ($method !== 'HEAD')
199 echo json_encode($out, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR);
200
201// ---- Helpers ----
202// (kept compatible with PHP 8.0: no array_is_list / never)
203
204function bulkIsList(array $a): bool
205{
206 $i = 0;
207 foreach ($a as $k => $_) {
208 if ($k !== $i++)
209 return false;
210 }
211 return true;
212}
213
214function bulkFail(int $code, string $message): void
215{
216 http_response_code($code);
217 echo json_encode(['error' => $message]);
218 exit;
219}
220
221/**
222 * Accept only plain relative paths like "folder/file.json":
223 * no absolute paths, no "..", no hidden segments, no backslashes, .json only.
224 */
225function isValidDataPath(string $p): bool
226{
227 if (strlen($p) > 255)
228 return false;
229 if (!preg_match('~^[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*\.json$~', $p))
230 return false;
231 foreach (explode('/', $p) as $seg) {
232 if ($seg === '..' || $seg === '.' || str_contains($seg, '..'))
233 return false;
234 }
235 return true;
236}
237
238/**
239 * Resolve a validated relative path to a real file inside web/data.
240 * Returns null if it doesn't exist or resolves (e.g. via symlink) outside web/data.
241 */
242function resolveDataFile(string $dataRoot, string $p): ?string
243{
244 $real = realpath($dataRoot . DIRECTORY_SEPARATOR . $p);
245 if ($real === false || !is_file($real))
246 return null;
247 if (!str_starts_with($real, $dataRoot . DIRECTORY_SEPARATOR))
248 return null;
249 if (strtolower(pathinfo($real, PATHINFO_EXTENSION)) !== 'json')
250 return null;
251 return $real;
252}