| 1 | <?php |
| 2 | /** |
| 3 | * Bulk data endpoint: fetch many data files in one request. |
| 4 | * |
| 5 | * GET dataBulk.php -> all files covered by dataIndex.php |
| 6 | * GET dataBulk.php?files=a/b.json,c/d.json -> only those files |
| 7 | * POST dataBulk.php {"files": [...]} -> only those files (may be combined with ?files=) |
| 8 | * POST dataBulk.php {"known": {"a/b.json": "<sha256>", ...}} |
| 9 | * -> only files that are new or whose sha256 differs from the given one, |
| 10 | * plus "deleted": known paths that are no longer in the index. |
| 11 | * "known" can be combined with "files" (then only the selected files are compared). |
| 12 | * |
| 13 | * Paths are relative to web/data, e.g. "powers/feint.json". |
| 14 | * Response (application/json, gzip if the client accepts it): |
| 15 | * { |
| 16 | * "indexSha256": "...", same value as dataIndex.php / X-Index-Sha256 |
| 17 | * "fileCount": 451, number of files in the index |
| 18 | * "files": { "<path>": { "sha256": "...", "content": "<exact file text>" } }, |
| 19 | * "missing": [...], valid-looking paths that don't exist |
| 20 | * "invalid": [...], rejected paths (traversal, absolute, not .json, ...) |
| 21 | * "deleted": [...], only with "known" |
| 22 | * "unchanged": 12 only with "known": number of files skipped |
| 23 | * } |
| 24 | * "content" is the raw file text (a JSON string), so sha256(content as UTF-8 bytes) |
| 25 | * equals "sha256" and equals the per-node sha256 in dataIndex.php. |
| 26 | */ |
| 27 | |
| 28 | ini_set('display_errors', '1'); |
| 29 | error_reporting(E_ALL); |
| 30 | |
| 31 | if (extension_loaded('zlib') && !ini_get('zlib.output_compression')) |
| 32 | ob_start('ob_gzhandler'); |
| 33 | |
| 34 | header('Content-Type: application/json; charset=utf-8'); |
| 35 | header('Access-Control-Allow-Origin: *'); |
| 36 | header('Access-Control-Allow-Methods: GET, HEAD, OPTIONS, POST'); |
| 37 | header('Access-Control-Allow-Headers: *'); |
| 38 | header('X-Content-Type-Options: nosniff'); |
| 39 | header('X-Robots-Tag: noindex'); |
| 40 | header('Cache-Control: no-store, no-cache, must-revalidate, max-age=0'); |
| 41 | header('Pragma: no-cache'); |
| 42 | |
| 43 | set_error_handler(function ($severity, $message, $file, $line) { |
| 44 | http_response_code(500); |
| 45 | echo json_encode(['error' => 'PHP Error', 'message' => $message, 'file' => basename($file), 'line' => $line]); |
| 46 | exit; |
| 47 | }); |
| 48 | |
| 49 | set_exception_handler(function ($e) { |
| 50 | http_response_code(500); |
| 51 | echo json_encode(['error' => 'Uncaught Exception', 'type' => get_class($e), 'message' => $e->getMessage()]); |
| 52 | exit; |
| 53 | }); |
| 54 | |
| 55 | require_once __DIR__ . '/dataIndexLib.php'; |
| 56 | |
| 57 | $method = $_SERVER['REQUEST_METHOD'] ?? 'GET'; |
| 58 | if ($method === 'OPTIONS') { |
| 59 | http_response_code(204); |
| 60 | exit; |
| 61 | } |
| 62 | if (!in_array($method, ['GET', 'HEAD', 'POST'], true)) |
| 63 | bulkFail(405, 'Method not allowed'); |
| 64 | |
| 65 | const BULK_MAX_BODY = 4 * 1024 * 1024; |
| 66 | const BULK_MAX_PATHS = 10000; |
| 67 | |
| 68 | // ---- Parse request ---- |
| 69 | $requested = []; |
| 70 | $known = null; |
| 71 | |
| 72 | if (isset($_GET['files']) && is_string($_GET['files'])) |
| 73 | $requested = array_merge($requested, explode(',', $_GET['files'])); |
| 74 | |
| 75 | if ($method === 'POST') { |
| 76 | $raw = file_get_contents('php://input', false, null, 0, BULK_MAX_BODY + 1); |
| 77 | if ($raw !== false && strlen($raw) > BULK_MAX_BODY) |
| 78 | bulkFail(413, 'Request body too large'); |
| 79 | |
| 80 | if ($raw !== false && trim($raw) !== '') { |
| 81 | try { |
| 82 | $body = json_decode($raw, true, 16, JSON_THROW_ON_ERROR); |
| 83 | } catch (Throwable $e) { |
| 84 | bulkFail(400, 'Body is not valid JSON'); |
| 85 | } |
| 86 | if (!is_array($body) || ($body !== [] && bulkIsList($body))) |
| 87 | bulkFail(400, 'Body must be a JSON object'); |
| 88 | |
| 89 | if (isset($body['files'])) { |
| 90 | if (!is_array($body['files']) || !bulkIsList($body['files'])) |
| 91 | bulkFail(400, '"files" must be an array of paths'); |
| 92 | $requested = array_merge($requested, $body['files']); |
| 93 | } |
| 94 | if (isset($body['known'])) { |
| 95 | if (!is_array($body['known']) || ($body['known'] !== [] && bulkIsList($body['known']))) |
| 96 | bulkFail(400, '"known" must be an object of path => sha256'); |
| 97 | $known = $body['known']; |
| 98 | } |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | if (count($requested) > BULK_MAX_PATHS || ($known !== null && count($known) > BULK_MAX_PATHS)) |
| 103 | bulkFail(413, 'Too many paths'); |
| 104 | |
| 105 | // ---- Build index (same logic and hashes as dataIndex.php) ---- |
| 106 | $built = buildDataIndex(); |
| 107 | $indexFiles = []; // "powers/feint.json" => sha256 |
| 108 | foreach ($built['files'] as $p => $h) { |
| 109 | if (str_starts_with($p, 'data/')) |
| 110 | $indexFiles[substr($p, 5)] = $h; |
| 111 | } |
| 112 | header('X-Index-Sha256: ' . $built['indexSha256']); |
| 113 | |
| 114 | $dataRoot = realpath(__DIR__ . DIRECTORY_SEPARATOR . 'data'); |
| 115 | if ($dataRoot === false) |
| 116 | bulkFail(500, 'Data directory not found'); |
| 117 | |
| 118 | // ---- Determine scope ---- |
| 119 | $missing = []; |
| 120 | $invalid = []; |
| 121 | |
| 122 | if ($requested) { |
| 123 | $scope = []; |
| 124 | foreach ($requested as $p) { |
| 125 | if (!is_string($p)) { |
| 126 | $invalid[] = $p; |
| 127 | continue; |
| 128 | } |
| 129 | $p = trim($p); |
| 130 | if ($p === '') |
| 131 | continue; |
| 132 | if (!isValidDataPath($p)) { |
| 133 | $invalid[] = $p; |
| 134 | continue; |
| 135 | } |
| 136 | $scope[$p] = true; |
| 137 | } |
| 138 | $scope = array_keys($scope); |
| 139 | } else { |
| 140 | $scope = array_keys($indexFiles); |
| 141 | } |
| 142 | sort($scope, SORT_STRING); |
| 143 | |
| 144 | // ---- Collect files ---- |
| 145 | $files = []; |
| 146 | $unchanged = 0; |
| 147 | |
| 148 | foreach ($scope as $p) { |
| 149 | $real = resolveDataFile($dataRoot, $p); |
| 150 | if ($real === null) { |
| 151 | $missing[] = $p; |
| 152 | continue; |
| 153 | } |
| 154 | |
| 155 | $raw = file_get_contents($real); |
| 156 | if ($raw === false) { |
| 157 | $missing[] = $p; |
| 158 | continue; |
| 159 | } |
| 160 | $hash = hash('sha256', $raw); |
| 161 | |
| 162 | if ($known !== null && isset($known[$p]) && is_string($known[$p]) && strtolower($known[$p]) === $hash) { |
| 163 | $unchanged++; |
| 164 | continue; |
| 165 | } |
| 166 | |
| 167 | $entry = ['sha256' => $hash]; |
| 168 | if (mb_check_encoding($raw, 'UTF-8')) |
| 169 | $entry['content'] = $raw; |
| 170 | else |
| 171 | $entry['contentBase64'] = base64_encode($raw); // should never happen (CI validates UTF-8 JSON) |
| 172 | |
| 173 | $files[$p] = $entry; |
| 174 | } |
| 175 | |
| 176 | $out = [ |
| 177 | 'indexSha256' => $built['indexSha256'], |
| 178 | 'fileCount' => count($indexFiles), |
| 179 | 'files' => (object) $files, |
| 180 | 'missing' => array_values($missing), |
| 181 | 'invalid' => array_values($invalid), |
| 182 | ]; |
| 183 | |
| 184 | if ($known !== null) { |
| 185 | $deleted = []; |
| 186 | foreach (array_keys($known) as $p) { |
| 187 | $p = (string) $p; |
| 188 | if ($requested && !in_array($p, $scope, true)) |
| 189 | continue; // selection mode: only report selected paths |
| 190 | if (!isset($indexFiles[$p]) && !isset($files[$p])) |
| 191 | $deleted[] = $p; |
| 192 | } |
| 193 | sort($deleted, SORT_STRING); |
| 194 | $out['deleted'] = $deleted; |
| 195 | $out['unchanged'] = $unchanged; |
| 196 | } |
| 197 | |
| 198 | if ($method !== 'HEAD') |
| 199 | echo json_encode($out, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR); |
| 200 | |
| 201 | // ---- Helpers ---- |
| 202 | // (kept compatible with PHP 8.0: no array_is_list / never) |
| 203 | |
| 204 | function bulkIsList(array $a): bool |
| 205 | { |
| 206 | $i = 0; |
| 207 | foreach ($a as $k => $_) { |
| 208 | if ($k !== $i++) |
| 209 | return false; |
| 210 | } |
| 211 | return true; |
| 212 | } |
| 213 | |
| 214 | function bulkFail(int $code, string $message): void |
| 215 | { |
| 216 | http_response_code($code); |
| 217 | echo json_encode(['error' => $message]); |
| 218 | exit; |
| 219 | } |
| 220 | |
| 221 | /** |
| 222 | * Accept only plain relative paths like "folder/file.json": |
| 223 | * no absolute paths, no "..", no hidden segments, no backslashes, .json only. |
| 224 | */ |
| 225 | function isValidDataPath(string $p): bool |
| 226 | { |
| 227 | if (strlen($p) > 255) |
| 228 | return false; |
| 229 | if (!preg_match('~^[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*\.json$~', $p)) |
| 230 | return false; |
| 231 | foreach (explode('/', $p) as $seg) { |
| 232 | if ($seg === '..' || $seg === '.' || str_contains($seg, '..')) |
| 233 | return false; |
| 234 | } |
| 235 | return true; |
| 236 | } |
| 237 | |
| 238 | /** |
| 239 | * Resolve a validated relative path to a real file inside web/data. |
| 240 | * Returns null if it doesn't exist or resolves (e.g. via symlink) outside web/data. |
| 241 | */ |
| 242 | function resolveDataFile(string $dataRoot, string $p): ?string |
| 243 | { |
| 244 | $real = realpath($dataRoot . DIRECTORY_SEPARATOR . $p); |
| 245 | if ($real === false || !is_file($real)) |
| 246 | return null; |
| 247 | if (!str_starts_with($real, $dataRoot . DIRECTORY_SEPARATOR)) |
| 248 | return null; |
| 249 | if (strtolower(pathinfo($real, PATHINFO_EXTENSION)) !== 'json') |
| 250 | return null; |
| 251 | return $real; |
| 252 | } |