all files covered by dataIndex.php * GET dataBulk.php?files=a/b.json,c/d.json -> only those files * POST dataBulk.php {"files": [...]} -> only those files (may be combined with ?files=) * POST dataBulk.php {"known": {"a/b.json": "", ...}} * -> only files that are new or whose sha256 differs from the given one, * plus "deleted": known paths that are no longer in the index. * "known" can be combined with "files" (then only the selected files are compared). * * Paths are relative to web/data, e.g. "powers/feint.json". * Response (application/json, gzip if the client accepts it): * { * "indexSha256": "...", same value as dataIndex.php / X-Index-Sha256 * "fileCount": 451, number of files in the index * "files": { "": { "sha256": "...", "content": "" } }, * "missing": [...], valid-looking paths that don't exist * "invalid": [...], rejected paths (traversal, absolute, not .json, ...) * "deleted": [...], only with "known" * "unchanged": 12 only with "known": number of files skipped * } * "content" is the raw file text (a JSON string), so sha256(content as UTF-8 bytes) * equals "sha256" and equals the per-node sha256 in dataIndex.php. */ ini_set('display_errors', '1'); error_reporting(E_ALL); if (extension_loaded('zlib') && !ini_get('zlib.output_compression')) ob_start('ob_gzhandler'); header('Content-Type: application/json; charset=utf-8'); header('Access-Control-Allow-Origin: *'); header('Access-Control-Allow-Methods: GET, HEAD, OPTIONS, POST'); header('Access-Control-Allow-Headers: *'); header('X-Content-Type-Options: nosniff'); header('X-Robots-Tag: noindex'); header('Cache-Control: no-store, no-cache, must-revalidate, max-age=0'); header('Pragma: no-cache'); set_error_handler(function ($severity, $message, $file, $line) { http_response_code(500); echo json_encode(['error' => 'PHP Error', 'message' => $message, 'file' => basename($file), 'line' => $line]); exit; }); set_exception_handler(function ($e) { http_response_code(500); echo json_encode(['error' => 'Uncaught Exception', 'type' => get_class($e), 'message' => $e->getMessage()]); exit; }); require_once __DIR__ . '/dataIndexLib.php'; $method = $_SERVER['REQUEST_METHOD'] ?? 'GET'; if ($method === 'OPTIONS') { http_response_code(204); exit; } if (!in_array($method, ['GET', 'HEAD', 'POST'], true)) bulkFail(405, 'Method not allowed'); const BULK_MAX_BODY = 4 * 1024 * 1024; const BULK_MAX_PATHS = 10000; // ---- Parse request ---- $requested = []; $known = null; if (isset($_GET['files']) && is_string($_GET['files'])) $requested = array_merge($requested, explode(',', $_GET['files'])); if ($method === 'POST') { $raw = file_get_contents('php://input', false, null, 0, BULK_MAX_BODY + 1); if ($raw !== false && strlen($raw) > BULK_MAX_BODY) bulkFail(413, 'Request body too large'); if ($raw !== false && trim($raw) !== '') { try { $body = json_decode($raw, true, 16, JSON_THROW_ON_ERROR); } catch (Throwable $e) { bulkFail(400, 'Body is not valid JSON'); } if (!is_array($body) || ($body !== [] && bulkIsList($body))) bulkFail(400, 'Body must be a JSON object'); if (isset($body['files'])) { if (!is_array($body['files']) || !bulkIsList($body['files'])) bulkFail(400, '"files" must be an array of paths'); $requested = array_merge($requested, $body['files']); } if (isset($body['known'])) { if (!is_array($body['known']) || ($body['known'] !== [] && bulkIsList($body['known']))) bulkFail(400, '"known" must be an object of path => sha256'); $known = $body['known']; } } } if (count($requested) > BULK_MAX_PATHS || ($known !== null && count($known) > BULK_MAX_PATHS)) bulkFail(413, 'Too many paths'); // ---- Build index (same logic and hashes as dataIndex.php) ---- $built = buildDataIndex(); $indexFiles = []; // "powers/feint.json" => sha256 foreach ($built['files'] as $p => $h) { if (str_starts_with($p, 'data/')) $indexFiles[substr($p, 5)] = $h; } header('X-Index-Sha256: ' . $built['indexSha256']); $dataRoot = realpath(__DIR__ . DIRECTORY_SEPARATOR . 'data'); if ($dataRoot === false) bulkFail(500, 'Data directory not found'); // ---- Determine scope ---- $missing = []; $invalid = []; if ($requested) { $scope = []; foreach ($requested as $p) { if (!is_string($p)) { $invalid[] = $p; continue; } $p = trim($p); if ($p === '') continue; if (!isValidDataPath($p)) { $invalid[] = $p; continue; } $scope[$p] = true; } $scope = array_keys($scope); } else { $scope = array_keys($indexFiles); } sort($scope, SORT_STRING); // ---- Collect files ---- $files = []; $unchanged = 0; foreach ($scope as $p) { $real = resolveDataFile($dataRoot, $p); if ($real === null) { $missing[] = $p; continue; } $raw = file_get_contents($real); if ($raw === false) { $missing[] = $p; continue; } $hash = hash('sha256', $raw); if ($known !== null && isset($known[$p]) && is_string($known[$p]) && strtolower($known[$p]) === $hash) { $unchanged++; continue; } $entry = ['sha256' => $hash]; if (mb_check_encoding($raw, 'UTF-8')) $entry['content'] = $raw; else $entry['contentBase64'] = base64_encode($raw); // should never happen (CI validates UTF-8 JSON) $files[$p] = $entry; } $out = [ 'indexSha256' => $built['indexSha256'], 'fileCount' => count($indexFiles), 'files' => (object) $files, 'missing' => array_values($missing), 'invalid' => array_values($invalid), ]; if ($known !== null) { $deleted = []; foreach (array_keys($known) as $p) { $p = (string) $p; if ($requested && !in_array($p, $scope, true)) continue; // selection mode: only report selected paths if (!isset($indexFiles[$p]) && !isset($files[$p])) $deleted[] = $p; } sort($deleted, SORT_STRING); $out['deleted'] = $deleted; $out['unchanged'] = $unchanged; } if ($method !== 'HEAD') echo json_encode($out, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR); // ---- Helpers ---- // (kept compatible with PHP 8.0: no array_is_list / never) function bulkIsList(array $a): bool { $i = 0; foreach ($a as $k => $_) { if ($k !== $i++) return false; } return true; } function bulkFail(int $code, string $message): void { http_response_code($code); echo json_encode(['error' => $message]); exit; } /** * Accept only plain relative paths like "folder/file.json": * no absolute paths, no "..", no hidden segments, no backslashes, .json only. */ function isValidDataPath(string $p): bool { if (strlen($p) > 255) return false; if (!preg_match('~^[A-Za-z0-9_-][A-Za-z0-9_.-]*(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*\.json$~', $p)) return false; foreach (explode('/', $p) as $seg) { if ($seg === '..' || $seg === '.' || str_contains($seg, '..')) return false; } return true; } /** * Resolve a validated relative path to a real file inside web/data. * Returns null if it doesn't exist or resolves (e.g. via symlink) outside web/data. */ function resolveDataFile(string $dataRoot, string $p): ?string { $real = realpath($dataRoot . DIRECTORY_SEPARATOR . $p); if ($real === false || !is_file($real)) return null; if (!str_starts_with($real, $dataRoot . DIRECTORY_SEPARATOR)) return null; if (strtolower(pathinfo($real, PATHINFO_EXTENSION)) !== 'json') return null; return $real; }