'use strict'; // Change journal for the sync API. Files can change through the web UI, the // API or directly on disk, so instead of hooking every writer we keep a // snapshot per root (api_entries) and diff it against the filesystem. The // diff is written to api_changes, which feeds the cursor-based delta API. // Moves/renames are detected by (device, inode) identity. const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const { getDb } = require('../db'); const DEFAULT_SCAN_INTERVAL_MS = 15 * 1000; const DEFAULT_MAX_ENTRIES = 500000; const CHANGE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; const TEMP_PREFIX = '.fmapi-'; const locks = new Map(); function shortHash (value, len = 22) { return crypto.createHash('sha256').update(String(value)).digest('base64url').slice(0, len); } function fileIdFor (stat) { return shortHash(`${stat.dev}:${stat.ino}`); } function versionFor (stat) { const mtime = Math.floor(Number(stat.mtimeMs) || 0); return shortHash(`${stat.ino}:${stat.isDirectory() ? 'd' : stat.size}:${mtime}:${Math.floor(Number(stat.ctimeMs) || 0)}`, 16); } function isIgnoredName (name) { return String(name).startsWith(TEMP_PREFIX); } function entryFromStat (relPath, stat) { return { rel_path: relPath, file_id: fileIdFor(stat), is_dir: stat.isDirectory() ? 1 : 0, size: stat.isDirectory() ? 0 : stat.size, mtime_ms: Math.floor(Number(stat.mtimeMs) || 0), version: versionFor(stat), }; } async function walk (rootAbs, maxEntries) { const out = new Map(); const stack = ['']; while (stack.length) { const rel = stack.pop(); let dirents; try { dirents = await fs.promises.readdir(path.join(rootAbs, rel), { withFileTypes: true }); } catch { continue; } for (const d of dirents) { if (isIgnoredName(d.name)) continue; if (d.isSymbolicLink()) continue; if (!d.isFile() && !d.isDirectory()) continue; const childRel = rel ? `${rel}/${d.name}` : d.name; let stat; try { stat = await fs.promises.lstat(path.join(rootAbs, childRel)); } catch { continue; } out.set(childRel, entryFromStat(childRel, stat)); if (out.size > maxEntries) { const err = new Error(`Root has more than ${maxEntries} entries; raise api.maxSyncEntries.`); err.code = 'too_many_entries'; throw err; } if (stat.isDirectory()) stack.push(childRel); } } return out; } function parentOf (rel) { const i = rel.lastIndexOf('/'); return i < 0 ? '' : rel.slice(0, i); } function depth (rel) { return rel.split('/').length; } function getState (rootKey) { return getDb().prepare('SELECT * FROM api_scan_state WHERE root_key = ?').get([rootKey]) || null; } function currentSeq () { const row = getDb().prepare("SELECT seq FROM sqlite_sequence WHERE name = 'api_changes'").get(); return row ? Number(row.seq) : 0; } function diff (oldMap, newMap) { const removed = []; const added = []; const modified = []; for (const [rel, o] of oldMap) { const n = newMap.get(rel); if (!n || n.file_id !== o.file_id || n.is_dir !== o.is_dir) removed.push(o); else if (n.version !== o.version && !n.is_dir) modified.push(n); } for (const [rel, n] of newMap) { const o = oldMap.get(rel); if (!o || o.file_id !== n.file_id || o.is_dir !== n.is_dir) added.push(n); } const removedById = new Map(removed.map((e) => [e.file_id, e])); const moves = []; const created = []; for (const n of added) { const o = removedById.get(n.file_id); if (o && o.is_dir === n.is_dir) { removedById.delete(n.file_id); moves.push({ from: o, to: n }); } else { created.push(n); } } const deleted = [...removedById.values()]; // Collapse implied children: a moved/deleted folder covers its contents. const movedFrom = new Map(moves.map((m) => [m.from.rel_path, m.to.rel_path])); const effMoves = moves.filter((m) => { const p = parentOf(m.from.rel_path); if (!p || !movedFrom.has(p)) return true; const impliedTo = `${movedFrom.get(p)}/${path.posix.basename(m.from.rel_path)}`; if (impliedTo !== m.to.rel_path) return true; // Implied by parent move, but report a content change if any. if (!m.to.is_dir && m.to.version !== m.from.version) modified.push(m.to); return false; }); const deletedSet = new Set(deleted.filter((e) => e.is_dir).map((e) => e.rel_path)); const effDeleted = deleted.filter((e) => { let p = parentOf(e.rel_path); while (p) { if (deletedSet.has(p)) return false; p = parentOf(p); } return true; }); effDeleted.sort((a, b) => depth(b.rel_path) - depth(a.rel_path)); effMoves.sort((a, b) => depth(a.to.rel_path) - depth(b.to.rel_path)); created.sort((a, b) => depth(a.rel_path) - depth(b.rel_path) || a.rel_path.localeCompare(b.rel_path)); // Order: moves first (so clients can rename locally), then deletes, creates, modifies. const changes = []; for (const m of effMoves) changes.push({ change: 'moved', ...m.to, old_rel_path: m.from.rel_path }); for (const e of effDeleted) changes.push({ change: 'deleted', ...e, old_rel_path: null }); for (const e of created) changes.push({ change: 'created', ...e, old_rel_path: null }); for (const e of modified) changes.push({ change: 'modified', ...e, old_rel_path: null }); return { changes, removed, added, modified }; } async function scanRoot (rootKey, rootAbs, opts = {}) { const maxEntries = opts.maxEntries || DEFAULT_MAX_ENTRIES; const newMap = await walk(rootAbs, maxEntries); const d = getDb(); const state = getState(rootKey); const t = Date.now(); const oldMap = new Map(); for (const row of d.prepare('SELECT rel_path, file_id, is_dir, size, mtime_ms, version FROM api_entries WHERE root_key = ?').all([rootKey])) { oldMap.set(row.rel_path, row); } const { changes } = state ? diff(oldMap, newMap) : { changes: [] }; d.exec('BEGIN'); try { const del = d.prepare('DELETE FROM api_entries WHERE root_key = ? AND rel_path = ?'); const up = d.prepare(`INSERT INTO api_entries (root_key, rel_path, file_id, is_dir, size, mtime_ms, version) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(root_key, rel_path) DO UPDATE SET file_id = excluded.file_id, is_dir = excluded.is_dir, size = excluded.size, mtime_ms = excluded.mtime_ms, version = excluded.version`); for (const rel of oldMap.keys()) { if (!newMap.has(rel)) del.run([rootKey, rel]); } for (const [rel, n] of newMap) { const o = oldMap.get(rel); if (!o || o.version !== n.version || o.file_id !== n.file_id || o.is_dir !== n.is_dir) { up.run([rootKey, rel, n.file_id, n.is_dir, n.size, n.mtime_ms, n.version]); } } del.finalize(); up.finalize(); if (changes.length) { const ins = d.prepare(`INSERT INTO api_changes (root_key, change, rel_path, old_rel_path, file_id, is_dir, size, mtime_ms, version, at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`); for (const c of changes) { ins.run([rootKey, c.change, c.rel_path, c.old_rel_path, c.file_id, c.is_dir, c.change === 'deleted' ? null : c.size, c.change === 'deleted' ? null : c.mtime_ms, c.change === 'deleted' ? null : c.version, t]); } ins.finalize(); } if (!state) { d.prepare('INSERT INTO api_scan_state (root_key, generation, last_scan_at, dirty, min_seq) VALUES (?, ?, ?, 0, ?)') .run([rootKey, crypto.randomBytes(6).toString('base64url'), t, currentSeq()]); } else { d.prepare('UPDATE api_scan_state SET last_scan_at = ?, dirty = 0 WHERE root_key = ?').run([t, rootKey]); } pruneChanges(rootKey, t); d.exec('COMMIT'); } catch (err) { try { d.exec('ROLLBACK'); } catch { /* ignore */ } throw err; } return { changes: changes.length }; } function pruneChanges (rootKey, t) { const d = getDb(); const cutoff = t - CHANGE_RETENTION_MS; const row = d.prepare('SELECT MAX(seq) AS s FROM api_changes WHERE root_key = ? AND at < ?').get([rootKey, cutoff]); if (row && row.s) { d.prepare('DELETE FROM api_changes WHERE root_key = ? AND seq <= ?').run([rootKey, row.s]); d.prepare('UPDATE api_scan_state SET min_seq = MAX(min_seq, ?) WHERE root_key = ?').run([row.s, rootKey]); } } /** Scan when dirty, never scanned, or older than the interval. Serialised per root. */ async function ensureFresh (rootKey, rootAbs, opts = {}) { const interval = opts.intervalMs != null ? opts.intervalMs : DEFAULT_SCAN_INTERVAL_MS; const running = locks.get(rootKey); if (running) await running.catch(() => {}); const state = getState(rootKey); if (state && !state.dirty && !opts.force && Date.now() - state.last_scan_at < interval) return state; const p = scanRoot(rootKey, rootAbs, opts).finally(() => locks.delete(rootKey)); locks.set(rootKey, p); await p; return getState(rootKey); } function markDirty (rootKey) { getDb().prepare('UPDATE api_scan_state SET dirty = 1 WHERE root_key = ?').run([rootKey]); } function listChanges (rootKey, afterSeq, limit) { return getDb().prepare('SELECT * FROM api_changes WHERE root_key = ? AND seq > ? ORDER BY seq LIMIT ?') .all([rootKey, afterSeq, limit]); } function listEntries (rootKey, offset, limit) { return getDb().prepare('SELECT * FROM api_entries WHERE root_key = ? ORDER BY rel_path LIMIT ? OFFSET ?') .all([rootKey, limit, offset]); } module.exports = { TEMP_PREFIX, fileIdFor, versionFor, isIgnoredName, entryFromStat, ensureFresh, scanRoot, markDirty, listChanges, listEntries, currentSeq, getState, diff, };