BlueGit

timfa filemanager

Files

filemanager / server / api / sync-index.js

sync-index.js

sync-index.js 9.6 KiB Download Raw
1'use strict';
2
3// Change journal for the sync API. Files can change through the web UI, the
4// API or directly on disk, so instead of hooking every writer we keep a
5// snapshot per root (api_entries) and diff it against the filesystem. The
6// diff is written to api_changes, which feeds the cursor-based delta API.
7// Moves/renames are detected by (device, inode) identity.
8
9const fs = require('fs');
10const path = require('path');
11const crypto = require('crypto');
12const { getDb } = require('../db');
13
14const DEFAULT_SCAN_INTERVAL_MS = 15 * 1000;
15const DEFAULT_MAX_ENTRIES = 500000;
16const CHANGE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
17const TEMP_PREFIX = '.fmapi-';
18
19const locks = new Map();
20
21function shortHash (value, len = 22)
22{
23 return crypto.createHash('sha256').update(String(value)).digest('base64url').slice(0, len);
24}
25
26function fileIdFor (stat)
27{
28 return shortHash(`${stat.dev}:${stat.ino}`);
29}
30
31function versionFor (stat)
32{
33 const mtime = Math.floor(Number(stat.mtimeMs) || 0);
34 return shortHash(`${stat.ino}:${stat.isDirectory() ? 'd' : stat.size}:${mtime}:${Math.floor(Number(stat.ctimeMs) || 0)}`, 16);
35}
36
37function isIgnoredName (name)
38{
39 return String(name).startsWith(TEMP_PREFIX);
40}
41
42function entryFromStat (relPath, stat)
43{
44 return {
45 rel_path: relPath,
46 file_id: fileIdFor(stat),
47 is_dir: stat.isDirectory() ? 1 : 0,
48 size: stat.isDirectory() ? 0 : stat.size,
49 mtime_ms: Math.floor(Number(stat.mtimeMs) || 0),
50 version: versionFor(stat),
51 };
52}
53
54async function walk (rootAbs, maxEntries)
55{
56 const out = new Map();
57 const stack = [''];
58 while (stack.length)
59 {
60 const rel = stack.pop();
61 let dirents;
62 try
63 {
64 dirents = await fs.promises.readdir(path.join(rootAbs, rel), { withFileTypes: true });
65 } catch
66 {
67 continue;
68 }
69 for (const d of dirents)
70 {
71 if (isIgnoredName(d.name)) continue;
72 if (d.isSymbolicLink()) continue;
73 if (!d.isFile() && !d.isDirectory()) continue;
74 const childRel = rel ? `${rel}/${d.name}` : d.name;
75 let stat;
76 try { stat = await fs.promises.lstat(path.join(rootAbs, childRel)); } catch { continue; }
77 out.set(childRel, entryFromStat(childRel, stat));
78 if (out.size > maxEntries)
79 {
80 const err = new Error(`Root has more than ${maxEntries} entries; raise api.maxSyncEntries.`);
81 err.code = 'too_many_entries';
82 throw err;
83 }
84 if (stat.isDirectory()) stack.push(childRel);
85 }
86 }
87 return out;
88}
89
90function parentOf (rel)
91{
92 const i = rel.lastIndexOf('/');
93 return i < 0 ? '' : rel.slice(0, i);
94}
95
96function depth (rel)
97{
98 return rel.split('/').length;
99}
100
101function getState (rootKey)
102{
103 return getDb().prepare('SELECT * FROM api_scan_state WHERE root_key = ?').get([rootKey]) || null;
104}
105
106function currentSeq ()
107{
108 const row = getDb().prepare("SELECT seq FROM sqlite_sequence WHERE name = 'api_changes'").get();
109 return row ? Number(row.seq) : 0;
110}
111
112function diff (oldMap, newMap)
113{
114 const removed = [];
115 const added = [];
116 const modified = [];
117 for (const [rel, o] of oldMap)
118 {
119 const n = newMap.get(rel);
120 if (!n || n.file_id !== o.file_id || n.is_dir !== o.is_dir) removed.push(o);
121 else if (n.version !== o.version && !n.is_dir) modified.push(n);
122 }
123 for (const [rel, n] of newMap)
124 {
125 const o = oldMap.get(rel);
126 if (!o || o.file_id !== n.file_id || o.is_dir !== n.is_dir) added.push(n);
127 }
128
129 const removedById = new Map(removed.map((e) => [e.file_id, e]));
130 const moves = [];
131 const created = [];
132 for (const n of added)
133 {
134 const o = removedById.get(n.file_id);
135 if (o && o.is_dir === n.is_dir)
136 {
137 removedById.delete(n.file_id);
138 moves.push({ from: o, to: n });
139 } else
140 {
141 created.push(n);
142 }
143 }
144 const deleted = [...removedById.values()];
145
146 // Collapse implied children: a moved/deleted folder covers its contents.
147 const movedFrom = new Map(moves.map((m) => [m.from.rel_path, m.to.rel_path]));
148 const effMoves = moves.filter((m) =>
149 {
150 const p = parentOf(m.from.rel_path);
151 if (!p || !movedFrom.has(p)) return true;
152 const impliedTo = `${movedFrom.get(p)}/${path.posix.basename(m.from.rel_path)}`;
153 if (impliedTo !== m.to.rel_path) return true;
154 // Implied by parent move, but report a content change if any.
155 if (!m.to.is_dir && m.to.version !== m.from.version) modified.push(m.to);
156 return false;
157 });
158 const deletedSet = new Set(deleted.filter((e) => e.is_dir).map((e) => e.rel_path));
159 const effDeleted = deleted.filter((e) =>
160 {
161 let p = parentOf(e.rel_path);
162 while (p)
163 {
164 if (deletedSet.has(p)) return false;
165 p = parentOf(p);
166 }
167 return true;
168 });
169
170 effDeleted.sort((a, b) => depth(b.rel_path) - depth(a.rel_path));
171 effMoves.sort((a, b) => depth(a.to.rel_path) - depth(b.to.rel_path));
172 created.sort((a, b) => depth(a.rel_path) - depth(b.rel_path) || a.rel_path.localeCompare(b.rel_path));
173
174 // Order: moves first (so clients can rename locally), then deletes, creates, modifies.
175 const changes = [];
176 for (const m of effMoves) changes.push({ change: 'moved', ...m.to, old_rel_path: m.from.rel_path });
177 for (const e of effDeleted) changes.push({ change: 'deleted', ...e, old_rel_path: null });
178 for (const e of created) changes.push({ change: 'created', ...e, old_rel_path: null });
179 for (const e of modified) changes.push({ change: 'modified', ...e, old_rel_path: null });
180 return { changes, removed, added, modified };
181}
182
183async function scanRoot (rootKey, rootAbs, opts = {})
184{
185 const maxEntries = opts.maxEntries || DEFAULT_MAX_ENTRIES;
186 const newMap = await walk(rootAbs, maxEntries);
187 const d = getDb();
188 const state = getState(rootKey);
189 const t = Date.now();
190
191 const oldMap = new Map();
192 for (const row of d.prepare('SELECT rel_path, file_id, is_dir, size, mtime_ms, version FROM api_entries WHERE root_key = ?').all([rootKey]))
193 {
194 oldMap.set(row.rel_path, row);
195 }
196
197 const { changes } = state ? diff(oldMap, newMap) : { changes: [] };
198
199 d.exec('BEGIN');
200 try
201 {
202 const del = d.prepare('DELETE FROM api_entries WHERE root_key = ? AND rel_path = ?');
203 const up = d.prepare(`INSERT INTO api_entries (root_key, rel_path, file_id, is_dir, size, mtime_ms, version) VALUES (?, ?, ?, ?, ?, ?, ?)
204 ON CONFLICT(root_key, rel_path) DO UPDATE SET file_id = excluded.file_id, is_dir = excluded.is_dir,
205 size = excluded.size, mtime_ms = excluded.mtime_ms, version = excluded.version`);
206 for (const rel of oldMap.keys())
207 {
208 if (!newMap.has(rel)) del.run([rootKey, rel]);
209 }
210 for (const [rel, n] of newMap)
211 {
212 const o = oldMap.get(rel);
213 if (!o || o.version !== n.version || o.file_id !== n.file_id || o.is_dir !== n.is_dir)
214 {
215 up.run([rootKey, rel, n.file_id, n.is_dir, n.size, n.mtime_ms, n.version]);
216 }
217 }
218 del.finalize();
219 up.finalize();
220 if (changes.length)
221 {
222 const ins = d.prepare(`INSERT INTO api_changes (root_key, change, rel_path, old_rel_path, file_id, is_dir, size, mtime_ms, version, at)
223 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
224 for (const c of changes)
225 {
226 ins.run([rootKey, c.change, c.rel_path, c.old_rel_path, c.file_id, c.is_dir,
227 c.change === 'deleted' ? null : c.size, c.change === 'deleted' ? null : c.mtime_ms, c.change === 'deleted' ? null : c.version, t]);
228 }
229 ins.finalize();
230 }
231 if (!state)
232 {
233 d.prepare('INSERT INTO api_scan_state (root_key, generation, last_scan_at, dirty, min_seq) VALUES (?, ?, ?, 0, ?)')
234 .run([rootKey, crypto.randomBytes(6).toString('base64url'), t, currentSeq()]);
235 } else
236 {
237 d.prepare('UPDATE api_scan_state SET last_scan_at = ?, dirty = 0 WHERE root_key = ?').run([t, rootKey]);
238 }
239 pruneChanges(rootKey, t);
240 d.exec('COMMIT');
241 } catch (err)
242 {
243 try { d.exec('ROLLBACK'); } catch { /* ignore */ }
244 throw err;
245 }
246 return { changes: changes.length };
247}
248
249function pruneChanges (rootKey, t)
250{
251 const d = getDb();
252 const cutoff = t - CHANGE_RETENTION_MS;
253 const row = d.prepare('SELECT MAX(seq) AS s FROM api_changes WHERE root_key = ? AND at < ?').get([rootKey, cutoff]);
254 if (row && row.s)
255 {
256 d.prepare('DELETE FROM api_changes WHERE root_key = ? AND seq <= ?').run([rootKey, row.s]);
257 d.prepare('UPDATE api_scan_state SET min_seq = MAX(min_seq, ?) WHERE root_key = ?').run([row.s, rootKey]);
258 }
259}
260
261/** Scan when dirty, never scanned, or older than the interval. Serialised per root. */
262async function ensureFresh (rootKey, rootAbs, opts = {})
263{
264 const interval = opts.intervalMs != null ? opts.intervalMs : DEFAULT_SCAN_INTERVAL_MS;
265 const running = locks.get(rootKey);
266 if (running) await running.catch(() => {});
267 const state = getState(rootKey);
268 if (state && !state.dirty && !opts.force && Date.now() - state.last_scan_at < interval) return state;
269 const p = scanRoot(rootKey, rootAbs, opts).finally(() => locks.delete(rootKey));
270 locks.set(rootKey, p);
271 await p;
272 return getState(rootKey);
273}
274
275function markDirty (rootKey)
276{
277 getDb().prepare('UPDATE api_scan_state SET dirty = 1 WHERE root_key = ?').run([rootKey]);
278}
279
280function listChanges (rootKey, afterSeq, limit)
281{
282 return getDb().prepare('SELECT * FROM api_changes WHERE root_key = ? AND seq > ? ORDER BY seq LIMIT ?')
283 .all([rootKey, afterSeq, limit]);
284}
285
286function listEntries (rootKey, offset, limit)
287{
288 return getDb().prepare('SELECT * FROM api_entries WHERE root_key = ? ORDER BY rel_path LIMIT ? OFFSET ?')
289 .all([rootKey, limit, offset]);
290}
291
292module.exports = {
293 TEMP_PREFIX,
294 fileIdFor,
295 versionFor,
296 isIgnoredName,
297 entryFromStat,
298 ensureFresh,
299 scanRoot,
300 markDirty,
301 listChanges,
302 listEntries,
303 currentSeq,
304 getState,
305 diff,
306};