BlueGit

timfa filemanager

Cursor Optimisation pass

Experimental

timfa Tim Falken · pushed by timfa

Authored 2026-05-30 · 5 months ago

Hash
7e6dfc5274c43db981e902d7d3f7303583ddbe04
Parent
fec9d013
14 files changed +1666 −338
.gitignore +2 −0
@@ -8,3 +8,5 @@ config.json
8 8 public/thumbs/
9 9 zip-cache/
10 10 logs.txt
11
12 *.db-journal
@@ -8,3 +8,5 @@ config.json
8public/thumbs/
8public/thumbs/
9zip-cache/
9zip-cache/
10logs.txt
10logs.txt
11
12*.db-journal
server/auth.js +74 −14
@@ -6,11 +6,16 @@ const geoip = require('geoip-lite');
6 6 const { getDb } = require('./db');
7 7
8 8 const SESSION_IDLE_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
9 const SESSION_LAST_SEEN_WRITE_INTERVAL_MS = 60 * 1000; // throttle DB writes on active sessions
9 10 const AUTH_SESSION_PURGE_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes
11 const AUTH_USER_CACHE_TTL_MS = 60 * 1000;
12 const AUTH_REGION_CACHE_MAX_ENTRIES = 2048;
10 13 const AUTH_SESSION_REVOKED_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours
11 14 const AUTH_SESSION_IDLE_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours beyond idle timeout
12 15
13 16 let authSessionPurgeTimer = null;
17 const regionByIp = new Map();
18 const userCacheByUsername = new Map();
14 19 const authSessionCleanupState = {
15 20 lastRunAt: null,
16 21 lastPurgedCount: 0,
@@ -40,10 +45,68 @@ function getClientIp (req)
40 45
41 46 function getRegion (ip)
42 47 {
48 const key = String(ip || '');
49 if (!key) return 'local:';
50
51 const cached = regionByIp.get(key);
52 if (cached) return cached;
53
43 54 // ip is already normalized by getClientIp (no ::ffff: prefix, no ::1)
44 const geo = geoip.lookup(ip);
55 const geo = geoip.lookup(key);
45 56 // Private / RFC1918 addresses won't resolve — use ip itself as region.
46 return geo ? `${geo.country}:${geo.region}` : `local:${ip}`;
57 const region = geo ? `${geo.country}:${geo.region}` : `local:${key}`;
58
59 if (regionByIp.size >= AUTH_REGION_CACHE_MAX_ENTRIES)
60 {
61 regionByIp.clear();
62 }
63 regionByIp.set(key, region);
64 return region;
65 }
66
67 function getCachedUser (username)
68 {
69 const key = String(username || '');
70 if (!key) return null;
71
72 const cached = userCacheByUsername.get(key);
73 const now = nowMs();
74 if (cached && (now - cached.cachedAt) < AUTH_USER_CACHE_TTL_MS)
75 {
76 return cached.user;
77 }
78
79 const db = getDb();
80 const user = db.prepare('SELECT id, username, is_admin, permissions FROM users WHERE username = ?')
81 .get([key]);
82 if (!user) return null;
83
84 const parsed = {
85 id: user.id,
86 username: user.username,
87 isAdmin: user.is_admin === 1,
88 permissions: JSON.parse(user.permissions || '{}'),
89 };
90
91 userCacheByUsername.set(key, { user: parsed, cachedAt: now });
92 return parsed;
93 }
94
95 function clearAuthUserCache (username)
96 {
97 if (username)
98 {
99 userCacheByUsername.delete(String(username));
100 return;
101 }
102 userCacheByUsername.clear();
103 }
104
105 function touchSessionLastSeen (sessionId, lastSeenAt)
106 {
107 const db = getDb();
108 db.prepare('UPDATE auth_sessions SET last_seen_at = ? WHERE session_id = ?')
109 .run([lastSeenAt, sessionId]);
47 110 }
48 111
49 112 function fingerprintHash (fingerprint)
@@ -269,29 +332,25 @@ function requireAuth (req, res, next)
269 332 return res.status(401).json({ error: 'Session invalid or expired.' });
270 333 }
271 334
272 if (nowMs() - Number(session.last_seen_at || 0) > SESSION_IDLE_TIMEOUT_MS)
335 const now = nowMs();
336 if (now - Number(session.last_seen_at || 0) > SESSION_IDLE_TIMEOUT_MS)
273 337 {
274 338 revokeSession(payload.sid);
275 339 return res.status(401).json({ error: 'Session invalid or expired.' });
276 340 }
277 341
278 db.prepare('UPDATE auth_sessions SET last_seen_at = ? WHERE session_id = ?').run([nowMs(), payload.sid]);
279
280 // Load user from DB
281 const user = db.prepare('SELECT id, username, is_admin, permissions FROM users WHERE username = ?')
282 .get([payload.u]);
342 if (now - Number(session.last_seen_at || 0) >= SESSION_LAST_SEEN_WRITE_INTERVAL_MS)
343 {
344 touchSessionLastSeen(payload.sid, now);
345 }
283 346
347 const user = getCachedUser(payload.u);
284 348 if (!user)
285 349 {
286 350 return res.status(401).json({ error: 'Session invalid or expired.' });
287 351 }
288 352
289 req.user = {
290 id: user.id,
291 username: user.username,
292 isAdmin: user.is_admin === 1,
293 permissions: JSON.parse(user.permissions || '{}'),
294 };
353 req.user = user;
295 354 req.authSessionId = payload.sid;
296 355
297 356 next();
@@ -315,6 +374,7 @@ module.exports = {
315 374 requireAdmin,
316 375 getClientIp,
317 376 getRegion,
377 clearAuthUserCache,
318 378 revokeSession,
319 379 purgeStaleSessions,
320 380 getAuthSessionCleanupSnapshot,
@@ -6,11 +6,16 @@ const geoip = require('geoip-lite');
6const { getDb } = require('./db');
6const { getDb } = require('./db');
7
7
8const SESSION_IDLE_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
8const SESSION_IDLE_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
9const SESSION_LAST_SEEN_WRITE_INTERVAL_MS = 60 * 1000; // throttle DB writes on active sessions
9const AUTH_SESSION_PURGE_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes
10const AUTH_SESSION_PURGE_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes
11const AUTH_USER_CACHE_TTL_MS = 60 * 1000;
12const AUTH_REGION_CACHE_MAX_ENTRIES = 2048;
10const AUTH_SESSION_REVOKED_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours
13const AUTH_SESSION_REVOKED_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours
11const AUTH_SESSION_IDLE_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours beyond idle timeout
14const AUTH_SESSION_IDLE_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours beyond idle timeout
12
15
13let authSessionPurgeTimer = null;
16let authSessionPurgeTimer = null;
17const regionByIp = new Map();
18const userCacheByUsername = new Map();
14const authSessionCleanupState = {
19const authSessionCleanupState = {
15 lastRunAt: null,
20 lastRunAt: null,
16 lastPurgedCount: 0,
21 lastPurgedCount: 0,
@@ -40,10 +45,68 @@ function getClientIp (req)
40
45
41function getRegion (ip)
46function getRegion (ip)
42{
47{
48 const key = String(ip || '');
49 if (!key) return 'local:';
50
51 const cached = regionByIp.get(key);
52 if (cached) return cached;
53
43 // ip is already normalized by getClientIp (no ::ffff: prefix, no ::1)
54 // ip is already normalized by getClientIp (no ::ffff: prefix, no ::1)
44 const geo = geoip.lookup(ip);
55 const geo = geoip.lookup(key);
45 // Private / RFC1918 addresses won't resolve — use ip itself as region.
56 // Private / RFC1918 addresses won't resolve — use ip itself as region.
46 return geo ? `${geo.country}:${geo.region}` : `local:${ip}`;
57 const region = geo ? `${geo.country}:${geo.region}` : `local:${key}`;
58
59 if (regionByIp.size >= AUTH_REGION_CACHE_MAX_ENTRIES)
60 {
61 regionByIp.clear();
62 }
63 regionByIp.set(key, region);
64 return region;
65}
66
67function getCachedUser (username)
68{
69 const key = String(username || '');
70 if (!key) return null;
71
72 const cached = userCacheByUsername.get(key);
73 const now = nowMs();
74 if (cached && (now - cached.cachedAt) < AUTH_USER_CACHE_TTL_MS)
75 {
76 return cached.user;
77 }
78
79 const db = getDb();
80 const user = db.prepare('SELECT id, username, is_admin, permissions FROM users WHERE username = ?')
81 .get([key]);
82 if (!user) return null;
83
84 const parsed = {
85 id: user.id,
86 username: user.username,
87 isAdmin: user.is_admin === 1,
88 permissions: JSON.parse(user.permissions || '{}'),
89 };
90
91 userCacheByUsername.set(key, { user: parsed, cachedAt: now });
92 return parsed;
93}
94
95function clearAuthUserCache (username)
96{
97 if (username)
98 {
99 userCacheByUsername.delete(String(username));
100 return;
101 }
102 userCacheByUsername.clear();
103}
104
105function touchSessionLastSeen (sessionId, lastSeenAt)
106{
107 const db = getDb();
108 db.prepare('UPDATE auth_sessions SET last_seen_at = ? WHERE session_id = ?')
109 .run([lastSeenAt, sessionId]);
47}
110}
48
111
49function fingerprintHash (fingerprint)
112function fingerprintHash (fingerprint)
@@ -269,29 +332,25 @@ function requireAuth (req, res, next)
269 return res.status(401).json({ error: 'Session invalid or expired.' });
332 return res.status(401).json({ error: 'Session invalid or expired.' });
270 }
333 }
271
334
272 if (nowMs() - Number(session.last_seen_at || 0) > SESSION_IDLE_TIMEOUT_MS)
335 const now = nowMs();
336 if (now - Number(session.last_seen_at || 0) > SESSION_IDLE_TIMEOUT_MS)
273 {
337 {
274 revokeSession(payload.sid);
338 revokeSession(payload.sid);
275 return res.status(401).json({ error: 'Session invalid or expired.' });
339 return res.status(401).json({ error: 'Session invalid or expired.' });
276 }
340 }
277
341
278 db.prepare('UPDATE auth_sessions SET last_seen_at = ? WHERE session_id = ?').run([nowMs(), payload.sid]);
342 if (now - Number(session.last_seen_at || 0) >= SESSION_LAST_SEEN_WRITE_INTERVAL_MS)
279
343 {
280 // Load user from DB
344 touchSessionLastSeen(payload.sid, now);
281 const user = db.prepare('SELECT id, username, is_admin, permissions FROM users WHERE username = ?')
345 }
282 .get([payload.u]);
283
346
347 const user = getCachedUser(payload.u);
284 if (!user)
348 if (!user)
285 {
349 {
286 return res.status(401).json({ error: 'Session invalid or expired.' });
350 return res.status(401).json({ error: 'Session invalid or expired.' });
287 }
351 }
288
352
289 req.user = {
353 req.user = user;
290 id: user.id,
291 username: user.username,
292 isAdmin: user.is_admin === 1,
293 permissions: JSON.parse(user.permissions || '{}'),
294 };
295 req.authSessionId = payload.sid;
354 req.authSessionId = payload.sid;
296
355
297 next();
356 next();
@@ -315,6 +374,7 @@ module.exports = {
315 requireAdmin,
374 requireAdmin,
316 getClientIp,
375 getClientIp,
317 getRegion,
376 getRegion,
377 clearAuthUserCache,
318 revokeSession,
378 revokeSession,
319 purgeStaleSessions,
379 purgeStaleSessions,
320 getAuthSessionCleanupSnapshot,
380 getAuthSessionCleanupSnapshot,
server/config.js +51 −10
@@ -5,6 +5,9 @@ const path = require('path');
5 5
6 6 const CONFIG_PATH = path.join(__dirname, '../config.json');
7 7
8 let cachedConfig = null;
9 let cachedConfigMtimeMs = null;
10
8 11 const DEFAULT_SCHEDULED_RESTART = {
9 12 enabled: false,
10 13 cadence: 'daily',
@@ -25,26 +28,42 @@ function normalizeScheduledRestart (value)
25 28 };
26 29 }
27 30
31 function normalizeLoadedConfig (parsed)
32 {
33 if (!parsed || typeof parsed !== 'object')
34 {
35 return { port: 3000, paths: [], rootAliases: {} };
36 }
37 if (!parsed.rootAliases || typeof parsed.rootAliases !== 'object')
38 {
39 parsed.rootAliases = {};
40 }
41 parsed.scheduledRestart = normalizeScheduledRestart(parsed.scheduledRestart);
42 return parsed;
43 }
44
28 45 function loadConfig ()
29 46 {
30 47 try
31 48 {
32 const raw = fs.readFileSync(CONFIG_PATH, 'utf8');
33 const parsed = JSON.parse(raw);
34 if (!parsed || typeof parsed !== 'object')
49 const stat = fs.statSync(CONFIG_PATH);
50 if (cachedConfig && cachedConfigMtimeMs === stat.mtimeMs)
35 51 {
36 return { port: 3000, paths: [], rootAliases: {} };
52 return cachedConfig;
37 53 }
38 if (!parsed.rootAliases || typeof parsed.rootAliases !== 'object')
39 {
40 parsed.rootAliases = {};
41 }
42 parsed.scheduledRestart = normalizeScheduledRestart(parsed.scheduledRestart);
54
55 const raw = fs.readFileSync(CONFIG_PATH, 'utf8');
56 const parsed = normalizeLoadedConfig(JSON.parse(raw));
57 cachedConfig = parsed;
58 cachedConfigMtimeMs = stat.mtimeMs;
43 59 return parsed;
44 60 } catch
45 61 {
46 62 const defaults = { port: 3000, paths: [], rootAliases: {}, scheduledRestart: { ...DEFAULT_SCHEDULED_RESTART } };
47 63 fs.writeFileSync(CONFIG_PATH, JSON.stringify(defaults, null, 2), 'utf8');
64 const stat = fs.statSync(CONFIG_PATH);
65 cachedConfig = defaults;
66 cachedConfigMtimeMs = stat.mtimeMs;
48 67 return defaults;
49 68 }
50 69 }
@@ -58,6 +77,22 @@ function saveConfig (config)
58 77 // Atomic replace to avoid partially-written config files.
59 78 fs.writeFileSync(tmp, payload, 'utf8');
60 79 fs.renameSync(tmp, CONFIG_PATH);
80
81 const normalized = normalizeLoadedConfig(config);
82 cachedConfig = normalized;
83 try
84 {
85 cachedConfigMtimeMs = fs.statSync(CONFIG_PATH).mtimeMs;
86 } catch
87 {
88 cachedConfigMtimeMs = null;
89 }
90 }
91
92 function invalidateConfigCache ()
93 {
94 cachedConfig = null;
95 cachedConfigMtimeMs = null;
61 96 }
62 97
63 98 function normalizeForCompare (p)
@@ -116,4 +151,10 @@ function getAllowedRoot (filePath, allowedRoots)
116 151 return null;
117 152 }
118 153
119 module.exports = { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot };
154 module.exports = {
155 loadConfig,
156 saveConfig,
157 invalidateConfigCache,
158 getEffectivePaths,
159 getAllowedRoot,
160 };
@@ -5,6 +5,9 @@ const path = require('path');
5
5
6const CONFIG_PATH = path.join(__dirname, '../config.json');
6const CONFIG_PATH = path.join(__dirname, '../config.json');
7
7
8let cachedConfig = null;
9let cachedConfigMtimeMs = null;
10
8const DEFAULT_SCHEDULED_RESTART = {
11const DEFAULT_SCHEDULED_RESTART = {
9 enabled: false,
12 enabled: false,
10 cadence: 'daily',
13 cadence: 'daily',
@@ -25,26 +28,42 @@ function normalizeScheduledRestart (value)
25 };
28 };
26}
29}
27
30
31function normalizeLoadedConfig (parsed)
32{
33 if (!parsed || typeof parsed !== 'object')
34 {
35 return { port: 3000, paths: [], rootAliases: {} };
36 }
37 if (!parsed.rootAliases || typeof parsed.rootAliases !== 'object')
38 {
39 parsed.rootAliases = {};
40 }
41 parsed.scheduledRestart = normalizeScheduledRestart(parsed.scheduledRestart);
42 return parsed;
43}
44
28function loadConfig ()
45function loadConfig ()
29{
46{
30 try
47 try
31 {
48 {
32 const raw = fs.readFileSync(CONFIG_PATH, 'utf8');
49 const stat = fs.statSync(CONFIG_PATH);
33 const parsed = JSON.parse(raw);
50 if (cachedConfig && cachedConfigMtimeMs === stat.mtimeMs)
34 if (!parsed || typeof parsed !== 'object')
35 {
51 {
36 return { port: 3000, paths: [], rootAliases: {} };
52 return cachedConfig;
37 }
53 }
38 if (!parsed.rootAliases || typeof parsed.rootAliases !== 'object')
54
39 {
55 const raw = fs.readFileSync(CONFIG_PATH, 'utf8');
40 parsed.rootAliases = {};
56 const parsed = normalizeLoadedConfig(JSON.parse(raw));
41 }
57 cachedConfig = parsed;
42 parsed.scheduledRestart = normalizeScheduledRestart(parsed.scheduledRestart);
58 cachedConfigMtimeMs = stat.mtimeMs;
43 return parsed;
59 return parsed;
44 } catch
60 } catch
45 {
61 {
46 const defaults = { port: 3000, paths: [], rootAliases: {}, scheduledRestart: { ...DEFAULT_SCHEDULED_RESTART } };
62 const defaults = { port: 3000, paths: [], rootAliases: {}, scheduledRestart: { ...DEFAULT_SCHEDULED_RESTART } };
47 fs.writeFileSync(CONFIG_PATH, JSON.stringify(defaults, null, 2), 'utf8');
63 fs.writeFileSync(CONFIG_PATH, JSON.stringify(defaults, null, 2), 'utf8');
64 const stat = fs.statSync(CONFIG_PATH);
65 cachedConfig = defaults;
66 cachedConfigMtimeMs = stat.mtimeMs;
48 return defaults;
67 return defaults;
49 }
68 }
50}
69}
@@ -58,6 +77,22 @@ function saveConfig (config)
58 // Atomic replace to avoid partially-written config files.
77 // Atomic replace to avoid partially-written config files.
59 fs.writeFileSync(tmp, payload, 'utf8');
78 fs.writeFileSync(tmp, payload, 'utf8');
60 fs.renameSync(tmp, CONFIG_PATH);
79 fs.renameSync(tmp, CONFIG_PATH);
80
81 const normalized = normalizeLoadedConfig(config);
82 cachedConfig = normalized;
83 try
84 {
85 cachedConfigMtimeMs = fs.statSync(CONFIG_PATH).mtimeMs;
86 } catch
87 {
88 cachedConfigMtimeMs = null;
89 }
90}
91
92function invalidateConfigCache ()
93{
94 cachedConfig = null;
95 cachedConfigMtimeMs = null;
61}
96}
62
97
63function normalizeForCompare (p)
98function normalizeForCompare (p)
@@ -116,4 +151,10 @@ function getAllowedRoot (filePath, allowedRoots)
116 return null;
151 return null;
117}
152}
118
153
119module.exports = { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot };
154module.exports = {
155 loadConfig,
156 saveConfig,
157 invalidateConfigCache,
158 getEffectivePaths,
159 getAllowedRoot,
160};
server/db.js +14 −0
@@ -70,6 +70,20 @@ function initDb ()
70 70 );
71 71
72 72 CREATE INDEX IF NOT EXISTS idx_share_tokens_expires ON share_tokens(expires_at);
73
74 CREATE TABLE IF NOT EXISTS file_search_index (
75 file_path TEXT PRIMARY KEY,
76 root_path TEXT NOT NULL,
77 name TEXT NOT NULL,
78 name_lower TEXT NOT NULL,
79 relative_path TEXT NOT NULL,
80 relative_lower TEXT NOT NULL,
81 size INTEGER,
82 mtime_ms INTEGER,
83 indexed_at INTEGER NOT NULL
84 );
85
86 CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
73 87 `);
74 88
75 89 const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();
@@ -70,6 +70,20 @@ function initDb ()
70 );
70 );
71
71
72 CREATE INDEX IF NOT EXISTS idx_share_tokens_expires ON share_tokens(expires_at);
72 CREATE INDEX IF NOT EXISTS idx_share_tokens_expires ON share_tokens(expires_at);
73
74 CREATE TABLE IF NOT EXISTS file_search_index (
75 file_path TEXT PRIMARY KEY,
76 root_path TEXT NOT NULL,
77 name TEXT NOT NULL,
78 name_lower TEXT NOT NULL,
79 relative_path TEXT NOT NULL,
80 relative_lower TEXT NOT NULL,
81 size INTEGER,
82 mtime_ms INTEGER,
83 indexed_at INTEGER NOT NULL
84 );
85
86 CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
73 `);
87 `);
74
88
75 const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();
89 const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();
server/file-hash-queue.js added +260 −0
@@ -0,0 +1,260 @@
1 'use strict';
2
3 const crypto = require('crypto');
4 const fs = require('fs');
5 const path = require('path');
6 const { getDb } = require('./db');
7
8 const MAX_QUEUE_LENGTH = toPositiveInt(process.env.FILE_HASH_QUEUE_MAX_LENGTH, 4096);
9 const BACKLOG_BATCH_SIZE = toPositiveInt(process.env.FILE_HASH_BACKLOG_BATCH_SIZE, 500);
10 const ACTIVE_STREAM_DELAY_MS = toPositiveInt(process.env.FILE_HASH_ACTIVE_STREAM_DELAY_MS, 15000);
11 const IDLE_TICK_MS = toPositiveInt(process.env.FILE_HASH_IDLE_TICK_MS, 250);
12
13 const queue = [];
14 const queued = new Set();
15
16 let loopStarted = false;
17 let loopTimer = null;
18 let workerRunning = false;
19
20 function toPositiveInt (value, fallback)
21 {
22 const parsed = Number.parseInt(String(value || ''), 10);
23 return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
24 }
25
26 function delay (ms)
27 {
28 return new Promise((resolve) => setTimeout(resolve, ms));
29 }
30
31 /**
32 * Computes the SHA-256 hex digest of the file at `filePath`.
33 * Streams the file so it works for large files without loading them into memory.
34 */
35 function computeFileSha256 (filePath)
36 {
37 return new Promise((resolve, reject) =>
38 {
39 const hash = crypto.createHash('sha256');
40 const stream = fs.createReadStream(filePath);
41 stream.on('error', reject);
42 stream.on('data', (chunk) => hash.update(chunk));
43 stream.on('end', () => resolve(hash.digest('hex')));
44 });
45 }
46
47 function hasActiveVideoStreams ()
48 {
49 try
50 {
51 const { getActiveStreamCount } = require('./video-stream');
52 return Number(getActiveStreamCount()) > 0;
53 } catch
54 {
55 return false;
56 }
57 }
58
59 function loopDelayMs ()
60 {
61 return hasActiveVideoStreams() ? ACTIVE_STREAM_DELAY_MS : IDLE_TICK_MS;
62 }
63
64 function enqueueFileHash (filePath)
65 {
66 const resolved = path.resolve(String(filePath || ''));
67 if (!resolved) return false;
68 if (queued.has(resolved)) return false;
69
70 if (queue.length >= MAX_QUEUE_LENGTH)
71 {
72 const dropped = queue.shift();
73 if (dropped) queued.delete(dropped);
74 }
75
76 queued.add(resolved);
77 queue.push(resolved);
78 scheduleWorker();
79 return true;
80 }
81
82 function persistFileHash (filePath, sha256)
83 {
84 const db = getDb();
85 const existing = db.prepare('SELECT file_path FROM file_meta WHERE file_path = ?').get([filePath]);
86
87 if (existing)
88 {
89 db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, filePath]);
90 return;
91 }
92
93 let uploadedAt = null;
94 try
95 {
96 const stat = fs.statSync(filePath);
97 const birthtimeMs = Number(stat.birthtimeMs);
98 const ctimeMs = Number(stat.ctimeMs);
99 const mtimeMs = Number(stat.mtimeMs);
100 const chosenMs = (Number.isFinite(birthtimeMs) && birthtimeMs > 0)
101 ? birthtimeMs
102 : (Number.isFinite(ctimeMs) && ctimeMs > 0)
103 ? ctimeMs
104 : (Number.isFinite(mtimeMs) && mtimeMs > 0)
105 ? mtimeMs
106 : null;
107 if (Number.isFinite(chosenMs) && chosenMs > 0)
108 {
109 uploadedAt = new Date(chosenMs).toISOString();
110 }
111 } catch
112 {
113 // Ignore missing files.
114 }
115
116 db.prepare(`
117 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256)
118 VALUES (?, ?, ?, ?, ?, ?)
119 ON CONFLICT(file_path) DO UPDATE SET sha256 = excluded.sha256
120 `).run([filePath, '', '[]', '{}', uploadedAt, sha256]);
121 }
122
123 async function processQueuedFile (filePath)
124 {
125 let stat;
126 try
127 {
128 stat = await fs.promises.stat(filePath);
129 } catch
130 {
131 return;
132 }
133
134 if (!stat.isFile()) return;
135
136 const sha256 = await computeFileSha256(filePath);
137 persistFileHash(filePath, sha256);
138 }
139
140 async function processQueueBatch (respectStreamDelay)
141 {
142 while (queue.length)
143 {
144 const filePath = queue.shift();
145 queued.delete(filePath);
146 if (!filePath) continue;
147
148 try
149 {
150 await processQueuedFile(filePath);
151 } catch (err)
152 {
153 console.error('[file-hash] hash failed:', filePath, err && err.message ? err.message : err);
154 }
155
156 if (queue.length && respectStreamDelay) await delay(loopDelayMs());
157 }
158 }
159
160 async function runWorkerPass ()
161 {
162 if (workerRunning) return;
163 workerRunning = true;
164
165 try
166 {
167 await processQueueBatch(true);
168 } finally
169 {
170 workerRunning = false;
171 }
172 }
173
174 function scheduleWorker ()
175 {
176 if (workerRunning || !queue.length) return;
177
178 void runWorkerPass().finally(() =>
179 {
180 if (queue.length) scheduleWorker();
181 });
182 }
183
184 function enqueueBacklogFromDb ()
185 {
186 const db = getDb();
187 const rows = db.prepare(`
188 SELECT file_path
189 FROM file_meta
190 WHERE sha256 IS NULL OR sha256 = ''
191 LIMIT ?
192 `).all([BACKLOG_BATCH_SIZE]);
193
194 for (const row of rows)
195 {
196 enqueueFileHash(row.file_path);
197 }
198 }
199
200 function startFileHashQueue ()
201 {
202 if (loopStarted) return;
203 loopStarted = true;
204 enqueueBacklogFromDb();
205 scheduleWorker();
206
207 loopTimer = setInterval(() =>
208 {
209 enqueueBacklogFromDb();
210 scheduleWorker();
211 }, 60 * 1000);
212 if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();
213 }
214
215 function stopFileHashQueue ()
216 {
217 loopStarted = false;
218 if (loopTimer)
219 {
220 clearInterval(loopTimer);
221 loopTimer = null;
222 }
223 queue.length = 0;
224 queued.clear();
225 }
226
227 /**
228 * Processes all queued hashes. Intended for tests and graceful shutdown drains.
229 */
230 async function drainFileHashQueue ()
231 {
232 while (workerRunning) await delay(5);
233 workerRunning = true;
234 try
235 {
236 await processQueueBatch(false);
237 } finally
238 {
239 workerRunning = false;
240 }
241 }
242
243 function getFileHashQueueSnapshot ()
244 {
245 return {
246 loopStarted,
247 queueLength: queue.length,
248 workerRunning,
249 hasActiveVideoStreams: hasActiveVideoStreams(),
250 };
251 }
252
253 module.exports = {
254 computeFileSha256,
255 enqueueFileHash,
256 drainFileHashQueue,
257 startFileHashQueue,
258 stopFileHashQueue,
259 getFileHashQueueSnapshot,
260 };
@@ -0,0 +1,260 @@
1'use strict';
2
3const crypto = require('crypto');
4const fs = require('fs');
5const path = require('path');
6const { getDb } = require('./db');
7
8const MAX_QUEUE_LENGTH = toPositiveInt(process.env.FILE_HASH_QUEUE_MAX_LENGTH, 4096);
9const BACKLOG_BATCH_SIZE = toPositiveInt(process.env.FILE_HASH_BACKLOG_BATCH_SIZE, 500);
10const ACTIVE_STREAM_DELAY_MS = toPositiveInt(process.env.FILE_HASH_ACTIVE_STREAM_DELAY_MS, 15000);
11const IDLE_TICK_MS = toPositiveInt(process.env.FILE_HASH_IDLE_TICK_MS, 250);
12
13const queue = [];
14const queued = new Set();
15
16let loopStarted = false;
17let loopTimer = null;
18let workerRunning = false;
19
20function toPositiveInt (value, fallback)
21{
22 const parsed = Number.parseInt(String(value || ''), 10);
23 return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
24}
25
26function delay (ms)
27{
28 return new Promise((resolve) => setTimeout(resolve, ms));
29}
30
31/**
32 * Computes the SHA-256 hex digest of the file at `filePath`.
33 * Streams the file so it works for large files without loading them into memory.
34 */
35function computeFileSha256 (filePath)
36{
37 return new Promise((resolve, reject) =>
38 {
39 const hash = crypto.createHash('sha256');
40 const stream = fs.createReadStream(filePath);
41 stream.on('error', reject);
42 stream.on('data', (chunk) => hash.update(chunk));
43 stream.on('end', () => resolve(hash.digest('hex')));
44 });
45}
46
47function hasActiveVideoStreams ()
48{
49 try
50 {
51 const { getActiveStreamCount } = require('./video-stream');
52 return Number(getActiveStreamCount()) > 0;
53 } catch
54 {
55 return false;
56 }
57}
58
59function loopDelayMs ()
60{
61 return hasActiveVideoStreams() ? ACTIVE_STREAM_DELAY_MS : IDLE_TICK_MS;
62}
63
64function enqueueFileHash (filePath)
65{
66 const resolved = path.resolve(String(filePath || ''));
67 if (!resolved) return false;
68 if (queued.has(resolved)) return false;
69
70 if (queue.length >= MAX_QUEUE_LENGTH)
71 {
72 const dropped = queue.shift();
73 if (dropped) queued.delete(dropped);
74 }
75
76 queued.add(resolved);
77 queue.push(resolved);
78 scheduleWorker();
79 return true;
80}
81
82function persistFileHash (filePath, sha256)
83{
84 const db = getDb();
85 const existing = db.prepare('SELECT file_path FROM file_meta WHERE file_path = ?').get([filePath]);
86
87 if (existing)
88 {
89 db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, filePath]);
90 return;
91 }
92
93 let uploadedAt = null;
94 try
95 {
96 const stat = fs.statSync(filePath);
97 const birthtimeMs = Number(stat.birthtimeMs);
98 const ctimeMs = Number(stat.ctimeMs);
99 const mtimeMs = Number(stat.mtimeMs);
100 const chosenMs = (Number.isFinite(birthtimeMs) && birthtimeMs > 0)
101 ? birthtimeMs
102 : (Number.isFinite(ctimeMs) && ctimeMs > 0)
103 ? ctimeMs
104 : (Number.isFinite(mtimeMs) && mtimeMs > 0)
105 ? mtimeMs
106 : null;
107 if (Number.isFinite(chosenMs) && chosenMs > 0)
108 {
109 uploadedAt = new Date(chosenMs).toISOString();
110 }
111 } catch
112 {
113 // Ignore missing files.
114 }
115
116 db.prepare(`
117 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256)
118 VALUES (?, ?, ?, ?, ?, ?)
119 ON CONFLICT(file_path) DO UPDATE SET sha256 = excluded.sha256
120 `).run([filePath, '', '[]', '{}', uploadedAt, sha256]);
121}
122
123async function processQueuedFile (filePath)
124{
125 let stat;
126 try
127 {
128 stat = await fs.promises.stat(filePath);
129 } catch
130 {
131 return;
132 }
133
134 if (!stat.isFile()) return;
135
136 const sha256 = await computeFileSha256(filePath);
137 persistFileHash(filePath, sha256);
138}
139
140async function processQueueBatch (respectStreamDelay)
141{
142 while (queue.length)
143 {
144 const filePath = queue.shift();
145 queued.delete(filePath);
146 if (!filePath) continue;
147
148 try
149 {
150 await processQueuedFile(filePath);
151 } catch (err)
152 {
153 console.error('[file-hash] hash failed:', filePath, err && err.message ? err.message : err);
154 }
155
156 if (queue.length && respectStreamDelay) await delay(loopDelayMs());
157 }
158}
159
160async function runWorkerPass ()
161{
162 if (workerRunning) return;
163 workerRunning = true;
164
165 try
166 {
167 await processQueueBatch(true);
168 } finally
169 {
170 workerRunning = false;
171 }
172}
173
174function scheduleWorker ()
175{
176 if (workerRunning || !queue.length) return;
177
178 void runWorkerPass().finally(() =>
179 {
180 if (queue.length) scheduleWorker();
181 });
182}
183
184function enqueueBacklogFromDb ()
185{
186 const db = getDb();
187 const rows = db.prepare(`
188 SELECT file_path
189 FROM file_meta
190 WHERE sha256 IS NULL OR sha256 = ''
191 LIMIT ?
192 `).all([BACKLOG_BATCH_SIZE]);
193
194 for (const row of rows)
195 {
196 enqueueFileHash(row.file_path);
197 }
198}
199
200function startFileHashQueue ()
201{
202 if (loopStarted) return;
203 loopStarted = true;
204 enqueueBacklogFromDb();
205 scheduleWorker();
206
207 loopTimer = setInterval(() =>
208 {
209 enqueueBacklogFromDb();
210 scheduleWorker();
211 }, 60 * 1000);
212 if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();
213}
214
215function stopFileHashQueue ()
216{
217 loopStarted = false;
218 if (loopTimer)
219 {
220 clearInterval(loopTimer);
221 loopTimer = null;
222 }
223 queue.length = 0;
224 queued.clear();
225}
226
227/**
228 * Processes all queued hashes. Intended for tests and graceful shutdown drains.
229 */
230async function drainFileHashQueue ()
231{
232 while (workerRunning) await delay(5);
233 workerRunning = true;
234 try
235 {
236 await processQueueBatch(false);
237 } finally
238 {
239 workerRunning = false;
240 }
241}
242
243function getFileHashQueueSnapshot ()
244{
245 return {
246 loopStarted,
247 queueLength: queue.length,
248 workerRunning,
249 hasActiveVideoStreams: hasActiveVideoStreams(),
250 };
251}
252
253module.exports = {
254 computeFileSha256,
255 enqueueFileHash,
256 drainFileHashQueue,
257 startFileHashQueue,
258 stopFileHashQueue,
259 getFileHashQueueSnapshot,
260};
server/index.js +6 −0
@@ -10,6 +10,8 @@ const { ensureServerKeys } = require('./crypto');
10 10 const { startAuthSessionCleanupLoop, stopAuthSessionCleanupLoop } = require('./auth');
11 11 const { loadConfig } = require('./config');
12 12 const { startThumbnailCacheLoop, stopThumbnailCacheLoop } = require('./thumbnail-cache');
13 const { startSearchIndexLoop, stopSearchIndexLoop } = require('./search-index');
14 const { startFileHashQueue, stopFileHashQueue } = require('./file-hash-queue');
13 15 const { startRestartScheduler, stopRestartScheduler } = require('./restart-scheduler');
14 16 const { logError } = require('./logger');
15 17 const authRoutes = require('./routes/auth');
@@ -65,6 +67,8 @@ function shutdown (opts = {}, ...servers)
65 67 active.forEach(s => { s.__isShuttingDown = true; });
66 68
67 69 stopThumbnailCacheLoop();
70 stopSearchIndexLoop();
71 stopFileHashQueue();
68 72 stopAuthSessionCleanupLoop();
69 73 stopShareMaintenanceLoop();
70 74 stopRestartScheduler();
@@ -127,6 +131,8 @@ function start ()
127 131 const shutdownForRestart = () => shutdown({ exitCode: 1 }, httpServer, httpsServer);
128 132
129 133 startThumbnailCacheLoop();
134 startSearchIndexLoop();
135 startFileHashQueue();
130 136 startAuthSessionCleanupLoop();
131 137 startShareMaintenanceLoop();
132 138 startRestartScheduler({ shutdownForRestart });
@@ -10,6 +10,8 @@ const { ensureServerKeys } = require('./crypto');
10const { startAuthSessionCleanupLoop, stopAuthSessionCleanupLoop } = require('./auth');
10const { startAuthSessionCleanupLoop, stopAuthSessionCleanupLoop } = require('./auth');
11const { loadConfig } = require('./config');
11const { loadConfig } = require('./config');
12const { startThumbnailCacheLoop, stopThumbnailCacheLoop } = require('./thumbnail-cache');
12const { startThumbnailCacheLoop, stopThumbnailCacheLoop } = require('./thumbnail-cache');
13const { startSearchIndexLoop, stopSearchIndexLoop } = require('./search-index');
14const { startFileHashQueue, stopFileHashQueue } = require('./file-hash-queue');
13const { startRestartScheduler, stopRestartScheduler } = require('./restart-scheduler');
15const { startRestartScheduler, stopRestartScheduler } = require('./restart-scheduler');
14const { logError } = require('./logger');
16const { logError } = require('./logger');
15const authRoutes = require('./routes/auth');
17const authRoutes = require('./routes/auth');
@@ -65,6 +67,8 @@ function shutdown (opts = {}, ...servers)
65 active.forEach(s => { s.__isShuttingDown = true; });
67 active.forEach(s => { s.__isShuttingDown = true; });
66
68
67 stopThumbnailCacheLoop();
69 stopThumbnailCacheLoop();
70 stopSearchIndexLoop();
71 stopFileHashQueue();
68 stopAuthSessionCleanupLoop();
72 stopAuthSessionCleanupLoop();
69 stopShareMaintenanceLoop();
73 stopShareMaintenanceLoop();
70 stopRestartScheduler();
74 stopRestartScheduler();
@@ -127,6 +131,8 @@ function start ()
127 const shutdownForRestart = () => shutdown({ exitCode: 1 }, httpServer, httpsServer);
131 const shutdownForRestart = () => shutdown({ exitCode: 1 }, httpServer, httpsServer);
128
132
129 startThumbnailCacheLoop();
133 startThumbnailCacheLoop();
134 startSearchIndexLoop();
135 startFileHashQueue();
130 startAuthSessionCleanupLoop();
136 startAuthSessionCleanupLoop();
131 startShareMaintenanceLoop();
137 startShareMaintenanceLoop();
132 startRestartScheduler({ shutdownForRestart });
138 startRestartScheduler({ shutdownForRestart });
server/routes/admin.js +4 −1
@@ -43,7 +43,7 @@ function ensureUserFolders (userPath)
43 43 }
44 44
45 45 const { getDb } = require('../db');
46 const { requireAuth, requireAdmin } = require('../auth');
46 const { requireAuth, requireAdmin, clearAuthUserCache } = require('../auth');
47 47 const { loadConfig, saveConfig, getEffectivePaths } = require('../config');
48 48 const debugRouter = require('./debug');
49 49
@@ -229,6 +229,7 @@ router.post('/users', (req, res) =>
229 229 try { fs.mkdirSync(folder, { recursive: true }); } catch { /* non-fatal */ }
230 230 }
231 231
232 clearAuthUserCache(username);
232 233 res.status(201).json({ ok: true });
233 234 });
234 235
@@ -279,6 +280,7 @@ router.patch('/users/:id', (req, res) =>
279 280 db.prepare('UPDATE users SET permissions = ? WHERE id = ?').run([JSON.stringify(normalizedPermissions), user.id]);
280 281 }
281 282
283 clearAuthUserCache(user.username);
282 284 res.json({ ok: true });
283 285 });
284 286
@@ -307,6 +309,7 @@ router.delete('/users/:id', (req, res) =>
307 309 }
308 310
309 311 db.prepare('DELETE FROM users WHERE id = ?').run([user.id]);
312 clearAuthUserCache(user.username);
310 313
311 314 // If the deleted user's personal folder is now empty, remove it automatically.
312 315 const config = loadConfig();
@@ -43,7 +43,7 @@ function ensureUserFolders (userPath)
43}
43}
44
44
45const { getDb } = require('../db');
45const { getDb } = require('../db');
46const { requireAuth, requireAdmin } = require('../auth');
46const { requireAuth, requireAdmin, clearAuthUserCache } = require('../auth');
47const { loadConfig, saveConfig, getEffectivePaths } = require('../config');
47const { loadConfig, saveConfig, getEffectivePaths } = require('../config');
48const debugRouter = require('./debug');
48const debugRouter = require('./debug');
49
49
@@ -229,6 +229,7 @@ router.post('/users', (req, res) =>
229 try { fs.mkdirSync(folder, { recursive: true }); } catch { /* non-fatal */ }
229 try { fs.mkdirSync(folder, { recursive: true }); } catch { /* non-fatal */ }
230 }
230 }
231
231
232 clearAuthUserCache(username);
232 res.status(201).json({ ok: true });
233 res.status(201).json({ ok: true });
233});
234});
234
235
@@ -279,6 +280,7 @@ router.patch('/users/:id', (req, res) =>
279 db.prepare('UPDATE users SET permissions = ? WHERE id = ?').run([JSON.stringify(normalizedPermissions), user.id]);
280 db.prepare('UPDATE users SET permissions = ? WHERE id = ?').run([JSON.stringify(normalizedPermissions), user.id]);
280 }
281 }
281
282
283 clearAuthUserCache(user.username);
282 res.json({ ok: true });
284 res.json({ ok: true });
283});
285});
284
286
@@ -307,6 +309,7 @@ router.delete('/users/:id', (req, res) =>
307 }
309 }
308
310
309 db.prepare('DELETE FROM users WHERE id = ?').run([user.id]);
311 db.prepare('DELETE FROM users WHERE id = ?').run([user.id]);
312 clearAuthUserCache(user.username);
310
313
311 // If the deleted user's personal folder is now empty, remove it automatically.
314 // If the deleted user's personal folder is now empty, remove it automatically.
312 const config = loadConfig();
315 const config = loadConfig();
server/routes/files.js +127 −272
@@ -10,34 +10,27 @@ const router = express.Router();
10 10 const { getDb } = require('../db');
11 11 const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');
12 12 const { requireAuth } = require('../auth');
13 const { getCachedThumbnailInfo, getFilesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
13 const {
14 getCachedThumbnailInfo,
15 getFilesWithThumbnailsInDir,
16 getFolderIconSourceFile,
17 setFolderIconAssignment,
18 getThumbnailManifestSourcePathSet,
19 THUMB_HTTP_MAX_AGE_SEC,
20 } = require('../thumbnail-cache');
21 const {
22 searchFiles,
23 deleteIndexPath,
24 deleteIndexUnder,
25 rewriteIndexPaths,
26 reindexDirectoryTree,
27 } = require('../search-index');
14 28 const { logError } = require('../logger');
15 29 const { renderImageSubtitleFrame, renderImageSubtitleFramesInRange } = require('../image-subtitles');
16 30 const { appendSubtitleDebugEntries, SUBTITLE_DEBUG_LOG_DIR } = require('../subtitle-debug-log');
17 31 const { buildZip } = require('./share');
18 32 const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
19
20 // ---------------------------------------------------------------------------
21 // SHA-256 hashing helper
22 // ---------------------------------------------------------------------------
23
24 /**
25 * Computes the SHA-256 hex digest of the file at `filePath`.
26 * Streams the file so it works for large files without loading them into memory.
27 * @param {string} filePath
28 * @returns {Promise<string>} hex digest
29 */
30 function computeFileSha256 (filePath)
31 {
32 return new Promise((resolve, reject) =>
33 {
34 const hash = crypto.createHash('sha256');
35 const stream = fs.createReadStream(filePath);
36 stream.on('error', reject);
37 stream.on('data', (chunk) => hash.update(chunk));
38 stream.on('end', () => resolve(hash.digest('hex')));
39 });
40 }
33 const { enqueueFileHash } = require('../file-hash-queue');
41 34
42 35 // ---------------------------------------------------------------------------
43 36 // Multer storage — files are staged into temporary files and only promoted
@@ -270,22 +263,43 @@ function effectivePaths ()
270 263 return getEffectivePaths(loadConfig().paths || []);
271 264 }
272 265
273 function ensureUserNavigationTable (db)
266 const META_BATCH_SIZE = 400;
267
268 function loadUploadedAtByPaths (db, paths)
269 {
270 const map = new Map();
271 const unique = [...new Set(paths.map((p) => String(p || '')).filter(Boolean))];
272 for (let i = 0; i < unique.length; i += META_BATCH_SIZE)
273 {
274 const chunk = unique.slice(i, i + META_BATCH_SIZE);
275 const placeholders = chunk.map(() => '?').join(', ');
276 const rows = db.prepare(`SELECT file_path, uploaded_at FROM file_meta WHERE file_path IN (${placeholders})`)
277 .all(chunk);
278 for (const row of rows)
279 {
280 map.set(row.file_path, String(row.uploaded_at || '').trim());
281 }
282 }
283 return map;
284 }
285
286 function persistMissingUploadedAt (db, rows)
274 287 {
275 db.exec(`
276 CREATE TABLE IF NOT EXISTS user_last_navigation (
277 username TEXT PRIMARY KEY,
278 last_dir TEXT,
279 updated_at INTEGER NOT NULL
280 );
288 if (!rows.length) return;
289
290 const stmt = db.prepare(`
291 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at)
292 VALUES (?, ?, ?, ?, ?)
293 ON CONFLICT(file_path) DO UPDATE SET
294 uploaded_at = COALESCE(NULLIF(file_meta.uploaded_at, ''), excluded.uploaded_at)
281 295 `);
296 for (const row of rows) stmt.run(row);
282 297 }
283 298
284 299 function setUserLastDir (username, dirPath)
285 300 {
286 301 if (!username) return;
287 302 const db = getDb();
288 ensureUserNavigationTable(db);
289 303 db.prepare(`
290 304 INSERT INTO user_last_navigation (username, last_dir, updated_at)
291 305 VALUES (?, ?, ?)
@@ -299,7 +313,6 @@ function getUserLastDir (username)
299 313 {
300 314 if (!username) return null;
301 315 const db = getDb();
302 ensureUserNavigationTable(db);
303 316 const row = db.prepare('SELECT last_dir FROM user_last_navigation WHERE username = ?').get([String(username)]);
304 317 const value = String(row && row.last_dir || '').trim();
305 318 return value || null;
@@ -309,7 +322,6 @@ function clearUserLastDir (username)
309 322 {
310 323 if (!username) return;
311 324 const db = getDb();
312 ensureUserNavigationTable(db);
313 325 db.prepare('DELETE FROM user_last_navigation WHERE username = ?').run([String(username)]);
314 326 }
315 327
@@ -355,45 +367,6 @@ function readableSearchRootsForUser (user, config, roots)
355 367 });
356 368 }
357 369
358 function fuzzyMatchScore (needle, haystack)
359 {
360 const q = String(needle || '').trim().toLowerCase();
361 const t = String(haystack || '').toLowerCase();
362 if (!q || !t) return null;
363
364 // Strong boost for contiguous substring matches.
365 const idx = t.indexOf(q);
366 if (idx >= 0)
367 {
368 const compactBonus = Math.max(0, 200 - idx);
369 const lengthBonus = Math.max(0, 120 - (t.length - q.length));
370 return 1200 + compactBonus + lengthBonus;
371 }
372
373 // Fallback: subsequence fuzzy match.
374 let qPos = 0;
375 let firstHit = -1;
376 let lastHit = -1;
377 let gaps = 0;
378
379 for (let i = 0; i < t.length && qPos < q.length; i++)
380 {
381 if (t[i] !== q[qPos]) continue;
382 if (firstHit === -1) firstHit = i;
383 if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1);
384 lastHit = i;
385 qPos += 1;
386 }
387
388 if (qPos !== q.length) return null;
389
390 const span = lastHit - firstHit + 1;
391 const compactness = Math.max(0, 300 - span);
392 const gapPenalty = Math.min(180, gaps * 3);
393 const startBonus = Math.max(0, 120 - firstHit);
394 return 700 + compactness + startBonus - gapPenalty;
395 }
396
397 370 const SEARCHABLE_IMAGE_EXTS = new Set([
398 371 '.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.svg', '.avif', '.ico', '.tif', '.tiff',
399 372 ]);
@@ -404,164 +377,28 @@ function isSearchableImageFileName (name)
404 377 return SEARCHABLE_IMAGE_EXTS.has(ext);
405 378 }
406 379
407 function searchFilesInRoots (query, roots, maxResults)
408 {
409 const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
410 const results = [];
411 const visitedDirs = new Set();
412
413 for (const root of roots)
414 {
415 const rootPath = path.resolve(root);
416 const stack = [rootPath];
417
418 while (stack.length)
419 {
420 const dir = stack.pop();
421 if (!dir) continue;
422 const dirKey = path.resolve(dir);
423 if (visitedDirs.has(dirKey)) continue;
424 visitedDirs.add(dirKey);
425
426 let entries;
427 try
428 {
429 entries = fs.readdirSync(dir, { withFileTypes: true });
430 } catch
431 {
432 continue;
433 }
434
435 for (const entry of entries)
436 {
437 const fullPath = path.join(dir, entry.name);
438 if (entry.isDirectory())
439 {
440 stack.push(fullPath);
441 continue;
442 }
443 if (!entry.isFile()) continue;
444
445 const relativePath = path.relative(rootPath, fullPath);
446 const score = fuzzyMatchScore(query, entry.name)
447 ?? fuzzyMatchScore(query, relativePath);
448 if (score == null) continue;
449
450 let stat = null;
451 try { stat = fs.statSync(fullPath); } catch { /* ignore disappearing files */ }
452
453 results.push({
454 name: entry.name,
455 path: fullPath,
456 dir: path.dirname(fullPath),
457 rootPath,
458 relativePath,
459 size: stat ? stat.size : null,
460 mtime: stat ? stat.mtime.toISOString() : null,
461 score,
462 });
463
464 if (results.length > limit * 4)
465 {
466 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
467 results.length = limit * 2;
468 }
469 }
470 }
471 }
472
473 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
474 return results.slice(0, limit);
475 }
476
477 380 async function searchImagesWithThumbnailsInRoots (query, roots, maxResults)
478 381 {
479 382 const limit = Math.max(1, Math.min(120, Number(maxResults) || 60));
480 const maxScannedFiles = 25000;
481 const results = [];
482 const visitedDirs = new Set();
483 let scannedFiles = 0;
484 let truncated = false;
383 const candidateLimit = Math.min(250, Math.max(limit * 4, 80));
384 const indexed = await searchFiles(query, roots, candidateLimit);
385 const thumbnailedPaths = await getThumbnailManifestSourcePathSet();
485 386
486 for (const root of roots)
387 const results = [];
388 for (const item of indexed)
487 389 {
488 if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
489
490 const rootPath = path.resolve(root);
491 const stack = [rootPath];
492
493 while (stack.length)
494 {
495 if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
496
497 const dir = stack.pop();
498 if (!dir) continue;
499 const dirKey = path.resolve(dir);
500 if (visitedDirs.has(dirKey)) continue;
501 visitedDirs.add(dirKey);
502
503 let entries;
504 try
505 {
506 entries = fs.readdirSync(dir, { withFileTypes: true });
507 } catch
508 {
509 continue;
510 }
390 if (!isSearchableImageFileName(item.name)) continue;
391 if (!thumbnailedPaths.has(path.resolve(item.path))) continue;
511 392
512 for (const entry of entries)
513 {
514 if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
515
516 const fullPath = path.join(dir, entry.name);
517 if (entry.isDirectory())
518 {
519 stack.push(fullPath);
520 continue;
521 }
522 if (!entry.isFile()) continue;
523
524 scannedFiles += 1;
525 if (!isSearchableImageFileName(entry.name)) continue;
526
527 const relativePath = path.relative(rootPath, fullPath);
528 const score = fuzzyMatchScore(query, entry.name)
529 ?? fuzzyMatchScore(query, relativePath);
530 if (score == null) continue;
531
532 // Only include files that already have a ready thumbnail.
533 const thumb = await getCachedThumbnailInfo(fullPath);
534 if (!thumb) continue;
535
536 let stat = null;
537 try { stat = fs.statSync(fullPath); } catch { /* ignore disappearing files */ }
538
539 results.push({
540 name: entry.name,
541 path: fullPath,
542 dir: path.dirname(fullPath),
543 rootPath,
544 relativePath,
545 size: stat ? stat.size : null,
546 mtime: stat ? stat.mtime.toISOString() : null,
547 score,
548 });
549
550 if (results.length > limit * 3)
551 {
552 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
553 results.length = limit;
554 }
555 }
556 }
393 const { score, ...rest } = item;
394 results.push(rest);
395 if (results.length >= limit) break;
557 396 }
558 397
559 if (scannedFiles >= maxScannedFiles) truncated = true;
560 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
561 398 return {
562 results: results.slice(0, limit),
563 scannedFiles,
564 truncated,
399 results,
400 scannedFiles: indexed.length,
401 truncated: indexed.length >= candidateLimit,
565 402 };
566 403 }
567 404
@@ -1180,7 +1017,7 @@ router.post('/root-alias', requireAuth, (req, res) =>
1180 1017 // GET /api/files/list?dir=<absolute-path>
1181 1018 // Lists files and folders in the given directory.
1182 1019 // ---------------------------------------------------------------------------
1183 router.get('/list', requireAuth, (req, res) =>
1020 router.get('/list', requireAuth, async (req, res) =>
1184 1021 {
1185 1022 const dir = req.query.dir;
1186 1023 if (!dir) return res.status(400).json({ error: 'dir is required.' });
@@ -1203,38 +1040,48 @@ router.get('/list', requireAuth, (req, res) =>
1203 1040 let entries;
1204 1041 try
1205 1042 {
1206 entries = fs.readdirSync(dir, { withFileTypes: true });
1043 entries = await fs.promises.readdir(dir, { withFileTypes: true });
1207 1044 } catch
1208 1045 {
1209 1046 return res.status(404).json({ error: 'Directory not found.' });
1210 1047 }
1211 1048
1212 const items = entries.map(e =>
1049 const fullPaths = entries.map((e) => path.join(dir, e.name));
1050 const db = getDb();
1051 const uploadedAtByPath = loadUploadedAtByPaths(db, fullPaths);
1052 const metaUpserts = [];
1053
1054 const statResults = await Promise.all(fullPaths.map(async (fullPath) =>
1055 {
1056 try
1057 {
1058 return { fullPath, stat: await fs.promises.stat(fullPath) };
1059 } catch
1060 {
1061 return { fullPath, stat: null };
1062 }
1063 }));
1064
1065 const statByPath = new Map(statResults.map((row) => [row.fullPath, row.stat]));
1066
1067 const items = entries.map((e) =>
1213 1068 {
1214 1069 const fullPath = path.join(dir, e.name);
1070 const stat = statByPath.get(fullPath);
1215 1071 let size = null;
1216 1072 let mtime = null;
1217 1073 let uploadedAt = null;
1218 try
1074
1075 if (stat)
1219 1076 {
1220 const stat = fs.statSync(fullPath);
1221 1077 size = stat.size;
1222 1078 mtime = stat.mtime.toISOString();
1223
1224 const db = getDb();
1225 const existing = db.prepare('SELECT uploaded_at FROM file_meta WHERE file_path = ?').get([fullPath]);
1226 uploadedAt = String(existing && existing.uploaded_at || '').trim() || uploadedAtFromStat(stat);
1227
1228 if (uploadedAt)
1079 uploadedAt = uploadedAtByPath.get(fullPath) || uploadedAtFromStat(stat) || null;
1080 if (uploadedAt && !uploadedAtByPath.has(fullPath))
1229 1081 {
1230 db.prepare(`
1231 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at)
1232 VALUES (?, ?, ?, ?, ?)
1233 ON CONFLICT(file_path) DO UPDATE SET
1234 uploaded_at = COALESCE(NULLIF(file_meta.uploaded_at, ''), excluded.uploaded_at)
1235 `).run([fullPath, '', '[]', '{}', uploadedAt]);
1082 metaUpserts.push([fullPath, '', '[]', '{}', uploadedAt]);
1236 1083 }
1237 } catch { /* ignore */ }
1084 }
1238 1085
1239 1086 return {
1240 1087 name: e.name,
@@ -1246,6 +1093,14 @@ router.get('/list', requireAuth, (req, res) =>
1246 1093 };
1247 1094 });
1248 1095
1096 try
1097 {
1098 persistMissingUploadedAt(db, metaUpserts);
1099 } catch
1100 {
1101 // Non-fatal metadata persistence.
1102 }
1103
1249 1104 try
1250 1105 {
1251 1106 setUserLastDir(req.user && req.user.username, dir);
@@ -1261,7 +1116,7 @@ router.get('/list', requireAuth, (req, res) =>
1261 1116 // GET /api/files/search?query=<text>&limit=<number>
1262 1117 // Fuzzy-searches all readable roots for files.
1263 1118 // ---------------------------------------------------------------------------
1264 router.get('/search', requireAuth, (req, res) =>
1119 router.get('/search', requireAuth, async (req, res) =>
1265 1120 {
1266 1121 const query = String(req.query.query || '').trim();
1267 1122 if (!query) return res.status(400).json({ error: 'query is required.' });
@@ -1279,7 +1134,7 @@ router.get('/search', requireAuth, (req, res) =>
1279 1134 }
1280 1135 });
1281 1136
1282 const results = searchFilesInRoots(query, readableRoots, req.query.limit);
1137 const results = await searchFiles(query, readableRoots, req.query.limit);
1283 1138 res.json({
1284 1139 query,
1285 1140 results,
@@ -1367,26 +1222,10 @@ router.get('/meta', requireAuth, async (req, res) =>
1367 1222 }
1368 1223 }
1369 1224
1370 // Lazily compute and persist the SHA-256 hash when it is absent.
1371 1225 let sha256 = meta ? (meta.sha256 || null) : null;
1372 1226 if (!sha256 && !stat.isDirectory())
1373 1227 {
1374 try
1375 {
1376 sha256 = await computeFileSha256(file);
1377 if (meta)
1378 {
1379 db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, file]);
1380 } else
1381 {
1382 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')
1383 .run([file, '', '[]', '{}', uploadedAt || uploadedAtFromStat(stat), sha256]);
1384 meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
1385 }
1386 } catch
1387 {
1388 // Non-fatal — hash will be retried next time.
1389 }
1228 enqueueFileHash(file);
1390 1229 }
1391 1230
1392 1231 res.json({
@@ -1422,6 +1261,7 @@ router.get('/duplicates', requireAuth, (req, res) =>
1422 1261 const hash = row ? (row.sha256 || null) : null;
1423 1262 if (!hash)
1424 1263 {
1264 enqueueFileHash(file);
1425 1265 return res.json({ duplicates: [] });
1426 1266 }
1427 1267
@@ -1990,7 +1830,7 @@ router.get('/thumbnail', requireAuth, async (req, res) =>
1990 1830 }
1991 1831
1992 1832 res.type(thumb.contentType);
1993 res.set('Cache-Control', 'private, max-age=10');
1833 res.set('Cache-Control', `private, max-age=${THUMB_HTTP_MAX_AGE_SEC}`);
1994 1834 res.sendFile(thumb.thumbPath);
1995 1835 });
1996 1836
@@ -2049,26 +1889,27 @@ router.post('/upload', requireAuth, (req, res, next) =>
2049 1889 const filePath = path.join(dir, f.originalname);
2050 1890 const uploadedAt = new Date().toISOString();
2051 1891
2052 let sha256 = null;
2053 try
2054 {
2055 sha256 = await computeFileSha256(filePath);
2056 } catch
2057 {
2058 // Non-fatal — hash can be computed lazily when the file is accessed.
2059 }
2060
2061 1892 const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);
2062 1893 if (existing)
2063 1894 {
2064 db.prepare('UPDATE file_meta SET uploader = ?, sha256 = ?, uploaded_at = ? WHERE file_path = ?')
2065 .run([req.user.username, sha256, uploadedAt, filePath]);
1895 db.prepare('UPDATE file_meta SET uploader = ?, sha256 = NULL, uploaded_at = ? WHERE file_path = ?')
1896 .run([req.user.username, uploadedAt, filePath]);
2066 1897 } else
2067 1898 {
2068 1899 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')
2069 .run([filePath, req.user.username, '[]', '{}', uploadedAt, sha256]);
1900 .run([filePath, req.user.username, '[]', '{}', uploadedAt, null]);
2070 1901 }
1902
1903 enqueueFileHash(filePath);
1904 }
1905 try
1906 {
1907 await reindexDirectoryTree(dir);
1908 } catch
1909 {
1910 // Non-fatal search index refresh.
2071 1911 }
1912
2072 1913 res.json({ uploaded: (req.files || []).map(f => f.originalname) });
2073 1914 });
2074 1915 });
@@ -2112,6 +1953,7 @@ router.post('/rename', requireAuth, (req, res) =>
2112 1953 }
2113 1954
2114 1955 rewriteMetaPaths(from, dest);
1956 rewriteIndexPaths(from, dest);
2115 1957
2116 1958 res.json({ ok: true });
2117 1959 });
@@ -2223,6 +2065,7 @@ router.post('/move', requireAuth, (req, res) =>
2223 2065 }
2224 2066
2225 2067 rewriteMetaPaths(from, dest);
2068 rewriteIndexPaths(from, dest);
2226 2069 res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });
2227 2070 });
2228 2071
@@ -2306,6 +2149,8 @@ router.post('/delete', requireAuth, (req, res) =>
2306 2149 db.prepare("DELETE FROM file_meta WHERE file_path = ? OR file_path LIKE ?")
2307 2150 .run([target, target + path.sep + '%']);
2308 2151
2152 deleteIndexUnder(target);
2153
2309 2154 res.json({ ok: true });
2310 2155 });
2311 2156
@@ -2482,7 +2327,7 @@ router.post('/global-favicon', requireAuth, async (req, res) =>
2482 2327 // Copies a file (not a directory) to the destination folder.
2483 2328 // Supports conflict: 'error' (default) | 'overwrite' | 'rename'
2484 2329 // ---------------------------------------------------------------------------
2485 router.post('/copy', requireAuth, (req, res) =>
2330 router.post('/copy', requireAuth, async (req, res) =>
2486 2331 {
2487 2332 const { from, toDir } = req.body;
2488 2333 const conflict = String(req.body && req.body.conflict || 'error').toLowerCase();
@@ -2562,6 +2407,16 @@ router.post('/copy', requireAuth, (req, res) =>
2562 2407 return res.status(500).json({ error: err.message });
2563 2408 }
2564 2409
2410 try
2411 {
2412 await reindexDirectoryTree(toDir);
2413 } catch
2414 {
2415 // Non-fatal search index refresh.
2416 }
2417
2418 enqueueFileHash(dest);
2419
2565 2420 res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });
2566 2421 });
2567 2422
@@ -10,34 +10,27 @@ const router = express.Router();
10const { getDb } = require('../db');
10const { getDb } = require('../db');
11const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');
11const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');
12const { requireAuth } = require('../auth');
12const { requireAuth } = require('../auth');
13const { getCachedThumbnailInfo, getFilesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
13const {
14 getCachedThumbnailInfo,
15 getFilesWithThumbnailsInDir,
16 getFolderIconSourceFile,
17 setFolderIconAssignment,
18 getThumbnailManifestSourcePathSet,
19 THUMB_HTTP_MAX_AGE_SEC,
20} = require('../thumbnail-cache');
21const {
22 searchFiles,
23 deleteIndexPath,
24 deleteIndexUnder,
25 rewriteIndexPaths,
26 reindexDirectoryTree,
27} = require('../search-index');
14const { logError } = require('../logger');
28const { logError } = require('../logger');
15const { renderImageSubtitleFrame, renderImageSubtitleFramesInRange } = require('../image-subtitles');
29const { renderImageSubtitleFrame, renderImageSubtitleFramesInRange } = require('../image-subtitles');
16const { appendSubtitleDebugEntries, SUBTITLE_DEBUG_LOG_DIR } = require('../subtitle-debug-log');
30const { appendSubtitleDebugEntries, SUBTITLE_DEBUG_LOG_DIR } = require('../subtitle-debug-log');
17const { buildZip } = require('./share');
31const { buildZip } = require('./share');
18const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
32const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
19
33const { enqueueFileHash } = require('../file-hash-queue');
20// ---------------------------------------------------------------------------
21// SHA-256 hashing helper
22// ---------------------------------------------------------------------------
23
24/**
25 * Computes the SHA-256 hex digest of the file at `filePath`.
26 * Streams the file so it works for large files without loading them into memory.
27 * @param {string} filePath
28 * @returns {Promise<string>} hex digest
29 */
30function computeFileSha256 (filePath)
31{
32 return new Promise((resolve, reject) =>
33 {
34 const hash = crypto.createHash('sha256');
35 const stream = fs.createReadStream(filePath);
36 stream.on('error', reject);
37 stream.on('data', (chunk) => hash.update(chunk));
38 stream.on('end', () => resolve(hash.digest('hex')));
39 });
40}
41
34
42// ---------------------------------------------------------------------------
35// ---------------------------------------------------------------------------
43// Multer storage — files are staged into temporary files and only promoted
36// Multer storage — files are staged into temporary files and only promoted
@@ -270,22 +263,43 @@ function effectivePaths ()
270 return getEffectivePaths(loadConfig().paths || []);
263 return getEffectivePaths(loadConfig().paths || []);
271}
264}
272
265
273function ensureUserNavigationTable (db)
266const META_BATCH_SIZE = 400;
267
268function loadUploadedAtByPaths (db, paths)
269{
270 const map = new Map();
271 const unique = [...new Set(paths.map((p) => String(p || '')).filter(Boolean))];
272 for (let i = 0; i < unique.length; i += META_BATCH_SIZE)
273 {
274 const chunk = unique.slice(i, i + META_BATCH_SIZE);
275 const placeholders = chunk.map(() => '?').join(', ');
276 const rows = db.prepare(`SELECT file_path, uploaded_at FROM file_meta WHERE file_path IN (${placeholders})`)
277 .all(chunk);
278 for (const row of rows)
279 {
280 map.set(row.file_path, String(row.uploaded_at || '').trim());
281 }
282 }
283 return map;
284}
285
286function persistMissingUploadedAt (db, rows)
274{
287{
275 db.exec(`
288 if (!rows.length) return;
276 CREATE TABLE IF NOT EXISTS user_last_navigation (
289
277 username TEXT PRIMARY KEY,
290 const stmt = db.prepare(`
278 last_dir TEXT,
291 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at)
279 updated_at INTEGER NOT NULL
292 VALUES (?, ?, ?, ?, ?)
280 );
293 ON CONFLICT(file_path) DO UPDATE SET
294 uploaded_at = COALESCE(NULLIF(file_meta.uploaded_at, ''), excluded.uploaded_at)
281 `);
295 `);
296 for (const row of rows) stmt.run(row);
282}
297}
283
298
284function setUserLastDir (username, dirPath)
299function setUserLastDir (username, dirPath)
285{
300{
286 if (!username) return;
301 if (!username) return;
287 const db = getDb();
302 const db = getDb();
288 ensureUserNavigationTable(db);
289 db.prepare(`
303 db.prepare(`
290 INSERT INTO user_last_navigation (username, last_dir, updated_at)
304 INSERT INTO user_last_navigation (username, last_dir, updated_at)
291 VALUES (?, ?, ?)
305 VALUES (?, ?, ?)
@@ -299,7 +313,6 @@ function getUserLastDir (username)
299{
313{
300 if (!username) return null;
314 if (!username) return null;
301 const db = getDb();
315 const db = getDb();
302 ensureUserNavigationTable(db);
303 const row = db.prepare('SELECT last_dir FROM user_last_navigation WHERE username = ?').get([String(username)]);
316 const row = db.prepare('SELECT last_dir FROM user_last_navigation WHERE username = ?').get([String(username)]);
304 const value = String(row && row.last_dir || '').trim();
317 const value = String(row && row.last_dir || '').trim();
305 return value || null;
318 return value || null;
@@ -309,7 +322,6 @@ function clearUserLastDir (username)
309{
322{
310 if (!username) return;
323 if (!username) return;
311 const db = getDb();
324 const db = getDb();
312 ensureUserNavigationTable(db);
313 db.prepare('DELETE FROM user_last_navigation WHERE username = ?').run([String(username)]);
325 db.prepare('DELETE FROM user_last_navigation WHERE username = ?').run([String(username)]);
314}
326}
315
327
@@ -355,45 +367,6 @@ function readableSearchRootsForUser (user, config, roots)
355 });
367 });
356}
368}
357
369
358function fuzzyMatchScore (needle, haystack)
359{
360 const q = String(needle || '').trim().toLowerCase();
361 const t = String(haystack || '').toLowerCase();
362 if (!q || !t) return null;
363
364 // Strong boost for contiguous substring matches.
365 const idx = t.indexOf(q);
366 if (idx >= 0)
367 {
368 const compactBonus = Math.max(0, 200 - idx);
369 const lengthBonus = Math.max(0, 120 - (t.length - q.length));
370 return 1200 + compactBonus + lengthBonus;
371 }
372
373 // Fallback: subsequence fuzzy match.
374 let qPos = 0;
375 let firstHit = -1;
376 let lastHit = -1;
377 let gaps = 0;
378
379 for (let i = 0; i < t.length && qPos < q.length; i++)
380 {
381 if (t[i] !== q[qPos]) continue;
382 if (firstHit === -1) firstHit = i;
383 if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1);
384 lastHit = i;
385 qPos += 1;
386 }
387
388 if (qPos !== q.length) return null;
389
390 const span = lastHit - firstHit + 1;
391 const compactness = Math.max(0, 300 - span);
392 const gapPenalty = Math.min(180, gaps * 3);
393 const startBonus = Math.max(0, 120 - firstHit);
394 return 700 + compactness + startBonus - gapPenalty;
395}
396
397const SEARCHABLE_IMAGE_EXTS = new Set([
370const SEARCHABLE_IMAGE_EXTS = new Set([
398 '.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.svg', '.avif', '.ico', '.tif', '.tiff',
371 '.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.svg', '.avif', '.ico', '.tif', '.tiff',
399]);
372]);
@@ -404,164 +377,28 @@ function isSearchableImageFileName (name)
404 return SEARCHABLE_IMAGE_EXTS.has(ext);
377 return SEARCHABLE_IMAGE_EXTS.has(ext);
405}
378}
406
379
407function searchFilesInRoots (query, roots, maxResults)
408{
409 const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
410 const results = [];
411 const visitedDirs = new Set();
412
413 for (const root of roots)
414 {
415 const rootPath = path.resolve(root);
416 const stack = [rootPath];
417
418 while (stack.length)
419 {
420 const dir = stack.pop();
421 if (!dir) continue;
422 const dirKey = path.resolve(dir);
423 if (visitedDirs.has(dirKey)) continue;
424 visitedDirs.add(dirKey);
425
426 let entries;
427 try
428 {
429 entries = fs.readdirSync(dir, { withFileTypes: true });
430 } catch
431 {
432 continue;
433 }
434
435 for (const entry of entries)
436 {
437 const fullPath = path.join(dir, entry.name);
438 if (entry.isDirectory())
439 {
440 stack.push(fullPath);
441 continue;
442 }
443 if (!entry.isFile()) continue;
444
445 const relativePath = path.relative(rootPath, fullPath);
446 const score = fuzzyMatchScore(query, entry.name)
447 ?? fuzzyMatchScore(query, relativePath);
448 if (score == null) continue;
449
450 let stat = null;
451 try { stat = fs.statSync(fullPath); } catch { /* ignore disappearing files */ }
452
453 results.push({
454 name: entry.name,
455 path: fullPath,
456 dir: path.dirname(fullPath),
457 rootPath,
458 relativePath,
459 size: stat ? stat.size : null,
460 mtime: stat ? stat.mtime.toISOString() : null,
461 score,
462 });
463
464 if (results.length > limit * 4)
465 {
466 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
467 results.length = limit * 2;
468 }
469 }
470 }
471 }
472
473 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
474 return results.slice(0, limit);
475}
476
477async function searchImagesWithThumbnailsInRoots (query, roots, maxResults)
380async function searchImagesWithThumbnailsInRoots (query, roots, maxResults)
478{
381{
479 const limit = Math.max(1, Math.min(120, Number(maxResults) || 60));
382 const limit = Math.max(1, Math.min(120, Number(maxResults) || 60));
480 const maxScannedFiles = 25000;
383 const candidateLimit = Math.min(250, Math.max(limit * 4, 80));
481 const results = [];
384 const indexed = await searchFiles(query, roots, candidateLimit);
482 const visitedDirs = new Set();
385 const thumbnailedPaths = await getThumbnailManifestSourcePathSet();
483 let scannedFiles = 0;
484 let truncated = false;
485
386
486 for (const root of roots)
387 const results = [];
388 for (const item of indexed)
487 {
389 {
488 if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
390 if (!isSearchableImageFileName(item.name)) continue;
489
391 if (!thumbnailedPaths.has(path.resolve(item.path))) continue;
490 const rootPath = path.resolve(root);
491 const stack = [rootPath];
492
493 while (stack.length)
494 {
495 if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
496
497 const dir = stack.pop();
498 if (!dir) continue;
499 const dirKey = path.resolve(dir);
500 if (visitedDirs.has(dirKey)) continue;
501 visitedDirs.add(dirKey);
502
503 let entries;
504 try
505 {
506 entries = fs.readdirSync(dir, { withFileTypes: true });
507 } catch
508 {
509 continue;
510 }
511
392
512 for (const entry of entries)
393 const { score, ...rest } = item;
513 {
394 results.push(rest);
514 if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
395 if (results.length >= limit) break;
515
516 const fullPath = path.join(dir, entry.name);
517 if (entry.isDirectory())
518 {
519 stack.push(fullPath);
520 continue;
521 }
522 if (!entry.isFile()) continue;
523
524 scannedFiles += 1;
525 if (!isSearchableImageFileName(entry.name)) continue;
526
527 const relativePath = path.relative(rootPath, fullPath);
528 const score = fuzzyMatchScore(query, entry.name)
529 ?? fuzzyMatchScore(query, relativePath);
530 if (score == null) continue;
531
532 // Only include files that already have a ready thumbnail.
533 const thumb = await getCachedThumbnailInfo(fullPath);
534 if (!thumb) continue;
535
536 let stat = null;
537 try { stat = fs.statSync(fullPath); } catch { /* ignore disappearing files */ }
538
539 results.push({
540 name: entry.name,
541 path: fullPath,
542 dir: path.dirname(fullPath),
543 rootPath,
544 relativePath,
545 size: stat ? stat.size : null,
546 mtime: stat ? stat.mtime.toISOString() : null,
547 score,
548 });
549
550 if (results.length > limit * 3)
551 {
552 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
553 results.length = limit;
554 }
555 }
556 }
557 }
396 }
558
397
559 if (scannedFiles >= maxScannedFiles) truncated = true;
560 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
561 return {
398 return {
562 results: results.slice(0, limit),
399 results,
563 scannedFiles,
400 scannedFiles: indexed.length,
564 truncated,
401 truncated: indexed.length >= candidateLimit,
565 };
402 };
566}
403}
567
404
@@ -1180,7 +1017,7 @@ router.post('/root-alias', requireAuth, (req, res) =>
1180// GET /api/files/list?dir=<absolute-path>
1017// GET /api/files/list?dir=<absolute-path>
1181// Lists files and folders in the given directory.
1018// Lists files and folders in the given directory.
1182// ---------------------------------------------------------------------------
1019// ---------------------------------------------------------------------------
1183router.get('/list', requireAuth, (req, res) =>
1020router.get('/list', requireAuth, async (req, res) =>
1184{
1021{
1185 const dir = req.query.dir;
1022 const dir = req.query.dir;
1186 if (!dir) return res.status(400).json({ error: 'dir is required.' });
1023 if (!dir) return res.status(400).json({ error: 'dir is required.' });
@@ -1203,38 +1040,48 @@ router.get('/list', requireAuth, (req, res) =>
1203 let entries;
1040 let entries;
1204 try
1041 try
1205 {
1042 {
1206 entries = fs.readdirSync(dir, { withFileTypes: true });
1043 entries = await fs.promises.readdir(dir, { withFileTypes: true });
1207 } catch
1044 } catch
1208 {
1045 {
1209 return res.status(404).json({ error: 'Directory not found.' });
1046 return res.status(404).json({ error: 'Directory not found.' });
1210 }
1047 }
1211
1048
1212 const items = entries.map(e =>
1049 const fullPaths = entries.map((e) => path.join(dir, e.name));
1050 const db = getDb();
1051 const uploadedAtByPath = loadUploadedAtByPaths(db, fullPaths);
1052 const metaUpserts = [];
1053
1054 const statResults = await Promise.all(fullPaths.map(async (fullPath) =>
1055 {
1056 try
1057 {
1058 return { fullPath, stat: await fs.promises.stat(fullPath) };
1059 } catch
1060 {
1061 return { fullPath, stat: null };
1062 }
1063 }));
1064
1065 const statByPath = new Map(statResults.map((row) => [row.fullPath, row.stat]));
1066
1067 const items = entries.map((e) =>
1213 {
1068 {
1214 const fullPath = path.join(dir, e.name);
1069 const fullPath = path.join(dir, e.name);
1070 const stat = statByPath.get(fullPath);
1215 let size = null;
1071 let size = null;
1216 let mtime = null;
1072 let mtime = null;
1217 let uploadedAt = null;
1073 let uploadedAt = null;
1218 try
1074
1075 if (stat)
1219 {
1076 {
1220 const stat = fs.statSync(fullPath);
1221 size = stat.size;
1077 size = stat.size;
1222 mtime = stat.mtime.toISOString();
1078 mtime = stat.mtime.toISOString();
1223
1079 uploadedAt = uploadedAtByPath.get(fullPath) || uploadedAtFromStat(stat) || null;
1224 const db = getDb();
1080 if (uploadedAt && !uploadedAtByPath.has(fullPath))
1225 const existing = db.prepare('SELECT uploaded_at FROM file_meta WHERE file_path = ?').get([fullPath]);
1226 uploadedAt = String(existing && existing.uploaded_at || '').trim() || uploadedAtFromStat(stat);
1227
1228 if (uploadedAt)
1229 {
1081 {
1230 db.prepare(`
1082 metaUpserts.push([fullPath, '', '[]', '{}', uploadedAt]);
1231 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at)
1232 VALUES (?, ?, ?, ?, ?)
1233 ON CONFLICT(file_path) DO UPDATE SET
1234 uploaded_at = COALESCE(NULLIF(file_meta.uploaded_at, ''), excluded.uploaded_at)
1235 `).run([fullPath, '', '[]', '{}', uploadedAt]);
1236 }
1083 }
1237 } catch { /* ignore */ }
1084 }
1238
1085
1239 return {
1086 return {
1240 name: e.name,
1087 name: e.name,
@@ -1246,6 +1093,14 @@ router.get('/list', requireAuth, (req, res) =>
1246 };
1093 };
1247 });
1094 });
1248
1095
1096 try
1097 {
1098 persistMissingUploadedAt(db, metaUpserts);
1099 } catch
1100 {
1101 // Non-fatal metadata persistence.
1102 }
1103
1249 try
1104 try
1250 {
1105 {
1251 setUserLastDir(req.user && req.user.username, dir);
1106 setUserLastDir(req.user && req.user.username, dir);
@@ -1261,7 +1116,7 @@ router.get('/list', requireAuth, (req, res) =>
1261// GET /api/files/search?query=<text>&limit=<number>
1116// GET /api/files/search?query=<text>&limit=<number>
1262// Fuzzy-searches all readable roots for files.
1117// Fuzzy-searches all readable roots for files.
1263// ---------------------------------------------------------------------------
1118// ---------------------------------------------------------------------------
1264router.get('/search', requireAuth, (req, res) =>
1119router.get('/search', requireAuth, async (req, res) =>
1265{
1120{
1266 const query = String(req.query.query || '').trim();
1121 const query = String(req.query.query || '').trim();
1267 if (!query) return res.status(400).json({ error: 'query is required.' });
1122 if (!query) return res.status(400).json({ error: 'query is required.' });
@@ -1279,7 +1134,7 @@ router.get('/search', requireAuth, (req, res) =>
1279 }
1134 }
1280 });
1135 });
1281
1136
1282 const results = searchFilesInRoots(query, readableRoots, req.query.limit);
1137 const results = await searchFiles(query, readableRoots, req.query.limit);
1283 res.json({
1138 res.json({
1284 query,
1139 query,
1285 results,
1140 results,
@@ -1367,26 +1222,10 @@ router.get('/meta', requireAuth, async (req, res) =>
1367 }
1222 }
1368 }
1223 }
1369
1224
1370 // Lazily compute and persist the SHA-256 hash when it is absent.
1371 let sha256 = meta ? (meta.sha256 || null) : null;
1225 let sha256 = meta ? (meta.sha256 || null) : null;
1372 if (!sha256 && !stat.isDirectory())
1226 if (!sha256 && !stat.isDirectory())
1373 {
1227 {
1374 try
1228 enqueueFileHash(file);
1375 {
1376 sha256 = await computeFileSha256(file);
1377 if (meta)
1378 {
1379 db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, file]);
1380 } else
1381 {
1382 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')
1383 .run([file, '', '[]', '{}', uploadedAt || uploadedAtFromStat(stat), sha256]);
1384 meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
1385 }
1386 } catch
1387 {
1388 // Non-fatal — hash will be retried next time.
1389 }
1390 }
1229 }
1391
1230
1392 res.json({
1231 res.json({
@@ -1422,6 +1261,7 @@ router.get('/duplicates', requireAuth, (req, res) =>
1422 const hash = row ? (row.sha256 || null) : null;
1261 const hash = row ? (row.sha256 || null) : null;
1423 if (!hash)
1262 if (!hash)
1424 {
1263 {
1264 enqueueFileHash(file);
1425 return res.json({ duplicates: [] });
1265 return res.json({ duplicates: [] });
1426 }
1266 }
1427
1267
@@ -1990,7 +1830,7 @@ router.get('/thumbnail', requireAuth, async (req, res) =>
1990 }
1830 }
1991
1831
1992 res.type(thumb.contentType);
1832 res.type(thumb.contentType);
1993 res.set('Cache-Control', 'private, max-age=10');
1833 res.set('Cache-Control', `private, max-age=${THUMB_HTTP_MAX_AGE_SEC}`);
1994 res.sendFile(thumb.thumbPath);
1834 res.sendFile(thumb.thumbPath);
1995});
1835});
1996
1836
@@ -2049,26 +1889,27 @@ router.post('/upload', requireAuth, (req, res, next) =>
2049 const filePath = path.join(dir, f.originalname);
1889 const filePath = path.join(dir, f.originalname);
2050 const uploadedAt = new Date().toISOString();
1890 const uploadedAt = new Date().toISOString();
2051
1891
2052 let sha256 = null;
2053 try
2054 {
2055 sha256 = await computeFileSha256(filePath);
2056 } catch
2057 {
2058 // Non-fatal — hash can be computed lazily when the file is accessed.
2059 }
2060
2061 const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);
1892 const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);
2062 if (existing)
1893 if (existing)
2063 {
1894 {
2064 db.prepare('UPDATE file_meta SET uploader = ?, sha256 = ?, uploaded_at = ? WHERE file_path = ?')
1895 db.prepare('UPDATE file_meta SET uploader = ?, sha256 = NULL, uploaded_at = ? WHERE file_path = ?')
2065 .run([req.user.username, sha256, uploadedAt, filePath]);
1896 .run([req.user.username, uploadedAt, filePath]);
2066 } else
1897 } else
2067 {
1898 {
2068 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')
1899 db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')
2069 .run([filePath, req.user.username, '[]', '{}', uploadedAt, sha256]);
1900 .run([filePath, req.user.username, '[]', '{}', uploadedAt, null]);
2070 }
1901 }
1902
1903 enqueueFileHash(filePath);
1904 }
1905 try
1906 {
1907 await reindexDirectoryTree(dir);
1908 } catch
1909 {
1910 // Non-fatal search index refresh.
2071 }
1911 }
1912
2072 res.json({ uploaded: (req.files || []).map(f => f.originalname) });
1913 res.json({ uploaded: (req.files || []).map(f => f.originalname) });
2073 });
1914 });
2074});
1915});
@@ -2112,6 +1953,7 @@ router.post('/rename', requireAuth, (req, res) =>
2112 }
1953 }
2113
1954
2114 rewriteMetaPaths(from, dest);
1955 rewriteMetaPaths(from, dest);
1956 rewriteIndexPaths(from, dest);
2115
1957
2116 res.json({ ok: true });
1958 res.json({ ok: true });
2117});
1959});
@@ -2223,6 +2065,7 @@ router.post('/move', requireAuth, (req, res) =>
2223 }
2065 }
2224
2066
2225 rewriteMetaPaths(from, dest);
2067 rewriteMetaPaths(from, dest);
2068 rewriteIndexPaths(from, dest);
2226 res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });
2069 res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });
2227});
2070});
2228
2071
@@ -2306,6 +2149,8 @@ router.post('/delete', requireAuth, (req, res) =>
2306 db.prepare("DELETE FROM file_meta WHERE file_path = ? OR file_path LIKE ?")
2149 db.prepare("DELETE FROM file_meta WHERE file_path = ? OR file_path LIKE ?")
2307 .run([target, target + path.sep + '%']);
2150 .run([target, target + path.sep + '%']);
2308
2151
2152 deleteIndexUnder(target);
2153
2309 res.json({ ok: true });
2154 res.json({ ok: true });
2310});
2155});
2311
2156
@@ -2482,7 +2327,7 @@ router.post('/global-favicon', requireAuth, async (req, res) =>
2482// Copies a file (not a directory) to the destination folder.
2327// Copies a file (not a directory) to the destination folder.
2483// Supports conflict: 'error' (default) | 'overwrite' | 'rename'
2328// Supports conflict: 'error' (default) | 'overwrite' | 'rename'
2484// ---------------------------------------------------------------------------
2329// ---------------------------------------------------------------------------
2485router.post('/copy', requireAuth, (req, res) =>
2330router.post('/copy', requireAuth, async (req, res) =>
2486{
2331{
2487 const { from, toDir } = req.body;
2332 const { from, toDir } = req.body;
2488 const conflict = String(req.body && req.body.conflict || 'error').toLowerCase();
2333 const conflict = String(req.body && req.body.conflict || 'error').toLowerCase();
@@ -2562,6 +2407,16 @@ router.post('/copy', requireAuth, (req, res) =>
2562 return res.status(500).json({ error: err.message });
2407 return res.status(500).json({ error: err.message });
2563 }
2408 }
2564
2409
2410 try
2411 {
2412 await reindexDirectoryTree(toDir);
2413 } catch
2414 {
2415 // Non-fatal search index refresh.
2416 }
2417
2418 enqueueFileHash(dest);
2419
2565 res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });
2420 res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });
2566});
2421});
2567
2422
server/search-index.js added +818 −0
@@ -0,0 +1,818 @@
1 'use strict';
2
3 const fs = require('fs');
4 const path = require('path');
5 const { getDb } = require('./db');
6 const { loadConfig, getEffectivePaths } = require('./config');
7
8 const ACTIVE_CLIENT_GRACE_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_CLIENT_GRACE_MS, 5 * 60 * 1000);
9 const IDLE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_IDLE_SYNC_INTERVAL_MS, 30 * 60 * 1000);
10 const ACTIVE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_SYNC_INTERVAL_MS, 10 * 60 * 1000);
11 const MAX_CANDIDATES_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_CANDIDATES, 8000);
12 const MAX_SUBSEQUENCE_SCAN = toPositiveInt(process.env.SEARCH_INDEX_MAX_SUBSEQUENCE_SCAN, 12000);
13 const MAX_VALIDATE_HITS = toPositiveInt(process.env.SEARCH_INDEX_MAX_VALIDATE_HITS, 250);
14 const MAX_RESCAN_DIRS_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_RESCAN_DIRS, 12);
15 const INDEX_BATCH_SIZE = 400;
16 const ROOT_CHUNK_SIZE = 50;
17
18 let loopTimer = null;
19 let loopStarted = false;
20 let backgroundSyncRunning = false;
21
22 function toPositiveInt (value, fallback)
23 {
24 const parsed = Number.parseInt(String(value || ''), 10);
25 return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
26 }
27
28 function yieldToEventLoop ()
29 {
30 return new Promise((resolve) => setImmediate(resolve));
31 }
32
33 function nowMs ()
34 {
35 return Date.now();
36 }
37
38 function fuzzyMatchScore (needle, haystack)
39 {
40 const q = String(needle || '').trim().toLowerCase();
41 const t = String(haystack || '').toLowerCase();
42 if (!q || !t) return null;
43
44 const idx = t.indexOf(q);
45 if (idx >= 0)
46 {
47 const compactBonus = Math.max(0, 200 - idx);
48 const lengthBonus = Math.max(0, 120 - (t.length - q.length));
49 return 1200 + compactBonus + lengthBonus;
50 }
51
52 let qPos = 0;
53 let firstHit = -1;
54 let lastHit = -1;
55 let gaps = 0;
56
57 for (let i = 0; i < t.length && qPos < q.length; i++)
58 {
59 if (t[i] !== q[qPos]) continue;
60 if (firstHit === -1) firstHit = i;
61 if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1);
62 lastHit = i;
63 qPos += 1;
64 }
65
66 if (qPos !== q.length) return null;
67
68 const span = lastHit - firstHit + 1;
69 const compactness = Math.max(0, 300 - span);
70 const gapPenalty = Math.min(180, gaps * 3);
71 const startBonus = Math.max(0, 120 - firstHit);
72 return 700 + compactness + startBonus - gapPenalty;
73 }
74
75 function effectiveIndexRoots ()
76 {
77 const config = loadConfig();
78 const roots = getEffectivePaths(config.paths || []);
79 const userPath = typeof config.userPath === 'string' ? config.userPath.trim() : '';
80 if (userPath) roots.push(userPath);
81
82 const seen = new Set();
83 const deduped = [];
84 for (const root of roots)
85 {
86 const key = path.resolve(root);
87 if (seen.has(key)) continue;
88 seen.add(key);
89 deduped.push(key);
90 }
91 return deduped;
92 }
93
94 function hasActiveClients ()
95 {
96 try
97 {
98 const { getActiveStreamCount } = require('./video-stream');
99 if (Number(getActiveStreamCount()) > 0) return true;
100 } catch
101 {
102 // Stream module unavailable — treat as no active streams.
103 }
104
105 const db = getDb();
106 const cutoff = nowMs() - ACTIVE_CLIENT_GRACE_MS;
107 const row = db.prepare(`
108 SELECT COUNT(*) AS n
109 FROM auth_sessions
110 WHERE revoked_at IS NULL AND last_seen_at >= ?
111 `).get([cutoff]);
112
113 return Number(row && row.n || 0) > 0;
114 }
115
116 function upsertIndexRow (db, row)
117 {
118 db.prepare(`
119 INSERT INTO file_search_index (
120 file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at
121 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
122 ON CONFLICT(file_path) DO UPDATE SET
123 root_path = excluded.root_path,
124 name = excluded.name,
125 name_lower = excluded.name_lower,
126 relative_path = excluded.relative_path,
127 relative_lower = excluded.relative_lower,
128 size = excluded.size,
129 mtime_ms = excluded.mtime_ms,
130 indexed_at = excluded.indexed_at
131 `).run([
132 row.file_path,
133 row.root_path,
134 row.name,
135 row.name_lower,
136 row.relative_path,
137 row.relative_lower,
138 row.size,
139 row.mtime_ms,
140 row.indexed_at,
141 ]);
142 }
143
144 function deleteIndexPath (filePath)
145 {
146 getDb().prepare('DELETE FROM file_search_index WHERE file_path = ?').run([path.resolve(String(filePath || ''))]);
147 }
148
149 function deleteIndexUnder (targetPath)
150 {
151 const resolved = path.resolve(String(targetPath || ''));
152 const db = getDb();
153 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([resolved]);
154 db.prepare('DELETE FROM file_search_index WHERE file_path LIKE ?')
155 .run([resolved + path.sep + '%']);
156 }
157
158 function rewriteIndexPaths (fromPath, toPath)
159 {
160 const from = path.resolve(String(fromPath || ''));
161 const to = path.resolve(String(toPath || ''));
162 const db = getDb();
163
164 const rows = db.prepare(`
165 SELECT file_path, root_path, name, relative_path, size, mtime_ms
166 FROM file_search_index
167 WHERE file_path = ? OR file_path LIKE ?
168 `).all([from, from + path.sep + '%']);
169
170 if (!rows.length) return;
171
172 const del = db.prepare('DELETE FROM file_search_index WHERE file_path = ?');
173 for (const row of rows)
174 {
175 del.run([row.file_path]);
176 const nextPath = row.file_path === from
177 ? to
178 : to + row.file_path.slice(from.length);
179 const nextName = path.basename(nextPath);
180 const rootPath = path.resolve(row.root_path);
181 const relativePath = path.relative(rootPath, nextPath);
182 upsertIndexRow(db, {
183 file_path: nextPath,
184 root_path: rootPath,
185 name: nextName,
186 name_lower: nextName.toLowerCase(),
187 relative_path: relativePath,
188 relative_lower: String(relativePath || '').toLowerCase(),
189 size: row.size,
190 mtime_ms: row.mtime_ms,
191 indexed_at: nowMs(),
192 });
193 }
194 }
195
196 async function walkFilesUnderRoot (storageRoot, onFile, startDir)
197 {
198 const resolvedRoot = path.resolve(storageRoot);
199 const stack = [path.resolve(startDir || storageRoot)];
200 const visitedDirs = new Set();
201 let dirsVisited = 0;
202
203 while (stack.length)
204 {
205 const dir = stack.pop();
206 if (!dir) continue;
207 const dirKey = path.resolve(dir);
208 if (visitedDirs.has(dirKey)) continue;
209 visitedDirs.add(dirKey);
210 dirsVisited += 1;
211 if (dirsVisited % 64 === 0) await yieldToEventLoop();
212
213 let entries;
214 try
215 {
216 entries = await fs.promises.readdir(dir, { withFileTypes: true });
217 } catch
218 {
219 continue;
220 }
221
222 for (const entry of entries)
223 {
224 const fullPath = path.join(dir, entry.name);
225 if (entry.isDirectory())
226 {
227 stack.push(fullPath);
228 continue;
229 }
230 if (!entry.isFile()) continue;
231
232 let stat;
233 try
234 {
235 stat = await fs.promises.stat(fullPath);
236 } catch
237 {
238 continue;
239 }
240
241 const relativePath = path.relative(resolvedRoot, fullPath);
242 await onFile({
243 file_path: fullPath,
244 root_path: resolvedRoot,
245 name: entry.name,
246 name_lower: entry.name.toLowerCase(),
247 relative_path: relativePath,
248 relative_lower: String(relativePath || '').toLowerCase(),
249 size: stat.size,
250 mtime_ms: stat.mtimeMs,
251 });
252 }
253 }
254 }
255
256 async function syncSearchIndexForRoot (rootPath)
257 {
258 const resolvedRoot = path.resolve(rootPath);
259 try
260 {
261 const stat = await fs.promises.stat(resolvedRoot);
262 if (!stat.isDirectory()) return { rootPath: resolvedRoot, indexed: 0, removed: 0 };
263 } catch
264 {
265 return { rootPath: resolvedRoot, indexed: 0, removed: 0, offline: true };
266 }
267
268 const db = getDb();
269 const seen = new Set();
270 const indexedAt = nowMs();
271 let indexed = 0;
272
273 await walkFilesUnderRoot(resolvedRoot, async (fileRow) =>
274 {
275 const filePath = path.resolve(fileRow.file_path);
276 seen.add(filePath);
277 upsertIndexRow(db, {
278 ...fileRow,
279 file_path: filePath,
280 root_path: resolvedRoot,
281 indexed_at: indexedAt,
282 });
283 indexed += 1;
284 });
285
286 const existing = db.prepare('SELECT file_path FROM file_search_index WHERE root_path = ?').all([resolvedRoot]);
287 let removed = 0;
288 for (const row of existing)
289 {
290 if (seen.has(path.resolve(row.file_path))) continue;
291 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([row.file_path]);
292 removed += 1;
293 }
294
295 return { rootPath: resolvedRoot, indexed, removed };
296 }
297
298 async function syncSearchIndexOnce (roots)
299 {
300 const targetRoots = Array.isArray(roots) && roots.length ? roots.map((r) => path.resolve(r)) : effectiveIndexRoots();
301 const summary = {
302 rootsScanned: 0,
303 filesIndexed: 0,
304 filesRemoved: 0,
305 offlineRoots: 0,
306 };
307
308 for (const root of targetRoots)
309 {
310 const result = await syncSearchIndexForRoot(root);
311 summary.rootsScanned += 1;
312 summary.filesIndexed += Number(result.indexed || 0);
313 summary.filesRemoved += Number(result.removed || 0);
314 if (result.offline) summary.offlineRoots += 1;
315 }
316
317 return summary;
318 }
319
320 function countIndexedFilesForRoots (roots)
321 {
322 const resolved = [...new Set((roots || []).map((r) => path.resolve(r)))];
323 if (!resolved.length) return 0;
324
325 let total = 0;
326 const db = getDb();
327 for (let i = 0; i < resolved.length; i += ROOT_CHUNK_SIZE)
328 {
329 const chunk = resolved.slice(i, i + ROOT_CHUNK_SIZE);
330 const placeholders = chunk.map(() => '?').join(', ');
331 const row = db.prepare(`SELECT COUNT(*) AS n FROM file_search_index WHERE root_path IN (${placeholders})`).get(chunk);
332 total += Number(row && row.n || 0);
333 }
334 return total;
335 }
336
337 function querySubstringCandidates (resolvedRoots, queryLower, rowLimit)
338 {
339 if (!resolvedRoots.length) return [];
340
341 const db = getDb();
342 const out = [];
343 const needle = queryLower;
344
345 for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE)
346 {
347 const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE);
348 const placeholders = chunk.map(() => '?').join(', ');
349 const rows = db.prepare(`
350 SELECT file_path, root_path, name, relative_path, size, mtime_ms
351 FROM file_search_index
352 WHERE root_path IN (${placeholders})
353 AND (instr(name_lower, ?) > 0 OR instr(relative_lower, ?) > 0)
354 LIMIT ?
355 `).all([...chunk, needle, needle, rowLimit]);
356
357 for (const row of rows) out.push(row);
358 if (out.length >= rowLimit) break;
359 }
360
361 return out.slice(0, rowLimit);
362 }
363
364 function queryBatchCandidates (resolvedRoots, offset, batchSize)
365 {
366 if (!resolvedRoots.length) return [];
367
368 const db = getDb();
369 const out = [];
370
371 for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE)
372 {
373 const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE);
374 const placeholders = chunk.map(() => '?').join(', ');
375 const rows = db.prepare(`
376 SELECT file_path, root_path, name, relative_path, size, mtime_ms
377 FROM file_search_index
378 WHERE root_path IN (${placeholders})
379 ORDER BY file_path
380 LIMIT ? OFFSET ?
381 `).all([...chunk, batchSize, offset]);
382
383 for (const row of rows) out.push(row);
384 if (out.length >= batchSize) return out.slice(0, batchSize);
385 }
386
387 return out;
388 }
389
390 function rowToSearchResult (row, query, score)
391 {
392 const filePath = path.resolve(row.file_path);
393 const rootPath = path.resolve(row.root_path);
394 const relativePath = row.relative_path != null
395 ? String(row.relative_path)
396 : path.relative(rootPath, filePath);
397
398 let mtime = null;
399 if (Number.isFinite(Number(row.mtime_ms)) && Number(row.mtime_ms) > 0)
400 {
401 mtime = new Date(Number(row.mtime_ms)).toISOString();
402 }
403
404 return {
405 name: row.name || path.basename(filePath),
406 path: filePath,
407 dir: path.dirname(filePath),
408 rootPath,
409 relativePath,
410 size: Number.isFinite(Number(row.size)) ? Number(row.size) : null,
411 mtime,
412 score,
413 };
414 }
415
416 function collectCandidatesFromIndex (resolvedRoots, query, maxCandidates)
417 {
418 const queryLower = String(query || '').trim().toLowerCase();
419 const byPath = new Map();
420
421 const substringRows = querySubstringCandidates(resolvedRoots, queryLower, maxCandidates);
422 for (const row of substringRows)
423 {
424 const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path);
425 if (score == null) continue;
426 byPath.set(path.resolve(row.file_path), { row, score });
427 }
428
429 if (byPath.size < maxCandidates)
430 {
431 let offset = 0;
432 let scanned = 0;
433 while (scanned < MAX_SUBSEQUENCE_SCAN && byPath.size < maxCandidates)
434 {
435 const batch = queryBatchCandidates(resolvedRoots, offset, INDEX_BATCH_SIZE);
436 if (!batch.length) break;
437 offset += batch.length;
438 scanned += batch.length;
439
440 for (const row of batch)
441 {
442 const filePath = path.resolve(row.file_path);
443 if (byPath.has(filePath)) continue;
444 const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path);
445 if (score == null) continue;
446 byPath.set(filePath, { row, score });
447 if (byPath.size >= maxCandidates) break;
448 }
449 }
450 }
451
452 return [...byPath.values()];
453 }
454
455 async function validateCandidates (candidates, dirtyDirs, db)
456 {
457 const validated = [];
458 const toCheck = candidates.slice(0, MAX_VALIDATE_HITS);
459
460 for (const entry of toCheck)
461 {
462 const filePath = path.resolve(entry.row.file_path);
463 let stat;
464 try
465 {
466 stat = await fs.promises.stat(filePath);
467 } catch
468 {
469 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
470 dirtyDirs.add(path.dirname(filePath));
471 continue;
472 }
473
474 if (!stat.isFile())
475 {
476 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
477 dirtyDirs.add(path.dirname(filePath));
478 continue;
479 }
480
481 const size = Number(stat.size);
482 const mtimeMs = Number(stat.mtimeMs);
483 const staleMeta = Number(entry.row.size) !== size || Number(entry.row.mtime_ms) !== mtimeMs;
484 if (staleMeta)
485 {
486 const rootPath = path.resolve(entry.row.root_path);
487 const name = path.basename(filePath);
488 const relativePath = path.relative(rootPath, filePath);
489 upsertIndexRow(db, {
490 file_path: filePath,
491 root_path: rootPath,
492 name,
493 name_lower: name.toLowerCase(),
494 relative_path: relativePath,
495 relative_lower: String(relativePath || '').toLowerCase(),
496 size,
497 mtime_ms: mtimeMs,
498 indexed_at: nowMs(),
499 });
500 entry.row.size = size;
501 entry.row.mtime_ms = mtimeMs;
502 dirtyDirs.add(path.dirname(filePath));
503 }
504
505 validated.push(entry);
506 }
507
508 for (const entry of candidates.slice(MAX_VALIDATE_HITS))
509 {
510 validated.push(entry);
511 }
512
513 return validated;
514 }
515
516 async function reindexDirectories (dirs, rootByDir)
517 {
518 const queue = [...dirs].slice(0, MAX_RESCAN_DIRS_PER_SEARCH);
519 const discovered = [];
520
521 for (const dir of queue)
522 {
523 const resolvedDir = path.resolve(dir);
524 const rootPath = rootByDir.get(resolvedDir);
525 if (!rootPath) continue;
526
527 try
528 {
529 const stat = await fs.promises.stat(resolvedDir);
530 if (!stat.isDirectory()) continue;
531 } catch
532 {
533 continue;
534 }
535
536 const db = getDb();
537 const seen = new Set();
538 const indexedAt = nowMs();
539
540 const resolvedRootPath = path.resolve(rootPath);
541 await walkFilesUnderRoot(resolvedRootPath, async (fileRow) =>
542 {
543 const filePath = path.resolve(fileRow.file_path);
544 seen.add(filePath);
545 upsertIndexRow(db, {
546 ...fileRow,
547 file_path: filePath,
548 root_path: resolvedRootPath,
549 indexed_at: indexedAt,
550 });
551 discovered.push(fileRow);
552 }, resolvedDir);
553
554 const prefix = resolvedDir + path.sep;
555 const existing = db.prepare(`
556 SELECT file_path FROM file_search_index
557 WHERE root_path = ? AND (file_path = ? OR file_path LIKE ?)
558 `).all([path.resolve(rootPath), resolvedDir, prefix + '%']);
559
560 for (const row of existing)
561 {
562 const filePath = path.resolve(row.file_path);
563 if (filePath === resolvedDir) continue;
564 if (seen.has(filePath)) continue;
565 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
566 }
567 }
568
569 return discovered;
570 }
571
572 function rootForPath (resolvedRoots, filePath)
573 {
574 const resolved = path.resolve(filePath);
575 for (const root of resolvedRoots)
576 {
577 const rel = path.relative(root, resolved);
578 if (rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))) return root;
579 }
580 return null;
581 }
582
583 function buildRootByDirMap (resolvedRoots)
584 {
585 const map = new Map();
586 for (const root of resolvedRoots)
587 {
588 map.set(path.resolve(root), path.resolve(root));
589 }
590 return map;
591 }
592
593 async function searchFilesFilesystemWalk (query, roots, maxResults)
594 {
595 const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
596 const results = [];
597 const visitedDirs = new Set();
598 let dirsVisited = 0;
599
600 for (const root of roots)
601 {
602 const rootPath = path.resolve(root);
603 const stack = [rootPath];
604
605 while (stack.length)
606 {
607 const dir = stack.pop();
608 if (!dir) continue;
609 const dirKey = path.resolve(dir);
610 if (visitedDirs.has(dirKey)) continue;
611 visitedDirs.add(dirKey);
612 dirsVisited += 1;
613 if (dirsVisited % 64 === 0) await yieldToEventLoop();
614
615 let entries;
616 try
617 {
618 entries = await fs.promises.readdir(dir, { withFileTypes: true });
619 } catch
620 {
621 continue;
622 }
623
624 for (const entry of entries)
625 {
626 const fullPath = path.join(dir, entry.name);
627 if (entry.isDirectory())
628 {
629 stack.push(fullPath);
630 continue;
631 }
632 if (!entry.isFile()) continue;
633
634 const relativePath = path.relative(rootPath, fullPath);
635 const score = fuzzyMatchScore(query, entry.name)
636 ?? fuzzyMatchScore(query, relativePath);
637 if (score == null) continue;
638
639 let stat = null;
640 try { stat = await fs.promises.stat(fullPath); } catch { /* ignore */ }
641
642 results.push(rowToSearchResult({
643 file_path: fullPath,
644 root_path: rootPath,
645 name: entry.name,
646 relative_path: relativePath,
647 size: stat ? stat.size : null,
648 mtime_ms: stat ? stat.mtimeMs : null,
649 }, query, score));
650
651 if (results.length > limit * 4)
652 {
653 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
654 results.length = limit * 2;
655 }
656 }
657 }
658 }
659
660 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
661 return results.slice(0, limit);
662 }
663
664 async function searchFiles (query, roots, maxResults)
665 {
666 const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
667 const resolvedRoots = [...new Set((roots || []).map((r) => path.resolve(r)))];
668 if (!resolvedRoots.length) return [];
669
670 if (countIndexedFilesForRoots(resolvedRoots) < 1)
671 {
672 scheduleBackgroundSync(resolvedRoots);
673 return searchFilesFilesystemWalk(query, roots, limit);
674 }
675
676 const candidates = collectCandidatesFromIndex(resolvedRoots, query, MAX_CANDIDATES_PER_SEARCH);
677 const dirtyDirs = new Set();
678 const db = getDb();
679 const validated = await validateCandidates(candidates, dirtyDirs, db);
680
681 const rootByDir = buildRootByDirMap(resolvedRoots);
682 for (const dir of dirtyDirs)
683 {
684 const resolvedDir = path.resolve(dir);
685 if (!rootByDir.has(resolvedDir))
686 {
687 const root = rootForPath(resolvedRoots, resolvedDir);
688 if (root) rootByDir.set(resolvedDir, root);
689 }
690 }
691
692 if (dirtyDirs.size)
693 {
694 const discovered = await reindexDirectories(dirtyDirs, rootByDir);
695 for (const fileRow of discovered)
696 {
697 const score = fuzzyMatchScore(query, fileRow.name) ?? fuzzyMatchScore(query, fileRow.relative_path);
698 if (score == null) continue;
699 validated.push({
700 row: {
701 file_path: fileRow.file_path,
702 root_path: fileRow.root_path,
703 name: fileRow.name,
704 relative_path: fileRow.relative_path,
705 size: fileRow.size,
706 mtime_ms: fileRow.mtime_ms,
707 },
708 score,
709 });
710 }
711 }
712
713 const merged = new Map();
714 for (const entry of validated)
715 {
716 const filePath = path.resolve(entry.row.file_path);
717 const prev = merged.get(filePath);
718 if (!prev || entry.score > prev.score) merged.set(filePath, entry);
719 }
720
721 const results = [...merged.values()]
722 .map((entry) => rowToSearchResult(entry.row, query, entry.score))
723 .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
724
725 return results.slice(0, limit);
726 }
727
728 function scheduleBackgroundSync (roots)
729 {
730 if (backgroundSyncRunning) return;
731 backgroundSyncRunning = true;
732 void syncSearchIndexOnce(roots)
733 .catch(() => { /* non-fatal */ })
734 .finally(() =>
735 {
736 backgroundSyncRunning = false;
737 });
738 }
739
740 function scheduleNextLoop (delayMs)
741 {
742 loopTimer = setTimeout(async () =>
743 {
744 loopTimer = null;
745 try
746 {
747 if (!hasActiveClients())
748 {
749 await syncSearchIndexOnce();
750 }
751 } catch (err)
752 {
753 console.error('[search-index] sync failed:', err && err.message ? err.message : err);
754 }
755 finally
756 {
757 if (loopStarted)
758 {
759 const delay = hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS;
760 scheduleNextLoop(delay);
761 }
762 }
763 }, delayMs);
764
765 if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();
766 }
767
768 function startSearchIndexLoop ()
769 {
770 if (loopStarted) return;
771 loopStarted = true;
772 scheduleBackgroundSync(effectiveIndexRoots());
773 scheduleNextLoop(hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS);
774 }
775
776 function stopSearchIndexLoop ()
777 {
778 loopStarted = false;
779 if (loopTimer)
780 {
781 clearTimeout(loopTimer);
782 loopTimer = null;
783 }
784 }
785
786 function getSearchIndexSnapshot ()
787 {
788 const db = getDb();
789 const row = db.prepare('SELECT COUNT(*) AS n FROM file_search_index').get();
790 return {
791 indexedFileCount: Number(row && row.n || 0),
792 loopStarted,
793 hasActiveClients: hasActiveClients(),
794 backgroundSyncRunning,
795 };
796 }
797
798 module.exports = {
799 fuzzyMatchScore,
800 searchFiles,
801 searchFilesFilesystemWalk,
802 syncSearchIndexOnce,
803 countIndexedFilesForRoots,
804 deleteIndexPath,
805 deleteIndexUnder,
806 rewriteIndexPaths,
807 reindexDirectoryTree: async (dirPath) =>
808 {
809 const resolvedDir = path.resolve(String(dirPath || ''));
810 const root = rootForPath(effectiveIndexRoots(), resolvedDir);
811 if (!root) return;
812 const rootByDir = new Map([[resolvedDir, root]]);
813 await reindexDirectories(new Set([resolvedDir]), rootByDir);
814 },
815 startSearchIndexLoop,
816 stopSearchIndexLoop,
817 getSearchIndexSnapshot,
818 };
@@ -0,0 +1,818 @@
1'use strict';
2
3const fs = require('fs');
4const path = require('path');
5const { getDb } = require('./db');
6const { loadConfig, getEffectivePaths } = require('./config');
7
8const ACTIVE_CLIENT_GRACE_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_CLIENT_GRACE_MS, 5 * 60 * 1000);
9const IDLE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_IDLE_SYNC_INTERVAL_MS, 30 * 60 * 1000);
10const ACTIVE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_SYNC_INTERVAL_MS, 10 * 60 * 1000);
11const MAX_CANDIDATES_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_CANDIDATES, 8000);
12const MAX_SUBSEQUENCE_SCAN = toPositiveInt(process.env.SEARCH_INDEX_MAX_SUBSEQUENCE_SCAN, 12000);
13const MAX_VALIDATE_HITS = toPositiveInt(process.env.SEARCH_INDEX_MAX_VALIDATE_HITS, 250);
14const MAX_RESCAN_DIRS_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_RESCAN_DIRS, 12);
15const INDEX_BATCH_SIZE = 400;
16const ROOT_CHUNK_SIZE = 50;
17
18let loopTimer = null;
19let loopStarted = false;
20let backgroundSyncRunning = false;
21
22function toPositiveInt (value, fallback)
23{
24 const parsed = Number.parseInt(String(value || ''), 10);
25 return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
26}
27
28function yieldToEventLoop ()
29{
30 return new Promise((resolve) => setImmediate(resolve));
31}
32
33function nowMs ()
34{
35 return Date.now();
36}
37
38function fuzzyMatchScore (needle, haystack)
39{
40 const q = String(needle || '').trim().toLowerCase();
41 const t = String(haystack || '').toLowerCase();
42 if (!q || !t) return null;
43
44 const idx = t.indexOf(q);
45 if (idx >= 0)
46 {
47 const compactBonus = Math.max(0, 200 - idx);
48 const lengthBonus = Math.max(0, 120 - (t.length - q.length));
49 return 1200 + compactBonus + lengthBonus;
50 }
51
52 let qPos = 0;
53 let firstHit = -1;
54 let lastHit = -1;
55 let gaps = 0;
56
57 for (let i = 0; i < t.length && qPos < q.length; i++)
58 {
59 if (t[i] !== q[qPos]) continue;
60 if (firstHit === -1) firstHit = i;
61 if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1);
62 lastHit = i;
63 qPos += 1;
64 }
65
66 if (qPos !== q.length) return null;
67
68 const span = lastHit - firstHit + 1;
69 const compactness = Math.max(0, 300 - span);
70 const gapPenalty = Math.min(180, gaps * 3);
71 const startBonus = Math.max(0, 120 - firstHit);
72 return 700 + compactness + startBonus - gapPenalty;
73}
74
75function effectiveIndexRoots ()
76{
77 const config = loadConfig();
78 const roots = getEffectivePaths(config.paths || []);
79 const userPath = typeof config.userPath === 'string' ? config.userPath.trim() : '';
80 if (userPath) roots.push(userPath);
81
82 const seen = new Set();
83 const deduped = [];
84 for (const root of roots)
85 {
86 const key = path.resolve(root);
87 if (seen.has(key)) continue;
88 seen.add(key);
89 deduped.push(key);
90 }
91 return deduped;
92}
93
94function hasActiveClients ()
95{
96 try
97 {
98 const { getActiveStreamCount } = require('./video-stream');
99 if (Number(getActiveStreamCount()) > 0) return true;
100 } catch
101 {
102 // Stream module unavailable — treat as no active streams.
103 }
104
105 const db = getDb();
106 const cutoff = nowMs() - ACTIVE_CLIENT_GRACE_MS;
107 const row = db.prepare(`
108 SELECT COUNT(*) AS n
109 FROM auth_sessions
110 WHERE revoked_at IS NULL AND last_seen_at >= ?
111 `).get([cutoff]);
112
113 return Number(row && row.n || 0) > 0;
114}
115
116function upsertIndexRow (db, row)
117{
118 db.prepare(`
119 INSERT INTO file_search_index (
120 file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at
121 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
122 ON CONFLICT(file_path) DO UPDATE SET
123 root_path = excluded.root_path,
124 name = excluded.name,
125 name_lower = excluded.name_lower,
126 relative_path = excluded.relative_path,
127 relative_lower = excluded.relative_lower,
128 size = excluded.size,
129 mtime_ms = excluded.mtime_ms,
130 indexed_at = excluded.indexed_at
131 `).run([
132 row.file_path,
133 row.root_path,
134 row.name,
135 row.name_lower,
136 row.relative_path,
137 row.relative_lower,
138 row.size,
139 row.mtime_ms,
140 row.indexed_at,
141 ]);
142}
143
144function deleteIndexPath (filePath)
145{
146 getDb().prepare('DELETE FROM file_search_index WHERE file_path = ?').run([path.resolve(String(filePath || ''))]);
147}
148
149function deleteIndexUnder (targetPath)
150{
151 const resolved = path.resolve(String(targetPath || ''));
152 const db = getDb();
153 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([resolved]);
154 db.prepare('DELETE FROM file_search_index WHERE file_path LIKE ?')
155 .run([resolved + path.sep + '%']);
156}
157
158function rewriteIndexPaths (fromPath, toPath)
159{
160 const from = path.resolve(String(fromPath || ''));
161 const to = path.resolve(String(toPath || ''));
162 const db = getDb();
163
164 const rows = db.prepare(`
165 SELECT file_path, root_path, name, relative_path, size, mtime_ms
166 FROM file_search_index
167 WHERE file_path = ? OR file_path LIKE ?
168 `).all([from, from + path.sep + '%']);
169
170 if (!rows.length) return;
171
172 const del = db.prepare('DELETE FROM file_search_index WHERE file_path = ?');
173 for (const row of rows)
174 {
175 del.run([row.file_path]);
176 const nextPath = row.file_path === from
177 ? to
178 : to + row.file_path.slice(from.length);
179 const nextName = path.basename(nextPath);
180 const rootPath = path.resolve(row.root_path);
181 const relativePath = path.relative(rootPath, nextPath);
182 upsertIndexRow(db, {
183 file_path: nextPath,
184 root_path: rootPath,
185 name: nextName,
186 name_lower: nextName.toLowerCase(),
187 relative_path: relativePath,
188 relative_lower: String(relativePath || '').toLowerCase(),
189 size: row.size,
190 mtime_ms: row.mtime_ms,
191 indexed_at: nowMs(),
192 });
193 }
194}
195
196async function walkFilesUnderRoot (storageRoot, onFile, startDir)
197{
198 const resolvedRoot = path.resolve(storageRoot);
199 const stack = [path.resolve(startDir || storageRoot)];
200 const visitedDirs = new Set();
201 let dirsVisited = 0;
202
203 while (stack.length)
204 {
205 const dir = stack.pop();
206 if (!dir) continue;
207 const dirKey = path.resolve(dir);
208 if (visitedDirs.has(dirKey)) continue;
209 visitedDirs.add(dirKey);
210 dirsVisited += 1;
211 if (dirsVisited % 64 === 0) await yieldToEventLoop();
212
213 let entries;
214 try
215 {
216 entries = await fs.promises.readdir(dir, { withFileTypes: true });
217 } catch
218 {
219 continue;
220 }
221
222 for (const entry of entries)
223 {
224 const fullPath = path.join(dir, entry.name);
225 if (entry.isDirectory())
226 {
227 stack.push(fullPath);
228 continue;
229 }
230 if (!entry.isFile()) continue;
231
232 let stat;
233 try
234 {
235 stat = await fs.promises.stat(fullPath);
236 } catch
237 {
238 continue;
239 }
240
241 const relativePath = path.relative(resolvedRoot, fullPath);
242 await onFile({
243 file_path: fullPath,
244 root_path: resolvedRoot,
245 name: entry.name,
246 name_lower: entry.name.toLowerCase(),
247 relative_path: relativePath,
248 relative_lower: String(relativePath || '').toLowerCase(),
249 size: stat.size,
250 mtime_ms: stat.mtimeMs,
251 });
252 }
253 }
254}
255
256async function syncSearchIndexForRoot (rootPath)
257{
258 const resolvedRoot = path.resolve(rootPath);
259 try
260 {
261 const stat = await fs.promises.stat(resolvedRoot);
262 if (!stat.isDirectory()) return { rootPath: resolvedRoot, indexed: 0, removed: 0 };
263 } catch
264 {
265 return { rootPath: resolvedRoot, indexed: 0, removed: 0, offline: true };
266 }
267
268 const db = getDb();
269 const seen = new Set();
270 const indexedAt = nowMs();
271 let indexed = 0;
272
273 await walkFilesUnderRoot(resolvedRoot, async (fileRow) =>
274 {
275 const filePath = path.resolve(fileRow.file_path);
276 seen.add(filePath);
277 upsertIndexRow(db, {
278 ...fileRow,
279 file_path: filePath,
280 root_path: resolvedRoot,
281 indexed_at: indexedAt,
282 });
283 indexed += 1;
284 });
285
286 const existing = db.prepare('SELECT file_path FROM file_search_index WHERE root_path = ?').all([resolvedRoot]);
287 let removed = 0;
288 for (const row of existing)
289 {
290 if (seen.has(path.resolve(row.file_path))) continue;
291 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([row.file_path]);
292 removed += 1;
293 }
294
295 return { rootPath: resolvedRoot, indexed, removed };
296}
297
298async function syncSearchIndexOnce (roots)
299{
300 const targetRoots = Array.isArray(roots) && roots.length ? roots.map((r) => path.resolve(r)) : effectiveIndexRoots();
301 const summary = {
302 rootsScanned: 0,
303 filesIndexed: 0,
304 filesRemoved: 0,
305 offlineRoots: 0,
306 };
307
308 for (const root of targetRoots)
309 {
310 const result = await syncSearchIndexForRoot(root);
311 summary.rootsScanned += 1;
312 summary.filesIndexed += Number(result.indexed || 0);
313 summary.filesRemoved += Number(result.removed || 0);
314 if (result.offline) summary.offlineRoots += 1;
315 }
316
317 return summary;
318}
319
320function countIndexedFilesForRoots (roots)
321{
322 const resolved = [...new Set((roots || []).map((r) => path.resolve(r)))];
323 if (!resolved.length) return 0;
324
325 let total = 0;
326 const db = getDb();
327 for (let i = 0; i < resolved.length; i += ROOT_CHUNK_SIZE)
328 {
329 const chunk = resolved.slice(i, i + ROOT_CHUNK_SIZE);
330 const placeholders = chunk.map(() => '?').join(', ');
331 const row = db.prepare(`SELECT COUNT(*) AS n FROM file_search_index WHERE root_path IN (${placeholders})`).get(chunk);
332 total += Number(row && row.n || 0);
333 }
334 return total;
335}
336
337function querySubstringCandidates (resolvedRoots, queryLower, rowLimit)
338{
339 if (!resolvedRoots.length) return [];
340
341 const db = getDb();
342 const out = [];
343 const needle = queryLower;
344
345 for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE)
346 {
347 const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE);
348 const placeholders = chunk.map(() => '?').join(', ');
349 const rows = db.prepare(`
350 SELECT file_path, root_path, name, relative_path, size, mtime_ms
351 FROM file_search_index
352 WHERE root_path IN (${placeholders})
353 AND (instr(name_lower, ?) > 0 OR instr(relative_lower, ?) > 0)
354 LIMIT ?
355 `).all([...chunk, needle, needle, rowLimit]);
356
357 for (const row of rows) out.push(row);
358 if (out.length >= rowLimit) break;
359 }
360
361 return out.slice(0, rowLimit);
362}
363
364function queryBatchCandidates (resolvedRoots, offset, batchSize)
365{
366 if (!resolvedRoots.length) return [];
367
368 const db = getDb();
369 const out = [];
370
371 for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE)
372 {
373 const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE);
374 const placeholders = chunk.map(() => '?').join(', ');
375 const rows = db.prepare(`
376 SELECT file_path, root_path, name, relative_path, size, mtime_ms
377 FROM file_search_index
378 WHERE root_path IN (${placeholders})
379 ORDER BY file_path
380 LIMIT ? OFFSET ?
381 `).all([...chunk, batchSize, offset]);
382
383 for (const row of rows) out.push(row);
384 if (out.length >= batchSize) return out.slice(0, batchSize);
385 }
386
387 return out;
388}
389
390function rowToSearchResult (row, query, score)
391{
392 const filePath = path.resolve(row.file_path);
393 const rootPath = path.resolve(row.root_path);
394 const relativePath = row.relative_path != null
395 ? String(row.relative_path)
396 : path.relative(rootPath, filePath);
397
398 let mtime = null;
399 if (Number.isFinite(Number(row.mtime_ms)) && Number(row.mtime_ms) > 0)
400 {
401 mtime = new Date(Number(row.mtime_ms)).toISOString();
402 }
403
404 return {
405 name: row.name || path.basename(filePath),
406 path: filePath,
407 dir: path.dirname(filePath),
408 rootPath,
409 relativePath,
410 size: Number.isFinite(Number(row.size)) ? Number(row.size) : null,
411 mtime,
412 score,
413 };
414}
415
416function collectCandidatesFromIndex (resolvedRoots, query, maxCandidates)
417{
418 const queryLower = String(query || '').trim().toLowerCase();
419 const byPath = new Map();
420
421 const substringRows = querySubstringCandidates(resolvedRoots, queryLower, maxCandidates);
422 for (const row of substringRows)
423 {
424 const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path);
425 if (score == null) continue;
426 byPath.set(path.resolve(row.file_path), { row, score });
427 }
428
429 if (byPath.size < maxCandidates)
430 {
431 let offset = 0;
432 let scanned = 0;
433 while (scanned < MAX_SUBSEQUENCE_SCAN && byPath.size < maxCandidates)
434 {
435 const batch = queryBatchCandidates(resolvedRoots, offset, INDEX_BATCH_SIZE);
436 if (!batch.length) break;
437 offset += batch.length;
438 scanned += batch.length;
439
440 for (const row of batch)
441 {
442 const filePath = path.resolve(row.file_path);
443 if (byPath.has(filePath)) continue;
444 const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path);
445 if (score == null) continue;
446 byPath.set(filePath, { row, score });
447 if (byPath.size >= maxCandidates) break;
448 }
449 }
450 }
451
452 return [...byPath.values()];
453}
454
455async function validateCandidates (candidates, dirtyDirs, db)
456{
457 const validated = [];
458 const toCheck = candidates.slice(0, MAX_VALIDATE_HITS);
459
460 for (const entry of toCheck)
461 {
462 const filePath = path.resolve(entry.row.file_path);
463 let stat;
464 try
465 {
466 stat = await fs.promises.stat(filePath);
467 } catch
468 {
469 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
470 dirtyDirs.add(path.dirname(filePath));
471 continue;
472 }
473
474 if (!stat.isFile())
475 {
476 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
477 dirtyDirs.add(path.dirname(filePath));
478 continue;
479 }
480
481 const size = Number(stat.size);
482 const mtimeMs = Number(stat.mtimeMs);
483 const staleMeta = Number(entry.row.size) !== size || Number(entry.row.mtime_ms) !== mtimeMs;
484 if (staleMeta)
485 {
486 const rootPath = path.resolve(entry.row.root_path);
487 const name = path.basename(filePath);
488 const relativePath = path.relative(rootPath, filePath);
489 upsertIndexRow(db, {
490 file_path: filePath,
491 root_path: rootPath,
492 name,
493 name_lower: name.toLowerCase(),
494 relative_path: relativePath,
495 relative_lower: String(relativePath || '').toLowerCase(),
496 size,
497 mtime_ms: mtimeMs,
498 indexed_at: nowMs(),
499 });
500 entry.row.size = size;
501 entry.row.mtime_ms = mtimeMs;
502 dirtyDirs.add(path.dirname(filePath));
503 }
504
505 validated.push(entry);
506 }
507
508 for (const entry of candidates.slice(MAX_VALIDATE_HITS))
509 {
510 validated.push(entry);
511 }
512
513 return validated;
514}
515
516async function reindexDirectories (dirs, rootByDir)
517{
518 const queue = [...dirs].slice(0, MAX_RESCAN_DIRS_PER_SEARCH);
519 const discovered = [];
520
521 for (const dir of queue)
522 {
523 const resolvedDir = path.resolve(dir);
524 const rootPath = rootByDir.get(resolvedDir);
525 if (!rootPath) continue;
526
527 try
528 {
529 const stat = await fs.promises.stat(resolvedDir);
530 if (!stat.isDirectory()) continue;
531 } catch
532 {
533 continue;
534 }
535
536 const db = getDb();
537 const seen = new Set();
538 const indexedAt = nowMs();
539
540 const resolvedRootPath = path.resolve(rootPath);
541 await walkFilesUnderRoot(resolvedRootPath, async (fileRow) =>
542 {
543 const filePath = path.resolve(fileRow.file_path);
544 seen.add(filePath);
545 upsertIndexRow(db, {
546 ...fileRow,
547 file_path: filePath,
548 root_path: resolvedRootPath,
549 indexed_at: indexedAt,
550 });
551 discovered.push(fileRow);
552 }, resolvedDir);
553
554 const prefix = resolvedDir + path.sep;
555 const existing = db.prepare(`
556 SELECT file_path FROM file_search_index
557 WHERE root_path = ? AND (file_path = ? OR file_path LIKE ?)
558 `).all([path.resolve(rootPath), resolvedDir, prefix + '%']);
559
560 for (const row of existing)
561 {
562 const filePath = path.resolve(row.file_path);
563 if (filePath === resolvedDir) continue;
564 if (seen.has(filePath)) continue;
565 db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
566 }
567 }
568
569 return discovered;
570}
571
572function rootForPath (resolvedRoots, filePath)
573{
574 const resolved = path.resolve(filePath);
575 for (const root of resolvedRoots)
576 {
577 const rel = path.relative(root, resolved);
578 if (rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))) return root;
579 }
580 return null;
581}
582
583function buildRootByDirMap (resolvedRoots)
584{
585 const map = new Map();
586 for (const root of resolvedRoots)
587 {
588 map.set(path.resolve(root), path.resolve(root));
589 }
590 return map;
591}
592
593async function searchFilesFilesystemWalk (query, roots, maxResults)
594{
595 const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
596 const results = [];
597 const visitedDirs = new Set();
598 let dirsVisited = 0;
599
600 for (const root of roots)
601 {
602 const rootPath = path.resolve(root);
603 const stack = [rootPath];
604
605 while (stack.length)
606 {
607 const dir = stack.pop();
608 if (!dir) continue;
609 const dirKey = path.resolve(dir);
610 if (visitedDirs.has(dirKey)) continue;
611 visitedDirs.add(dirKey);
612 dirsVisited += 1;
613 if (dirsVisited % 64 === 0) await yieldToEventLoop();
614
615 let entries;
616 try
617 {
618 entries = await fs.promises.readdir(dir, { withFileTypes: true });
619 } catch
620 {
621 continue;
622 }
623
624 for (const entry of entries)
625 {
626 const fullPath = path.join(dir, entry.name);
627 if (entry.isDirectory())
628 {
629 stack.push(fullPath);
630 continue;
631 }
632 if (!entry.isFile()) continue;
633
634 const relativePath = path.relative(rootPath, fullPath);
635 const score = fuzzyMatchScore(query, entry.name)
636 ?? fuzzyMatchScore(query, relativePath);
637 if (score == null) continue;
638
639 let stat = null;
640 try { stat = await fs.promises.stat(fullPath); } catch { /* ignore */ }
641
642 results.push(rowToSearchResult({
643 file_path: fullPath,
644 root_path: rootPath,
645 name: entry.name,
646 relative_path: relativePath,
647 size: stat ? stat.size : null,
648 mtime_ms: stat ? stat.mtimeMs : null,
649 }, query, score));
650
651 if (results.length > limit * 4)
652 {
653 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
654 results.length = limit * 2;
655 }
656 }
657 }
658 }
659
660 results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
661 return results.slice(0, limit);
662}
663
664async function searchFiles (query, roots, maxResults)
665{
666 const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
667 const resolvedRoots = [...new Set((roots || []).map((r) => path.resolve(r)))];
668 if (!resolvedRoots.length) return [];
669
670 if (countIndexedFilesForRoots(resolvedRoots) < 1)
671 {
672 scheduleBackgroundSync(resolvedRoots);
673 return searchFilesFilesystemWalk(query, roots, limit);
674 }
675
676 const candidates = collectCandidatesFromIndex(resolvedRoots, query, MAX_CANDIDATES_PER_SEARCH);
677 const dirtyDirs = new Set();
678 const db = getDb();
679 const validated = await validateCandidates(candidates, dirtyDirs, db);
680
681 const rootByDir = buildRootByDirMap(resolvedRoots);
682 for (const dir of dirtyDirs)
683 {
684 const resolvedDir = path.resolve(dir);
685 if (!rootByDir.has(resolvedDir))
686 {
687 const root = rootForPath(resolvedRoots, resolvedDir);
688 if (root) rootByDir.set(resolvedDir, root);
689 }
690 }
691
692 if (dirtyDirs.size)
693 {
694 const discovered = await reindexDirectories(dirtyDirs, rootByDir);
695 for (const fileRow of discovered)
696 {
697 const score = fuzzyMatchScore(query, fileRow.name) ?? fuzzyMatchScore(query, fileRow.relative_path);
698 if (score == null) continue;
699 validated.push({
700 row: {
701 file_path: fileRow.file_path,
702 root_path: fileRow.root_path,
703 name: fileRow.name,
704 relative_path: fileRow.relative_path,
705 size: fileRow.size,
706 mtime_ms: fileRow.mtime_ms,
707 },
708 score,
709 });
710 }
711 }
712
713 const merged = new Map();
714 for (const entry of validated)
715 {
716 const filePath = path.resolve(entry.row.file_path);
717 const prev = merged.get(filePath);
718 if (!prev || entry.score > prev.score) merged.set(filePath, entry);
719 }
720
721 const results = [...merged.values()]
722 .map((entry) => rowToSearchResult(entry.row, query, entry.score))
723 .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
724
725 return results.slice(0, limit);
726}
727
728function scheduleBackgroundSync (roots)
729{
730 if (backgroundSyncRunning) return;
731 backgroundSyncRunning = true;
732 void syncSearchIndexOnce(roots)
733 .catch(() => { /* non-fatal */ })
734 .finally(() =>
735 {
736 backgroundSyncRunning = false;
737 });
738}
739
740function scheduleNextLoop (delayMs)
741{
742 loopTimer = setTimeout(async () =>
743 {
744 loopTimer = null;
745 try
746 {
747 if (!hasActiveClients())
748 {
749 await syncSearchIndexOnce();
750 }
751 } catch (err)
752 {
753 console.error('[search-index] sync failed:', err && err.message ? err.message : err);
754 }
755 finally
756 {
757 if (loopStarted)
758 {
759 const delay = hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS;
760 scheduleNextLoop(delay);
761 }
762 }
763 }, delayMs);
764
765 if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();
766}
767
768function startSearchIndexLoop ()
769{
770 if (loopStarted) return;
771 loopStarted = true;
772 scheduleBackgroundSync(effectiveIndexRoots());
773 scheduleNextLoop(hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS);
774}
775
776function stopSearchIndexLoop ()
777{
778 loopStarted = false;
779 if (loopTimer)
780 {
781 clearTimeout(loopTimer);
782 loopTimer = null;
783 }
784}
785
786function getSearchIndexSnapshot ()
787{
788 const db = getDb();
789 const row = db.prepare('SELECT COUNT(*) AS n FROM file_search_index').get();
790 return {
791 indexedFileCount: Number(row && row.n || 0),
792 loopStarted,
793 hasActiveClients: hasActiveClients(),
794 backgroundSyncRunning,
795 };
796}
797
798module.exports = {
799 fuzzyMatchScore,
800 searchFiles,
801 searchFilesFilesystemWalk,
802 syncSearchIndexOnce,
803 countIndexedFilesForRoots,
804 deleteIndexPath,
805 deleteIndexUnder,
806 rewriteIndexPaths,
807 reindexDirectoryTree: async (dirPath) =>
808 {
809 const resolvedDir = path.resolve(String(dirPath || ''));
810 const root = rootForPath(effectiveIndexRoots(), resolvedDir);
811 if (!root) return;
812 const rootByDir = new Map([[resolvedDir, root]]);
813 await reindexDirectories(new Set([resolvedDir]), rootByDir);
814 },
815 startSearchIndexLoop,
816 stopSearchIndexLoop,
817 getSearchIndexSnapshot,
818};
server/thumbnail-cache.js +60 −32
@@ -2,7 +2,6 @@
2 2
3 3 const fs = require('fs');
4 4 const path = require('path');
5 const crypto = require('crypto');
6 5 const { spawn } = require('child_process');
7 6 const sharp = require('sharp');
8 7
@@ -12,8 +11,9 @@ const PUBLIC_DIR = path.resolve(process.env.FILEMANAGER_PUBLIC_DIR || path.join(
12 11 const THUMBS_DIR = path.resolve(process.env.FILEMANAGER_THUMBS_DIR || path.join(PUBLIC_DIR, 'thumbs'));
13 12 const MANIFEST_PATH = path.join(THUMBS_DIR, '.manifest.json');
14 13 const FOLDER_ICONS_PATH = path.join(THUMBS_DIR, '.folder-icons.json');
15 const LOOP_DELAY_MS = 5000;
14 const LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_LOOP_DELAY_MS, 15000);
16 15 const ACTIVE_STREAM_LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_ACTIVE_STREAM_DELAY_MS, 120000);
16 const THUMB_HTTP_MAX_AGE_SEC = toPositiveInt(process.env.THUMBNAIL_HTTP_MAX_AGE_SEC, 3600);
17 17
18 18 const IMAGE_EXTS = new Set([
19 19 'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tif', 'tiff', 'avif', 'svg', 'ico',
@@ -25,6 +25,8 @@ const MODEL_EXTS = new Set(['stl', 'obj']);
25 25
26 26 let loopTimer = null;
27 27 let loopStarted = false;
28 let manifestCache = null;
29 let manifestCacheMtimeMs = null;
28 30
29 31 function toPosix (p)
30 32 {
@@ -213,24 +215,61 @@ async function readManifest ()
213 215 return { version: 1, files: {} };
214 216 }
215 217
218 function invalidateManifestCache ()
219 {
220 manifestCache = null;
221 manifestCacheMtimeMs = null;
222 }
223
224 async function readManifestCached ()
225 {
226 try
227 {
228 const stat = await fs.promises.stat(MANIFEST_PATH);
229 if (manifestCache && manifestCacheMtimeMs === stat.mtimeMs)
230 {
231 return manifestCache;
232 }
233
234 const manifest = await readManifest();
235 manifestCache = manifest;
236 manifestCacheMtimeMs = stat.mtimeMs;
237 return manifest;
238 } catch
239 {
240 invalidateManifestCache();
241 return { version: 1, files: {} };
242 }
243 }
244
245 /**
246 * Returns a Set of source file paths that have a manifest entry (thumbnail expected).
247 */
248 async function getThumbnailManifestSourcePathSet ()
249 {
250 const manifest = await readManifestCached();
251 const paths = new Set();
252 for (const filePath of Object.keys(manifest.files || {}))
253 {
254 paths.add(path.resolve(filePath));
255 }
256 return paths;
257 }
258
216 259 async function writeManifest (manifest)
217 260 {
218 261 const payload = JSON.stringify(manifest, null, 2);
219 262 const tmp = MANIFEST_PATH + '.tmp';
220 263 await fs.promises.writeFile(tmp, payload, 'utf8');
221 264 await fs.promises.rename(tmp, MANIFEST_PATH);
222 }
223
224 function sha256File (target)
225 {
226 return new Promise((resolve, reject) =>
265 manifestCache = manifest;
266 try
227 267 {
228 const hash = crypto.createHash('sha256');
229 const stream = fs.createReadStream(target);
230 stream.on('error', reject);
231 stream.on('data', chunk => hash.update(chunk));
232 stream.on('end', () => resolve(hash.digest('hex')));
233 });
268 manifestCacheMtimeMs = (await fs.promises.stat(MANIFEST_PATH)).mtimeMs;
269 } catch
270 {
271 invalidateManifestCache();
272 }
234 273 }
235 274
236 275 async function walkDir (dirPath, onDir, onFile)
@@ -697,17 +736,7 @@ async function syncThumbnailCacheOnce ()
697 736
698 737 if (kind === 'image')
699 738 {
700 let sha256;
701 try
702 {
703 sha256 = await sha256File(filePath);
704 } catch
705 {
706 return;
707 }
708
709 const unchangedByHash = prev && prev.sha256 === sha256 && await exists(thumbPath);
710 if (!unchangedByHash)
739 if (!unchangedByStat || !(await exists(thumbPath)))
711 740 {
712 741 try
713 742 {
@@ -719,7 +748,7 @@ async function syncThumbnailCacheOnce ()
719 748 }
720 749
721 750 nextManifest.files[filePath] = {
722 sha256,
751 sha256: prev && prev.sha256 ? prev.sha256 : null,
723 752 size: stat.size,
724 753 mtimeMs: stat.mtimeMs,
725 754 thumbRelPath,
@@ -869,20 +898,17 @@ async function writeFolderIcons (obj)
869 898 */
870 899 async function getFilesWithThumbnailsInDir (dirPath)
871 900 {
872 const manifest = await readManifest();
901 const manifest = await readManifestCached();
873 902 const resolved = path.resolve(dirPath);
903 const prefix = resolved + path.sep;
874 904 const results = [];
875 905
876 906 for (const [filePath, meta] of Object.entries(manifest.files || {}))
877 907 {
878 const resolvedFile = path.resolve(filePath);
879 const relative = path.relative(resolved, resolvedFile);
880 if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) continue;
881 908 if (!meta || !meta.thumbRelPath) continue;
882 909
883 // Verify the thumbnail file actually exists on disk.
884 const thumbPath = path.resolve(path.join(PUBLIC_DIR, meta.thumbRelPath));
885 if (!(await exists(thumbPath))) continue;
910 const resolvedFile = path.resolve(filePath);
911 if (resolvedFile !== resolved && !resolvedFile.startsWith(prefix)) continue;
886 912
887 913 results.push({ filePath, thumbRelPath: meta.thumbRelPath });
888 914 }
@@ -945,6 +971,7 @@ module.exports = {
945 971 getCachedThumbnailInfo,
946 972 getThumbPathForSourceFile,
947 973 getThumbRelPathForSourceFile,
974 getThumbnailManifestSourcePathSet,
948 975 getFilesWithThumbnailsInDir,
949 976 getImagesWithThumbnailsInDir,
950 977 getFolderIconSourceFile,
@@ -952,4 +979,5 @@ module.exports = {
952 979 startThumbnailCacheLoop,
953 980 stopThumbnailCacheLoop,
954 981 syncThumbnailCacheOnce,
982 THUMB_HTTP_MAX_AGE_SEC,
955 983 };
@@ -2,7 +2,6 @@
2
2
3const fs = require('fs');
3const fs = require('fs');
4const path = require('path');
4const path = require('path');
5const crypto = require('crypto');
6const { spawn } = require('child_process');
5const { spawn } = require('child_process');
7const sharp = require('sharp');
6const sharp = require('sharp');
8
7
@@ -12,8 +11,9 @@ const PUBLIC_DIR = path.resolve(process.env.FILEMANAGER_PUBLIC_DIR || path.join(
12const THUMBS_DIR = path.resolve(process.env.FILEMANAGER_THUMBS_DIR || path.join(PUBLIC_DIR, 'thumbs'));
11const THUMBS_DIR = path.resolve(process.env.FILEMANAGER_THUMBS_DIR || path.join(PUBLIC_DIR, 'thumbs'));
13const MANIFEST_PATH = path.join(THUMBS_DIR, '.manifest.json');
12const MANIFEST_PATH = path.join(THUMBS_DIR, '.manifest.json');
14const FOLDER_ICONS_PATH = path.join(THUMBS_DIR, '.folder-icons.json');
13const FOLDER_ICONS_PATH = path.join(THUMBS_DIR, '.folder-icons.json');
15const LOOP_DELAY_MS = 5000;
14const LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_LOOP_DELAY_MS, 15000);
16const ACTIVE_STREAM_LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_ACTIVE_STREAM_DELAY_MS, 120000);
15const ACTIVE_STREAM_LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_ACTIVE_STREAM_DELAY_MS, 120000);
16const THUMB_HTTP_MAX_AGE_SEC = toPositiveInt(process.env.THUMBNAIL_HTTP_MAX_AGE_SEC, 3600);
17
17
18const IMAGE_EXTS = new Set([
18const IMAGE_EXTS = new Set([
19 'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tif', 'tiff', 'avif', 'svg', 'ico',
19 'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tif', 'tiff', 'avif', 'svg', 'ico',
@@ -25,6 +25,8 @@ const MODEL_EXTS = new Set(['stl', 'obj']);
25
25
26let loopTimer = null;
26let loopTimer = null;
27let loopStarted = false;
27let loopStarted = false;
28let manifestCache = null;
29let manifestCacheMtimeMs = null;
28
30
29function toPosix (p)
31function toPosix (p)
30{
32{
@@ -213,24 +215,61 @@ async function readManifest ()
213 return { version: 1, files: {} };
215 return { version: 1, files: {} };
214}
216}
215
217
218function invalidateManifestCache ()
219{
220 manifestCache = null;
221 manifestCacheMtimeMs = null;
222}
223
224async function readManifestCached ()
225{
226 try
227 {
228 const stat = await fs.promises.stat(MANIFEST_PATH);
229 if (manifestCache && manifestCacheMtimeMs === stat.mtimeMs)
230 {
231 return manifestCache;
232 }
233
234 const manifest = await readManifest();
235 manifestCache = manifest;
236 manifestCacheMtimeMs = stat.mtimeMs;
237 return manifest;
238 } catch
239 {
240 invalidateManifestCache();
241 return { version: 1, files: {} };
242 }
243}
244
245/**
246 * Returns a Set of source file paths that have a manifest entry (thumbnail expected).
247 */
248async function getThumbnailManifestSourcePathSet ()
249{
250 const manifest = await readManifestCached();
251 const paths = new Set();
252 for (const filePath of Object.keys(manifest.files || {}))
253 {
254 paths.add(path.resolve(filePath));
255 }
256 return paths;
257}
258
216async function writeManifest (manifest)
259async function writeManifest (manifest)
217{
260{
218 const payload = JSON.stringify(manifest, null, 2);
261 const payload = JSON.stringify(manifest, null, 2);
219 const tmp = MANIFEST_PATH + '.tmp';
262 const tmp = MANIFEST_PATH + '.tmp';
220 await fs.promises.writeFile(tmp, payload, 'utf8');
263 await fs.promises.writeFile(tmp, payload, 'utf8');
221 await fs.promises.rename(tmp, MANIFEST_PATH);
264 await fs.promises.rename(tmp, MANIFEST_PATH);
222}
265 manifestCache = manifest;
223
266 try
224function sha256File (target)
225{
226 return new Promise((resolve, reject) =>
227 {
267 {
228 const hash = crypto.createHash('sha256');
268 manifestCacheMtimeMs = (await fs.promises.stat(MANIFEST_PATH)).mtimeMs;
229 const stream = fs.createReadStream(target);
269 } catch
230 stream.on('error', reject);
270 {
231 stream.on('data', chunk => hash.update(chunk));
271 invalidateManifestCache();
232 stream.on('end', () => resolve(hash.digest('hex')));
272 }
233 });
234}
273}
235
274
236async function walkDir (dirPath, onDir, onFile)
275async function walkDir (dirPath, onDir, onFile)
@@ -697,17 +736,7 @@ async function syncThumbnailCacheOnce ()
697
736
698 if (kind === 'image')
737 if (kind === 'image')
699 {
738 {
700 let sha256;
739 if (!unchangedByStat || !(await exists(thumbPath)))
701 try
702 {
703 sha256 = await sha256File(filePath);
704 } catch
705 {
706 return;
707 }
708
709 const unchangedByHash = prev && prev.sha256 === sha256 && await exists(thumbPath);
710 if (!unchangedByHash)
711 {
740 {
712 try
741 try
713 {
742 {
@@ -719,7 +748,7 @@ async function syncThumbnailCacheOnce ()
719 }
748 }
720
749
721 nextManifest.files[filePath] = {
750 nextManifest.files[filePath] = {
722 sha256,
751 sha256: prev && prev.sha256 ? prev.sha256 : null,
723 size: stat.size,
752 size: stat.size,
724 mtimeMs: stat.mtimeMs,
753 mtimeMs: stat.mtimeMs,
725 thumbRelPath,
754 thumbRelPath,
@@ -869,20 +898,17 @@ async function writeFolderIcons (obj)
869 */
898 */
870async function getFilesWithThumbnailsInDir (dirPath)
899async function getFilesWithThumbnailsInDir (dirPath)
871{
900{
872 const manifest = await readManifest();
901 const manifest = await readManifestCached();
873 const resolved = path.resolve(dirPath);
902 const resolved = path.resolve(dirPath);
903 const prefix = resolved + path.sep;
874 const results = [];
904 const results = [];
875
905
876 for (const [filePath, meta] of Object.entries(manifest.files || {}))
906 for (const [filePath, meta] of Object.entries(manifest.files || {}))
877 {
907 {
878 const resolvedFile = path.resolve(filePath);
879 const relative = path.relative(resolved, resolvedFile);
880 if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) continue;
881 if (!meta || !meta.thumbRelPath) continue;
908 if (!meta || !meta.thumbRelPath) continue;
882
909
883 // Verify the thumbnail file actually exists on disk.
910 const resolvedFile = path.resolve(filePath);
884 const thumbPath = path.resolve(path.join(PUBLIC_DIR, meta.thumbRelPath));
911 if (resolvedFile !== resolved && !resolvedFile.startsWith(prefix)) continue;
885 if (!(await exists(thumbPath))) continue;
886
912
887 results.push({ filePath, thumbRelPath: meta.thumbRelPath });
913 results.push({ filePath, thumbRelPath: meta.thumbRelPath });
888 }
914 }
@@ -945,6 +971,7 @@ module.exports = {
945 getCachedThumbnailInfo,
971 getCachedThumbnailInfo,
946 getThumbPathForSourceFile,
972 getThumbPathForSourceFile,
947 getThumbRelPathForSourceFile,
973 getThumbRelPathForSourceFile,
974 getThumbnailManifestSourcePathSet,
948 getFilesWithThumbnailsInDir,
975 getFilesWithThumbnailsInDir,
949 getImagesWithThumbnailsInDir,
976 getImagesWithThumbnailsInDir,
950 getFolderIconSourceFile,
977 getFolderIconSourceFile,
@@ -952,4 +979,5 @@ module.exports = {
952 startThumbnailCacheLoop,
979 startThumbnailCacheLoop,
953 stopThumbnailCacheLoop,
980 stopThumbnailCacheLoop,
954 syncThumbnailCacheOnce,
981 syncThumbnailCacheOnce,
982 THUMB_HTTP_MAX_AGE_SEC,
955};
983};
tests/admin.routes.test.js +1 −0
@@ -147,6 +147,7 @@ jest.mock('../server/auth', () => ({
147 147 createToken: () => 'mock-token',
148 148 getClientIp: () => '127.0.0.1',
149 149 getRegion: () => 'US:CA',
150 clearAuthUserCache: () => {},
150 151 }));
151 152
152 153 const adminRoutes = require('../server/routes/admin');
@@ -147,6 +147,7 @@ jest.mock('../server/auth', () => ({
147 createToken: () => 'mock-token',
147 createToken: () => 'mock-token',
148 getClientIp: () => '127.0.0.1',
148 getClientIp: () => '127.0.0.1',
149 getRegion: () => 'US:CA',
149 getRegion: () => 'US:CA',
150 clearAuthUserCache: () => {},
150}));
151}));
151
152
152const adminRoutes = require('../server/routes/admin');
153const adminRoutes = require('../server/routes/admin');
tests/file-hash-queue.test.js added +65 −0
@@ -0,0 +1,65 @@
1 'use strict';
2
3 const crypto = require('crypto');
4 const fs = require('fs');
5 const os = require('os');
6 const path = require('path');
7
8 jest.mock('../server/db', () =>
9 {
10 const { Database } = require('node-sqlite3-wasm');
11 let db;
12 function getDb ()
13 {
14 if (!db)
15 {
16 db = new Database(':memory:');
17 db.exec(`
18 CREATE TABLE IF NOT EXISTS file_meta (
19 file_path TEXT PRIMARY KEY,
20 uploader TEXT NOT NULL DEFAULT '',
21 downloaders TEXT NOT NULL DEFAULT '[]',
22 watched_by TEXT NOT NULL DEFAULT '{}',
23 uploaded_at TEXT,
24 sha256 TEXT
25 );
26 `);
27 }
28 return db;
29 }
30 return { getDb, initDb: getDb };
31 });
32
33 describe('file hash queue', () =>
34 {
35 let tempDir;
36
37 beforeEach(() =>
38 {
39 tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filemanager-file-hash-'));
40 });
41
42 afterEach(() =>
43 {
44 fs.rmSync(tempDir, { recursive: true, force: true });
45 });
46
47 test('drainFileHashQueue persists sha256 for queued files', async () =>
48 {
49 const filePath = path.join(tempDir, 'queued.txt');
50 const content = 'background hash';
51 fs.writeFileSync(filePath, content);
52
53 const { getDb } = require('../server/db');
54 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
55 .run([filePath, '', '[]']);
56
57 const hashQueue = require('../server/file-hash-queue');
58 hashQueue.enqueueFileHash(filePath);
59 await hashQueue.drainFileHashQueue();
60
61 const expected = crypto.createHash('sha256').update(content).digest('hex');
62 const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
63 expect(row.sha256).toBe(expected);
64 });
65 });
@@ -0,0 +1,65 @@
1'use strict';
2
3const crypto = require('crypto');
4const fs = require('fs');
5const os = require('os');
6const path = require('path');
7
8jest.mock('../server/db', () =>
9{
10 const { Database } = require('node-sqlite3-wasm');
11 let db;
12 function getDb ()
13 {
14 if (!db)
15 {
16 db = new Database(':memory:');
17 db.exec(`
18 CREATE TABLE IF NOT EXISTS file_meta (
19 file_path TEXT PRIMARY KEY,
20 uploader TEXT NOT NULL DEFAULT '',
21 downloaders TEXT NOT NULL DEFAULT '[]',
22 watched_by TEXT NOT NULL DEFAULT '{}',
23 uploaded_at TEXT,
24 sha256 TEXT
25 );
26 `);
27 }
28 return db;
29 }
30 return { getDb, initDb: getDb };
31});
32
33describe('file hash queue', () =>
34{
35 let tempDir;
36
37 beforeEach(() =>
38 {
39 tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filemanager-file-hash-'));
40 });
41
42 afterEach(() =>
43 {
44 fs.rmSync(tempDir, { recursive: true, force: true });
45 });
46
47 test('drainFileHashQueue persists sha256 for queued files', async () =>
48 {
49 const filePath = path.join(tempDir, 'queued.txt');
50 const content = 'background hash';
51 fs.writeFileSync(filePath, content);
52
53 const { getDb } = require('../server/db');
54 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
55 .run([filePath, '', '[]']);
56
57 const hashQueue = require('../server/file-hash-queue');
58 hashQueue.enqueueFileHash(filePath);
59 await hashQueue.drainFileHashQueue();
60
61 const expected = crypto.createHash('sha256').update(content).digest('hex');
62 const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
63 expect(row.sha256).toBe(expected);
64 });
65});
tests/files.routes.test.js +44 −9
@@ -13,6 +13,7 @@ let mockRootAliases = {};
13 13 let mockGlobalFaviconPath = null;
14 14 let mockUser = { username: 'admin', isAdmin: true, permissions: {} };
15 15 let mockThumbPath = null;
16 let mockThumbnailManifestPaths = null;
16 17 let mockFolderImages = [];
17 18 let mockFolderIconFile = null;
18 19 let lastFolderIconAssignment = undefined;
@@ -51,6 +52,23 @@ jest.mock('../server/db', () =>
51 52 sha256 TEXT
52 53 );
53 54 CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);
55 CREATE TABLE IF NOT EXISTS user_last_navigation (
56 username TEXT PRIMARY KEY,
57 last_dir TEXT,
58 updated_at INTEGER NOT NULL
59 );
60 CREATE TABLE IF NOT EXISTS file_search_index (
61 file_path TEXT PRIMARY KEY,
62 root_path TEXT NOT NULL,
63 name TEXT NOT NULL,
64 name_lower TEXT NOT NULL,
65 relative_path TEXT NOT NULL,
66 relative_lower TEXT NOT NULL,
67 size INTEGER,
68 mtime_ms INTEGER,
69 indexed_at INTEGER NOT NULL
70 );
71 CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
54 72 `);
55 73 }
56 74 return db;
@@ -95,12 +113,18 @@ jest.mock('../server/thumbnail-cache', () => ({
95 113 if (!mockThumbPath) return null;
96 114 return { thumbPath: mockThumbPath, contentType: 'image/webp' };
97 115 },
116 getThumbnailManifestSourcePathSet: async () =>
117 {
118 if (mockThumbnailManifestPaths) return new Set(mockThumbnailManifestPaths);
119 return new Set();
120 },
98 121 getFilesWithThumbnailsInDir: async () => mockFolderImages.slice(),
99 122 getFolderIconSourceFile: async () => mockFolderIconFile,
100 123 setFolderIconAssignment: async (folderPath, imagePath) =>
101 124 {
102 125 lastFolderIconAssignment = { folderPath, imagePath };
103 126 },
127 THUMB_HTTP_MAX_AGE_SEC: 3600,
104 128 }));
105 129
106 130 jest.mock('../server/video-stream', () => ({
@@ -145,6 +169,7 @@ jest.mock('../server/subtitle-debug-log', () => ({
145 169
146 170 const filesRoutes = require('../server/routes/files');
147 171 const { getDb } = require('../server/db');
172 const { drainFileHashQueue, stopFileHashQueue } = require('../server/file-hash-queue');
148 173
149 174 let server;
150 175 let baseUrl;
@@ -193,6 +218,7 @@ beforeEach(() =>
193 218 mockUserPath = null;
194 219 mockUser = { username: 'admin', isAdmin: true, permissions: {} };
195 220 mockThumbPath = null;
221 mockThumbnailManifestPaths = null;
196 222 mockFolderImages = [];
197 223 mockFolderIconFile = null;
198 224 lastFolderIconAssignment = undefined;
@@ -206,6 +232,8 @@ beforeEach(() =>
206 232 const db = getDb();
207 233 db.prepare('DELETE FROM file_meta').run([]);
208 234 db.prepare('DELETE FROM users').run([]);
235 db.exec('DELETE FROM file_search_index');
236 stopFileHashQueue();
209 237 });
210 238
211 239 function req (method, routePath, body)
@@ -1058,9 +1086,11 @@ describe('image search endpoint', () =>
1058 1086 fs.writeFileSync(path.join(picsDir, 'dragon-cover.jpg'), 'img');
1059 1087 fs.writeFileSync(path.join(picsDir, 'dragon-notes.txt'), 'txt');
1060 1088
1089 const imgPath = path.join(picsDir, 'dragon-cover.jpg');
1061 1090 const thumbPath = path.join(rootDir, 'thumb.webp');
1062 1091 fs.writeFileSync(thumbPath, 'thumb');
1063 1092 mockThumbPath = thumbPath;
1093 mockThumbnailManifestPaths = [imgPath];
1064 1094
1065 1095 const { status, body } = await req('GET', '/api/files/search-images?query=dragon');
1066 1096 expect(status).toBe(200);
@@ -1436,17 +1466,18 @@ describe('video streaming routes', () =>
1436 1466
1437 1467 describe('SHA-256 hashing and duplicate detection', () =>
1438 1468 {
1439 test('POST /api/files/upload stores sha256 hash for uploaded files', async () =>
1469 test('POST /api/files/upload queues sha256 hash for uploaded files', async () =>
1440 1470 {
1441 1471 const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
1442 1472 { name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },
1443 1473 ]);
1444 1474 expect(status).toBe(200);
1445 1475
1476 await drainFileHashQueue();
1477
1446 1478 const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
1447 1479 .get([path.join(rootDir, 'hashme.txt')]);
1448 1480 expect(meta).toBeTruthy();
1449 // SHA-256 is a 64-char hex string.
1450 1481 expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);
1451 1482 });
1452 1483
@@ -1463,22 +1494,26 @@ describe('SHA-256 hashing and duplicate detection', () =>
1463 1494 expect(String(meta.uploaded_at || '')).toMatch(/^\d{4}-\d{2}-\d{2}T/);
1464 1495 });
1465 1496
1466 test('GET /api/files/meta returns sha256 and computes it lazily when missing', async () =>
1497 test('GET /api/files/meta enqueues sha256 and background worker persists it', async () =>
1467 1498 {
1468 1499 const filePath = path.join(rootDir, 'lazy.txt');
1469 1500 fs.writeFileSync(filePath, 'lazy content');
1470 1501
1471 // Insert a row without a hash to simulate a pre-existing file.
1472 1502 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
1473 1503 .run([filePath, 'admin', '[]']);
1474 1504
1475 const { status, body } = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1476 expect(status).toBe(200);
1477 expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
1505 const first = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1506 expect(first.status).toBe(200);
1507 expect(first.body.sha256).toBeNull();
1508
1509 await drainFileHashQueue();
1478 1510
1479 // Verify it was persisted.
1480 1511 const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
1481 expect(row.sha256).toBe(body.sha256);
1512 expect(row.sha256).toMatch(/^[0-9a-f]{64}$/);
1513
1514 const second = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1515 expect(second.status).toBe(200);
1516 expect(second.body.sha256).toBe(row.sha256);
1482 1517 });
1483 1518
1484 1519 test('GET /api/files/meta backfills uploaded_at when missing', async () =>
@@ -13,6 +13,7 @@ let mockRootAliases = {};
13let mockGlobalFaviconPath = null;
13let mockGlobalFaviconPath = null;
14let mockUser = { username: 'admin', isAdmin: true, permissions: {} };
14let mockUser = { username: 'admin', isAdmin: true, permissions: {} };
15let mockThumbPath = null;
15let mockThumbPath = null;
16let mockThumbnailManifestPaths = null;
16let mockFolderImages = [];
17let mockFolderImages = [];
17let mockFolderIconFile = null;
18let mockFolderIconFile = null;
18let lastFolderIconAssignment = undefined;
19let lastFolderIconAssignment = undefined;
@@ -51,6 +52,23 @@ jest.mock('../server/db', () =>
51 sha256 TEXT
52 sha256 TEXT
52 );
53 );
53 CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);
54 CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);
55 CREATE TABLE IF NOT EXISTS user_last_navigation (
56 username TEXT PRIMARY KEY,
57 last_dir TEXT,
58 updated_at INTEGER NOT NULL
59 );
60 CREATE TABLE IF NOT EXISTS file_search_index (
61 file_path TEXT PRIMARY KEY,
62 root_path TEXT NOT NULL,
63 name TEXT NOT NULL,
64 name_lower TEXT NOT NULL,
65 relative_path TEXT NOT NULL,
66 relative_lower TEXT NOT NULL,
67 size INTEGER,
68 mtime_ms INTEGER,
69 indexed_at INTEGER NOT NULL
70 );
71 CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
54 `);
72 `);
55 }
73 }
56 return db;
74 return db;
@@ -95,12 +113,18 @@ jest.mock('../server/thumbnail-cache', () => ({
95 if (!mockThumbPath) return null;
113 if (!mockThumbPath) return null;
96 return { thumbPath: mockThumbPath, contentType: 'image/webp' };
114 return { thumbPath: mockThumbPath, contentType: 'image/webp' };
97 },
115 },
116 getThumbnailManifestSourcePathSet: async () =>
117 {
118 if (mockThumbnailManifestPaths) return new Set(mockThumbnailManifestPaths);
119 return new Set();
120 },
98 getFilesWithThumbnailsInDir: async () => mockFolderImages.slice(),
121 getFilesWithThumbnailsInDir: async () => mockFolderImages.slice(),
99 getFolderIconSourceFile: async () => mockFolderIconFile,
122 getFolderIconSourceFile: async () => mockFolderIconFile,
100 setFolderIconAssignment: async (folderPath, imagePath) =>
123 setFolderIconAssignment: async (folderPath, imagePath) =>
101 {
124 {
102 lastFolderIconAssignment = { folderPath, imagePath };
125 lastFolderIconAssignment = { folderPath, imagePath };
103 },
126 },
127 THUMB_HTTP_MAX_AGE_SEC: 3600,
104}));
128}));
105
129
106jest.mock('../server/video-stream', () => ({
130jest.mock('../server/video-stream', () => ({
@@ -145,6 +169,7 @@ jest.mock('../server/subtitle-debug-log', () => ({
145
169
146const filesRoutes = require('../server/routes/files');
170const filesRoutes = require('../server/routes/files');
147const { getDb } = require('../server/db');
171const { getDb } = require('../server/db');
172const { drainFileHashQueue, stopFileHashQueue } = require('../server/file-hash-queue');
148
173
149let server;
174let server;
150let baseUrl;
175let baseUrl;
@@ -193,6 +218,7 @@ beforeEach(() =>
193 mockUserPath = null;
218 mockUserPath = null;
194 mockUser = { username: 'admin', isAdmin: true, permissions: {} };
219 mockUser = { username: 'admin', isAdmin: true, permissions: {} };
195 mockThumbPath = null;
220 mockThumbPath = null;
221 mockThumbnailManifestPaths = null;
196 mockFolderImages = [];
222 mockFolderImages = [];
197 mockFolderIconFile = null;
223 mockFolderIconFile = null;
198 lastFolderIconAssignment = undefined;
224 lastFolderIconAssignment = undefined;
@@ -206,6 +232,8 @@ beforeEach(() =>
206 const db = getDb();
232 const db = getDb();
207 db.prepare('DELETE FROM file_meta').run([]);
233 db.prepare('DELETE FROM file_meta').run([]);
208 db.prepare('DELETE FROM users').run([]);
234 db.prepare('DELETE FROM users').run([]);
235 db.exec('DELETE FROM file_search_index');
236 stopFileHashQueue();
209});
237});
210
238
211function req (method, routePath, body)
239function req (method, routePath, body)
@@ -1058,9 +1086,11 @@ describe('image search endpoint', () =>
1058 fs.writeFileSync(path.join(picsDir, 'dragon-cover.jpg'), 'img');
1086 fs.writeFileSync(path.join(picsDir, 'dragon-cover.jpg'), 'img');
1059 fs.writeFileSync(path.join(picsDir, 'dragon-notes.txt'), 'txt');
1087 fs.writeFileSync(path.join(picsDir, 'dragon-notes.txt'), 'txt');
1060
1088
1089 const imgPath = path.join(picsDir, 'dragon-cover.jpg');
1061 const thumbPath = path.join(rootDir, 'thumb.webp');
1090 const thumbPath = path.join(rootDir, 'thumb.webp');
1062 fs.writeFileSync(thumbPath, 'thumb');
1091 fs.writeFileSync(thumbPath, 'thumb');
1063 mockThumbPath = thumbPath;
1092 mockThumbPath = thumbPath;
1093 mockThumbnailManifestPaths = [imgPath];
1064
1094
1065 const { status, body } = await req('GET', '/api/files/search-images?query=dragon');
1095 const { status, body } = await req('GET', '/api/files/search-images?query=dragon');
1066 expect(status).toBe(200);
1096 expect(status).toBe(200);
@@ -1436,17 +1466,18 @@ describe('video streaming routes', () =>
1436
1466
1437describe('SHA-256 hashing and duplicate detection', () =>
1467describe('SHA-256 hashing and duplicate detection', () =>
1438{
1468{
1439 test('POST /api/files/upload stores sha256 hash for uploaded files', async () =>
1469 test('POST /api/files/upload queues sha256 hash for uploaded files', async () =>
1440 {
1470 {
1441 const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
1471 const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
1442 { name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },
1472 { name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },
1443 ]);
1473 ]);
1444 expect(status).toBe(200);
1474 expect(status).toBe(200);
1445
1475
1476 await drainFileHashQueue();
1477
1446 const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
1478 const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
1447 .get([path.join(rootDir, 'hashme.txt')]);
1479 .get([path.join(rootDir, 'hashme.txt')]);
1448 expect(meta).toBeTruthy();
1480 expect(meta).toBeTruthy();
1449 // SHA-256 is a 64-char hex string.
1450 expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);
1481 expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);
1451 });
1482 });
1452
1483
@@ -1463,22 +1494,26 @@ describe('SHA-256 hashing and duplicate detection', () =>
1463 expect(String(meta.uploaded_at || '')).toMatch(/^\d{4}-\d{2}-\d{2}T/);
1494 expect(String(meta.uploaded_at || '')).toMatch(/^\d{4}-\d{2}-\d{2}T/);
1464 });
1495 });
1465
1496
1466 test('GET /api/files/meta returns sha256 and computes it lazily when missing', async () =>
1497 test('GET /api/files/meta enqueues sha256 and background worker persists it', async () =>
1467 {
1498 {
1468 const filePath = path.join(rootDir, 'lazy.txt');
1499 const filePath = path.join(rootDir, 'lazy.txt');
1469 fs.writeFileSync(filePath, 'lazy content');
1500 fs.writeFileSync(filePath, 'lazy content');
1470
1501
1471 // Insert a row without a hash to simulate a pre-existing file.
1472 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
1502 getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
1473 .run([filePath, 'admin', '[]']);
1503 .run([filePath, 'admin', '[]']);
1474
1504
1475 const { status, body } = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1505 const first = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1476 expect(status).toBe(200);
1506 expect(first.status).toBe(200);
1477 expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
1507 expect(first.body.sha256).toBeNull();
1508
1509 await drainFileHashQueue();
1478
1510
1479 // Verify it was persisted.
1480 const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
1511 const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
1481 expect(row.sha256).toBe(body.sha256);
1512 expect(row.sha256).toMatch(/^[0-9a-f]{64}$/);
1513
1514 const second = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1515 expect(second.status).toBe(200);
1516 expect(second.body.sha256).toBe(row.sha256);
1482 });
1517 });
1483
1518
1484 test('GET /api/files/meta backfills uploaded_at when missing', async () =>
1519 test('GET /api/files/meta backfills uploaded_at when missing', async () =>
tests/search-index.test.js added +140 −0
@@ -0,0 +1,140 @@
1 'use strict';
2
3 const fs = require('fs');
4 const os = require('os');
5 const path = require('path');
6
7 let mockPaths = [];
8
9 jest.mock('../server/db', () =>
10 {
11 const { Database } = require('node-sqlite3-wasm');
12 let db;
13 function getDb ()
14 {
15 if (!db)
16 {
17 db = new Database(':memory:');
18 db.exec(`
19 CREATE TABLE IF NOT EXISTS file_search_index (
20 file_path TEXT PRIMARY KEY,
21 root_path TEXT NOT NULL,
22 name TEXT NOT NULL,
23 name_lower TEXT NOT NULL,
24 relative_path TEXT NOT NULL,
25 relative_lower TEXT NOT NULL,
26 size INTEGER,
27 mtime_ms INTEGER,
28 indexed_at INTEGER NOT NULL
29 );
30 CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
31 `);
32 }
33 return db;
34 }
35 return { getDb, initDb: getDb };
36 });
37
38 jest.mock('../server/config', () => ({
39 loadConfig: () => ({ paths: [...mockPaths] }),
40 getEffectivePaths: (paths) => paths,
41 }));
42
43 describe('search index', () =>
44 {
45 let tempDir;
46 let sourceRoot;
47
48 function loadModule ()
49 {
50 jest.resetModules();
51 return require('../server/search-index');
52 }
53
54 beforeEach(() =>
55 {
56 tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filemanager-search-index-'));
57 sourceRoot = path.join(tempDir, 'root');
58 fs.mkdirSync(sourceRoot, { recursive: true });
59 mockPaths = [sourceRoot];
60 });
61
62 afterEach(() =>
63 {
64 mockPaths = [];
65 fs.rmSync(tempDir, { recursive: true, force: true });
66 });
67
68 test('syncSearchIndexOnce indexes files and removes deleted entries', async () =>
69 {
70 const keepPath = path.join(sourceRoot, 'dragon-ball.mkv');
71 const removePath = path.join(sourceRoot, 'old.txt');
72 fs.writeFileSync(keepPath, 'vid');
73 fs.writeFileSync(removePath, 'txt');
74
75 const searchIndex = loadModule();
76 await searchIndex.syncSearchIndexOnce([sourceRoot]);
77 expect(searchIndex.countIndexedFilesForRoots([sourceRoot])).toBe(2);
78
79 fs.unlinkSync(removePath);
80 await searchIndex.syncSearchIndexOnce([sourceRoot]);
81 expect(searchIndex.countIndexedFilesForRoots([sourceRoot])).toBe(1);
82
83 const results = await searchIndex.searchFiles('dragon', [sourceRoot], 20);
84 expect(results.length).toBe(1);
85 expect(results[0].name).toBe('dragon-ball.mkv');
86 });
87
88 test('searchFiles removes stale ghost hits and rescans for new external files', async () =>
89 {
90 const searchIndex = loadModule();
91 const ghostPath = path.join(sourceRoot, 'ghost.txt');
92 const livePath = path.join(sourceRoot, 'fresh-notes.txt');
93
94 const { getDb } = require('../server/db');
95 const db = getDb();
96 db.prepare(`
97 INSERT INTO file_search_index (
98 file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at
99 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
100 `).run([
101 ghostPath,
102 path.resolve(sourceRoot),
103 'ghost.txt',
104 'ghost.txt',
105 'ghost.txt',
106 'ghost.txt',
107 3,
108 Date.now(),
109 Date.now(),
110 ]);
111
112 fs.writeFileSync(livePath, 'hello');
113
114 const ghostResults = await searchIndex.searchFiles('ghost', [sourceRoot], 20);
115 expect(ghostResults.some((row) => row.name === 'ghost.txt')).toBe(false);
116 const ghostRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([ghostPath]);
117 expect(ghostRow).toBeFalsy();
118
119 const freshResults = await searchIndex.searchFiles('fresh', [sourceRoot], 20);
120 expect(freshResults.some((row) => row.name === 'fresh-notes.txt')).toBe(true);
121 });
122
123 test('rewriteIndexPaths moves indexed paths', async () =>
124 {
125 const from = path.join(sourceRoot, 'old-name.txt');
126 const to = path.join(sourceRoot, 'new-name.txt');
127 fs.writeFileSync(from, 'data');
128
129 const searchIndex = loadModule();
130 await searchIndex.syncSearchIndexOnce([sourceRoot]);
131 searchIndex.rewriteIndexPaths(from, to);
132
133 const { getDb } = require('../server/db');
134 const db = getDb();
135 const oldRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([from]);
136 const newRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([to]);
137 expect(oldRow).toBeFalsy();
138 expect(newRow).toBeTruthy();
139 });
140 });
@@ -0,0 +1,140 @@
1'use strict';
2
3const fs = require('fs');
4const os = require('os');
5const path = require('path');
6
7let mockPaths = [];
8
9jest.mock('../server/db', () =>
10{
11 const { Database } = require('node-sqlite3-wasm');
12 let db;
13 function getDb ()
14 {
15 if (!db)
16 {
17 db = new Database(':memory:');
18 db.exec(`
19 CREATE TABLE IF NOT EXISTS file_search_index (
20 file_path TEXT PRIMARY KEY,
21 root_path TEXT NOT NULL,
22 name TEXT NOT NULL,
23 name_lower TEXT NOT NULL,
24 relative_path TEXT NOT NULL,
25 relative_lower TEXT NOT NULL,
26 size INTEGER,
27 mtime_ms INTEGER,
28 indexed_at INTEGER NOT NULL
29 );
30 CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
31 `);
32 }
33 return db;
34 }
35 return { getDb, initDb: getDb };
36});
37
38jest.mock('../server/config', () => ({
39 loadConfig: () => ({ paths: [...mockPaths] }),
40 getEffectivePaths: (paths) => paths,
41}));
42
43describe('search index', () =>
44{
45 let tempDir;
46 let sourceRoot;
47
48 function loadModule ()
49 {
50 jest.resetModules();
51 return require('../server/search-index');
52 }
53
54 beforeEach(() =>
55 {
56 tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filemanager-search-index-'));
57 sourceRoot = path.join(tempDir, 'root');
58 fs.mkdirSync(sourceRoot, { recursive: true });
59 mockPaths = [sourceRoot];
60 });
61
62 afterEach(() =>
63 {
64 mockPaths = [];
65 fs.rmSync(tempDir, { recursive: true, force: true });
66 });
67
68 test('syncSearchIndexOnce indexes files and removes deleted entries', async () =>
69 {
70 const keepPath = path.join(sourceRoot, 'dragon-ball.mkv');
71 const removePath = path.join(sourceRoot, 'old.txt');
72 fs.writeFileSync(keepPath, 'vid');
73 fs.writeFileSync(removePath, 'txt');
74
75 const searchIndex = loadModule();
76 await searchIndex.syncSearchIndexOnce([sourceRoot]);
77 expect(searchIndex.countIndexedFilesForRoots([sourceRoot])).toBe(2);
78
79 fs.unlinkSync(removePath);
80 await searchIndex.syncSearchIndexOnce([sourceRoot]);
81 expect(searchIndex.countIndexedFilesForRoots([sourceRoot])).toBe(1);
82
83 const results = await searchIndex.searchFiles('dragon', [sourceRoot], 20);
84 expect(results.length).toBe(1);
85 expect(results[0].name).toBe('dragon-ball.mkv');
86 });
87
88 test('searchFiles removes stale ghost hits and rescans for new external files', async () =>
89 {
90 const searchIndex = loadModule();
91 const ghostPath = path.join(sourceRoot, 'ghost.txt');
92 const livePath = path.join(sourceRoot, 'fresh-notes.txt');
93
94 const { getDb } = require('../server/db');
95 const db = getDb();
96 db.prepare(`
97 INSERT INTO file_search_index (
98 file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at
99 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
100 `).run([
101 ghostPath,
102 path.resolve(sourceRoot),
103 'ghost.txt',
104 'ghost.txt',
105 'ghost.txt',
106 'ghost.txt',
107 3,
108 Date.now(),
109 Date.now(),
110 ]);
111
112 fs.writeFileSync(livePath, 'hello');
113
114 const ghostResults = await searchIndex.searchFiles('ghost', [sourceRoot], 20);
115 expect(ghostResults.some((row) => row.name === 'ghost.txt')).toBe(false);
116 const ghostRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([ghostPath]);
117 expect(ghostRow).toBeFalsy();
118
119 const freshResults = await searchIndex.searchFiles('fresh', [sourceRoot], 20);
120 expect(freshResults.some((row) => row.name === 'fresh-notes.txt')).toBe(true);
121 });
122
123 test('rewriteIndexPaths moves indexed paths', async () =>
124 {
125 const from = path.join(sourceRoot, 'old-name.txt');
126 const to = path.join(sourceRoot, 'new-name.txt');
127 fs.writeFileSync(from, 'data');
128
129 const searchIndex = loadModule();
130 await searchIndex.syncSearchIndexOnce([sourceRoot]);
131 searchIndex.rewriteIndexPaths(from, to);
132
133 const { getDb } = require('../server/db');
134 const db = getDb();
135 const oldRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([from]);
136 const newRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([to]);
137 expect(oldRow).toBeFalsy();
138 expect(newRow).toBeTruthy();
139 });
140});