Cursor Optimisation pass
Experimental
@@ -8,3 +8,5 @@ config.json
8
8
public/thumbs/
9
9
zip-cache/
10
10
logs.txt
11
12
*.db-journal
@@ -8,3 +8,5 @@ config.json8
public/thumbs/
8
public/thumbs/
9
zip-cache/
9
zip-cache/
10
logs.txt
10
logs.txt
11
12
*.db-journal@@ -6,11 +6,16 @@ const geoip = require('geoip-lite');
6
6
const { getDb } = require('./db');
7
7
8
8
const SESSION_IDLE_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
9
const SESSION_LAST_SEEN_WRITE_INTERVAL_MS = 60 * 1000; // throttle DB writes on active sessions
9
10
const AUTH_SESSION_PURGE_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes
11
const AUTH_USER_CACHE_TTL_MS = 60 * 1000;
12
const AUTH_REGION_CACHE_MAX_ENTRIES = 2048;
10
13
const AUTH_SESSION_REVOKED_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours
11
14
const AUTH_SESSION_IDLE_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours beyond idle timeout
12
15
13
16
let authSessionPurgeTimer = null;
17
const regionByIp = new Map();
18
const userCacheByUsername = new Map();
14
19
const authSessionCleanupState = {
15
20
lastRunAt: null,
16
21
lastPurgedCount: 0,
@@ -40,10 +45,68 @@ function getClientIp (req)
40
45
41
46
function getRegion (ip)
42
47
{
48
const key = String(ip || '');
49
if (!key) return 'local:';
50
51
const cached = regionByIp.get(key);
52
if (cached) return cached;
53
43
54
// ip is already normalized by getClientIp (no ::ffff: prefix, no ::1)
44
const geo = geoip.lookup(ip);
55
const geo = geoip.lookup(key);
45
56
// Private / RFC1918 addresses won't resolve — use ip itself as region.
46
return geo ? `${geo.country}:${geo.region}` : `local:${ip}`;
57
const region = geo ? `${geo.country}:${geo.region}` : `local:${key}`;
58
59
if (regionByIp.size >= AUTH_REGION_CACHE_MAX_ENTRIES)
60
{
61
regionByIp.clear();
62
}
63
regionByIp.set(key, region);
64
return region;
65
}
66
67
function getCachedUser (username)
68
{
69
const key = String(username || '');
70
if (!key) return null;
71
72
const cached = userCacheByUsername.get(key);
73
const now = nowMs();
74
if (cached && (now - cached.cachedAt) < AUTH_USER_CACHE_TTL_MS)
75
{
76
return cached.user;
77
}
78
79
const db = getDb();
80
const user = db.prepare('SELECT id, username, is_admin, permissions FROM users WHERE username = ?')
81
.get([key]);
82
if (!user) return null;
83
84
const parsed = {
85
id: user.id,
86
username: user.username,
87
isAdmin: user.is_admin === 1,
88
permissions: JSON.parse(user.permissions || '{}'),
89
};
90
91
userCacheByUsername.set(key, { user: parsed, cachedAt: now });
92
return parsed;
93
}
94
95
function clearAuthUserCache (username)
96
{
97
if (username)
98
{
99
userCacheByUsername.delete(String(username));
100
return;
101
}
102
userCacheByUsername.clear();
103
}
104
105
function touchSessionLastSeen (sessionId, lastSeenAt)
106
{
107
const db = getDb();
108
db.prepare('UPDATE auth_sessions SET last_seen_at = ? WHERE session_id = ?')
109
.run([lastSeenAt, sessionId]);
47
110
}
48
111
49
112
function fingerprintHash (fingerprint)
@@ -269,29 +332,25 @@ function requireAuth (req, res, next)
269
332
return res.status(401).json({ error: 'Session invalid or expired.' });
270
333
}
271
334
272
if (nowMs() - Number(session.last_seen_at || 0) > SESSION_IDLE_TIMEOUT_MS)
335
const now = nowMs();
336
if (now - Number(session.last_seen_at || 0) > SESSION_IDLE_TIMEOUT_MS)
273
337
{
274
338
revokeSession(payload.sid);
275
339
return res.status(401).json({ error: 'Session invalid or expired.' });
276
340
}
277
341
278
db.prepare('UPDATE auth_sessions SET last_seen_at = ? WHERE session_id = ?').run([nowMs(), payload.sid]);
279
280
// Load user from DB
281
const user = db.prepare('SELECT id, username, is_admin, permissions FROM users WHERE username = ?')
282
.get([payload.u]);
342
if (now - Number(session.last_seen_at || 0) >= SESSION_LAST_SEEN_WRITE_INTERVAL_MS)
343
{
344
touchSessionLastSeen(payload.sid, now);
345
}
283
346
347
const user = getCachedUser(payload.u);
284
348
if (!user)
285
349
{
286
350
return res.status(401).json({ error: 'Session invalid or expired.' });
287
351
}
288
352
289
req.user = {
290
id: user.id,
291
username: user.username,
292
isAdmin: user.is_admin === 1,
293
permissions: JSON.parse(user.permissions || '{}'),
294
};
353
req.user = user;
295
354
req.authSessionId = payload.sid;
296
355
297
356
next();
@@ -315,6 +374,7 @@ module.exports = {
315
374
requireAdmin,
316
375
getClientIp,
317
376
getRegion,
377
clearAuthUserCache,
318
378
revokeSession,
319
379
purgeStaleSessions,
320
380
getAuthSessionCleanupSnapshot,
@@ -6,11 +6,16 @@ const geoip = require('geoip-lite');6
const { getDb } = require('./db');6
const { getDb } = require('./db');7
7
8
const SESSION_IDLE_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour8
const SESSION_IDLE_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour9
const SESSION_LAST_SEEN_WRITE_INTERVAL_MS = 60 * 1000; // throttle DB writes on active sessions9
const AUTH_SESSION_PURGE_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes10
const AUTH_SESSION_PURGE_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes11
const AUTH_USER_CACHE_TTL_MS = 60 * 1000;12
const AUTH_REGION_CACHE_MAX_ENTRIES = 2048;10
const AUTH_SESSION_REVOKED_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours13
const AUTH_SESSION_REVOKED_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours11
const AUTH_SESSION_IDLE_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours beyond idle timeout14
const AUTH_SESSION_IDLE_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours beyond idle timeout12
15
13
let authSessionPurgeTimer = null;16
let authSessionPurgeTimer = null;17
const regionByIp = new Map();18
const userCacheByUsername = new Map();14
const authSessionCleanupState = {19
const authSessionCleanupState = {15
lastRunAt: null,20
lastRunAt: null,16
lastPurgedCount: 0,21
lastPurgedCount: 0,@@ -40,10 +45,68 @@ function getClientIp (req)40
45
41
function getRegion (ip)46
function getRegion (ip)42
{47
{48
const key = String(ip || '');49
if (!key) return 'local:';50
51
const cached = regionByIp.get(key);52
if (cached) return cached;53
43
// ip is already normalized by getClientIp (no ::ffff: prefix, no ::1)54
// ip is already normalized by getClientIp (no ::ffff: prefix, no ::1)44
const geo = geoip.lookup(ip);55
const geo = geoip.lookup(key);45
// Private / RFC1918 addresses won't resolve — use ip itself as region.56
// Private / RFC1918 addresses won't resolve — use ip itself as region.46
return geo ? `${geo.country}:${geo.region}` : `local:${ip}`;57
const region = geo ? `${geo.country}:${geo.region}` : `local:${key}`;58
59
if (regionByIp.size >= AUTH_REGION_CACHE_MAX_ENTRIES)60
{61
regionByIp.clear();62
}63
regionByIp.set(key, region);64
return region;65
}66
67
function getCachedUser (username)68
{69
const key = String(username || '');70
if (!key) return null;71
72
const cached = userCacheByUsername.get(key);73
const now = nowMs();74
if (cached && (now - cached.cachedAt) < AUTH_USER_CACHE_TTL_MS)75
{76
return cached.user;77
}78
79
const db = getDb();80
const user = db.prepare('SELECT id, username, is_admin, permissions FROM users WHERE username = ?')81
.get([key]);82
if (!user) return null;83
84
const parsed = {85
id: user.id,86
username: user.username,87
isAdmin: user.is_admin === 1,88
permissions: JSON.parse(user.permissions || '{}'),89
};90
91
userCacheByUsername.set(key, { user: parsed, cachedAt: now });92
return parsed;93
}94
95
function clearAuthUserCache (username)96
{97
if (username)98
{99
userCacheByUsername.delete(String(username));100
return;101
}102
userCacheByUsername.clear();103
}104
105
function touchSessionLastSeen (sessionId, lastSeenAt)106
{107
const db = getDb();108
db.prepare('UPDATE auth_sessions SET last_seen_at = ? WHERE session_id = ?')109
.run([lastSeenAt, sessionId]);47
}110
}48
111
49
function fingerprintHash (fingerprint)112
function fingerprintHash (fingerprint)@@ -269,29 +332,25 @@ function requireAuth (req, res, next)269
return res.status(401).json({ error: 'Session invalid or expired.' });332
return res.status(401).json({ error: 'Session invalid or expired.' });270
}333
}271
334
272
if (nowMs() - Number(session.last_seen_at || 0) > SESSION_IDLE_TIMEOUT_MS)335
const now = nowMs();336
if (now - Number(session.last_seen_at || 0) > SESSION_IDLE_TIMEOUT_MS)273
{337
{274
revokeSession(payload.sid);338
revokeSession(payload.sid);275
return res.status(401).json({ error: 'Session invalid or expired.' });339
return res.status(401).json({ error: 'Session invalid or expired.' });276
}340
}277
341
278
db.prepare('UPDATE auth_sessions SET last_seen_at = ? WHERE session_id = ?').run([nowMs(), payload.sid]);342
if (now - Number(session.last_seen_at || 0) >= SESSION_LAST_SEEN_WRITE_INTERVAL_MS)279
343
{280
// Load user from DB344
touchSessionLastSeen(payload.sid, now);281
const user = db.prepare('SELECT id, username, is_admin, permissions FROM users WHERE username = ?')345
}282
.get([payload.u]);283
346
347
const user = getCachedUser(payload.u);284
if (!user)348
if (!user)285
{349
{286
return res.status(401).json({ error: 'Session invalid or expired.' });350
return res.status(401).json({ error: 'Session invalid or expired.' });287
}351
}288
352
289
req.user = {353
req.user = user;290
id: user.id,291
username: user.username,292
isAdmin: user.is_admin === 1,293
permissions: JSON.parse(user.permissions || '{}'),294
};295
req.authSessionId = payload.sid;354
req.authSessionId = payload.sid;296
355
297
next();356
next();@@ -315,6 +374,7 @@ module.exports = {315
requireAdmin,374
requireAdmin,316
getClientIp,375
getClientIp,317
getRegion,376
getRegion,377
clearAuthUserCache,318
revokeSession,378
revokeSession,319
purgeStaleSessions,379
purgeStaleSessions,320
getAuthSessionCleanupSnapshot,380
getAuthSessionCleanupSnapshot,@@ -5,6 +5,9 @@ const path = require('path');
5
5
6
6
const CONFIG_PATH = path.join(__dirname, '../config.json');
7
7
8
let cachedConfig = null;
9
let cachedConfigMtimeMs = null;
10
8
11
const DEFAULT_SCHEDULED_RESTART = {
9
12
enabled: false,
10
13
cadence: 'daily',
@@ -25,26 +28,42 @@ function normalizeScheduledRestart (value)
25
28
};
26
29
}
27
30
31
function normalizeLoadedConfig (parsed)
32
{
33
if (!parsed || typeof parsed !== 'object')
34
{
35
return { port: 3000, paths: [], rootAliases: {} };
36
}
37
if (!parsed.rootAliases || typeof parsed.rootAliases !== 'object')
38
{
39
parsed.rootAliases = {};
40
}
41
parsed.scheduledRestart = normalizeScheduledRestart(parsed.scheduledRestart);
42
return parsed;
43
}
44
28
45
function loadConfig ()
29
46
{
30
47
try
31
48
{
32
const raw = fs.readFileSync(CONFIG_PATH, 'utf8');
33
const parsed = JSON.parse(raw);
34
if (!parsed || typeof parsed !== 'object')
49
const stat = fs.statSync(CONFIG_PATH);
50
if (cachedConfig && cachedConfigMtimeMs === stat.mtimeMs)
35
51
{
36
return { port: 3000, paths: [], rootAliases: {} };
52
return cachedConfig;
37
53
}
38
if (!parsed.rootAliases || typeof parsed.rootAliases !== 'object')
39
{
40
parsed.rootAliases = {};
41
}
42
parsed.scheduledRestart = normalizeScheduledRestart(parsed.scheduledRestart);
54
55
const raw = fs.readFileSync(CONFIG_PATH, 'utf8');
56
const parsed = normalizeLoadedConfig(JSON.parse(raw));
57
cachedConfig = parsed;
58
cachedConfigMtimeMs = stat.mtimeMs;
43
59
return parsed;
44
60
} catch
45
61
{
46
62
const defaults = { port: 3000, paths: [], rootAliases: {}, scheduledRestart: { ...DEFAULT_SCHEDULED_RESTART } };
47
63
fs.writeFileSync(CONFIG_PATH, JSON.stringify(defaults, null, 2), 'utf8');
64
const stat = fs.statSync(CONFIG_PATH);
65
cachedConfig = defaults;
66
cachedConfigMtimeMs = stat.mtimeMs;
48
67
return defaults;
49
68
}
50
69
}
@@ -58,6 +77,22 @@ function saveConfig (config)
58
77
// Atomic replace to avoid partially-written config files.
59
78
fs.writeFileSync(tmp, payload, 'utf8');
60
79
fs.renameSync(tmp, CONFIG_PATH);
80
81
const normalized = normalizeLoadedConfig(config);
82
cachedConfig = normalized;
83
try
84
{
85
cachedConfigMtimeMs = fs.statSync(CONFIG_PATH).mtimeMs;
86
} catch
87
{
88
cachedConfigMtimeMs = null;
89
}
90
}
91
92
function invalidateConfigCache ()
93
{
94
cachedConfig = null;
95
cachedConfigMtimeMs = null;
61
96
}
62
97
63
98
function normalizeForCompare (p)
@@ -116,4 +151,10 @@ function getAllowedRoot (filePath, allowedRoots)
116
151
return null;
117
152
}
118
153
119
module.exports = { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot };
154
module.exports = {
155
loadConfig,
156
saveConfig,
157
invalidateConfigCache,
158
getEffectivePaths,
159
getAllowedRoot,
160
};
@@ -5,6 +5,9 @@ const path = require('path');5
5
6
const CONFIG_PATH = path.join(__dirname, '../config.json');6
const CONFIG_PATH = path.join(__dirname, '../config.json');7
7
8
let cachedConfig = null;9
let cachedConfigMtimeMs = null;10
8
const DEFAULT_SCHEDULED_RESTART = {11
const DEFAULT_SCHEDULED_RESTART = {9
enabled: false,12
enabled: false,10
cadence: 'daily',13
cadence: 'daily',@@ -25,26 +28,42 @@ function normalizeScheduledRestart (value)25
};28
};26
}29
}27
30
31
function normalizeLoadedConfig (parsed)32
{33
if (!parsed || typeof parsed !== 'object')34
{35
return { port: 3000, paths: [], rootAliases: {} };36
}37
if (!parsed.rootAliases || typeof parsed.rootAliases !== 'object')38
{39
parsed.rootAliases = {};40
}41
parsed.scheduledRestart = normalizeScheduledRestart(parsed.scheduledRestart);42
return parsed;43
}44
28
function loadConfig ()45
function loadConfig ()29
{46
{30
try47
try31
{48
{32
const raw = fs.readFileSync(CONFIG_PATH, 'utf8');49
const stat = fs.statSync(CONFIG_PATH);33
const parsed = JSON.parse(raw);50
if (cachedConfig && cachedConfigMtimeMs === stat.mtimeMs)34
if (!parsed || typeof parsed !== 'object')35
{51
{36
return { port: 3000, paths: [], rootAliases: {} };52
return cachedConfig;37
}53
}38
if (!parsed.rootAliases || typeof parsed.rootAliases !== 'object')54
39
{55
const raw = fs.readFileSync(CONFIG_PATH, 'utf8');40
parsed.rootAliases = {};56
const parsed = normalizeLoadedConfig(JSON.parse(raw));41
}57
cachedConfig = parsed;42
parsed.scheduledRestart = normalizeScheduledRestart(parsed.scheduledRestart);58
cachedConfigMtimeMs = stat.mtimeMs;43
return parsed;59
return parsed;44
} catch60
} catch45
{61
{46
const defaults = { port: 3000, paths: [], rootAliases: {}, scheduledRestart: { ...DEFAULT_SCHEDULED_RESTART } };62
const defaults = { port: 3000, paths: [], rootAliases: {}, scheduledRestart: { ...DEFAULT_SCHEDULED_RESTART } };47
fs.writeFileSync(CONFIG_PATH, JSON.stringify(defaults, null, 2), 'utf8');63
fs.writeFileSync(CONFIG_PATH, JSON.stringify(defaults, null, 2), 'utf8');64
const stat = fs.statSync(CONFIG_PATH);65
cachedConfig = defaults;66
cachedConfigMtimeMs = stat.mtimeMs;48
return defaults;67
return defaults;49
}68
}50
}69
}@@ -58,6 +77,22 @@ function saveConfig (config)58
// Atomic replace to avoid partially-written config files.77
// Atomic replace to avoid partially-written config files.59
fs.writeFileSync(tmp, payload, 'utf8');78
fs.writeFileSync(tmp, payload, 'utf8');60
fs.renameSync(tmp, CONFIG_PATH);79
fs.renameSync(tmp, CONFIG_PATH);80
81
const normalized = normalizeLoadedConfig(config);82
cachedConfig = normalized;83
try84
{85
cachedConfigMtimeMs = fs.statSync(CONFIG_PATH).mtimeMs;86
} catch87
{88
cachedConfigMtimeMs = null;89
}90
}91
92
function invalidateConfigCache ()93
{94
cachedConfig = null;95
cachedConfigMtimeMs = null;61
}96
}62
97
63
function normalizeForCompare (p)98
function normalizeForCompare (p)@@ -116,4 +151,10 @@ function getAllowedRoot (filePath, allowedRoots)116
return null;151
return null;117
}152
}118
153
119
module.exports = { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot };154
module.exports = {155
loadConfig,156
saveConfig,157
invalidateConfigCache,158
getEffectivePaths,159
getAllowedRoot,160
};@@ -70,6 +70,20 @@ function initDb ()
70
70
);
71
71
72
72
CREATE INDEX IF NOT EXISTS idx_share_tokens_expires ON share_tokens(expires_at);
73
74
CREATE TABLE IF NOT EXISTS file_search_index (
75
file_path TEXT PRIMARY KEY,
76
root_path TEXT NOT NULL,
77
name TEXT NOT NULL,
78
name_lower TEXT NOT NULL,
79
relative_path TEXT NOT NULL,
80
relative_lower TEXT NOT NULL,
81
size INTEGER,
82
mtime_ms INTEGER,
83
indexed_at INTEGER NOT NULL
84
);
85
86
CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
73
87
`);
74
88
75
89
const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();
@@ -70,6 +70,20 @@ function initDb ()70
);70
);71
71
72
CREATE INDEX IF NOT EXISTS idx_share_tokens_expires ON share_tokens(expires_at);72
CREATE INDEX IF NOT EXISTS idx_share_tokens_expires ON share_tokens(expires_at);73
74
CREATE TABLE IF NOT EXISTS file_search_index (75
file_path TEXT PRIMARY KEY,76
root_path TEXT NOT NULL,77
name TEXT NOT NULL,78
name_lower TEXT NOT NULL,79
relative_path TEXT NOT NULL,80
relative_lower TEXT NOT NULL,81
size INTEGER,82
mtime_ms INTEGER,83
indexed_at INTEGER NOT NULL84
);85
86
CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);73
`);87
`);74
88
75
const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();89
const fileMetaCols = d.prepare('PRAGMA table_info(file_meta)').all();@@ -0,0 +1,260 @@
1
'use strict';
2
3
const crypto = require('crypto');
4
const fs = require('fs');
5
const path = require('path');
6
const { getDb } = require('./db');
7
8
const MAX_QUEUE_LENGTH = toPositiveInt(process.env.FILE_HASH_QUEUE_MAX_LENGTH, 4096);
9
const BACKLOG_BATCH_SIZE = toPositiveInt(process.env.FILE_HASH_BACKLOG_BATCH_SIZE, 500);
10
const ACTIVE_STREAM_DELAY_MS = toPositiveInt(process.env.FILE_HASH_ACTIVE_STREAM_DELAY_MS, 15000);
11
const IDLE_TICK_MS = toPositiveInt(process.env.FILE_HASH_IDLE_TICK_MS, 250);
12
13
const queue = [];
14
const queued = new Set();
15
16
let loopStarted = false;
17
let loopTimer = null;
18
let workerRunning = false;
19
20
function toPositiveInt (value, fallback)
21
{
22
const parsed = Number.parseInt(String(value || ''), 10);
23
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
24
}
25
26
function delay (ms)
27
{
28
return new Promise((resolve) => setTimeout(resolve, ms));
29
}
30
31
/**
32
* Computes the SHA-256 hex digest of the file at `filePath`.
33
* Streams the file so it works for large files without loading them into memory.
34
*/
35
function computeFileSha256 (filePath)
36
{
37
return new Promise((resolve, reject) =>
38
{
39
const hash = crypto.createHash('sha256');
40
const stream = fs.createReadStream(filePath);
41
stream.on('error', reject);
42
stream.on('data', (chunk) => hash.update(chunk));
43
stream.on('end', () => resolve(hash.digest('hex')));
44
});
45
}
46
47
function hasActiveVideoStreams ()
48
{
49
try
50
{
51
const { getActiveStreamCount } = require('./video-stream');
52
return Number(getActiveStreamCount()) > 0;
53
} catch
54
{
55
return false;
56
}
57
}
58
59
function loopDelayMs ()
60
{
61
return hasActiveVideoStreams() ? ACTIVE_STREAM_DELAY_MS : IDLE_TICK_MS;
62
}
63
64
function enqueueFileHash (filePath)
65
{
66
const resolved = path.resolve(String(filePath || ''));
67
if (!resolved) return false;
68
if (queued.has(resolved)) return false;
69
70
if (queue.length >= MAX_QUEUE_LENGTH)
71
{
72
const dropped = queue.shift();
73
if (dropped) queued.delete(dropped);
74
}
75
76
queued.add(resolved);
77
queue.push(resolved);
78
scheduleWorker();
79
return true;
80
}
81
82
function persistFileHash (filePath, sha256)
83
{
84
const db = getDb();
85
const existing = db.prepare('SELECT file_path FROM file_meta WHERE file_path = ?').get([filePath]);
86
87
if (existing)
88
{
89
db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, filePath]);
90
return;
91
}
92
93
let uploadedAt = null;
94
try
95
{
96
const stat = fs.statSync(filePath);
97
const birthtimeMs = Number(stat.birthtimeMs);
98
const ctimeMs = Number(stat.ctimeMs);
99
const mtimeMs = Number(stat.mtimeMs);
100
const chosenMs = (Number.isFinite(birthtimeMs) && birthtimeMs > 0)
101
? birthtimeMs
102
: (Number.isFinite(ctimeMs) && ctimeMs > 0)
103
? ctimeMs
104
: (Number.isFinite(mtimeMs) && mtimeMs > 0)
105
? mtimeMs
106
: null;
107
if (Number.isFinite(chosenMs) && chosenMs > 0)
108
{
109
uploadedAt = new Date(chosenMs).toISOString();
110
}
111
} catch
112
{
113
// Ignore missing files.
114
}
115
116
db.prepare(`
117
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256)
118
VALUES (?, ?, ?, ?, ?, ?)
119
ON CONFLICT(file_path) DO UPDATE SET sha256 = excluded.sha256
120
`).run([filePath, '', '[]', '{}', uploadedAt, sha256]);
121
}
122
123
async function processQueuedFile (filePath)
124
{
125
let stat;
126
try
127
{
128
stat = await fs.promises.stat(filePath);
129
} catch
130
{
131
return;
132
}
133
134
if (!stat.isFile()) return;
135
136
const sha256 = await computeFileSha256(filePath);
137
persistFileHash(filePath, sha256);
138
}
139
140
async function processQueueBatch (respectStreamDelay)
141
{
142
while (queue.length)
143
{
144
const filePath = queue.shift();
145
queued.delete(filePath);
146
if (!filePath) continue;
147
148
try
149
{
150
await processQueuedFile(filePath);
151
} catch (err)
152
{
153
console.error('[file-hash] hash failed:', filePath, err && err.message ? err.message : err);
154
}
155
156
if (queue.length && respectStreamDelay) await delay(loopDelayMs());
157
}
158
}
159
160
async function runWorkerPass ()
161
{
162
if (workerRunning) return;
163
workerRunning = true;
164
165
try
166
{
167
await processQueueBatch(true);
168
} finally
169
{
170
workerRunning = false;
171
}
172
}
173
174
function scheduleWorker ()
175
{
176
if (workerRunning || !queue.length) return;
177
178
void runWorkerPass().finally(() =>
179
{
180
if (queue.length) scheduleWorker();
181
});
182
}
183
184
function enqueueBacklogFromDb ()
185
{
186
const db = getDb();
187
const rows = db.prepare(`
188
SELECT file_path
189
FROM file_meta
190
WHERE sha256 IS NULL OR sha256 = ''
191
LIMIT ?
192
`).all([BACKLOG_BATCH_SIZE]);
193
194
for (const row of rows)
195
{
196
enqueueFileHash(row.file_path);
197
}
198
}
199
200
function startFileHashQueue ()
201
{
202
if (loopStarted) return;
203
loopStarted = true;
204
enqueueBacklogFromDb();
205
scheduleWorker();
206
207
loopTimer = setInterval(() =>
208
{
209
enqueueBacklogFromDb();
210
scheduleWorker();
211
}, 60 * 1000);
212
if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();
213
}
214
215
function stopFileHashQueue ()
216
{
217
loopStarted = false;
218
if (loopTimer)
219
{
220
clearInterval(loopTimer);
221
loopTimer = null;
222
}
223
queue.length = 0;
224
queued.clear();
225
}
226
227
/**
228
* Processes all queued hashes. Intended for tests and graceful shutdown drains.
229
*/
230
async function drainFileHashQueue ()
231
{
232
while (workerRunning) await delay(5);
233
workerRunning = true;
234
try
235
{
236
await processQueueBatch(false);
237
} finally
238
{
239
workerRunning = false;
240
}
241
}
242
243
function getFileHashQueueSnapshot ()
244
{
245
return {
246
loopStarted,
247
queueLength: queue.length,
248
workerRunning,
249
hasActiveVideoStreams: hasActiveVideoStreams(),
250
};
251
}
252
253
module.exports = {
254
computeFileSha256,
255
enqueueFileHash,
256
drainFileHashQueue,
257
startFileHashQueue,
258
stopFileHashQueue,
259
getFileHashQueueSnapshot,
260
};
@@ -0,0 +1,260 @@1
'use strict';2
3
const crypto = require('crypto');4
const fs = require('fs');5
const path = require('path');6
const { getDb } = require('./db');7
8
const MAX_QUEUE_LENGTH = toPositiveInt(process.env.FILE_HASH_QUEUE_MAX_LENGTH, 4096);9
const BACKLOG_BATCH_SIZE = toPositiveInt(process.env.FILE_HASH_BACKLOG_BATCH_SIZE, 500);10
const ACTIVE_STREAM_DELAY_MS = toPositiveInt(process.env.FILE_HASH_ACTIVE_STREAM_DELAY_MS, 15000);11
const IDLE_TICK_MS = toPositiveInt(process.env.FILE_HASH_IDLE_TICK_MS, 250);12
13
const queue = [];14
const queued = new Set();15
16
let loopStarted = false;17
let loopTimer = null;18
let workerRunning = false;19
20
function toPositiveInt (value, fallback)21
{22
const parsed = Number.parseInt(String(value || ''), 10);23
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;24
}25
26
function delay (ms)27
{28
return new Promise((resolve) => setTimeout(resolve, ms));29
}30
31
/**32
* Computes the SHA-256 hex digest of the file at `filePath`.33
* Streams the file so it works for large files without loading them into memory.34
*/35
function computeFileSha256 (filePath)36
{37
return new Promise((resolve, reject) =>38
{39
const hash = crypto.createHash('sha256');40
const stream = fs.createReadStream(filePath);41
stream.on('error', reject);42
stream.on('data', (chunk) => hash.update(chunk));43
stream.on('end', () => resolve(hash.digest('hex')));44
});45
}46
47
function hasActiveVideoStreams ()48
{49
try50
{51
const { getActiveStreamCount } = require('./video-stream');52
return Number(getActiveStreamCount()) > 0;53
} catch54
{55
return false;56
}57
}58
59
function loopDelayMs ()60
{61
return hasActiveVideoStreams() ? ACTIVE_STREAM_DELAY_MS : IDLE_TICK_MS;62
}63
64
function enqueueFileHash (filePath)65
{66
const resolved = path.resolve(String(filePath || ''));67
if (!resolved) return false;68
if (queued.has(resolved)) return false;69
70
if (queue.length >= MAX_QUEUE_LENGTH)71
{72
const dropped = queue.shift();73
if (dropped) queued.delete(dropped);74
}75
76
queued.add(resolved);77
queue.push(resolved);78
scheduleWorker();79
return true;80
}81
82
function persistFileHash (filePath, sha256)83
{84
const db = getDb();85
const existing = db.prepare('SELECT file_path FROM file_meta WHERE file_path = ?').get([filePath]);86
87
if (existing)88
{89
db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, filePath]);90
return;91
}92
93
let uploadedAt = null;94
try95
{96
const stat = fs.statSync(filePath);97
const birthtimeMs = Number(stat.birthtimeMs);98
const ctimeMs = Number(stat.ctimeMs);99
const mtimeMs = Number(stat.mtimeMs);100
const chosenMs = (Number.isFinite(birthtimeMs) && birthtimeMs > 0)101
? birthtimeMs102
: (Number.isFinite(ctimeMs) && ctimeMs > 0)103
? ctimeMs104
: (Number.isFinite(mtimeMs) && mtimeMs > 0)105
? mtimeMs106
: null;107
if (Number.isFinite(chosenMs) && chosenMs > 0)108
{109
uploadedAt = new Date(chosenMs).toISOString();110
}111
} catch112
{113
// Ignore missing files.114
}115
116
db.prepare(`117
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256)118
VALUES (?, ?, ?, ?, ?, ?)119
ON CONFLICT(file_path) DO UPDATE SET sha256 = excluded.sha256120
`).run([filePath, '', '[]', '{}', uploadedAt, sha256]);121
}122
123
async function processQueuedFile (filePath)124
{125
let stat;126
try127
{128
stat = await fs.promises.stat(filePath);129
} catch130
{131
return;132
}133
134
if (!stat.isFile()) return;135
136
const sha256 = await computeFileSha256(filePath);137
persistFileHash(filePath, sha256);138
}139
140
async function processQueueBatch (respectStreamDelay)141
{142
while (queue.length)143
{144
const filePath = queue.shift();145
queued.delete(filePath);146
if (!filePath) continue;147
148
try149
{150
await processQueuedFile(filePath);151
} catch (err)152
{153
console.error('[file-hash] hash failed:', filePath, err && err.message ? err.message : err);154
}155
156
if (queue.length && respectStreamDelay) await delay(loopDelayMs());157
}158
}159
160
async function runWorkerPass ()161
{162
if (workerRunning) return;163
workerRunning = true;164
165
try166
{167
await processQueueBatch(true);168
} finally169
{170
workerRunning = false;171
}172
}173
174
function scheduleWorker ()175
{176
if (workerRunning || !queue.length) return;177
178
void runWorkerPass().finally(() =>179
{180
if (queue.length) scheduleWorker();181
});182
}183
184
function enqueueBacklogFromDb ()185
{186
const db = getDb();187
const rows = db.prepare(`188
SELECT file_path189
FROM file_meta190
WHERE sha256 IS NULL OR sha256 = ''191
LIMIT ?192
`).all([BACKLOG_BATCH_SIZE]);193
194
for (const row of rows)195
{196
enqueueFileHash(row.file_path);197
}198
}199
200
function startFileHashQueue ()201
{202
if (loopStarted) return;203
loopStarted = true;204
enqueueBacklogFromDb();205
scheduleWorker();206
207
loopTimer = setInterval(() =>208
{209
enqueueBacklogFromDb();210
scheduleWorker();211
}, 60 * 1000);212
if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();213
}214
215
function stopFileHashQueue ()216
{217
loopStarted = false;218
if (loopTimer)219
{220
clearInterval(loopTimer);221
loopTimer = null;222
}223
queue.length = 0;224
queued.clear();225
}226
227
/**228
* Processes all queued hashes. Intended for tests and graceful shutdown drains.229
*/230
async function drainFileHashQueue ()231
{232
while (workerRunning) await delay(5);233
workerRunning = true;234
try235
{236
await processQueueBatch(false);237
} finally238
{239
workerRunning = false;240
}241
}242
243
function getFileHashQueueSnapshot ()244
{245
return {246
loopStarted,247
queueLength: queue.length,248
workerRunning,249
hasActiveVideoStreams: hasActiveVideoStreams(),250
};251
}252
253
module.exports = {254
computeFileSha256,255
enqueueFileHash,256
drainFileHashQueue,257
startFileHashQueue,258
stopFileHashQueue,259
getFileHashQueueSnapshot,260
};@@ -10,6 +10,8 @@ const { ensureServerKeys } = require('./crypto');
10
10
const { startAuthSessionCleanupLoop, stopAuthSessionCleanupLoop } = require('./auth');
11
11
const { loadConfig } = require('./config');
12
12
const { startThumbnailCacheLoop, stopThumbnailCacheLoop } = require('./thumbnail-cache');
13
const { startSearchIndexLoop, stopSearchIndexLoop } = require('./search-index');
14
const { startFileHashQueue, stopFileHashQueue } = require('./file-hash-queue');
13
15
const { startRestartScheduler, stopRestartScheduler } = require('./restart-scheduler');
14
16
const { logError } = require('./logger');
15
17
const authRoutes = require('./routes/auth');
@@ -65,6 +67,8 @@ function shutdown (opts = {}, ...servers)
65
67
active.forEach(s => { s.__isShuttingDown = true; });
66
68
67
69
stopThumbnailCacheLoop();
70
stopSearchIndexLoop();
71
stopFileHashQueue();
68
72
stopAuthSessionCleanupLoop();
69
73
stopShareMaintenanceLoop();
70
74
stopRestartScheduler();
@@ -127,6 +131,8 @@ function start ()
127
131
const shutdownForRestart = () => shutdown({ exitCode: 1 }, httpServer, httpsServer);
128
132
129
133
startThumbnailCacheLoop();
134
startSearchIndexLoop();
135
startFileHashQueue();
130
136
startAuthSessionCleanupLoop();
131
137
startShareMaintenanceLoop();
132
138
startRestartScheduler({ shutdownForRestart });
@@ -10,6 +10,8 @@ const { ensureServerKeys } = require('./crypto');10
const { startAuthSessionCleanupLoop, stopAuthSessionCleanupLoop } = require('./auth');10
const { startAuthSessionCleanupLoop, stopAuthSessionCleanupLoop } = require('./auth');11
const { loadConfig } = require('./config');11
const { loadConfig } = require('./config');12
const { startThumbnailCacheLoop, stopThumbnailCacheLoop } = require('./thumbnail-cache');12
const { startThumbnailCacheLoop, stopThumbnailCacheLoop } = require('./thumbnail-cache');13
const { startSearchIndexLoop, stopSearchIndexLoop } = require('./search-index');14
const { startFileHashQueue, stopFileHashQueue } = require('./file-hash-queue');13
const { startRestartScheduler, stopRestartScheduler } = require('./restart-scheduler');15
const { startRestartScheduler, stopRestartScheduler } = require('./restart-scheduler');14
const { logError } = require('./logger');16
const { logError } = require('./logger');15
const authRoutes = require('./routes/auth');17
const authRoutes = require('./routes/auth');@@ -65,6 +67,8 @@ function shutdown (opts = {}, ...servers)65
active.forEach(s => { s.__isShuttingDown = true; });67
active.forEach(s => { s.__isShuttingDown = true; });66
68
67
stopThumbnailCacheLoop();69
stopThumbnailCacheLoop();70
stopSearchIndexLoop();71
stopFileHashQueue();68
stopAuthSessionCleanupLoop();72
stopAuthSessionCleanupLoop();69
stopShareMaintenanceLoop();73
stopShareMaintenanceLoop();70
stopRestartScheduler();74
stopRestartScheduler();@@ -127,6 +131,8 @@ function start ()127
const shutdownForRestart = () => shutdown({ exitCode: 1 }, httpServer, httpsServer);131
const shutdownForRestart = () => shutdown({ exitCode: 1 }, httpServer, httpsServer);128
132
129
startThumbnailCacheLoop();133
startThumbnailCacheLoop();134
startSearchIndexLoop();135
startFileHashQueue();130
startAuthSessionCleanupLoop();136
startAuthSessionCleanupLoop();131
startShareMaintenanceLoop();137
startShareMaintenanceLoop();132
startRestartScheduler({ shutdownForRestart });138
startRestartScheduler({ shutdownForRestart });@@ -43,7 +43,7 @@ function ensureUserFolders (userPath)
43
43
}
44
44
45
45
const { getDb } = require('../db');
46
const { requireAuth, requireAdmin } = require('../auth');
46
const { requireAuth, requireAdmin, clearAuthUserCache } = require('../auth');
47
47
const { loadConfig, saveConfig, getEffectivePaths } = require('../config');
48
48
const debugRouter = require('./debug');
49
49
@@ -229,6 +229,7 @@ router.post('/users', (req, res) =>
229
229
try { fs.mkdirSync(folder, { recursive: true }); } catch { /* non-fatal */ }
230
230
}
231
231
232
clearAuthUserCache(username);
232
233
res.status(201).json({ ok: true });
233
234
});
234
235
@@ -279,6 +280,7 @@ router.patch('/users/:id', (req, res) =>
279
280
db.prepare('UPDATE users SET permissions = ? WHERE id = ?').run([JSON.stringify(normalizedPermissions), user.id]);
280
281
}
281
282
283
clearAuthUserCache(user.username);
282
284
res.json({ ok: true });
283
285
});
284
286
@@ -307,6 +309,7 @@ router.delete('/users/:id', (req, res) =>
307
309
}
308
310
309
311
db.prepare('DELETE FROM users WHERE id = ?').run([user.id]);
312
clearAuthUserCache(user.username);
310
313
311
314
// If the deleted user's personal folder is now empty, remove it automatically.
312
315
const config = loadConfig();
@@ -43,7 +43,7 @@ function ensureUserFolders (userPath)43
}43
}44
44
45
const { getDb } = require('../db');45
const { getDb } = require('../db');46
const { requireAuth, requireAdmin } = require('../auth');46
const { requireAuth, requireAdmin, clearAuthUserCache } = require('../auth');47
const { loadConfig, saveConfig, getEffectivePaths } = require('../config');47
const { loadConfig, saveConfig, getEffectivePaths } = require('../config');48
const debugRouter = require('./debug');48
const debugRouter = require('./debug');49
49
@@ -229,6 +229,7 @@ router.post('/users', (req, res) =>229
try { fs.mkdirSync(folder, { recursive: true }); } catch { /* non-fatal */ }229
try { fs.mkdirSync(folder, { recursive: true }); } catch { /* non-fatal */ }230
}230
}231
231
232
clearAuthUserCache(username);232
res.status(201).json({ ok: true });233
res.status(201).json({ ok: true });233
});234
});234
235
@@ -279,6 +280,7 @@ router.patch('/users/:id', (req, res) =>279
db.prepare('UPDATE users SET permissions = ? WHERE id = ?').run([JSON.stringify(normalizedPermissions), user.id]);280
db.prepare('UPDATE users SET permissions = ? WHERE id = ?').run([JSON.stringify(normalizedPermissions), user.id]);280
}281
}281
282
283
clearAuthUserCache(user.username);282
res.json({ ok: true });284
res.json({ ok: true });283
});285
});284
286
@@ -307,6 +309,7 @@ router.delete('/users/:id', (req, res) =>307
}309
}308
310
309
db.prepare('DELETE FROM users WHERE id = ?').run([user.id]);311
db.prepare('DELETE FROM users WHERE id = ?').run([user.id]);312
clearAuthUserCache(user.username);310
313
311
// If the deleted user's personal folder is now empty, remove it automatically.314
// If the deleted user's personal folder is now empty, remove it automatically.312
const config = loadConfig();315
const config = loadConfig();@@ -10,34 +10,27 @@ const router = express.Router();
10
10
const { getDb } = require('../db');
11
11
const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');
12
12
const { requireAuth } = require('../auth');
13
const { getCachedThumbnailInfo, getFilesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
13
const {
14
getCachedThumbnailInfo,
15
getFilesWithThumbnailsInDir,
16
getFolderIconSourceFile,
17
setFolderIconAssignment,
18
getThumbnailManifestSourcePathSet,
19
THUMB_HTTP_MAX_AGE_SEC,
20
} = require('../thumbnail-cache');
21
const {
22
searchFiles,
23
deleteIndexPath,
24
deleteIndexUnder,
25
rewriteIndexPaths,
26
reindexDirectoryTree,
27
} = require('../search-index');
14
28
const { logError } = require('../logger');
15
29
const { renderImageSubtitleFrame, renderImageSubtitleFramesInRange } = require('../image-subtitles');
16
30
const { appendSubtitleDebugEntries, SUBTITLE_DEBUG_LOG_DIR } = require('../subtitle-debug-log');
17
31
const { buildZip } = require('./share');
18
32
const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
19
20
// ---------------------------------------------------------------------------
21
// SHA-256 hashing helper
22
// ---------------------------------------------------------------------------
23
24
/**
25
* Computes the SHA-256 hex digest of the file at `filePath`.
26
* Streams the file so it works for large files without loading them into memory.
27
* @param {string} filePath
28
* @returns {Promise<string>} hex digest
29
*/
30
function computeFileSha256 (filePath)
31
{
32
return new Promise((resolve, reject) =>
33
{
34
const hash = crypto.createHash('sha256');
35
const stream = fs.createReadStream(filePath);
36
stream.on('error', reject);
37
stream.on('data', (chunk) => hash.update(chunk));
38
stream.on('end', () => resolve(hash.digest('hex')));
39
});
40
}
33
const { enqueueFileHash } = require('../file-hash-queue');
41
34
42
35
// ---------------------------------------------------------------------------
43
36
// Multer storage — files are staged into temporary files and only promoted
@@ -270,22 +263,43 @@ function effectivePaths ()
270
263
return getEffectivePaths(loadConfig().paths || []);
271
264
}
272
265
273
function ensureUserNavigationTable (db)
266
const META_BATCH_SIZE = 400;
267
268
function loadUploadedAtByPaths (db, paths)
269
{
270
const map = new Map();
271
const unique = [...new Set(paths.map((p) => String(p || '')).filter(Boolean))];
272
for (let i = 0; i < unique.length; i += META_BATCH_SIZE)
273
{
274
const chunk = unique.slice(i, i + META_BATCH_SIZE);
275
const placeholders = chunk.map(() => '?').join(', ');
276
const rows = db.prepare(`SELECT file_path, uploaded_at FROM file_meta WHERE file_path IN (${placeholders})`)
277
.all(chunk);
278
for (const row of rows)
279
{
280
map.set(row.file_path, String(row.uploaded_at || '').trim());
281
}
282
}
283
return map;
284
}
285
286
function persistMissingUploadedAt (db, rows)
274
287
{
275
db.exec(`
276
CREATE TABLE IF NOT EXISTS user_last_navigation (
277
username TEXT PRIMARY KEY,
278
last_dir TEXT,
279
updated_at INTEGER NOT NULL
280
);
288
if (!rows.length) return;
289
290
const stmt = db.prepare(`
291
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at)
292
VALUES (?, ?, ?, ?, ?)
293
ON CONFLICT(file_path) DO UPDATE SET
294
uploaded_at = COALESCE(NULLIF(file_meta.uploaded_at, ''), excluded.uploaded_at)
281
295
`);
296
for (const row of rows) stmt.run(row);
282
297
}
283
298
284
299
function setUserLastDir (username, dirPath)
285
300
{
286
301
if (!username) return;
287
302
const db = getDb();
288
ensureUserNavigationTable(db);
289
303
db.prepare(`
290
304
INSERT INTO user_last_navigation (username, last_dir, updated_at)
291
305
VALUES (?, ?, ?)
@@ -299,7 +313,6 @@ function getUserLastDir (username)
299
313
{
300
314
if (!username) return null;
301
315
const db = getDb();
302
ensureUserNavigationTable(db);
303
316
const row = db.prepare('SELECT last_dir FROM user_last_navigation WHERE username = ?').get([String(username)]);
304
317
const value = String(row && row.last_dir || '').trim();
305
318
return value || null;
@@ -309,7 +322,6 @@ function clearUserLastDir (username)
309
322
{
310
323
if (!username) return;
311
324
const db = getDb();
312
ensureUserNavigationTable(db);
313
325
db.prepare('DELETE FROM user_last_navigation WHERE username = ?').run([String(username)]);
314
326
}
315
327
@@ -355,45 +367,6 @@ function readableSearchRootsForUser (user, config, roots)
355
367
});
356
368
}
357
369
358
function fuzzyMatchScore (needle, haystack)
359
{
360
const q = String(needle || '').trim().toLowerCase();
361
const t = String(haystack || '').toLowerCase();
362
if (!q || !t) return null;
363
364
// Strong boost for contiguous substring matches.
365
const idx = t.indexOf(q);
366
if (idx >= 0)
367
{
368
const compactBonus = Math.max(0, 200 - idx);
369
const lengthBonus = Math.max(0, 120 - (t.length - q.length));
370
return 1200 + compactBonus + lengthBonus;
371
}
372
373
// Fallback: subsequence fuzzy match.
374
let qPos = 0;
375
let firstHit = -1;
376
let lastHit = -1;
377
let gaps = 0;
378
379
for (let i = 0; i < t.length && qPos < q.length; i++)
380
{
381
if (t[i] !== q[qPos]) continue;
382
if (firstHit === -1) firstHit = i;
383
if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1);
384
lastHit = i;
385
qPos += 1;
386
}
387
388
if (qPos !== q.length) return null;
389
390
const span = lastHit - firstHit + 1;
391
const compactness = Math.max(0, 300 - span);
392
const gapPenalty = Math.min(180, gaps * 3);
393
const startBonus = Math.max(0, 120 - firstHit);
394
return 700 + compactness + startBonus - gapPenalty;
395
}
396
397
370
const SEARCHABLE_IMAGE_EXTS = new Set([
398
371
'.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.svg', '.avif', '.ico', '.tif', '.tiff',
399
372
]);
@@ -404,164 +377,28 @@ function isSearchableImageFileName (name)
404
377
return SEARCHABLE_IMAGE_EXTS.has(ext);
405
378
}
406
379
407
function searchFilesInRoots (query, roots, maxResults)
408
{
409
const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
410
const results = [];
411
const visitedDirs = new Set();
412
413
for (const root of roots)
414
{
415
const rootPath = path.resolve(root);
416
const stack = [rootPath];
417
418
while (stack.length)
419
{
420
const dir = stack.pop();
421
if (!dir) continue;
422
const dirKey = path.resolve(dir);
423
if (visitedDirs.has(dirKey)) continue;
424
visitedDirs.add(dirKey);
425
426
let entries;
427
try
428
{
429
entries = fs.readdirSync(dir, { withFileTypes: true });
430
} catch
431
{
432
continue;
433
}
434
435
for (const entry of entries)
436
{
437
const fullPath = path.join(dir, entry.name);
438
if (entry.isDirectory())
439
{
440
stack.push(fullPath);
441
continue;
442
}
443
if (!entry.isFile()) continue;
444
445
const relativePath = path.relative(rootPath, fullPath);
446
const score = fuzzyMatchScore(query, entry.name)
447
?? fuzzyMatchScore(query, relativePath);
448
if (score == null) continue;
449
450
let stat = null;
451
try { stat = fs.statSync(fullPath); } catch { /* ignore disappearing files */ }
452
453
results.push({
454
name: entry.name,
455
path: fullPath,
456
dir: path.dirname(fullPath),
457
rootPath,
458
relativePath,
459
size: stat ? stat.size : null,
460
mtime: stat ? stat.mtime.toISOString() : null,
461
score,
462
});
463
464
if (results.length > limit * 4)
465
{
466
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
467
results.length = limit * 2;
468
}
469
}
470
}
471
}
472
473
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
474
return results.slice(0, limit);
475
}
476
477
380
async function searchImagesWithThumbnailsInRoots (query, roots, maxResults)
478
381
{
479
382
const limit = Math.max(1, Math.min(120, Number(maxResults) || 60));
480
const maxScannedFiles = 25000;
481
const results = [];
482
const visitedDirs = new Set();
483
let scannedFiles = 0;
484
let truncated = false;
383
const candidateLimit = Math.min(250, Math.max(limit * 4, 80));
384
const indexed = await searchFiles(query, roots, candidateLimit);
385
const thumbnailedPaths = await getThumbnailManifestSourcePathSet();
485
386
486
for (const root of roots)
387
const results = [];
388
for (const item of indexed)
487
389
{
488
if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
489
490
const rootPath = path.resolve(root);
491
const stack = [rootPath];
492
493
while (stack.length)
494
{
495
if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
496
497
const dir = stack.pop();
498
if (!dir) continue;
499
const dirKey = path.resolve(dir);
500
if (visitedDirs.has(dirKey)) continue;
501
visitedDirs.add(dirKey);
502
503
let entries;
504
try
505
{
506
entries = fs.readdirSync(dir, { withFileTypes: true });
507
} catch
508
{
509
continue;
510
}
390
if (!isSearchableImageFileName(item.name)) continue;
391
if (!thumbnailedPaths.has(path.resolve(item.path))) continue;
511
392
512
for (const entry of entries)
513
{
514
if (results.length >= limit || scannedFiles >= maxScannedFiles) break;
515
516
const fullPath = path.join(dir, entry.name);
517
if (entry.isDirectory())
518
{
519
stack.push(fullPath);
520
continue;
521
}
522
if (!entry.isFile()) continue;
523
524
scannedFiles += 1;
525
if (!isSearchableImageFileName(entry.name)) continue;
526
527
const relativePath = path.relative(rootPath, fullPath);
528
const score = fuzzyMatchScore(query, entry.name)
529
?? fuzzyMatchScore(query, relativePath);
530
if (score == null) continue;
531
532
// Only include files that already have a ready thumbnail.
533
const thumb = await getCachedThumbnailInfo(fullPath);
534
if (!thumb) continue;
535
536
let stat = null;
537
try { stat = fs.statSync(fullPath); } catch { /* ignore disappearing files */ }
538
539
results.push({
540
name: entry.name,
541
path: fullPath,
542
dir: path.dirname(fullPath),
543
rootPath,
544
relativePath,
545
size: stat ? stat.size : null,
546
mtime: stat ? stat.mtime.toISOString() : null,
547
score,
548
});
549
550
if (results.length > limit * 3)
551
{
552
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
553
results.length = limit;
554
}
555
}
556
}
393
const { score, ...rest } = item;
394
results.push(rest);
395
if (results.length >= limit) break;
557
396
}
558
397
559
if (scannedFiles >= maxScannedFiles) truncated = true;
560
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
561
398
return {
562
results: results.slice(0, limit),
563
scannedFiles,
564
truncated,
399
results,
400
scannedFiles: indexed.length,
401
truncated: indexed.length >= candidateLimit,
565
402
};
566
403
}
567
404
@@ -1180,7 +1017,7 @@ router.post('/root-alias', requireAuth, (req, res) =>
1180
1017
// GET /api/files/list?dir=<absolute-path>
1181
1018
// Lists files and folders in the given directory.
1182
1019
// ---------------------------------------------------------------------------
1183
router.get('/list', requireAuth, (req, res) =>
1020
router.get('/list', requireAuth, async (req, res) =>
1184
1021
{
1185
1022
const dir = req.query.dir;
1186
1023
if (!dir) return res.status(400).json({ error: 'dir is required.' });
@@ -1203,38 +1040,48 @@ router.get('/list', requireAuth, (req, res) =>
1203
1040
let entries;
1204
1041
try
1205
1042
{
1206
entries = fs.readdirSync(dir, { withFileTypes: true });
1043
entries = await fs.promises.readdir(dir, { withFileTypes: true });
1207
1044
} catch
1208
1045
{
1209
1046
return res.status(404).json({ error: 'Directory not found.' });
1210
1047
}
1211
1048
1212
const items = entries.map(e =>
1049
const fullPaths = entries.map((e) => path.join(dir, e.name));
1050
const db = getDb();
1051
const uploadedAtByPath = loadUploadedAtByPaths(db, fullPaths);
1052
const metaUpserts = [];
1053
1054
const statResults = await Promise.all(fullPaths.map(async (fullPath) =>
1055
{
1056
try
1057
{
1058
return { fullPath, stat: await fs.promises.stat(fullPath) };
1059
} catch
1060
{
1061
return { fullPath, stat: null };
1062
}
1063
}));
1064
1065
const statByPath = new Map(statResults.map((row) => [row.fullPath, row.stat]));
1066
1067
const items = entries.map((e) =>
1213
1068
{
1214
1069
const fullPath = path.join(dir, e.name);
1070
const stat = statByPath.get(fullPath);
1215
1071
let size = null;
1216
1072
let mtime = null;
1217
1073
let uploadedAt = null;
1218
try
1074
1075
if (stat)
1219
1076
{
1220
const stat = fs.statSync(fullPath);
1221
1077
size = stat.size;
1222
1078
mtime = stat.mtime.toISOString();
1223
1224
const db = getDb();
1225
const existing = db.prepare('SELECT uploaded_at FROM file_meta WHERE file_path = ?').get([fullPath]);
1226
uploadedAt = String(existing && existing.uploaded_at || '').trim() || uploadedAtFromStat(stat);
1227
1228
if (uploadedAt)
1079
uploadedAt = uploadedAtByPath.get(fullPath) || uploadedAtFromStat(stat) || null;
1080
if (uploadedAt && !uploadedAtByPath.has(fullPath))
1229
1081
{
1230
db.prepare(`
1231
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at)
1232
VALUES (?, ?, ?, ?, ?)
1233
ON CONFLICT(file_path) DO UPDATE SET
1234
uploaded_at = COALESCE(NULLIF(file_meta.uploaded_at, ''), excluded.uploaded_at)
1235
`).run([fullPath, '', '[]', '{}', uploadedAt]);
1082
metaUpserts.push([fullPath, '', '[]', '{}', uploadedAt]);
1236
1083
}
1237
} catch { /* ignore */ }
1084
}
1238
1085
1239
1086
return {
1240
1087
name: e.name,
@@ -1246,6 +1093,14 @@ router.get('/list', requireAuth, (req, res) =>
1246
1093
};
1247
1094
});
1248
1095
1096
try
1097
{
1098
persistMissingUploadedAt(db, metaUpserts);
1099
} catch
1100
{
1101
// Non-fatal metadata persistence.
1102
}
1103
1249
1104
try
1250
1105
{
1251
1106
setUserLastDir(req.user && req.user.username, dir);
@@ -1261,7 +1116,7 @@ router.get('/list', requireAuth, (req, res) =>
1261
1116
// GET /api/files/search?query=<text>&limit=<number>
1262
1117
// Fuzzy-searches all readable roots for files.
1263
1118
// ---------------------------------------------------------------------------
1264
router.get('/search', requireAuth, (req, res) =>
1119
router.get('/search', requireAuth, async (req, res) =>
1265
1120
{
1266
1121
const query = String(req.query.query || '').trim();
1267
1122
if (!query) return res.status(400).json({ error: 'query is required.' });
@@ -1279,7 +1134,7 @@ router.get('/search', requireAuth, (req, res) =>
1279
1134
}
1280
1135
});
1281
1136
1282
const results = searchFilesInRoots(query, readableRoots, req.query.limit);
1137
const results = await searchFiles(query, readableRoots, req.query.limit);
1283
1138
res.json({
1284
1139
query,
1285
1140
results,
@@ -1367,26 +1222,10 @@ router.get('/meta', requireAuth, async (req, res) =>
1367
1222
}
1368
1223
}
1369
1224
1370
// Lazily compute and persist the SHA-256 hash when it is absent.
1371
1225
let sha256 = meta ? (meta.sha256 || null) : null;
1372
1226
if (!sha256 && !stat.isDirectory())
1373
1227
{
1374
try
1375
{
1376
sha256 = await computeFileSha256(file);
1377
if (meta)
1378
{
1379
db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, file]);
1380
} else
1381
{
1382
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')
1383
.run([file, '', '[]', '{}', uploadedAt || uploadedAtFromStat(stat), sha256]);
1384
meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);
1385
}
1386
} catch
1387
{
1388
// Non-fatal — hash will be retried next time.
1389
}
1228
enqueueFileHash(file);
1390
1229
}
1391
1230
1392
1231
res.json({
@@ -1422,6 +1261,7 @@ router.get('/duplicates', requireAuth, (req, res) =>
1422
1261
const hash = row ? (row.sha256 || null) : null;
1423
1262
if (!hash)
1424
1263
{
1264
enqueueFileHash(file);
1425
1265
return res.json({ duplicates: [] });
1426
1266
}
1427
1267
@@ -1990,7 +1830,7 @@ router.get('/thumbnail', requireAuth, async (req, res) =>
1990
1830
}
1991
1831
1992
1832
res.type(thumb.contentType);
1993
res.set('Cache-Control', 'private, max-age=10');
1833
res.set('Cache-Control', `private, max-age=${THUMB_HTTP_MAX_AGE_SEC}`);
1994
1834
res.sendFile(thumb.thumbPath);
1995
1835
});
1996
1836
@@ -2049,26 +1889,27 @@ router.post('/upload', requireAuth, (req, res, next) =>
2049
1889
const filePath = path.join(dir, f.originalname);
2050
1890
const uploadedAt = new Date().toISOString();
2051
1891
2052
let sha256 = null;
2053
try
2054
{
2055
sha256 = await computeFileSha256(filePath);
2056
} catch
2057
{
2058
// Non-fatal — hash can be computed lazily when the file is accessed.
2059
}
2060
2061
1892
const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);
2062
1893
if (existing)
2063
1894
{
2064
db.prepare('UPDATE file_meta SET uploader = ?, sha256 = ?, uploaded_at = ? WHERE file_path = ?')
2065
.run([req.user.username, sha256, uploadedAt, filePath]);
1895
db.prepare('UPDATE file_meta SET uploader = ?, sha256 = NULL, uploaded_at = ? WHERE file_path = ?')
1896
.run([req.user.username, uploadedAt, filePath]);
2066
1897
} else
2067
1898
{
2068
1899
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')
2069
.run([filePath, req.user.username, '[]', '{}', uploadedAt, sha256]);
1900
.run([filePath, req.user.username, '[]', '{}', uploadedAt, null]);
2070
1901
}
1902
1903
enqueueFileHash(filePath);
1904
}
1905
try
1906
{
1907
await reindexDirectoryTree(dir);
1908
} catch
1909
{
1910
// Non-fatal search index refresh.
2071
1911
}
1912
2072
1913
res.json({ uploaded: (req.files || []).map(f => f.originalname) });
2073
1914
});
2074
1915
});
@@ -2112,6 +1953,7 @@ router.post('/rename', requireAuth, (req, res) =>
2112
1953
}
2113
1954
2114
1955
rewriteMetaPaths(from, dest);
1956
rewriteIndexPaths(from, dest);
2115
1957
2116
1958
res.json({ ok: true });
2117
1959
});
@@ -2223,6 +2065,7 @@ router.post('/move', requireAuth, (req, res) =>
2223
2065
}
2224
2066
2225
2067
rewriteMetaPaths(from, dest);
2068
rewriteIndexPaths(from, dest);
2226
2069
res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });
2227
2070
});
2228
2071
@@ -2306,6 +2149,8 @@ router.post('/delete', requireAuth, (req, res) =>
2306
2149
db.prepare("DELETE FROM file_meta WHERE file_path = ? OR file_path LIKE ?")
2307
2150
.run([target, target + path.sep + '%']);
2308
2151
2152
deleteIndexUnder(target);
2153
2309
2154
res.json({ ok: true });
2310
2155
});
2311
2156
@@ -2482,7 +2327,7 @@ router.post('/global-favicon', requireAuth, async (req, res) =>
2482
2327
// Copies a file (not a directory) to the destination folder.
2483
2328
// Supports conflict: 'error' (default) | 'overwrite' | 'rename'
2484
2329
// ---------------------------------------------------------------------------
2485
router.post('/copy', requireAuth, (req, res) =>
2330
router.post('/copy', requireAuth, async (req, res) =>
2486
2331
{
2487
2332
const { from, toDir } = req.body;
2488
2333
const conflict = String(req.body && req.body.conflict || 'error').toLowerCase();
@@ -2562,6 +2407,16 @@ router.post('/copy', requireAuth, (req, res) =>
2562
2407
return res.status(500).json({ error: err.message });
2563
2408
}
2564
2409
2410
try
2411
{
2412
await reindexDirectoryTree(toDir);
2413
} catch
2414
{
2415
// Non-fatal search index refresh.
2416
}
2417
2418
enqueueFileHash(dest);
2419
2565
2420
res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });
2566
2421
});
2567
2422
@@ -10,34 +10,27 @@ const router = express.Router();10
const { getDb } = require('../db');10
const { getDb } = require('../db');11
const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');11
const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');12
const { requireAuth } = require('../auth');12
const { requireAuth } = require('../auth');13
const { getCachedThumbnailInfo, getFilesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');13
const {14
getCachedThumbnailInfo,15
getFilesWithThumbnailsInDir,16
getFolderIconSourceFile,17
setFolderIconAssignment,18
getThumbnailManifestSourcePathSet,19
THUMB_HTTP_MAX_AGE_SEC,20
} = require('../thumbnail-cache');21
const {22
searchFiles,23
deleteIndexPath,24
deleteIndexUnder,25
rewriteIndexPaths,26
reindexDirectoryTree,27
} = require('../search-index');14
const { logError } = require('../logger');28
const { logError } = require('../logger');15
const { renderImageSubtitleFrame, renderImageSubtitleFramesInRange } = require('../image-subtitles');29
const { renderImageSubtitleFrame, renderImageSubtitleFramesInRange } = require('../image-subtitles');16
const { appendSubtitleDebugEntries, SUBTITLE_DEBUG_LOG_DIR } = require('../subtitle-debug-log');30
const { appendSubtitleDebugEntries, SUBTITLE_DEBUG_LOG_DIR } = require('../subtitle-debug-log');17
const { buildZip } = require('./share');31
const { buildZip } = require('./share');18
const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');32
const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');19
33
const { enqueueFileHash } = require('../file-hash-queue');20
// ---------------------------------------------------------------------------21
// SHA-256 hashing helper22
// ---------------------------------------------------------------------------23
24
/**25
* Computes the SHA-256 hex digest of the file at `filePath`.26
* Streams the file so it works for large files without loading them into memory.27
* @param {string} filePath28
* @returns {Promise<string>} hex digest29
*/30
function computeFileSha256 (filePath)31
{32
return new Promise((resolve, reject) =>33
{34
const hash = crypto.createHash('sha256');35
const stream = fs.createReadStream(filePath);36
stream.on('error', reject);37
stream.on('data', (chunk) => hash.update(chunk));38
stream.on('end', () => resolve(hash.digest('hex')));39
});40
}41
34
42
// ---------------------------------------------------------------------------35
// ---------------------------------------------------------------------------43
// Multer storage — files are staged into temporary files and only promoted36
// Multer storage — files are staged into temporary files and only promoted@@ -270,22 +263,43 @@ function effectivePaths ()270
return getEffectivePaths(loadConfig().paths || []);263
return getEffectivePaths(loadConfig().paths || []);271
}264
}272
265
273
function ensureUserNavigationTable (db)266
const META_BATCH_SIZE = 400;267
268
function loadUploadedAtByPaths (db, paths)269
{270
const map = new Map();271
const unique = [...new Set(paths.map((p) => String(p || '')).filter(Boolean))];272
for (let i = 0; i < unique.length; i += META_BATCH_SIZE)273
{274
const chunk = unique.slice(i, i + META_BATCH_SIZE);275
const placeholders = chunk.map(() => '?').join(', ');276
const rows = db.prepare(`SELECT file_path, uploaded_at FROM file_meta WHERE file_path IN (${placeholders})`)277
.all(chunk);278
for (const row of rows)279
{280
map.set(row.file_path, String(row.uploaded_at || '').trim());281
}282
}283
return map;284
}285
286
function persistMissingUploadedAt (db, rows)274
{287
{275
db.exec(`288
if (!rows.length) return;276
CREATE TABLE IF NOT EXISTS user_last_navigation (289
277
username TEXT PRIMARY KEY,290
const stmt = db.prepare(`278
last_dir TEXT,291
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at)279
updated_at INTEGER NOT NULL292
VALUES (?, ?, ?, ?, ?)280
);293
ON CONFLICT(file_path) DO UPDATE SET294
uploaded_at = COALESCE(NULLIF(file_meta.uploaded_at, ''), excluded.uploaded_at)281
`);295
`);296
for (const row of rows) stmt.run(row);282
}297
}283
298
284
function setUserLastDir (username, dirPath)299
function setUserLastDir (username, dirPath)285
{300
{286
if (!username) return;301
if (!username) return;287
const db = getDb();302
const db = getDb();288
ensureUserNavigationTable(db);289
db.prepare(`303
db.prepare(`290
INSERT INTO user_last_navigation (username, last_dir, updated_at)304
INSERT INTO user_last_navigation (username, last_dir, updated_at)291
VALUES (?, ?, ?)305
VALUES (?, ?, ?)@@ -299,7 +313,6 @@ function getUserLastDir (username)299
{313
{300
if (!username) return null;314
if (!username) return null;301
const db = getDb();315
const db = getDb();302
ensureUserNavigationTable(db);303
const row = db.prepare('SELECT last_dir FROM user_last_navigation WHERE username = ?').get([String(username)]);316
const row = db.prepare('SELECT last_dir FROM user_last_navigation WHERE username = ?').get([String(username)]);304
const value = String(row && row.last_dir || '').trim();317
const value = String(row && row.last_dir || '').trim();305
return value || null;318
return value || null;@@ -309,7 +322,6 @@ function clearUserLastDir (username)309
{322
{310
if (!username) return;323
if (!username) return;311
const db = getDb();324
const db = getDb();312
ensureUserNavigationTable(db);313
db.prepare('DELETE FROM user_last_navigation WHERE username = ?').run([String(username)]);325
db.prepare('DELETE FROM user_last_navigation WHERE username = ?').run([String(username)]);314
}326
}315
327
@@ -355,45 +367,6 @@ function readableSearchRootsForUser (user, config, roots)355
});367
});356
}368
}357
369
358
function fuzzyMatchScore (needle, haystack)359
{360
const q = String(needle || '').trim().toLowerCase();361
const t = String(haystack || '').toLowerCase();362
if (!q || !t) return null;363
364
// Strong boost for contiguous substring matches.365
const idx = t.indexOf(q);366
if (idx >= 0)367
{368
const compactBonus = Math.max(0, 200 - idx);369
const lengthBonus = Math.max(0, 120 - (t.length - q.length));370
return 1200 + compactBonus + lengthBonus;371
}372
373
// Fallback: subsequence fuzzy match.374
let qPos = 0;375
let firstHit = -1;376
let lastHit = -1;377
let gaps = 0;378
379
for (let i = 0; i < t.length && qPos < q.length; i++)380
{381
if (t[i] !== q[qPos]) continue;382
if (firstHit === -1) firstHit = i;383
if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1);384
lastHit = i;385
qPos += 1;386
}387
388
if (qPos !== q.length) return null;389
390
const span = lastHit - firstHit + 1;391
const compactness = Math.max(0, 300 - span);392
const gapPenalty = Math.min(180, gaps * 3);393
const startBonus = Math.max(0, 120 - firstHit);394
return 700 + compactness + startBonus - gapPenalty;395
}396
397
const SEARCHABLE_IMAGE_EXTS = new Set([370
const SEARCHABLE_IMAGE_EXTS = new Set([398
'.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.svg', '.avif', '.ico', '.tif', '.tiff',371
'.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.svg', '.avif', '.ico', '.tif', '.tiff',399
]);372
]);@@ -404,164 +377,28 @@ function isSearchableImageFileName (name)404
return SEARCHABLE_IMAGE_EXTS.has(ext);377
return SEARCHABLE_IMAGE_EXTS.has(ext);405
}378
}406
379
407
function searchFilesInRoots (query, roots, maxResults)408
{409
const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));410
const results = [];411
const visitedDirs = new Set();412
413
for (const root of roots)414
{415
const rootPath = path.resolve(root);416
const stack = [rootPath];417
418
while (stack.length)419
{420
const dir = stack.pop();421
if (!dir) continue;422
const dirKey = path.resolve(dir);423
if (visitedDirs.has(dirKey)) continue;424
visitedDirs.add(dirKey);425
426
let entries;427
try428
{429
entries = fs.readdirSync(dir, { withFileTypes: true });430
} catch431
{432
continue;433
}434
435
for (const entry of entries)436
{437
const fullPath = path.join(dir, entry.name);438
if (entry.isDirectory())439
{440
stack.push(fullPath);441
continue;442
}443
if (!entry.isFile()) continue;444
445
const relativePath = path.relative(rootPath, fullPath);446
const score = fuzzyMatchScore(query, entry.name)447
?? fuzzyMatchScore(query, relativePath);448
if (score == null) continue;449
450
let stat = null;451
try { stat = fs.statSync(fullPath); } catch { /* ignore disappearing files */ }452
453
results.push({454
name: entry.name,455
path: fullPath,456
dir: path.dirname(fullPath),457
rootPath,458
relativePath,459
size: stat ? stat.size : null,460
mtime: stat ? stat.mtime.toISOString() : null,461
score,462
});463
464
if (results.length > limit * 4)465
{466
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));467
results.length = limit * 2;468
}469
}470
}471
}472
473
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));474
return results.slice(0, limit);475
}476
477
async function searchImagesWithThumbnailsInRoots (query, roots, maxResults)380
async function searchImagesWithThumbnailsInRoots (query, roots, maxResults)478
{381
{479
const limit = Math.max(1, Math.min(120, Number(maxResults) || 60));382
const limit = Math.max(1, Math.min(120, Number(maxResults) || 60));480
const maxScannedFiles = 25000;383
const candidateLimit = Math.min(250, Math.max(limit * 4, 80));481
const results = [];384
const indexed = await searchFiles(query, roots, candidateLimit);482
const visitedDirs = new Set();385
const thumbnailedPaths = await getThumbnailManifestSourcePathSet();483
let scannedFiles = 0;484
let truncated = false;485
386
486
for (const root of roots)387
const results = [];388
for (const item of indexed)487
{389
{488
if (results.length >= limit || scannedFiles >= maxScannedFiles) break;390
if (!isSearchableImageFileName(item.name)) continue;489
391
if (!thumbnailedPaths.has(path.resolve(item.path))) continue;490
const rootPath = path.resolve(root);491
const stack = [rootPath];492
493
while (stack.length)494
{495
if (results.length >= limit || scannedFiles >= maxScannedFiles) break;496
497
const dir = stack.pop();498
if (!dir) continue;499
const dirKey = path.resolve(dir);500
if (visitedDirs.has(dirKey)) continue;501
visitedDirs.add(dirKey);502
503
let entries;504
try505
{506
entries = fs.readdirSync(dir, { withFileTypes: true });507
} catch508
{509
continue;510
}511
392
512
for (const entry of entries)393
const { score, ...rest } = item;513
{394
results.push(rest);514
if (results.length >= limit || scannedFiles >= maxScannedFiles) break;395
if (results.length >= limit) break;515
516
const fullPath = path.join(dir, entry.name);517
if (entry.isDirectory())518
{519
stack.push(fullPath);520
continue;521
}522
if (!entry.isFile()) continue;523
524
scannedFiles += 1;525
if (!isSearchableImageFileName(entry.name)) continue;526
527
const relativePath = path.relative(rootPath, fullPath);528
const score = fuzzyMatchScore(query, entry.name)529
?? fuzzyMatchScore(query, relativePath);530
if (score == null) continue;531
532
// Only include files that already have a ready thumbnail.533
const thumb = await getCachedThumbnailInfo(fullPath);534
if (!thumb) continue;535
536
let stat = null;537
try { stat = fs.statSync(fullPath); } catch { /* ignore disappearing files */ }538
539
results.push({540
name: entry.name,541
path: fullPath,542
dir: path.dirname(fullPath),543
rootPath,544
relativePath,545
size: stat ? stat.size : null,546
mtime: stat ? stat.mtime.toISOString() : null,547
score,548
});549
550
if (results.length > limit * 3)551
{552
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));553
results.length = limit;554
}555
}556
}557
}396
}558
397
559
if (scannedFiles >= maxScannedFiles) truncated = true;560
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));561
return {398
return {562
results: results.slice(0, limit),399
results,563
scannedFiles,400
scannedFiles: indexed.length,564
truncated,401
truncated: indexed.length >= candidateLimit,565
};402
};566
}403
}567
404
@@ -1180,7 +1017,7 @@ router.post('/root-alias', requireAuth, (req, res) =>1180
// GET /api/files/list?dir=<absolute-path>1017
// GET /api/files/list?dir=<absolute-path>1181
// Lists files and folders in the given directory.1018
// Lists files and folders in the given directory.1182
// ---------------------------------------------------------------------------1019
// ---------------------------------------------------------------------------1183
router.get('/list', requireAuth, (req, res) =>1020
router.get('/list', requireAuth, async (req, res) =>1184
{1021
{1185
const dir = req.query.dir;1022
const dir = req.query.dir;1186
if (!dir) return res.status(400).json({ error: 'dir is required.' });1023
if (!dir) return res.status(400).json({ error: 'dir is required.' });@@ -1203,38 +1040,48 @@ router.get('/list', requireAuth, (req, res) =>1203
let entries;1040
let entries;1204
try1041
try1205
{1042
{1206
entries = fs.readdirSync(dir, { withFileTypes: true });1043
entries = await fs.promises.readdir(dir, { withFileTypes: true });1207
} catch1044
} catch1208
{1045
{1209
return res.status(404).json({ error: 'Directory not found.' });1046
return res.status(404).json({ error: 'Directory not found.' });1210
}1047
}1211
1048
1212
const items = entries.map(e =>1049
const fullPaths = entries.map((e) => path.join(dir, e.name));1050
const db = getDb();1051
const uploadedAtByPath = loadUploadedAtByPaths(db, fullPaths);1052
const metaUpserts = [];1053
1054
const statResults = await Promise.all(fullPaths.map(async (fullPath) =>1055
{1056
try1057
{1058
return { fullPath, stat: await fs.promises.stat(fullPath) };1059
} catch1060
{1061
return { fullPath, stat: null };1062
}1063
}));1064
1065
const statByPath = new Map(statResults.map((row) => [row.fullPath, row.stat]));1066
1067
const items = entries.map((e) =>1213
{1068
{1214
const fullPath = path.join(dir, e.name);1069
const fullPath = path.join(dir, e.name);1070
const stat = statByPath.get(fullPath);1215
let size = null;1071
let size = null;1216
let mtime = null;1072
let mtime = null;1217
let uploadedAt = null;1073
let uploadedAt = null;1218
try1074
1075
if (stat)1219
{1076
{1220
const stat = fs.statSync(fullPath);1221
size = stat.size;1077
size = stat.size;1222
mtime = stat.mtime.toISOString();1078
mtime = stat.mtime.toISOString();1223
1079
uploadedAt = uploadedAtByPath.get(fullPath) || uploadedAtFromStat(stat) || null;1224
const db = getDb();1080
if (uploadedAt && !uploadedAtByPath.has(fullPath))1225
const existing = db.prepare('SELECT uploaded_at FROM file_meta WHERE file_path = ?').get([fullPath]);1226
uploadedAt = String(existing && existing.uploaded_at || '').trim() || uploadedAtFromStat(stat);1227
1228
if (uploadedAt)1229
{1081
{1230
db.prepare(`1082
metaUpserts.push([fullPath, '', '[]', '{}', uploadedAt]);1231
INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at)1232
VALUES (?, ?, ?, ?, ?)1233
ON CONFLICT(file_path) DO UPDATE SET1234
uploaded_at = COALESCE(NULLIF(file_meta.uploaded_at, ''), excluded.uploaded_at)1235
`).run([fullPath, '', '[]', '{}', uploadedAt]);1236
}1083
}1237
} catch { /* ignore */ }1084
}1238
1085
1239
return {1086
return {1240
name: e.name,1087
name: e.name,@@ -1246,6 +1093,14 @@ router.get('/list', requireAuth, (req, res) =>1246
};1093
};1247
});1094
});1248
1095
1096
try1097
{1098
persistMissingUploadedAt(db, metaUpserts);1099
} catch1100
{1101
// Non-fatal metadata persistence.1102
}1103
1249
try1104
try1250
{1105
{1251
setUserLastDir(req.user && req.user.username, dir);1106
setUserLastDir(req.user && req.user.username, dir);@@ -1261,7 +1116,7 @@ router.get('/list', requireAuth, (req, res) =>1261
// GET /api/files/search?query=<text>&limit=<number>1116
// GET /api/files/search?query=<text>&limit=<number>1262
// Fuzzy-searches all readable roots for files.1117
// Fuzzy-searches all readable roots for files.1263
// ---------------------------------------------------------------------------1118
// ---------------------------------------------------------------------------1264
router.get('/search', requireAuth, (req, res) =>1119
router.get('/search', requireAuth, async (req, res) =>1265
{1120
{1266
const query = String(req.query.query || '').trim();1121
const query = String(req.query.query || '').trim();1267
if (!query) return res.status(400).json({ error: 'query is required.' });1122
if (!query) return res.status(400).json({ error: 'query is required.' });@@ -1279,7 +1134,7 @@ router.get('/search', requireAuth, (req, res) =>1279
}1134
}1280
});1135
});1281
1136
1282
const results = searchFilesInRoots(query, readableRoots, req.query.limit);1137
const results = await searchFiles(query, readableRoots, req.query.limit);1283
res.json({1138
res.json({1284
query,1139
query,1285
results,1140
results,@@ -1367,26 +1222,10 @@ router.get('/meta', requireAuth, async (req, res) =>1367
}1222
}1368
}1223
}1369
1224
1370
// Lazily compute and persist the SHA-256 hash when it is absent.1371
let sha256 = meta ? (meta.sha256 || null) : null;1225
let sha256 = meta ? (meta.sha256 || null) : null;1372
if (!sha256 && !stat.isDirectory())1226
if (!sha256 && !stat.isDirectory())1373
{1227
{1374
try1228
enqueueFileHash(file);1375
{1376
sha256 = await computeFileSha256(file);1377
if (meta)1378
{1379
db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, file]);1380
} else1381
{1382
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')1383
.run([file, '', '[]', '{}', uploadedAt || uploadedAtFromStat(stat), sha256]);1384
meta = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([file]);1385
}1386
} catch1387
{1388
// Non-fatal — hash will be retried next time.1389
}1390
}1229
}1391
1230
1392
res.json({1231
res.json({@@ -1422,6 +1261,7 @@ router.get('/duplicates', requireAuth, (req, res) =>1422
const hash = row ? (row.sha256 || null) : null;1261
const hash = row ? (row.sha256 || null) : null;1423
if (!hash)1262
if (!hash)1424
{1263
{1264
enqueueFileHash(file);1425
return res.json({ duplicates: [] });1265
return res.json({ duplicates: [] });1426
}1266
}1427
1267
@@ -1990,7 +1830,7 @@ router.get('/thumbnail', requireAuth, async (req, res) =>1990
}1830
}1991
1831
1992
res.type(thumb.contentType);1832
res.type(thumb.contentType);1993
res.set('Cache-Control', 'private, max-age=10');1833
res.set('Cache-Control', `private, max-age=${THUMB_HTTP_MAX_AGE_SEC}`);1994
res.sendFile(thumb.thumbPath);1834
res.sendFile(thumb.thumbPath);1995
});1835
});1996
1836
@@ -2049,26 +1889,27 @@ router.post('/upload', requireAuth, (req, res, next) =>2049
const filePath = path.join(dir, f.originalname);1889
const filePath = path.join(dir, f.originalname);2050
const uploadedAt = new Date().toISOString();1890
const uploadedAt = new Date().toISOString();2051
1891
2052
let sha256 = null;2053
try2054
{2055
sha256 = await computeFileSha256(filePath);2056
} catch2057
{2058
// Non-fatal — hash can be computed lazily when the file is accessed.2059
}2060
2061
const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);1892
const existing = db.prepare('SELECT * FROM file_meta WHERE file_path = ?').get([filePath]);2062
if (existing)1893
if (existing)2063
{1894
{2064
db.prepare('UPDATE file_meta SET uploader = ?, sha256 = ?, uploaded_at = ? WHERE file_path = ?')1895
db.prepare('UPDATE file_meta SET uploader = ?, sha256 = NULL, uploaded_at = ? WHERE file_path = ?')2065
.run([req.user.username, sha256, uploadedAt, filePath]);1896
.run([req.user.username, uploadedAt, filePath]);2066
} else1897
} else2067
{1898
{2068
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')1899
db.prepare('INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256) VALUES (?, ?, ?, ?, ?, ?)')2069
.run([filePath, req.user.username, '[]', '{}', uploadedAt, sha256]);1900
.run([filePath, req.user.username, '[]', '{}', uploadedAt, null]);2070
}1901
}1902
1903
enqueueFileHash(filePath);1904
}1905
try1906
{1907
await reindexDirectoryTree(dir);1908
} catch1909
{1910
// Non-fatal search index refresh.2071
}1911
}1912
2072
res.json({ uploaded: (req.files || []).map(f => f.originalname) });1913
res.json({ uploaded: (req.files || []).map(f => f.originalname) });2073
});1914
});2074
});1915
});@@ -2112,6 +1953,7 @@ router.post('/rename', requireAuth, (req, res) =>2112
}1953
}2113
1954
2114
rewriteMetaPaths(from, dest);1955
rewriteMetaPaths(from, dest);1956
rewriteIndexPaths(from, dest);2115
1957
2116
res.json({ ok: true });1958
res.json({ ok: true });2117
});1959
});@@ -2223,6 +2065,7 @@ router.post('/move', requireAuth, (req, res) =>2223
}2065
}2224
2066
2225
rewriteMetaPaths(from, dest);2067
rewriteMetaPaths(from, dest);2068
rewriteIndexPaths(from, dest);2226
res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });2069
res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });2227
});2070
});2228
2071
@@ -2306,6 +2149,8 @@ router.post('/delete', requireAuth, (req, res) =>2306
db.prepare("DELETE FROM file_meta WHERE file_path = ? OR file_path LIKE ?")2149
db.prepare("DELETE FROM file_meta WHERE file_path = ? OR file_path LIKE ?")2307
.run([target, target + path.sep + '%']);2150
.run([target, target + path.sep + '%']);2308
2151
2152
deleteIndexUnder(target);2153
2309
res.json({ ok: true });2154
res.json({ ok: true });2310
});2155
});2311
2156
@@ -2482,7 +2327,7 @@ router.post('/global-favicon', requireAuth, async (req, res) =>2482
// Copies a file (not a directory) to the destination folder.2327
// Copies a file (not a directory) to the destination folder.2483
// Supports conflict: 'error' (default) | 'overwrite' | 'rename'2328
// Supports conflict: 'error' (default) | 'overwrite' | 'rename'2484
// ---------------------------------------------------------------------------2329
// ---------------------------------------------------------------------------2485
router.post('/copy', requireAuth, (req, res) =>2330
router.post('/copy', requireAuth, async (req, res) =>2486
{2331
{2487
const { from, toDir } = req.body;2332
const { from, toDir } = req.body;2488
const conflict = String(req.body && req.body.conflict || 'error').toLowerCase();2333
const conflict = String(req.body && req.body.conflict || 'error').toLowerCase();@@ -2562,6 +2407,16 @@ router.post('/copy', requireAuth, (req, res) =>2562
return res.status(500).json({ error: err.message });2407
return res.status(500).json({ error: err.message });2563
}2408
}2564
2409
2410
try2411
{2412
await reindexDirectoryTree(toDir);2413
} catch2414
{2415
// Non-fatal search index refresh.2416
}2417
2418
enqueueFileHash(dest);2419
2565
res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });2420
res.json({ ok: true, path: dest, conflictResolution: resolvedConflict });2566
});2421
});2567
2422
@@ -0,0 +1,818 @@
1
'use strict';
2
3
const fs = require('fs');
4
const path = require('path');
5
const { getDb } = require('./db');
6
const { loadConfig, getEffectivePaths } = require('./config');
7
8
const ACTIVE_CLIENT_GRACE_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_CLIENT_GRACE_MS, 5 * 60 * 1000);
9
const IDLE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_IDLE_SYNC_INTERVAL_MS, 30 * 60 * 1000);
10
const ACTIVE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_SYNC_INTERVAL_MS, 10 * 60 * 1000);
11
const MAX_CANDIDATES_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_CANDIDATES, 8000);
12
const MAX_SUBSEQUENCE_SCAN = toPositiveInt(process.env.SEARCH_INDEX_MAX_SUBSEQUENCE_SCAN, 12000);
13
const MAX_VALIDATE_HITS = toPositiveInt(process.env.SEARCH_INDEX_MAX_VALIDATE_HITS, 250);
14
const MAX_RESCAN_DIRS_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_RESCAN_DIRS, 12);
15
const INDEX_BATCH_SIZE = 400;
16
const ROOT_CHUNK_SIZE = 50;
17
18
let loopTimer = null;
19
let loopStarted = false;
20
let backgroundSyncRunning = false;
21
22
function toPositiveInt (value, fallback)
23
{
24
const parsed = Number.parseInt(String(value || ''), 10);
25
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
26
}
27
28
function yieldToEventLoop ()
29
{
30
return new Promise((resolve) => setImmediate(resolve));
31
}
32
33
function nowMs ()
34
{
35
return Date.now();
36
}
37
38
function fuzzyMatchScore (needle, haystack)
39
{
40
const q = String(needle || '').trim().toLowerCase();
41
const t = String(haystack || '').toLowerCase();
42
if (!q || !t) return null;
43
44
const idx = t.indexOf(q);
45
if (idx >= 0)
46
{
47
const compactBonus = Math.max(0, 200 - idx);
48
const lengthBonus = Math.max(0, 120 - (t.length - q.length));
49
return 1200 + compactBonus + lengthBonus;
50
}
51
52
let qPos = 0;
53
let firstHit = -1;
54
let lastHit = -1;
55
let gaps = 0;
56
57
for (let i = 0; i < t.length && qPos < q.length; i++)
58
{
59
if (t[i] !== q[qPos]) continue;
60
if (firstHit === -1) firstHit = i;
61
if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1);
62
lastHit = i;
63
qPos += 1;
64
}
65
66
if (qPos !== q.length) return null;
67
68
const span = lastHit - firstHit + 1;
69
const compactness = Math.max(0, 300 - span);
70
const gapPenalty = Math.min(180, gaps * 3);
71
const startBonus = Math.max(0, 120 - firstHit);
72
return 700 + compactness + startBonus - gapPenalty;
73
}
74
75
function effectiveIndexRoots ()
76
{
77
const config = loadConfig();
78
const roots = getEffectivePaths(config.paths || []);
79
const userPath = typeof config.userPath === 'string' ? config.userPath.trim() : '';
80
if (userPath) roots.push(userPath);
81
82
const seen = new Set();
83
const deduped = [];
84
for (const root of roots)
85
{
86
const key = path.resolve(root);
87
if (seen.has(key)) continue;
88
seen.add(key);
89
deduped.push(key);
90
}
91
return deduped;
92
}
93
94
function hasActiveClients ()
95
{
96
try
97
{
98
const { getActiveStreamCount } = require('./video-stream');
99
if (Number(getActiveStreamCount()) > 0) return true;
100
} catch
101
{
102
// Stream module unavailable — treat as no active streams.
103
}
104
105
const db = getDb();
106
const cutoff = nowMs() - ACTIVE_CLIENT_GRACE_MS;
107
const row = db.prepare(`
108
SELECT COUNT(*) AS n
109
FROM auth_sessions
110
WHERE revoked_at IS NULL AND last_seen_at >= ?
111
`).get([cutoff]);
112
113
return Number(row && row.n || 0) > 0;
114
}
115
116
function upsertIndexRow (db, row)
117
{
118
db.prepare(`
119
INSERT INTO file_search_index (
120
file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at
121
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
122
ON CONFLICT(file_path) DO UPDATE SET
123
root_path = excluded.root_path,
124
name = excluded.name,
125
name_lower = excluded.name_lower,
126
relative_path = excluded.relative_path,
127
relative_lower = excluded.relative_lower,
128
size = excluded.size,
129
mtime_ms = excluded.mtime_ms,
130
indexed_at = excluded.indexed_at
131
`).run([
132
row.file_path,
133
row.root_path,
134
row.name,
135
row.name_lower,
136
row.relative_path,
137
row.relative_lower,
138
row.size,
139
row.mtime_ms,
140
row.indexed_at,
141
]);
142
}
143
144
function deleteIndexPath (filePath)
145
{
146
getDb().prepare('DELETE FROM file_search_index WHERE file_path = ?').run([path.resolve(String(filePath || ''))]);
147
}
148
149
function deleteIndexUnder (targetPath)
150
{
151
const resolved = path.resolve(String(targetPath || ''));
152
const db = getDb();
153
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([resolved]);
154
db.prepare('DELETE FROM file_search_index WHERE file_path LIKE ?')
155
.run([resolved + path.sep + '%']);
156
}
157
158
function rewriteIndexPaths (fromPath, toPath)
159
{
160
const from = path.resolve(String(fromPath || ''));
161
const to = path.resolve(String(toPath || ''));
162
const db = getDb();
163
164
const rows = db.prepare(`
165
SELECT file_path, root_path, name, relative_path, size, mtime_ms
166
FROM file_search_index
167
WHERE file_path = ? OR file_path LIKE ?
168
`).all([from, from + path.sep + '%']);
169
170
if (!rows.length) return;
171
172
const del = db.prepare('DELETE FROM file_search_index WHERE file_path = ?');
173
for (const row of rows)
174
{
175
del.run([row.file_path]);
176
const nextPath = row.file_path === from
177
? to
178
: to + row.file_path.slice(from.length);
179
const nextName = path.basename(nextPath);
180
const rootPath = path.resolve(row.root_path);
181
const relativePath = path.relative(rootPath, nextPath);
182
upsertIndexRow(db, {
183
file_path: nextPath,
184
root_path: rootPath,
185
name: nextName,
186
name_lower: nextName.toLowerCase(),
187
relative_path: relativePath,
188
relative_lower: String(relativePath || '').toLowerCase(),
189
size: row.size,
190
mtime_ms: row.mtime_ms,
191
indexed_at: nowMs(),
192
});
193
}
194
}
195
196
async function walkFilesUnderRoot (storageRoot, onFile, startDir)
197
{
198
const resolvedRoot = path.resolve(storageRoot);
199
const stack = [path.resolve(startDir || storageRoot)];
200
const visitedDirs = new Set();
201
let dirsVisited = 0;
202
203
while (stack.length)
204
{
205
const dir = stack.pop();
206
if (!dir) continue;
207
const dirKey = path.resolve(dir);
208
if (visitedDirs.has(dirKey)) continue;
209
visitedDirs.add(dirKey);
210
dirsVisited += 1;
211
if (dirsVisited % 64 === 0) await yieldToEventLoop();
212
213
let entries;
214
try
215
{
216
entries = await fs.promises.readdir(dir, { withFileTypes: true });
217
} catch
218
{
219
continue;
220
}
221
222
for (const entry of entries)
223
{
224
const fullPath = path.join(dir, entry.name);
225
if (entry.isDirectory())
226
{
227
stack.push(fullPath);
228
continue;
229
}
230
if (!entry.isFile()) continue;
231
232
let stat;
233
try
234
{
235
stat = await fs.promises.stat(fullPath);
236
} catch
237
{
238
continue;
239
}
240
241
const relativePath = path.relative(resolvedRoot, fullPath);
242
await onFile({
243
file_path: fullPath,
244
root_path: resolvedRoot,
245
name: entry.name,
246
name_lower: entry.name.toLowerCase(),
247
relative_path: relativePath,
248
relative_lower: String(relativePath || '').toLowerCase(),
249
size: stat.size,
250
mtime_ms: stat.mtimeMs,
251
});
252
}
253
}
254
}
255
256
async function syncSearchIndexForRoot (rootPath)
257
{
258
const resolvedRoot = path.resolve(rootPath);
259
try
260
{
261
const stat = await fs.promises.stat(resolvedRoot);
262
if (!stat.isDirectory()) return { rootPath: resolvedRoot, indexed: 0, removed: 0 };
263
} catch
264
{
265
return { rootPath: resolvedRoot, indexed: 0, removed: 0, offline: true };
266
}
267
268
const db = getDb();
269
const seen = new Set();
270
const indexedAt = nowMs();
271
let indexed = 0;
272
273
await walkFilesUnderRoot(resolvedRoot, async (fileRow) =>
274
{
275
const filePath = path.resolve(fileRow.file_path);
276
seen.add(filePath);
277
upsertIndexRow(db, {
278
...fileRow,
279
file_path: filePath,
280
root_path: resolvedRoot,
281
indexed_at: indexedAt,
282
});
283
indexed += 1;
284
});
285
286
const existing = db.prepare('SELECT file_path FROM file_search_index WHERE root_path = ?').all([resolvedRoot]);
287
let removed = 0;
288
for (const row of existing)
289
{
290
if (seen.has(path.resolve(row.file_path))) continue;
291
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([row.file_path]);
292
removed += 1;
293
}
294
295
return { rootPath: resolvedRoot, indexed, removed };
296
}
297
298
async function syncSearchIndexOnce (roots)
299
{
300
const targetRoots = Array.isArray(roots) && roots.length ? roots.map((r) => path.resolve(r)) : effectiveIndexRoots();
301
const summary = {
302
rootsScanned: 0,
303
filesIndexed: 0,
304
filesRemoved: 0,
305
offlineRoots: 0,
306
};
307
308
for (const root of targetRoots)
309
{
310
const result = await syncSearchIndexForRoot(root);
311
summary.rootsScanned += 1;
312
summary.filesIndexed += Number(result.indexed || 0);
313
summary.filesRemoved += Number(result.removed || 0);
314
if (result.offline) summary.offlineRoots += 1;
315
}
316
317
return summary;
318
}
319
320
function countIndexedFilesForRoots (roots)
321
{
322
const resolved = [...new Set((roots || []).map((r) => path.resolve(r)))];
323
if (!resolved.length) return 0;
324
325
let total = 0;
326
const db = getDb();
327
for (let i = 0; i < resolved.length; i += ROOT_CHUNK_SIZE)
328
{
329
const chunk = resolved.slice(i, i + ROOT_CHUNK_SIZE);
330
const placeholders = chunk.map(() => '?').join(', ');
331
const row = db.prepare(`SELECT COUNT(*) AS n FROM file_search_index WHERE root_path IN (${placeholders})`).get(chunk);
332
total += Number(row && row.n || 0);
333
}
334
return total;
335
}
336
337
function querySubstringCandidates (resolvedRoots, queryLower, rowLimit)
338
{
339
if (!resolvedRoots.length) return [];
340
341
const db = getDb();
342
const out = [];
343
const needle = queryLower;
344
345
for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE)
346
{
347
const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE);
348
const placeholders = chunk.map(() => '?').join(', ');
349
const rows = db.prepare(`
350
SELECT file_path, root_path, name, relative_path, size, mtime_ms
351
FROM file_search_index
352
WHERE root_path IN (${placeholders})
353
AND (instr(name_lower, ?) > 0 OR instr(relative_lower, ?) > 0)
354
LIMIT ?
355
`).all([...chunk, needle, needle, rowLimit]);
356
357
for (const row of rows) out.push(row);
358
if (out.length >= rowLimit) break;
359
}
360
361
return out.slice(0, rowLimit);
362
}
363
364
function queryBatchCandidates (resolvedRoots, offset, batchSize)
365
{
366
if (!resolvedRoots.length) return [];
367
368
const db = getDb();
369
const out = [];
370
371
for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE)
372
{
373
const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE);
374
const placeholders = chunk.map(() => '?').join(', ');
375
const rows = db.prepare(`
376
SELECT file_path, root_path, name, relative_path, size, mtime_ms
377
FROM file_search_index
378
WHERE root_path IN (${placeholders})
379
ORDER BY file_path
380
LIMIT ? OFFSET ?
381
`).all([...chunk, batchSize, offset]);
382
383
for (const row of rows) out.push(row);
384
if (out.length >= batchSize) return out.slice(0, batchSize);
385
}
386
387
return out;
388
}
389
390
function rowToSearchResult (row, query, score)
391
{
392
const filePath = path.resolve(row.file_path);
393
const rootPath = path.resolve(row.root_path);
394
const relativePath = row.relative_path != null
395
? String(row.relative_path)
396
: path.relative(rootPath, filePath);
397
398
let mtime = null;
399
if (Number.isFinite(Number(row.mtime_ms)) && Number(row.mtime_ms) > 0)
400
{
401
mtime = new Date(Number(row.mtime_ms)).toISOString();
402
}
403
404
return {
405
name: row.name || path.basename(filePath),
406
path: filePath,
407
dir: path.dirname(filePath),
408
rootPath,
409
relativePath,
410
size: Number.isFinite(Number(row.size)) ? Number(row.size) : null,
411
mtime,
412
score,
413
};
414
}
415
416
function collectCandidatesFromIndex (resolvedRoots, query, maxCandidates)
417
{
418
const queryLower = String(query || '').trim().toLowerCase();
419
const byPath = new Map();
420
421
const substringRows = querySubstringCandidates(resolvedRoots, queryLower, maxCandidates);
422
for (const row of substringRows)
423
{
424
const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path);
425
if (score == null) continue;
426
byPath.set(path.resolve(row.file_path), { row, score });
427
}
428
429
if (byPath.size < maxCandidates)
430
{
431
let offset = 0;
432
let scanned = 0;
433
while (scanned < MAX_SUBSEQUENCE_SCAN && byPath.size < maxCandidates)
434
{
435
const batch = queryBatchCandidates(resolvedRoots, offset, INDEX_BATCH_SIZE);
436
if (!batch.length) break;
437
offset += batch.length;
438
scanned += batch.length;
439
440
for (const row of batch)
441
{
442
const filePath = path.resolve(row.file_path);
443
if (byPath.has(filePath)) continue;
444
const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path);
445
if (score == null) continue;
446
byPath.set(filePath, { row, score });
447
if (byPath.size >= maxCandidates) break;
448
}
449
}
450
}
451
452
return [...byPath.values()];
453
}
454
455
async function validateCandidates (candidates, dirtyDirs, db)
456
{
457
const validated = [];
458
const toCheck = candidates.slice(0, MAX_VALIDATE_HITS);
459
460
for (const entry of toCheck)
461
{
462
const filePath = path.resolve(entry.row.file_path);
463
let stat;
464
try
465
{
466
stat = await fs.promises.stat(filePath);
467
} catch
468
{
469
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
470
dirtyDirs.add(path.dirname(filePath));
471
continue;
472
}
473
474
if (!stat.isFile())
475
{
476
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
477
dirtyDirs.add(path.dirname(filePath));
478
continue;
479
}
480
481
const size = Number(stat.size);
482
const mtimeMs = Number(stat.mtimeMs);
483
const staleMeta = Number(entry.row.size) !== size || Number(entry.row.mtime_ms) !== mtimeMs;
484
if (staleMeta)
485
{
486
const rootPath = path.resolve(entry.row.root_path);
487
const name = path.basename(filePath);
488
const relativePath = path.relative(rootPath, filePath);
489
upsertIndexRow(db, {
490
file_path: filePath,
491
root_path: rootPath,
492
name,
493
name_lower: name.toLowerCase(),
494
relative_path: relativePath,
495
relative_lower: String(relativePath || '').toLowerCase(),
496
size,
497
mtime_ms: mtimeMs,
498
indexed_at: nowMs(),
499
});
500
entry.row.size = size;
501
entry.row.mtime_ms = mtimeMs;
502
dirtyDirs.add(path.dirname(filePath));
503
}
504
505
validated.push(entry);
506
}
507
508
for (const entry of candidates.slice(MAX_VALIDATE_HITS))
509
{
510
validated.push(entry);
511
}
512
513
return validated;
514
}
515
516
async function reindexDirectories (dirs, rootByDir)
517
{
518
const queue = [...dirs].slice(0, MAX_RESCAN_DIRS_PER_SEARCH);
519
const discovered = [];
520
521
for (const dir of queue)
522
{
523
const resolvedDir = path.resolve(dir);
524
const rootPath = rootByDir.get(resolvedDir);
525
if (!rootPath) continue;
526
527
try
528
{
529
const stat = await fs.promises.stat(resolvedDir);
530
if (!stat.isDirectory()) continue;
531
} catch
532
{
533
continue;
534
}
535
536
const db = getDb();
537
const seen = new Set();
538
const indexedAt = nowMs();
539
540
const resolvedRootPath = path.resolve(rootPath);
541
await walkFilesUnderRoot(resolvedRootPath, async (fileRow) =>
542
{
543
const filePath = path.resolve(fileRow.file_path);
544
seen.add(filePath);
545
upsertIndexRow(db, {
546
...fileRow,
547
file_path: filePath,
548
root_path: resolvedRootPath,
549
indexed_at: indexedAt,
550
});
551
discovered.push(fileRow);
552
}, resolvedDir);
553
554
const prefix = resolvedDir + path.sep;
555
const existing = db.prepare(`
556
SELECT file_path FROM file_search_index
557
WHERE root_path = ? AND (file_path = ? OR file_path LIKE ?)
558
`).all([path.resolve(rootPath), resolvedDir, prefix + '%']);
559
560
for (const row of existing)
561
{
562
const filePath = path.resolve(row.file_path);
563
if (filePath === resolvedDir) continue;
564
if (seen.has(filePath)) continue;
565
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);
566
}
567
}
568
569
return discovered;
570
}
571
572
function rootForPath (resolvedRoots, filePath)
573
{
574
const resolved = path.resolve(filePath);
575
for (const root of resolvedRoots)
576
{
577
const rel = path.relative(root, resolved);
578
if (rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))) return root;
579
}
580
return null;
581
}
582
583
function buildRootByDirMap (resolvedRoots)
584
{
585
const map = new Map();
586
for (const root of resolvedRoots)
587
{
588
map.set(path.resolve(root), path.resolve(root));
589
}
590
return map;
591
}
592
593
async function searchFilesFilesystemWalk (query, roots, maxResults)
594
{
595
const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
596
const results = [];
597
const visitedDirs = new Set();
598
let dirsVisited = 0;
599
600
for (const root of roots)
601
{
602
const rootPath = path.resolve(root);
603
const stack = [rootPath];
604
605
while (stack.length)
606
{
607
const dir = stack.pop();
608
if (!dir) continue;
609
const dirKey = path.resolve(dir);
610
if (visitedDirs.has(dirKey)) continue;
611
visitedDirs.add(dirKey);
612
dirsVisited += 1;
613
if (dirsVisited % 64 === 0) await yieldToEventLoop();
614
615
let entries;
616
try
617
{
618
entries = await fs.promises.readdir(dir, { withFileTypes: true });
619
} catch
620
{
621
continue;
622
}
623
624
for (const entry of entries)
625
{
626
const fullPath = path.join(dir, entry.name);
627
if (entry.isDirectory())
628
{
629
stack.push(fullPath);
630
continue;
631
}
632
if (!entry.isFile()) continue;
633
634
const relativePath = path.relative(rootPath, fullPath);
635
const score = fuzzyMatchScore(query, entry.name)
636
?? fuzzyMatchScore(query, relativePath);
637
if (score == null) continue;
638
639
let stat = null;
640
try { stat = await fs.promises.stat(fullPath); } catch { /* ignore */ }
641
642
results.push(rowToSearchResult({
643
file_path: fullPath,
644
root_path: rootPath,
645
name: entry.name,
646
relative_path: relativePath,
647
size: stat ? stat.size : null,
648
mtime_ms: stat ? stat.mtimeMs : null,
649
}, query, score));
650
651
if (results.length > limit * 4)
652
{
653
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
654
results.length = limit * 2;
655
}
656
}
657
}
658
}
659
660
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
661
return results.slice(0, limit);
662
}
663
664
async function searchFiles (query, roots, maxResults)
665
{
666
const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));
667
const resolvedRoots = [...new Set((roots || []).map((r) => path.resolve(r)))];
668
if (!resolvedRoots.length) return [];
669
670
if (countIndexedFilesForRoots(resolvedRoots) < 1)
671
{
672
scheduleBackgroundSync(resolvedRoots);
673
return searchFilesFilesystemWalk(query, roots, limit);
674
}
675
676
const candidates = collectCandidatesFromIndex(resolvedRoots, query, MAX_CANDIDATES_PER_SEARCH);
677
const dirtyDirs = new Set();
678
const db = getDb();
679
const validated = await validateCandidates(candidates, dirtyDirs, db);
680
681
const rootByDir = buildRootByDirMap(resolvedRoots);
682
for (const dir of dirtyDirs)
683
{
684
const resolvedDir = path.resolve(dir);
685
if (!rootByDir.has(resolvedDir))
686
{
687
const root = rootForPath(resolvedRoots, resolvedDir);
688
if (root) rootByDir.set(resolvedDir, root);
689
}
690
}
691
692
if (dirtyDirs.size)
693
{
694
const discovered = await reindexDirectories(dirtyDirs, rootByDir);
695
for (const fileRow of discovered)
696
{
697
const score = fuzzyMatchScore(query, fileRow.name) ?? fuzzyMatchScore(query, fileRow.relative_path);
698
if (score == null) continue;
699
validated.push({
700
row: {
701
file_path: fileRow.file_path,
702
root_path: fileRow.root_path,
703
name: fileRow.name,
704
relative_path: fileRow.relative_path,
705
size: fileRow.size,
706
mtime_ms: fileRow.mtime_ms,
707
},
708
score,
709
});
710
}
711
}
712
713
const merged = new Map();
714
for (const entry of validated)
715
{
716
const filePath = path.resolve(entry.row.file_path);
717
const prev = merged.get(filePath);
718
if (!prev || entry.score > prev.score) merged.set(filePath, entry);
719
}
720
721
const results = [...merged.values()]
722
.map((entry) => rowToSearchResult(entry.row, query, entry.score))
723
.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
724
725
return results.slice(0, limit);
726
}
727
728
function scheduleBackgroundSync (roots)
729
{
730
if (backgroundSyncRunning) return;
731
backgroundSyncRunning = true;
732
void syncSearchIndexOnce(roots)
733
.catch(() => { /* non-fatal */ })
734
.finally(() =>
735
{
736
backgroundSyncRunning = false;
737
});
738
}
739
740
function scheduleNextLoop (delayMs)
741
{
742
loopTimer = setTimeout(async () =>
743
{
744
loopTimer = null;
745
try
746
{
747
if (!hasActiveClients())
748
{
749
await syncSearchIndexOnce();
750
}
751
} catch (err)
752
{
753
console.error('[search-index] sync failed:', err && err.message ? err.message : err);
754
}
755
finally
756
{
757
if (loopStarted)
758
{
759
const delay = hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS;
760
scheduleNextLoop(delay);
761
}
762
}
763
}, delayMs);
764
765
if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();
766
}
767
768
function startSearchIndexLoop ()
769
{
770
if (loopStarted) return;
771
loopStarted = true;
772
scheduleBackgroundSync(effectiveIndexRoots());
773
scheduleNextLoop(hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS);
774
}
775
776
function stopSearchIndexLoop ()
777
{
778
loopStarted = false;
779
if (loopTimer)
780
{
781
clearTimeout(loopTimer);
782
loopTimer = null;
783
}
784
}
785
786
function getSearchIndexSnapshot ()
787
{
788
const db = getDb();
789
const row = db.prepare('SELECT COUNT(*) AS n FROM file_search_index').get();
790
return {
791
indexedFileCount: Number(row && row.n || 0),
792
loopStarted,
793
hasActiveClients: hasActiveClients(),
794
backgroundSyncRunning,
795
};
796
}
797
798
module.exports = {
799
fuzzyMatchScore,
800
searchFiles,
801
searchFilesFilesystemWalk,
802
syncSearchIndexOnce,
803
countIndexedFilesForRoots,
804
deleteIndexPath,
805
deleteIndexUnder,
806
rewriteIndexPaths,
807
reindexDirectoryTree: async (dirPath) =>
808
{
809
const resolvedDir = path.resolve(String(dirPath || ''));
810
const root = rootForPath(effectiveIndexRoots(), resolvedDir);
811
if (!root) return;
812
const rootByDir = new Map([[resolvedDir, root]]);
813
await reindexDirectories(new Set([resolvedDir]), rootByDir);
814
},
815
startSearchIndexLoop,
816
stopSearchIndexLoop,
817
getSearchIndexSnapshot,
818
};
@@ -0,0 +1,818 @@1
'use strict';2
3
const fs = require('fs');4
const path = require('path');5
const { getDb } = require('./db');6
const { loadConfig, getEffectivePaths } = require('./config');7
8
const ACTIVE_CLIENT_GRACE_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_CLIENT_GRACE_MS, 5 * 60 * 1000);9
const IDLE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_IDLE_SYNC_INTERVAL_MS, 30 * 60 * 1000);10
const ACTIVE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_SYNC_INTERVAL_MS, 10 * 60 * 1000);11
const MAX_CANDIDATES_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_CANDIDATES, 8000);12
const MAX_SUBSEQUENCE_SCAN = toPositiveInt(process.env.SEARCH_INDEX_MAX_SUBSEQUENCE_SCAN, 12000);13
const MAX_VALIDATE_HITS = toPositiveInt(process.env.SEARCH_INDEX_MAX_VALIDATE_HITS, 250);14
const MAX_RESCAN_DIRS_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_RESCAN_DIRS, 12);15
const INDEX_BATCH_SIZE = 400;16
const ROOT_CHUNK_SIZE = 50;17
18
let loopTimer = null;19
let loopStarted = false;20
let backgroundSyncRunning = false;21
22
function toPositiveInt (value, fallback)23
{24
const parsed = Number.parseInt(String(value || ''), 10);25
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;26
}27
28
function yieldToEventLoop ()29
{30
return new Promise((resolve) => setImmediate(resolve));31
}32
33
function nowMs ()34
{35
return Date.now();36
}37
38
function fuzzyMatchScore (needle, haystack)39
{40
const q = String(needle || '').trim().toLowerCase();41
const t = String(haystack || '').toLowerCase();42
if (!q || !t) return null;43
44
const idx = t.indexOf(q);45
if (idx >= 0)46
{47
const compactBonus = Math.max(0, 200 - idx);48
const lengthBonus = Math.max(0, 120 - (t.length - q.length));49
return 1200 + compactBonus + lengthBonus;50
}51
52
let qPos = 0;53
let firstHit = -1;54
let lastHit = -1;55
let gaps = 0;56
57
for (let i = 0; i < t.length && qPos < q.length; i++)58
{59
if (t[i] !== q[qPos]) continue;60
if (firstHit === -1) firstHit = i;61
if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1);62
lastHit = i;63
qPos += 1;64
}65
66
if (qPos !== q.length) return null;67
68
const span = lastHit - firstHit + 1;69
const compactness = Math.max(0, 300 - span);70
const gapPenalty = Math.min(180, gaps * 3);71
const startBonus = Math.max(0, 120 - firstHit);72
return 700 + compactness + startBonus - gapPenalty;73
}74
75
function effectiveIndexRoots ()76
{77
const config = loadConfig();78
const roots = getEffectivePaths(config.paths || []);79
const userPath = typeof config.userPath === 'string' ? config.userPath.trim() : '';80
if (userPath) roots.push(userPath);81
82
const seen = new Set();83
const deduped = [];84
for (const root of roots)85
{86
const key = path.resolve(root);87
if (seen.has(key)) continue;88
seen.add(key);89
deduped.push(key);90
}91
return deduped;92
}93
94
function hasActiveClients ()95
{96
try97
{98
const { getActiveStreamCount } = require('./video-stream');99
if (Number(getActiveStreamCount()) > 0) return true;100
} catch101
{102
// Stream module unavailable — treat as no active streams.103
}104
105
const db = getDb();106
const cutoff = nowMs() - ACTIVE_CLIENT_GRACE_MS;107
const row = db.prepare(`108
SELECT COUNT(*) AS n109
FROM auth_sessions110
WHERE revoked_at IS NULL AND last_seen_at >= ?111
`).get([cutoff]);112
113
return Number(row && row.n || 0) > 0;114
}115
116
function upsertIndexRow (db, row)117
{118
db.prepare(`119
INSERT INTO file_search_index (120
file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at121
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)122
ON CONFLICT(file_path) DO UPDATE SET123
root_path = excluded.root_path,124
name = excluded.name,125
name_lower = excluded.name_lower,126
relative_path = excluded.relative_path,127
relative_lower = excluded.relative_lower,128
size = excluded.size,129
mtime_ms = excluded.mtime_ms,130
indexed_at = excluded.indexed_at131
`).run([132
row.file_path,133
row.root_path,134
row.name,135
row.name_lower,136
row.relative_path,137
row.relative_lower,138
row.size,139
row.mtime_ms,140
row.indexed_at,141
]);142
}143
144
function deleteIndexPath (filePath)145
{146
getDb().prepare('DELETE FROM file_search_index WHERE file_path = ?').run([path.resolve(String(filePath || ''))]);147
}148
149
function deleteIndexUnder (targetPath)150
{151
const resolved = path.resolve(String(targetPath || ''));152
const db = getDb();153
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([resolved]);154
db.prepare('DELETE FROM file_search_index WHERE file_path LIKE ?')155
.run([resolved + path.sep + '%']);156
}157
158
function rewriteIndexPaths (fromPath, toPath)159
{160
const from = path.resolve(String(fromPath || ''));161
const to = path.resolve(String(toPath || ''));162
const db = getDb();163
164
const rows = db.prepare(`165
SELECT file_path, root_path, name, relative_path, size, mtime_ms166
FROM file_search_index167
WHERE file_path = ? OR file_path LIKE ?168
`).all([from, from + path.sep + '%']);169
170
if (!rows.length) return;171
172
const del = db.prepare('DELETE FROM file_search_index WHERE file_path = ?');173
for (const row of rows)174
{175
del.run([row.file_path]);176
const nextPath = row.file_path === from177
? to178
: to + row.file_path.slice(from.length);179
const nextName = path.basename(nextPath);180
const rootPath = path.resolve(row.root_path);181
const relativePath = path.relative(rootPath, nextPath);182
upsertIndexRow(db, {183
file_path: nextPath,184
root_path: rootPath,185
name: nextName,186
name_lower: nextName.toLowerCase(),187
relative_path: relativePath,188
relative_lower: String(relativePath || '').toLowerCase(),189
size: row.size,190
mtime_ms: row.mtime_ms,191
indexed_at: nowMs(),192
});193
}194
}195
196
async function walkFilesUnderRoot (storageRoot, onFile, startDir)197
{198
const resolvedRoot = path.resolve(storageRoot);199
const stack = [path.resolve(startDir || storageRoot)];200
const visitedDirs = new Set();201
let dirsVisited = 0;202
203
while (stack.length)204
{205
const dir = stack.pop();206
if (!dir) continue;207
const dirKey = path.resolve(dir);208
if (visitedDirs.has(dirKey)) continue;209
visitedDirs.add(dirKey);210
dirsVisited += 1;211
if (dirsVisited % 64 === 0) await yieldToEventLoop();212
213
let entries;214
try215
{216
entries = await fs.promises.readdir(dir, { withFileTypes: true });217
} catch218
{219
continue;220
}221
222
for (const entry of entries)223
{224
const fullPath = path.join(dir, entry.name);225
if (entry.isDirectory())226
{227
stack.push(fullPath);228
continue;229
}230
if (!entry.isFile()) continue;231
232
let stat;233
try234
{235
stat = await fs.promises.stat(fullPath);236
} catch237
{238
continue;239
}240
241
const relativePath = path.relative(resolvedRoot, fullPath);242
await onFile({243
file_path: fullPath,244
root_path: resolvedRoot,245
name: entry.name,246
name_lower: entry.name.toLowerCase(),247
relative_path: relativePath,248
relative_lower: String(relativePath || '').toLowerCase(),249
size: stat.size,250
mtime_ms: stat.mtimeMs,251
});252
}253
}254
}255
256
async function syncSearchIndexForRoot (rootPath)257
{258
const resolvedRoot = path.resolve(rootPath);259
try260
{261
const stat = await fs.promises.stat(resolvedRoot);262
if (!stat.isDirectory()) return { rootPath: resolvedRoot, indexed: 0, removed: 0 };263
} catch264
{265
return { rootPath: resolvedRoot, indexed: 0, removed: 0, offline: true };266
}267
268
const db = getDb();269
const seen = new Set();270
const indexedAt = nowMs();271
let indexed = 0;272
273
await walkFilesUnderRoot(resolvedRoot, async (fileRow) =>274
{275
const filePath = path.resolve(fileRow.file_path);276
seen.add(filePath);277
upsertIndexRow(db, {278
...fileRow,279
file_path: filePath,280
root_path: resolvedRoot,281
indexed_at: indexedAt,282
});283
indexed += 1;284
});285
286
const existing = db.prepare('SELECT file_path FROM file_search_index WHERE root_path = ?').all([resolvedRoot]);287
let removed = 0;288
for (const row of existing)289
{290
if (seen.has(path.resolve(row.file_path))) continue;291
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([row.file_path]);292
removed += 1;293
}294
295
return { rootPath: resolvedRoot, indexed, removed };296
}297
298
async function syncSearchIndexOnce (roots)299
{300
const targetRoots = Array.isArray(roots) && roots.length ? roots.map((r) => path.resolve(r)) : effectiveIndexRoots();301
const summary = {302
rootsScanned: 0,303
filesIndexed: 0,304
filesRemoved: 0,305
offlineRoots: 0,306
};307
308
for (const root of targetRoots)309
{310
const result = await syncSearchIndexForRoot(root);311
summary.rootsScanned += 1;312
summary.filesIndexed += Number(result.indexed || 0);313
summary.filesRemoved += Number(result.removed || 0);314
if (result.offline) summary.offlineRoots += 1;315
}316
317
return summary;318
}319
320
function countIndexedFilesForRoots (roots)321
{322
const resolved = [...new Set((roots || []).map((r) => path.resolve(r)))];323
if (!resolved.length) return 0;324
325
let total = 0;326
const db = getDb();327
for (let i = 0; i < resolved.length; i += ROOT_CHUNK_SIZE)328
{329
const chunk = resolved.slice(i, i + ROOT_CHUNK_SIZE);330
const placeholders = chunk.map(() => '?').join(', ');331
const row = db.prepare(`SELECT COUNT(*) AS n FROM file_search_index WHERE root_path IN (${placeholders})`).get(chunk);332
total += Number(row && row.n || 0);333
}334
return total;335
}336
337
function querySubstringCandidates (resolvedRoots, queryLower, rowLimit)338
{339
if (!resolvedRoots.length) return [];340
341
const db = getDb();342
const out = [];343
const needle = queryLower;344
345
for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE)346
{347
const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE);348
const placeholders = chunk.map(() => '?').join(', ');349
const rows = db.prepare(`350
SELECT file_path, root_path, name, relative_path, size, mtime_ms351
FROM file_search_index352
WHERE root_path IN (${placeholders})353
AND (instr(name_lower, ?) > 0 OR instr(relative_lower, ?) > 0)354
LIMIT ?355
`).all([...chunk, needle, needle, rowLimit]);356
357
for (const row of rows) out.push(row);358
if (out.length >= rowLimit) break;359
}360
361
return out.slice(0, rowLimit);362
}363
364
function queryBatchCandidates (resolvedRoots, offset, batchSize)365
{366
if (!resolvedRoots.length) return [];367
368
const db = getDb();369
const out = [];370
371
for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE)372
{373
const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE);374
const placeholders = chunk.map(() => '?').join(', ');375
const rows = db.prepare(`376
SELECT file_path, root_path, name, relative_path, size, mtime_ms377
FROM file_search_index378
WHERE root_path IN (${placeholders})379
ORDER BY file_path380
LIMIT ? OFFSET ?381
`).all([...chunk, batchSize, offset]);382
383
for (const row of rows) out.push(row);384
if (out.length >= batchSize) return out.slice(0, batchSize);385
}386
387
return out;388
}389
390
function rowToSearchResult (row, query, score)391
{392
const filePath = path.resolve(row.file_path);393
const rootPath = path.resolve(row.root_path);394
const relativePath = row.relative_path != null395
? String(row.relative_path)396
: path.relative(rootPath, filePath);397
398
let mtime = null;399
if (Number.isFinite(Number(row.mtime_ms)) && Number(row.mtime_ms) > 0)400
{401
mtime = new Date(Number(row.mtime_ms)).toISOString();402
}403
404
return {405
name: row.name || path.basename(filePath),406
path: filePath,407
dir: path.dirname(filePath),408
rootPath,409
relativePath,410
size: Number.isFinite(Number(row.size)) ? Number(row.size) : null,411
mtime,412
score,413
};414
}415
416
function collectCandidatesFromIndex (resolvedRoots, query, maxCandidates)417
{418
const queryLower = String(query || '').trim().toLowerCase();419
const byPath = new Map();420
421
const substringRows = querySubstringCandidates(resolvedRoots, queryLower, maxCandidates);422
for (const row of substringRows)423
{424
const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path);425
if (score == null) continue;426
byPath.set(path.resolve(row.file_path), { row, score });427
}428
429
if (byPath.size < maxCandidates)430
{431
let offset = 0;432
let scanned = 0;433
while (scanned < MAX_SUBSEQUENCE_SCAN && byPath.size < maxCandidates)434
{435
const batch = queryBatchCandidates(resolvedRoots, offset, INDEX_BATCH_SIZE);436
if (!batch.length) break;437
offset += batch.length;438
scanned += batch.length;439
440
for (const row of batch)441
{442
const filePath = path.resolve(row.file_path);443
if (byPath.has(filePath)) continue;444
const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path);445
if (score == null) continue;446
byPath.set(filePath, { row, score });447
if (byPath.size >= maxCandidates) break;448
}449
}450
}451
452
return [...byPath.values()];453
}454
455
async function validateCandidates (candidates, dirtyDirs, db)456
{457
const validated = [];458
const toCheck = candidates.slice(0, MAX_VALIDATE_HITS);459
460
for (const entry of toCheck)461
{462
const filePath = path.resolve(entry.row.file_path);463
let stat;464
try465
{466
stat = await fs.promises.stat(filePath);467
} catch468
{469
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);470
dirtyDirs.add(path.dirname(filePath));471
continue;472
}473
474
if (!stat.isFile())475
{476
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);477
dirtyDirs.add(path.dirname(filePath));478
continue;479
}480
481
const size = Number(stat.size);482
const mtimeMs = Number(stat.mtimeMs);483
const staleMeta = Number(entry.row.size) !== size || Number(entry.row.mtime_ms) !== mtimeMs;484
if (staleMeta)485
{486
const rootPath = path.resolve(entry.row.root_path);487
const name = path.basename(filePath);488
const relativePath = path.relative(rootPath, filePath);489
upsertIndexRow(db, {490
file_path: filePath,491
root_path: rootPath,492
name,493
name_lower: name.toLowerCase(),494
relative_path: relativePath,495
relative_lower: String(relativePath || '').toLowerCase(),496
size,497
mtime_ms: mtimeMs,498
indexed_at: nowMs(),499
});500
entry.row.size = size;501
entry.row.mtime_ms = mtimeMs;502
dirtyDirs.add(path.dirname(filePath));503
}504
505
validated.push(entry);506
}507
508
for (const entry of candidates.slice(MAX_VALIDATE_HITS))509
{510
validated.push(entry);511
}512
513
return validated;514
}515
516
async function reindexDirectories (dirs, rootByDir)517
{518
const queue = [...dirs].slice(0, MAX_RESCAN_DIRS_PER_SEARCH);519
const discovered = [];520
521
for (const dir of queue)522
{523
const resolvedDir = path.resolve(dir);524
const rootPath = rootByDir.get(resolvedDir);525
if (!rootPath) continue;526
527
try528
{529
const stat = await fs.promises.stat(resolvedDir);530
if (!stat.isDirectory()) continue;531
} catch532
{533
continue;534
}535
536
const db = getDb();537
const seen = new Set();538
const indexedAt = nowMs();539
540
const resolvedRootPath = path.resolve(rootPath);541
await walkFilesUnderRoot(resolvedRootPath, async (fileRow) =>542
{543
const filePath = path.resolve(fileRow.file_path);544
seen.add(filePath);545
upsertIndexRow(db, {546
...fileRow,547
file_path: filePath,548
root_path: resolvedRootPath,549
indexed_at: indexedAt,550
});551
discovered.push(fileRow);552
}, resolvedDir);553
554
const prefix = resolvedDir + path.sep;555
const existing = db.prepare(`556
SELECT file_path FROM file_search_index557
WHERE root_path = ? AND (file_path = ? OR file_path LIKE ?)558
`).all([path.resolve(rootPath), resolvedDir, prefix + '%']);559
560
for (const row of existing)561
{562
const filePath = path.resolve(row.file_path);563
if (filePath === resolvedDir) continue;564
if (seen.has(filePath)) continue;565
db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]);566
}567
}568
569
return discovered;570
}571
572
function rootForPath (resolvedRoots, filePath)573
{574
const resolved = path.resolve(filePath);575
for (const root of resolvedRoots)576
{577
const rel = path.relative(root, resolved);578
if (rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))) return root;579
}580
return null;581
}582
583
function buildRootByDirMap (resolvedRoots)584
{585
const map = new Map();586
for (const root of resolvedRoots)587
{588
map.set(path.resolve(root), path.resolve(root));589
}590
return map;591
}592
593
async function searchFilesFilesystemWalk (query, roots, maxResults)594
{595
const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));596
const results = [];597
const visitedDirs = new Set();598
let dirsVisited = 0;599
600
for (const root of roots)601
{602
const rootPath = path.resolve(root);603
const stack = [rootPath];604
605
while (stack.length)606
{607
const dir = stack.pop();608
if (!dir) continue;609
const dirKey = path.resolve(dir);610
if (visitedDirs.has(dirKey)) continue;611
visitedDirs.add(dirKey);612
dirsVisited += 1;613
if (dirsVisited % 64 === 0) await yieldToEventLoop();614
615
let entries;616
try617
{618
entries = await fs.promises.readdir(dir, { withFileTypes: true });619
} catch620
{621
continue;622
}623
624
for (const entry of entries)625
{626
const fullPath = path.join(dir, entry.name);627
if (entry.isDirectory())628
{629
stack.push(fullPath);630
continue;631
}632
if (!entry.isFile()) continue;633
634
const relativePath = path.relative(rootPath, fullPath);635
const score = fuzzyMatchScore(query, entry.name)636
?? fuzzyMatchScore(query, relativePath);637
if (score == null) continue;638
639
let stat = null;640
try { stat = await fs.promises.stat(fullPath); } catch { /* ignore */ }641
642
results.push(rowToSearchResult({643
file_path: fullPath,644
root_path: rootPath,645
name: entry.name,646
relative_path: relativePath,647
size: stat ? stat.size : null,648
mtime_ms: stat ? stat.mtimeMs : null,649
}, query, score));650
651
if (results.length > limit * 4)652
{653
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));654
results.length = limit * 2;655
}656
}657
}658
}659
660
results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));661
return results.slice(0, limit);662
}663
664
async function searchFiles (query, roots, maxResults)665
{666
const limit = Math.max(1, Math.min(500, Number(maxResults) || 200));667
const resolvedRoots = [...new Set((roots || []).map((r) => path.resolve(r)))];668
if (!resolvedRoots.length) return [];669
670
if (countIndexedFilesForRoots(resolvedRoots) < 1)671
{672
scheduleBackgroundSync(resolvedRoots);673
return searchFilesFilesystemWalk(query, roots, limit);674
}675
676
const candidates = collectCandidatesFromIndex(resolvedRoots, query, MAX_CANDIDATES_PER_SEARCH);677
const dirtyDirs = new Set();678
const db = getDb();679
const validated = await validateCandidates(candidates, dirtyDirs, db);680
681
const rootByDir = buildRootByDirMap(resolvedRoots);682
for (const dir of dirtyDirs)683
{684
const resolvedDir = path.resolve(dir);685
if (!rootByDir.has(resolvedDir))686
{687
const root = rootForPath(resolvedRoots, resolvedDir);688
if (root) rootByDir.set(resolvedDir, root);689
}690
}691
692
if (dirtyDirs.size)693
{694
const discovered = await reindexDirectories(dirtyDirs, rootByDir);695
for (const fileRow of discovered)696
{697
const score = fuzzyMatchScore(query, fileRow.name) ?? fuzzyMatchScore(query, fileRow.relative_path);698
if (score == null) continue;699
validated.push({700
row: {701
file_path: fileRow.file_path,702
root_path: fileRow.root_path,703
name: fileRow.name,704
relative_path: fileRow.relative_path,705
size: fileRow.size,706
mtime_ms: fileRow.mtime_ms,707
},708
score,709
});710
}711
}712
713
const merged = new Map();714
for (const entry of validated)715
{716
const filePath = path.resolve(entry.row.file_path);717
const prev = merged.get(filePath);718
if (!prev || entry.score > prev.score) merged.set(filePath, entry);719
}720
721
const results = [...merged.values()]722
.map((entry) => rowToSearchResult(entry.row, query, entry.score))723
.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));724
725
return results.slice(0, limit);726
}727
728
function scheduleBackgroundSync (roots)729
{730
if (backgroundSyncRunning) return;731
backgroundSyncRunning = true;732
void syncSearchIndexOnce(roots)733
.catch(() => { /* non-fatal */ })734
.finally(() =>735
{736
backgroundSyncRunning = false;737
});738
}739
740
function scheduleNextLoop (delayMs)741
{742
loopTimer = setTimeout(async () =>743
{744
loopTimer = null;745
try746
{747
if (!hasActiveClients())748
{749
await syncSearchIndexOnce();750
}751
} catch (err)752
{753
console.error('[search-index] sync failed:', err && err.message ? err.message : err);754
}755
finally756
{757
if (loopStarted)758
{759
const delay = hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS;760
scheduleNextLoop(delay);761
}762
}763
}, delayMs);764
765
if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();766
}767
768
function startSearchIndexLoop ()769
{770
if (loopStarted) return;771
loopStarted = true;772
scheduleBackgroundSync(effectiveIndexRoots());773
scheduleNextLoop(hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS);774
}775
776
function stopSearchIndexLoop ()777
{778
loopStarted = false;779
if (loopTimer)780
{781
clearTimeout(loopTimer);782
loopTimer = null;783
}784
}785
786
function getSearchIndexSnapshot ()787
{788
const db = getDb();789
const row = db.prepare('SELECT COUNT(*) AS n FROM file_search_index').get();790
return {791
indexedFileCount: Number(row && row.n || 0),792
loopStarted,793
hasActiveClients: hasActiveClients(),794
backgroundSyncRunning,795
};796
}797
798
module.exports = {799
fuzzyMatchScore,800
searchFiles,801
searchFilesFilesystemWalk,802
syncSearchIndexOnce,803
countIndexedFilesForRoots,804
deleteIndexPath,805
deleteIndexUnder,806
rewriteIndexPaths,807
reindexDirectoryTree: async (dirPath) =>808
{809
const resolvedDir = path.resolve(String(dirPath || ''));810
const root = rootForPath(effectiveIndexRoots(), resolvedDir);811
if (!root) return;812
const rootByDir = new Map([[resolvedDir, root]]);813
await reindexDirectories(new Set([resolvedDir]), rootByDir);814
},815
startSearchIndexLoop,816
stopSearchIndexLoop,817
getSearchIndexSnapshot,818
};@@ -2,7 +2,6 @@
2
2
3
3
const fs = require('fs');
4
4
const path = require('path');
5
const crypto = require('crypto');
6
5
const { spawn } = require('child_process');
7
6
const sharp = require('sharp');
8
7
@@ -12,8 +11,9 @@ const PUBLIC_DIR = path.resolve(process.env.FILEMANAGER_PUBLIC_DIR || path.join(
12
11
const THUMBS_DIR = path.resolve(process.env.FILEMANAGER_THUMBS_DIR || path.join(PUBLIC_DIR, 'thumbs'));
13
12
const MANIFEST_PATH = path.join(THUMBS_DIR, '.manifest.json');
14
13
const FOLDER_ICONS_PATH = path.join(THUMBS_DIR, '.folder-icons.json');
15
const LOOP_DELAY_MS = 5000;
14
const LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_LOOP_DELAY_MS, 15000);
16
15
const ACTIVE_STREAM_LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_ACTIVE_STREAM_DELAY_MS, 120000);
16
const THUMB_HTTP_MAX_AGE_SEC = toPositiveInt(process.env.THUMBNAIL_HTTP_MAX_AGE_SEC, 3600);
17
17
18
18
const IMAGE_EXTS = new Set([
19
19
'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tif', 'tiff', 'avif', 'svg', 'ico',
@@ -25,6 +25,8 @@ const MODEL_EXTS = new Set(['stl', 'obj']);
25
25
26
26
let loopTimer = null;
27
27
let loopStarted = false;
28
let manifestCache = null;
29
let manifestCacheMtimeMs = null;
28
30
29
31
function toPosix (p)
30
32
{
@@ -213,24 +215,61 @@ async function readManifest ()
213
215
return { version: 1, files: {} };
214
216
}
215
217
218
function invalidateManifestCache ()
219
{
220
manifestCache = null;
221
manifestCacheMtimeMs = null;
222
}
223
224
async function readManifestCached ()
225
{
226
try
227
{
228
const stat = await fs.promises.stat(MANIFEST_PATH);
229
if (manifestCache && manifestCacheMtimeMs === stat.mtimeMs)
230
{
231
return manifestCache;
232
}
233
234
const manifest = await readManifest();
235
manifestCache = manifest;
236
manifestCacheMtimeMs = stat.mtimeMs;
237
return manifest;
238
} catch
239
{
240
invalidateManifestCache();
241
return { version: 1, files: {} };
242
}
243
}
244
245
/**
246
* Returns a Set of source file paths that have a manifest entry (thumbnail expected).
247
*/
248
async function getThumbnailManifestSourcePathSet ()
249
{
250
const manifest = await readManifestCached();
251
const paths = new Set();
252
for (const filePath of Object.keys(manifest.files || {}))
253
{
254
paths.add(path.resolve(filePath));
255
}
256
return paths;
257
}
258
216
259
async function writeManifest (manifest)
217
260
{
218
261
const payload = JSON.stringify(manifest, null, 2);
219
262
const tmp = MANIFEST_PATH + '.tmp';
220
263
await fs.promises.writeFile(tmp, payload, 'utf8');
221
264
await fs.promises.rename(tmp, MANIFEST_PATH);
222
}
223
224
function sha256File (target)
225
{
226
return new Promise((resolve, reject) =>
265
manifestCache = manifest;
266
try
227
267
{
228
const hash = crypto.createHash('sha256');
229
const stream = fs.createReadStream(target);
230
stream.on('error', reject);
231
stream.on('data', chunk => hash.update(chunk));
232
stream.on('end', () => resolve(hash.digest('hex')));
233
});
268
manifestCacheMtimeMs = (await fs.promises.stat(MANIFEST_PATH)).mtimeMs;
269
} catch
270
{
271
invalidateManifestCache();
272
}
234
273
}
235
274
236
275
async function walkDir (dirPath, onDir, onFile)
@@ -697,17 +736,7 @@ async function syncThumbnailCacheOnce ()
697
736
698
737
if (kind === 'image')
699
738
{
700
let sha256;
701
try
702
{
703
sha256 = await sha256File(filePath);
704
} catch
705
{
706
return;
707
}
708
709
const unchangedByHash = prev && prev.sha256 === sha256 && await exists(thumbPath);
710
if (!unchangedByHash)
739
if (!unchangedByStat || !(await exists(thumbPath)))
711
740
{
712
741
try
713
742
{
@@ -719,7 +748,7 @@ async function syncThumbnailCacheOnce ()
719
748
}
720
749
721
750
nextManifest.files[filePath] = {
722
sha256,
751
sha256: prev && prev.sha256 ? prev.sha256 : null,
723
752
size: stat.size,
724
753
mtimeMs: stat.mtimeMs,
725
754
thumbRelPath,
@@ -869,20 +898,17 @@ async function writeFolderIcons (obj)
869
898
*/
870
899
async function getFilesWithThumbnailsInDir (dirPath)
871
900
{
872
const manifest = await readManifest();
901
const manifest = await readManifestCached();
873
902
const resolved = path.resolve(dirPath);
903
const prefix = resolved + path.sep;
874
904
const results = [];
875
905
876
906
for (const [filePath, meta] of Object.entries(manifest.files || {}))
877
907
{
878
const resolvedFile = path.resolve(filePath);
879
const relative = path.relative(resolved, resolvedFile);
880
if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) continue;
881
908
if (!meta || !meta.thumbRelPath) continue;
882
909
883
// Verify the thumbnail file actually exists on disk.
884
const thumbPath = path.resolve(path.join(PUBLIC_DIR, meta.thumbRelPath));
885
if (!(await exists(thumbPath))) continue;
910
const resolvedFile = path.resolve(filePath);
911
if (resolvedFile !== resolved && !resolvedFile.startsWith(prefix)) continue;
886
912
887
913
results.push({ filePath, thumbRelPath: meta.thumbRelPath });
888
914
}
@@ -945,6 +971,7 @@ module.exports = {
945
971
getCachedThumbnailInfo,
946
972
getThumbPathForSourceFile,
947
973
getThumbRelPathForSourceFile,
974
getThumbnailManifestSourcePathSet,
948
975
getFilesWithThumbnailsInDir,
949
976
getImagesWithThumbnailsInDir,
950
977
getFolderIconSourceFile,
@@ -952,4 +979,5 @@ module.exports = {
952
979
startThumbnailCacheLoop,
953
980
stopThumbnailCacheLoop,
954
981
syncThumbnailCacheOnce,
982
THUMB_HTTP_MAX_AGE_SEC,
955
983
};
@@ -2,7 +2,6 @@2
2
3
const fs = require('fs');3
const fs = require('fs');4
const path = require('path');4
const path = require('path');5
const crypto = require('crypto');6
const { spawn } = require('child_process');5
const { spawn } = require('child_process');7
const sharp = require('sharp');6
const sharp = require('sharp');8
7
@@ -12,8 +11,9 @@ const PUBLIC_DIR = path.resolve(process.env.FILEMANAGER_PUBLIC_DIR || path.join(12
const THUMBS_DIR = path.resolve(process.env.FILEMANAGER_THUMBS_DIR || path.join(PUBLIC_DIR, 'thumbs'));11
const THUMBS_DIR = path.resolve(process.env.FILEMANAGER_THUMBS_DIR || path.join(PUBLIC_DIR, 'thumbs'));13
const MANIFEST_PATH = path.join(THUMBS_DIR, '.manifest.json');12
const MANIFEST_PATH = path.join(THUMBS_DIR, '.manifest.json');14
const FOLDER_ICONS_PATH = path.join(THUMBS_DIR, '.folder-icons.json');13
const FOLDER_ICONS_PATH = path.join(THUMBS_DIR, '.folder-icons.json');15
const LOOP_DELAY_MS = 5000;14
const LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_LOOP_DELAY_MS, 15000);16
const ACTIVE_STREAM_LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_ACTIVE_STREAM_DELAY_MS, 120000);15
const ACTIVE_STREAM_LOOP_DELAY_MS = toPositiveInt(process.env.THUMBNAIL_CACHE_ACTIVE_STREAM_DELAY_MS, 120000);16
const THUMB_HTTP_MAX_AGE_SEC = toPositiveInt(process.env.THUMBNAIL_HTTP_MAX_AGE_SEC, 3600);17
17
18
const IMAGE_EXTS = new Set([18
const IMAGE_EXTS = new Set([19
'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tif', 'tiff', 'avif', 'svg', 'ico',19
'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tif', 'tiff', 'avif', 'svg', 'ico',@@ -25,6 +25,8 @@ const MODEL_EXTS = new Set(['stl', 'obj']);25
25
26
let loopTimer = null;26
let loopTimer = null;27
let loopStarted = false;27
let loopStarted = false;28
let manifestCache = null;29
let manifestCacheMtimeMs = null;28
30
29
function toPosix (p)31
function toPosix (p)30
{32
{@@ -213,24 +215,61 @@ async function readManifest ()213
return { version: 1, files: {} };215
return { version: 1, files: {} };214
}216
}215
217
218
function invalidateManifestCache ()219
{220
manifestCache = null;221
manifestCacheMtimeMs = null;222
}223
224
async function readManifestCached ()225
{226
try227
{228
const stat = await fs.promises.stat(MANIFEST_PATH);229
if (manifestCache && manifestCacheMtimeMs === stat.mtimeMs)230
{231
return manifestCache;232
}233
234
const manifest = await readManifest();235
manifestCache = manifest;236
manifestCacheMtimeMs = stat.mtimeMs;237
return manifest;238
} catch239
{240
invalidateManifestCache();241
return { version: 1, files: {} };242
}243
}244
245
/**246
* Returns a Set of source file paths that have a manifest entry (thumbnail expected).247
*/248
async function getThumbnailManifestSourcePathSet ()249
{250
const manifest = await readManifestCached();251
const paths = new Set();252
for (const filePath of Object.keys(manifest.files || {}))253
{254
paths.add(path.resolve(filePath));255
}256
return paths;257
}258
216
async function writeManifest (manifest)259
async function writeManifest (manifest)217
{260
{218
const payload = JSON.stringify(manifest, null, 2);261
const payload = JSON.stringify(manifest, null, 2);219
const tmp = MANIFEST_PATH + '.tmp';262
const tmp = MANIFEST_PATH + '.tmp';220
await fs.promises.writeFile(tmp, payload, 'utf8');263
await fs.promises.writeFile(tmp, payload, 'utf8');221
await fs.promises.rename(tmp, MANIFEST_PATH);264
await fs.promises.rename(tmp, MANIFEST_PATH);222
}265
manifestCache = manifest;223
266
try224
function sha256File (target)225
{226
return new Promise((resolve, reject) =>227
{267
{228
const hash = crypto.createHash('sha256');268
manifestCacheMtimeMs = (await fs.promises.stat(MANIFEST_PATH)).mtimeMs;229
const stream = fs.createReadStream(target);269
} catch230
stream.on('error', reject);270
{231
stream.on('data', chunk => hash.update(chunk));271
invalidateManifestCache();232
stream.on('end', () => resolve(hash.digest('hex')));272
}233
});234
}273
}235
274
236
async function walkDir (dirPath, onDir, onFile)275
async function walkDir (dirPath, onDir, onFile)@@ -697,17 +736,7 @@ async function syncThumbnailCacheOnce ()697
736
698
if (kind === 'image')737
if (kind === 'image')699
{738
{700
let sha256;739
if (!unchangedByStat || !(await exists(thumbPath)))701
try702
{703
sha256 = await sha256File(filePath);704
} catch705
{706
return;707
}708
709
const unchangedByHash = prev && prev.sha256 === sha256 && await exists(thumbPath);710
if (!unchangedByHash)711
{740
{712
try741
try713
{742
{@@ -719,7 +748,7 @@ async function syncThumbnailCacheOnce ()719
}748
}720
749
721
nextManifest.files[filePath] = {750
nextManifest.files[filePath] = {722
sha256,751
sha256: prev && prev.sha256 ? prev.sha256 : null,723
size: stat.size,752
size: stat.size,724
mtimeMs: stat.mtimeMs,753
mtimeMs: stat.mtimeMs,725
thumbRelPath,754
thumbRelPath,@@ -869,20 +898,17 @@ async function writeFolderIcons (obj)869
*/898
*/870
async function getFilesWithThumbnailsInDir (dirPath)899
async function getFilesWithThumbnailsInDir (dirPath)871
{900
{872
const manifest = await readManifest();901
const manifest = await readManifestCached();873
const resolved = path.resolve(dirPath);902
const resolved = path.resolve(dirPath);903
const prefix = resolved + path.sep;874
const results = [];904
const results = [];875
905
876
for (const [filePath, meta] of Object.entries(manifest.files || {}))906
for (const [filePath, meta] of Object.entries(manifest.files || {}))877
{907
{878
const resolvedFile = path.resolve(filePath);879
const relative = path.relative(resolved, resolvedFile);880
if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) continue;881
if (!meta || !meta.thumbRelPath) continue;908
if (!meta || !meta.thumbRelPath) continue;882
909
883
// Verify the thumbnail file actually exists on disk.910
const resolvedFile = path.resolve(filePath);884
const thumbPath = path.resolve(path.join(PUBLIC_DIR, meta.thumbRelPath));911
if (resolvedFile !== resolved && !resolvedFile.startsWith(prefix)) continue;885
if (!(await exists(thumbPath))) continue;886
912
887
results.push({ filePath, thumbRelPath: meta.thumbRelPath });913
results.push({ filePath, thumbRelPath: meta.thumbRelPath });888
}914
}@@ -945,6 +971,7 @@ module.exports = {945
getCachedThumbnailInfo,971
getCachedThumbnailInfo,946
getThumbPathForSourceFile,972
getThumbPathForSourceFile,947
getThumbRelPathForSourceFile,973
getThumbRelPathForSourceFile,974
getThumbnailManifestSourcePathSet,948
getFilesWithThumbnailsInDir,975
getFilesWithThumbnailsInDir,949
getImagesWithThumbnailsInDir,976
getImagesWithThumbnailsInDir,950
getFolderIconSourceFile,977
getFolderIconSourceFile,@@ -952,4 +979,5 @@ module.exports = {952
startThumbnailCacheLoop,979
startThumbnailCacheLoop,953
stopThumbnailCacheLoop,980
stopThumbnailCacheLoop,954
syncThumbnailCacheOnce,981
syncThumbnailCacheOnce,982
THUMB_HTTP_MAX_AGE_SEC,955
};983
};@@ -147,6 +147,7 @@ jest.mock('../server/auth', () => ({
147
147
createToken: () => 'mock-token',
148
148
getClientIp: () => '127.0.0.1',
149
149
getRegion: () => 'US:CA',
150
clearAuthUserCache: () => {},
150
151
}));
151
152
152
153
const adminRoutes = require('../server/routes/admin');
@@ -147,6 +147,7 @@ jest.mock('../server/auth', () => ({147
createToken: () => 'mock-token',147
createToken: () => 'mock-token',148
getClientIp: () => '127.0.0.1',148
getClientIp: () => '127.0.0.1',149
getRegion: () => 'US:CA',149
getRegion: () => 'US:CA',150
clearAuthUserCache: () => {},150
}));151
}));151
152
152
const adminRoutes = require('../server/routes/admin');153
const adminRoutes = require('../server/routes/admin');@@ -0,0 +1,65 @@
1
'use strict';
2
3
const crypto = require('crypto');
4
const fs = require('fs');
5
const os = require('os');
6
const path = require('path');
7
8
jest.mock('../server/db', () =>
9
{
10
const { Database } = require('node-sqlite3-wasm');
11
let db;
12
function getDb ()
13
{
14
if (!db)
15
{
16
db = new Database(':memory:');
17
db.exec(`
18
CREATE TABLE IF NOT EXISTS file_meta (
19
file_path TEXT PRIMARY KEY,
20
uploader TEXT NOT NULL DEFAULT '',
21
downloaders TEXT NOT NULL DEFAULT '[]',
22
watched_by TEXT NOT NULL DEFAULT '{}',
23
uploaded_at TEXT,
24
sha256 TEXT
25
);
26
`);
27
}
28
return db;
29
}
30
return { getDb, initDb: getDb };
31
});
32
33
describe('file hash queue', () =>
34
{
35
let tempDir;
36
37
beforeEach(() =>
38
{
39
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filemanager-file-hash-'));
40
});
41
42
afterEach(() =>
43
{
44
fs.rmSync(tempDir, { recursive: true, force: true });
45
});
46
47
test('drainFileHashQueue persists sha256 for queued files', async () =>
48
{
49
const filePath = path.join(tempDir, 'queued.txt');
50
const content = 'background hash';
51
fs.writeFileSync(filePath, content);
52
53
const { getDb } = require('../server/db');
54
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
55
.run([filePath, '', '[]']);
56
57
const hashQueue = require('../server/file-hash-queue');
58
hashQueue.enqueueFileHash(filePath);
59
await hashQueue.drainFileHashQueue();
60
61
const expected = crypto.createHash('sha256').update(content).digest('hex');
62
const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
63
expect(row.sha256).toBe(expected);
64
});
65
});
@@ -0,0 +1,65 @@1
'use strict';2
3
const crypto = require('crypto');4
const fs = require('fs');5
const os = require('os');6
const path = require('path');7
8
jest.mock('../server/db', () =>9
{10
const { Database } = require('node-sqlite3-wasm');11
let db;12
function getDb ()13
{14
if (!db)15
{16
db = new Database(':memory:');17
db.exec(`18
CREATE TABLE IF NOT EXISTS file_meta (19
file_path TEXT PRIMARY KEY,20
uploader TEXT NOT NULL DEFAULT '',21
downloaders TEXT NOT NULL DEFAULT '[]',22
watched_by TEXT NOT NULL DEFAULT '{}',23
uploaded_at TEXT,24
sha256 TEXT25
);26
`);27
}28
return db;29
}30
return { getDb, initDb: getDb };31
});32
33
describe('file hash queue', () =>34
{35
let tempDir;36
37
beforeEach(() =>38
{39
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filemanager-file-hash-'));40
});41
42
afterEach(() =>43
{44
fs.rmSync(tempDir, { recursive: true, force: true });45
});46
47
test('drainFileHashQueue persists sha256 for queued files', async () =>48
{49
const filePath = path.join(tempDir, 'queued.txt');50
const content = 'background hash';51
fs.writeFileSync(filePath, content);52
53
const { getDb } = require('../server/db');54
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')55
.run([filePath, '', '[]']);56
57
const hashQueue = require('../server/file-hash-queue');58
hashQueue.enqueueFileHash(filePath);59
await hashQueue.drainFileHashQueue();60
61
const expected = crypto.createHash('sha256').update(content).digest('hex');62
const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);63
expect(row.sha256).toBe(expected);64
});65
});@@ -13,6 +13,7 @@ let mockRootAliases = {};
13
13
let mockGlobalFaviconPath = null;
14
14
let mockUser = { username: 'admin', isAdmin: true, permissions: {} };
15
15
let mockThumbPath = null;
16
let mockThumbnailManifestPaths = null;
16
17
let mockFolderImages = [];
17
18
let mockFolderIconFile = null;
18
19
let lastFolderIconAssignment = undefined;
@@ -51,6 +52,23 @@ jest.mock('../server/db', () =>
51
52
sha256 TEXT
52
53
);
53
54
CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);
55
CREATE TABLE IF NOT EXISTS user_last_navigation (
56
username TEXT PRIMARY KEY,
57
last_dir TEXT,
58
updated_at INTEGER NOT NULL
59
);
60
CREATE TABLE IF NOT EXISTS file_search_index (
61
file_path TEXT PRIMARY KEY,
62
root_path TEXT NOT NULL,
63
name TEXT NOT NULL,
64
name_lower TEXT NOT NULL,
65
relative_path TEXT NOT NULL,
66
relative_lower TEXT NOT NULL,
67
size INTEGER,
68
mtime_ms INTEGER,
69
indexed_at INTEGER NOT NULL
70
);
71
CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
54
72
`);
55
73
}
56
74
return db;
@@ -95,12 +113,18 @@ jest.mock('../server/thumbnail-cache', () => ({
95
113
if (!mockThumbPath) return null;
96
114
return { thumbPath: mockThumbPath, contentType: 'image/webp' };
97
115
},
116
getThumbnailManifestSourcePathSet: async () =>
117
{
118
if (mockThumbnailManifestPaths) return new Set(mockThumbnailManifestPaths);
119
return new Set();
120
},
98
121
getFilesWithThumbnailsInDir: async () => mockFolderImages.slice(),
99
122
getFolderIconSourceFile: async () => mockFolderIconFile,
100
123
setFolderIconAssignment: async (folderPath, imagePath) =>
101
124
{
102
125
lastFolderIconAssignment = { folderPath, imagePath };
103
126
},
127
THUMB_HTTP_MAX_AGE_SEC: 3600,
104
128
}));
105
129
106
130
jest.mock('../server/video-stream', () => ({
@@ -145,6 +169,7 @@ jest.mock('../server/subtitle-debug-log', () => ({
145
169
146
170
const filesRoutes = require('../server/routes/files');
147
171
const { getDb } = require('../server/db');
172
const { drainFileHashQueue, stopFileHashQueue } = require('../server/file-hash-queue');
148
173
149
174
let server;
150
175
let baseUrl;
@@ -193,6 +218,7 @@ beforeEach(() =>
193
218
mockUserPath = null;
194
219
mockUser = { username: 'admin', isAdmin: true, permissions: {} };
195
220
mockThumbPath = null;
221
mockThumbnailManifestPaths = null;
196
222
mockFolderImages = [];
197
223
mockFolderIconFile = null;
198
224
lastFolderIconAssignment = undefined;
@@ -206,6 +232,8 @@ beforeEach(() =>
206
232
const db = getDb();
207
233
db.prepare('DELETE FROM file_meta').run([]);
208
234
db.prepare('DELETE FROM users').run([]);
235
db.exec('DELETE FROM file_search_index');
236
stopFileHashQueue();
209
237
});
210
238
211
239
function req (method, routePath, body)
@@ -1058,9 +1086,11 @@ describe('image search endpoint', () =>
1058
1086
fs.writeFileSync(path.join(picsDir, 'dragon-cover.jpg'), 'img');
1059
1087
fs.writeFileSync(path.join(picsDir, 'dragon-notes.txt'), 'txt');
1060
1088
1089
const imgPath = path.join(picsDir, 'dragon-cover.jpg');
1061
1090
const thumbPath = path.join(rootDir, 'thumb.webp');
1062
1091
fs.writeFileSync(thumbPath, 'thumb');
1063
1092
mockThumbPath = thumbPath;
1093
mockThumbnailManifestPaths = [imgPath];
1064
1094
1065
1095
const { status, body } = await req('GET', '/api/files/search-images?query=dragon');
1066
1096
expect(status).toBe(200);
@@ -1436,17 +1466,18 @@ describe('video streaming routes', () =>
1436
1466
1437
1467
describe('SHA-256 hashing and duplicate detection', () =>
1438
1468
{
1439
test('POST /api/files/upload stores sha256 hash for uploaded files', async () =>
1469
test('POST /api/files/upload queues sha256 hash for uploaded files', async () =>
1440
1470
{
1441
1471
const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [
1442
1472
{ name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },
1443
1473
]);
1444
1474
expect(status).toBe(200);
1445
1475
1476
await drainFileHashQueue();
1477
1446
1478
const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')
1447
1479
.get([path.join(rootDir, 'hashme.txt')]);
1448
1480
expect(meta).toBeTruthy();
1449
// SHA-256 is a 64-char hex string.
1450
1481
expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);
1451
1482
});
1452
1483
@@ -1463,22 +1494,26 @@ describe('SHA-256 hashing and duplicate detection', () =>
1463
1494
expect(String(meta.uploaded_at || '')).toMatch(/^\d{4}-\d{2}-\d{2}T/);
1464
1495
});
1465
1496
1466
test('GET /api/files/meta returns sha256 and computes it lazily when missing', async () =>
1497
test('GET /api/files/meta enqueues sha256 and background worker persists it', async () =>
1467
1498
{
1468
1499
const filePath = path.join(rootDir, 'lazy.txt');
1469
1500
fs.writeFileSync(filePath, 'lazy content');
1470
1501
1471
// Insert a row without a hash to simulate a pre-existing file.
1472
1502
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')
1473
1503
.run([filePath, 'admin', '[]']);
1474
1504
1475
const { status, body } = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1476
expect(status).toBe(200);
1477
expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);
1505
const first = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1506
expect(first.status).toBe(200);
1507
expect(first.body.sha256).toBeNull();
1508
1509
await drainFileHashQueue();
1478
1510
1479
// Verify it was persisted.
1480
1511
const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);
1481
expect(row.sha256).toBe(body.sha256);
1512
expect(row.sha256).toMatch(/^[0-9a-f]{64}$/);
1513
1514
const second = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);
1515
expect(second.status).toBe(200);
1516
expect(second.body.sha256).toBe(row.sha256);
1482
1517
});
1483
1518
1484
1519
test('GET /api/files/meta backfills uploaded_at when missing', async () =>
@@ -13,6 +13,7 @@ let mockRootAliases = {};13
let mockGlobalFaviconPath = null;13
let mockGlobalFaviconPath = null;14
let mockUser = { username: 'admin', isAdmin: true, permissions: {} };14
let mockUser = { username: 'admin', isAdmin: true, permissions: {} };15
let mockThumbPath = null;15
let mockThumbPath = null;16
let mockThumbnailManifestPaths = null;16
let mockFolderImages = [];17
let mockFolderImages = [];17
let mockFolderIconFile = null;18
let mockFolderIconFile = null;18
let lastFolderIconAssignment = undefined;19
let lastFolderIconAssignment = undefined;@@ -51,6 +52,23 @@ jest.mock('../server/db', () =>51
sha256 TEXT52
sha256 TEXT52
);53
);53
CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);54
CREATE INDEX IF NOT EXISTS idx_file_meta_sha256 ON file_meta(sha256);55
CREATE TABLE IF NOT EXISTS user_last_navigation (56
username TEXT PRIMARY KEY,57
last_dir TEXT,58
updated_at INTEGER NOT NULL59
);60
CREATE TABLE IF NOT EXISTS file_search_index (61
file_path TEXT PRIMARY KEY,62
root_path TEXT NOT NULL,63
name TEXT NOT NULL,64
name_lower TEXT NOT NULL,65
relative_path TEXT NOT NULL,66
relative_lower TEXT NOT NULL,67
size INTEGER,68
mtime_ms INTEGER,69
indexed_at INTEGER NOT NULL70
);71
CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);54
`);72
`);55
}73
}56
return db;74
return db;@@ -95,12 +113,18 @@ jest.mock('../server/thumbnail-cache', () => ({95
if (!mockThumbPath) return null;113
if (!mockThumbPath) return null;96
return { thumbPath: mockThumbPath, contentType: 'image/webp' };114
return { thumbPath: mockThumbPath, contentType: 'image/webp' };97
},115
},116
getThumbnailManifestSourcePathSet: async () =>117
{118
if (mockThumbnailManifestPaths) return new Set(mockThumbnailManifestPaths);119
return new Set();120
},98
getFilesWithThumbnailsInDir: async () => mockFolderImages.slice(),121
getFilesWithThumbnailsInDir: async () => mockFolderImages.slice(),99
getFolderIconSourceFile: async () => mockFolderIconFile,122
getFolderIconSourceFile: async () => mockFolderIconFile,100
setFolderIconAssignment: async (folderPath, imagePath) =>123
setFolderIconAssignment: async (folderPath, imagePath) =>101
{124
{102
lastFolderIconAssignment = { folderPath, imagePath };125
lastFolderIconAssignment = { folderPath, imagePath };103
},126
},127
THUMB_HTTP_MAX_AGE_SEC: 3600,104
}));128
}));105
129
106
jest.mock('../server/video-stream', () => ({130
jest.mock('../server/video-stream', () => ({@@ -145,6 +169,7 @@ jest.mock('../server/subtitle-debug-log', () => ({145
169
146
const filesRoutes = require('../server/routes/files');170
const filesRoutes = require('../server/routes/files');147
const { getDb } = require('../server/db');171
const { getDb } = require('../server/db');172
const { drainFileHashQueue, stopFileHashQueue } = require('../server/file-hash-queue');148
173
149
let server;174
let server;150
let baseUrl;175
let baseUrl;@@ -193,6 +218,7 @@ beforeEach(() =>193
mockUserPath = null;218
mockUserPath = null;194
mockUser = { username: 'admin', isAdmin: true, permissions: {} };219
mockUser = { username: 'admin', isAdmin: true, permissions: {} };195
mockThumbPath = null;220
mockThumbPath = null;221
mockThumbnailManifestPaths = null;196
mockFolderImages = [];222
mockFolderImages = [];197
mockFolderIconFile = null;223
mockFolderIconFile = null;198
lastFolderIconAssignment = undefined;224
lastFolderIconAssignment = undefined;@@ -206,6 +232,8 @@ beforeEach(() =>206
const db = getDb();232
const db = getDb();207
db.prepare('DELETE FROM file_meta').run([]);233
db.prepare('DELETE FROM file_meta').run([]);208
db.prepare('DELETE FROM users').run([]);234
db.prepare('DELETE FROM users').run([]);235
db.exec('DELETE FROM file_search_index');236
stopFileHashQueue();209
});237
});210
238
211
function req (method, routePath, body)239
function req (method, routePath, body)@@ -1058,9 +1086,11 @@ describe('image search endpoint', () =>1058
fs.writeFileSync(path.join(picsDir, 'dragon-cover.jpg'), 'img');1086
fs.writeFileSync(path.join(picsDir, 'dragon-cover.jpg'), 'img');1059
fs.writeFileSync(path.join(picsDir, 'dragon-notes.txt'), 'txt');1087
fs.writeFileSync(path.join(picsDir, 'dragon-notes.txt'), 'txt');1060
1088
1089
const imgPath = path.join(picsDir, 'dragon-cover.jpg');1061
const thumbPath = path.join(rootDir, 'thumb.webp');1090
const thumbPath = path.join(rootDir, 'thumb.webp');1062
fs.writeFileSync(thumbPath, 'thumb');1091
fs.writeFileSync(thumbPath, 'thumb');1063
mockThumbPath = thumbPath;1092
mockThumbPath = thumbPath;1093
mockThumbnailManifestPaths = [imgPath];1064
1094
1065
const { status, body } = await req('GET', '/api/files/search-images?query=dragon');1095
const { status, body } = await req('GET', '/api/files/search-images?query=dragon');1066
expect(status).toBe(200);1096
expect(status).toBe(200);@@ -1436,17 +1466,18 @@ describe('video streaming routes', () =>1436
1466
1437
describe('SHA-256 hashing and duplicate detection', () =>1467
describe('SHA-256 hashing and duplicate detection', () =>1438
{1468
{1439
test('POST /api/files/upload stores sha256 hash for uploaded files', async () =>1469
test('POST /api/files/upload queues sha256 hash for uploaded files', async () =>1440
{1470
{1441
const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [1471
const { status } = await reqMultipart(`/api/files/upload?dir=${encodeURIComponent(rootDir)}`, [1442
{ name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },1472
{ name: 'hashme.txt', content: 'hello hash', type: 'text/plain' },1443
]);1473
]);1444
expect(status).toBe(200);1474
expect(status).toBe(200);1445
1475
1476
await drainFileHashQueue();1477
1446
const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')1478
const meta = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?')1447
.get([path.join(rootDir, 'hashme.txt')]);1479
.get([path.join(rootDir, 'hashme.txt')]);1448
expect(meta).toBeTruthy();1480
expect(meta).toBeTruthy();1449
// SHA-256 is a 64-char hex string.1450
expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);1481
expect(meta.sha256).toMatch(/^[0-9a-f]{64}$/);1451
});1482
});1452
1483
@@ -1463,22 +1494,26 @@ describe('SHA-256 hashing and duplicate detection', () =>1463
expect(String(meta.uploaded_at || '')).toMatch(/^\d{4}-\d{2}-\d{2}T/);1494
expect(String(meta.uploaded_at || '')).toMatch(/^\d{4}-\d{2}-\d{2}T/);1464
});1495
});1465
1496
1466
test('GET /api/files/meta returns sha256 and computes it lazily when missing', async () =>1497
test('GET /api/files/meta enqueues sha256 and background worker persists it', async () =>1467
{1498
{1468
const filePath = path.join(rootDir, 'lazy.txt');1499
const filePath = path.join(rootDir, 'lazy.txt');1469
fs.writeFileSync(filePath, 'lazy content');1500
fs.writeFileSync(filePath, 'lazy content');1470
1501
1471
// Insert a row without a hash to simulate a pre-existing file.1472
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')1502
getDb().prepare('INSERT INTO file_meta (file_path, uploader, downloaders) VALUES (?, ?, ?)')1473
.run([filePath, 'admin', '[]']);1503
.run([filePath, 'admin', '[]']);1474
1504
1475
const { status, body } = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);1505
const first = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);1476
expect(status).toBe(200);1506
expect(first.status).toBe(200);1477
expect(body.sha256).toMatch(/^[0-9a-f]{64}$/);1507
expect(first.body.sha256).toBeNull();1508
1509
await drainFileHashQueue();1478
1510
1479
// Verify it was persisted.1480
const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);1511
const row = getDb().prepare('SELECT sha256 FROM file_meta WHERE file_path = ?').get([filePath]);1481
expect(row.sha256).toBe(body.sha256);1512
expect(row.sha256).toMatch(/^[0-9a-f]{64}$/);1513
1514
const second = await req('GET', `/api/files/meta?file=${encodeURIComponent(filePath)}`);1515
expect(second.status).toBe(200);1516
expect(second.body.sha256).toBe(row.sha256);1482
});1517
});1483
1518
1484
test('GET /api/files/meta backfills uploaded_at when missing', async () =>1519
test('GET /api/files/meta backfills uploaded_at when missing', async () =>@@ -0,0 +1,140 @@
1
'use strict';
2
3
const fs = require('fs');
4
const os = require('os');
5
const path = require('path');
6
7
let mockPaths = [];
8
9
jest.mock('../server/db', () =>
10
{
11
const { Database } = require('node-sqlite3-wasm');
12
let db;
13
function getDb ()
14
{
15
if (!db)
16
{
17
db = new Database(':memory:');
18
db.exec(`
19
CREATE TABLE IF NOT EXISTS file_search_index (
20
file_path TEXT PRIMARY KEY,
21
root_path TEXT NOT NULL,
22
name TEXT NOT NULL,
23
name_lower TEXT NOT NULL,
24
relative_path TEXT NOT NULL,
25
relative_lower TEXT NOT NULL,
26
size INTEGER,
27
mtime_ms INTEGER,
28
indexed_at INTEGER NOT NULL
29
);
30
CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);
31
`);
32
}
33
return db;
34
}
35
return { getDb, initDb: getDb };
36
});
37
38
jest.mock('../server/config', () => ({
39
loadConfig: () => ({ paths: [...mockPaths] }),
40
getEffectivePaths: (paths) => paths,
41
}));
42
43
describe('search index', () =>
44
{
45
let tempDir;
46
let sourceRoot;
47
48
function loadModule ()
49
{
50
jest.resetModules();
51
return require('../server/search-index');
52
}
53
54
beforeEach(() =>
55
{
56
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filemanager-search-index-'));
57
sourceRoot = path.join(tempDir, 'root');
58
fs.mkdirSync(sourceRoot, { recursive: true });
59
mockPaths = [sourceRoot];
60
});
61
62
afterEach(() =>
63
{
64
mockPaths = [];
65
fs.rmSync(tempDir, { recursive: true, force: true });
66
});
67
68
test('syncSearchIndexOnce indexes files and removes deleted entries', async () =>
69
{
70
const keepPath = path.join(sourceRoot, 'dragon-ball.mkv');
71
const removePath = path.join(sourceRoot, 'old.txt');
72
fs.writeFileSync(keepPath, 'vid');
73
fs.writeFileSync(removePath, 'txt');
74
75
const searchIndex = loadModule();
76
await searchIndex.syncSearchIndexOnce([sourceRoot]);
77
expect(searchIndex.countIndexedFilesForRoots([sourceRoot])).toBe(2);
78
79
fs.unlinkSync(removePath);
80
await searchIndex.syncSearchIndexOnce([sourceRoot]);
81
expect(searchIndex.countIndexedFilesForRoots([sourceRoot])).toBe(1);
82
83
const results = await searchIndex.searchFiles('dragon', [sourceRoot], 20);
84
expect(results.length).toBe(1);
85
expect(results[0].name).toBe('dragon-ball.mkv');
86
});
87
88
test('searchFiles removes stale ghost hits and rescans for new external files', async () =>
89
{
90
const searchIndex = loadModule();
91
const ghostPath = path.join(sourceRoot, 'ghost.txt');
92
const livePath = path.join(sourceRoot, 'fresh-notes.txt');
93
94
const { getDb } = require('../server/db');
95
const db = getDb();
96
db.prepare(`
97
INSERT INTO file_search_index (
98
file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at
99
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
100
`).run([
101
ghostPath,
102
path.resolve(sourceRoot),
103
'ghost.txt',
104
'ghost.txt',
105
'ghost.txt',
106
'ghost.txt',
107
3,
108
Date.now(),
109
Date.now(),
110
]);
111
112
fs.writeFileSync(livePath, 'hello');
113
114
const ghostResults = await searchIndex.searchFiles('ghost', [sourceRoot], 20);
115
expect(ghostResults.some((row) => row.name === 'ghost.txt')).toBe(false);
116
const ghostRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([ghostPath]);
117
expect(ghostRow).toBeFalsy();
118
119
const freshResults = await searchIndex.searchFiles('fresh', [sourceRoot], 20);
120
expect(freshResults.some((row) => row.name === 'fresh-notes.txt')).toBe(true);
121
});
122
123
test('rewriteIndexPaths moves indexed paths', async () =>
124
{
125
const from = path.join(sourceRoot, 'old-name.txt');
126
const to = path.join(sourceRoot, 'new-name.txt');
127
fs.writeFileSync(from, 'data');
128
129
const searchIndex = loadModule();
130
await searchIndex.syncSearchIndexOnce([sourceRoot]);
131
searchIndex.rewriteIndexPaths(from, to);
132
133
const { getDb } = require('../server/db');
134
const db = getDb();
135
const oldRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([from]);
136
const newRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([to]);
137
expect(oldRow).toBeFalsy();
138
expect(newRow).toBeTruthy();
139
});
140
});
@@ -0,0 +1,140 @@1
'use strict';2
3
const fs = require('fs');4
const os = require('os');5
const path = require('path');6
7
let mockPaths = [];8
9
jest.mock('../server/db', () =>10
{11
const { Database } = require('node-sqlite3-wasm');12
let db;13
function getDb ()14
{15
if (!db)16
{17
db = new Database(':memory:');18
db.exec(`19
CREATE TABLE IF NOT EXISTS file_search_index (20
file_path TEXT PRIMARY KEY,21
root_path TEXT NOT NULL,22
name TEXT NOT NULL,23
name_lower TEXT NOT NULL,24
relative_path TEXT NOT NULL,25
relative_lower TEXT NOT NULL,26
size INTEGER,27
mtime_ms INTEGER,28
indexed_at INTEGER NOT NULL29
);30
CREATE INDEX IF NOT EXISTS idx_file_search_root ON file_search_index(root_path);31
`);32
}33
return db;34
}35
return { getDb, initDb: getDb };36
});37
38
jest.mock('../server/config', () => ({39
loadConfig: () => ({ paths: [...mockPaths] }),40
getEffectivePaths: (paths) => paths,41
}));42
43
describe('search index', () =>44
{45
let tempDir;46
let sourceRoot;47
48
function loadModule ()49
{50
jest.resetModules();51
return require('../server/search-index');52
}53
54
beforeEach(() =>55
{56
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'filemanager-search-index-'));57
sourceRoot = path.join(tempDir, 'root');58
fs.mkdirSync(sourceRoot, { recursive: true });59
mockPaths = [sourceRoot];60
});61
62
afterEach(() =>63
{64
mockPaths = [];65
fs.rmSync(tempDir, { recursive: true, force: true });66
});67
68
test('syncSearchIndexOnce indexes files and removes deleted entries', async () =>69
{70
const keepPath = path.join(sourceRoot, 'dragon-ball.mkv');71
const removePath = path.join(sourceRoot, 'old.txt');72
fs.writeFileSync(keepPath, 'vid');73
fs.writeFileSync(removePath, 'txt');74
75
const searchIndex = loadModule();76
await searchIndex.syncSearchIndexOnce([sourceRoot]);77
expect(searchIndex.countIndexedFilesForRoots([sourceRoot])).toBe(2);78
79
fs.unlinkSync(removePath);80
await searchIndex.syncSearchIndexOnce([sourceRoot]);81
expect(searchIndex.countIndexedFilesForRoots([sourceRoot])).toBe(1);82
83
const results = await searchIndex.searchFiles('dragon', [sourceRoot], 20);84
expect(results.length).toBe(1);85
expect(results[0].name).toBe('dragon-ball.mkv');86
});87
88
test('searchFiles removes stale ghost hits and rescans for new external files', async () =>89
{90
const searchIndex = loadModule();91
const ghostPath = path.join(sourceRoot, 'ghost.txt');92
const livePath = path.join(sourceRoot, 'fresh-notes.txt');93
94
const { getDb } = require('../server/db');95
const db = getDb();96
db.prepare(`97
INSERT INTO file_search_index (98
file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at99
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)100
`).run([101
ghostPath,102
path.resolve(sourceRoot),103
'ghost.txt',104
'ghost.txt',105
'ghost.txt',106
'ghost.txt',107
3,108
Date.now(),109
Date.now(),110
]);111
112
fs.writeFileSync(livePath, 'hello');113
114
const ghostResults = await searchIndex.searchFiles('ghost', [sourceRoot], 20);115
expect(ghostResults.some((row) => row.name === 'ghost.txt')).toBe(false);116
const ghostRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([ghostPath]);117
expect(ghostRow).toBeFalsy();118
119
const freshResults = await searchIndex.searchFiles('fresh', [sourceRoot], 20);120
expect(freshResults.some((row) => row.name === 'fresh-notes.txt')).toBe(true);121
});122
123
test('rewriteIndexPaths moves indexed paths', async () =>124
{125
const from = path.join(sourceRoot, 'old-name.txt');126
const to = path.join(sourceRoot, 'new-name.txt');127
fs.writeFileSync(from, 'data');128
129
const searchIndex = loadModule();130
await searchIndex.syncSearchIndexOnce([sourceRoot]);131
searchIndex.rewriteIndexPaths(from, to);132
133
const { getDb } = require('../server/db');134
const db = getDb();135
const oldRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([from]);136
const newRow = db.prepare('SELECT file_path FROM file_search_index WHERE file_path = ?').get([to]);137
expect(oldRow).toBeFalsy();138
expect(newRow).toBeTruthy();139
});140
});