'use strict'; const fs = require('fs'); const path = require('path'); const { getDb } = require('./db'); const { loadConfig, getEffectivePaths } = require('./config'); const ACTIVE_CLIENT_GRACE_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_CLIENT_GRACE_MS, 5 * 60 * 1000); const IDLE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_IDLE_SYNC_INTERVAL_MS, 30 * 60 * 1000); const ACTIVE_SYNC_INTERVAL_MS = toPositiveInt(process.env.SEARCH_INDEX_ACTIVE_SYNC_INTERVAL_MS, 10 * 60 * 1000); const MAX_CANDIDATES_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_CANDIDATES, 8000); const MAX_SUBSEQUENCE_SCAN = toPositiveInt(process.env.SEARCH_INDEX_MAX_SUBSEQUENCE_SCAN, 12000); const MAX_VALIDATE_HITS = toPositiveInt(process.env.SEARCH_INDEX_MAX_VALIDATE_HITS, 250); const MAX_RESCAN_DIRS_PER_SEARCH = toPositiveInt(process.env.SEARCH_INDEX_MAX_RESCAN_DIRS, 12); const INDEX_BATCH_SIZE = 400; const ROOT_CHUNK_SIZE = 50; let loopTimer = null; let loopStarted = false; let backgroundSyncRunning = false; function toPositiveInt (value, fallback) { const parsed = Number.parseInt(String(value || ''), 10); return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; } function yieldToEventLoop () { return new Promise((resolve) => setImmediate(resolve)); } function nowMs () { return Date.now(); } function fuzzyMatchScore (needle, haystack) { const q = String(needle || '').trim().toLowerCase(); const t = String(haystack || '').toLowerCase(); if (!q || !t) return null; const idx = t.indexOf(q); if (idx >= 0) { const compactBonus = Math.max(0, 200 - idx); const lengthBonus = Math.max(0, 120 - (t.length - q.length)); return 1200 + compactBonus + lengthBonus; } let qPos = 0; let firstHit = -1; let lastHit = -1; let gaps = 0; for (let i = 0; i < t.length && qPos < q.length; i++) { if (t[i] !== q[qPos]) continue; if (firstHit === -1) firstHit = i; if (lastHit >= 0) gaps += Math.max(0, i - lastHit - 1); lastHit = i; qPos += 1; } if (qPos !== q.length) return null; const span = lastHit - firstHit + 1; const compactness = Math.max(0, 300 - span); const gapPenalty = Math.min(180, gaps * 3); const startBonus = Math.max(0, 120 - firstHit); return 700 + compactness + startBonus - gapPenalty; } function effectiveIndexRoots () { const config = loadConfig(); const roots = getEffectivePaths(config.paths || []); const userPath = typeof config.userPath === 'string' ? config.userPath.trim() : ''; if (userPath) roots.push(userPath); const seen = new Set(); const deduped = []; for (const root of roots) { const key = path.resolve(root); if (seen.has(key)) continue; seen.add(key); deduped.push(key); } return deduped; } function hasActiveClients () { try { const { getActiveStreamCount } = require('./video-stream'); if (Number(getActiveStreamCount()) > 0) return true; } catch { // Stream module unavailable — treat as no active streams. } const db = getDb(); const cutoff = nowMs() - ACTIVE_CLIENT_GRACE_MS; const row = db.prepare(` SELECT COUNT(*) AS n FROM auth_sessions WHERE revoked_at IS NULL AND last_seen_at >= ? `).get([cutoff]); return Number(row && row.n || 0) > 0; } function upsertIndexRow (db, row) { db.prepare(` INSERT INTO file_search_index ( file_path, root_path, name, name_lower, relative_path, relative_lower, size, mtime_ms, indexed_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(file_path) DO UPDATE SET root_path = excluded.root_path, name = excluded.name, name_lower = excluded.name_lower, relative_path = excluded.relative_path, relative_lower = excluded.relative_lower, size = excluded.size, mtime_ms = excluded.mtime_ms, indexed_at = excluded.indexed_at `).run([ row.file_path, row.root_path, row.name, row.name_lower, row.relative_path, row.relative_lower, row.size, row.mtime_ms, row.indexed_at, ]); } function deleteIndexPath (filePath) { getDb().prepare('DELETE FROM file_search_index WHERE file_path = ?').run([path.resolve(String(filePath || ''))]); } function deleteIndexUnder (targetPath) { const resolved = path.resolve(String(targetPath || '')); const db = getDb(); db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([resolved]); db.prepare('DELETE FROM file_search_index WHERE file_path LIKE ?') .run([resolved + path.sep + '%']); } function rewriteIndexPaths (fromPath, toPath) { const from = path.resolve(String(fromPath || '')); const to = path.resolve(String(toPath || '')); const db = getDb(); const rows = db.prepare(` SELECT file_path, root_path, name, relative_path, size, mtime_ms FROM file_search_index WHERE file_path = ? OR file_path LIKE ? `).all([from, from + path.sep + '%']); if (!rows.length) return; const del = db.prepare('DELETE FROM file_search_index WHERE file_path = ?'); for (const row of rows) { del.run([row.file_path]); const nextPath = row.file_path === from ? to : to + row.file_path.slice(from.length); const nextName = path.basename(nextPath); const rootPath = path.resolve(row.root_path); const relativePath = path.relative(rootPath, nextPath); upsertIndexRow(db, { file_path: nextPath, root_path: rootPath, name: nextName, name_lower: nextName.toLowerCase(), relative_path: relativePath, relative_lower: String(relativePath || '').toLowerCase(), size: row.size, mtime_ms: row.mtime_ms, indexed_at: nowMs(), }); } } async function walkFilesUnderRoot (storageRoot, onFile, startDir) { const resolvedRoot = path.resolve(storageRoot); const stack = [path.resolve(startDir || storageRoot)]; const visitedDirs = new Set(); let dirsVisited = 0; while (stack.length) { const dir = stack.pop(); if (!dir) continue; const dirKey = path.resolve(dir); if (visitedDirs.has(dirKey)) continue; visitedDirs.add(dirKey); dirsVisited += 1; if (dirsVisited % 64 === 0) await yieldToEventLoop(); let entries; try { entries = await fs.promises.readdir(dir, { withFileTypes: true }); } catch { continue; } for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { stack.push(fullPath); continue; } if (!entry.isFile()) continue; let stat; try { stat = await fs.promises.stat(fullPath); } catch { continue; } const relativePath = path.relative(resolvedRoot, fullPath); await onFile({ file_path: fullPath, root_path: resolvedRoot, name: entry.name, name_lower: entry.name.toLowerCase(), relative_path: relativePath, relative_lower: String(relativePath || '').toLowerCase(), size: stat.size, mtime_ms: stat.mtimeMs, }); } } } async function syncSearchIndexForRoot (rootPath) { const resolvedRoot = path.resolve(rootPath); try { const stat = await fs.promises.stat(resolvedRoot); if (!stat.isDirectory()) return { rootPath: resolvedRoot, indexed: 0, removed: 0 }; } catch { return { rootPath: resolvedRoot, indexed: 0, removed: 0, offline: true }; } const db = getDb(); const seen = new Set(); const indexedAt = nowMs(); let indexed = 0; await walkFilesUnderRoot(resolvedRoot, async (fileRow) => { const filePath = path.resolve(fileRow.file_path); seen.add(filePath); upsertIndexRow(db, { ...fileRow, file_path: filePath, root_path: resolvedRoot, indexed_at: indexedAt, }); indexed += 1; }); const existing = db.prepare('SELECT file_path FROM file_search_index WHERE root_path = ?').all([resolvedRoot]); let removed = 0; for (const row of existing) { if (seen.has(path.resolve(row.file_path))) continue; db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([row.file_path]); removed += 1; } return { rootPath: resolvedRoot, indexed, removed }; } async function syncSearchIndexOnce (roots) { const targetRoots = Array.isArray(roots) && roots.length ? roots.map((r) => path.resolve(r)) : effectiveIndexRoots(); const summary = { rootsScanned: 0, filesIndexed: 0, filesRemoved: 0, offlineRoots: 0, }; for (const root of targetRoots) { const result = await syncSearchIndexForRoot(root); summary.rootsScanned += 1; summary.filesIndexed += Number(result.indexed || 0); summary.filesRemoved += Number(result.removed || 0); if (result.offline) summary.offlineRoots += 1; } return summary; } function countIndexedFilesForRoots (roots) { const resolved = [...new Set((roots || []).map((r) => path.resolve(r)))]; if (!resolved.length) return 0; let total = 0; const db = getDb(); for (let i = 0; i < resolved.length; i += ROOT_CHUNK_SIZE) { const chunk = resolved.slice(i, i + ROOT_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(', '); const row = db.prepare(`SELECT COUNT(*) AS n FROM file_search_index WHERE root_path IN (${placeholders})`).get(chunk); total += Number(row && row.n || 0); } return total; } function querySubstringCandidates (resolvedRoots, queryLower, rowLimit) { if (!resolvedRoots.length) return []; const db = getDb(); const out = []; const needle = queryLower; for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE) { const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(', '); const rows = db.prepare(` SELECT file_path, root_path, name, relative_path, size, mtime_ms FROM file_search_index WHERE root_path IN (${placeholders}) AND (instr(name_lower, ?) > 0 OR instr(relative_lower, ?) > 0) LIMIT ? `).all([...chunk, needle, needle, rowLimit]); for (const row of rows) out.push(row); if (out.length >= rowLimit) break; } return out.slice(0, rowLimit); } function queryBatchCandidates (resolvedRoots, offset, batchSize) { if (!resolvedRoots.length) return []; const db = getDb(); const out = []; for (let i = 0; i < resolvedRoots.length; i += ROOT_CHUNK_SIZE) { const chunk = resolvedRoots.slice(i, i + ROOT_CHUNK_SIZE); const placeholders = chunk.map(() => '?').join(', '); const rows = db.prepare(` SELECT file_path, root_path, name, relative_path, size, mtime_ms FROM file_search_index WHERE root_path IN (${placeholders}) ORDER BY file_path LIMIT ? OFFSET ? `).all([...chunk, batchSize, offset]); for (const row of rows) out.push(row); if (out.length >= batchSize) return out.slice(0, batchSize); } return out; } function rowToSearchResult (row, query, score) { const filePath = path.resolve(row.file_path); const rootPath = path.resolve(row.root_path); const relativePath = row.relative_path != null ? String(row.relative_path) : path.relative(rootPath, filePath); let mtime = null; if (Number.isFinite(Number(row.mtime_ms)) && Number(row.mtime_ms) > 0) { mtime = new Date(Number(row.mtime_ms)).toISOString(); } return { name: row.name || path.basename(filePath), path: filePath, dir: path.dirname(filePath), rootPath, relativePath, size: Number.isFinite(Number(row.size)) ? Number(row.size) : null, mtime, score, }; } function collectCandidatesFromIndex (resolvedRoots, query, maxCandidates) { const queryLower = String(query || '').trim().toLowerCase(); const byPath = new Map(); const substringRows = querySubstringCandidates(resolvedRoots, queryLower, maxCandidates); for (const row of substringRows) { const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path); if (score == null) continue; byPath.set(path.resolve(row.file_path), { row, score }); } if (byPath.size < maxCandidates) { let offset = 0; let scanned = 0; while (scanned < MAX_SUBSEQUENCE_SCAN && byPath.size < maxCandidates) { const batch = queryBatchCandidates(resolvedRoots, offset, INDEX_BATCH_SIZE); if (!batch.length) break; offset += batch.length; scanned += batch.length; for (const row of batch) { const filePath = path.resolve(row.file_path); if (byPath.has(filePath)) continue; const score = fuzzyMatchScore(query, row.name) ?? fuzzyMatchScore(query, row.relative_path); if (score == null) continue; byPath.set(filePath, { row, score }); if (byPath.size >= maxCandidates) break; } } } return [...byPath.values()]; } async function validateCandidates (candidates, dirtyDirs, db) { const validated = []; const toCheck = candidates.slice(0, MAX_VALIDATE_HITS); for (const entry of toCheck) { const filePath = path.resolve(entry.row.file_path); let stat; try { stat = await fs.promises.stat(filePath); } catch { db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]); dirtyDirs.add(path.dirname(filePath)); continue; } if (!stat.isFile()) { db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]); dirtyDirs.add(path.dirname(filePath)); continue; } const size = Number(stat.size); const mtimeMs = Number(stat.mtimeMs); const staleMeta = Number(entry.row.size) !== size || Number(entry.row.mtime_ms) !== mtimeMs; if (staleMeta) { const rootPath = path.resolve(entry.row.root_path); const name = path.basename(filePath); const relativePath = path.relative(rootPath, filePath); upsertIndexRow(db, { file_path: filePath, root_path: rootPath, name, name_lower: name.toLowerCase(), relative_path: relativePath, relative_lower: String(relativePath || '').toLowerCase(), size, mtime_ms: mtimeMs, indexed_at: nowMs(), }); entry.row.size = size; entry.row.mtime_ms = mtimeMs; dirtyDirs.add(path.dirname(filePath)); } validated.push(entry); } for (const entry of candidates.slice(MAX_VALIDATE_HITS)) { validated.push(entry); } return validated; } async function reindexDirectories (dirs, rootByDir) { const queue = [...dirs].slice(0, MAX_RESCAN_DIRS_PER_SEARCH); const discovered = []; for (const dir of queue) { const resolvedDir = path.resolve(dir); const rootPath = rootByDir.get(resolvedDir); if (!rootPath) continue; try { const stat = await fs.promises.stat(resolvedDir); if (!stat.isDirectory()) continue; } catch { continue; } const db = getDb(); const seen = new Set(); const indexedAt = nowMs(); const resolvedRootPath = path.resolve(rootPath); await walkFilesUnderRoot(resolvedRootPath, async (fileRow) => { const filePath = path.resolve(fileRow.file_path); seen.add(filePath); upsertIndexRow(db, { ...fileRow, file_path: filePath, root_path: resolvedRootPath, indexed_at: indexedAt, }); discovered.push(fileRow); }, resolvedDir); const prefix = resolvedDir + path.sep; const existing = db.prepare(` SELECT file_path FROM file_search_index WHERE root_path = ? AND (file_path = ? OR file_path LIKE ?) `).all([path.resolve(rootPath), resolvedDir, prefix + '%']); for (const row of existing) { const filePath = path.resolve(row.file_path); if (filePath === resolvedDir) continue; if (seen.has(filePath)) continue; db.prepare('DELETE FROM file_search_index WHERE file_path = ?').run([filePath]); } } return discovered; } function rootForPath (resolvedRoots, filePath) { const resolved = path.resolve(filePath); for (const root of resolvedRoots) { const rel = path.relative(root, resolved); if (rel === '' || (!rel.startsWith('..') && !path.isAbsolute(rel))) return root; } return null; } function buildRootByDirMap (resolvedRoots) { const map = new Map(); for (const root of resolvedRoots) { map.set(path.resolve(root), path.resolve(root)); } return map; } async function searchFilesFilesystemWalk (query, roots, maxResults) { const limit = Math.max(1, Math.min(500, Number(maxResults) || 200)); const results = []; const visitedDirs = new Set(); let dirsVisited = 0; for (const root of roots) { const rootPath = path.resolve(root); const stack = [rootPath]; while (stack.length) { const dir = stack.pop(); if (!dir) continue; const dirKey = path.resolve(dir); if (visitedDirs.has(dirKey)) continue; visitedDirs.add(dirKey); dirsVisited += 1; if (dirsVisited % 64 === 0) await yieldToEventLoop(); let entries; try { entries = await fs.promises.readdir(dir, { withFileTypes: true }); } catch { continue; } for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { stack.push(fullPath); continue; } if (!entry.isFile()) continue; const relativePath = path.relative(rootPath, fullPath); const score = fuzzyMatchScore(query, entry.name) ?? fuzzyMatchScore(query, relativePath); if (score == null) continue; let stat = null; try { stat = await fs.promises.stat(fullPath); } catch { /* ignore */ } results.push(rowToSearchResult({ file_path: fullPath, root_path: rootPath, name: entry.name, relative_path: relativePath, size: stat ? stat.size : null, mtime_ms: stat ? stat.mtimeMs : null, }, query, score)); if (results.length > limit * 4) { results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); results.length = limit * 2; } } } } results.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); return results.slice(0, limit); } async function searchFiles (query, roots, maxResults) { const limit = Math.max(1, Math.min(500, Number(maxResults) || 200)); const resolvedRoots = [...new Set((roots || []).map((r) => path.resolve(r)))]; if (!resolvedRoots.length) return []; if (countIndexedFilesForRoots(resolvedRoots) < 1) { scheduleBackgroundSync(resolvedRoots); return searchFilesFilesystemWalk(query, roots, limit); } const candidates = collectCandidatesFromIndex(resolvedRoots, query, MAX_CANDIDATES_PER_SEARCH); const dirtyDirs = new Set(); const db = getDb(); const validated = await validateCandidates(candidates, dirtyDirs, db); const rootByDir = buildRootByDirMap(resolvedRoots); for (const dir of dirtyDirs) { const resolvedDir = path.resolve(dir); if (!rootByDir.has(resolvedDir)) { const root = rootForPath(resolvedRoots, resolvedDir); if (root) rootByDir.set(resolvedDir, root); } } if (dirtyDirs.size) { const discovered = await reindexDirectories(dirtyDirs, rootByDir); for (const fileRow of discovered) { const score = fuzzyMatchScore(query, fileRow.name) ?? fuzzyMatchScore(query, fileRow.relative_path); if (score == null) continue; validated.push({ row: { file_path: fileRow.file_path, root_path: fileRow.root_path, name: fileRow.name, relative_path: fileRow.relative_path, size: fileRow.size, mtime_ms: fileRow.mtime_ms, }, score, }); } } const merged = new Map(); for (const entry of validated) { const filePath = path.resolve(entry.row.file_path); const prev = merged.get(filePath); if (!prev || entry.score > prev.score) merged.set(filePath, entry); } const results = [...merged.values()] .map((entry) => rowToSearchResult(entry.row, query, entry.score)) .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); return results.slice(0, limit); } function scheduleBackgroundSync (roots) { if (backgroundSyncRunning) return; backgroundSyncRunning = true; void syncSearchIndexOnce(roots) .catch(() => { /* non-fatal */ }) .finally(() => { backgroundSyncRunning = false; }); } function scheduleNextLoop (delayMs) { loopTimer = setTimeout(async () => { loopTimer = null; try { if (!hasActiveClients()) { await syncSearchIndexOnce(); } } catch (err) { console.error('[search-index] sync failed:', err && err.message ? err.message : err); } finally { if (loopStarted) { const delay = hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS; scheduleNextLoop(delay); } } }, delayMs); if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref(); } function startSearchIndexLoop () { if (loopStarted) return; loopStarted = true; scheduleBackgroundSync(effectiveIndexRoots()); scheduleNextLoop(hasActiveClients() ? ACTIVE_SYNC_INTERVAL_MS : IDLE_SYNC_INTERVAL_MS); } function stopSearchIndexLoop () { loopStarted = false; if (loopTimer) { clearTimeout(loopTimer); loopTimer = null; } } function getSearchIndexSnapshot () { const db = getDb(); const row = db.prepare('SELECT COUNT(*) AS n FROM file_search_index').get(); return { indexedFileCount: Number(row && row.n || 0), loopStarted, hasActiveClients: hasActiveClients(), backgroundSyncRunning, }; } module.exports = { fuzzyMatchScore, searchFiles, searchFilesFilesystemWalk, syncSearchIndexOnce, countIndexedFilesForRoots, deleteIndexPath, deleteIndexUnder, rewriteIndexPaths, reindexDirectoryTree: async (dirPath) => { const resolvedDir = path.resolve(String(dirPath || '')); const root = rootForPath(effectiveIndexRoots(), resolvedDir); if (!root) return; const rootByDir = new Map([[resolvedDir, root]]); await reindexDirectories(new Set([resolvedDir]), rootByDir); }, startSearchIndexLoop, stopSearchIndexLoop, getSearchIndexSnapshot, };