BlueGit

timfa filemanager

Files

filemanager / server / file-hash-queue.js

file-hash-queue.js

Cursor Optimisation pass
timfa Tim Falken · pushed by timfa · 5 months ago
7e6dfc52 History
file-hash-queue.js 5.5 KiB Download Raw
1'use strict';
2
3const crypto = require('crypto');
4const fs = require('fs');
5const path = require('path');
6const { getDb } = require('./db');
7
8const MAX_QUEUE_LENGTH = toPositiveInt(process.env.FILE_HASH_QUEUE_MAX_LENGTH, 4096);
9const BACKLOG_BATCH_SIZE = toPositiveInt(process.env.FILE_HASH_BACKLOG_BATCH_SIZE, 500);
10const ACTIVE_STREAM_DELAY_MS = toPositiveInt(process.env.FILE_HASH_ACTIVE_STREAM_DELAY_MS, 15000);
11const IDLE_TICK_MS = toPositiveInt(process.env.FILE_HASH_IDLE_TICK_MS, 250);
12
13const queue = [];
14const queued = new Set();
15
16let loopStarted = false;
17let loopTimer = null;
18let workerRunning = false;
19
20function toPositiveInt (value, fallback)
21{
22 const parsed = Number.parseInt(String(value || ''), 10);
23 return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
24}
25
26function delay (ms)
27{
28 return new Promise((resolve) => setTimeout(resolve, ms));
29}
30
31/**
32 * Computes the SHA-256 hex digest of the file at `filePath`.
33 * Streams the file so it works for large files without loading them into memory.
34 */
35function computeFileSha256 (filePath)
36{
37 return new Promise((resolve, reject) =>
38 {
39 const hash = crypto.createHash('sha256');
40 const stream = fs.createReadStream(filePath);
41 stream.on('error', reject);
42 stream.on('data', (chunk) => hash.update(chunk));
43 stream.on('end', () => resolve(hash.digest('hex')));
44 });
45}
46
47function hasActiveVideoStreams ()
48{
49 try
50 {
51 const { getActiveStreamCount } = require('./video-stream');
52 return Number(getActiveStreamCount()) > 0;
53 } catch
54 {
55 return false;
56 }
57}
58
59function loopDelayMs ()
60{
61 return hasActiveVideoStreams() ? ACTIVE_STREAM_DELAY_MS : IDLE_TICK_MS;
62}
63
64function enqueueFileHash (filePath)
65{
66 const resolved = path.resolve(String(filePath || ''));
67 if (!resolved) return false;
68 if (queued.has(resolved)) return false;
69
70 if (queue.length >= MAX_QUEUE_LENGTH)
71 {
72 const dropped = queue.shift();
73 if (dropped) queued.delete(dropped);
74 }
75
76 queued.add(resolved);
77 queue.push(resolved);
78 scheduleWorker();
79 return true;
80}
81
82function persistFileHash (filePath, sha256)
83{
84 const db = getDb();
85 const existing = db.prepare('SELECT file_path FROM file_meta WHERE file_path = ?').get([filePath]);
86
87 if (existing)
88 {
89 db.prepare('UPDATE file_meta SET sha256 = ? WHERE file_path = ?').run([sha256, filePath]);
90 return;
91 }
92
93 let uploadedAt = null;
94 try
95 {
96 const stat = fs.statSync(filePath);
97 const birthtimeMs = Number(stat.birthtimeMs);
98 const ctimeMs = Number(stat.ctimeMs);
99 const mtimeMs = Number(stat.mtimeMs);
100 const chosenMs = (Number.isFinite(birthtimeMs) && birthtimeMs > 0)
101 ? birthtimeMs
102 : (Number.isFinite(ctimeMs) && ctimeMs > 0)
103 ? ctimeMs
104 : (Number.isFinite(mtimeMs) && mtimeMs > 0)
105 ? mtimeMs
106 : null;
107 if (Number.isFinite(chosenMs) && chosenMs > 0)
108 {
109 uploadedAt = new Date(chosenMs).toISOString();
110 }
111 } catch
112 {
113 // Ignore missing files.
114 }
115
116 db.prepare(`
117 INSERT INTO file_meta (file_path, uploader, downloaders, watched_by, uploaded_at, sha256)
118 VALUES (?, ?, ?, ?, ?, ?)
119 ON CONFLICT(file_path) DO UPDATE SET sha256 = excluded.sha256
120 `).run([filePath, '', '[]', '{}', uploadedAt, sha256]);
121}
122
123async function processQueuedFile (filePath)
124{
125 let stat;
126 try
127 {
128 stat = await fs.promises.stat(filePath);
129 } catch
130 {
131 return;
132 }
133
134 if (!stat.isFile()) return;
135
136 const sha256 = await computeFileSha256(filePath);
137 persistFileHash(filePath, sha256);
138}
139
140async function processQueueBatch (respectStreamDelay)
141{
142 while (queue.length)
143 {
144 const filePath = queue.shift();
145 queued.delete(filePath);
146 if (!filePath) continue;
147
148 try
149 {
150 await processQueuedFile(filePath);
151 } catch (err)
152 {
153 console.error('[file-hash] hash failed:', filePath, err && err.message ? err.message : err);
154 }
155
156 if (queue.length && respectStreamDelay) await delay(loopDelayMs());
157 }
158}
159
160async function runWorkerPass ()
161{
162 if (workerRunning) return;
163 workerRunning = true;
164
165 try
166 {
167 await processQueueBatch(true);
168 } finally
169 {
170 workerRunning = false;
171 }
172}
173
174function scheduleWorker ()
175{
176 if (workerRunning || !queue.length) return;
177
178 void runWorkerPass().finally(() =>
179 {
180 if (queue.length) scheduleWorker();
181 });
182}
183
184function enqueueBacklogFromDb ()
185{
186 const db = getDb();
187 const rows = db.prepare(`
188 SELECT file_path
189 FROM file_meta
190 WHERE sha256 IS NULL OR sha256 = ''
191 LIMIT ?
192 `).all([BACKLOG_BATCH_SIZE]);
193
194 for (const row of rows)
195 {
196 enqueueFileHash(row.file_path);
197 }
198}
199
200function startFileHashQueue ()
201{
202 if (loopStarted) return;
203 loopStarted = true;
204 enqueueBacklogFromDb();
205 scheduleWorker();
206
207 loopTimer = setInterval(() =>
208 {
209 enqueueBacklogFromDb();
210 scheduleWorker();
211 }, 60 * 1000);
212 if (loopTimer && typeof loopTimer.unref === 'function') loopTimer.unref();
213}
214
215function stopFileHashQueue ()
216{
217 loopStarted = false;
218 if (loopTimer)
219 {
220 clearInterval(loopTimer);
221 loopTimer = null;
222 }
223 queue.length = 0;
224 queued.clear();
225}
226
227/**
228 * Processes all queued hashes. Intended for tests and graceful shutdown drains.
229 */
230async function drainFileHashQueue ()
231{
232 while (workerRunning) await delay(5);
233 workerRunning = true;
234 try
235 {
236 await processQueueBatch(false);
237 } finally
238 {
239 workerRunning = false;
240 }
241}
242
243function getFileHashQueueSnapshot ()
244{
245 return {
246 loopStarted,
247 queueLength: queue.length,
248 workerRunning,
249 hasActiveVideoStreams: hasActiveVideoStreams(),
250 };
251}
252
253module.exports = {
254 computeFileSha256,
255 enqueueFileHash,
256 drainFileHashQueue,
257 startFileHashQueue,
258 stopFileHashQueue,
259 getFileHashQueueSnapshot,
260};