BlueGit

timfa filemanager

Tune stream retention for CPU and RAM limits

timfa Tim Falken · pushed by timfa

Authored 2026-05-14 · 6 months ago

Hash
c171b6b772bc7293791385445a947879c3006cdc
Parent
9b31f226
3 files changed +84 −4
.github/technical-reference.md +1 −1
@@ -98,7 +98,7 @@
98 98 - STREAM_ROOT_DIR - Temporary root directory for active ffmpeg-backed stream sessions.
99 99 - STREAM_SESSION_TTL_MS - Idle lifetime for a stream session before cleanup.
100 100 - STREAM_IDLE_STOP_MS - Maximum client inactivity window before an active stream session is terminated (5 minutes).
101 - STREAM_SEGMENT_RETENTION_COUNT - Number of most-recent HLS segment indices kept on disk while older segments are pruned (default 180, overridable via VIDEO_STREAM_SEGMENT_RETENTION_COUNT).
101 - STREAM_SEGMENT_RETENTION_COUNT - Number of most-recent HLS segment indices kept on disk while older segments are pruned; when not overridden with VIDEO_STREAM_SEGMENT_RETENTION_COUNT it is derived from hardware capacity (RAM + CPU), current memory/CPU pressure, and Raspberry Pi 5 profile bounds.
102 102
103 103 ## server/db.js
104 104 - initDb() - Initializes SQLite schema for users, file metadata, auth_sessions, and share_tokens (expires_at index).
@@ -98,7 +98,7 @@
98- STREAM_ROOT_DIR - Temporary root directory for active ffmpeg-backed stream sessions.
98- STREAM_ROOT_DIR - Temporary root directory for active ffmpeg-backed stream sessions.
99- STREAM_SESSION_TTL_MS - Idle lifetime for a stream session before cleanup.
99- STREAM_SESSION_TTL_MS - Idle lifetime for a stream session before cleanup.
100- STREAM_IDLE_STOP_MS - Maximum client inactivity window before an active stream session is terminated (5 minutes).
100- STREAM_IDLE_STOP_MS - Maximum client inactivity window before an active stream session is terminated (5 minutes).
101- STREAM_SEGMENT_RETENTION_COUNT - Number of most-recent HLS segment indices kept on disk while older segments are pruned (default 180, overridable via VIDEO_STREAM_SEGMENT_RETENTION_COUNT).
101- STREAM_SEGMENT_RETENTION_COUNT - Number of most-recent HLS segment indices kept on disk while older segments are pruned; when not overridden with VIDEO_STREAM_SEGMENT_RETENTION_COUNT it is derived from hardware capacity (RAM + CPU), current memory/CPU pressure, and Raspberry Pi 5 profile bounds.
102
102
103## server/db.js
103## server/db.js
104- initDb() - Initializes SQLite schema for users, file metadata, auth_sessions, and share_tokens (expires_at index).
104- initDb() - Initializes SQLite schema for users, file metadata, auth_sessions, and share_tokens (expires_at index).
server/video-stream.js +74 −1
@@ -10,6 +10,7 @@ const STREAM_ROOT_DIR = path.join(os.tmpdir(), 'filemanager-video-streams');
10 10 const STREAM_SESSION_TTL_MS = 30 * 60 * 1000;
11 11 const STREAM_READY_TIMEOUT_MS = 20 * 1000;
12 12 const STREAM_IDLE_STOP_MS = 5 * 60 * 1000;
13 const ESTIMATED_SEGMENT_BYTES = 1024 * 1024;
13 14 const STREAM_SEGMENT_RETENTION_COUNT = parseSegmentRetentionCount(process.env.VIDEO_STREAM_SEGMENT_RETENTION_COUNT);
14 15 const sessions = new Map();
15 16
@@ -17,7 +18,78 @@ function parseSegmentRetentionCount (value)
17 18 {
18 19 const parsed = Number.parseInt(String(value || ''), 10);
19 20 if (Number.isInteger(parsed) && parsed >= 30) return parsed;
20 return 180;
21 return defaultSegmentRetentionCount();
22 }
23
24 function readRaspberryPiModel ()
25 {
26 try
27 {
28 return fs.readFileSync('/proc/device-tree/model', 'utf8').replace(/\u0000/g, '').trim();
29 } catch
30 {
31 return '';
32 }
33 }
34
35 function clampNumber (value, min, max)
36 {
37 return Math.min(max, Math.max(min, value));
38 }
39
40 function currentCpuPressureFactor (cpuCount)
41 {
42 if (typeof os.loadavg !== 'function') return 1;
43 if (!Number.isFinite(cpuCount) || cpuCount <= 0) return 1;
44
45 const [load1] = os.loadavg();
46 if (!Number.isFinite(load1) || load1 < 0) return 1;
47
48 const saturation = load1 / cpuCount;
49 if (saturation >= 1.5) return 0.72;
50 if (saturation >= 1.2) return 0.8;
51 if (saturation >= 1.0) return 0.88;
52 if (saturation >= 0.8) return 0.94;
53 return 1;
54 }
55
56 function defaultSegmentRetentionCount ()
57 {
58 const totalMemBytes = Number(os.totalmem()) || 0;
59 const freeMemBytes = Number(os.freemem()) || 0;
60 const cpuCount = Math.max(1, ((os.cpus() || []).length || 1));
61 const arch = String(os.arch() || '').toLowerCase();
62 const piModel = readRaspberryPiModel().toLowerCase();
63 const isPi5 = piModel.includes('raspberry pi 5');
64 const isArm = arch === 'arm' || arch === 'arm64' || arch.startsWith('armv');
65 const cpuPressure = currentCpuPressureFactor(cpuCount);
66
67 const freeMemRatio = totalMemBytes > 0 ? (freeMemBytes / totalMemBytes) : 1;
68 const memoryPressure = freeMemRatio < 0.12 ? 0.72 : freeMemRatio < 0.2 ? 0.84 : freeMemRatio < 0.3 ? 0.92 : 1;
69
70 if (isPi5)
71 {
72 const memoryBudgetBytes = Math.floor(totalMemBytes * 0.03);
73 const byMemory = Math.floor(memoryBudgetBytes / ESTIMATED_SEGMENT_BYTES);
74 const byCpu = 120 + ((cpuCount - 2) * 20);
75 const combined = Math.min(byMemory, byCpu);
76 return clampNumber(Math.floor(combined * cpuPressure * memoryPressure), 100, 220);
77 }
78
79 if (isArm)
80 {
81 const memoryBudgetBytes = Math.floor(totalMemBytes * 0.025);
82 const byMemory = Math.floor(memoryBudgetBytes / ESTIMATED_SEGMENT_BYTES);
83 const byCpu = 90 + ((cpuCount - 2) * 14);
84 const combined = Math.min(byMemory, byCpu);
85 return clampNumber(Math.floor(combined * cpuPressure * memoryPressure), 80, 180);
86 }
87
88 const memoryBudgetBytes = Math.floor(totalMemBytes * 0.05);
89 const byMemory = Math.floor(memoryBudgetBytes / ESTIMATED_SEGMENT_BYTES);
90 const byCpu = 140 + ((cpuCount - 4) * 16);
91 const combined = Math.min(byMemory, byCpu);
92 return clampNumber(Math.floor(combined * cpuPressure * memoryPressure), 120, 320);
21 93 }
22 94
23 95 function hasInitialHlsAssets (outputDir)
@@ -727,6 +799,7 @@ module.exports = {
727 799 STREAM_ROOT_DIR,
728 800 STREAM_SESSION_TTL_MS,
729 801 STREAM_IDLE_STOP_MS,
802 STREAM_SEGMENT_RETENTION_COUNT,
730 803 createVideoStreamSession,
731 804 sendVideoStreamAsset,
732 805 releaseVideoStreamSession,
@@ -10,6 +10,7 @@ const STREAM_ROOT_DIR = path.join(os.tmpdir(), 'filemanager-video-streams');
10const STREAM_SESSION_TTL_MS = 30 * 60 * 1000;
10const STREAM_SESSION_TTL_MS = 30 * 60 * 1000;
11const STREAM_READY_TIMEOUT_MS = 20 * 1000;
11const STREAM_READY_TIMEOUT_MS = 20 * 1000;
12const STREAM_IDLE_STOP_MS = 5 * 60 * 1000;
12const STREAM_IDLE_STOP_MS = 5 * 60 * 1000;
13const ESTIMATED_SEGMENT_BYTES = 1024 * 1024;
13const STREAM_SEGMENT_RETENTION_COUNT = parseSegmentRetentionCount(process.env.VIDEO_STREAM_SEGMENT_RETENTION_COUNT);
14const STREAM_SEGMENT_RETENTION_COUNT = parseSegmentRetentionCount(process.env.VIDEO_STREAM_SEGMENT_RETENTION_COUNT);
14const sessions = new Map();
15const sessions = new Map();
15
16
@@ -17,7 +18,78 @@ function parseSegmentRetentionCount (value)
17{
18{
18 const parsed = Number.parseInt(String(value || ''), 10);
19 const parsed = Number.parseInt(String(value || ''), 10);
19 if (Number.isInteger(parsed) && parsed >= 30) return parsed;
20 if (Number.isInteger(parsed) && parsed >= 30) return parsed;
20 return 180;
21 return defaultSegmentRetentionCount();
22}
23
24function readRaspberryPiModel ()
25{
26 try
27 {
28 return fs.readFileSync('/proc/device-tree/model', 'utf8').replace(/\u0000/g, '').trim();
29 } catch
30 {
31 return '';
32 }
33}
34
35function clampNumber (value, min, max)
36{
37 return Math.min(max, Math.max(min, value));
38}
39
40function currentCpuPressureFactor (cpuCount)
41{
42 if (typeof os.loadavg !== 'function') return 1;
43 if (!Number.isFinite(cpuCount) || cpuCount <= 0) return 1;
44
45 const [load1] = os.loadavg();
46 if (!Number.isFinite(load1) || load1 < 0) return 1;
47
48 const saturation = load1 / cpuCount;
49 if (saturation >= 1.5) return 0.72;
50 if (saturation >= 1.2) return 0.8;
51 if (saturation >= 1.0) return 0.88;
52 if (saturation >= 0.8) return 0.94;
53 return 1;
54}
55
56function defaultSegmentRetentionCount ()
57{
58 const totalMemBytes = Number(os.totalmem()) || 0;
59 const freeMemBytes = Number(os.freemem()) || 0;
60 const cpuCount = Math.max(1, ((os.cpus() || []).length || 1));
61 const arch = String(os.arch() || '').toLowerCase();
62 const piModel = readRaspberryPiModel().toLowerCase();
63 const isPi5 = piModel.includes('raspberry pi 5');
64 const isArm = arch === 'arm' || arch === 'arm64' || arch.startsWith('armv');
65 const cpuPressure = currentCpuPressureFactor(cpuCount);
66
67 const freeMemRatio = totalMemBytes > 0 ? (freeMemBytes / totalMemBytes) : 1;
68 const memoryPressure = freeMemRatio < 0.12 ? 0.72 : freeMemRatio < 0.2 ? 0.84 : freeMemRatio < 0.3 ? 0.92 : 1;
69
70 if (isPi5)
71 {
72 const memoryBudgetBytes = Math.floor(totalMemBytes * 0.03);
73 const byMemory = Math.floor(memoryBudgetBytes / ESTIMATED_SEGMENT_BYTES);
74 const byCpu = 120 + ((cpuCount - 2) * 20);
75 const combined = Math.min(byMemory, byCpu);
76 return clampNumber(Math.floor(combined * cpuPressure * memoryPressure), 100, 220);
77 }
78
79 if (isArm)
80 {
81 const memoryBudgetBytes = Math.floor(totalMemBytes * 0.025);
82 const byMemory = Math.floor(memoryBudgetBytes / ESTIMATED_SEGMENT_BYTES);
83 const byCpu = 90 + ((cpuCount - 2) * 14);
84 const combined = Math.min(byMemory, byCpu);
85 return clampNumber(Math.floor(combined * cpuPressure * memoryPressure), 80, 180);
86 }
87
88 const memoryBudgetBytes = Math.floor(totalMemBytes * 0.05);
89 const byMemory = Math.floor(memoryBudgetBytes / ESTIMATED_SEGMENT_BYTES);
90 const byCpu = 140 + ((cpuCount - 4) * 16);
91 const combined = Math.min(byMemory, byCpu);
92 return clampNumber(Math.floor(combined * cpuPressure * memoryPressure), 120, 320);
21}
93}
22
94
23function hasInitialHlsAssets (outputDir)
95function hasInitialHlsAssets (outputDir)
@@ -727,6 +799,7 @@ module.exports = {
727 STREAM_ROOT_DIR,
799 STREAM_ROOT_DIR,
728 STREAM_SESSION_TTL_MS,
800 STREAM_SESSION_TTL_MS,
729 STREAM_IDLE_STOP_MS,
801 STREAM_IDLE_STOP_MS,
802 STREAM_SEGMENT_RETENTION_COUNT,
730 createVideoStreamSession,
803 createVideoStreamSession,
731 sendVideoStreamAsset,
804 sendVideoStreamAsset,
732 releaseVideoStreamSession,
805 releaseVideoStreamSession,
tests/video-stream.test.js +9 −2
@@ -382,15 +382,22 @@ describe('video stream bootstrap', () =>
382 382 const streamModule = require('../server/video-stream');
383 383 const session = await streamModule.createVideoStreamSession(sourceFile);
384 384 const sessionDir = path.join(streamModule.STREAM_ROOT_DIR, session.sessionId);
385 expect(Number.isInteger(streamModule.STREAM_SEGMENT_RETENTION_COUNT)).toBe(true);
386 const targetSegmentIndex = streamModule.STREAM_SEGMENT_RETENTION_COUNT + 40;
385 387
386 for (let segmentIndex = 0; segmentIndex <= 220; segmentIndex += 1)
388 for (let segmentIndex = 0; segmentIndex <= targetSegmentIndex; segmentIndex += 1)
387 389 {
388 390 const fileName = `stream_0_segment_${String(segmentIndex).padStart(6, '0')}.ts`;
389 391 fs.writeFileSync(path.join(sessionDir, fileName), `segment-${segmentIndex}`);
390 392 }
391 393
392 394 const { req, res } = makeReqRes();
393 await streamModule.sendVideoStreamAsset(session.sessionId, 'stream_0_segment_000220.ts', req, res);
395 await streamModule.sendVideoStreamAsset(
396 session.sessionId,
397 `stream_0_segment_${String(targetSegmentIndex).padStart(6, '0')}.ts`,
398 req,
399 res
400 );
394 401
395 402 await new Promise((resolve) => setTimeout(resolve, 80));
396 403
@@ -382,15 +382,22 @@ describe('video stream bootstrap', () =>
382 const streamModule = require('../server/video-stream');
382 const streamModule = require('../server/video-stream');
383 const session = await streamModule.createVideoStreamSession(sourceFile);
383 const session = await streamModule.createVideoStreamSession(sourceFile);
384 const sessionDir = path.join(streamModule.STREAM_ROOT_DIR, session.sessionId);
384 const sessionDir = path.join(streamModule.STREAM_ROOT_DIR, session.sessionId);
385 expect(Number.isInteger(streamModule.STREAM_SEGMENT_RETENTION_COUNT)).toBe(true);
386 const targetSegmentIndex = streamModule.STREAM_SEGMENT_RETENTION_COUNT + 40;
385
387
386 for (let segmentIndex = 0; segmentIndex <= 220; segmentIndex += 1)
388 for (let segmentIndex = 0; segmentIndex <= targetSegmentIndex; segmentIndex += 1)
387 {
389 {
388 const fileName = `stream_0_segment_${String(segmentIndex).padStart(6, '0')}.ts`;
390 const fileName = `stream_0_segment_${String(segmentIndex).padStart(6, '0')}.ts`;
389 fs.writeFileSync(path.join(sessionDir, fileName), `segment-${segmentIndex}`);
391 fs.writeFileSync(path.join(sessionDir, fileName), `segment-${segmentIndex}`);
390 }
392 }
391
393
392 const { req, res } = makeReqRes();
394 const { req, res } = makeReqRes();
393 await streamModule.sendVideoStreamAsset(session.sessionId, 'stream_0_segment_000220.ts', req, res);
395 await streamModule.sendVideoStreamAsset(
396 session.sessionId,
397 `stream_0_segment_${String(targetSegmentIndex).padStart(6, '0')}.ts`,
398 req,
399 res
400 );
394
401
395 await new Promise((resolve) => setTimeout(resolve, 80));
402 await new Promise((resolve) => setTimeout(resolve, 80));
396
403