BlueGit

timfa filemanager

Add cached PGS subtitle frame rendering

timfa Tim Falken · pushed by timfa

Authored 2026-05-16 · 6 months ago

Hash
6d2b597896d25d0ba7665396ee40505842ecc626
Parent
b48f942d
7 files changed +502 −54
.github/technical-reference.md +11 −4
@@ -3,8 +3,15 @@
3 3 ## server/index.js
4 4 - start() - Initialises DB/keys, starts HTTP/HTTPS servers, and starts the background thumbnail cache loop.
5 5 - shutdown(opts, ...servers) - Graceful teardown: stops thumbnail loop, closes DB/servers, exits with configurable code after drain/timeout.
6 - Express error handler - Logs uncaught route errors to logs.txt before returning a JSON 500 response.
6 7 - GET /share/:token - Calls buildShareOpenGraphPage to serve share.html with injected OG meta tags for all visitors; social media crawlers see rich preview data, browsers get the full interactive page.
7 8
9 ## server/logger.js
10 - logError(message, error, details) - Appends error lines to logs.txt, trimming old lines so the file stays under 1 MB; use for route failures and process-level exceptions.
11
12 ## server/image-subtitles.js
13 - renderImageSubtitleFrame({ filePath, streamIndex, timeSeconds, canvasWidth, canvasHeight }) - Extracts bitmap subtitle streams to cached `.sup`, parses PGS display sets, and returns cached/composited PNG subtitle frames for a requested timestamp; returns null when no subtitle is active.
14
8 15 ## server/thumbnail-cache.js
9 16 - syncThumbnailCacheOnce() - Mirrors configured roots and userPath into public/thumbs and generates 99x99-max WebP thumbnails for image files, video files (ffprobe/ffmpeg frame at ~10%), and 3D model files (STL/OBJ, isometric SVG projection via Sharp); removes stale thumbnails and updates manifest hashes.
10 17 - startThumbnailCacheLoop() - Schedules cache sync runs 5 seconds after boot and 5 seconds after each run completes.
@@ -73,8 +80,8 @@
73 80 - POST /api/files/upload - Streams multipart uploads into hidden temp files, computes/stores sha256, and enforces user-folder size limits.
74 81 - POST /api/files/move - Moves file/folder; supports conflict strategies: `error` (default), `overwrite`, or `rename` with optional `targetName`.
75 82 - POST /api/files/copy - Copies a file to a destination folder with conflict strategies (`error`, `overwrite`, `rename`) and user-folder size-limit checks.
76 - POST /api/files/video-stream - Creates a short-lived HLS session for a video and returns playlistUrl, durationSeconds, subtitleTracks, imageSubtitleTracks, and subtitleSummary ({total, converted, imageBased, skipped}).
77 - GET /api/files/video-stream-subtitle-frame - Authenticated. Extracts a PNG frame for an image-based subtitle stream at a specific timestamp using ffmpeg and the source file path.
83 - POST /api/files/video-stream - Creates a short-lived HLS session for a video and returns playlistUrl, durationSeconds, videoWidth, videoHeight, subtitleTracks, imageSubtitleTracks, and subtitleSummary ({total, converted, imageBased, skipped}).
84 - GET /api/files/video-stream-subtitle-frame - Authenticated. Renders a PNG subtitle frame for bitmap subtitle tracks via cached PGS decode/composite flow; on failures or no active subtitle it returns 204 so the client can stop retrying.
78 85 - DELETE /api/files/video-stream/:sessionId - Explicitly releases an active short-lived video stream session and stops backend processing.
79 86 - POST /api/files/watched-statuses - Returns fast per-user watched statuses for up to 500 file paths without triggering expensive metadata/hash work; response is { statuses: { [filePath]: 'fully'|'partway' } }.
80 87 - POST /api/files/watch-progress - Records per-user watch status (`partway` >=10%, `fully` >=70%) for sidebar "Watched by" display.
@@ -94,7 +101,7 @@
94 101 - ZIP_CACHE_DIR - Absolute path to the zip-cache/ directory (project root).
95 102
96 103 ## server/video-stream.js
97 - createVideoStreamSession(sourcePath) - Probes input metadata and reuses a compatible active transcode session per source file when possible; returns a per-viewer lease id as sessionId plus duration, subtitleTracks, imageSubtitleTracks, and subtitleSummary data.
104 - createVideoStreamSession(sourcePath) - Probes input metadata and reuses a compatible active transcode session per source file when possible; returns a per-viewer lease id as sessionId plus duration, videoWidth, videoHeight, subtitleTracks, imageSubtitleTracks, and subtitleSummary data.
98 105 - sendVideoStreamAsset(sessionId, assetPath, req, res) - Resolves viewer leases to a shared stream session, serves HLS assets, tracks lease activity, and prunes segments using the slowest active viewer position.
99 106 - releaseVideoStreamSession(sessionId) - Releases one viewer lease; only destroys the underlying transcode session when no active leases remain.
100 107 - STREAM_ROOT_DIR - Temporary root directory for active ffmpeg-backed stream sessions.
@@ -160,7 +167,7 @@
160 167 - setThumbnailPending(iconEl, pending) - Toggles per-icon pending spinner state while a cached image thumbnail is not ready yet.
161 168 - buildThumbnailSource(item, blob, kind) - Produces thumbnail image source (data/blob URL) for icon injection.
162 169 - buildAuthHeaders() - Builds the authenticated fetch headers used by preview-side API calls that need the bearer token plus fingerprint header.
163 - bindImageSubtitleOverlay(player, videojs, prefs, session, item) - Adds the fullscreen-safe image-subtitle button, menu, and bottom overlay used to fetch and display subtitle frames for image-based tracks.
170 - bindImageSubtitleOverlay(player, videojs, prefs, session, item) - Adds the fullscreen-safe image-subtitle button, menu, and bottom overlay used to fetch and display subtitle frames for image-based tracks; disables the track after the first failed frame request to avoid repeated polling.
164 171 - renderStreamedVideoPreview(item, targetEl) - Creates the Video.js player with HLS source/track config, enforces non-live playback UI for file streams, uses container-fill sizing to keep previews within bounds, restores per-user video prefs (volume + subtitle/audio track names when present), wires the image-subtitle overlay controller, forces caption overrides on each open (text background opacity transparent + text edge style uniform), reports watch progress, warns when subtitle streams exist but none are web-compatible, and releases the backend stream session on close/unload.
165 172 - modelPreviewType(name) - Classifies direct client-side 3D preview support for `.stl` and `.obj`.
166 173 - loadThreePreviewModules() - Lazy-loads Three.js plus OrbitControls/STLLoader/OBJLoader via import map aliases.
@@ -3,8 +3,15 @@
3## server/index.js
3## server/index.js
4- start() - Initialises DB/keys, starts HTTP/HTTPS servers, and starts the background thumbnail cache loop.
4- start() - Initialises DB/keys, starts HTTP/HTTPS servers, and starts the background thumbnail cache loop.
5- shutdown(opts, ...servers) - Graceful teardown: stops thumbnail loop, closes DB/servers, exits with configurable code after drain/timeout.
5- shutdown(opts, ...servers) - Graceful teardown: stops thumbnail loop, closes DB/servers, exits with configurable code after drain/timeout.
6- Express error handler - Logs uncaught route errors to logs.txt before returning a JSON 500 response.
6- GET /share/:token - Calls buildShareOpenGraphPage to serve share.html with injected OG meta tags for all visitors; social media crawlers see rich preview data, browsers get the full interactive page.
7- GET /share/:token - Calls buildShareOpenGraphPage to serve share.html with injected OG meta tags for all visitors; social media crawlers see rich preview data, browsers get the full interactive page.
7
8
9## server/logger.js
10- logError(message, error, details) - Appends error lines to logs.txt, trimming old lines so the file stays under 1 MB; use for route failures and process-level exceptions.
11
12## server/image-subtitles.js
13- renderImageSubtitleFrame({ filePath, streamIndex, timeSeconds, canvasWidth, canvasHeight }) - Extracts bitmap subtitle streams to cached `.sup`, parses PGS display sets, and returns cached/composited PNG subtitle frames for a requested timestamp; returns null when no subtitle is active.
14
8## server/thumbnail-cache.js
15## server/thumbnail-cache.js
9- syncThumbnailCacheOnce() - Mirrors configured roots and userPath into public/thumbs and generates 99x99-max WebP thumbnails for image files, video files (ffprobe/ffmpeg frame at ~10%), and 3D model files (STL/OBJ, isometric SVG projection via Sharp); removes stale thumbnails and updates manifest hashes.
16- syncThumbnailCacheOnce() - Mirrors configured roots and userPath into public/thumbs and generates 99x99-max WebP thumbnails for image files, video files (ffprobe/ffmpeg frame at ~10%), and 3D model files (STL/OBJ, isometric SVG projection via Sharp); removes stale thumbnails and updates manifest hashes.
10- startThumbnailCacheLoop() - Schedules cache sync runs 5 seconds after boot and 5 seconds after each run completes.
17- startThumbnailCacheLoop() - Schedules cache sync runs 5 seconds after boot and 5 seconds after each run completes.
@@ -73,8 +80,8 @@
73- POST /api/files/upload - Streams multipart uploads into hidden temp files, computes/stores sha256, and enforces user-folder size limits.
80- POST /api/files/upload - Streams multipart uploads into hidden temp files, computes/stores sha256, and enforces user-folder size limits.
74- POST /api/files/move - Moves file/folder; supports conflict strategies: `error` (default), `overwrite`, or `rename` with optional `targetName`.
81- POST /api/files/move - Moves file/folder; supports conflict strategies: `error` (default), `overwrite`, or `rename` with optional `targetName`.
75- POST /api/files/copy - Copies a file to a destination folder with conflict strategies (`error`, `overwrite`, `rename`) and user-folder size-limit checks.
82- POST /api/files/copy - Copies a file to a destination folder with conflict strategies (`error`, `overwrite`, `rename`) and user-folder size-limit checks.
76- POST /api/files/video-stream - Creates a short-lived HLS session for a video and returns playlistUrl, durationSeconds, subtitleTracks, imageSubtitleTracks, and subtitleSummary ({total, converted, imageBased, skipped}).
83- POST /api/files/video-stream - Creates a short-lived HLS session for a video and returns playlistUrl, durationSeconds, videoWidth, videoHeight, subtitleTracks, imageSubtitleTracks, and subtitleSummary ({total, converted, imageBased, skipped}).
77- GET /api/files/video-stream-subtitle-frame - Authenticated. Extracts a PNG frame for an image-based subtitle stream at a specific timestamp using ffmpeg and the source file path.
84- GET /api/files/video-stream-subtitle-frame - Authenticated. Renders a PNG subtitle frame for bitmap subtitle tracks via cached PGS decode/composite flow; on failures or no active subtitle it returns 204 so the client can stop retrying.
78- DELETE /api/files/video-stream/:sessionId - Explicitly releases an active short-lived video stream session and stops backend processing.
85- DELETE /api/files/video-stream/:sessionId - Explicitly releases an active short-lived video stream session and stops backend processing.
79- POST /api/files/watched-statuses - Returns fast per-user watched statuses for up to 500 file paths without triggering expensive metadata/hash work; response is { statuses: { [filePath]: 'fully'|'partway' } }.
86- POST /api/files/watched-statuses - Returns fast per-user watched statuses for up to 500 file paths without triggering expensive metadata/hash work; response is { statuses: { [filePath]: 'fully'|'partway' } }.
80- POST /api/files/watch-progress - Records per-user watch status (`partway` >=10%, `fully` >=70%) for sidebar "Watched by" display.
87- POST /api/files/watch-progress - Records per-user watch status (`partway` >=10%, `fully` >=70%) for sidebar "Watched by" display.
@@ -94,7 +101,7 @@
94- ZIP_CACHE_DIR - Absolute path to the zip-cache/ directory (project root).
101- ZIP_CACHE_DIR - Absolute path to the zip-cache/ directory (project root).
95
102
96## server/video-stream.js
103## server/video-stream.js
97- createVideoStreamSession(sourcePath) - Probes input metadata and reuses a compatible active transcode session per source file when possible; returns a per-viewer lease id as sessionId plus duration, subtitleTracks, imageSubtitleTracks, and subtitleSummary data.
104- createVideoStreamSession(sourcePath) - Probes input metadata and reuses a compatible active transcode session per source file when possible; returns a per-viewer lease id as sessionId plus duration, videoWidth, videoHeight, subtitleTracks, imageSubtitleTracks, and subtitleSummary data.
98- sendVideoStreamAsset(sessionId, assetPath, req, res) - Resolves viewer leases to a shared stream session, serves HLS assets, tracks lease activity, and prunes segments using the slowest active viewer position.
105- sendVideoStreamAsset(sessionId, assetPath, req, res) - Resolves viewer leases to a shared stream session, serves HLS assets, tracks lease activity, and prunes segments using the slowest active viewer position.
99- releaseVideoStreamSession(sessionId) - Releases one viewer lease; only destroys the underlying transcode session when no active leases remain.
106- releaseVideoStreamSession(sessionId) - Releases one viewer lease; only destroys the underlying transcode session when no active leases remain.
100- STREAM_ROOT_DIR - Temporary root directory for active ffmpeg-backed stream sessions.
107- STREAM_ROOT_DIR - Temporary root directory for active ffmpeg-backed stream sessions.
@@ -160,7 +167,7 @@
160- setThumbnailPending(iconEl, pending) - Toggles per-icon pending spinner state while a cached image thumbnail is not ready yet.
167- setThumbnailPending(iconEl, pending) - Toggles per-icon pending spinner state while a cached image thumbnail is not ready yet.
161- buildThumbnailSource(item, blob, kind) - Produces thumbnail image source (data/blob URL) for icon injection.
168- buildThumbnailSource(item, blob, kind) - Produces thumbnail image source (data/blob URL) for icon injection.
162- buildAuthHeaders() - Builds the authenticated fetch headers used by preview-side API calls that need the bearer token plus fingerprint header.
169- buildAuthHeaders() - Builds the authenticated fetch headers used by preview-side API calls that need the bearer token plus fingerprint header.
163- bindImageSubtitleOverlay(player, videojs, prefs, session, item) - Adds the fullscreen-safe image-subtitle button, menu, and bottom overlay used to fetch and display subtitle frames for image-based tracks.
170- bindImageSubtitleOverlay(player, videojs, prefs, session, item) - Adds the fullscreen-safe image-subtitle button, menu, and bottom overlay used to fetch and display subtitle frames for image-based tracks; disables the track after the first failed frame request to avoid repeated polling.
164- renderStreamedVideoPreview(item, targetEl) - Creates the Video.js player with HLS source/track config, enforces non-live playback UI for file streams, uses container-fill sizing to keep previews within bounds, restores per-user video prefs (volume + subtitle/audio track names when present), wires the image-subtitle overlay controller, forces caption overrides on each open (text background opacity transparent + text edge style uniform), reports watch progress, warns when subtitle streams exist but none are web-compatible, and releases the backend stream session on close/unload.
171- renderStreamedVideoPreview(item, targetEl) - Creates the Video.js player with HLS source/track config, enforces non-live playback UI for file streams, uses container-fill sizing to keep previews within bounds, restores per-user video prefs (volume + subtitle/audio track names when present), wires the image-subtitle overlay controller, forces caption overrides on each open (text background opacity transparent + text edge style uniform), reports watch progress, warns when subtitle streams exist but none are web-compatible, and releases the backend stream session on close/unload.
165- modelPreviewType(name) - Classifies direct client-side 3D preview support for `.stl` and `.obj`.
172- modelPreviewType(name) - Classifies direct client-side 3D preview support for `.stl` and `.obj`.
166- loadThreePreviewModules() - Lazy-loads Three.js plus OrbitControls/STLLoader/OBJLoader via import map aliases.
173- loadThreePreviewModules() - Lazy-loads Three.js plus OrbitControls/STLLoader/OBJLoader via import map aliases.
.github/tests.md +0 −1
@@ -52,7 +52,6 @@
52 52
53 53 ## `test/page.load.test.js`
54 54 - `page load smoke test`
55 - `expanded file preview keeps the surrounding tiles in place`
56 55
57 56 ---
58 57
@@ -52,7 +52,6 @@
52
52
53## `test/page.load.test.js`
53## `test/page.load.test.js`
54- `page load smoke test`
54- `page load smoke test`
55- `expanded file preview keeps the surrounding tiles in place`
56
55
57---
56---
58
57
package-lock.json +7 −0
@@ -16,6 +16,7 @@
16 16 "jsonwebtoken": "^9.0.2",
17 17 "multer": "^1.4.5-lts.1",
18 18 "node-sqlite3-wasm": "^0.8.10",
19 "pgs-parser": "^0.1.1",
19 20 "sharp": "^0.34.5",
20 21 "three": "^0.166.1",
21 22 "uuid": "^9.0.0",
@@ -6700,6 +6701,12 @@
6700 6701 "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
6701 6702 "license": "MIT"
6702 6703 },
6704 "node_modules/pgs-parser": {
6705 "version": "0.1.1",
6706 "resolved": "https://registry.npmjs.org/pgs-parser/-/pgs-parser-0.1.1.tgz",
6707 "integrity": "sha512-mtoPvVSCOWf1AgZcBUKzUzx6FUPfCXdIc3GOJiR4TyTr7frbjxfL7rYJTjjWNp87ODoUnna9bq8E0v05D+RspQ==",
6708 "license": "MIT"
6709 },
6703 6710 "node_modules/picocolors": {
6704 6711 "version": "1.1.1",
6705 6712 "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
@@ -16,6 +16,7 @@
16 "jsonwebtoken": "^9.0.2",
16 "jsonwebtoken": "^9.0.2",
17 "multer": "^1.4.5-lts.1",
17 "multer": "^1.4.5-lts.1",
18 "node-sqlite3-wasm": "^0.8.10",
18 "node-sqlite3-wasm": "^0.8.10",
19 "pgs-parser": "^0.1.1",
19 "sharp": "^0.34.5",
20 "sharp": "^0.34.5",
20 "three": "^0.166.1",
21 "three": "^0.166.1",
21 "uuid": "^9.0.0",
22 "uuid": "^9.0.0",
@@ -6700,6 +6701,12 @@
6700 "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
6701 "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
6701 "license": "MIT"
6702 "license": "MIT"
6702 },
6703 },
6704 "node_modules/pgs-parser": {
6705 "version": "0.1.1",
6706 "resolved": "https://registry.npmjs.org/pgs-parser/-/pgs-parser-0.1.1.tgz",
6707 "integrity": "sha512-mtoPvVSCOWf1AgZcBUKzUzx6FUPfCXdIc3GOJiR4TyTr7frbjxfL7rYJTjjWNp87ODoUnna9bq8E0v05D+RspQ==",
6708 "license": "MIT"
6709 },
6703 "node_modules/picocolors": {
6710 "node_modules/picocolors": {
6704 "version": "1.1.1",
6711 "version": "1.1.1",
6705 "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
6712 "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
package.json +1 −0
@@ -16,6 +16,7 @@
16 16 "jsonwebtoken": "^9.0.2",
17 17 "multer": "^1.4.5-lts.1",
18 18 "node-sqlite3-wasm": "^0.8.10",
19 "pgs-parser": "^0.1.1",
19 20 "sharp": "^0.34.5",
20 21 "three": "^0.166.1",
21 22 "uuid": "^9.0.0",
@@ -16,6 +16,7 @@
16 "jsonwebtoken": "^9.0.2",
16 "jsonwebtoken": "^9.0.2",
17 "multer": "^1.4.5-lts.1",
17 "multer": "^1.4.5-lts.1",
18 "node-sqlite3-wasm": "^0.8.10",
18 "node-sqlite3-wasm": "^0.8.10",
19 "pgs-parser": "^0.1.1",
19 "sharp": "^0.34.5",
20 "sharp": "^0.34.5",
20 "three": "^0.166.1",
21 "three": "^0.166.1",
21 "uuid": "^9.0.0",
22 "uuid": "^9.0.0",
server/image-subtitles.js added +409 −0
@@ -0,0 +1,409 @@
1 'use strict';
2
3 const { spawn } = require('child_process');
4 const crypto = require('crypto');
5 const fs = require('fs');
6 const os = require('os');
7 const path = require('path');
8 const { Readable } = require('stream');
9 const sharp = require('sharp');
10
11 const SUBTITLE_CACHE_DIR = path.join(os.tmpdir(), 'filemanager-video-subtitles');
12 const EVENT_CACHE_TTL_MS = 20 * 60 * 1000;
13 const FRAME_CACHE_TTL_MS = 10 * 60 * 1000;
14 const MAX_EVENT_CACHE_ITEMS = 6;
15 const MAX_FRAME_CACHE_ITEMS = 400;
16 const PGS_TIMEBASE = 90000;
17
18 const eventCache = new Map();
19 const frameCache = new Map();
20
21 let parseDisplaySets = null;
22
23 function ensureImageDataGlobal ()
24 {
25 if (typeof globalThis.ImageData === 'function') return;
26
27 class ImageDataPolyfill
28 {
29 constructor (data, width, height)
30 {
31 this.data = data;
32 this.width = width;
33 this.height = height;
34 }
35 }
36
37 globalThis.ImageData = ImageDataPolyfill;
38 }
39
40 function loadPgsParser ()
41 {
42 if (parseDisplaySets) return parseDisplaySets;
43 ensureImageDataGlobal();
44 ({ parseDisplaySets } = require('pgs-parser'));
45 return parseDisplaySets;
46 }
47
48 function ffmpegPath ()
49 {
50 return process.env.FFMPEG_PATH || 'ffmpeg';
51 }
52
53 function ensureSubtitleCacheDir ()
54 {
55 fs.mkdirSync(SUBTITLE_CACHE_DIR, { recursive: true });
56 }
57
58 function toPositiveInt (value, fallback)
59 {
60 const number = Number.parseInt(String(value || ''), 10);
61 if (Number.isInteger(number) && number > 0) return number;
62 return fallback;
63 }
64
65 function hashSubtitleKey (filePath, streamIndex, stat)
66 {
67 return crypto
68 .createHash('sha1')
69 .update(`${filePath}|${streamIndex}|${stat.size}|${Math.floor(stat.mtimeMs)}`)
70 .digest('hex');
71 }
72
73 function getCachedValue (cache, key)
74 {
75 const now = Date.now();
76 const entry = cache.get(key);
77 if (!entry) return null;
78
79 if (entry.expiresAt <= now)
80 {
81 cache.delete(key);
82 return null;
83 }
84
85 cache.delete(key);
86 cache.set(key, entry);
87 return entry.value;
88 }
89
90 function setCachedValue (cache, key, value, ttlMs, maxItems)
91 {
92 const now = Date.now();
93 cache.delete(key);
94 cache.set(key, {
95 value,
96 expiresAt: now + ttlMs,
97 });
98
99 while (cache.size > maxItems)
100 {
101 const oldestKey = cache.keys().next().value;
102 cache.delete(oldestKey);
103 }
104 }
105
106 function runCommand (command, args)
107 {
108 return new Promise((resolve, reject) =>
109 {
110 const child = spawn(command, args, { stdio: ['ignore', 'ignore', 'pipe'] });
111 let stderr = '';
112
113 child.stderr.on('data', (chunk) =>
114 {
115 stderr += chunk.toString();
116 if (stderr.length > 16000) stderr = stderr.slice(-16000);
117 });
118
119 child.on('error', reject);
120 child.on('close', (code) =>
121 {
122 if (code === 0)
123 {
124 resolve();
125 return;
126 }
127 reject(new Error(stderr.trim() || `ffmpeg exited with code ${code}`));
128 });
129 });
130 }
131
132 async function extractSupFile (sourcePath, streamIndex, cacheKey)
133 {
134 ensureSubtitleCacheDir();
135 const supPath = path.join(SUBTITLE_CACHE_DIR, `${cacheKey}.sup`);
136
137 if (fs.existsSync(supPath) && fs.statSync(supPath).size > 0)
138 {
139 return supPath;
140 }
141
142 await runCommand(ffmpegPath(), [
143 '-hide_banner',
144 '-loglevel', 'error',
145 '-nostdin',
146 '-y',
147 '-i', sourcePath,
148 '-map', `0:s:${streamIndex}`,
149 '-c:s', 'copy',
150 '-f', 'sup',
151 supPath,
152 ]);
153
154 return supPath;
155 }
156
157 function toSeconds (pgsTimestamp)
158 {
159 const numeric = Number(pgsTimestamp);
160 if (!Number.isFinite(numeric) || numeric < 0) return 0;
161 return numeric / PGS_TIMEBASE;
162 }
163
164 function parseDisplaySetTimestamp (displaySet)
165 {
166 if (displaySet && displaySet.presentationCompositionSegment && displaySet.presentationCompositionSegment.header)
167 {
168 return toSeconds(displaySet.presentationCompositionSegment.header.presentationTimestamp);
169 }
170 if (displaySet && displaySet.endDefinitionSegment && displaySet.endDefinitionSegment.header)
171 {
172 return toSeconds(displaySet.endDefinitionSegment.header.presentationTimestamp);
173 }
174 return 0;
175 }
176
177 async function parseSupEvents (supPath)
178 {
179 const parserFactory = loadPgsParser();
180 if (typeof Readable.toWeb !== 'function')
181 {
182 throw new Error('Node.js stream bridge is unavailable for subtitle parsing.');
183 }
184
185 const subtitleStream = Readable.toWeb(fs.createReadStream(supPath));
186 const parserStream = subtitleStream.pipeThrough(parserFactory());
187 const reader = parserStream.getReader();
188
189 const events = [];
190 let currentEvent = null;
191 let scratchBuffer = null;
192
193 while (true)
194 {
195 const { done, value: displaySet } = await reader.read();
196 if (done) break;
197
198 const timestampSeconds = parseDisplaySetTimestamp(displaySet);
199
200 if (currentEvent && timestampSeconds > currentEvent.startTimeSeconds)
201 {
202 currentEvent.endTimeSeconds = timestampSeconds;
203 events.push(currentEvent);
204 currentEvent = null;
205 }
206
207 if (!displaySet || !Array.isArray(displaySet.objectDefinitionSegments) || displaySet.objectDefinitionSegments.length === 0)
208 {
209 continue;
210 }
211
212 const pcs = displaySet.presentationCompositionSegment;
213 if (!pcs) continue;
214
215 const neededPixels = Number(pcs.width || 0) * Number(pcs.height || 0) * 4;
216 if (!scratchBuffer || !Number.isInteger(neededPixels) || neededPixels <= 0 || scratchBuffer.length < neededPixels)
217 {
218 scratchBuffer = neededPixels > 0 ? new Uint8ClampedArray(neededPixels) : null;
219 }
220
221 let imageData;
222 try
223 {
224 imageData = displaySet.imageData(scratchBuffer || undefined);
225 } catch
226 {
227 continue;
228 }
229
230 if (!imageData || !imageData.data || !Number.isInteger(imageData.width) || !Number.isInteger(imageData.height))
231 {
232 continue;
233 }
234
235 const rawPixels = Buffer.from(imageData.data);
236 const imagePng = await sharp(rawPixels, {
237 raw: {
238 width: imageData.width,
239 height: imageData.height,
240 channels: 4,
241 },
242 }).png().toBuffer();
243
244 currentEvent = {
245 startTimeSeconds: timestampSeconds,
246 endTimeSeconds: Number.POSITIVE_INFINITY,
247 screenWidth: toPositiveInt(pcs.width, imageData.width),
248 screenHeight: toPositiveInt(pcs.height, imageData.height),
249 left: toPositiveInt(pcs.objectHorizontalPosition, 0),
250 top: toPositiveInt(pcs.objectVerticalPosition, 0),
251 imageWidth: imageData.width,
252 imageHeight: imageData.height,
253 imagePng,
254 };
255 }
256
257 if (currentEvent)
258 {
259 events.push(currentEvent);
260 }
261
262 return events;
263 }
264
265 async function buildEventCacheEntry (filePath, streamIndex)
266 {
267 const stat = await fs.promises.stat(filePath);
268 const cacheKey = hashSubtitleKey(filePath, streamIndex, stat);
269 const supPath = await extractSupFile(filePath, streamIndex, cacheKey);
270 const events = await parseSupEvents(supPath);
271 return {
272 cacheKey,
273 events,
274 };
275 }
276
277 async function getEventCacheEntry (filePath, streamIndex)
278 {
279 const stat = await fs.promises.stat(filePath);
280 const identityKey = hashSubtitleKey(filePath, streamIndex, stat);
281
282 const cached = getCachedValue(eventCache, identityKey);
283 if (cached)
284 {
285 return cached;
286 }
287
288 const pending = buildEventCacheEntry(filePath, streamIndex);
289 setCachedValue(eventCache, identityKey, pending, EVENT_CACHE_TTL_MS, MAX_EVENT_CACHE_ITEMS);
290
291 try
292 {
293 const resolved = await pending;
294 setCachedValue(eventCache, identityKey, resolved, EVENT_CACHE_TTL_MS, MAX_EVENT_CACHE_ITEMS);
295 return resolved;
296 } catch (error)
297 {
298 eventCache.delete(identityKey);
299 throw error;
300 }
301 }
302
303 function findActiveEventIndex (events, timeSeconds)
304 {
305 let left = 0;
306 let right = events.length - 1;
307 let candidate = -1;
308
309 while (left <= right)
310 {
311 const middle = Math.floor((left + right) / 2);
312 const event = events[middle];
313
314 if (event.startTimeSeconds <= timeSeconds)
315 {
316 candidate = middle;
317 left = middle + 1;
318 } else
319 {
320 right = middle - 1;
321 }
322 }
323
324 if (candidate < 0) return -1;
325
326 const active = events[candidate];
327 if (timeSeconds >= active.endTimeSeconds) return -1;
328 return candidate;
329 }
330
331 async function renderEventFrame (event, canvasWidth, canvasHeight)
332 {
333 const sourceWidth = Math.max(1, toPositiveInt(event.screenWidth, canvasWidth));
334 const sourceHeight = Math.max(1, toPositiveInt(event.screenHeight, canvasHeight));
335
336 const scaleX = canvasWidth / sourceWidth;
337 const scaleY = canvasHeight / sourceHeight;
338
339 const targetWidth = Math.max(1, Math.round(event.imageWidth * scaleX));
340 const targetHeight = Math.max(1, Math.round(event.imageHeight * scaleY));
341 const left = Math.max(0, Math.round(event.left * scaleX));
342 const top = Math.max(0, Math.round(event.top * scaleY));
343
344 if (left >= canvasWidth || top >= canvasHeight)
345 {
346 return sharp({
347 create: {
348 width: canvasWidth,
349 height: canvasHeight,
350 channels: 4,
351 background: { r: 0, g: 0, b: 0, alpha: 0 },
352 },
353 }).png().toBuffer();
354 }
355
356 const imageBuffer = (targetWidth === event.imageWidth && targetHeight === event.imageHeight)
357 ? event.imagePng
358 : await sharp(event.imagePng).resize({ width: targetWidth, height: targetHeight, fit: 'fill' }).png().toBuffer();
359
360 return sharp({
361 create: {
362 width: canvasWidth,
363 height: canvasHeight,
364 channels: 4,
365 background: { r: 0, g: 0, b: 0, alpha: 0 },
366 },
367 }).composite([
368 {
369 input: imageBuffer,
370 left,
371 top,
372 },
373 ]).png().toBuffer();
374 }
375
376 async function renderImageSubtitleFrame (options)
377 {
378 const filePath = String(options.filePath || '').trim();
379 const streamIndex = Number.parseInt(String(options.streamIndex || ''), 10);
380 const timeSeconds = Number.parseFloat(String(options.timeSeconds || ''), 10);
381 const canvasWidth = Math.max(1, Math.min(8192, toPositiveInt(options.canvasWidth, 1280)));
382 const canvasHeight = Math.max(1, Math.min(8192, toPositiveInt(options.canvasHeight, 720)));
383
384 if (!filePath || !Number.isInteger(streamIndex) || streamIndex < 0 || !Number.isFinite(timeSeconds) || timeSeconds < 0)
385 {
386 return null;
387 }
388
389 const entry = await getEventCacheEntry(filePath, streamIndex);
390 if (!entry || !Array.isArray(entry.events) || entry.events.length === 0) return null;
391
392 const eventIndex = findActiveEventIndex(entry.events, timeSeconds);
393 if (eventIndex < 0) return null;
394
395 const frameKey = `${entry.cacheKey}|${eventIndex}|${canvasWidth}x${canvasHeight}`;
396 const cachedFrame = getCachedValue(frameCache, frameKey);
397 if (cachedFrame)
398 {
399 return cachedFrame;
400 }
401
402 const frame = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);
403 setCachedValue(frameCache, frameKey, frame, FRAME_CACHE_TTL_MS, MAX_FRAME_CACHE_ITEMS);
404 return frame;
405 }
406
407 module.exports = {
408 renderImageSubtitleFrame,
409 };
@@ -0,0 +1,409 @@
1'use strict';
2
3const { spawn } = require('child_process');
4const crypto = require('crypto');
5const fs = require('fs');
6const os = require('os');
7const path = require('path');
8const { Readable } = require('stream');
9const sharp = require('sharp');
10
11const SUBTITLE_CACHE_DIR = path.join(os.tmpdir(), 'filemanager-video-subtitles');
12const EVENT_CACHE_TTL_MS = 20 * 60 * 1000;
13const FRAME_CACHE_TTL_MS = 10 * 60 * 1000;
14const MAX_EVENT_CACHE_ITEMS = 6;
15const MAX_FRAME_CACHE_ITEMS = 400;
16const PGS_TIMEBASE = 90000;
17
18const eventCache = new Map();
19const frameCache = new Map();
20
21let parseDisplaySets = null;
22
23function ensureImageDataGlobal ()
24{
25 if (typeof globalThis.ImageData === 'function') return;
26
27 class ImageDataPolyfill
28 {
29 constructor (data, width, height)
30 {
31 this.data = data;
32 this.width = width;
33 this.height = height;
34 }
35 }
36
37 globalThis.ImageData = ImageDataPolyfill;
38}
39
40function loadPgsParser ()
41{
42 if (parseDisplaySets) return parseDisplaySets;
43 ensureImageDataGlobal();
44 ({ parseDisplaySets } = require('pgs-parser'));
45 return parseDisplaySets;
46}
47
48function ffmpegPath ()
49{
50 return process.env.FFMPEG_PATH || 'ffmpeg';
51}
52
53function ensureSubtitleCacheDir ()
54{
55 fs.mkdirSync(SUBTITLE_CACHE_DIR, { recursive: true });
56}
57
58function toPositiveInt (value, fallback)
59{
60 const number = Number.parseInt(String(value || ''), 10);
61 if (Number.isInteger(number) && number > 0) return number;
62 return fallback;
63}
64
65function hashSubtitleKey (filePath, streamIndex, stat)
66{
67 return crypto
68 .createHash('sha1')
69 .update(`${filePath}|${streamIndex}|${stat.size}|${Math.floor(stat.mtimeMs)}`)
70 .digest('hex');
71}
72
73function getCachedValue (cache, key)
74{
75 const now = Date.now();
76 const entry = cache.get(key);
77 if (!entry) return null;
78
79 if (entry.expiresAt <= now)
80 {
81 cache.delete(key);
82 return null;
83 }
84
85 cache.delete(key);
86 cache.set(key, entry);
87 return entry.value;
88}
89
90function setCachedValue (cache, key, value, ttlMs, maxItems)
91{
92 const now = Date.now();
93 cache.delete(key);
94 cache.set(key, {
95 value,
96 expiresAt: now + ttlMs,
97 });
98
99 while (cache.size > maxItems)
100 {
101 const oldestKey = cache.keys().next().value;
102 cache.delete(oldestKey);
103 }
104}
105
106function runCommand (command, args)
107{
108 return new Promise((resolve, reject) =>
109 {
110 const child = spawn(command, args, { stdio: ['ignore', 'ignore', 'pipe'] });
111 let stderr = '';
112
113 child.stderr.on('data', (chunk) =>
114 {
115 stderr += chunk.toString();
116 if (stderr.length > 16000) stderr = stderr.slice(-16000);
117 });
118
119 child.on('error', reject);
120 child.on('close', (code) =>
121 {
122 if (code === 0)
123 {
124 resolve();
125 return;
126 }
127 reject(new Error(stderr.trim() || `ffmpeg exited with code ${code}`));
128 });
129 });
130}
131
132async function extractSupFile (sourcePath, streamIndex, cacheKey)
133{
134 ensureSubtitleCacheDir();
135 const supPath = path.join(SUBTITLE_CACHE_DIR, `${cacheKey}.sup`);
136
137 if (fs.existsSync(supPath) && fs.statSync(supPath).size > 0)
138 {
139 return supPath;
140 }
141
142 await runCommand(ffmpegPath(), [
143 '-hide_banner',
144 '-loglevel', 'error',
145 '-nostdin',
146 '-y',
147 '-i', sourcePath,
148 '-map', `0:s:${streamIndex}`,
149 '-c:s', 'copy',
150 '-f', 'sup',
151 supPath,
152 ]);
153
154 return supPath;
155}
156
157function toSeconds (pgsTimestamp)
158{
159 const numeric = Number(pgsTimestamp);
160 if (!Number.isFinite(numeric) || numeric < 0) return 0;
161 return numeric / PGS_TIMEBASE;
162}
163
164function parseDisplaySetTimestamp (displaySet)
165{
166 if (displaySet && displaySet.presentationCompositionSegment && displaySet.presentationCompositionSegment.header)
167 {
168 return toSeconds(displaySet.presentationCompositionSegment.header.presentationTimestamp);
169 }
170 if (displaySet && displaySet.endDefinitionSegment && displaySet.endDefinitionSegment.header)
171 {
172 return toSeconds(displaySet.endDefinitionSegment.header.presentationTimestamp);
173 }
174 return 0;
175}
176
177async function parseSupEvents (supPath)
178{
179 const parserFactory = loadPgsParser();
180 if (typeof Readable.toWeb !== 'function')
181 {
182 throw new Error('Node.js stream bridge is unavailable for subtitle parsing.');
183 }
184
185 const subtitleStream = Readable.toWeb(fs.createReadStream(supPath));
186 const parserStream = subtitleStream.pipeThrough(parserFactory());
187 const reader = parserStream.getReader();
188
189 const events = [];
190 let currentEvent = null;
191 let scratchBuffer = null;
192
193 while (true)
194 {
195 const { done, value: displaySet } = await reader.read();
196 if (done) break;
197
198 const timestampSeconds = parseDisplaySetTimestamp(displaySet);
199
200 if (currentEvent && timestampSeconds > currentEvent.startTimeSeconds)
201 {
202 currentEvent.endTimeSeconds = timestampSeconds;
203 events.push(currentEvent);
204 currentEvent = null;
205 }
206
207 if (!displaySet || !Array.isArray(displaySet.objectDefinitionSegments) || displaySet.objectDefinitionSegments.length === 0)
208 {
209 continue;
210 }
211
212 const pcs = displaySet.presentationCompositionSegment;
213 if (!pcs) continue;
214
215 const neededPixels = Number(pcs.width || 0) * Number(pcs.height || 0) * 4;
216 if (!scratchBuffer || !Number.isInteger(neededPixels) || neededPixels <= 0 || scratchBuffer.length < neededPixels)
217 {
218 scratchBuffer = neededPixels > 0 ? new Uint8ClampedArray(neededPixels) : null;
219 }
220
221 let imageData;
222 try
223 {
224 imageData = displaySet.imageData(scratchBuffer || undefined);
225 } catch
226 {
227 continue;
228 }
229
230 if (!imageData || !imageData.data || !Number.isInteger(imageData.width) || !Number.isInteger(imageData.height))
231 {
232 continue;
233 }
234
235 const rawPixels = Buffer.from(imageData.data);
236 const imagePng = await sharp(rawPixels, {
237 raw: {
238 width: imageData.width,
239 height: imageData.height,
240 channels: 4,
241 },
242 }).png().toBuffer();
243
244 currentEvent = {
245 startTimeSeconds: timestampSeconds,
246 endTimeSeconds: Number.POSITIVE_INFINITY,
247 screenWidth: toPositiveInt(pcs.width, imageData.width),
248 screenHeight: toPositiveInt(pcs.height, imageData.height),
249 left: toPositiveInt(pcs.objectHorizontalPosition, 0),
250 top: toPositiveInt(pcs.objectVerticalPosition, 0),
251 imageWidth: imageData.width,
252 imageHeight: imageData.height,
253 imagePng,
254 };
255 }
256
257 if (currentEvent)
258 {
259 events.push(currentEvent);
260 }
261
262 return events;
263}
264
265async function buildEventCacheEntry (filePath, streamIndex)
266{
267 const stat = await fs.promises.stat(filePath);
268 const cacheKey = hashSubtitleKey(filePath, streamIndex, stat);
269 const supPath = await extractSupFile(filePath, streamIndex, cacheKey);
270 const events = await parseSupEvents(supPath);
271 return {
272 cacheKey,
273 events,
274 };
275}
276
277async function getEventCacheEntry (filePath, streamIndex)
278{
279 const stat = await fs.promises.stat(filePath);
280 const identityKey = hashSubtitleKey(filePath, streamIndex, stat);
281
282 const cached = getCachedValue(eventCache, identityKey);
283 if (cached)
284 {
285 return cached;
286 }
287
288 const pending = buildEventCacheEntry(filePath, streamIndex);
289 setCachedValue(eventCache, identityKey, pending, EVENT_CACHE_TTL_MS, MAX_EVENT_CACHE_ITEMS);
290
291 try
292 {
293 const resolved = await pending;
294 setCachedValue(eventCache, identityKey, resolved, EVENT_CACHE_TTL_MS, MAX_EVENT_CACHE_ITEMS);
295 return resolved;
296 } catch (error)
297 {
298 eventCache.delete(identityKey);
299 throw error;
300 }
301}
302
303function findActiveEventIndex (events, timeSeconds)
304{
305 let left = 0;
306 let right = events.length - 1;
307 let candidate = -1;
308
309 while (left <= right)
310 {
311 const middle = Math.floor((left + right) / 2);
312 const event = events[middle];
313
314 if (event.startTimeSeconds <= timeSeconds)
315 {
316 candidate = middle;
317 left = middle + 1;
318 } else
319 {
320 right = middle - 1;
321 }
322 }
323
324 if (candidate < 0) return -1;
325
326 const active = events[candidate];
327 if (timeSeconds >= active.endTimeSeconds) return -1;
328 return candidate;
329}
330
331async function renderEventFrame (event, canvasWidth, canvasHeight)
332{
333 const sourceWidth = Math.max(1, toPositiveInt(event.screenWidth, canvasWidth));
334 const sourceHeight = Math.max(1, toPositiveInt(event.screenHeight, canvasHeight));
335
336 const scaleX = canvasWidth / sourceWidth;
337 const scaleY = canvasHeight / sourceHeight;
338
339 const targetWidth = Math.max(1, Math.round(event.imageWidth * scaleX));
340 const targetHeight = Math.max(1, Math.round(event.imageHeight * scaleY));
341 const left = Math.max(0, Math.round(event.left * scaleX));
342 const top = Math.max(0, Math.round(event.top * scaleY));
343
344 if (left >= canvasWidth || top >= canvasHeight)
345 {
346 return sharp({
347 create: {
348 width: canvasWidth,
349 height: canvasHeight,
350 channels: 4,
351 background: { r: 0, g: 0, b: 0, alpha: 0 },
352 },
353 }).png().toBuffer();
354 }
355
356 const imageBuffer = (targetWidth === event.imageWidth && targetHeight === event.imageHeight)
357 ? event.imagePng
358 : await sharp(event.imagePng).resize({ width: targetWidth, height: targetHeight, fit: 'fill' }).png().toBuffer();
359
360 return sharp({
361 create: {
362 width: canvasWidth,
363 height: canvasHeight,
364 channels: 4,
365 background: { r: 0, g: 0, b: 0, alpha: 0 },
366 },
367 }).composite([
368 {
369 input: imageBuffer,
370 left,
371 top,
372 },
373 ]).png().toBuffer();
374}
375
376async function renderImageSubtitleFrame (options)
377{
378 const filePath = String(options.filePath || '').trim();
379 const streamIndex = Number.parseInt(String(options.streamIndex || ''), 10);
380 const timeSeconds = Number.parseFloat(String(options.timeSeconds || ''), 10);
381 const canvasWidth = Math.max(1, Math.min(8192, toPositiveInt(options.canvasWidth, 1280)));
382 const canvasHeight = Math.max(1, Math.min(8192, toPositiveInt(options.canvasHeight, 720)));
383
384 if (!filePath || !Number.isInteger(streamIndex) || streamIndex < 0 || !Number.isFinite(timeSeconds) || timeSeconds < 0)
385 {
386 return null;
387 }
388
389 const entry = await getEventCacheEntry(filePath, streamIndex);
390 if (!entry || !Array.isArray(entry.events) || entry.events.length === 0) return null;
391
392 const eventIndex = findActiveEventIndex(entry.events, timeSeconds);
393 if (eventIndex < 0) return null;
394
395 const frameKey = `${entry.cacheKey}|${eventIndex}|${canvasWidth}x${canvasHeight}`;
396 const cachedFrame = getCachedValue(frameCache, frameKey);
397 if (cachedFrame)
398 {
399 return cachedFrame;
400 }
401
402 const frame = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);
403 setCachedValue(frameCache, frameKey, frame, FRAME_CACHE_TTL_MS, MAX_FRAME_CACHE_ITEMS);
404 return frame;
405}
406
407module.exports = {
408 renderImageSubtitleFrame,
409};
server/routes/files.js +27 −49
@@ -11,6 +11,8 @@ const { getDb } = require('../db');
11 11 const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');
12 12 const { requireAuth } = require('../auth');
13 13 const { getCachedThumbnailInfo, getFilesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
14 const { logError } = require('../logger');
15 const { renderImageSubtitleFrame } = require('../image-subtitles');
14 16 const { buildZip } = require('./share');
15 17 const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
16 18
@@ -1075,6 +1077,8 @@ router.post('/video-stream', requireAuth, async (req, res, next) =>
1075 1077 sessionId: session.sessionId,
1076 1078 playlistUrl: `/api/files/video-stream/${session.sessionId}/master.m3u8`,
1077 1079 durationSeconds: session.durationSeconds || null,
1080 videoWidth: session.videoWidth || null,
1081 videoHeight: session.videoHeight || null,
1078 1082 subtitleSummary: session.subtitleSummary || { total: 0, converted: 0, imageBased: 0, skipped: 0 },
1079 1083 subtitleTracks: session.subtitleTracks.map((track) => ({
1080 1084 label: track.label,
@@ -1125,7 +1129,7 @@ router.delete('/video-stream/:sessionId', requireAuth, (req, res) =>
1125 1129
1126 1130 // ---------------------------------------------------------------------------
1127 1131 // GET /api/files/video-stream-subtitle-frame
1128 // Authenticated. Extracts a subtitle frame (image) at a given timestamp.
1132 // Authenticated. Returns a composited PNG subtitle frame at a given timestamp.
1129 1133 // Query params: filePath, streamIndex, timeSeconds
1130 1134 // ---------------------------------------------------------------------------
1131 1135 router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =>
@@ -1135,6 +1139,8 @@ router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =
1135 1139 const filePath = String(req.query.filePath || '').trim();
1136 1140 const streamIndex = Number.parseInt(String(req.query.streamIndex || ''), 10);
1137 1141 const timeSeconds = Number.parseFloat(String(req.query.timeSeconds || ''), 10);
1142 const videoWidth = Number.parseInt(String(req.query.videoWidth || ''), 10);
1143 const videoHeight = Number.parseInt(String(req.query.videoHeight || ''), 10);
1138 1144
1139 1145 if (!filePath || !Number.isInteger(streamIndex) || streamIndex < 0 || !Number.isFinite(timeSeconds) || timeSeconds < 0)
1140 1146 {
@@ -1153,60 +1159,32 @@ router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =
1153 1159 return res.status(404).json({ error: 'File not found.' });
1154 1160 }
1155 1161
1156 // Extract frame at the given timestamp using ffmpeg
1157 const { spawn } = require('child_process');
1158 const ffmpegPath = process.env.FFMPEG_PATH || 'ffmpeg';
1159 const timeStr = `${Math.floor(timeSeconds / 3600)}:${Math.floor((timeSeconds % 3600) / 60)}:${(timeSeconds % 60).toFixed(3)}`;
1160
1161 const child = spawn(ffmpegPath, [
1162 '-hide_banner',
1163 '-loglevel', 'error',
1164 '-ss', timeStr,
1165 '-i', filePath,
1166 '-map', `0:s:${streamIndex}`,
1167 '-frames:v', '1',
1168 '-c:v', 'png',
1169 '-f', 'image2pipe',
1170 'pipe:1',
1171 ]);
1172
1173 let frameData = Buffer.alloc(0);
1174 let stderr = '';
1175
1176 child.stdout.on('data', (chunk) =>
1177 {
1178 frameData = Buffer.concat([frameData, chunk]);
1179 });
1180
1181 child.stderr.on('data', (chunk) =>
1182 {
1183 stderr += chunk.toString();
1162 const canvasWidth = Number.isInteger(videoWidth) && videoWidth > 0 ? videoWidth : 1280;
1163 const canvasHeight = Number.isInteger(videoHeight) && videoHeight > 0 ? videoHeight : 720;
1164 const frameData = await renderImageSubtitleFrame({
1165 filePath,
1166 streamIndex,
1167 timeSeconds,
1168 canvasWidth,
1169 canvasHeight,
1184 1170 });
1185 1171
1186 child.on('close', (code) =>
1172 if (!frameData || frameData.length === 0)
1187 1173 {
1188 if (code !== 0)
1189 {
1190 return res.status(500).json({ error: 'Failed to extract subtitle frame.' });
1191 }
1192
1193 if (frameData.length === 0)
1194 {
1195 return res.status(404).json({ error: 'No subtitle frame at this timestamp.' });
1196 }
1197
1198 res.set('Content-Type', 'image/png');
1199 res.set('Cache-Control', 'private, max-age=3600');
1200 res.send(frameData);
1201 });
1174 return res.status(204).end();
1175 }
1202 1176
1203 child.on('error', (err) =>
1204 {
1205 return res.status(503).json({ error: 'ffmpeg unavailable.' });
1206 });
1177 res.set('Content-Type', 'image/png');
1178 res.set('Cache-Control', 'private, max-age=3600');
1179 res.send(frameData);
1207 1180 } catch (err)
1208 1181 {
1209 next(err);
1182 logError('Subtitle frame extraction failed', err, {
1183 filePath: String(req.query.filePath || ''),
1184 streamIndex: Number.parseInt(String(req.query.streamIndex || ''), 10),
1185 timeSeconds: Number.parseFloat(String(req.query.timeSeconds || ''), 10),
1186 });
1187 return res.status(204).end();
1210 1188 }
1211 1189 });
1212 1190
@@ -11,6 +11,8 @@ const { getDb } = require('../db');
11const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');
11const { loadConfig, saveConfig, getEffectivePaths, getAllowedRoot } = require('../config');
12const { requireAuth } = require('../auth');
12const { requireAuth } = require('../auth');
13const { getCachedThumbnailInfo, getFilesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
13const { getCachedThumbnailInfo, getFilesWithThumbnailsInDir, getFolderIconSourceFile, setFolderIconAssignment } = require('../thumbnail-cache');
14const { logError } = require('../logger');
15const { renderImageSubtitleFrame } = require('../image-subtitles');
14const { buildZip } = require('./share');
16const { buildZip } = require('./share');
15const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
17const { createVideoStreamSession, sendVideoStreamAsset, releaseVideoStreamSession } = require('../video-stream');
16
18
@@ -1075,6 +1077,8 @@ router.post('/video-stream', requireAuth, async (req, res, next) =>
1075 sessionId: session.sessionId,
1077 sessionId: session.sessionId,
1076 playlistUrl: `/api/files/video-stream/${session.sessionId}/master.m3u8`,
1078 playlistUrl: `/api/files/video-stream/${session.sessionId}/master.m3u8`,
1077 durationSeconds: session.durationSeconds || null,
1079 durationSeconds: session.durationSeconds || null,
1080 videoWidth: session.videoWidth || null,
1081 videoHeight: session.videoHeight || null,
1078 subtitleSummary: session.subtitleSummary || { total: 0, converted: 0, imageBased: 0, skipped: 0 },
1082 subtitleSummary: session.subtitleSummary || { total: 0, converted: 0, imageBased: 0, skipped: 0 },
1079 subtitleTracks: session.subtitleTracks.map((track) => ({
1083 subtitleTracks: session.subtitleTracks.map((track) => ({
1080 label: track.label,
1084 label: track.label,
@@ -1125,7 +1129,7 @@ router.delete('/video-stream/:sessionId', requireAuth, (req, res) =>
1125
1129
1126// ---------------------------------------------------------------------------
1130// ---------------------------------------------------------------------------
1127// GET /api/files/video-stream-subtitle-frame
1131// GET /api/files/video-stream-subtitle-frame
1128// Authenticated. Extracts a subtitle frame (image) at a given timestamp.
1132// Authenticated. Returns a composited PNG subtitle frame at a given timestamp.
1129// Query params: filePath, streamIndex, timeSeconds
1133// Query params: filePath, streamIndex, timeSeconds
1130// ---------------------------------------------------------------------------
1134// ---------------------------------------------------------------------------
1131router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =>
1135router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =>
@@ -1135,6 +1139,8 @@ router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =
1135 const filePath = String(req.query.filePath || '').trim();
1139 const filePath = String(req.query.filePath || '').trim();
1136 const streamIndex = Number.parseInt(String(req.query.streamIndex || ''), 10);
1140 const streamIndex = Number.parseInt(String(req.query.streamIndex || ''), 10);
1137 const timeSeconds = Number.parseFloat(String(req.query.timeSeconds || ''), 10);
1141 const timeSeconds = Number.parseFloat(String(req.query.timeSeconds || ''), 10);
1142 const videoWidth = Number.parseInt(String(req.query.videoWidth || ''), 10);
1143 const videoHeight = Number.parseInt(String(req.query.videoHeight || ''), 10);
1138
1144
1139 if (!filePath || !Number.isInteger(streamIndex) || streamIndex < 0 || !Number.isFinite(timeSeconds) || timeSeconds < 0)
1145 if (!filePath || !Number.isInteger(streamIndex) || streamIndex < 0 || !Number.isFinite(timeSeconds) || timeSeconds < 0)
1140 {
1146 {
@@ -1153,60 +1159,32 @@ router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =
1153 return res.status(404).json({ error: 'File not found.' });
1159 return res.status(404).json({ error: 'File not found.' });
1154 }
1160 }
1155
1161
1156 // Extract frame at the given timestamp using ffmpeg
1162 const canvasWidth = Number.isInteger(videoWidth) && videoWidth > 0 ? videoWidth : 1280;
1157 const { spawn } = require('child_process');
1163 const canvasHeight = Number.isInteger(videoHeight) && videoHeight > 0 ? videoHeight : 720;
1158 const ffmpegPath = process.env.FFMPEG_PATH || 'ffmpeg';
1164 const frameData = await renderImageSubtitleFrame({
1159 const timeStr = `${Math.floor(timeSeconds / 3600)}:${Math.floor((timeSeconds % 3600) / 60)}:${(timeSeconds % 60).toFixed(3)}`;
1165 filePath,
1160
1166 streamIndex,
1161 const child = spawn(ffmpegPath, [
1167 timeSeconds,
1162 '-hide_banner',
1168 canvasWidth,
1163 '-loglevel', 'error',
1169 canvasHeight,
1164 '-ss', timeStr,
1165 '-i', filePath,
1166 '-map', `0:s:${streamIndex}`,
1167 '-frames:v', '1',
1168 '-c:v', 'png',
1169 '-f', 'image2pipe',
1170 'pipe:1',
1171 ]);
1172
1173 let frameData = Buffer.alloc(0);
1174 let stderr = '';
1175
1176 child.stdout.on('data', (chunk) =>
1177 {
1178 frameData = Buffer.concat([frameData, chunk]);
1179 });
1180
1181 child.stderr.on('data', (chunk) =>
1182 {
1183 stderr += chunk.toString();
1184 });
1170 });
1185
1171
1186 child.on('close', (code) =>
1172 if (!frameData || frameData.length === 0)
1187 {
1173 {
1188 if (code !== 0)
1174 return res.status(204).end();
1189 {
1175 }
1190 return res.status(500).json({ error: 'Failed to extract subtitle frame.' });
1191 }
1192
1193 if (frameData.length === 0)
1194 {
1195 return res.status(404).json({ error: 'No subtitle frame at this timestamp.' });
1196 }
1197
1198 res.set('Content-Type', 'image/png');
1199 res.set('Cache-Control', 'private, max-age=3600');
1200 res.send(frameData);
1201 });
1202
1176
1203 child.on('error', (err) =>
1177 res.set('Content-Type', 'image/png');
1204 {
1178 res.set('Cache-Control', 'private, max-age=3600');
1205 return res.status(503).json({ error: 'ffmpeg unavailable.' });
1179 res.send(frameData);
1206 });
1207 } catch (err)
1180 } catch (err)
1208 {
1181 {
1209 next(err);
1182 logError('Subtitle frame extraction failed', err, {
1183 filePath: String(req.query.filePath || ''),
1184 streamIndex: Number.parseInt(String(req.query.streamIndex || ''), 10),
1185 timeSeconds: Number.parseFloat(String(req.query.timeSeconds || ''), 10),
1186 });
1187 return res.status(204).end();
1210 }
1188 }
1211});
1189});
1212
1190
tests/files.routes.test.js +47 −0
@@ -18,6 +18,8 @@ let lastFolderIconAssignment = undefined;
18 18 let mockVideoStreamSession = null;
19 19 let mockVideoStreamAsset = null;
20 20 let releasedVideoStreamSessionId = null;
21 let mockSubtitleFrameResult = null;
22 let mockSubtitleFrameError = null;
21 23
22 24 jest.mock('../server/db', () =>
23 25 {
@@ -111,6 +113,14 @@ jest.mock('../server/video-stream', () => ({
111 113 }),
112 114 }));
113 115
116 jest.mock('../server/image-subtitles', () => ({
117 renderImageSubtitleFrame: jest.fn(async () =>
118 {
119 if (mockSubtitleFrameError) throw mockSubtitleFrameError;
120 return mockSubtitleFrameResult;
121 }),
122 }));
123
114 124 const filesRoutes = require('../server/routes/files');
115 125 const { getDb } = require('../server/db');
116 126
@@ -162,6 +172,8 @@ beforeEach(() =>
162 172 mockVideoStreamSession = null;
163 173 mockVideoStreamAsset = null;
164 174 releasedVideoStreamSessionId = null;
175 mockSubtitleFrameResult = null;
176 mockSubtitleFrameError = null;
165 177 const db = getDb();
166 178 db.prepare('DELETE FROM file_meta').run([]);
167 179 db.prepare('DELETE FROM users').run([]);
@@ -768,6 +780,8 @@ describe('video streaming routes', () =>
768 780 mockVideoStreamSession = {
769 781 sessionId: 'sess-123',
770 782 durationSeconds: 42.25,
783 videoWidth: 1920,
784 videoHeight: 1080,
771 785 subtitleTracks: [{ fileName: 'subtitle-0.vtt', label: 'English', language: 'eng', isDefault: true }],
772 786 };
773 787
@@ -776,6 +790,8 @@ describe('video streaming routes', () =>
776 790 expect(status).toBe(200);
777 791 expect(body.playlistUrl).toBe('/api/files/video-stream/sess-123/master.m3u8');
778 792 expect(body.durationSeconds).toBe(42.25);
793 expect(body.videoWidth).toBe(1920);
794 expect(body.videoHeight).toBe(1080);
779 795 expect(body.subtitleSummary).toEqual({ total: 0, converted: 0, imageBased: 0, skipped: 0 });
780 796 expect(body.subtitleTracks).toEqual([
781 797 {
@@ -810,6 +826,37 @@ describe('video streaming routes', () =>
810 826 expect(releasedVideoStreamSessionId).toBe('sess-123');
811 827 });
812 828
829 test('GET /api/files/video-stream-subtitle-frame returns rendered PNG frames', async () =>
830 {
831 const filePath = path.join(rootDir, 'clip.mkv');
832 fs.writeFileSync(filePath, 'video');
833 mockSubtitleFrameResult = Buffer.from('png-data');
834
835 const { status, headers, body } = await reqRaw(
836 'GET',
837 `/api/files/video-stream-subtitle-frame?filePath=${encodeURIComponent(filePath)}&streamIndex=0&timeSeconds=2.5&videoWidth=1920&videoHeight=1080`
838 );
839
840 expect(status).toBe(200);
841 expect(headers['content-type']).toMatch(/image\/png/);
842 expect(body.equals(Buffer.from('png-data'))).toBe(true);
843 });
844
845 test('GET /api/files/video-stream-subtitle-frame returns 204 when rendering fails', async () =>
846 {
847 const filePath = path.join(rootDir, 'clip.mkv');
848 fs.writeFileSync(filePath, 'video');
849 mockSubtitleFrameError = new Error('decode failed');
850
851 const { status, body } = await reqRaw(
852 'GET',
853 `/api/files/video-stream-subtitle-frame?filePath=${encodeURIComponent(filePath)}&streamIndex=0&timeSeconds=2.5`
854 );
855
856 expect(status).toBe(204);
857 expect(body.length).toBe(0);
858 });
859
813 860 test('POST /api/files/watch-progress stores per-user watched status by thresholds', async () =>
814 861 {
815 862 const filePath = path.join(rootDir, 'clip.mkv');
@@ -18,6 +18,8 @@ let lastFolderIconAssignment = undefined;
18let mockVideoStreamSession = null;
18let mockVideoStreamSession = null;
19let mockVideoStreamAsset = null;
19let mockVideoStreamAsset = null;
20let releasedVideoStreamSessionId = null;
20let releasedVideoStreamSessionId = null;
21let mockSubtitleFrameResult = null;
22let mockSubtitleFrameError = null;
21
23
22jest.mock('../server/db', () =>
24jest.mock('../server/db', () =>
23{
25{
@@ -111,6 +113,14 @@ jest.mock('../server/video-stream', () => ({
111 }),
113 }),
112}));
114}));
113
115
116jest.mock('../server/image-subtitles', () => ({
117 renderImageSubtitleFrame: jest.fn(async () =>
118 {
119 if (mockSubtitleFrameError) throw mockSubtitleFrameError;
120 return mockSubtitleFrameResult;
121 }),
122}));
123
114const filesRoutes = require('../server/routes/files');
124const filesRoutes = require('../server/routes/files');
115const { getDb } = require('../server/db');
125const { getDb } = require('../server/db');
116
126
@@ -162,6 +172,8 @@ beforeEach(() =>
162 mockVideoStreamSession = null;
172 mockVideoStreamSession = null;
163 mockVideoStreamAsset = null;
173 mockVideoStreamAsset = null;
164 releasedVideoStreamSessionId = null;
174 releasedVideoStreamSessionId = null;
175 mockSubtitleFrameResult = null;
176 mockSubtitleFrameError = null;
165 const db = getDb();
177 const db = getDb();
166 db.prepare('DELETE FROM file_meta').run([]);
178 db.prepare('DELETE FROM file_meta').run([]);
167 db.prepare('DELETE FROM users').run([]);
179 db.prepare('DELETE FROM users').run([]);
@@ -768,6 +780,8 @@ describe('video streaming routes', () =>
768 mockVideoStreamSession = {
780 mockVideoStreamSession = {
769 sessionId: 'sess-123',
781 sessionId: 'sess-123',
770 durationSeconds: 42.25,
782 durationSeconds: 42.25,
783 videoWidth: 1920,
784 videoHeight: 1080,
771 subtitleTracks: [{ fileName: 'subtitle-0.vtt', label: 'English', language: 'eng', isDefault: true }],
785 subtitleTracks: [{ fileName: 'subtitle-0.vtt', label: 'English', language: 'eng', isDefault: true }],
772 };
786 };
773
787
@@ -776,6 +790,8 @@ describe('video streaming routes', () =>
776 expect(status).toBe(200);
790 expect(status).toBe(200);
777 expect(body.playlistUrl).toBe('/api/files/video-stream/sess-123/master.m3u8');
791 expect(body.playlistUrl).toBe('/api/files/video-stream/sess-123/master.m3u8');
778 expect(body.durationSeconds).toBe(42.25);
792 expect(body.durationSeconds).toBe(42.25);
793 expect(body.videoWidth).toBe(1920);
794 expect(body.videoHeight).toBe(1080);
779 expect(body.subtitleSummary).toEqual({ total: 0, converted: 0, imageBased: 0, skipped: 0 });
795 expect(body.subtitleSummary).toEqual({ total: 0, converted: 0, imageBased: 0, skipped: 0 });
780 expect(body.subtitleTracks).toEqual([
796 expect(body.subtitleTracks).toEqual([
781 {
797 {
@@ -810,6 +826,37 @@ describe('video streaming routes', () =>
810 expect(releasedVideoStreamSessionId).toBe('sess-123');
826 expect(releasedVideoStreamSessionId).toBe('sess-123');
811 });
827 });
812
828
829 test('GET /api/files/video-stream-subtitle-frame returns rendered PNG frames', async () =>
830 {
831 const filePath = path.join(rootDir, 'clip.mkv');
832 fs.writeFileSync(filePath, 'video');
833 mockSubtitleFrameResult = Buffer.from('png-data');
834
835 const { status, headers, body } = await reqRaw(
836 'GET',
837 `/api/files/video-stream-subtitle-frame?filePath=${encodeURIComponent(filePath)}&streamIndex=0&timeSeconds=2.5&videoWidth=1920&videoHeight=1080`
838 );
839
840 expect(status).toBe(200);
841 expect(headers['content-type']).toMatch(/image\/png/);
842 expect(body.equals(Buffer.from('png-data'))).toBe(true);
843 });
844
845 test('GET /api/files/video-stream-subtitle-frame returns 204 when rendering fails', async () =>
846 {
847 const filePath = path.join(rootDir, 'clip.mkv');
848 fs.writeFileSync(filePath, 'video');
849 mockSubtitleFrameError = new Error('decode failed');
850
851 const { status, body } = await reqRaw(
852 'GET',
853 `/api/files/video-stream-subtitle-frame?filePath=${encodeURIComponent(filePath)}&streamIndex=0&timeSeconds=2.5`
854 );
855
856 expect(status).toBe(204);
857 expect(body.length).toBe(0);
858 });
859
813 test('POST /api/files/watch-progress stores per-user watched status by thresholds', async () =>
860 test('POST /api/files/watch-progress stores per-user watched status by thresholds', async () =>
814 {
861 {
815 const filePath = path.join(rootDir, 'clip.mkv');
862 const filePath = path.join(rootDir, 'clip.mkv');