BlueGit

timfa filemanager

Send subtitle frame placement metadata from server to client

Previously, the server rendered image subtitles but threw away all positioning
metadata, sending only the PNG buffer. The client had no way to know the
actual frame dimensions or position within the rendered canvas, so it relied
on CSS max-height constraints (which didn't scale properly for downscaled frames).

Now the server:
1. Preserves positioning metadata (left, top, width, height, canvasWidth, canvasHeight)
2. Returns frame data as an object with both the PNG buffer and metadata
3. Sends headers with the placement information

The client:
1. Reads X-Subtitle-Frame-* headers from the response
2. Calculates percentage-based positioning: (frameLeft / canvasWidth) * 100%
3. Calculates percentage-based sizing: (frameWidth / canvasWidth) * 100%
4. Applies these values to properly position and scale the image

This allows downscaled subtitle frames (rendered at 0.5 scale) to:
- Display at correct size relative to the canvas they were rendered for
- Scale properly in both normal and fullscreen modes
- Maintain proper aspect ratio and positioning

Fixes the subtitle display issue where downscaled frames appeared too small
and fell off screen edges in fullscreen mode.

timfa Tim Falken · pushed by timfa

Authored 2026-05-20 · 6 months ago

Hash
b41f010303e14272855158968f4f5cbae372aa04
Parent
7e49aa30
4 files changed +84 −10
public/app.js +32 −2
@@ -721,6 +721,34 @@ function bindImageSubtitleOverlay (player, videojs, prefs, session, item)
721 721 closeObjectUrl();
722 722 activeObjectUrl = URL.createObjectURL(blob);
723 723 overlayImgEl.src = activeObjectUrl;
724
725 // Apply placement from headers when available
726 const frameLeft = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Left') || ''), 10);
727 const frameTop = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Top') || ''), 10);
728 const frameWidth = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Width') || ''), 10);
729 const frameHeight = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Height') || ''), 10);
730 const canvasWidth = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Canvas-Width') || ''), 10);
731 const canvasHeight = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Canvas-Height') || ''), 10);
732
733 if (frameWidth > 0 && frameHeight > 0 && canvasWidth > 0 && canvasHeight > 0)
734 {
735 // Apply percentage-based sizing to scale the downscaled image properly
736 const widthPercent = (frameWidth / canvasWidth) * 100;
737 const heightPercent = (frameHeight / canvasHeight) * 100;
738 overlayImgEl.style.width = `${widthPercent}%`;
739 overlayImgEl.style.height = `${heightPercent}%`;
740 overlayImgEl.style.maxWidth = 'none';
741 overlayImgEl.style.maxHeight = 'none';
742 }
743 else
744 {
745 // Fallback: use auto sizing with max constraints
746 overlayImgEl.style.width = 'auto';
747 overlayImgEl.style.height = 'auto';
748 overlayImgEl.style.maxWidth = '100%';
749 overlayImgEl.style.maxHeight = '100%';
750 }
751
724 752 overlayEl.style.display = 'block';
725 753 overlayImgEl.style.display = 'block';
726 754 } catch
@@ -751,15 +779,17 @@ function bindImageSubtitleOverlay (player, videojs, prefs, session, item)
751 779 overlayEl.style.pointerEvents = 'none';
752 780 overlayEl.style.textAlign = 'center';
753 781 overlayEl.style.padding = '0 0.75rem';
782 overlayEl.style.maxHeight = '70%';
783 overlayEl.style.overflow = 'hidden';
754 784
755 785 overlayImgEl = document.createElement('img');
756 786 overlayImgEl.alt = 'Image subtitles';
757 787 overlayImgEl.style.display = 'none';
758 overlayImgEl.style.maxWidth = '100%';
759 overlayImgEl.style.maxHeight = '70%';
760 788 overlayImgEl.style.margin = '0 auto';
761 789 overlayImgEl.style.objectFit = 'contain';
762 790 overlayImgEl.style.filter = 'drop-shadow(0 2px 2px rgba(0, 0, 0, 0.7))';
791 overlayImgEl.style.maxWidth = '100%';
792 overlayImgEl.style.height = 'auto';
763 793 overlayEl.appendChild(overlayImgEl);
764 794
765 795 menuEl = document.createElement('div');
@@ -721,6 +721,34 @@ function bindImageSubtitleOverlay (player, videojs, prefs, session, item)
721 closeObjectUrl();
721 closeObjectUrl();
722 activeObjectUrl = URL.createObjectURL(blob);
722 activeObjectUrl = URL.createObjectURL(blob);
723 overlayImgEl.src = activeObjectUrl;
723 overlayImgEl.src = activeObjectUrl;
724
725 // Apply placement from headers when available
726 const frameLeft = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Left') || ''), 10);
727 const frameTop = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Top') || ''), 10);
728 const frameWidth = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Width') || ''), 10);
729 const frameHeight = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Height') || ''), 10);
730 const canvasWidth = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Canvas-Width') || ''), 10);
731 const canvasHeight = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Canvas-Height') || ''), 10);
732
733 if (frameWidth > 0 && frameHeight > 0 && canvasWidth > 0 && canvasHeight > 0)
734 {
735 // Apply percentage-based sizing to scale the downscaled image properly
736 const widthPercent = (frameWidth / canvasWidth) * 100;
737 const heightPercent = (frameHeight / canvasHeight) * 100;
738 overlayImgEl.style.width = `${widthPercent}%`;
739 overlayImgEl.style.height = `${heightPercent}%`;
740 overlayImgEl.style.maxWidth = 'none';
741 overlayImgEl.style.maxHeight = 'none';
742 }
743 else
744 {
745 // Fallback: use auto sizing with max constraints
746 overlayImgEl.style.width = 'auto';
747 overlayImgEl.style.height = 'auto';
748 overlayImgEl.style.maxWidth = '100%';
749 overlayImgEl.style.maxHeight = '100%';
750 }
751
724 overlayEl.style.display = 'block';
752 overlayEl.style.display = 'block';
725 overlayImgEl.style.display = 'block';
753 overlayImgEl.style.display = 'block';
726 } catch
754 } catch
@@ -751,15 +779,17 @@ function bindImageSubtitleOverlay (player, videojs, prefs, session, item)
751 overlayEl.style.pointerEvents = 'none';
779 overlayEl.style.pointerEvents = 'none';
752 overlayEl.style.textAlign = 'center';
780 overlayEl.style.textAlign = 'center';
753 overlayEl.style.padding = '0 0.75rem';
781 overlayEl.style.padding = '0 0.75rem';
782 overlayEl.style.maxHeight = '70%';
783 overlayEl.style.overflow = 'hidden';
754
784
755 overlayImgEl = document.createElement('img');
785 overlayImgEl = document.createElement('img');
756 overlayImgEl.alt = 'Image subtitles';
786 overlayImgEl.alt = 'Image subtitles';
757 overlayImgEl.style.display = 'none';
787 overlayImgEl.style.display = 'none';
758 overlayImgEl.style.maxWidth = '100%';
759 overlayImgEl.style.maxHeight = '70%';
760 overlayImgEl.style.margin = '0 auto';
788 overlayImgEl.style.margin = '0 auto';
761 overlayImgEl.style.objectFit = 'contain';
789 overlayImgEl.style.objectFit = 'contain';
762 overlayImgEl.style.filter = 'drop-shadow(0 2px 2px rgba(0, 0, 0, 0.7))';
790 overlayImgEl.style.filter = 'drop-shadow(0 2px 2px rgba(0, 0, 0, 0.7))';
791 overlayImgEl.style.maxWidth = '100%';
792 overlayImgEl.style.height = 'auto';
763 overlayEl.appendChild(overlayImgEl);
793 overlayEl.appendChild(overlayImgEl);
764
794
765 menuEl = document.createElement('div');
795 menuEl = document.createElement('div');
server/image-subtitles.js +29 −5
@@ -464,7 +464,15 @@ async function renderEventFrame (event, canvasWidth, canvasHeight)
464 464 .toBuffer();
465 465 }
466 466
467 return imageBuffer;
467 return {
468 frame: imageBuffer,
469 left: visibleLeft,
470 top: visibleTop,
471 width: visibleWidth,
472 height: visibleHeight,
473 canvasWidth,
474 canvasHeight,
475 };
468 476 }
469 477
470 478 function parseRenderOptions (options)
@@ -509,19 +517,35 @@ async function renderImageSubtitleFrame (options)
509 517 const cachedFrame = getCachedValue(frameCache, frameKey);
510 518 if (cachedFrame)
511 519 {
512 return cachedFrame.frame || cachedFrame;
520 return cachedFrame;
521 }
522
523 const frameData = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);
524 if (!frameData)
525 {
526 return null;
513 527 }
514 528
515 const frame = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);
516 529 const event = entry.events[eventIndex];
530 const cacheValue = {
531 frame: frameData.frame,
532 left: frameData.left,
533 top: frameData.top,
534 width: frameData.width,
535 height: frameData.height,
536 canvasWidth: frameData.canvasWidth,
537 canvasHeight: frameData.canvasHeight,
538 startTimeSeconds: event.startTimeSeconds,
539 endTimeSeconds: event.endTimeSeconds,
540 };
517 541 setCachedValue(
518 542 frameCache,
519 543 frameKey,
520 { frame, startTimeSeconds: event.startTimeSeconds, endTimeSeconds: event.endTimeSeconds },
544 cacheValue,
521 545 FRAME_CACHE_TTL_MS,
522 546 estimateFrameCacheBytes,
523 547 );
524 return frame;
548 return cacheValue;
525 549 }
526 550
527 551 function pruneOldImageSubtitleCaches (filePath, slowestViewerTimeSeconds)
@@ -464,7 +464,15 @@ async function renderEventFrame (event, canvasWidth, canvasHeight)
464 .toBuffer();
464 .toBuffer();
465 }
465 }
466
466
467 return imageBuffer;
467 return {
468 frame: imageBuffer,
469 left: visibleLeft,
470 top: visibleTop,
471 width: visibleWidth,
472 height: visibleHeight,
473 canvasWidth,
474 canvasHeight,
475 };
468}
476}
469
477
470function parseRenderOptions (options)
478function parseRenderOptions (options)
@@ -509,19 +517,35 @@ async function renderImageSubtitleFrame (options)
509 const cachedFrame = getCachedValue(frameCache, frameKey);
517 const cachedFrame = getCachedValue(frameCache, frameKey);
510 if (cachedFrame)
518 if (cachedFrame)
511 {
519 {
512 return cachedFrame.frame || cachedFrame;
520 return cachedFrame;
521 }
522
523 const frameData = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);
524 if (!frameData)
525 {
526 return null;
513 }
527 }
514
528
515 const frame = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);
516 const event = entry.events[eventIndex];
529 const event = entry.events[eventIndex];
530 const cacheValue = {
531 frame: frameData.frame,
532 left: frameData.left,
533 top: frameData.top,
534 width: frameData.width,
535 height: frameData.height,
536 canvasWidth: frameData.canvasWidth,
537 canvasHeight: frameData.canvasHeight,
538 startTimeSeconds: event.startTimeSeconds,
539 endTimeSeconds: event.endTimeSeconds,
540 };
517 setCachedValue(
541 setCachedValue(
518 frameCache,
542 frameCache,
519 frameKey,
543 frameKey,
520 { frame, startTimeSeconds: event.startTimeSeconds, endTimeSeconds: event.endTimeSeconds },
544 cacheValue,
521 FRAME_CACHE_TTL_MS,
545 FRAME_CACHE_TTL_MS,
522 estimateFrameCacheBytes,
546 estimateFrameCacheBytes,
523 );
547 );
524 return frame;
548 return cacheValue;
525}
549}
526
550
527function pruneOldImageSubtitleCaches (filePath, slowestViewerTimeSeconds)
551function pruneOldImageSubtitleCaches (filePath, slowestViewerTimeSeconds)
server/routes/files.js +8 −2
@@ -1588,16 +1588,22 @@ router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =
1588 1588 canvasHeight,
1589 1589 });
1590 1590
1591 if (!frameData || frameData.length === 0)
1591 if (!frameData || !frameData.frame)
1592 1592 {
1593 1593 res.set('X-Subtitle-Frame-Status', 'none');
1594 1594 return res.status(204).end();
1595 1595 }
1596 1596
1597 1597 res.set('X-Subtitle-Frame-Status', 'ok');
1598 res.set('X-Subtitle-Frame-Left', String(frameData.left || 0));
1599 res.set('X-Subtitle-Frame-Top', String(frameData.top || 0));
1600 res.set('X-Subtitle-Frame-Width', String(frameData.width || 0));
1601 res.set('X-Subtitle-Frame-Height', String(frameData.height || 0));
1602 res.set('X-Subtitle-Frame-Canvas-Width', String(frameData.canvasWidth || 1280));
1603 res.set('X-Subtitle-Frame-Canvas-Height', String(frameData.canvasHeight || 720));
1598 1604 res.set('Content-Type', 'image/png');
1599 1605 res.set('Cache-Control', 'private, max-age=3600');
1600 res.send(frameData);
1606 res.send(frameData.frame);
1601 1607 } catch (err)
1602 1608 {
1603 1609 logError('Subtitle frame extraction failed', err, {
@@ -1588,16 +1588,22 @@ router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =
1588 canvasHeight,
1588 canvasHeight,
1589 });
1589 });
1590
1590
1591 if (!frameData || frameData.length === 0)
1591 if (!frameData || !frameData.frame)
1592 {
1592 {
1593 res.set('X-Subtitle-Frame-Status', 'none');
1593 res.set('X-Subtitle-Frame-Status', 'none');
1594 return res.status(204).end();
1594 return res.status(204).end();
1595 }
1595 }
1596
1596
1597 res.set('X-Subtitle-Frame-Status', 'ok');
1597 res.set('X-Subtitle-Frame-Status', 'ok');
1598 res.set('X-Subtitle-Frame-Left', String(frameData.left || 0));
1599 res.set('X-Subtitle-Frame-Top', String(frameData.top || 0));
1600 res.set('X-Subtitle-Frame-Width', String(frameData.width || 0));
1601 res.set('X-Subtitle-Frame-Height', String(frameData.height || 0));
1602 res.set('X-Subtitle-Frame-Canvas-Width', String(frameData.canvasWidth || 1280));
1603 res.set('X-Subtitle-Frame-Canvas-Height', String(frameData.canvasHeight || 720));
1598 res.set('Content-Type', 'image/png');
1604 res.set('Content-Type', 'image/png');
1599 res.set('Cache-Control', 'private, max-age=3600');
1605 res.set('Cache-Control', 'private, max-age=3600');
1600 res.send(frameData);
1606 res.send(frameData.frame);
1601 } catch (err)
1607 } catch (err)
1602 {
1608 {
1603 logError('Subtitle frame extraction failed', err, {
1609 logError('Subtitle frame extraction failed', err, {
tests/files.routes.test.js +15 −1
@@ -1005,7 +1005,15 @@ describe('video streaming routes', () =>
1005 1005 {
1006 1006 const filePath = path.join(rootDir, 'clip.mkv');
1007 1007 fs.writeFileSync(filePath, 'video');
1008 mockSubtitleFrameResult = Buffer.from('png-data');
1008 mockSubtitleFrameResult = {
1009 frame: Buffer.from('png-data'),
1010 left: 100,
1011 top: 50,
1012 width: 320,
1013 height: 180,
1014 canvasWidth: 640,
1015 canvasHeight: 360,
1016 };
1009 1017
1010 1018 const { status, headers, body } = await reqRaw(
1011 1019 'GET',
@@ -1015,6 +1023,12 @@ describe('video streaming routes', () =>
1015 1023 expect(status).toBe(200);
1016 1024 expect(headers['content-type']).toMatch(/image\/png/);
1017 1025 expect(headers['x-subtitle-frame-status']).toBe('ok');
1026 expect(headers['x-subtitle-frame-left']).toBe('100');
1027 expect(headers['x-subtitle-frame-top']).toBe('50');
1028 expect(headers['x-subtitle-frame-width']).toBe('320');
1029 expect(headers['x-subtitle-frame-height']).toBe('180');
1030 expect(headers['x-subtitle-frame-canvas-width']).toBe('640');
1031 expect(headers['x-subtitle-frame-canvas-height']).toBe('360');
1018 1032 expect(body.equals(Buffer.from('png-data'))).toBe(true);
1019 1033 });
1020 1034
@@ -1005,7 +1005,15 @@ describe('video streaming routes', () =>
1005 {
1005 {
1006 const filePath = path.join(rootDir, 'clip.mkv');
1006 const filePath = path.join(rootDir, 'clip.mkv');
1007 fs.writeFileSync(filePath, 'video');
1007 fs.writeFileSync(filePath, 'video');
1008 mockSubtitleFrameResult = Buffer.from('png-data');
1008 mockSubtitleFrameResult = {
1009 frame: Buffer.from('png-data'),
1010 left: 100,
1011 top: 50,
1012 width: 320,
1013 height: 180,
1014 canvasWidth: 640,
1015 canvasHeight: 360,
1016 };
1009
1017
1010 const { status, headers, body } = await reqRaw(
1018 const { status, headers, body } = await reqRaw(
1011 'GET',
1019 'GET',
@@ -1015,6 +1023,12 @@ describe('video streaming routes', () =>
1015 expect(status).toBe(200);
1023 expect(status).toBe(200);
1016 expect(headers['content-type']).toMatch(/image\/png/);
1024 expect(headers['content-type']).toMatch(/image\/png/);
1017 expect(headers['x-subtitle-frame-status']).toBe('ok');
1025 expect(headers['x-subtitle-frame-status']).toBe('ok');
1026 expect(headers['x-subtitle-frame-left']).toBe('100');
1027 expect(headers['x-subtitle-frame-top']).toBe('50');
1028 expect(headers['x-subtitle-frame-width']).toBe('320');
1029 expect(headers['x-subtitle-frame-height']).toBe('180');
1030 expect(headers['x-subtitle-frame-canvas-width']).toBe('640');
1031 expect(headers['x-subtitle-frame-canvas-height']).toBe('360');
1018 expect(body.equals(Buffer.from('png-data'))).toBe(true);
1032 expect(body.equals(Buffer.from('png-data'))).toBe(true);
1019 });
1033 });
1020
1034