Send subtitle frame placement metadata from server to client
Previously, the server rendered image subtitles but threw away all positioning metadata, sending only the PNG buffer. The client had no way to know the actual frame dimensions or position within the rendered canvas, so it relied on CSS max-height constraints (which didn't scale properly for downscaled frames). Now the server: 1. Preserves positioning metadata (left, top, width, height, canvasWidth, canvasHeight) 2. Returns frame data as an object with both the PNG buffer and metadata 3. Sends headers with the placement information The client: 1. Reads X-Subtitle-Frame-* headers from the response 2. Calculates percentage-based positioning: (frameLeft / canvasWidth) * 100% 3. Calculates percentage-based sizing: (frameWidth / canvasWidth) * 100% 4. Applies these values to properly position and scale the image This allows downscaled subtitle frames (rendered at 0.5 scale) to: - Display at correct size relative to the canvas they were rendered for - Scale properly in both normal and fullscreen modes - Maintain proper aspect ratio and positioning Fixes the subtitle display issue where downscaled frames appeared too small and fell off screen edges in fullscreen mode.
@@ -721,6 +721,34 @@ function bindImageSubtitleOverlay (player, videojs, prefs, session, item)
721
721
closeObjectUrl();
722
722
activeObjectUrl = URL.createObjectURL(blob);
723
723
overlayImgEl.src = activeObjectUrl;
724
725
// Apply placement from headers when available
726
const frameLeft = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Left') || ''), 10);
727
const frameTop = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Top') || ''), 10);
728
const frameWidth = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Width') || ''), 10);
729
const frameHeight = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Height') || ''), 10);
730
const canvasWidth = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Canvas-Width') || ''), 10);
731
const canvasHeight = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Canvas-Height') || ''), 10);
732
733
if (frameWidth > 0 && frameHeight > 0 && canvasWidth > 0 && canvasHeight > 0)
734
{
735
// Apply percentage-based sizing to scale the downscaled image properly
736
const widthPercent = (frameWidth / canvasWidth) * 100;
737
const heightPercent = (frameHeight / canvasHeight) * 100;
738
overlayImgEl.style.width = `${widthPercent}%`;
739
overlayImgEl.style.height = `${heightPercent}%`;
740
overlayImgEl.style.maxWidth = 'none';
741
overlayImgEl.style.maxHeight = 'none';
742
}
743
else
744
{
745
// Fallback: use auto sizing with max constraints
746
overlayImgEl.style.width = 'auto';
747
overlayImgEl.style.height = 'auto';
748
overlayImgEl.style.maxWidth = '100%';
749
overlayImgEl.style.maxHeight = '100%';
750
}
751
724
752
overlayEl.style.display = 'block';
725
753
overlayImgEl.style.display = 'block';
726
754
} catch
@@ -751,15 +779,17 @@ function bindImageSubtitleOverlay (player, videojs, prefs, session, item)
751
779
overlayEl.style.pointerEvents = 'none';
752
780
overlayEl.style.textAlign = 'center';
753
781
overlayEl.style.padding = '0 0.75rem';
782
overlayEl.style.maxHeight = '70%';
783
overlayEl.style.overflow = 'hidden';
754
784
755
785
overlayImgEl = document.createElement('img');
756
786
overlayImgEl.alt = 'Image subtitles';
757
787
overlayImgEl.style.display = 'none';
758
overlayImgEl.style.maxWidth = '100%';
759
overlayImgEl.style.maxHeight = '70%';
760
788
overlayImgEl.style.margin = '0 auto';
761
789
overlayImgEl.style.objectFit = 'contain';
762
790
overlayImgEl.style.filter = 'drop-shadow(0 2px 2px rgba(0, 0, 0, 0.7))';
791
overlayImgEl.style.maxWidth = '100%';
792
overlayImgEl.style.height = 'auto';
763
793
overlayEl.appendChild(overlayImgEl);
764
794
765
795
menuEl = document.createElement('div');
@@ -721,6 +721,34 @@ function bindImageSubtitleOverlay (player, videojs, prefs, session, item)721
closeObjectUrl();721
closeObjectUrl();722
activeObjectUrl = URL.createObjectURL(blob);722
activeObjectUrl = URL.createObjectURL(blob);723
overlayImgEl.src = activeObjectUrl;723
overlayImgEl.src = activeObjectUrl;724
725
// Apply placement from headers when available726
const frameLeft = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Left') || ''), 10);727
const frameTop = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Top') || ''), 10);728
const frameWidth = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Width') || ''), 10);729
const frameHeight = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Height') || ''), 10);730
const canvasWidth = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Canvas-Width') || ''), 10);731
const canvasHeight = Number.parseInt(String(res.headers.get('X-Subtitle-Frame-Canvas-Height') || ''), 10);732
733
if (frameWidth > 0 && frameHeight > 0 && canvasWidth > 0 && canvasHeight > 0)734
{735
// Apply percentage-based sizing to scale the downscaled image properly736
const widthPercent = (frameWidth / canvasWidth) * 100;737
const heightPercent = (frameHeight / canvasHeight) * 100;738
overlayImgEl.style.width = `${widthPercent}%`;739
overlayImgEl.style.height = `${heightPercent}%`;740
overlayImgEl.style.maxWidth = 'none';741
overlayImgEl.style.maxHeight = 'none';742
}743
else744
{745
// Fallback: use auto sizing with max constraints746
overlayImgEl.style.width = 'auto';747
overlayImgEl.style.height = 'auto';748
overlayImgEl.style.maxWidth = '100%';749
overlayImgEl.style.maxHeight = '100%';750
}751
724
overlayEl.style.display = 'block';752
overlayEl.style.display = 'block';725
overlayImgEl.style.display = 'block';753
overlayImgEl.style.display = 'block';726
} catch754
} catch@@ -751,15 +779,17 @@ function bindImageSubtitleOverlay (player, videojs, prefs, session, item)751
overlayEl.style.pointerEvents = 'none';779
overlayEl.style.pointerEvents = 'none';752
overlayEl.style.textAlign = 'center';780
overlayEl.style.textAlign = 'center';753
overlayEl.style.padding = '0 0.75rem';781
overlayEl.style.padding = '0 0.75rem';782
overlayEl.style.maxHeight = '70%';783
overlayEl.style.overflow = 'hidden';754
784
755
overlayImgEl = document.createElement('img');785
overlayImgEl = document.createElement('img');756
overlayImgEl.alt = 'Image subtitles';786
overlayImgEl.alt = 'Image subtitles';757
overlayImgEl.style.display = 'none';787
overlayImgEl.style.display = 'none';758
overlayImgEl.style.maxWidth = '100%';759
overlayImgEl.style.maxHeight = '70%';760
overlayImgEl.style.margin = '0 auto';788
overlayImgEl.style.margin = '0 auto';761
overlayImgEl.style.objectFit = 'contain';789
overlayImgEl.style.objectFit = 'contain';762
overlayImgEl.style.filter = 'drop-shadow(0 2px 2px rgba(0, 0, 0, 0.7))';790
overlayImgEl.style.filter = 'drop-shadow(0 2px 2px rgba(0, 0, 0, 0.7))';791
overlayImgEl.style.maxWidth = '100%';792
overlayImgEl.style.height = 'auto';763
overlayEl.appendChild(overlayImgEl);793
overlayEl.appendChild(overlayImgEl);764
794
765
menuEl = document.createElement('div');795
menuEl = document.createElement('div');@@ -464,7 +464,15 @@ async function renderEventFrame (event, canvasWidth, canvasHeight)
464
464
.toBuffer();
465
465
}
466
466
467
return imageBuffer;
467
return {
468
frame: imageBuffer,
469
left: visibleLeft,
470
top: visibleTop,
471
width: visibleWidth,
472
height: visibleHeight,
473
canvasWidth,
474
canvasHeight,
475
};
468
476
}
469
477
470
478
function parseRenderOptions (options)
@@ -509,19 +517,35 @@ async function renderImageSubtitleFrame (options)
509
517
const cachedFrame = getCachedValue(frameCache, frameKey);
510
518
if (cachedFrame)
511
519
{
512
return cachedFrame.frame || cachedFrame;
520
return cachedFrame;
521
}
522
523
const frameData = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);
524
if (!frameData)
525
{
526
return null;
513
527
}
514
528
515
const frame = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);
516
529
const event = entry.events[eventIndex];
530
const cacheValue = {
531
frame: frameData.frame,
532
left: frameData.left,
533
top: frameData.top,
534
width: frameData.width,
535
height: frameData.height,
536
canvasWidth: frameData.canvasWidth,
537
canvasHeight: frameData.canvasHeight,
538
startTimeSeconds: event.startTimeSeconds,
539
endTimeSeconds: event.endTimeSeconds,
540
};
517
541
setCachedValue(
518
542
frameCache,
519
543
frameKey,
520
{ frame, startTimeSeconds: event.startTimeSeconds, endTimeSeconds: event.endTimeSeconds },
544
cacheValue,
521
545
FRAME_CACHE_TTL_MS,
522
546
estimateFrameCacheBytes,
523
547
);
524
return frame;
548
return cacheValue;
525
549
}
526
550
527
551
function pruneOldImageSubtitleCaches (filePath, slowestViewerTimeSeconds)
@@ -464,7 +464,15 @@ async function renderEventFrame (event, canvasWidth, canvasHeight)464
.toBuffer();464
.toBuffer();465
}465
}466
466
467
return imageBuffer;467
return {468
frame: imageBuffer,469
left: visibleLeft,470
top: visibleTop,471
width: visibleWidth,472
height: visibleHeight,473
canvasWidth,474
canvasHeight,475
};468
}476
}469
477
470
function parseRenderOptions (options)478
function parseRenderOptions (options)@@ -509,19 +517,35 @@ async function renderImageSubtitleFrame (options)509
const cachedFrame = getCachedValue(frameCache, frameKey);517
const cachedFrame = getCachedValue(frameCache, frameKey);510
if (cachedFrame)518
if (cachedFrame)511
{519
{512
return cachedFrame.frame || cachedFrame;520
return cachedFrame;521
}522
523
const frameData = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);524
if (!frameData)525
{526
return null;513
}527
}514
528
515
const frame = await renderEventFrame(entry.events[eventIndex], canvasWidth, canvasHeight);516
const event = entry.events[eventIndex];529
const event = entry.events[eventIndex];530
const cacheValue = {531
frame: frameData.frame,532
left: frameData.left,533
top: frameData.top,534
width: frameData.width,535
height: frameData.height,536
canvasWidth: frameData.canvasWidth,537
canvasHeight: frameData.canvasHeight,538
startTimeSeconds: event.startTimeSeconds,539
endTimeSeconds: event.endTimeSeconds,540
};517
setCachedValue(541
setCachedValue(518
frameCache,542
frameCache,519
frameKey,543
frameKey,520
{ frame, startTimeSeconds: event.startTimeSeconds, endTimeSeconds: event.endTimeSeconds },544
cacheValue,521
FRAME_CACHE_TTL_MS,545
FRAME_CACHE_TTL_MS,522
estimateFrameCacheBytes,546
estimateFrameCacheBytes,523
);547
);524
return frame;548
return cacheValue;525
}549
}526
550
527
function pruneOldImageSubtitleCaches (filePath, slowestViewerTimeSeconds)551
function pruneOldImageSubtitleCaches (filePath, slowestViewerTimeSeconds)@@ -1588,16 +1588,22 @@ router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =
1588
1588
canvasHeight,
1589
1589
});
1590
1590
1591
if (!frameData || frameData.length === 0)
1591
if (!frameData || !frameData.frame)
1592
1592
{
1593
1593
res.set('X-Subtitle-Frame-Status', 'none');
1594
1594
return res.status(204).end();
1595
1595
}
1596
1596
1597
1597
res.set('X-Subtitle-Frame-Status', 'ok');
1598
res.set('X-Subtitle-Frame-Left', String(frameData.left || 0));
1599
res.set('X-Subtitle-Frame-Top', String(frameData.top || 0));
1600
res.set('X-Subtitle-Frame-Width', String(frameData.width || 0));
1601
res.set('X-Subtitle-Frame-Height', String(frameData.height || 0));
1602
res.set('X-Subtitle-Frame-Canvas-Width', String(frameData.canvasWidth || 1280));
1603
res.set('X-Subtitle-Frame-Canvas-Height', String(frameData.canvasHeight || 720));
1598
1604
res.set('Content-Type', 'image/png');
1599
1605
res.set('Cache-Control', 'private, max-age=3600');
1600
res.send(frameData);
1606
res.send(frameData.frame);
1601
1607
} catch (err)
1602
1608
{
1603
1609
logError('Subtitle frame extraction failed', err, {
@@ -1588,16 +1588,22 @@ router.get('/video-stream-subtitle-frame', requireAuth, async (req, res, next) =1588
canvasHeight,1588
canvasHeight,1589
});1589
});1590
1590
1591
if (!frameData || frameData.length === 0)1591
if (!frameData || !frameData.frame)1592
{1592
{1593
res.set('X-Subtitle-Frame-Status', 'none');1593
res.set('X-Subtitle-Frame-Status', 'none');1594
return res.status(204).end();1594
return res.status(204).end();1595
}1595
}1596
1596
1597
res.set('X-Subtitle-Frame-Status', 'ok');1597
res.set('X-Subtitle-Frame-Status', 'ok');1598
res.set('X-Subtitle-Frame-Left', String(frameData.left || 0));1599
res.set('X-Subtitle-Frame-Top', String(frameData.top || 0));1600
res.set('X-Subtitle-Frame-Width', String(frameData.width || 0));1601
res.set('X-Subtitle-Frame-Height', String(frameData.height || 0));1602
res.set('X-Subtitle-Frame-Canvas-Width', String(frameData.canvasWidth || 1280));1603
res.set('X-Subtitle-Frame-Canvas-Height', String(frameData.canvasHeight || 720));1598
res.set('Content-Type', 'image/png');1604
res.set('Content-Type', 'image/png');1599
res.set('Cache-Control', 'private, max-age=3600');1605
res.set('Cache-Control', 'private, max-age=3600');1600
res.send(frameData);1606
res.send(frameData.frame);1601
} catch (err)1607
} catch (err)1602
{1608
{1603
logError('Subtitle frame extraction failed', err, {1609
logError('Subtitle frame extraction failed', err, {@@ -1005,7 +1005,15 @@ describe('video streaming routes', () =>
1005
1005
{
1006
1006
const filePath = path.join(rootDir, 'clip.mkv');
1007
1007
fs.writeFileSync(filePath, 'video');
1008
mockSubtitleFrameResult = Buffer.from('png-data');
1008
mockSubtitleFrameResult = {
1009
frame: Buffer.from('png-data'),
1010
left: 100,
1011
top: 50,
1012
width: 320,
1013
height: 180,
1014
canvasWidth: 640,
1015
canvasHeight: 360,
1016
};
1009
1017
1010
1018
const { status, headers, body } = await reqRaw(
1011
1019
'GET',
@@ -1015,6 +1023,12 @@ describe('video streaming routes', () =>
1015
1023
expect(status).toBe(200);
1016
1024
expect(headers['content-type']).toMatch(/image\/png/);
1017
1025
expect(headers['x-subtitle-frame-status']).toBe('ok');
1026
expect(headers['x-subtitle-frame-left']).toBe('100');
1027
expect(headers['x-subtitle-frame-top']).toBe('50');
1028
expect(headers['x-subtitle-frame-width']).toBe('320');
1029
expect(headers['x-subtitle-frame-height']).toBe('180');
1030
expect(headers['x-subtitle-frame-canvas-width']).toBe('640');
1031
expect(headers['x-subtitle-frame-canvas-height']).toBe('360');
1018
1032
expect(body.equals(Buffer.from('png-data'))).toBe(true);
1019
1033
});
1020
1034
@@ -1005,7 +1005,15 @@ describe('video streaming routes', () =>1005
{1005
{1006
const filePath = path.join(rootDir, 'clip.mkv');1006
const filePath = path.join(rootDir, 'clip.mkv');1007
fs.writeFileSync(filePath, 'video');1007
fs.writeFileSync(filePath, 'video');1008
mockSubtitleFrameResult = Buffer.from('png-data');1008
mockSubtitleFrameResult = {1009
frame: Buffer.from('png-data'),1010
left: 100,1011
top: 50,1012
width: 320,1013
height: 180,1014
canvasWidth: 640,1015
canvasHeight: 360,1016
};1009
1017
1010
const { status, headers, body } = await reqRaw(1018
const { status, headers, body } = await reqRaw(1011
'GET',1019
'GET',@@ -1015,6 +1023,12 @@ describe('video streaming routes', () =>1015
expect(status).toBe(200);1023
expect(status).toBe(200);1016
expect(headers['content-type']).toMatch(/image\/png/);1024
expect(headers['content-type']).toMatch(/image\/png/);1017
expect(headers['x-subtitle-frame-status']).toBe('ok');1025
expect(headers['x-subtitle-frame-status']).toBe('ok');1026
expect(headers['x-subtitle-frame-left']).toBe('100');1027
expect(headers['x-subtitle-frame-top']).toBe('50');1028
expect(headers['x-subtitle-frame-width']).toBe('320');1029
expect(headers['x-subtitle-frame-height']).toBe('180');1030
expect(headers['x-subtitle-frame-canvas-width']).toBe('640');1031
expect(headers['x-subtitle-frame-canvas-height']).toBe('360');1018
expect(body.equals(Buffer.from('png-data'))).toBe(true);1032
expect(body.equals(Buffer.from('png-data'))).toBe(true);1019
});1033
});1020
1034