Improve subtitle clock precision and prewarming
Image subtitle overlay timing now prefers the video renderer media clock when available, which improves boundary accuracy for cached captions that previously appeared late or were missed. Also adds proactive secondary-buffer prewarming with the next cached frame so rapid caption transitions have the next image ready before the swap point, reducing visible misses on dense subtitle sequences. Includes a regression test that verifies media-time based sync behaviour.
timfa Tim Falken · pushed by timfa
Authored 2026-05-29 · 5 months ago
- Hash
f4321073f7044974c5ed92c125f9490b5ba510ae- Parent
a40f0816
2 files changed
+135
−4
@@ -16,7 +16,7 @@
16
16
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_DEFAULT = 0.5;
17
17
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MIN = 0.25;
18
18
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MAX = 30;
19
const IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS = 0.05;
19
const IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS = 0.005;
20
20
const IMAGE_SUBTITLE_CACHE_MISS_FALLBACK_INTERVAL_MS_DEFAULT = 2000;
21
21
const IMAGE_SUBTITLE_CURRENT_REQUEST_TIMEOUT_MS_DEFAULT = 4000;
22
22
const IMAGE_SUBTITLE_PREFETCH_REQUEST_TIMEOUT_MS_DEFAULT = 4000;
@@ -393,6 +393,17 @@
393
393
394
394
const requestFrame = (fn) =>
395
395
{
396
const playerEl = player.el && player.el();
397
const videoEl = playerEl ? playerEl.querySelector('video') : null;
398
if (videoEl && typeof videoEl.requestVideoFrameCallback === 'function')
399
{
400
const id = videoEl.requestVideoFrameCallback((_, metadata) =>
401
{
402
fn(metadata);
403
});
404
return { kind: 'video-frame', id, videoEl };
405
}
406
396
407
if (typeof window.requestAnimationFrame === 'function')
397
408
{
398
409
return { kind: 'raf', id: window.requestAnimationFrame(fn) };
@@ -403,6 +414,11 @@
403
414
const cancelFrameRequest = (handle) =>
404
415
{
405
416
if (!handle || !Number.isFinite(Number(handle.id))) return;
417
if (handle.kind === 'video-frame' && handle.videoEl && typeof handle.videoEl.cancelVideoFrameCallback === 'function')
418
{
419
handle.videoEl.cancelVideoFrameCallback(handle.id);
420
return;
421
}
406
422
if (handle.kind === 'raf' && typeof window.cancelAnimationFrame === 'function')
407
423
{
408
424
window.cancelAnimationFrame(handle.id);
@@ -547,6 +563,7 @@
547
563
const closeObjectUrl = () =>
548
564
{
549
565
activeObjectUrl = null;
566
primaryBufferIndex = 0;
550
567
primaryFrameKey = null;
551
568
secondaryFrameKey = null;
552
569
};
@@ -554,7 +571,16 @@
554
571
const hideOverlay = () =>
555
572
{
556
573
if (overlayEl) overlayEl.style.display = 'none';
557
if (overlayImgEl) overlayImgEl.removeAttribute('src');
574
if (overlayImgEl)
575
{
576
overlayImgEl.style.display = 'none';
577
overlayImgEl.removeAttribute('src');
578
}
579
if (overlayImgElSecondary)
580
{
581
overlayImgElSecondary.style.display = 'none';
582
overlayImgElSecondary.removeAttribute('src');
583
}
558
584
closeObjectUrl();
559
585
};
560
586
@@ -669,6 +695,24 @@
669
695
return best;
670
696
};
671
697
698
const findNextCachedFrameAfterTime = (streamIndex, currentTime) =>
699
{
700
if (!Number.isInteger(streamIndex) || streamIndex < 0) return null;
701
if (!Number.isFinite(currentTime) || currentTime < 0) return null;
702
703
let best = null;
704
const threshold = currentTime + IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS;
705
for (const frame of cachedSubtitleFrames.values())
706
{
707
if (!frame || frame.streamIndex !== streamIndex) continue;
708
if (!Number.isFinite(frame.startTimeSeconds)) continue;
709
if (frame.startTimeSeconds <= threshold) continue;
710
if (!best || frame.startTimeSeconds < best.startTimeSeconds) best = frame;
711
}
712
713
return best;
714
};
715
672
716
const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>
673
717
{
674
718
if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)
@@ -769,7 +813,19 @@
769
813
showPrimaryBuffer();
770
814
};
771
815
772
const syncOverlayToPlaybackClock = () =>
816
const prewarmSecondaryBuffer = (streamIndex, currentTime, currentFrame) =>
817
{
818
if (!currentFrame || !Number.isInteger(streamIndex) || streamIndex < 0) return;
819
820
const secondaryIdx = primaryBufferIndex === 0 ? 1 : 0;
821
const secondaryKey = primaryBufferIndex === 0 ? secondaryFrameKey : primaryFrameKey;
822
const nextFrame = findNextCachedFrameAfterTime(streamIndex, currentTime);
823
if (!nextFrame || nextFrame.key === currentFrame.key || nextFrame.key === secondaryKey) return;
824
825
renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);
826
};
827
828
const syncOverlayToPlaybackClock = (frameMetadata) =>
773
829
{
774
830
if (selectedStreamIndex === null || trackDisabled)
775
831
{
@@ -777,7 +833,10 @@
777
833
return;
778
834
}
779
835
780
const currentTime = Number(player.currentTime());
836
const metadataTime = Number(frameMetadata && frameMetadata.mediaTime);
837
const currentTime = Number.isFinite(metadataTime) && metadataTime >= 0
838
? metadataTime
839
: Number(player.currentTime());
781
840
if (Number.isFinite(currentTime) && currentTime >= 0)
782
841
{
783
842
const cachedFrame = findCachedFrameForTime(selectedStreamIndex, currentTime);
@@ -806,6 +865,8 @@
806
865
activeObjectUrl = cachedFrame.objectUrl;
807
866
}
808
867
}
868
869
prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);
809
870
} else
810
871
{
811
872
hideOverlay();
@@ -16,7 +16,7 @@16
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_DEFAULT = 0.5;16
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_DEFAULT = 0.5;17
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MIN = 0.25;17
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MIN = 0.25;18
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MAX = 30;18
const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MAX = 30;19
const IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS = 0.05;19
const IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS = 0.005;20
const IMAGE_SUBTITLE_CACHE_MISS_FALLBACK_INTERVAL_MS_DEFAULT = 2000;20
const IMAGE_SUBTITLE_CACHE_MISS_FALLBACK_INTERVAL_MS_DEFAULT = 2000;21
const IMAGE_SUBTITLE_CURRENT_REQUEST_TIMEOUT_MS_DEFAULT = 4000;21
const IMAGE_SUBTITLE_CURRENT_REQUEST_TIMEOUT_MS_DEFAULT = 4000;22
const IMAGE_SUBTITLE_PREFETCH_REQUEST_TIMEOUT_MS_DEFAULT = 4000;22
const IMAGE_SUBTITLE_PREFETCH_REQUEST_TIMEOUT_MS_DEFAULT = 4000;@@ -393,6 +393,17 @@393
393
394
const requestFrame = (fn) =>394
const requestFrame = (fn) =>395
{395
{396
const playerEl = player.el && player.el();397
const videoEl = playerEl ? playerEl.querySelector('video') : null;398
if (videoEl && typeof videoEl.requestVideoFrameCallback === 'function')399
{400
const id = videoEl.requestVideoFrameCallback((_, metadata) =>401
{402
fn(metadata);403
});404
return { kind: 'video-frame', id, videoEl };405
}406
396
if (typeof window.requestAnimationFrame === 'function')407
if (typeof window.requestAnimationFrame === 'function')397
{408
{398
return { kind: 'raf', id: window.requestAnimationFrame(fn) };409
return { kind: 'raf', id: window.requestAnimationFrame(fn) };@@ -403,6 +414,11 @@403
const cancelFrameRequest = (handle) =>414
const cancelFrameRequest = (handle) =>404
{415
{405
if (!handle || !Number.isFinite(Number(handle.id))) return;416
if (!handle || !Number.isFinite(Number(handle.id))) return;417
if (handle.kind === 'video-frame' && handle.videoEl && typeof handle.videoEl.cancelVideoFrameCallback === 'function')418
{419
handle.videoEl.cancelVideoFrameCallback(handle.id);420
return;421
}406
if (handle.kind === 'raf' && typeof window.cancelAnimationFrame === 'function')422
if (handle.kind === 'raf' && typeof window.cancelAnimationFrame === 'function')407
{423
{408
window.cancelAnimationFrame(handle.id);424
window.cancelAnimationFrame(handle.id);@@ -547,6 +563,7 @@547
const closeObjectUrl = () =>563
const closeObjectUrl = () =>548
{564
{549
activeObjectUrl = null;565
activeObjectUrl = null;566
primaryBufferIndex = 0;550
primaryFrameKey = null;567
primaryFrameKey = null;551
secondaryFrameKey = null;568
secondaryFrameKey = null;552
};569
};@@ -554,7 +571,16 @@554
const hideOverlay = () =>571
const hideOverlay = () =>555
{572
{556
if (overlayEl) overlayEl.style.display = 'none';573
if (overlayEl) overlayEl.style.display = 'none';557
if (overlayImgEl) overlayImgEl.removeAttribute('src');574
if (overlayImgEl)575
{576
overlayImgEl.style.display = 'none';577
overlayImgEl.removeAttribute('src');578
}579
if (overlayImgElSecondary)580
{581
overlayImgElSecondary.style.display = 'none';582
overlayImgElSecondary.removeAttribute('src');583
}558
closeObjectUrl();584
closeObjectUrl();559
};585
};560
586
@@ -669,6 +695,24 @@669
return best;695
return best;670
};696
};671
697
698
const findNextCachedFrameAfterTime = (streamIndex, currentTime) =>699
{700
if (!Number.isInteger(streamIndex) || streamIndex < 0) return null;701
if (!Number.isFinite(currentTime) || currentTime < 0) return null;702
703
let best = null;704
const threshold = currentTime + IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS;705
for (const frame of cachedSubtitleFrames.values())706
{707
if (!frame || frame.streamIndex !== streamIndex) continue;708
if (!Number.isFinite(frame.startTimeSeconds)) continue;709
if (frame.startTimeSeconds <= threshold) continue;710
if (!best || frame.startTimeSeconds < best.startTimeSeconds) best = frame;711
}712
713
return best;714
};715
672
const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>716
const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>673
{717
{674
if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)718
if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)@@ -769,7 +813,19 @@769
showPrimaryBuffer();813
showPrimaryBuffer();770
};814
};771
815
772
const syncOverlayToPlaybackClock = () =>816
const prewarmSecondaryBuffer = (streamIndex, currentTime, currentFrame) =>817
{818
if (!currentFrame || !Number.isInteger(streamIndex) || streamIndex < 0) return;819
820
const secondaryIdx = primaryBufferIndex === 0 ? 1 : 0;821
const secondaryKey = primaryBufferIndex === 0 ? secondaryFrameKey : primaryFrameKey;822
const nextFrame = findNextCachedFrameAfterTime(streamIndex, currentTime);823
if (!nextFrame || nextFrame.key === currentFrame.key || nextFrame.key === secondaryKey) return;824
825
renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);826
};827
828
const syncOverlayToPlaybackClock = (frameMetadata) =>773
{829
{774
if (selectedStreamIndex === null || trackDisabled)830
if (selectedStreamIndex === null || trackDisabled)775
{831
{@@ -777,7 +833,10 @@777
return;833
return;778
}834
}779
835
780
const currentTime = Number(player.currentTime());836
const metadataTime = Number(frameMetadata && frameMetadata.mediaTime);837
const currentTime = Number.isFinite(metadataTime) && metadataTime >= 0838
? metadataTime839
: Number(player.currentTime());781
if (Number.isFinite(currentTime) && currentTime >= 0)840
if (Number.isFinite(currentTime) && currentTime >= 0)782
{841
{783
const cachedFrame = findCachedFrameForTime(selectedStreamIndex, currentTime);842
const cachedFrame = findCachedFrameForTime(selectedStreamIndex, currentTime);@@ -806,6 +865,8 @@806
activeObjectUrl = cachedFrame.objectUrl;865
activeObjectUrl = cachedFrame.objectUrl;807
}866
}808
}867
}868
869
prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);809
} else870
} else810
{871
{811
hideOverlay();872
hideOverlay();@@ -785,6 +785,76 @@ describe('shared image subtitle prefetching', () =>
785
785
window.close();
786
786
});
787
787
788
test('uses video-frame media clock for precise cached caption timing', async () =>
789
{
790
const window = createWindow();
791
const helper = loadSharedVideoPlayer(window);
792
const { player, videojs } = buildPlayerHarness(window);
793
794
let frameCallback = null;
795
let frameCallbackId = 0;
796
const videoEl = window.document.createElement('video');
797
videoEl.requestVideoFrameCallback = jest.fn((cb) =>
798
{
799
frameCallback = cb;
800
frameCallbackId += 1;
801
return frameCallbackId;
802
});
803
videoEl.cancelVideoFrameCallback = jest.fn();
804
player.el().appendChild(videoEl);
805
806
window.fetch = jest.fn(async () =>
807
{
808
return new Response(new Uint8Array([137, 80, 78, 71]), {
809
status: 200,
810
headers: {
811
'Content-Type': 'image/png',
812
'X-Subtitle-Frame-Start-Time': '1',
813
'X-Subtitle-Frame-End-Time': '2',
814
'X-Subtitle-Frame-Width': '640',
815
'X-Subtitle-Frame-Height': '80',
816
'X-Subtitle-Frame-Canvas-Width': '1920',
817
'X-Subtitle-Frame-Canvas-Height': '1080',
818
},
819
});
820
});
821
822
const controller = helper.bindImageSubtitleOverlay(player, videojs, {}, {
823
imageSubtitleTracks: [{ label: 'PGS English', streamIndex: 0, default: true }],
824
}, {
825
lookaheadSeconds: 0,
826
initialPrefetchSeconds: 0,
827
currentFetchEnabled: false,
828
cacheMissFallbackEnabled: false,
829
buildFrameRequest: ({ currentTime, streamIndex }) => ({
830
url: `/subtitle-frame?streamIndex=${streamIndex}&timeSeconds=${currentTime}`,
831
}),
832
});
833
834
await flushAsyncWork(window, 8);
835
836
const overlay = player.el().querySelector('.video-subtitle-overlay');
837
expect(overlay).toBeTruthy();
838
expect(videoEl.requestVideoFrameCallback).toHaveBeenCalled();
839
expect(typeof frameCallback).toBe('function');
840
841
player.setCurrentTime(9);
842
player.trigger('play');
843
844
frameCallback(1000, { mediaTime: 1.5 });
845
await flushAsyncWork(window, 3);
846
expect(overlay.style.display).toBe('block');
847
848
const nextFrameCallback = frameCallback;
849
nextFrameCallback(1016, { mediaTime: 2.2 });
850
await flushAsyncWork(window, 3);
851
expect(overlay.style.display).toBe('none');
852
853
controller.detach();
854
await flushAsyncWork(window, 2);
855
window.close();
856
});
857
788
858
test('aborts in-flight prefetch requests when the overlay is detached', async () =>
789
859
{
790
860
const window = createWindow();
@@ -785,6 +785,76 @@ describe('shared image subtitle prefetching', () =>785
window.close();785
window.close();786
});786
});787
787
788
test('uses video-frame media clock for precise cached caption timing', async () =>789
{790
const window = createWindow();791
const helper = loadSharedVideoPlayer(window);792
const { player, videojs } = buildPlayerHarness(window);793
794
let frameCallback = null;795
let frameCallbackId = 0;796
const videoEl = window.document.createElement('video');797
videoEl.requestVideoFrameCallback = jest.fn((cb) =>798
{799
frameCallback = cb;800
frameCallbackId += 1;801
return frameCallbackId;802
});803
videoEl.cancelVideoFrameCallback = jest.fn();804
player.el().appendChild(videoEl);805
806
window.fetch = jest.fn(async () =>807
{808
return new Response(new Uint8Array([137, 80, 78, 71]), {809
status: 200,810
headers: {811
'Content-Type': 'image/png',812
'X-Subtitle-Frame-Start-Time': '1',813
'X-Subtitle-Frame-End-Time': '2',814
'X-Subtitle-Frame-Width': '640',815
'X-Subtitle-Frame-Height': '80',816
'X-Subtitle-Frame-Canvas-Width': '1920',817
'X-Subtitle-Frame-Canvas-Height': '1080',818
},819
});820
});821
822
const controller = helper.bindImageSubtitleOverlay(player, videojs, {}, {823
imageSubtitleTracks: [{ label: 'PGS English', streamIndex: 0, default: true }],824
}, {825
lookaheadSeconds: 0,826
initialPrefetchSeconds: 0,827
currentFetchEnabled: false,828
cacheMissFallbackEnabled: false,829
buildFrameRequest: ({ currentTime, streamIndex }) => ({830
url: `/subtitle-frame?streamIndex=${streamIndex}&timeSeconds=${currentTime}`,831
}),832
});833
834
await flushAsyncWork(window, 8);835
836
const overlay = player.el().querySelector('.video-subtitle-overlay');837
expect(overlay).toBeTruthy();838
expect(videoEl.requestVideoFrameCallback).toHaveBeenCalled();839
expect(typeof frameCallback).toBe('function');840
841
player.setCurrentTime(9);842
player.trigger('play');843
844
frameCallback(1000, { mediaTime: 1.5 });845
await flushAsyncWork(window, 3);846
expect(overlay.style.display).toBe('block');847
848
const nextFrameCallback = frameCallback;849
nextFrameCallback(1016, { mediaTime: 2.2 });850
await flushAsyncWork(window, 3);851
expect(overlay.style.display).toBe('none');852
853
controller.detach();854
await flushAsyncWork(window, 2);855
window.close();856
});857
788
test('aborts in-flight prefetch requests when the overlay is detached', async () =>858
test('aborts in-flight prefetch requests when the overlay is detached', async () =>789
{859
{790
const window = createWindow();860
const window = createWindow();