Add cue-boundary subtitle scheduler
Subtitle overlay timing now wakes on the next cue boundary in addition to the media-frame sync loop, which improves precision for densely packed captions and keeps the active caption aligned when the playhead lands near a start or end boundary. Also resyncs immediately on seeked and ratechange so pausing, jumping, or changing playback speed still snaps to the correct cached subtitle.
timfa Tim Falken · pushed by timfa
Authored 2026-05-29 · 5 months ago
- Hash
d3ee4c42fbe877339eca6afe7c19b55596868c6d- Parent
aa3efeba
2 files changed
+140
−0
@@ -389,6 +389,7 @@
389
389
let subtitleLogFlushTimer = null;
390
390
let lastCacheMissFallbackAtMs = 0;
391
391
let playbackClockSyncHandle = null;
392
let cueBoundarySyncHandle = null;
392
393
const cachedSubtitleFrames = new Map();
393
394
394
395
const requestFrame = (fn) =>
@@ -558,6 +559,14 @@
558
559
if (!playbackClockSyncHandle) return;
559
560
cancelFrameRequest(playbackClockSyncHandle);
560
561
playbackClockSyncHandle = null;
562
stopCueBoundarySync();
563
};
564
565
const stopCueBoundarySync = () =>
566
{
567
if (!cueBoundarySyncHandle) return;
568
window.clearTimeout(cueBoundarySyncHandle);
569
cueBoundarySyncHandle = null;
561
570
};
562
571
563
572
const closeObjectUrl = () =>
@@ -584,6 +593,12 @@
584
593
closeObjectUrl();
585
594
};
586
595
596
const getPlaybackRate = () =>
597
{
598
const rate = Number(player.playbackRate && player.playbackRate());
599
return Number.isFinite(rate) && rate > 0 ? rate : 1;
600
};
601
587
602
const revokeCachedFrameUrls = () =>
588
603
{
589
604
for (const frame of cachedSubtitleFrames.values())
@@ -713,6 +728,29 @@
713
728
return best;
714
729
};
715
730
731
const getNextCueBoundaryTimeSeconds = (streamIndex, currentTime, currentFrame = null) =>
732
{
733
if (!Number.isInteger(streamIndex) || streamIndex < 0) return null;
734
if (!Number.isFinite(currentTime) || currentTime < 0) return null;
735
736
const activeFrame = currentFrame || findCachedFrameForTime(streamIndex, currentTime);
737
const nextFrameSearchTime = activeFrame && Number.isFinite(activeFrame.startTimeSeconds) && activeFrame.startTimeSeconds > currentTime
738
? activeFrame.startTimeSeconds
739
: currentTime;
740
const nextFrame = findNextCachedFrameAfterTime(streamIndex, nextFrameSearchTime);
741
742
let boundaryTime = null;
743
if (activeFrame && Number.isFinite(activeFrame.endTimeSeconds)) boundaryTime = activeFrame.endTimeSeconds;
744
if (nextFrame && Number.isFinite(nextFrame.startTimeSeconds))
745
{
746
boundaryTime = Number.isFinite(boundaryTime)
747
? Math.min(boundaryTime, nextFrame.startTimeSeconds)
748
: nextFrame.startTimeSeconds;
749
}
750
751
return boundaryTime;
752
};
753
716
754
const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>
717
755
{
718
756
if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)
@@ -828,8 +866,26 @@
828
866
renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);
829
867
};
830
868
869
const scheduleCueBoundarySync = (streamIndex, currentTime, currentFrame = null) =>
870
{
871
stopCueBoundarySync();
872
if (selectedStreamIndex === null || trackDisabled) return;
873
if (typeof player.paused === 'function' && player.paused()) return;
874
875
const boundaryTime = getNextCueBoundaryTimeSeconds(streamIndex, currentTime, currentFrame);
876
if (!Number.isFinite(boundaryTime) || boundaryTime <= currentTime) return;
877
878
const delayMs = Math.max(1, Math.round(((boundaryTime - currentTime) / getPlaybackRate()) * 1000));
879
cueBoundarySyncHandle = window.setTimeout(() =>
880
{
881
cueBoundarySyncHandle = null;
882
syncOverlayToPlaybackClock();
883
}, delayMs);
884
};
885
831
886
const syncOverlayToPlaybackClock = (frameMetadata) =>
832
887
{
888
stopCueBoundarySync();
833
889
if (selectedStreamIndex === null || trackDisabled)
834
890
{
835
891
stopPlaybackClockSync();
@@ -870,9 +926,11 @@
870
926
}
871
927
872
928
prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);
929
scheduleCueBoundarySync(selectedStreamIndex, currentTime, cachedFrame);
873
930
} else
874
931
{
875
932
hideOverlay();
933
scheduleCueBoundarySync(selectedStreamIndex, currentTime, null);
876
934
}
877
935
}
878
936
@@ -1675,6 +1733,7 @@
1675
1733
const dispose = () =>
1676
1734
{
1677
1735
stopPlaybackClockSync();
1736
stopCueBoundarySync();
1678
1737
resetRequestState();
1679
1738
flushSubtitleLogs();
1680
1739
closeMenu();
@@ -1859,10 +1918,12 @@
1859
1918
1860
1919
player.on('timeupdate', () => { void refreshFrame(); });
1861
1920
player.on('seeking', () => { void refreshFrame(true); });
1921
player.on('seeked', () => { void refreshFrame(true); });
1862
1922
player.on('play', () => { void refreshFrame(true); });
1863
1923
player.on('play', ensurePlaybackClockSync);
1864
1924
player.on('pause', stopPlaybackClockSync);
1865
1925
player.on('ended', stopPlaybackClockSync);
1926
player.on('ratechange', () => { void refreshFrame(true); });
1866
1927
player.on('fullscreenchange', () => { closeMenu(); void refreshFrame(true); });
1867
1928
player.on('dispose', dispose);
1868
1929
player.ready(() =>
@@ -389,6 +389,7 @@389
let subtitleLogFlushTimer = null;389
let subtitleLogFlushTimer = null;390
let lastCacheMissFallbackAtMs = 0;390
let lastCacheMissFallbackAtMs = 0;391
let playbackClockSyncHandle = null;391
let playbackClockSyncHandle = null;392
let cueBoundarySyncHandle = null;392
const cachedSubtitleFrames = new Map();393
const cachedSubtitleFrames = new Map();393
394
394
const requestFrame = (fn) =>395
const requestFrame = (fn) =>@@ -558,6 +559,14 @@558
if (!playbackClockSyncHandle) return;559
if (!playbackClockSyncHandle) return;559
cancelFrameRequest(playbackClockSyncHandle);560
cancelFrameRequest(playbackClockSyncHandle);560
playbackClockSyncHandle = null;561
playbackClockSyncHandle = null;562
stopCueBoundarySync();563
};564
565
const stopCueBoundarySync = () =>566
{567
if (!cueBoundarySyncHandle) return;568
window.clearTimeout(cueBoundarySyncHandle);569
cueBoundarySyncHandle = null;561
};570
};562
571
563
const closeObjectUrl = () =>572
const closeObjectUrl = () =>@@ -584,6 +593,12 @@584
closeObjectUrl();593
closeObjectUrl();585
};594
};586
595
596
const getPlaybackRate = () =>597
{598
const rate = Number(player.playbackRate && player.playbackRate());599
return Number.isFinite(rate) && rate > 0 ? rate : 1;600
};601
587
const revokeCachedFrameUrls = () =>602
const revokeCachedFrameUrls = () =>588
{603
{589
for (const frame of cachedSubtitleFrames.values())604
for (const frame of cachedSubtitleFrames.values())@@ -713,6 +728,29 @@713
return best;728
return best;714
};729
};715
730
731
const getNextCueBoundaryTimeSeconds = (streamIndex, currentTime, currentFrame = null) =>732
{733
if (!Number.isInteger(streamIndex) || streamIndex < 0) return null;734
if (!Number.isFinite(currentTime) || currentTime < 0) return null;735
736
const activeFrame = currentFrame || findCachedFrameForTime(streamIndex, currentTime);737
const nextFrameSearchTime = activeFrame && Number.isFinite(activeFrame.startTimeSeconds) && activeFrame.startTimeSeconds > currentTime738
? activeFrame.startTimeSeconds739
: currentTime;740
const nextFrame = findNextCachedFrameAfterTime(streamIndex, nextFrameSearchTime);741
742
let boundaryTime = null;743
if (activeFrame && Number.isFinite(activeFrame.endTimeSeconds)) boundaryTime = activeFrame.endTimeSeconds;744
if (nextFrame && Number.isFinite(nextFrame.startTimeSeconds))745
{746
boundaryTime = Number.isFinite(boundaryTime)747
? Math.min(boundaryTime, nextFrame.startTimeSeconds)748
: nextFrame.startTimeSeconds;749
}750
751
return boundaryTime;752
};753
716
const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>754
const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>717
{755
{718
if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)756
if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)@@ -828,8 +866,26 @@828
renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);866
renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);829
};867
};830
868
869
const scheduleCueBoundarySync = (streamIndex, currentTime, currentFrame = null) =>870
{871
stopCueBoundarySync();872
if (selectedStreamIndex === null || trackDisabled) return;873
if (typeof player.paused === 'function' && player.paused()) return;874
875
const boundaryTime = getNextCueBoundaryTimeSeconds(streamIndex, currentTime, currentFrame);876
if (!Number.isFinite(boundaryTime) || boundaryTime <= currentTime) return;877
878
const delayMs = Math.max(1, Math.round(((boundaryTime - currentTime) / getPlaybackRate()) * 1000));879
cueBoundarySyncHandle = window.setTimeout(() =>880
{881
cueBoundarySyncHandle = null;882
syncOverlayToPlaybackClock();883
}, delayMs);884
};885
831
const syncOverlayToPlaybackClock = (frameMetadata) =>886
const syncOverlayToPlaybackClock = (frameMetadata) =>832
{887
{888
stopCueBoundarySync();833
if (selectedStreamIndex === null || trackDisabled)889
if (selectedStreamIndex === null || trackDisabled)834
{890
{835
stopPlaybackClockSync();891
stopPlaybackClockSync();@@ -870,9 +926,11 @@870
}926
}871
927
872
prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);928
prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);929
scheduleCueBoundarySync(selectedStreamIndex, currentTime, cachedFrame);873
} else930
} else874
{931
{875
hideOverlay();932
hideOverlay();933
scheduleCueBoundarySync(selectedStreamIndex, currentTime, null);876
}934
}877
}935
}878
936
@@ -1675,6 +1733,7 @@1675
const dispose = () =>1733
const dispose = () =>1676
{1734
{1677
stopPlaybackClockSync();1735
stopPlaybackClockSync();1736
stopCueBoundarySync();1678
resetRequestState();1737
resetRequestState();1679
flushSubtitleLogs();1738
flushSubtitleLogs();1680
closeMenu();1739
closeMenu();@@ -1859,10 +1918,12 @@1859
1918
1860
player.on('timeupdate', () => { void refreshFrame(); });1919
player.on('timeupdate', () => { void refreshFrame(); });1861
player.on('seeking', () => { void refreshFrame(true); });1920
player.on('seeking', () => { void refreshFrame(true); });1921
player.on('seeked', () => { void refreshFrame(true); });1862
player.on('play', () => { void refreshFrame(true); });1922
player.on('play', () => { void refreshFrame(true); });1863
player.on('play', ensurePlaybackClockSync);1923
player.on('play', ensurePlaybackClockSync);1864
player.on('pause', stopPlaybackClockSync);1924
player.on('pause', stopPlaybackClockSync);1865
player.on('ended', stopPlaybackClockSync);1925
player.on('ended', stopPlaybackClockSync);1926
player.on('ratechange', () => { void refreshFrame(true); });1866
player.on('fullscreenchange', () => { closeMenu(); void refreshFrame(true); });1927
player.on('fullscreenchange', () => { closeMenu(); void refreshFrame(true); });1867
player.on('dispose', dispose);1928
player.on('dispose', dispose);1868
player.ready(() =>1929
player.ready(() =>@@ -923,6 +923,85 @@ describe('shared image subtitle prefetching', () =>
923
923
window.close();
924
924
});
925
925
926
test('resyncs the displayed caption after a seek without waiting for timeupdate', async () =>
927
{
928
const window = createWindow();
929
const helper = loadSharedVideoPlayer(window);
930
const { player, videojs } = buildPlayerHarness(window);
931
932
window.requestAnimationFrame = undefined;
933
window.cancelAnimationFrame = undefined;
934
935
window.fetch = jest.fn(async (url) =>
936
{
937
const parsed = new URL(url, 'http://localhost/');
938
const timeSeconds = Number(parsed.searchParams.get('timeSeconds'));
939
const headers = {
940
'Content-Type': 'image/png',
941
'X-Subtitle-Frame-Width': '640',
942
'X-Subtitle-Frame-Height': '80',
943
'X-Subtitle-Frame-Canvas-Width': '1920',
944
'X-Subtitle-Frame-Canvas-Height': '1080',
945
};
946
947
if (Math.abs(timeSeconds - 1.2) < 0.001)
948
{
949
headers['X-Subtitle-Frame-Start-Time'] = '1.0';
950
headers['X-Subtitle-Frame-End-Time'] = '2.0';
951
} else if (Math.abs(timeSeconds - 3.2) < 0.001)
952
{
953
headers['X-Subtitle-Frame-Start-Time'] = '3.0';
954
headers['X-Subtitle-Frame-End-Time'] = '4.0';
955
} else
956
{
957
return new Response(null, {
958
status: 204,
959
headers: { 'X-Subtitle-Frame-Status': 'none' },
960
});
961
}
962
963
return new Response(new Uint8Array([137, 80, 78, 71]), {
964
status: 200,
965
headers,
966
});
967
});
968
969
const controller = helper.bindImageSubtitleOverlay(player, videojs, {}, {
970
imageSubtitleTracks: [{ label: 'PGS English', streamIndex: 0, default: true }],
971
}, {
972
currentFetchEnabled: false,
973
cacheMissFallbackEnabled: false,
974
lookaheadSeconds: 0,
975
initialPrefetchSeconds: 0,
976
buildFrameRequest: ({ currentTime, streamIndex }) => ({
977
url: `/subtitle-frame?streamIndex=${streamIndex}&timeSeconds=${currentTime}`,
978
}),
979
});
980
981
await flushAsyncWork(window, 8);
982
983
const overlay = player.el().querySelector('.video-subtitle-overlay');
984
expect(overlay).toBeTruthy();
985
986
player.setCurrentTime(1.2);
987
player.trigger('play');
988
await new Promise((resolve) => window.setTimeout(resolve, 50));
989
await flushAsyncWork(window, 2);
990
expect(overlay.style.display).toBe('block');
991
992
player.trigger('pause');
993
player.setCurrentTime(3.2);
994
player.trigger('seeked');
995
await flushAsyncWork(window, 6);
996
997
expect(overlay.style.display).toBe('block');
998
expect(overlay.querySelector('img').getAttribute('src')).toBe('blob:subtitle');
999
1000
controller.detach();
1001
await flushAsyncWork(window, 2);
1002
window.close();
1003
});
1004
926
1005
test('aborts in-flight prefetch requests when the overlay is detached', async () =>
927
1006
{
928
1007
const window = createWindow();
@@ -923,6 +923,85 @@ describe('shared image subtitle prefetching', () =>923
window.close();923
window.close();924
});924
});925
925
926
test('resyncs the displayed caption after a seek without waiting for timeupdate', async () =>927
{928
const window = createWindow();929
const helper = loadSharedVideoPlayer(window);930
const { player, videojs } = buildPlayerHarness(window);931
932
window.requestAnimationFrame = undefined;933
window.cancelAnimationFrame = undefined;934
935
window.fetch = jest.fn(async (url) =>936
{937
const parsed = new URL(url, 'http://localhost/');938
const timeSeconds = Number(parsed.searchParams.get('timeSeconds'));939
const headers = {940
'Content-Type': 'image/png',941
'X-Subtitle-Frame-Width': '640',942
'X-Subtitle-Frame-Height': '80',943
'X-Subtitle-Frame-Canvas-Width': '1920',944
'X-Subtitle-Frame-Canvas-Height': '1080',945
};946
947
if (Math.abs(timeSeconds - 1.2) < 0.001)948
{949
headers['X-Subtitle-Frame-Start-Time'] = '1.0';950
headers['X-Subtitle-Frame-End-Time'] = '2.0';951
} else if (Math.abs(timeSeconds - 3.2) < 0.001)952
{953
headers['X-Subtitle-Frame-Start-Time'] = '3.0';954
headers['X-Subtitle-Frame-End-Time'] = '4.0';955
} else956
{957
return new Response(null, {958
status: 204,959
headers: { 'X-Subtitle-Frame-Status': 'none' },960
});961
}962
963
return new Response(new Uint8Array([137, 80, 78, 71]), {964
status: 200,965
headers,966
});967
});968
969
const controller = helper.bindImageSubtitleOverlay(player, videojs, {}, {970
imageSubtitleTracks: [{ label: 'PGS English', streamIndex: 0, default: true }],971
}, {972
currentFetchEnabled: false,973
cacheMissFallbackEnabled: false,974
lookaheadSeconds: 0,975
initialPrefetchSeconds: 0,976
buildFrameRequest: ({ currentTime, streamIndex }) => ({977
url: `/subtitle-frame?streamIndex=${streamIndex}&timeSeconds=${currentTime}`,978
}),979
});980
981
await flushAsyncWork(window, 8);982
983
const overlay = player.el().querySelector('.video-subtitle-overlay');984
expect(overlay).toBeTruthy();985
986
player.setCurrentTime(1.2);987
player.trigger('play');988
await new Promise((resolve) => window.setTimeout(resolve, 50));989
await flushAsyncWork(window, 2);990
expect(overlay.style.display).toBe('block');991
992
player.trigger('pause');993
player.setCurrentTime(3.2);994
player.trigger('seeked');995
await flushAsyncWork(window, 6);996
997
expect(overlay.style.display).toBe('block');998
expect(overlay.querySelector('img').getAttribute('src')).toBe('blob:subtitle');999
1000
controller.detach();1001
await flushAsyncWork(window, 2);1002
window.close();1003
});1004
926
test('aborts in-flight prefetch requests when the overlay is detached', async () =>1005
test('aborts in-flight prefetch requests when the overlay is detached', async () =>927
{1006
{928
const window = createWindow();1007
const window = createWindow();