BlueGit

timfa filemanager

Add cue-boundary subtitle scheduler

Subtitle overlay timing now wakes on the next cue boundary in addition to the media-frame sync loop, which improves precision for densely packed captions and keeps the active caption aligned when the playhead lands near a start or end boundary.

Also resyncs immediately on seeked and ratechange so pausing, jumping, or changing playback speed still snaps to the correct cached subtitle.

timfa Tim Falken · pushed by timfa

Authored 2026-05-29 · 5 months ago

Hash
d3ee4c42fbe877339eca6afe7c19b55596868c6d
Parent
aa3efeba
2 files changed +140 −0
public/shared-video-player.js +61 −0
@@ -389,6 +389,7 @@
389 389 let subtitleLogFlushTimer = null;
390 390 let lastCacheMissFallbackAtMs = 0;
391 391 let playbackClockSyncHandle = null;
392 let cueBoundarySyncHandle = null;
392 393 const cachedSubtitleFrames = new Map();
393 394
394 395 const requestFrame = (fn) =>
@@ -558,6 +559,14 @@
558 559 if (!playbackClockSyncHandle) return;
559 560 cancelFrameRequest(playbackClockSyncHandle);
560 561 playbackClockSyncHandle = null;
562 stopCueBoundarySync();
563 };
564
565 const stopCueBoundarySync = () =>
566 {
567 if (!cueBoundarySyncHandle) return;
568 window.clearTimeout(cueBoundarySyncHandle);
569 cueBoundarySyncHandle = null;
561 570 };
562 571
563 572 const closeObjectUrl = () =>
@@ -584,6 +593,12 @@
584 593 closeObjectUrl();
585 594 };
586 595
596 const getPlaybackRate = () =>
597 {
598 const rate = Number(player.playbackRate && player.playbackRate());
599 return Number.isFinite(rate) && rate > 0 ? rate : 1;
600 };
601
587 602 const revokeCachedFrameUrls = () =>
588 603 {
589 604 for (const frame of cachedSubtitleFrames.values())
@@ -713,6 +728,29 @@
713 728 return best;
714 729 };
715 730
731 const getNextCueBoundaryTimeSeconds = (streamIndex, currentTime, currentFrame = null) =>
732 {
733 if (!Number.isInteger(streamIndex) || streamIndex < 0) return null;
734 if (!Number.isFinite(currentTime) || currentTime < 0) return null;
735
736 const activeFrame = currentFrame || findCachedFrameForTime(streamIndex, currentTime);
737 const nextFrameSearchTime = activeFrame && Number.isFinite(activeFrame.startTimeSeconds) && activeFrame.startTimeSeconds > currentTime
738 ? activeFrame.startTimeSeconds
739 : currentTime;
740 const nextFrame = findNextCachedFrameAfterTime(streamIndex, nextFrameSearchTime);
741
742 let boundaryTime = null;
743 if (activeFrame && Number.isFinite(activeFrame.endTimeSeconds)) boundaryTime = activeFrame.endTimeSeconds;
744 if (nextFrame && Number.isFinite(nextFrame.startTimeSeconds))
745 {
746 boundaryTime = Number.isFinite(boundaryTime)
747 ? Math.min(boundaryTime, nextFrame.startTimeSeconds)
748 : nextFrame.startTimeSeconds;
749 }
750
751 return boundaryTime;
752 };
753
716 754 const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>
717 755 {
718 756 if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)
@@ -828,8 +866,26 @@
828 866 renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);
829 867 };
830 868
869 const scheduleCueBoundarySync = (streamIndex, currentTime, currentFrame = null) =>
870 {
871 stopCueBoundarySync();
872 if (selectedStreamIndex === null || trackDisabled) return;
873 if (typeof player.paused === 'function' && player.paused()) return;
874
875 const boundaryTime = getNextCueBoundaryTimeSeconds(streamIndex, currentTime, currentFrame);
876 if (!Number.isFinite(boundaryTime) || boundaryTime <= currentTime) return;
877
878 const delayMs = Math.max(1, Math.round(((boundaryTime - currentTime) / getPlaybackRate()) * 1000));
879 cueBoundarySyncHandle = window.setTimeout(() =>
880 {
881 cueBoundarySyncHandle = null;
882 syncOverlayToPlaybackClock();
883 }, delayMs);
884 };
885
831 886 const syncOverlayToPlaybackClock = (frameMetadata) =>
832 887 {
888 stopCueBoundarySync();
833 889 if (selectedStreamIndex === null || trackDisabled)
834 890 {
835 891 stopPlaybackClockSync();
@@ -870,9 +926,11 @@
870 926 }
871 927
872 928 prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);
929 scheduleCueBoundarySync(selectedStreamIndex, currentTime, cachedFrame);
873 930 } else
874 931 {
875 932 hideOverlay();
933 scheduleCueBoundarySync(selectedStreamIndex, currentTime, null);
876 934 }
877 935 }
878 936
@@ -1675,6 +1733,7 @@
1675 1733 const dispose = () =>
1676 1734 {
1677 1735 stopPlaybackClockSync();
1736 stopCueBoundarySync();
1678 1737 resetRequestState();
1679 1738 flushSubtitleLogs();
1680 1739 closeMenu();
@@ -1859,10 +1918,12 @@
1859 1918
1860 1919 player.on('timeupdate', () => { void refreshFrame(); });
1861 1920 player.on('seeking', () => { void refreshFrame(true); });
1921 player.on('seeked', () => { void refreshFrame(true); });
1862 1922 player.on('play', () => { void refreshFrame(true); });
1863 1923 player.on('play', ensurePlaybackClockSync);
1864 1924 player.on('pause', stopPlaybackClockSync);
1865 1925 player.on('ended', stopPlaybackClockSync);
1926 player.on('ratechange', () => { void refreshFrame(true); });
1866 1927 player.on('fullscreenchange', () => { closeMenu(); void refreshFrame(true); });
1867 1928 player.on('dispose', dispose);
1868 1929 player.ready(() =>
@@ -389,6 +389,7 @@
389 let subtitleLogFlushTimer = null;
389 let subtitleLogFlushTimer = null;
390 let lastCacheMissFallbackAtMs = 0;
390 let lastCacheMissFallbackAtMs = 0;
391 let playbackClockSyncHandle = null;
391 let playbackClockSyncHandle = null;
392 let cueBoundarySyncHandle = null;
392 const cachedSubtitleFrames = new Map();
393 const cachedSubtitleFrames = new Map();
393
394
394 const requestFrame = (fn) =>
395 const requestFrame = (fn) =>
@@ -558,6 +559,14 @@
558 if (!playbackClockSyncHandle) return;
559 if (!playbackClockSyncHandle) return;
559 cancelFrameRequest(playbackClockSyncHandle);
560 cancelFrameRequest(playbackClockSyncHandle);
560 playbackClockSyncHandle = null;
561 playbackClockSyncHandle = null;
562 stopCueBoundarySync();
563 };
564
565 const stopCueBoundarySync = () =>
566 {
567 if (!cueBoundarySyncHandle) return;
568 window.clearTimeout(cueBoundarySyncHandle);
569 cueBoundarySyncHandle = null;
561 };
570 };
562
571
563 const closeObjectUrl = () =>
572 const closeObjectUrl = () =>
@@ -584,6 +593,12 @@
584 closeObjectUrl();
593 closeObjectUrl();
585 };
594 };
586
595
596 const getPlaybackRate = () =>
597 {
598 const rate = Number(player.playbackRate && player.playbackRate());
599 return Number.isFinite(rate) && rate > 0 ? rate : 1;
600 };
601
587 const revokeCachedFrameUrls = () =>
602 const revokeCachedFrameUrls = () =>
588 {
603 {
589 for (const frame of cachedSubtitleFrames.values())
604 for (const frame of cachedSubtitleFrames.values())
@@ -713,6 +728,29 @@
713 return best;
728 return best;
714 };
729 };
715
730
731 const getNextCueBoundaryTimeSeconds = (streamIndex, currentTime, currentFrame = null) =>
732 {
733 if (!Number.isInteger(streamIndex) || streamIndex < 0) return null;
734 if (!Number.isFinite(currentTime) || currentTime < 0) return null;
735
736 const activeFrame = currentFrame || findCachedFrameForTime(streamIndex, currentTime);
737 const nextFrameSearchTime = activeFrame && Number.isFinite(activeFrame.startTimeSeconds) && activeFrame.startTimeSeconds > currentTime
738 ? activeFrame.startTimeSeconds
739 : currentTime;
740 const nextFrame = findNextCachedFrameAfterTime(streamIndex, nextFrameSearchTime);
741
742 let boundaryTime = null;
743 if (activeFrame && Number.isFinite(activeFrame.endTimeSeconds)) boundaryTime = activeFrame.endTimeSeconds;
744 if (nextFrame && Number.isFinite(nextFrame.startTimeSeconds))
745 {
746 boundaryTime = Number.isFinite(boundaryTime)
747 ? Math.min(boundaryTime, nextFrame.startTimeSeconds)
748 : nextFrame.startTimeSeconds;
749 }
750
751 return boundaryTime;
752 };
753
716 const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>
754 const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>
717 {
755 {
718 if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)
756 if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)
@@ -828,8 +866,26 @@
828 renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);
866 renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);
829 };
867 };
830
868
869 const scheduleCueBoundarySync = (streamIndex, currentTime, currentFrame = null) =>
870 {
871 stopCueBoundarySync();
872 if (selectedStreamIndex === null || trackDisabled) return;
873 if (typeof player.paused === 'function' && player.paused()) return;
874
875 const boundaryTime = getNextCueBoundaryTimeSeconds(streamIndex, currentTime, currentFrame);
876 if (!Number.isFinite(boundaryTime) || boundaryTime <= currentTime) return;
877
878 const delayMs = Math.max(1, Math.round(((boundaryTime - currentTime) / getPlaybackRate()) * 1000));
879 cueBoundarySyncHandle = window.setTimeout(() =>
880 {
881 cueBoundarySyncHandle = null;
882 syncOverlayToPlaybackClock();
883 }, delayMs);
884 };
885
831 const syncOverlayToPlaybackClock = (frameMetadata) =>
886 const syncOverlayToPlaybackClock = (frameMetadata) =>
832 {
887 {
888 stopCueBoundarySync();
833 if (selectedStreamIndex === null || trackDisabled)
889 if (selectedStreamIndex === null || trackDisabled)
834 {
890 {
835 stopPlaybackClockSync();
891 stopPlaybackClockSync();
@@ -870,9 +926,11 @@
870 }
926 }
871
927
872 prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);
928 prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);
929 scheduleCueBoundarySync(selectedStreamIndex, currentTime, cachedFrame);
873 } else
930 } else
874 {
931 {
875 hideOverlay();
932 hideOverlay();
933 scheduleCueBoundarySync(selectedStreamIndex, currentTime, null);
876 }
934 }
877 }
935 }
878
936
@@ -1675,6 +1733,7 @@
1675 const dispose = () =>
1733 const dispose = () =>
1676 {
1734 {
1677 stopPlaybackClockSync();
1735 stopPlaybackClockSync();
1736 stopCueBoundarySync();
1678 resetRequestState();
1737 resetRequestState();
1679 flushSubtitleLogs();
1738 flushSubtitleLogs();
1680 closeMenu();
1739 closeMenu();
@@ -1859,10 +1918,12 @@
1859
1918
1860 player.on('timeupdate', () => { void refreshFrame(); });
1919 player.on('timeupdate', () => { void refreshFrame(); });
1861 player.on('seeking', () => { void refreshFrame(true); });
1920 player.on('seeking', () => { void refreshFrame(true); });
1921 player.on('seeked', () => { void refreshFrame(true); });
1862 player.on('play', () => { void refreshFrame(true); });
1922 player.on('play', () => { void refreshFrame(true); });
1863 player.on('play', ensurePlaybackClockSync);
1923 player.on('play', ensurePlaybackClockSync);
1864 player.on('pause', stopPlaybackClockSync);
1924 player.on('pause', stopPlaybackClockSync);
1865 player.on('ended', stopPlaybackClockSync);
1925 player.on('ended', stopPlaybackClockSync);
1926 player.on('ratechange', () => { void refreshFrame(true); });
1866 player.on('fullscreenchange', () => { closeMenu(); void refreshFrame(true); });
1927 player.on('fullscreenchange', () => { closeMenu(); void refreshFrame(true); });
1867 player.on('dispose', dispose);
1928 player.on('dispose', dispose);
1868 player.ready(() =>
1929 player.ready(() =>
tests/shared-video-player.test.js +79 −0
@@ -923,6 +923,85 @@ describe('shared image subtitle prefetching', () =>
923 923 window.close();
924 924 });
925 925
926 test('resyncs the displayed caption after a seek without waiting for timeupdate', async () =>
927 {
928 const window = createWindow();
929 const helper = loadSharedVideoPlayer(window);
930 const { player, videojs } = buildPlayerHarness(window);
931
932 window.requestAnimationFrame = undefined;
933 window.cancelAnimationFrame = undefined;
934
935 window.fetch = jest.fn(async (url) =>
936 {
937 const parsed = new URL(url, 'http://localhost/');
938 const timeSeconds = Number(parsed.searchParams.get('timeSeconds'));
939 const headers = {
940 'Content-Type': 'image/png',
941 'X-Subtitle-Frame-Width': '640',
942 'X-Subtitle-Frame-Height': '80',
943 'X-Subtitle-Frame-Canvas-Width': '1920',
944 'X-Subtitle-Frame-Canvas-Height': '1080',
945 };
946
947 if (Math.abs(timeSeconds - 1.2) < 0.001)
948 {
949 headers['X-Subtitle-Frame-Start-Time'] = '1.0';
950 headers['X-Subtitle-Frame-End-Time'] = '2.0';
951 } else if (Math.abs(timeSeconds - 3.2) < 0.001)
952 {
953 headers['X-Subtitle-Frame-Start-Time'] = '3.0';
954 headers['X-Subtitle-Frame-End-Time'] = '4.0';
955 } else
956 {
957 return new Response(null, {
958 status: 204,
959 headers: { 'X-Subtitle-Frame-Status': 'none' },
960 });
961 }
962
963 return new Response(new Uint8Array([137, 80, 78, 71]), {
964 status: 200,
965 headers,
966 });
967 });
968
969 const controller = helper.bindImageSubtitleOverlay(player, videojs, {}, {
970 imageSubtitleTracks: [{ label: 'PGS English', streamIndex: 0, default: true }],
971 }, {
972 currentFetchEnabled: false,
973 cacheMissFallbackEnabled: false,
974 lookaheadSeconds: 0,
975 initialPrefetchSeconds: 0,
976 buildFrameRequest: ({ currentTime, streamIndex }) => ({
977 url: `/subtitle-frame?streamIndex=${streamIndex}&timeSeconds=${currentTime}`,
978 }),
979 });
980
981 await flushAsyncWork(window, 8);
982
983 const overlay = player.el().querySelector('.video-subtitle-overlay');
984 expect(overlay).toBeTruthy();
985
986 player.setCurrentTime(1.2);
987 player.trigger('play');
988 await new Promise((resolve) => window.setTimeout(resolve, 50));
989 await flushAsyncWork(window, 2);
990 expect(overlay.style.display).toBe('block');
991
992 player.trigger('pause');
993 player.setCurrentTime(3.2);
994 player.trigger('seeked');
995 await flushAsyncWork(window, 6);
996
997 expect(overlay.style.display).toBe('block');
998 expect(overlay.querySelector('img').getAttribute('src')).toBe('blob:subtitle');
999
1000 controller.detach();
1001 await flushAsyncWork(window, 2);
1002 window.close();
1003 });
1004
926 1005 test('aborts in-flight prefetch requests when the overlay is detached', async () =>
927 1006 {
928 1007 const window = createWindow();
@@ -923,6 +923,85 @@ describe('shared image subtitle prefetching', () =>
923 window.close();
923 window.close();
924 });
924 });
925
925
926 test('resyncs the displayed caption after a seek without waiting for timeupdate', async () =>
927 {
928 const window = createWindow();
929 const helper = loadSharedVideoPlayer(window);
930 const { player, videojs } = buildPlayerHarness(window);
931
932 window.requestAnimationFrame = undefined;
933 window.cancelAnimationFrame = undefined;
934
935 window.fetch = jest.fn(async (url) =>
936 {
937 const parsed = new URL(url, 'http://localhost/');
938 const timeSeconds = Number(parsed.searchParams.get('timeSeconds'));
939 const headers = {
940 'Content-Type': 'image/png',
941 'X-Subtitle-Frame-Width': '640',
942 'X-Subtitle-Frame-Height': '80',
943 'X-Subtitle-Frame-Canvas-Width': '1920',
944 'X-Subtitle-Frame-Canvas-Height': '1080',
945 };
946
947 if (Math.abs(timeSeconds - 1.2) < 0.001)
948 {
949 headers['X-Subtitle-Frame-Start-Time'] = '1.0';
950 headers['X-Subtitle-Frame-End-Time'] = '2.0';
951 } else if (Math.abs(timeSeconds - 3.2) < 0.001)
952 {
953 headers['X-Subtitle-Frame-Start-Time'] = '3.0';
954 headers['X-Subtitle-Frame-End-Time'] = '4.0';
955 } else
956 {
957 return new Response(null, {
958 status: 204,
959 headers: { 'X-Subtitle-Frame-Status': 'none' },
960 });
961 }
962
963 return new Response(new Uint8Array([137, 80, 78, 71]), {
964 status: 200,
965 headers,
966 });
967 });
968
969 const controller = helper.bindImageSubtitleOverlay(player, videojs, {}, {
970 imageSubtitleTracks: [{ label: 'PGS English', streamIndex: 0, default: true }],
971 }, {
972 currentFetchEnabled: false,
973 cacheMissFallbackEnabled: false,
974 lookaheadSeconds: 0,
975 initialPrefetchSeconds: 0,
976 buildFrameRequest: ({ currentTime, streamIndex }) => ({
977 url: `/subtitle-frame?streamIndex=${streamIndex}&timeSeconds=${currentTime}`,
978 }),
979 });
980
981 await flushAsyncWork(window, 8);
982
983 const overlay = player.el().querySelector('.video-subtitle-overlay');
984 expect(overlay).toBeTruthy();
985
986 player.setCurrentTime(1.2);
987 player.trigger('play');
988 await new Promise((resolve) => window.setTimeout(resolve, 50));
989 await flushAsyncWork(window, 2);
990 expect(overlay.style.display).toBe('block');
991
992 player.trigger('pause');
993 player.setCurrentTime(3.2);
994 player.trigger('seeked');
995 await flushAsyncWork(window, 6);
996
997 expect(overlay.style.display).toBe('block');
998 expect(overlay.querySelector('img').getAttribute('src')).toBe('blob:subtitle');
999
1000 controller.detach();
1001 await flushAsyncWork(window, 2);
1002 window.close();
1003 });
1004
926 test('aborts in-flight prefetch requests when the overlay is detached', async () =>
1005 test('aborts in-flight prefetch requests when the overlay is detached', async () =>
927 {
1006 {
928 const window = createWindow();
1007 const window = createWindow();