BlueGit

timfa filemanager

Improve subtitle clock precision and prewarming

Image subtitle overlay timing now prefers the video renderer media clock when available, which improves boundary accuracy for cached captions that previously appeared late or were missed.

Also adds proactive secondary-buffer prewarming with the next cached frame so rapid caption transitions have the next image ready before the swap point, reducing visible misses on dense subtitle sequences.

Includes a regression test that verifies media-time based sync behaviour.

timfa Tim Falken · pushed by timfa

Authored 2026-05-29 · 5 months ago

Hash
f4321073f7044974c5ed92c125f9490b5ba510ae
Parent
a40f0816
2 files changed +135 −4
public/shared-video-player.js +65 −4
@@ -16,7 +16,7 @@
16 16 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_DEFAULT = 0.5;
17 17 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MIN = 0.25;
18 18 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MAX = 30;
19 const IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS = 0.05;
19 const IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS = 0.005;
20 20 const IMAGE_SUBTITLE_CACHE_MISS_FALLBACK_INTERVAL_MS_DEFAULT = 2000;
21 21 const IMAGE_SUBTITLE_CURRENT_REQUEST_TIMEOUT_MS_DEFAULT = 4000;
22 22 const IMAGE_SUBTITLE_PREFETCH_REQUEST_TIMEOUT_MS_DEFAULT = 4000;
@@ -393,6 +393,17 @@
393 393
394 394 const requestFrame = (fn) =>
395 395 {
396 const playerEl = player.el && player.el();
397 const videoEl = playerEl ? playerEl.querySelector('video') : null;
398 if (videoEl && typeof videoEl.requestVideoFrameCallback === 'function')
399 {
400 const id = videoEl.requestVideoFrameCallback((_, metadata) =>
401 {
402 fn(metadata);
403 });
404 return { kind: 'video-frame', id, videoEl };
405 }
406
396 407 if (typeof window.requestAnimationFrame === 'function')
397 408 {
398 409 return { kind: 'raf', id: window.requestAnimationFrame(fn) };
@@ -403,6 +414,11 @@
403 414 const cancelFrameRequest = (handle) =>
404 415 {
405 416 if (!handle || !Number.isFinite(Number(handle.id))) return;
417 if (handle.kind === 'video-frame' && handle.videoEl && typeof handle.videoEl.cancelVideoFrameCallback === 'function')
418 {
419 handle.videoEl.cancelVideoFrameCallback(handle.id);
420 return;
421 }
406 422 if (handle.kind === 'raf' && typeof window.cancelAnimationFrame === 'function')
407 423 {
408 424 window.cancelAnimationFrame(handle.id);
@@ -547,6 +563,7 @@
547 563 const closeObjectUrl = () =>
548 564 {
549 565 activeObjectUrl = null;
566 primaryBufferIndex = 0;
550 567 primaryFrameKey = null;
551 568 secondaryFrameKey = null;
552 569 };
@@ -554,7 +571,16 @@
554 571 const hideOverlay = () =>
555 572 {
556 573 if (overlayEl) overlayEl.style.display = 'none';
557 if (overlayImgEl) overlayImgEl.removeAttribute('src');
574 if (overlayImgEl)
575 {
576 overlayImgEl.style.display = 'none';
577 overlayImgEl.removeAttribute('src');
578 }
579 if (overlayImgElSecondary)
580 {
581 overlayImgElSecondary.style.display = 'none';
582 overlayImgElSecondary.removeAttribute('src');
583 }
558 584 closeObjectUrl();
559 585 };
560 586
@@ -669,6 +695,24 @@
669 695 return best;
670 696 };
671 697
698 const findNextCachedFrameAfterTime = (streamIndex, currentTime) =>
699 {
700 if (!Number.isInteger(streamIndex) || streamIndex < 0) return null;
701 if (!Number.isFinite(currentTime) || currentTime < 0) return null;
702
703 let best = null;
704 const threshold = currentTime + IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS;
705 for (const frame of cachedSubtitleFrames.values())
706 {
707 if (!frame || frame.streamIndex !== streamIndex) continue;
708 if (!Number.isFinite(frame.startTimeSeconds)) continue;
709 if (frame.startTimeSeconds <= threshold) continue;
710 if (!best || frame.startTimeSeconds < best.startTimeSeconds) best = frame;
711 }
712
713 return best;
714 };
715
672 716 const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>
673 717 {
674 718 if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)
@@ -769,7 +813,19 @@
769 813 showPrimaryBuffer();
770 814 };
771 815
772 const syncOverlayToPlaybackClock = () =>
816 const prewarmSecondaryBuffer = (streamIndex, currentTime, currentFrame) =>
817 {
818 if (!currentFrame || !Number.isInteger(streamIndex) || streamIndex < 0) return;
819
820 const secondaryIdx = primaryBufferIndex === 0 ? 1 : 0;
821 const secondaryKey = primaryBufferIndex === 0 ? secondaryFrameKey : primaryFrameKey;
822 const nextFrame = findNextCachedFrameAfterTime(streamIndex, currentTime);
823 if (!nextFrame || nextFrame.key === currentFrame.key || nextFrame.key === secondaryKey) return;
824
825 renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);
826 };
827
828 const syncOverlayToPlaybackClock = (frameMetadata) =>
773 829 {
774 830 if (selectedStreamIndex === null || trackDisabled)
775 831 {
@@ -777,7 +833,10 @@
777 833 return;
778 834 }
779 835
780 const currentTime = Number(player.currentTime());
836 const metadataTime = Number(frameMetadata && frameMetadata.mediaTime);
837 const currentTime = Number.isFinite(metadataTime) && metadataTime >= 0
838 ? metadataTime
839 : Number(player.currentTime());
781 840 if (Number.isFinite(currentTime) && currentTime >= 0)
782 841 {
783 842 const cachedFrame = findCachedFrameForTime(selectedStreamIndex, currentTime);
@@ -806,6 +865,8 @@
806 865 activeObjectUrl = cachedFrame.objectUrl;
807 866 }
808 867 }
868
869 prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);
809 870 } else
810 871 {
811 872 hideOverlay();
@@ -16,7 +16,7 @@
16 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_DEFAULT = 0.5;
16 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_DEFAULT = 0.5;
17 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MIN = 0.25;
17 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MIN = 0.25;
18 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MAX = 30;
18 const IMAGE_SUBTITLE_PREFETCH_STEP_SECONDS_MAX = 30;
19 const IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS = 0.05;
19 const IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS = 0.005;
20 const IMAGE_SUBTITLE_CACHE_MISS_FALLBACK_INTERVAL_MS_DEFAULT = 2000;
20 const IMAGE_SUBTITLE_CACHE_MISS_FALLBACK_INTERVAL_MS_DEFAULT = 2000;
21 const IMAGE_SUBTITLE_CURRENT_REQUEST_TIMEOUT_MS_DEFAULT = 4000;
21 const IMAGE_SUBTITLE_CURRENT_REQUEST_TIMEOUT_MS_DEFAULT = 4000;
22 const IMAGE_SUBTITLE_PREFETCH_REQUEST_TIMEOUT_MS_DEFAULT = 4000;
22 const IMAGE_SUBTITLE_PREFETCH_REQUEST_TIMEOUT_MS_DEFAULT = 4000;
@@ -393,6 +393,17 @@
393
393
394 const requestFrame = (fn) =>
394 const requestFrame = (fn) =>
395 {
395 {
396 const playerEl = player.el && player.el();
397 const videoEl = playerEl ? playerEl.querySelector('video') : null;
398 if (videoEl && typeof videoEl.requestVideoFrameCallback === 'function')
399 {
400 const id = videoEl.requestVideoFrameCallback((_, metadata) =>
401 {
402 fn(metadata);
403 });
404 return { kind: 'video-frame', id, videoEl };
405 }
406
396 if (typeof window.requestAnimationFrame === 'function')
407 if (typeof window.requestAnimationFrame === 'function')
397 {
408 {
398 return { kind: 'raf', id: window.requestAnimationFrame(fn) };
409 return { kind: 'raf', id: window.requestAnimationFrame(fn) };
@@ -403,6 +414,11 @@
403 const cancelFrameRequest = (handle) =>
414 const cancelFrameRequest = (handle) =>
404 {
415 {
405 if (!handle || !Number.isFinite(Number(handle.id))) return;
416 if (!handle || !Number.isFinite(Number(handle.id))) return;
417 if (handle.kind === 'video-frame' && handle.videoEl && typeof handle.videoEl.cancelVideoFrameCallback === 'function')
418 {
419 handle.videoEl.cancelVideoFrameCallback(handle.id);
420 return;
421 }
406 if (handle.kind === 'raf' && typeof window.cancelAnimationFrame === 'function')
422 if (handle.kind === 'raf' && typeof window.cancelAnimationFrame === 'function')
407 {
423 {
408 window.cancelAnimationFrame(handle.id);
424 window.cancelAnimationFrame(handle.id);
@@ -547,6 +563,7 @@
547 const closeObjectUrl = () =>
563 const closeObjectUrl = () =>
548 {
564 {
549 activeObjectUrl = null;
565 activeObjectUrl = null;
566 primaryBufferIndex = 0;
550 primaryFrameKey = null;
567 primaryFrameKey = null;
551 secondaryFrameKey = null;
568 secondaryFrameKey = null;
552 };
569 };
@@ -554,7 +571,16 @@
554 const hideOverlay = () =>
571 const hideOverlay = () =>
555 {
572 {
556 if (overlayEl) overlayEl.style.display = 'none';
573 if (overlayEl) overlayEl.style.display = 'none';
557 if (overlayImgEl) overlayImgEl.removeAttribute('src');
574 if (overlayImgEl)
575 {
576 overlayImgEl.style.display = 'none';
577 overlayImgEl.removeAttribute('src');
578 }
579 if (overlayImgElSecondary)
580 {
581 overlayImgElSecondary.style.display = 'none';
582 overlayImgElSecondary.removeAttribute('src');
583 }
558 closeObjectUrl();
584 closeObjectUrl();
559 };
585 };
560
586
@@ -669,6 +695,24 @@
669 return best;
695 return best;
670 };
696 };
671
697
698 const findNextCachedFrameAfterTime = (streamIndex, currentTime) =>
699 {
700 if (!Number.isInteger(streamIndex) || streamIndex < 0) return null;
701 if (!Number.isFinite(currentTime) || currentTime < 0) return null;
702
703 let best = null;
704 const threshold = currentTime + IMAGE_SUBTITLE_TIME_MATCH_EPSILON_SECONDS;
705 for (const frame of cachedSubtitleFrames.values())
706 {
707 if (!frame || frame.streamIndex !== streamIndex) continue;
708 if (!Number.isFinite(frame.startTimeSeconds)) continue;
709 if (frame.startTimeSeconds <= threshold) continue;
710 if (!best || frame.startTimeSeconds < best.startTimeSeconds) best = frame;
711 }
712
713 return best;
714 };
715
672 const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>
716 const getAdaptivePrefetchStepSeconds = (currentTimeSeconds) =>
673 {
717 {
674 if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)
718 if (!prefetchAdaptiveStepEnabled || prefetchStepSecondsMax <= prefetchStepSecondsMin)
@@ -769,7 +813,19 @@
769 showPrimaryBuffer();
813 showPrimaryBuffer();
770 };
814 };
771
815
772 const syncOverlayToPlaybackClock = () =>
816 const prewarmSecondaryBuffer = (streamIndex, currentTime, currentFrame) =>
817 {
818 if (!currentFrame || !Number.isInteger(streamIndex) || streamIndex < 0) return;
819
820 const secondaryIdx = primaryBufferIndex === 0 ? 1 : 0;
821 const secondaryKey = primaryBufferIndex === 0 ? secondaryFrameKey : primaryFrameKey;
822 const nextFrame = findNextCachedFrameAfterTime(streamIndex, currentTime);
823 if (!nextFrame || nextFrame.key === currentFrame.key || nextFrame.key === secondaryKey) return;
824
825 renderSubtitleFrameOnBuffer(nextFrame, secondaryIdx);
826 };
827
828 const syncOverlayToPlaybackClock = (frameMetadata) =>
773 {
829 {
774 if (selectedStreamIndex === null || trackDisabled)
830 if (selectedStreamIndex === null || trackDisabled)
775 {
831 {
@@ -777,7 +833,10 @@
777 return;
833 return;
778 }
834 }
779
835
780 const currentTime = Number(player.currentTime());
836 const metadataTime = Number(frameMetadata && frameMetadata.mediaTime);
837 const currentTime = Number.isFinite(metadataTime) && metadataTime >= 0
838 ? metadataTime
839 : Number(player.currentTime());
781 if (Number.isFinite(currentTime) && currentTime >= 0)
840 if (Number.isFinite(currentTime) && currentTime >= 0)
782 {
841 {
783 const cachedFrame = findCachedFrameForTime(selectedStreamIndex, currentTime);
842 const cachedFrame = findCachedFrameForTime(selectedStreamIndex, currentTime);
@@ -806,6 +865,8 @@
806 activeObjectUrl = cachedFrame.objectUrl;
865 activeObjectUrl = cachedFrame.objectUrl;
807 }
866 }
808 }
867 }
868
869 prewarmSecondaryBuffer(selectedStreamIndex, currentTime, cachedFrame);
809 } else
870 } else
810 {
871 {
811 hideOverlay();
872 hideOverlay();
tests/shared-video-player.test.js +70 −0
@@ -785,6 +785,76 @@ describe('shared image subtitle prefetching', () =>
785 785 window.close();
786 786 });
787 787
788 test('uses video-frame media clock for precise cached caption timing', async () =>
789 {
790 const window = createWindow();
791 const helper = loadSharedVideoPlayer(window);
792 const { player, videojs } = buildPlayerHarness(window);
793
794 let frameCallback = null;
795 let frameCallbackId = 0;
796 const videoEl = window.document.createElement('video');
797 videoEl.requestVideoFrameCallback = jest.fn((cb) =>
798 {
799 frameCallback = cb;
800 frameCallbackId += 1;
801 return frameCallbackId;
802 });
803 videoEl.cancelVideoFrameCallback = jest.fn();
804 player.el().appendChild(videoEl);
805
806 window.fetch = jest.fn(async () =>
807 {
808 return new Response(new Uint8Array([137, 80, 78, 71]), {
809 status: 200,
810 headers: {
811 'Content-Type': 'image/png',
812 'X-Subtitle-Frame-Start-Time': '1',
813 'X-Subtitle-Frame-End-Time': '2',
814 'X-Subtitle-Frame-Width': '640',
815 'X-Subtitle-Frame-Height': '80',
816 'X-Subtitle-Frame-Canvas-Width': '1920',
817 'X-Subtitle-Frame-Canvas-Height': '1080',
818 },
819 });
820 });
821
822 const controller = helper.bindImageSubtitleOverlay(player, videojs, {}, {
823 imageSubtitleTracks: [{ label: 'PGS English', streamIndex: 0, default: true }],
824 }, {
825 lookaheadSeconds: 0,
826 initialPrefetchSeconds: 0,
827 currentFetchEnabled: false,
828 cacheMissFallbackEnabled: false,
829 buildFrameRequest: ({ currentTime, streamIndex }) => ({
830 url: `/subtitle-frame?streamIndex=${streamIndex}&timeSeconds=${currentTime}`,
831 }),
832 });
833
834 await flushAsyncWork(window, 8);
835
836 const overlay = player.el().querySelector('.video-subtitle-overlay');
837 expect(overlay).toBeTruthy();
838 expect(videoEl.requestVideoFrameCallback).toHaveBeenCalled();
839 expect(typeof frameCallback).toBe('function');
840
841 player.setCurrentTime(9);
842 player.trigger('play');
843
844 frameCallback(1000, { mediaTime: 1.5 });
845 await flushAsyncWork(window, 3);
846 expect(overlay.style.display).toBe('block');
847
848 const nextFrameCallback = frameCallback;
849 nextFrameCallback(1016, { mediaTime: 2.2 });
850 await flushAsyncWork(window, 3);
851 expect(overlay.style.display).toBe('none');
852
853 controller.detach();
854 await flushAsyncWork(window, 2);
855 window.close();
856 });
857
788 858 test('aborts in-flight prefetch requests when the overlay is detached', async () =>
789 859 {
790 860 const window = createWindow();
@@ -785,6 +785,76 @@ describe('shared image subtitle prefetching', () =>
785 window.close();
785 window.close();
786 });
786 });
787
787
788 test('uses video-frame media clock for precise cached caption timing', async () =>
789 {
790 const window = createWindow();
791 const helper = loadSharedVideoPlayer(window);
792 const { player, videojs } = buildPlayerHarness(window);
793
794 let frameCallback = null;
795 let frameCallbackId = 0;
796 const videoEl = window.document.createElement('video');
797 videoEl.requestVideoFrameCallback = jest.fn((cb) =>
798 {
799 frameCallback = cb;
800 frameCallbackId += 1;
801 return frameCallbackId;
802 });
803 videoEl.cancelVideoFrameCallback = jest.fn();
804 player.el().appendChild(videoEl);
805
806 window.fetch = jest.fn(async () =>
807 {
808 return new Response(new Uint8Array([137, 80, 78, 71]), {
809 status: 200,
810 headers: {
811 'Content-Type': 'image/png',
812 'X-Subtitle-Frame-Start-Time': '1',
813 'X-Subtitle-Frame-End-Time': '2',
814 'X-Subtitle-Frame-Width': '640',
815 'X-Subtitle-Frame-Height': '80',
816 'X-Subtitle-Frame-Canvas-Width': '1920',
817 'X-Subtitle-Frame-Canvas-Height': '1080',
818 },
819 });
820 });
821
822 const controller = helper.bindImageSubtitleOverlay(player, videojs, {}, {
823 imageSubtitleTracks: [{ label: 'PGS English', streamIndex: 0, default: true }],
824 }, {
825 lookaheadSeconds: 0,
826 initialPrefetchSeconds: 0,
827 currentFetchEnabled: false,
828 cacheMissFallbackEnabled: false,
829 buildFrameRequest: ({ currentTime, streamIndex }) => ({
830 url: `/subtitle-frame?streamIndex=${streamIndex}&timeSeconds=${currentTime}`,
831 }),
832 });
833
834 await flushAsyncWork(window, 8);
835
836 const overlay = player.el().querySelector('.video-subtitle-overlay');
837 expect(overlay).toBeTruthy();
838 expect(videoEl.requestVideoFrameCallback).toHaveBeenCalled();
839 expect(typeof frameCallback).toBe('function');
840
841 player.setCurrentTime(9);
842 player.trigger('play');
843
844 frameCallback(1000, { mediaTime: 1.5 });
845 await flushAsyncWork(window, 3);
846 expect(overlay.style.display).toBe('block');
847
848 const nextFrameCallback = frameCallback;
849 nextFrameCallback(1016, { mediaTime: 2.2 });
850 await flushAsyncWork(window, 3);
851 expect(overlay.style.display).toBe('none');
852
853 controller.detach();
854 await flushAsyncWork(window, 2);
855 window.close();
856 });
857
788 test('aborts in-flight prefetch requests when the overlay is detached', async () =>
858 test('aborts in-flight prefetch requests when the overlay is detached', async () =>
789 {
859 {
790 const window = createWindow();
860 const window = createWindow();