diff --git a/packages/core/src/runtime/audioClockSource.test.ts b/packages/core/src/runtime/audioClockSource.test.ts index 9c8c1787438..c3e4997b63a 100644 --- a/packages/core/src/runtime/audioClockSource.test.ts +++ b/packages/core/src/runtime/audioClockSource.test.ts @@ -323,23 +323,20 @@ describe("the audio the playhead follows", () => { expect(window.__player!.getTime()).toBeGreaterThan(1.4); }); - it("holds the playhead for a playing clip that has no data, then lets the picture run on", async () => { - const stuck = await playStarved(); + it("holds the playhead for a playing clip that has no data until it has some", async () => { + await playStarved(); stepFrames(1); const held = window.__player!.getTime(); - stepFrames(60); + stepFrames(5 * 60); expect(window.__player!.getTime()).toBe(held); - stepFrames(4 * 60); // past the 3 s hold - expect(window.__player!.getTime()).toBeGreaterThan(0.5); - expect(stuck.paused).toBe(false); }); - it("brings a clip whose data arrives late in where the playhead is, without moving it back", async () => { + it("plays on from where it held once a clip's late data arrives", async () => { const late = await playStarved(); stepFrames(5 * 60); const before = window.__player!.getTime(); - expect(before).toBeGreaterThan(1); - late.readyState = 4; // its data arrives and it plays on from its start, until media sync seeks it + expect(before).toBeLessThan(0.1); + late.readyState = 4; for (let frame = 0; frame < 30; frame++) { late.currentTime += 1 / 60; stepFrames(1); diff --git a/packages/core/src/runtime/capturedMediaIdleRoute.test.ts b/packages/core/src/runtime/capturedMediaIdleRoute.test.ts index d0dd8289aeb..6ec6b0e8481 100644 --- a/packages/core/src/runtime/capturedMediaIdleRoute.test.ts +++ b/packages/core/src/runtime/capturedMediaIdleRoute.test.ts @@ -98,6 +98,7 @@ function mount(bodyHtml: string): void { `
${bodyHtml}
`; for (const el of document.querySelectorAll("audio")) { + Object.defineProperty(el, "readyState", { value: 4, configurable: true }); el.load = () => {}; el.play = vi.fn(() => Promise.resolve()); el.pause = vi.fn(); diff --git a/packages/core/src/runtime/init.buffering.test.ts b/packages/core/src/runtime/init.buffering.test.ts new file mode 100644 index 00000000000..306d1d6899a --- /dev/null +++ b/packages/core/src/runtime/init.buffering.test.ts @@ -0,0 +1,123 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { initSandboxRuntimeModular } from "./init"; +import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers"; + +function audio(id: string, duration: number, readyState: { value: number; bufferedTo?: number }) { + const el = document.getElementById(id) as HTMLAudioElement; + let paused = true; + Object.defineProperties(el, { + duration: { value: duration, configurable: true }, + readyState: { get: () => readyState.value, configurable: true }, + buffered: { + get: () => ({ length: 1, start: () => 0, end: () => readyState.bufferedTo ?? 0 }), + configurable: true, + }, + paused: { get: () => paused, configurable: true }, + }); + el.play = vi.fn(() => { + paused = false; + return Promise.resolve(); + }); + el.pause = vi.fn(() => { + if (!paused) el.dispatchEvent(new Event("pause")); + paused = true; + }); + el.load = vi.fn(); + return el; +} + +function playPodcast(voiceStart = 0) { + resetRuntimeFixtureDom(); + document.body.innerHTML = + '
' + + '' + + `` + + "
"; + const voiceReady: { value: number; bufferedTo?: number } = { value: 1 }; + const music = audio("music", 10, { value: 4 }); + const voice = audio("voice", 4, voiceReady); + window.__timelines = { main: createMockTimeline(10) }; + + let now = 0; + const frames = new Map(); + let nextFrame = 0; + vi.spyOn(performance, "now").mockImplementation(() => now); + window.requestAnimationFrame = (cb) => (frames.set(++nextFrame, cb), nextFrame); + window.cancelAnimationFrame = (id) => void frames.delete(id); + const states: Array<{ isPlaying: boolean; buffering: boolean }> = []; + vi.spyOn(window.parent, "postMessage").mockImplementation((message: unknown) => { + const m = message as { type?: string; isPlaying: boolean; buffering: boolean }; + if (m?.type === "state") states.push({ isPlaying: m.isPlaying, buffering: m.buffering }); + }); + + initSandboxRuntimeModular(); + window.__player!.play(); + const step = (ms: number) => { + now += ms; + const due = [...frames.values()]; + frames.clear(); + for (const cb of due) cb(now); + }; + const playFor = (seconds: number) => { + for (let ms = 0; ms < seconds * 1000; ms += 1000 / 60) step(1000 / 60); + }; + return { music, voice, voiceReady, states, playFor, step }; +} + +it("a track not ready pauses everything with Buffering, and resumes when it can play", () => { + const { music, voice, voiceReady, states, playFor } = playPodcast(); + + playFor(1); + expect(window.__player!.getTime()).toBeLessThan(0.1); + expect(music.paused).toBe(true); + expect(states.at(-1)).toEqual({ isPlaying: true, buffering: true }); + + voiceReady.value = 4; + playFor(1); + expect(window.__player!.getTime()).toBeGreaterThan(0.9); + expect(music.paused).toBe(false); + expect(voice.paused).toBe(false); + expect(states.at(-1)).toEqual({ isPlaying: true, buffering: false }); +}); + +it("holds on the frame a slow tick skips into a clip with no data, after asking it to play", () => { + const { voice, states, playFor, step } = playPodcast(0.5); + + playFor(0.4); + expect(voice.play).not.toHaveBeenCalled(); + step(250); + expect(voice.play).toHaveBeenCalled(); + expect(states.at(-1)).toEqual({ isPlaying: true, buffering: true }); +}); + +it("resumes for a held clip with data buffered ahead that reports only its current frame, as WebKit's do", () => { + const { voiceReady, states, playFor } = playPodcast(); + + playFor(0.5); + voiceReady.value = 2; + playFor(0.5); + expect(states.at(-1)).toEqual({ isPlaying: true, buffering: true }); + + voiceReady.bufferedTo = 2; + playFor(1); + expect(window.__player!.getTime()).toBeGreaterThan(0.9); + expect(states.at(-1)).toEqual({ isPlaying: true, buffering: false }); +}); + +it("a pause while buffering stays paused once the track is ready", () => { + const { music, voiceReady, states, playFor } = playPodcast(); + + playFor(0.5); + window.__player!.pause(); + voiceReady.value = 4; + playFor(1); + expect(window.__player!.getTime()).toBeLessThan(0.1); + expect(music.paused).toBe(true); + expect(states.at(-1)).toEqual({ isPlaying: false, buffering: false }); +}); + +afterEach(() => { + window.__hfRuntimeTeardown?.(); + vi.restoreAllMocks(); + resetRuntimeFixtureDom(); +}); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 2f6b0720c96..dbb3a59b1d3 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -3419,6 +3419,8 @@ export function initSandboxRuntimeModular(): void { // boundaries crossed in between. let lastSyncedMediaTimeSeconds: number | null = null; let mediaClipsInWindow: RuntimeMediaClip[] = []; + // Clips due now that could not play: every track holds until they can, so none runs on alone. + let bufferingFor: HTMLMediaElement[] = []; // The one definition of "media the transport drives": timed itself, or hosted // by a composition whose timing it inherits. Both the media cache and the @@ -3752,12 +3754,12 @@ export function initSandboxRuntimeModular(): void { const shouldPost = force || frame !== state.bridgeLastPostedFrame || - state.isPlaying !== state.bridgeLastPostedPlaying || + (state.isPlaying || bufferingFor.length > 0) !== state.bridgeLastPostedPlaying || state.bridgeMuted !== state.bridgeLastPostedMuted || now - state.bridgeLastPostedAt >= state.bridgeMaxPostIntervalMs; if (!shouldPost) return; state.bridgeLastPostedFrame = frame; - state.bridgeLastPostedPlaying = state.isPlaying; + state.bridgeLastPostedPlaying = state.isPlaying || bufferingFor.length > 0; state.bridgeLastPostedMuted = state.bridgeMuted; state.bridgeLastPostedAt = now; postRuntimeMessage({ @@ -3766,7 +3768,8 @@ export function initSandboxRuntimeModular(): void { frame, currentTime: state.currentTime || 0, ended: clock.reachedEnd(), - isPlaying: state.isPlaying, + isPlaying: state.bridgeLastPostedPlaying, + buffering: bufferingFor.length > 0, muted: state.bridgeMuted, playbackRate: state.playbackRate, }); @@ -4709,9 +4712,28 @@ export function initSandboxRuntimeModular(): void { postState(true); }; + const haltPlayback = () => { + webAudio.stopAll(); + clock.detachAudioSource(); + clock.pause(); + state.isPlaying = false; + state.currentTime = clock.now(); + state.mediaForceSyncNextTick = true; + hardSyncAllMedia(state.currentTime); + const tl = state.capturedTimeline; + pauseTimelineIfPossible(tl); + runAdapters("pause"); + syncMediaForCurrentState(); + colorGrading.redraw(); + // Paused: back to plain DOM so Studio can select inside the composition. + void paintCaptureMotionBlur(state.currentTime); + postState(true); + }; + const transport: RuntimePlayerTransport = { play: () => { flushHeldSeek(); + bufferingFor = []; const tl = state.capturedTimeline; if (clock.isPlaying()) return; const dur = getSafeTimelineDurationSeconds(tl, 0); @@ -4742,22 +4764,10 @@ export function initSandboxRuntimeModular(): void { postState(true); }, pause: () => { - if (!clock.isPlaying()) return; - webAudio.stopAll(); - clock.detachAudioSource(); - clock.pause(); - state.isPlaying = false; - state.currentTime = clock.now(); - state.mediaForceSyncNextTick = true; - hardSyncAllMedia(state.currentTime); - const tl = state.capturedTimeline; - pauseTimelineIfPossible(tl); - runAdapters("pause"); - syncMediaForCurrentState(); - colorGrading.redraw(); - // Paused: back to plain DOM so Studio can select inside the composition. - void paintCaptureMotionBlur(state.currentTime); - postState(true); + const wasBuffering = bufferingFor.length > 0; + bufferingFor = []; + if (clock.isPlaying()) haltPlayback(); + else if (wasBuffering) postState(true); }, seek: (timeSeconds, options) => { const quantized = quantizeTimeToFrame( @@ -5402,6 +5412,7 @@ export function initSandboxRuntimeModular(): void { */ const canParkTransport = (): boolean => !clock.isPlaying() && + bufferingFor.length === 0 && !isExportRenderDrivingFrames() && // Without MutationObserver nothing can PUSH a DOM change: neither the // composition-timing watcher nor the gesture watch attaches, and both fall @@ -5547,15 +5558,20 @@ export function initSandboxRuntimeModular(): void { requestTransportTick(); }; + const isPlayableTrack = (el: HTMLMediaElement): boolean => + el.isConnected && + !isSilencedByHidden(el) && + !isUnplayable(el) && + !(el.ended && !el.loop) && + (el.hasAttribute("src") || el.querySelector("source[src]") !== null); + const followedOrLongestRunningAudio = ( followed: HTMLMediaElement | null, ): { el: HTMLMediaElement; start: number } | null => { const audioEls = queryForTransport("audio[data-start]"); let longest: { el: HTMLMediaElement; start: number; runsUntil: number } | null = null; for (const el of followed ? [followed, ...audioEls] : audioEls) { - if (!isMediaElement(el) || !el.isConnected) continue; - if (isSilencedByHidden(el) || isUnplayable(el) || (el.ended && !el.loop)) continue; - if (!el.hasAttribute("src") && !el.querySelector("source[src]")) continue; + if (!isMediaElement(el) || !isPlayableTrack(el)) continue; const start = resolveAbsoluteMediaStartSeconds(el); const kept = playedMediaWindow(el, start); if (!Number.isFinite(start) || !isInClipWindow(state.currentTime, kept.start, kept.end)) @@ -5620,6 +5636,31 @@ export function initSandboxRuntimeModular(): void { } } + const READY_AHEAD_SECONDS = 1; + // WebKit reports a paused element as HAVE_CURRENT_DATA however much it has buffered, so read the ranges too. + const hasDataAhead = (el: HTMLMediaElement): boolean => { + if (el.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) return true; + if (el.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return false; + const until = Math.min(el.currentTime + READY_AHEAD_SECONDS, el.duration) - 0.01; + for (let i = 0; i < el.buffered.length; i++) { + if (el.buffered.start(i) <= el.currentTime && el.buffered.end(i) >= until) return true; + } + return false; + }; + + const mediaNotReadyAt = (t: number): HTMLMediaElement[] => + state.nativeMediaSyncDisabled + ? [] + : mediaClipsInWindow + .filter( + ({ el, start, end }) => + isInClipWindow(t, start, end) && + !hasDataAhead(el) && + isPlayableTrack(el) && + !webAudio.ownsElement(el), + ) + .map(({ el }) => el); + const transportTick = () => { if (state.tornDown || inTransportTick) return; inTransportTick = true; @@ -5789,6 +5830,14 @@ export function initSandboxRuntimeModular(): void { lastSyncedMediaTimeSeconds = null; syncMediaForCurrentState(timingRevision); } + // After media sync, so a clip that just fell due was asked to play, and fetch, before it holds. + const notReady = mediaNotReadyAt(state.currentTime); + if (clock.isPlaying() && notReady.length > 0) { + bufferingFor = notReady; + haltPlayback(); + } else if (bufferingFor.length > 0 && !notReady.some((el) => bufferingFor.includes(el))) { + transport.play(); + } postState(false); } finally { inTransportTick = false; diff --git a/packages/core/src/runtime/types.ts b/packages/core/src/runtime/types.ts index f322cd4b26f..9147d3270d5 100644 --- a/packages/core/src/runtime/types.ts +++ b/packages/core/src/runtime/types.ts @@ -62,6 +62,7 @@ export type RuntimeStateMessage = { currentTime: number; ended: boolean; isPlaying: boolean; + buffering: boolean; muted: boolean; playbackRate: number; }; diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index 6df44244161..30db868caed 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -1822,6 +1822,25 @@ describe("WebAudioTransport", () => { vi.unstubAllGlobals(); }); + it("hands a clip that decodes short to its element, and keeps one short only by its priming", async () => { + // Measured in Chromium 156 and WebKit 27.2: a 4 s AAC file whose mdhd timescale says 44.1 kHz over a + // 48 kHz stream decodes to 3.669 s; Apple's HE-AAC of the same tone decodes to 4.0 s under a 4.133 s element. + const withDuration = (src: string, duration: number) => + Object.assign(el(src), { duration }) as HTMLMediaElement; + const transport = transportWithDecode(async () => ({ duration: 3.669 }) as AudioBuffer); + const fetchMock = vi + .fn() + .mockResolvedValue({ ok: true, arrayBuffer: async () => new ArrayBuffer(8) }); + vi.stubGlobal("fetch", fetchMock); + expect(await transport.decodeAudioElement(withDuration("vo.m4a", 4))).toBeNull(); + expect(failedSrcs(transport).has(key("vo.m4a"))).toBe(true); + + const primed = transportWithDecode(async () => ({ duration: 4 }) as AudioBuffer); + expect(await primed.decodeAudioElement(withDuration("he.m4a", 4.133))).not.toBeNull(); + expect(await primed.decodeAudioElement(withDuration("rec.webm", Infinity))).not.toBeNull(); + vi.unstubAllGlobals(); + }); + it("fetches and decodes a source once when a play and a seek both ask before it decodes", async () => { const decoded = {} as AudioBuffer; const transport = transportWithDecode(async () => decoded); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index d123820f13c..0c12849c6b4 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -228,6 +228,7 @@ const FOLLOWED_EVENTS = ["playing", "seeked", "ratechange"] as const; // A pause to scrub or a reschedule mid-play keeps the output awake; a real idle lets it sleep. const IDLE_SUSPEND_MS = 10000; +const SHORT_DECODE_SECONDS = 0.25; export class WebAudioTransport { private _ctx: AudioContext | null = null; @@ -394,6 +395,14 @@ export class WebAudioTransport { // the same bad payload on every generation. try { const audioBuffer = await this._ctx.decodeAudioData(arrayBuffer); + // A file whose container timescale disagrees with its stream decodes short; its element plays it whole. + if ( + Number.isFinite(el.duration) && + el.duration - audioBuffer.duration > SHORT_DECODE_SECONDS + ) { + this._failedSrcs.add(src); + return null; + } this._bufferCache.set(src, audioBuffer); return audioBuffer; } catch (err) { diff --git a/packages/player/src/hyperframes-player.test.ts b/packages/player/src/hyperframes-player.test.ts index c16d9e41e31..9be3c73bc9c 100644 --- a/packages/player/src/hyperframes-player.test.ts +++ b/packages/player/src/hyperframes-player.test.ts @@ -4236,6 +4236,32 @@ describe("HyperframesPlayer asset-ready gate", () => { player.remove(); }); + it("shows Buffering while the runtime holds playback for a track, and hides it on resume", async () => { + const player = (await createConnectedPlayer()) as PlayerInternal & { + _onMessage: (event: MessageEvent) => void; + paused: boolean; + }; + const showSpy = vi.spyOn(player.shaderLoader, "showAssetsLoading"); + const post = (buffering: boolean) => + player._onMessage( + new MessageEvent("message", { + source: player.iframe.contentWindow, + data: { source: "hf-preview", type: "state", frame: 30, isPlaying: true, buffering }, + }), + ); + + post(true); + await new Promise((resolve) => setTimeout(resolve, 160)); + expect(player.hasAttribute("buffering")).toBe(true); + expect(showSpy).toHaveBeenCalledWith("Buffering", ""); + expect(player.paused).toBe(false); + + post(false); + expect(player.hasAttribute("buffering")).toBe(false); + + player.remove(); + }); + it("keeps a shader load drawn over the loading card when assets-loading-ui switches to none", async () => { const player = await createConnectedPlayer(); const { doc } = createStalledVideoDoc(); diff --git a/packages/player/src/hyperframes-player.ts b/packages/player/src/hyperframes-player.ts index f78aace8b89..7db233f58d8 100644 --- a/packages/player/src/hyperframes-player.ts +++ b/packages/player/src/hyperframes-player.ts @@ -52,6 +52,7 @@ const ASSETS_READY_TIMEOUT_MS = 8_000; const ASSETS_LOADING_ATTR = "assets-loading"; // "player" (default) draws the loading-assets card; "none" never does, like shader-loading="none". const ASSETS_LOADING_UI_ATTR = "assets-loading-ui"; +const BUFFERING_ATTR = "buffering"; const LOW_POWER_IDLE_ATTR = "low-power-idle"; const DISABLE_CLICK_TO_PLAY_ATTR = "disable-click-to-play"; const RANGE_START_ATTR = "range-start"; @@ -136,6 +137,7 @@ class HyperframesPlayer extends HTMLElement { private _pendingPlay = false; private _assetsGeneration = 0; private _assetsLoadingShowTimer: ReturnType | null = null; + private _bufferingShowTimer: ReturnType | null = null; private _currentTime = 0; private _duration = 0; private _paused = true; @@ -1272,6 +1274,7 @@ class HyperframesPlayer extends HTMLElement { this._rejectRuntimeDataDelivery(channel, requestId, message), onRuntimeTimelineReady: (duration, assetsReady) => this._onRuntimeTimelineReady(duration, assetsReady), + setBuffering: (buffering) => this._setBuffering(buffering), setRuntimeFps: (fps) => { this._runtimeFps = fps; }, @@ -1375,6 +1378,23 @@ class HyperframesPlayer extends HTMLElement { }, ASSETS_LOADING_SHOW_DELAY_MS); } + private _setBuffering(buffering: boolean): void { + if (buffering) { + if (this._bufferingShowTimer !== null || this.hasAttribute(BUFFERING_ATTR)) return; + this._bufferingShowTimer = setTimeout(() => { + this._bufferingShowTimer = null; + this.setAttribute(BUFFERING_ATTR, ""); + if (this.assetsLoadingUi !== "none") this.shaderLoader.showAssetsLoading("Buffering", ""); + }, ASSETS_LOADING_SHOW_DELAY_MS); + return; + } + if (this._bufferingShowTimer !== null) clearTimeout(this._bufferingShowTimer); + this._bufferingShowTimer = null; + if (!this.hasAttribute(BUFFERING_ATTR)) return; + this.removeAttribute(BUFFERING_ATTR); + this.shaderLoader.hideAssetsLoading(); + } + private _hasPendingFirstFrameAssets(doc: Document): boolean { const { pendingMedia, pendingImages, fontsLoading } = scanPendingCompositionAssets(doc, { scope: FIRST_FRAME_READINESS_SCOPE, @@ -1442,6 +1462,7 @@ class HyperframesPlayer extends HTMLElement { this._directTimelineAdapter = null; this._directTimelineClock.stop(); this._stopParentTickClock(); + this._setBuffering(false); this.shaderLoader.reset(); this._media.resetForIframeLoad(); } diff --git a/packages/player/src/runtime-message-handler.ts b/packages/player/src/runtime-message-handler.ts index 3d73ec072a9..1ae13ac1208 100644 --- a/packages/player/src/runtime-message-handler.ts +++ b/packages/player/src/runtime-message-handler.ts @@ -48,6 +48,7 @@ export interface MessageHandlerCallbacks extends PlaybackStateCallbacks { * player uses this as the cross-origin readiness signal because the * same-origin composition probe cannot inspect CDN iframes. */ onRuntimeTimelineReady: (duration: number, assetsReady: boolean | undefined) => void; + setBuffering?: (buffering: boolean) => void; setRuntimeFps?: (fps: number) => void; setRuntimeOwnsPlayRange?: (owns: boolean) => void; /** Called with the scene list whenever a "timeline" message is received. */ @@ -130,6 +131,7 @@ export function handleRuntimeMessage( callbacks, ), ); + callbacks.setBuffering?.(data["buffering"] === true); return; } diff --git a/packages/player/src/shader-loader-state.ts b/packages/player/src/shader-loader-state.ts index 5de4e008c3d..e9005a7e1ea 100644 --- a/packages/player/src/shader-loader-state.ts +++ b/packages/player/src/shader-loader-state.ts @@ -145,11 +145,14 @@ export class ShaderLoaderState { /** Second reason to show the same overlay — reuses the shader-transition * panel's DOM rather than drawing a second one. */ - showAssetsLoading(): void { + showAssetsLoading( + title = "Loading assets", + detail = "Waiting for images, video and fonts to finish loading.", + ): void { this.reset(); - this._el.title.textContent = "Loading assets"; - this._el.detail.textContent = "Waiting for images, video and fonts to finish loading."; - this._el.root.setAttribute("aria-label", "Loading assets"); + this._el.title.textContent = title; + this._el.detail.textContent = detail; + this._el.root.setAttribute("aria-label", title); this._drawingAssets = true; this.show(); }