fix(player): preview voice no longer goes silent while other tracks play

packages/core/src/runtime/audioClockSource.test.ts

@@ -323,23 +323,20 @@ describe("the audio the playhead follows", () => {
     expect(window.__player!.getTime()).toBeGreaterThan(1.4);
   });
 
-  it("holds the playhead for a playing clip that has no data, then lets the picture run on", async () => {
-    const stuck = await playStarved();
+  it("holds the playhead for a playing clip that has no data until it has some", async () => {
+    await playStarved();
     stepFrames(1);
     const held = window.__player!.getTime();
-    stepFrames(60);
+    stepFrames(5 * 60);
     expect(window.__player!.getTime()).toBe(held);
-    stepFrames(4 * 60); // past the 3 s hold
-    expect(window.__player!.getTime()).toBeGreaterThan(0.5);
-    expect(stuck.paused).toBe(false);
   });
 
-  it("brings a clip whose data arrives late in where the playhead is, without moving it back", async () => {
+  it("plays on from where it held once a clip's late data arrives", async () => {
     const late = await playStarved();
     stepFrames(5 * 60);
     const before = window.__player!.getTime();
-    expect(before).toBeGreaterThan(1);
-    late.readyState = 4; // its data arrives and it plays on from its start, until media sync seeks it
+    expect(before).toBeLessThan(0.1);
+    late.readyState = 4;
     for (let frame = 0; frame < 30; frame++) {
       late.currentTime += 1 / 60;
       stepFrames(1);

packages/core/src/runtime/capturedMediaIdleRoute.test.ts

@@ -98,6 +98,7 @@ function mount(bodyHtml: string): void {
     `<div data-composition-id="main" data-root="true" data-start="0" data-duration="10"` +
     ` data-width="1920" data-height="1080">${bodyHtml}</div>`;
   for (const el of document.querySelectorAll("audio")) {
+    Object.defineProperty(el, "readyState", { value: 4, configurable: true });
     el.load = () => {};
     el.play = vi.fn(() => Promise.resolve());
     el.pause = vi.fn();

packages/core/src/runtime/init.buffering.test.ts

@@ -0,0 +1,123 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { initSandboxRuntimeModular } from "./init";
+import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
+
+function audio(id: string, duration: number, readyState: { value: number; bufferedTo?: number }) {
+  const el = document.getElementById(id) as HTMLAudioElement;
+  let paused = true;
+  Object.defineProperties(el, {
+    duration: { value: duration, configurable: true },
+    readyState: { get: () => readyState.value, configurable: true },
+    buffered: {
+      get: () => ({ length: 1, start: () => 0, end: () => readyState.bufferedTo ?? 0 }),
+      configurable: true,
+    },
+    paused: { get: () => paused, configurable: true },
+  });
+  el.play = vi.fn(() => {
+    paused = false;
+    return Promise.resolve();
+  });
+  el.pause = vi.fn(() => {
+    if (!paused) el.dispatchEvent(new Event("pause"));
+    paused = true;
+  });
+  el.load = vi.fn();
+  return el;
+}
+
+function playPodcast(voiceStart = 0) {
+  resetRuntimeFixtureDom();
+  document.body.innerHTML =
+    '<div data-composition-id="main" data-root="true" data-duration="10">' +
+    '<audio id="music" data-start="0" data-duration="10" src="/music.m4a"></audio>' +
+    `<audio id="voice" data-start="${voiceStart}" data-duration="4" src="/voice.m4a"></audio>` +
+    "</div>";
+  const voiceReady: { value: number; bufferedTo?: number } = { value: 1 };
+  const music = audio("music", 10, { value: 4 });
+  const voice = audio("voice", 4, voiceReady);
+  window.__timelines = { main: createMockTimeline(10) };
+
+  let now = 0;
+  const frames = new Map<number, FrameRequestCallback>();
+  let nextFrame = 0;
+  vi.spyOn(performance, "now").mockImplementation(() => now);
+  window.requestAnimationFrame = (cb) => (frames.set(++nextFrame, cb), nextFrame);
+  window.cancelAnimationFrame = (id) => void frames.delete(id);
+  const states: Array<{ isPlaying: boolean; buffering: boolean }> = [];
+  vi.spyOn(window.parent, "postMessage").mockImplementation((message: unknown) => {
+    const m = message as { type?: string; isPlaying: boolean; buffering: boolean };
+    if (m?.type === "state") states.push({ isPlaying: m.isPlaying, buffering: m.buffering });
+  });
+
+  initSandboxRuntimeModular();
+  window.__player!.play();
+  const step = (ms: number) => {
+    now += ms;
+    const due = [...frames.values()];
+    frames.clear();
+    for (const cb of due) cb(now);
+  };
+  const playFor = (seconds: number) => {
+    for (let ms = 0; ms < seconds * 1000; ms += 1000 / 60) step(1000 / 60);
+  };
+  return { music, voice, voiceReady, states, playFor, step };
+}
+
+it("a track not ready pauses everything with Buffering, and resumes when it can play", () => {
+  const { music, voice, voiceReady, states, playFor } = playPodcast();
+
+  playFor(1);
+  expect(window.__player!.getTime()).toBeLessThan(0.1);
+  expect(music.paused).toBe(true);
+  expect(states.at(-1)).toEqual({ isPlaying: true, buffering: true });
+
+  voiceReady.value = 4;
+  playFor(1);
+  expect(window.__player!.getTime()).toBeGreaterThan(0.9);
+  expect(music.paused).toBe(false);
+  expect(voice.paused).toBe(false);
+  expect(states.at(-1)).toEqual({ isPlaying: true, buffering: false });
+});
+
+it("holds on the frame a slow tick skips into a clip with no data, after asking it to play", () => {
+  const { voice, states, playFor, step } = playPodcast(0.5);
+
+  playFor(0.4);
+  expect(voice.play).not.toHaveBeenCalled();
+  step(250);
+  expect(voice.play).toHaveBeenCalled();
+  expect(states.at(-1)).toEqual({ isPlaying: true, buffering: true });
+});
+
+it("resumes for a held clip with data buffered ahead that reports only its current frame, as WebKit's do", () => {
+  const { voiceReady, states, playFor } = playPodcast();
+
+  playFor(0.5);
+  voiceReady.value = 2;
+  playFor(0.5);
+  expect(states.at(-1)).toEqual({ isPlaying: true, buffering: true });
+
+  voiceReady.bufferedTo = 2;
+  playFor(1);
+  expect(window.__player!.getTime()).toBeGreaterThan(0.9);
+  expect(states.at(-1)).toEqual({ isPlaying: true, buffering: false });
+});
+
+it("a pause while buffering stays paused once the track is ready", () => {
+  const { music, voiceReady, states, playFor } = playPodcast();
+
+  playFor(0.5);
+  window.__player!.pause();
+  voiceReady.value = 4;
+  playFor(1);
+  expect(window.__player!.getTime()).toBeLessThan(0.1);
+  expect(music.paused).toBe(true);
+  expect(states.at(-1)).toEqual({ isPlaying: false, buffering: false });
+});
+
+afterEach(() => {
+  window.__hfRuntimeTeardown?.();
+  vi.restoreAllMocks();
+  resetRuntimeFixtureDom();
+});

packages/core/src/runtime/init.ts

@@ -3419,6 +3419,8 @@ export function initSandboxRuntimeModular(): void {
   // boundaries crossed in between.
   let lastSyncedMediaTimeSeconds: number | null = null;
   let mediaClipsInWindow: RuntimeMediaClip[] = [];
+  // Clips due now that could not play: every track holds until they can, so none runs on alone.
+  let bufferingFor: HTMLMediaElement[] = [];
 
   // The one definition of "media the transport drives": timed itself, or hosted
   // by a composition whose timing it inherits. Both the media cache and the
@@ -3752,12 +3754,12 @@ export function initSandboxRuntimeModular(): void {
     const shouldPost =
       force ||
       frame !== state.bridgeLastPostedFrame ||
-      state.isPlaying !== state.bridgeLastPostedPlaying ||
+      (state.isPlaying || bufferingFor.length > 0) !== state.bridgeLastPostedPlaying ||
       state.bridgeMuted !== state.bridgeLastPostedMuted ||
       now - state.bridgeLastPostedAt >= state.bridgeMaxPostIntervalMs;
     if (!shouldPost) return;
     state.bridgeLastPostedFrame = frame;
-    state.bridgeLastPostedPlaying = state.isPlaying;
+    state.bridgeLastPostedPlaying = state.isPlaying || bufferingFor.length > 0;
     state.bridgeLastPostedMuted = state.bridgeMuted;
     state.bridgeLastPostedAt = now;
     postRuntimeMessage({
@@ -3766,7 +3768,8 @@ export function initSandboxRuntimeModular(): void {
       frame,
       currentTime: state.currentTime || 0,
       ended: clock.reachedEnd(),
-      isPlaying: state.isPlaying,
+      isPlaying: state.bridgeLastPostedPlaying,
+      buffering: bufferingFor.length > 0,
       muted: state.bridgeMuted,
       playbackRate: state.playbackRate,
     });
@@ -4709,9 +4712,28 @@ export function initSandboxRuntimeModular(): void {
     postState(true);
   };
 
+  const haltPlayback = () => {
+    webAudio.stopAll();
+    clock.detachAudioSource();
+    clock.pause();
+    state.isPlaying = false;
+    state.currentTime = clock.now();
+    state.mediaForceSyncNextTick = true;
+    hardSyncAllMedia(state.currentTime);
+    const tl = state.capturedTimeline;
+    pauseTimelineIfPossible(tl);
+    runAdapters("pause");
+    syncMediaForCurrentState();
+    colorGrading.redraw();
+    // Paused: back to plain DOM so Studio can select inside the composition.
+    void paintCaptureMotionBlur(state.currentTime);
+    postState(true);
+  };
+
   const transport: RuntimePlayerTransport = {
     play: () => {
       flushHeldSeek();
+      bufferingFor = [];
       const tl = state.capturedTimeline;
       if (clock.isPlaying()) return;
       const dur = getSafeTimelineDurationSeconds(tl, 0);
@@ -4742,22 +4764,10 @@ export function initSandboxRuntimeModular(): void {
       postState(true);
     },
     pause: () => {
-      if (!clock.isPlaying()) return;
-      webAudio.stopAll();
-      clock.detachAudioSource();
-      clock.pause();
-      state.isPlaying = false;
-      state.currentTime = clock.now();
-      state.mediaForceSyncNextTick = true;
-      hardSyncAllMedia(state.currentTime);
-      const tl = state.capturedTimeline;
-      pauseTimelineIfPossible(tl);
-      runAdapters("pause");
-      syncMediaForCurrentState();
-      colorGrading.redraw();
-      // Paused: back to plain DOM so Studio can select inside the composition.
-      void paintCaptureMotionBlur(state.currentTime);
-      postState(true);
+      const wasBuffering = bufferingFor.length > 0;
+      bufferingFor = [];
+      if (clock.isPlaying()) haltPlayback();
+      else if (wasBuffering) postState(true);
     },
     seek: (timeSeconds, options) => {
       const quantized = quantizeTimeToFrame(
@@ -5402,6 +5412,7 @@ export function initSandboxRuntimeModular(): void {
    */
   const canParkTransport = (): boolean =>
     !clock.isPlaying() &&
+    bufferingFor.length === 0 &&
     !isExportRenderDrivingFrames() &&
     // Without MutationObserver nothing can PUSH a DOM change: neither the
     // composition-timing watcher nor the gesture watch attaches, and both fall
@@ -5547,15 +5558,20 @@ export function initSandboxRuntimeModular(): void {
     requestTransportTick();
   };
 
+  const isPlayableTrack = (el: HTMLMediaElement): boolean =>
+    el.isConnected &&
+    !isSilencedByHidden(el) &&
+    !isUnplayable(el) &&
+    !(el.ended && !el.loop) &&
+    (el.hasAttribute("src") || el.querySelector("source[src]") !== null);
+
   const followedOrLongestRunningAudio = (
     followed: HTMLMediaElement | null,
   ): { el: HTMLMediaElement; start: number } | null => {
     const audioEls = queryForTransport("audio[data-start]");
     let longest: { el: HTMLMediaElement; start: number; runsUntil: number } | null = null;
     for (const el of followed ? [followed, ...audioEls] : audioEls) {
-      if (!isMediaElement(el) || !el.isConnected) continue;
-      if (isSilencedByHidden(el) || isUnplayable(el) || (el.ended && !el.loop)) continue;
-      if (!el.hasAttribute("src") && !el.querySelector("source[src]")) continue;
+      if (!isMediaElement(el) || !isPlayableTrack(el)) continue;
       const start = resolveAbsoluteMediaStartSeconds(el);
       const kept = playedMediaWindow(el, start);
       if (!Number.isFinite(start) || !isInClipWindow(state.currentTime, kept.start, kept.end))
@@ -5620,6 +5636,31 @@ export function initSandboxRuntimeModular(): void {
     }
   }
 
+  const READY_AHEAD_SECONDS = 1;
+  // WebKit reports a paused element as HAVE_CURRENT_DATA however much it has buffered, so read the ranges too.
+  const hasDataAhead = (el: HTMLMediaElement): boolean => {
+    if (el.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) return true;
+    if (el.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return false;
+    const until = Math.min(el.currentTime + READY_AHEAD_SECONDS, el.duration) - 0.01;
+    for (let i = 0; i < el.buffered.length; i++) {
+      if (el.buffered.start(i) <= el.currentTime && el.buffered.end(i) >= until) return true;
+    }
+    return false;
+  };
+
+  const mediaNotReadyAt = (t: number): HTMLMediaElement[] =>
+    state.nativeMediaSyncDisabled
+      ? []
+      : mediaClipsInWindow
+          .filter(
+            ({ el, start, end }) =>
+              isInClipWindow(t, start, end) &&
+              !hasDataAhead(el) &&
+              isPlayableTrack(el) &&
+              !webAudio.ownsElement(el),
+          )
+          .map(({ el }) => el);
+
   const transportTick = () => {
     if (state.tornDown || inTransportTick) return;
     inTransportTick = true;
@@ -5789,6 +5830,14 @@ export function initSandboxRuntimeModular(): void {
         lastSyncedMediaTimeSeconds = null;
         syncMediaForCurrentState(timingRevision);
       }
+      // After media sync, so a clip that just fell due was asked to play, and fetch, before it holds.
+      const notReady = mediaNotReadyAt(state.currentTime);
+      if (clock.isPlaying() && notReady.length > 0) {
+        bufferingFor = notReady;
+        haltPlayback();
+      } else if (bufferingFor.length > 0 && !notReady.some((el) => bufferingFor.includes(el))) {
+        transport.play();
+      }
       postState(false);
     } finally {
       inTransportTick = false;

packages/core/src/runtime/types.ts

@@ -62,6 +62,7 @@ export type RuntimeStateMessage = {
   currentTime: number;
   ended: boolean;
   isPlaying: boolean;
+  buffering: boolean;
   muted: boolean;
   playbackRate: number;
 };

packages/core/src/runtime/webAudioTransport.test.ts

@@ -1822,6 +1822,25 @@ describe("WebAudioTransport", () => {
       vi.unstubAllGlobals();
     });
 
+    it("hands a clip that decodes short to its element, and keeps one short only by its priming", async () => {
+      // Measured in Chromium 156 and WebKit 27.2: a 4 s AAC file whose mdhd timescale says 44.1 kHz over a
+      // 48 kHz stream decodes to 3.669 s; Apple's HE-AAC of the same tone decodes to 4.0 s under a 4.133 s element.
+      const withDuration = (src: string, duration: number) =>
+        Object.assign(el(src), { duration }) as HTMLMediaElement;
+      const transport = transportWithDecode(async () => ({ duration: 3.669 }) as AudioBuffer);
+      const fetchMock = vi
+        .fn()
+        .mockResolvedValue({ ok: true, arrayBuffer: async () => new ArrayBuffer(8) });
+      vi.stubGlobal("fetch", fetchMock);
+      expect(await transport.decodeAudioElement(withDuration("vo.m4a", 4))).toBeNull();
+      expect(failedSrcs(transport).has(key("vo.m4a"))).toBe(true);
+
+      const primed = transportWithDecode(async () => ({ duration: 4 }) as AudioBuffer);
+      expect(await primed.decodeAudioElement(withDuration("he.m4a", 4.133))).not.toBeNull();
+      expect(await primed.decodeAudioElement(withDuration("rec.webm", Infinity))).not.toBeNull();
+      vi.unstubAllGlobals();
+    });
+
     it("fetches and decodes a source once when a play and a seek both ask before it decodes", async () => {
       const decoded = {} as AudioBuffer;
       const transport = transportWithDecode(async () => decoded);

packages/core/src/runtime/webAudioTransport.ts

@@ -228,6 +228,7 @@ const FOLLOWED_EVENTS = ["playing", "seeked", "ratechange"] as const;
 
 // A pause to scrub or a reschedule mid-play keeps the output awake; a real idle lets it sleep.
 const IDLE_SUSPEND_MS = 10000;
+const SHORT_DECODE_SECONDS = 0.25;
 
 export class WebAudioTransport {
   private _ctx: AudioContext | null = null;
@@ -394,6 +395,14 @@ export class WebAudioTransport {
     // the same bad payload on every generation.
     try {
       const audioBuffer = await this._ctx.decodeAudioData(arrayBuffer);
+      // A file whose container timescale disagrees with its stream decodes short; its element plays it whole.
+      if (
+        Number.isFinite(el.duration) &&
+        el.duration - audioBuffer.duration > SHORT_DECODE_SECONDS
+      ) {
+        this._failedSrcs.add(src);
+        return null;
+      }
       this._bufferCache.set(src, audioBuffer);
       return audioBuffer;
     } catch (err) {