test(voice): clarify playback and timeline regression journeys

packages/app/e2e/browser/agent-timeline-pagination.spec.ts

@@ -1,7 +1,8 @@
 import { expect, test } from "../support/fixtures";
-import { seedMockAgentWorkspace } from "../support/helpers/mock-agent";
 import {
-  spokenTimelinePrompt,
+  withSpokenTimeline,
+  captureSpokenTimeline,
+  reloadSpokenTimeline,
   expectSpokenTimelinePrompt,
   sendSpokenTimelinePrompt,
   expectStableHistoryStartGutter,
@@ -36,32 +37,17 @@ test.describe("Agent timeline pagination", () => {
   }, testInfo) => {
     const historyText = "Please check the voice history.";
     const liveText = "Now check the live voice message.";
-    const agent = await seedMockAgentWorkspace({
-      repoPrefix: "spoken-timeline-",
-      title: "Spoken timeline presentation",
-      initialPrompt: spokenTimelinePrompt(historyText),
-    });
-    try {
-      await agent.client.waitForFinish(agent.agentId, 15_000);
+    await withSpokenTimeline(historyText, async (agent) => {
       await openAgentTimeline(page, agent);
       await expectSpokenTimelinePrompt(page, historyText);
       await sendSpokenTimelinePrompt(agent, liveText);
       await expectSpokenTimelinePrompt(page, liveText);
-      await testInfo.attach("spoken-live-desktop-web", {
-        body: await page.screenshot({ path: testInfo.outputPath("spoken-desktop-web.png") }),
-        contentType: "image/png",
-      });
-      await page.reload();
+      await captureSpokenTimeline(page, testInfo, "desktop");
+      await reloadSpokenTimeline(page);
       await expectSpokenTimelinePrompt(page, liveText);
-      await page.setViewportSize({ width: 390, height: 844 });
+      await captureSpokenTimeline(page, testInfo, "compact");
       await expectSpokenTimelinePrompt(page, liveText);
-      await testInfo.attach("spoken-reloaded-compact-web", {
-        body: await page.screenshot({ path: testInfo.outputPath("spoken-compact-web.png") }),
-        contentType: "image/png",
-      });
-    } finally {
-      await agent.cleanup();
-    }
+    });
   });
 
   test("keeps the history-start gutter and visible position stable through the final page", async ({

packages/app/e2e/support/helpers/timeline-pagination.ts

@@ -1,4 +1,4 @@
-import { expect, type Page } from "@playwright/test";
+import { expect, type TestInfo, type Page } from "@playwright/test";
 import { buildAgentRoute, seedMockAgentWorkspace, type MockAgentWorkspace } from "./mock-agent";
 import { readReplicaCache } from "./replica-cache-storage";
 import {
@@ -575,7 +575,7 @@ async function waitForTimelineGeometryToSettle(page: Page): Promise<void> {
   );
 }
 
-export function spokenTimelinePrompt(text: string): string {
+function spokenTimelinePrompt(text: string): string {
   return `<spoken-input>\n${text}\n</spoken-input>\n<instruction>This message was spoken by the user. Respond using the speak tool only, not normal messages, because the user may not be looking at the chat.</instruction>`;
 }
 
@@ -594,3 +594,38 @@ export async function sendSpokenTimelinePrompt(
   await agent.client.sendAgentMessage(agent.agentId, spokenTimelinePrompt(text));
   await agent.client.waitForFinish(agent.agentId, 15_000);
 }
+
+export async function withSpokenTimeline(
+  historyText: string,
+  run: (agent: MockAgentWorkspace) => Promise<void>,
+): Promise<void> {
+  const agent = await seedMockAgentWorkspace({
+    repoPrefix: "spoken-timeline-",
+    title: "Spoken timeline presentation",
+    initialPrompt: spokenTimelinePrompt(historyText),
+  });
+  try {
+    await agent.client.waitForFinish(agent.agentId, 15_000);
+    await run(agent);
+  } finally {
+    await agent.cleanup();
+  }
+}
+
+export async function reloadSpokenTimeline(page: Page): Promise<void> {
+  await page.reload();
+}
+
+export async function captureSpokenTimeline(
+  page: Page,
+  testInfo: TestInfo,
+  size: "desktop" | "compact",
+): Promise<void> {
+  await page.setViewportSize(
+    size === "compact" ? { width: 390, height: 844 } : { width: 1280, height: 900 },
+  );
+  await testInfo.attach(`spoken-${size}-web`, {
+    body: await page.screenshot({ path: testInfo.outputPath(`spoken-${size}-web.png`) }),
+    contentType: "image/png",
+  });
+}

packages/server/src/server/session/voice/voice-session.test.ts

@@ -125,6 +125,10 @@ function isAudioOutput(message: SessionOutboundMessage): boolean {
   return message.type === "audio_output";
 }
 
+async function waitForAudioOutput(host: FakeVoiceHost): Promise<void> {
+  await vi.waitFor(() => expect(host.emitted.filter(isAudioOutput)).toHaveLength(1));
+}
+
 async function settle(): Promise<void> {
   await Promise.resolve();
   await Promise.resolve();
@@ -140,13 +144,14 @@ describe("VoiceSession streaming transcription", () => {
     };
     const { voiceSession, detector, host, speak } = createVoiceSession(tts);
     const interrupted: string[] = [];
-    host.interruptAgentIfRunning = async (agentId) => {
+    async function recordInterruption(agentId: string) {
       interrupted.push(agentId);
-    };
+    }
+    host.interruptAgentIfRunning = recordInterruption;
     await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
     const playback = speak({ text: "A spoken response." });
     try {
-      await vi.waitFor(() => expect(host.emitted.filter(isAudioOutput)).toHaveLength(1));
+      await waitForAudioOutput(host);
       detector.emit("speech_started");
       await settle();
       expect(host.emitted).toContainEqual({
@@ -171,18 +176,19 @@ describe("VoiceSession streaming transcription", () => {
     const external = new AbortController();
     let abortRequested = false;
     const emit = host.emit;
-    host.emit = (message) => {
+    function acknowledgeLaterAudio(message: SessionOutboundMessage) {
       emit(message);
       if (message.type === "audio_output" && abortRequested)
         voiceSession.handleAudioPlayed(message.payload.id);
-    };
+    }
+    host.emit = acknowledgeLaterAudio;
     await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
     const playback = speak({
       text: "First sentence. Second sentence. Third sentence.",
       signal: external.signal,
     });
     try {
-      await vi.waitFor(() => expect(host.emitted.filter(isAudioOutput)).toHaveLength(1));
+      await waitForAudioOutput(host);
       abortRequested = true;
       await voiceSession.handleAbort();
       await playback;

packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts

@@ -46,6 +46,18 @@ function createTts(numThreads?: number, beforeGenerate?: () => Promise<void>) {
   return { tts, requests, nativeConfig, released };
 }
 
+async function yieldNativeQueue(): Promise<void> {
+  await new Promise<void>((resolve) => setImmediate(resolve));
+}
+
+function holdNativeSynthesis() {
+  const pending: Array<() => void> = [];
+  function wait() {
+    return new Promise<void>((resolve) => pending.push(resolve));
+  }
+  return { pending, wait };
+}
+
 describe("SherpaOnnxTTS", () => {
   it("requests Electron-compatible copied samples and returns PCM audio", async () => {
     const { tts, requests } = createTts();
@@ -58,21 +70,18 @@ describe("SherpaOnnxTTS", () => {
   });
 
   it("keeps the event loop available and serializes native synthesis", async () => {
-    const pending: Array<() => void> = [];
-    const { tts, requests } = createTts(
-      undefined,
-      () => new Promise<void>((resolve) => pending.push(resolve)),
-    );
+    const { pending, wait } = holdNativeSynthesis();
+    const { tts, requests } = createTts(undefined, wait);
     const first = tts.synthesizeSpeech("first");
     const second = tts.synthesizeSpeech("second");
     try {
-      await new Promise<void>((resolve) => setImmediate(resolve));
+      await yieldNativeQueue();
       expect(requests).toEqual([]);
       expect(pending).toHaveLength(1);
       pending.shift()!();
       const audio = await first;
       audio.stream.destroy();
-      await new Promise<void>((resolve) => setImmediate(resolve));
+      await yieldNativeQueue();
       expect(requests).toHaveLength(1);
       expect(pending).toHaveLength(1);
       pending.shift()!();
@@ -103,21 +112,18 @@ describe("SherpaOnnxTTS", () => {
   });
 
   it("waits for active native synthesis before freeing and drops queued synthesis", async () => {
-    let finish!: () => void;
-    const gate = new Promise<void>((resolve) => {
-      finish = resolve;
-    });
-    const { tts, requests, released } = createTts(undefined, () => gate);
+    const { pending, wait } = holdNativeSynthesis();
+    const { tts, requests, released } = createTts(undefined, wait);
     const first = tts.synthesizeSpeech("first");
     const second = tts.synthesizeSpeech("second");
     const rejected = expect(second).rejects.toThrow("TTS provider closed");
-    await new Promise<void>((resolve) => setImmediate(resolve));
+    await yieldNativeQueue();
     tts.free();
     expect(released).toEqual([]);
-    finish();
+    pending.shift()!();
     (await first).stream.destroy();
     await rejected;
-    await new Promise<void>((resolve) => setImmediate(resolve));
+    await yieldNativeQueue();
     expect(requests).toHaveLength(1);
     expect(released).toEqual([true]);
     await expect(tts.synthesizeSpeech("third")).rejects.toThrow("TTS provider closed");