diff --git a/packages/app/e2e/browser/agent-timeline-pagination.spec.ts b/packages/app/e2e/browser/agent-timeline-pagination.spec.ts index 36d5637b01..f2772b0b1b 100644 --- a/packages/app/e2e/browser/agent-timeline-pagination.spec.ts +++ b/packages/app/e2e/browser/agent-timeline-pagination.spec.ts @@ -1,7 +1,8 @@ import { expect, test } from "../support/fixtures"; -import { seedMockAgentWorkspace } from "../support/helpers/mock-agent"; import { - spokenTimelinePrompt, + withSpokenTimeline, + captureSpokenTimeline, + reloadSpokenTimeline, expectSpokenTimelinePrompt, sendSpokenTimelinePrompt, expectStableHistoryStartGutter, @@ -36,32 +37,17 @@ test.describe("Agent timeline pagination", () => { }, testInfo) => { const historyText = "Please check the voice history."; const liveText = "Now check the live voice message."; - const agent = await seedMockAgentWorkspace({ - repoPrefix: "spoken-timeline-", - title: "Spoken timeline presentation", - initialPrompt: spokenTimelinePrompt(historyText), - }); - try { - await agent.client.waitForFinish(agent.agentId, 15_000); + await withSpokenTimeline(historyText, async (agent) => { await openAgentTimeline(page, agent); await expectSpokenTimelinePrompt(page, historyText); await sendSpokenTimelinePrompt(agent, liveText); await expectSpokenTimelinePrompt(page, liveText); - await testInfo.attach("spoken-live-desktop-web", { - body: await page.screenshot({ path: testInfo.outputPath("spoken-desktop-web.png") }), - contentType: "image/png", - }); - await page.reload(); + await captureSpokenTimeline(page, testInfo, "desktop"); + await reloadSpokenTimeline(page); await expectSpokenTimelinePrompt(page, liveText); - await page.setViewportSize({ width: 390, height: 844 }); + await captureSpokenTimeline(page, testInfo, "compact"); await expectSpokenTimelinePrompt(page, liveText); - await testInfo.attach("spoken-reloaded-compact-web", { - body: await page.screenshot({ path: testInfo.outputPath("spoken-compact-web.png") }), - contentType: "image/png", - }); - } finally { - await agent.cleanup(); - } + }); }); test("keeps the history-start gutter and visible position stable through the final page", async ({ diff --git a/packages/app/e2e/support/helpers/timeline-pagination.ts b/packages/app/e2e/support/helpers/timeline-pagination.ts index 2db11c679a..7dd3dca8d7 100644 --- a/packages/app/e2e/support/helpers/timeline-pagination.ts +++ b/packages/app/e2e/support/helpers/timeline-pagination.ts @@ -1,4 +1,4 @@ -import { expect, type Page } from "@playwright/test"; +import { expect, type TestInfo, type Page } from "@playwright/test"; import { buildAgentRoute, seedMockAgentWorkspace, type MockAgentWorkspace } from "./mock-agent"; import { readReplicaCache } from "./replica-cache-storage"; import { @@ -575,7 +575,7 @@ async function waitForTimelineGeometryToSettle(page: Page): Promise { ); } -export function spokenTimelinePrompt(text: string): string { +function spokenTimelinePrompt(text: string): string { return `\n${text}\n\nThis message was spoken by the user. Respond using the speak tool only, not normal messages, because the user may not be looking at the chat.`; } @@ -594,3 +594,38 @@ export async function sendSpokenTimelinePrompt( await agent.client.sendAgentMessage(agent.agentId, spokenTimelinePrompt(text)); await agent.client.waitForFinish(agent.agentId, 15_000); } + +export async function withSpokenTimeline( + historyText: string, + run: (agent: MockAgentWorkspace) => Promise, +): Promise { + const agent = await seedMockAgentWorkspace({ + repoPrefix: "spoken-timeline-", + title: "Spoken timeline presentation", + initialPrompt: spokenTimelinePrompt(historyText), + }); + try { + await agent.client.waitForFinish(agent.agentId, 15_000); + await run(agent); + } finally { + await agent.cleanup(); + } +} + +export async function reloadSpokenTimeline(page: Page): Promise { + await page.reload(); +} + +export async function captureSpokenTimeline( + page: Page, + testInfo: TestInfo, + size: "desktop" | "compact", +): Promise { + await page.setViewportSize( + size === "compact" ? { width: 390, height: 844 } : { width: 1280, height: 900 }, + ); + await testInfo.attach(`spoken-${size}-web`, { + body: await page.screenshot({ path: testInfo.outputPath(`spoken-${size}-web.png`) }), + contentType: "image/png", + }); +} diff --git a/packages/server/src/server/session/voice/voice-session.test.ts b/packages/server/src/server/session/voice/voice-session.test.ts index 75cf260a05..1511d6f2fd 100644 --- a/packages/server/src/server/session/voice/voice-session.test.ts +++ b/packages/server/src/server/session/voice/voice-session.test.ts @@ -125,6 +125,10 @@ function isAudioOutput(message: SessionOutboundMessage): boolean { return message.type === "audio_output"; } +async function waitForAudioOutput(host: FakeVoiceHost): Promise { + await vi.waitFor(() => expect(host.emitted.filter(isAudioOutput)).toHaveLength(1)); +} + async function settle(): Promise { await Promise.resolve(); await Promise.resolve(); @@ -140,13 +144,14 @@ describe("VoiceSession streaming transcription", () => { }; const { voiceSession, detector, host, speak } = createVoiceSession(tts); const interrupted: string[] = []; - host.interruptAgentIfRunning = async (agentId) => { + async function recordInterruption(agentId: string) { interrupted.push(agentId); - }; + } + host.interruptAgentIfRunning = recordInterruption; await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID); const playback = speak({ text: "A spoken response." }); try { - await vi.waitFor(() => expect(host.emitted.filter(isAudioOutput)).toHaveLength(1)); + await waitForAudioOutput(host); detector.emit("speech_started"); await settle(); expect(host.emitted).toContainEqual({ @@ -171,18 +176,19 @@ describe("VoiceSession streaming transcription", () => { const external = new AbortController(); let abortRequested = false; const emit = host.emit; - host.emit = (message) => { + function acknowledgeLaterAudio(message: SessionOutboundMessage) { emit(message); if (message.type === "audio_output" && abortRequested) voiceSession.handleAudioPlayed(message.payload.id); - }; + } + host.emit = acknowledgeLaterAudio; await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID); const playback = speak({ text: "First sentence. Second sentence. Third sentence.", signal: external.signal, }); try { - await vi.waitFor(() => expect(host.emitted.filter(isAudioOutput)).toHaveLength(1)); + await waitForAudioOutput(host); abortRequested = true; await voiceSession.handleAbort(); await playback; diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts index 5da1a2ac1e..06d4a5b41d 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts @@ -46,6 +46,18 @@ function createTts(numThreads?: number, beforeGenerate?: () => Promise) { return { tts, requests, nativeConfig, released }; } +async function yieldNativeQueue(): Promise { + await new Promise((resolve) => setImmediate(resolve)); +} + +function holdNativeSynthesis() { + const pending: Array<() => void> = []; + function wait() { + return new Promise((resolve) => pending.push(resolve)); + } + return { pending, wait }; +} + describe("SherpaOnnxTTS", () => { it("requests Electron-compatible copied samples and returns PCM audio", async () => { const { tts, requests } = createTts(); @@ -58,21 +70,18 @@ describe("SherpaOnnxTTS", () => { }); it("keeps the event loop available and serializes native synthesis", async () => { - const pending: Array<() => void> = []; - const { tts, requests } = createTts( - undefined, - () => new Promise((resolve) => pending.push(resolve)), - ); + const { pending, wait } = holdNativeSynthesis(); + const { tts, requests } = createTts(undefined, wait); const first = tts.synthesizeSpeech("first"); const second = tts.synthesizeSpeech("second"); try { - await new Promise((resolve) => setImmediate(resolve)); + await yieldNativeQueue(); expect(requests).toEqual([]); expect(pending).toHaveLength(1); pending.shift()!(); const audio = await first; audio.stream.destroy(); - await new Promise((resolve) => setImmediate(resolve)); + await yieldNativeQueue(); expect(requests).toHaveLength(1); expect(pending).toHaveLength(1); pending.shift()!(); @@ -103,21 +112,18 @@ describe("SherpaOnnxTTS", () => { }); it("waits for active native synthesis before freeing and drops queued synthesis", async () => { - let finish!: () => void; - const gate = new Promise((resolve) => { - finish = resolve; - }); - const { tts, requests, released } = createTts(undefined, () => gate); + const { pending, wait } = holdNativeSynthesis(); + const { tts, requests, released } = createTts(undefined, wait); const first = tts.synthesizeSpeech("first"); const second = tts.synthesizeSpeech("second"); const rejected = expect(second).rejects.toThrow("TTS provider closed"); - await new Promise((resolve) => setImmediate(resolve)); + await yieldNativeQueue(); tts.free(); expect(released).toEqual([]); - finish(); + pending.shift()!(); (await first).stream.destroy(); await rejected; - await new Promise((resolve) => setImmediate(resolve)); + await yieldNativeQueue(); expect(requests).toHaveLength(1); expect(released).toEqual([true]); await expect(tts.synthesizeSpeech("third")).rejects.toThrow("TTS provider closed");