From 3e859625ed431f9c5b38eedad961be5dd932165e Mon Sep 17 00:00:00 2001 From: "paseo-bot[bot]" <266920839+paseo-bot[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:42:30 +0000 Subject: [PATCH] test(app): verify dictation retries through the real daemon Observe actual dictation frames instead of constructing protocol success inside Playwright. Silence is finalized by the daemon's existing filter, so the test needs no transcription response replacement. Keep the completion decision and recording ownership behind a typed port for deterministic empty, successful, and partial-result unit coverage. --- .../browser/dictation-empty-result.spec.ts | 61 +++---- .../new-workspace-dictation-submit.spec.ts | 70 +++++++- packages/app/e2e/support/helpers/dictation.ts | 161 +++++------------- .../support/helpers/isolated-host-daemon.ts | 6 +- .../app/src/components/dictation-controls.tsx | 2 +- .../dictation/empty-transcript-reason.test.ts | 12 -- .../src/dictation/empty-transcript-reason.ts | 6 - packages/app/src/dictation/result.test.ts | 49 ++++++ packages/app/src/dictation/result.ts | 26 +++ .../app/src/hooks/use-dictation.shared.ts | 2 +- packages/app/src/hooks/use-dictation.ts | 36 ++-- 11 files changed, 230 insertions(+), 201 deletions(-) delete mode 100644 packages/app/src/dictation/empty-transcript-reason.test.ts delete mode 100644 packages/app/src/dictation/empty-transcript-reason.ts create mode 100644 packages/app/src/dictation/result.test.ts create mode 100644 packages/app/src/dictation/result.ts diff --git a/packages/app/e2e/browser/dictation-empty-result.spec.ts b/packages/app/e2e/browser/dictation-empty-result.spec.ts index a08f9f5e09..c2095933a3 100644 --- a/packages/app/e2e/browser/dictation-empty-result.spec.ts +++ b/packages/app/e2e/browser/dictation-empty-result.spec.ts @@ -3,39 +3,52 @@ import { composerLocator, expectComposerVisible } from "../support/helpers/compo import { discardDictation, expectEmptyDictation, - installDictationTranscriptHarness, installSyntheticMicrophone, + observeDictation, recordDictation, retryDictation, } from "../support/helpers/dictation"; import { openAgentRoute, seedMockAgentWorkspace } from "../support/helpers/mock-agent"; -const TRANSCRIPT = "Recovered on the second retry"; +// Silence is finalized by the real daemon without calling a transcription service. +// A closed local endpoint makes an unexpected transcription request fail the test. +test.use({ + e2eDaemonEnvironment: { + PASEO_DICTATION_ENABLED: "1", + OPENAI_API_KEY: "unused-for-silent-recordings", + OPENAI_BASE_URL: "http://127.0.0.1:9/v1", + }, +}); test.describe("Dictation empty result", () => { - test("explains an empty result and replays the same audio through repeated retries", async ({ + test("the daemon accepts the same recording through repeated empty-result retries", async ({ page, }) => { const agent = await seedMockAgentWorkspace({ repoPrefix: "dictation-empty-", title: "Empty dictation", }); - await installSyntheticMicrophone(page); - const harness = await installDictationTranscriptHarness(page, ["", "", TRANSCRIPT]); + await installSyntheticMicrophone(page, { silent: true }); + const observed = observeDictation(page); try { await openAgentRoute(page, agent); await expectComposerVisible(page); - await recordDictation(page, harness); + await recordDictation(page, observed); await expectEmptyDictation(page); await retryDictation(page); await expectEmptyDictation(page); await retryDictation(page); + await expectEmptyDictation(page); - await expect(composerLocator(page)).toHaveValue(TRANSCRIPT); - expect(harness.recordings).toHaveLength(3); - expect(harness.recordings[0].length).toBeGreaterThan(0); - expect(harness.recordings[1]).toEqual(harness.recordings[0]); - expect(harness.recordings[2]).toEqual(harness.recordings[0]); + expect(observed.recordings).toHaveLength(3); + expect(observed.recordings[0].audio.length).toBeGreaterThan(0); + expect(observed.recordings[0].finalSeq).toBe(observed.recordings[0].audio.length - 1); + expect(observed.recordings[0].text).toBe(""); + expect(observed.recordings[1]).toEqual(observed.recordings[0]); + expect(observed.recordings[2]).toEqual(observed.recordings[0]); + await discardDictation(page); + await expect(composerLocator(page)).toBeEditable(); + await expect(composerLocator(page)).toHaveValue(""); } finally { await agent.cleanup(); } @@ -43,18 +56,17 @@ test.describe("Dictation empty result", () => { test.describe("narrow composer", () => { test.use({ viewport: { width: 390, height: 844 } }); - - test("explains and discards an empty result from a silent microphone", async ({ page }) => { + test("explains and discards an empty result", async ({ page }) => { const agent = await seedMockAgentWorkspace({ repoPrefix: "dictation-silent-", title: "Silent dictation", }); await installSyntheticMicrophone(page, { silent: true }); - const harness = await installDictationTranscriptHarness(page, [""]); + const observed = observeDictation(page); try { await openAgentRoute(page, agent); await expectComposerVisible(page); - await recordDictation(page, harness); + await recordDictation(page, observed); await expectEmptyDictation(page); await discardDictation(page); await expect(composerLocator(page)).toBeEditable(); @@ -64,23 +76,4 @@ test.describe("Dictation empty result", () => { } }); }); - - test("keeps the latest partial text when the final response is empty", async ({ page }) => { - const agent = await seedMockAgentWorkspace({ - repoPrefix: "dictation-partial-", - title: "Partial dictation", - }); - await installSyntheticMicrophone(page); - const harness = await installDictationTranscriptHarness(page, [""], { - partialTranscript: "Words already recognized", - }); - try { - await openAgentRoute(page, agent); - await expectComposerVisible(page); - await recordDictation(page, harness); - await expect(composerLocator(page)).toHaveValue("Words already recognized"); - } finally { - await agent.cleanup(); - } - }); }); diff --git a/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts b/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts index b5ac8f6a18..22b2b8a6c5 100644 --- a/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts +++ b/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts @@ -16,16 +16,74 @@ import { expectComposerFullscreenControls, } from "../support/helpers/composer"; import { waitForSidebarHydration } from "../support/helpers/workspace-ui"; -import { - enableDictationCapability, - installSyntheticMicrophone, - sendSessionMessage, - sessionMessage, -} from "../support/helpers/dictation"; + +type WebSocketMessage = string | Buffer; const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace"; const CREATE_FAILURE = "Synthetic workspace creation failure"; +function parseEnvelope(message: WebSocketMessage): { + type?: unknown; + message?: Record; +} | null { + const raw = typeof message === "string" ? message : message.toString("utf8"); + try { + return JSON.parse(raw) as { type?: unknown; message?: Record }; + } catch { + return null; + } +} + +function sessionMessage(message: WebSocketMessage): Record | null { + const envelope = parseEnvelope(message); + return envelope?.type === "session" && envelope.message ? envelope.message : null; +} + +function sendSessionMessage( + ws: { send(message: string): void }, + message: Record, +): void { + ws.send(JSON.stringify({ type: "session", message })); +} + +function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { + const envelope = parseEnvelope(message); + const payload = envelope?.message?.payload; + if ( + envelope?.message?.type !== "status" || + !payload || + typeof payload !== "object" || + (payload as { status?: unknown }).status !== "server_info" + ) { + return message; + } + + (payload as Record).capabilities = { + voice: { + dictation: { enabled: true, reason: "" }, + voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, + }, + }; + return JSON.stringify(envelope); +} + +async function installSyntheticMicrophone(page: Page): Promise { + await page.addInitScript(() => { + const mediaDevices = navigator.mediaDevices; + Object.defineProperty(mediaDevices, "getUserMedia", { + configurable: true, + value: async () => { + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const destination = context.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + return destination.stream; + }, + }); + }); +} + async function installDictationFailureHarness(page: Page) { let resolveAudioChunk!: () => void; let resolveCreateRequest!: () => void; diff --git a/packages/app/e2e/support/helpers/dictation.ts b/packages/app/e2e/support/helpers/dictation.ts index d9554e292b..f99ddede05 100644 --- a/packages/app/e2e/support/helpers/dictation.ts +++ b/packages/app/e2e/support/helpers/dictation.ts @@ -1,51 +1,4 @@ import { expect, type Page } from "../fixtures"; -import { daemonWsRoutePattern } from "./daemon-port"; - -type WebSocketMessage = string | Buffer; - -function parseEnvelope(message: WebSocketMessage): { - type?: unknown; - message?: Record; -} | null { - const raw = typeof message === "string" ? message : message.toString("utf8"); - try { - return JSON.parse(raw) as { type?: unknown; message?: Record }; - } catch { - return null; - } -} - -export function sessionMessage(message: WebSocketMessage): Record | null { - const envelope = parseEnvelope(message); - return envelope?.type === "session" && envelope.message ? envelope.message : null; -} - -export function sendSessionMessage( - ws: { send(message: string): void }, - message: Record, -): void { - ws.send(JSON.stringify({ type: "session", message })); -} - -export function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { - const envelope = parseEnvelope(message); - const payload = envelope?.message?.payload; - if ( - envelope?.message?.type !== "status" || - !payload || - typeof payload !== "object" || - (payload as { status?: unknown }).status !== "server_info" - ) { - return message; - } - (payload as Record).capabilities = { - voice: { - dictation: { enabled: true, reason: "" }, - voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, - }, - }; - return JSON.stringify(envelope); -} /** * Replaces the browser microphone with an oscillator. `silent` keeps the stream @@ -74,88 +27,58 @@ export async function installSyntheticMicrophone( }, options.silent ?? false); } -/** - * Answers dictation stream requests in place of the daemon. Each finish is - * answered with the next queued transcript; everything else reaches the daemon. - */ -export async function installDictationTranscriptHarness( - page: Page, - transcripts: string[], - options: { partialTranscript?: string } = {}, -) { - const queue = [...transcripts]; - const streams = new Map>(); - const recordings: string[][] = []; - let resolveAudioChunk!: () => void; - const audioChunk = new Promise((resolve) => { - resolveAudioChunk = resolve; +/** Observe real browser/daemon traffic without intercepting or answering it. */ +export function observeDictation(page: Page) { + const streams = new Map; finalSeq: number }>(); + const recordings: Array<{ audio: string[]; finalSeq: number; text: string }> = []; + let resolveAudio!: () => void; + const audio = new Promise((resolve) => { + resolveAudio = resolve; }); - await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { - const server = ws.connectToServer(); - - ws.onMessage((message) => { - const request = sessionMessage(message); - if (!request) { - server.send(message); - return; + page.on("websocket", (socket) => { + socket.on("framesent", ({ payload }) => { + const request = sessionMessage(payload); + if (!request || typeof request.dictationId !== "string") return; + const id = request.dictationId; + if (request.type === "dictation_stream_start") { + streams.set(id, { audio: new Map(), finalSeq: -1 }); } - const type = request.type; - const dictationId = typeof request.dictationId === "string" ? request.dictationId : null; - - if (type === "dictation_stream_start" && dictationId) { - streams.set(dictationId, new Map()); - sendSessionMessage(ws, { - type: "dictation_stream_ack", - payload: { dictationId, ackSeq: -1 }, - }); - return; + if (request.type === "dictation_stream_chunk") { + streams.get(id)!.audio.set(request.seq as number, request.audio as string); + resolveAudio(); } - if (type === "dictation_stream_chunk" && dictationId) { - const seq = typeof request?.seq === "number" ? request.seq : 0; - sendSessionMessage(ws, { - type: "dictation_stream_ack", - payload: { dictationId, ackSeq: seq }, - }); - streams.get(dictationId)?.set(seq, String(request.audio)); - if (options.partialTranscript) { - sendSessionMessage(ws, { - type: "dictation_stream_partial", - payload: { dictationId, text: options.partialTranscript }, - }); - } - resolveAudioChunk(); - return; + if (request.type === "dictation_stream_finish") { + streams.get(id)!.finalSeq = request.finalSeq as number; } - if (type === "dictation_stream_finish" && dictationId) { - recordings.push( - [...(streams.get(dictationId)?.entries() ?? [])] - .sort(([a], [b]) => a - b) - .map(([, audio]) => audio), - ); - sendSessionMessage(ws, { - type: "dictation_stream_finish_accepted", - payload: { dictationId, timeoutMs: 5_000 }, - }); - sendSessionMessage(ws, { - type: "dictation_stream_final", - payload: { dictationId, text: queue.shift() ?? "" }, - }); - return; - } - - server.send(message); }); - - server.onMessage((message) => ws.send(enableDictationCapability(message))); + socket.on("framereceived", ({ payload }) => { + const response = sessionMessage(payload); + if (response?.type !== "dictation_stream_final") return; + const final = response.payload as { dictationId: string; text: string }; + const stream = streams.get(final.dictationId)!; + recordings.push({ + audio: [...stream.audio.entries()].sort(([a], [b]) => a - b).map(([, chunk]) => chunk), + finalSeq: stream.finalSeq, + text: final.text, + }); + }); }); - - return { waitForAudio: () => audioChunk, recordings }; + return { waitForAudio: () => audio, recordings }; } -export async function recordDictation(page: Page, harness: { waitForAudio(): Promise }) { +function sessionMessage(payload: string | Buffer): Record | null { + try { + const envelope = JSON.parse(typeof payload === "string" ? payload : payload.toString("utf8")); + return envelope.type === "session" ? envelope.message : null; + } catch { + return null; + } +} + +export async function recordDictation(page: Page, observed: { waitForAudio(): Promise }) { await page.getByRole("button", { name: "Start dictation", exact: true }).click(); - await harness.waitForAudio(); + await observed.waitForAudio(); await page.getByRole("button", { name: "Insert transcription", exact: true }).click(); } diff --git a/packages/app/e2e/support/helpers/isolated-host-daemon.ts b/packages/app/e2e/support/helpers/isolated-host-daemon.ts index c8dd3eaa44..b19cd72cdb 100644 --- a/packages/app/e2e/support/helpers/isolated-host-daemon.ts +++ b/packages/app/e2e/support/helpers/isolated-host-daemon.ts @@ -182,8 +182,8 @@ export async function startIsolatedHostDaemon( const spawnDaemon = async (): Promise => { const spawnOptions: SpawnOptions = { cwd: serverDir, - env: withDisabledE2ESpeechEnv({ - ...process.env, + env: { + ...withDisabledE2ESpeechEnv(process.env), ...options.environment, PASEO_HOME: paseoHome, PASEO_SERVER_ID: serverId, @@ -192,7 +192,7 @@ export async function startIsolatedHostDaemon( PASEO_RELAY_ENABLED: options.mutableRelay ? undefined : "0", PASEO_NODE_ENV: "development", NODE_ENV: "development", - }), + }, stdio: ["ignore", "ignore", "pipe"], detached: false, }; diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index f95607b93f..a646f94605 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -8,7 +8,7 @@ import type { TFunction } from "i18next"; import { VolumeMeter } from "./volume-meter"; import { FOOTER_HEIGHT } from "@/constants/layout"; import type { DictationStatus } from "@/hooks/use-dictation"; -import type { DictationEmptyReason } from "@/dictation/empty-transcript-reason"; +import type { DictationEmptyReason } from "@/dictation/result"; interface DictationControlsProps { volume: number; diff --git a/packages/app/src/dictation/empty-transcript-reason.test.ts b/packages/app/src/dictation/empty-transcript-reason.test.ts deleted file mode 100644 index 2a73e813dd..0000000000 --- a/packages/app/src/dictation/empty-transcript-reason.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { resolveDictationEmptyReason } from "@/dictation/empty-transcript-reason"; - -describe("resolveDictationEmptyReason", () => { - it("reports when the microphone delivered no audio", () => { - expect(resolveDictationEmptyReason({ segmentCount: 0 })).toBe("no-audio"); - }); - - it("does not infer the server's reason from recorded audio", () => { - expect(resolveDictationEmptyReason({ segmentCount: 40 })).toBe("no-transcript"); - }); -}); diff --git a/packages/app/src/dictation/empty-transcript-reason.ts b/packages/app/src/dictation/empty-transcript-reason.ts deleted file mode 100644 index 1c3e49d866..0000000000 --- a/packages/app/src/dictation/empty-transcript-reason.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** What the client knows after a dictation finishes without text. */ -export type DictationEmptyReason = "no-audio" | "no-transcript"; - -export function resolveDictationEmptyReason(input: { segmentCount: number }): DictationEmptyReason { - return input.segmentCount === 0 ? "no-audio" : "no-transcript"; -} diff --git a/packages/app/src/dictation/result.test.ts b/packages/app/src/dictation/result.test.ts new file mode 100644 index 0000000000..e0db8fd370 --- /dev/null +++ b/packages/app/src/dictation/result.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { completeDictation, type DictationRecording } from "./result"; + +class RecordedAudio implements DictationRecording { + constructor(private segments: string[]) {} + getSegmentCount() { + return this.segments.length; + } + clearAll() { + this.segments = []; + } +} + +describe("completing dictation", () => { + it("keeps a recording through repeated empty results until a retry returns text", () => { + const recording = new RecordedAudio(["first PCM chunk", "second PCM chunk"]); + const empty = { finalText: " ", partialText: "", recording }; + const result = completeDictation(empty); + expect(recording.getSegmentCount()).toBe(2); + expect(result).toEqual({ status: "empty", reason: "no-transcript" }); + expect(completeDictation(empty)).toEqual({ status: "empty", reason: "no-transcript" }); + expect(recording.getSegmentCount()).toBe(2); + expect(completeDictation({ ...empty, finalText: " recovered " })).toEqual({ + status: "transcribed", + text: "recovered", + }); + expect(recording.getSegmentCount()).toBe(0); + }); + + it("keeps usable partial text when the final response is empty", () => { + const recording = new RecordedAudio(["PCM chunk"]); + expect( + completeDictation({ finalText: "", partialText: " recognized words ", recording }), + ).toEqual({ + status: "transcribed", + text: "recognized words", + }); + expect(recording.getSegmentCount()).toBe(0); + }); + + it("reports a missing recording without inventing a reason for the empty result", () => { + expect( + completeDictation({ finalText: "", partialText: "", recording: new RecordedAudio([]) }), + ).toEqual({ + status: "empty", + reason: "no-audio", + }); + }); +}); diff --git a/packages/app/src/dictation/result.ts b/packages/app/src/dictation/result.ts new file mode 100644 index 0000000000..97ee926c3d --- /dev/null +++ b/packages/app/src/dictation/result.ts @@ -0,0 +1,26 @@ +export type DictationEmptyReason = "no-audio" | "no-transcript"; + +export interface DictationRecording { + getSegmentCount(): number; + clearAll(): void; +} + +export type DictationResult = + | { status: "empty"; reason: DictationEmptyReason } + | { status: "transcribed"; text: string }; + +export function completeDictation(input: { + finalText: string; + partialText: string; + recording: DictationRecording; +}): DictationResult { + const text = input.finalText.trim() || input.partialText.trim(); + if (!text) { + return { + status: "empty", + reason: input.recording.getSegmentCount() === 0 ? "no-audio" : "no-transcript", + }; + } + input.recording.clearAll(); + return { status: "transcribed", text }; +} diff --git a/packages/app/src/hooks/use-dictation.shared.ts b/packages/app/src/hooks/use-dictation.shared.ts index e2e885428f..b76eb076ed 100644 --- a/packages/app/src/hooks/use-dictation.shared.ts +++ b/packages/app/src/hooks/use-dictation.shared.ts @@ -1,4 +1,4 @@ -import type { DictationEmptyReason } from "@/dictation/empty-transcript-reason"; +import type { DictationEmptyReason } from "@/dictation/result"; import { i18n } from "@/i18n/i18next"; export type DictationStatus = diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index f2c8f050f9..81bf4cfffb 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -2,10 +2,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { DictationStreamSender } from "@/dictation/dictation-stream-sender"; -import { - resolveDictationEmptyReason, - type DictationEmptyReason, -} from "@/dictation/empty-transcript-reason"; +import { completeDictation, type DictationEmptyReason } from "@/dictation/result"; import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source"; import { generateMessageId } from "@/types/stream"; import { AttemptGuard } from "@/utils/attempt-guard"; @@ -136,13 +133,17 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { [setError], ); - const clearStreamingState = useCallback(() => { - senderRef.current?.clearAll(); + const clearTranscriptState = useCallback(() => { latestPartialTranscriptRef.current = ""; setPartialTranscript(""); setEmptyReason(null); }, []); + const clearStreamingState = useCallback(() => { + senderRef.current?.clearAll(); + clearTranscriptState(); + }, [clearTranscriptState]); + const startNewStream = useCallback(async (reason: string) => { await senderRef.current?.restartStream(reason); }, []); @@ -198,23 +199,20 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { setDuration(0); setStatus("idle"); - const transcriptText = - text.trim().length > 0 ? text.trim() : latestPartialTranscriptRef.current.trim(); - - if (!transcriptText) { - // Keep the audio until the user retries or discards this empty result. + const result = completeDictation({ + finalText: text, + partialText: latestPartialTranscriptRef.current, + recording: senderRef.current!, + }); + if (result.status === "empty") { setStatus("empty"); - setEmptyReason( - resolveDictationEmptyReason({ - segmentCount: senderRef.current?.getSegmentCount() ?? 0, - }), - ); + setEmptyReason(result.reason); return; } - clearStreamingState(); - onTranscriptRef.current?.(transcriptText, { requestId }); + clearTranscriptState(); + onTranscriptRef.current?.(result.text, { requestId }); }, - [clearStreamingState], + [clearTranscriptState], ); const handleDictationFailure = useCallback(