diff --git a/packages/app/e2e/browser/dictation-empty-result.spec.ts b/packages/app/e2e/browser/dictation-empty-result.spec.ts index a08f9f5e09..c2095933a3 100644 --- a/packages/app/e2e/browser/dictation-empty-result.spec.ts +++ b/packages/app/e2e/browser/dictation-empty-result.spec.ts @@ -3,39 +3,52 @@ import { composerLocator, expectComposerVisible } from "../support/helpers/compo import { discardDictation, expectEmptyDictation, - installDictationTranscriptHarness, installSyntheticMicrophone, + observeDictation, recordDictation, retryDictation, } from "../support/helpers/dictation"; import { openAgentRoute, seedMockAgentWorkspace } from "../support/helpers/mock-agent"; -const TRANSCRIPT = "Recovered on the second retry"; +// Silence is finalized by the real daemon without calling a transcription service. +// A closed local endpoint makes an unexpected transcription request fail the test. +test.use({ + e2eDaemonEnvironment: { + PASEO_DICTATION_ENABLED: "1", + OPENAI_API_KEY: "unused-for-silent-recordings", + OPENAI_BASE_URL: "http://127.0.0.1:9/v1", + }, +}); test.describe("Dictation empty result", () => { - test("explains an empty result and replays the same audio through repeated retries", async ({ + test("the daemon accepts the same recording through repeated empty-result retries", async ({ page, }) => { const agent = await seedMockAgentWorkspace({ repoPrefix: "dictation-empty-", title: "Empty dictation", }); - await installSyntheticMicrophone(page); - const harness = await installDictationTranscriptHarness(page, ["", "", TRANSCRIPT]); + await installSyntheticMicrophone(page, { silent: true }); + const observed = observeDictation(page); try { await openAgentRoute(page, agent); await expectComposerVisible(page); - await recordDictation(page, harness); + await recordDictation(page, observed); await expectEmptyDictation(page); await retryDictation(page); await expectEmptyDictation(page); await retryDictation(page); + await expectEmptyDictation(page); - await expect(composerLocator(page)).toHaveValue(TRANSCRIPT); - expect(harness.recordings).toHaveLength(3); - expect(harness.recordings[0].length).toBeGreaterThan(0); - expect(harness.recordings[1]).toEqual(harness.recordings[0]); - expect(harness.recordings[2]).toEqual(harness.recordings[0]); + expect(observed.recordings).toHaveLength(3); + expect(observed.recordings[0].audio.length).toBeGreaterThan(0); + expect(observed.recordings[0].finalSeq).toBe(observed.recordings[0].audio.length - 1); + expect(observed.recordings[0].text).toBe(""); + expect(observed.recordings[1]).toEqual(observed.recordings[0]); + expect(observed.recordings[2]).toEqual(observed.recordings[0]); + await discardDictation(page); + await expect(composerLocator(page)).toBeEditable(); + await expect(composerLocator(page)).toHaveValue(""); } finally { await agent.cleanup(); } @@ -43,18 +56,17 @@ test.describe("Dictation empty result", () => { test.describe("narrow composer", () => { test.use({ viewport: { width: 390, height: 844 } }); - - test("explains and discards an empty result from a silent microphone", async ({ page }) => { + test("explains and discards an empty result", async ({ page }) => { const agent = await seedMockAgentWorkspace({ repoPrefix: "dictation-silent-", title: "Silent dictation", }); await installSyntheticMicrophone(page, { silent: true }); - const harness = await installDictationTranscriptHarness(page, [""]); + const observed = observeDictation(page); try { await openAgentRoute(page, agent); await expectComposerVisible(page); - await recordDictation(page, harness); + await recordDictation(page, observed); await expectEmptyDictation(page); await discardDictation(page); await expect(composerLocator(page)).toBeEditable(); @@ -64,23 +76,4 @@ test.describe("Dictation empty result", () => { } }); }); - - test("keeps the latest partial text when the final response is empty", async ({ page }) => { - const agent = await seedMockAgentWorkspace({ - repoPrefix: "dictation-partial-", - title: "Partial dictation", - }); - await installSyntheticMicrophone(page); - const harness = await installDictationTranscriptHarness(page, [""], { - partialTranscript: "Words already recognized", - }); - try { - await openAgentRoute(page, agent); - await expectComposerVisible(page); - await recordDictation(page, harness); - await expect(composerLocator(page)).toHaveValue("Words already recognized"); - } finally { - await agent.cleanup(); - } - }); }); diff --git a/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts b/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts index b5ac8f6a18..22b2b8a6c5 100644 --- a/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts +++ b/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts @@ -16,16 +16,74 @@ import { expectComposerFullscreenControls, } from "../support/helpers/composer"; import { waitForSidebarHydration } from "../support/helpers/workspace-ui"; -import { - enableDictationCapability, - installSyntheticMicrophone, - sendSessionMessage, - sessionMessage, -} from "../support/helpers/dictation"; + +type WebSocketMessage = string | Buffer; const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace"; const CREATE_FAILURE = "Synthetic workspace creation failure"; +function parseEnvelope(message: WebSocketMessage): { + type?: unknown; + message?: Record; +} | null { + const raw = typeof message === "string" ? message : message.toString("utf8"); + try { + return JSON.parse(raw) as { type?: unknown; message?: Record }; + } catch { + return null; + } +} + +function sessionMessage(message: WebSocketMessage): Record | null { + const envelope = parseEnvelope(message); + return envelope?.type === "session" && envelope.message ? envelope.message : null; +} + +function sendSessionMessage( + ws: { send(message: string): void }, + message: Record, +): void { + ws.send(JSON.stringify({ type: "session", message })); +} + +function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { + const envelope = parseEnvelope(message); + const payload = envelope?.message?.payload; + if ( + envelope?.message?.type !== "status" || + !payload || + typeof payload !== "object" || + (payload as { status?: unknown }).status !== "server_info" + ) { + return message; + } + + (payload as Record).capabilities = { + voice: { + dictation: { enabled: true, reason: "" }, + voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, + }, + }; + return JSON.stringify(envelope); +} + +async function installSyntheticMicrophone(page: Page): Promise { + await page.addInitScript(() => { + const mediaDevices = navigator.mediaDevices; + Object.defineProperty(mediaDevices, "getUserMedia", { + configurable: true, + value: async () => { + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const destination = context.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + return destination.stream; + }, + }); + }); +} + async function installDictationFailureHarness(page: Page) { let resolveAudioChunk!: () => void; let resolveCreateRequest!: () => void; diff --git a/packages/app/e2e/support/helpers/dictation.ts b/packages/app/e2e/support/helpers/dictation.ts index d9554e292b..f99ddede05 100644 --- a/packages/app/e2e/support/helpers/dictation.ts +++ b/packages/app/e2e/support/helpers/dictation.ts @@ -1,51 +1,4 @@ import { expect, type Page } from "../fixtures"; -import { daemonWsRoutePattern } from "./daemon-port"; - -type WebSocketMessage = string | Buffer; - -function parseEnvelope(message: WebSocketMessage): { - type?: unknown; - message?: Record; -} | null { - const raw = typeof message === "string" ? message : message.toString("utf8"); - try { - return JSON.parse(raw) as { type?: unknown; message?: Record }; - } catch { - return null; - } -} - -export function sessionMessage(message: WebSocketMessage): Record | null { - const envelope = parseEnvelope(message); - return envelope?.type === "session" && envelope.message ? envelope.message : null; -} - -export function sendSessionMessage( - ws: { send(message: string): void }, - message: Record, -): void { - ws.send(JSON.stringify({ type: "session", message })); -} - -export function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { - const envelope = parseEnvelope(message); - const payload = envelope?.message?.payload; - if ( - envelope?.message?.type !== "status" || - !payload || - typeof payload !== "object" || - (payload as { status?: unknown }).status !== "server_info" - ) { - return message; - } - (payload as Record).capabilities = { - voice: { - dictation: { enabled: true, reason: "" }, - voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, - }, - }; - return JSON.stringify(envelope); -} /** * Replaces the browser microphone with an oscillator. `silent` keeps the stream @@ -74,88 +27,58 @@ export async function installSyntheticMicrophone( }, options.silent ?? false); } -/** - * Answers dictation stream requests in place of the daemon. Each finish is - * answered with the next queued transcript; everything else reaches the daemon. - */ -export async function installDictationTranscriptHarness( - page: Page, - transcripts: string[], - options: { partialTranscript?: string } = {}, -) { - const queue = [...transcripts]; - const streams = new Map>(); - const recordings: string[][] = []; - let resolveAudioChunk!: () => void; - const audioChunk = new Promise((resolve) => { - resolveAudioChunk = resolve; +/** Observe real browser/daemon traffic without intercepting or answering it. */ +export function observeDictation(page: Page) { + const streams = new Map; finalSeq: number }>(); + const recordings: Array<{ audio: string[]; finalSeq: number; text: string }> = []; + let resolveAudio!: () => void; + const audio = new Promise((resolve) => { + resolveAudio = resolve; }); - await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { - const server = ws.connectToServer(); - - ws.onMessage((message) => { - const request = sessionMessage(message); - if (!request) { - server.send(message); - return; + page.on("websocket", (socket) => { + socket.on("framesent", ({ payload }) => { + const request = sessionMessage(payload); + if (!request || typeof request.dictationId !== "string") return; + const id = request.dictationId; + if (request.type === "dictation_stream_start") { + streams.set(id, { audio: new Map(), finalSeq: -1 }); } - const type = request.type; - const dictationId = typeof request.dictationId === "string" ? request.dictationId : null; - - if (type === "dictation_stream_start" && dictationId) { - streams.set(dictationId, new Map()); - sendSessionMessage(ws, { - type: "dictation_stream_ack", - payload: { dictationId, ackSeq: -1 }, - }); - return; + if (request.type === "dictation_stream_chunk") { + streams.get(id)!.audio.set(request.seq as number, request.audio as string); + resolveAudio(); } - if (type === "dictation_stream_chunk" && dictationId) { - const seq = typeof request?.seq === "number" ? request.seq : 0; - sendSessionMessage(ws, { - type: "dictation_stream_ack", - payload: { dictationId, ackSeq: seq }, - }); - streams.get(dictationId)?.set(seq, String(request.audio)); - if (options.partialTranscript) { - sendSessionMessage(ws, { - type: "dictation_stream_partial", - payload: { dictationId, text: options.partialTranscript }, - }); - } - resolveAudioChunk(); - return; + if (request.type === "dictation_stream_finish") { + streams.get(id)!.finalSeq = request.finalSeq as number; } - if (type === "dictation_stream_finish" && dictationId) { - recordings.push( - [...(streams.get(dictationId)?.entries() ?? [])] - .sort(([a], [b]) => a - b) - .map(([, audio]) => audio), - ); - sendSessionMessage(ws, { - type: "dictation_stream_finish_accepted", - payload: { dictationId, timeoutMs: 5_000 }, - }); - sendSessionMessage(ws, { - type: "dictation_stream_final", - payload: { dictationId, text: queue.shift() ?? "" }, - }); - return; - } - - server.send(message); }); - - server.onMessage((message) => ws.send(enableDictationCapability(message))); + socket.on("framereceived", ({ payload }) => { + const response = sessionMessage(payload); + if (response?.type !== "dictation_stream_final") return; + const final = response.payload as { dictationId: string; text: string }; + const stream = streams.get(final.dictationId)!; + recordings.push({ + audio: [...stream.audio.entries()].sort(([a], [b]) => a - b).map(([, chunk]) => chunk), + finalSeq: stream.finalSeq, + text: final.text, + }); + }); }); - - return { waitForAudio: () => audioChunk, recordings }; + return { waitForAudio: () => audio, recordings }; } -export async function recordDictation(page: Page, harness: { waitForAudio(): Promise }) { +function sessionMessage(payload: string | Buffer): Record | null { + try { + const envelope = JSON.parse(typeof payload === "string" ? payload : payload.toString("utf8")); + return envelope.type === "session" ? envelope.message : null; + } catch { + return null; + } +} + +export async function recordDictation(page: Page, observed: { waitForAudio(): Promise }) { await page.getByRole("button", { name: "Start dictation", exact: true }).click(); - await harness.waitForAudio(); + await observed.waitForAudio(); await page.getByRole("button", { name: "Insert transcription", exact: true }).click(); } diff --git a/packages/app/e2e/support/helpers/isolated-host-daemon.ts b/packages/app/e2e/support/helpers/isolated-host-daemon.ts index c8dd3eaa44..b19cd72cdb 100644 --- a/packages/app/e2e/support/helpers/isolated-host-daemon.ts +++ b/packages/app/e2e/support/helpers/isolated-host-daemon.ts @@ -182,8 +182,8 @@ export async function startIsolatedHostDaemon( const spawnDaemon = async (): Promise => { const spawnOptions: SpawnOptions = { cwd: serverDir, - env: withDisabledE2ESpeechEnv({ - ...process.env, + env: { + ...withDisabledE2ESpeechEnv(process.env), ...options.environment, PASEO_HOME: paseoHome, PASEO_SERVER_ID: serverId, @@ -192,7 +192,7 @@ export async function startIsolatedHostDaemon( PASEO_RELAY_ENABLED: options.mutableRelay ? undefined : "0", PASEO_NODE_ENV: "development", NODE_ENV: "development", - }), + }, stdio: ["ignore", "ignore", "pipe"], detached: false, }; diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index f95607b93f..a646f94605 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -8,7 +8,7 @@ import type { TFunction } from "i18next"; import { VolumeMeter } from "./volume-meter"; import { FOOTER_HEIGHT } from "@/constants/layout"; import type { DictationStatus } from "@/hooks/use-dictation"; -import type { DictationEmptyReason } from "@/dictation/empty-transcript-reason"; +import type { DictationEmptyReason } from "@/dictation/result"; interface DictationControlsProps { volume: number; diff --git a/packages/app/src/dictation/empty-transcript-reason.test.ts b/packages/app/src/dictation/empty-transcript-reason.test.ts deleted file mode 100644 index 2a73e813dd..0000000000 --- a/packages/app/src/dictation/empty-transcript-reason.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { resolveDictationEmptyReason } from "@/dictation/empty-transcript-reason"; - -describe("resolveDictationEmptyReason", () => { - it("reports when the microphone delivered no audio", () => { - expect(resolveDictationEmptyReason({ segmentCount: 0 })).toBe("no-audio"); - }); - - it("does not infer the server's reason from recorded audio", () => { - expect(resolveDictationEmptyReason({ segmentCount: 40 })).toBe("no-transcript"); - }); -}); diff --git a/packages/app/src/dictation/empty-transcript-reason.ts b/packages/app/src/dictation/empty-transcript-reason.ts deleted file mode 100644 index 1c3e49d866..0000000000 --- a/packages/app/src/dictation/empty-transcript-reason.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** What the client knows after a dictation finishes without text. */ -export type DictationEmptyReason = "no-audio" | "no-transcript"; - -export function resolveDictationEmptyReason(input: { segmentCount: number }): DictationEmptyReason { - return input.segmentCount === 0 ? "no-audio" : "no-transcript"; -} diff --git a/packages/app/src/dictation/result.test.ts b/packages/app/src/dictation/result.test.ts new file mode 100644 index 0000000000..e0db8fd370 --- /dev/null +++ b/packages/app/src/dictation/result.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { completeDictation, type DictationRecording } from "./result"; + +class RecordedAudio implements DictationRecording { + constructor(private segments: string[]) {} + getSegmentCount() { + return this.segments.length; + } + clearAll() { + this.segments = []; + } +} + +describe("completing dictation", () => { + it("keeps a recording through repeated empty results until a retry returns text", () => { + const recording = new RecordedAudio(["first PCM chunk", "second PCM chunk"]); + const empty = { finalText: " ", partialText: "", recording }; + const result = completeDictation(empty); + expect(recording.getSegmentCount()).toBe(2); + expect(result).toEqual({ status: "empty", reason: "no-transcript" }); + expect(completeDictation(empty)).toEqual({ status: "empty", reason: "no-transcript" }); + expect(recording.getSegmentCount()).toBe(2); + expect(completeDictation({ ...empty, finalText: " recovered " })).toEqual({ + status: "transcribed", + text: "recovered", + }); + expect(recording.getSegmentCount()).toBe(0); + }); + + it("keeps usable partial text when the final response is empty", () => { + const recording = new RecordedAudio(["PCM chunk"]); + expect( + completeDictation({ finalText: "", partialText: " recognized words ", recording }), + ).toEqual({ + status: "transcribed", + text: "recognized words", + }); + expect(recording.getSegmentCount()).toBe(0); + }); + + it("reports a missing recording without inventing a reason for the empty result", () => { + expect( + completeDictation({ finalText: "", partialText: "", recording: new RecordedAudio([]) }), + ).toEqual({ + status: "empty", + reason: "no-audio", + }); + }); +}); diff --git a/packages/app/src/dictation/result.ts b/packages/app/src/dictation/result.ts new file mode 100644 index 0000000000..97ee926c3d --- /dev/null +++ b/packages/app/src/dictation/result.ts @@ -0,0 +1,26 @@ +export type DictationEmptyReason = "no-audio" | "no-transcript"; + +export interface DictationRecording { + getSegmentCount(): number; + clearAll(): void; +} + +export type DictationResult = + | { status: "empty"; reason: DictationEmptyReason } + | { status: "transcribed"; text: string }; + +export function completeDictation(input: { + finalText: string; + partialText: string; + recording: DictationRecording; +}): DictationResult { + const text = input.finalText.trim() || input.partialText.trim(); + if (!text) { + return { + status: "empty", + reason: input.recording.getSegmentCount() === 0 ? "no-audio" : "no-transcript", + }; + } + input.recording.clearAll(); + return { status: "transcribed", text }; +} diff --git a/packages/app/src/hooks/use-dictation.shared.ts b/packages/app/src/hooks/use-dictation.shared.ts index e2e885428f..b76eb076ed 100644 --- a/packages/app/src/hooks/use-dictation.shared.ts +++ b/packages/app/src/hooks/use-dictation.shared.ts @@ -1,4 +1,4 @@ -import type { DictationEmptyReason } from "@/dictation/empty-transcript-reason"; +import type { DictationEmptyReason } from "@/dictation/result"; import { i18n } from "@/i18n/i18next"; export type DictationStatus = diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index f2c8f050f9..81bf4cfffb 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -2,10 +2,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { DictationStreamSender } from "@/dictation/dictation-stream-sender"; -import { - resolveDictationEmptyReason, - type DictationEmptyReason, -} from "@/dictation/empty-transcript-reason"; +import { completeDictation, type DictationEmptyReason } from "@/dictation/result"; import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source"; import { generateMessageId } from "@/types/stream"; import { AttemptGuard } from "@/utils/attempt-guard"; @@ -136,13 +133,17 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { [setError], ); - const clearStreamingState = useCallback(() => { - senderRef.current?.clearAll(); + const clearTranscriptState = useCallback(() => { latestPartialTranscriptRef.current = ""; setPartialTranscript(""); setEmptyReason(null); }, []); + const clearStreamingState = useCallback(() => { + senderRef.current?.clearAll(); + clearTranscriptState(); + }, [clearTranscriptState]); + const startNewStream = useCallback(async (reason: string) => { await senderRef.current?.restartStream(reason); }, []); @@ -198,23 +199,20 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { setDuration(0); setStatus("idle"); - const transcriptText = - text.trim().length > 0 ? text.trim() : latestPartialTranscriptRef.current.trim(); - - if (!transcriptText) { - // Keep the audio until the user retries or discards this empty result. + const result = completeDictation({ + finalText: text, + partialText: latestPartialTranscriptRef.current, + recording: senderRef.current!, + }); + if (result.status === "empty") { setStatus("empty"); - setEmptyReason( - resolveDictationEmptyReason({ - segmentCount: senderRef.current?.getSegmentCount() ?? 0, - }), - ); + setEmptyReason(result.reason); return; } - clearStreamingState(); - onTranscriptRef.current?.(transcriptText, { requestId }); + clearTranscriptState(); + onTranscriptRef.current?.(result.text, { requestId }); }, - [clearStreamingState], + [clearTranscriptState], ); const handleDictationFailure = useCallback(