From f8985372331a6d18c661ad026a4e8a879d905948 Mon Sep 17 00:00:00 2001 From: d0zingcat Date: Sun, 18 Oct 2026 14:22:09 +0800 Subject: [PATCH] fix(dictation): keep transcription context opt-in An English instruction used as transcript context caused the transcription model to discard Portuguese list items. Omit built-in context and retain explicit vocabulary overrides, including an empty override to disable it. Co-authored-by: d0zingcat Co-authored-by: Joey Hotz --- .../scripts/dictation-generate-baseline.ts | 8 +-- .../dictation-stream-manager.test.ts | 57 +++++++++++++++++++ .../dictation/dictation-stream-manager.ts | 14 +++-- public-docs/voice.md | 11 ++++ 4 files changed, 82 insertions(+), 8 deletions(-) diff --git a/packages/server/scripts/dictation-generate-baseline.ts b/packages/server/scripts/dictation-generate-baseline.ts index 8cc69ddf85..8db36c5763 100644 --- a/packages/server/scripts/dictation-generate-baseline.ts +++ b/packages/server/scripts/dictation-generate-baseline.ts @@ -33,9 +33,9 @@ async function main(): Promise { const apiKey = requireEnv("OPENAI_API_KEY"); const transcriptionModel = process.env.STT_MODEL ?? "gpt-4o-transcribe"; - const prompt = - process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT ?? - "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript."; + // Mirrors the daemon: no built-in prompt, only an explicit override. Whisper + // treats `prompt` as preceding transcript, so a default would bias the baseline. + const prompt = process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT?.trim() || undefined; const openai = new OpenAI({ apiKey }); @@ -43,7 +43,7 @@ async function main(): Promise { file: await import("node:fs").then((fs) => fs.createReadStream(audioPath)), language: "en", model: transcriptionModel, - prompt, + ...(prompt ? { prompt } : {}), response_format: "json", // Aim for determinism. temperature: 0, diff --git a/packages/server/src/server/dictation/dictation-stream-manager.test.ts b/packages/server/src/server/dictation/dictation-stream-manager.test.ts index 1d258553ca..cd8e7c1e5b 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.test.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.test.ts @@ -57,11 +57,13 @@ class FakeRealtimeSession extends EventEmitter implements StreamingTranscription class FakeSttProvider implements SpeechToTextProvider { public readonly id = "fake"; public lastLanguage?: string; + public lastPrompt?: string; constructor(private readonly session: FakeRealtimeSession) {} createSession( params: Parameters[0], ): StreamingTranscriptionSession { this.lastLanguage = params.language; + this.lastPrompt = params.prompt; return this.session; } } @@ -594,3 +596,58 @@ it("closes every dictation stream when one provider cleanup fails", async () => expect(second.closed).toBe(true); expect(manager.hasDemand).toBe(false); }); + +describe("DictationStreamManager (transcription prompt)", () => { + const originalPromptEnv = process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT; + + afterEach(() => { + if (originalPromptEnv === undefined) { + delete process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT; + } else { + process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT = originalPromptEnv; + } + }); + + const startDictation = async ( + options: { language?: string } = {}, + ): Promise<{ provider: FakeSttProvider; manager: DictationStreamManager }> => { + const session = new FakeRealtimeSession(); + const provider = new FakeSttProvider(session); + const manager = new DictationStreamManager({ + logger: pino({ level: "silent" }), + emit: () => {}, + sessionId: "prompt", + stt: provider, + ...(options.language ? { language: options.language } : {}), + }); + await manager.handleStart("d1", "audio/pcm;rate=24000;bits=16"); + manager.cleanupAll(); + return { provider, manager }; + }; + + it("sends no prompt by default", async () => { + delete process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT; + const { provider } = await startDictation(); + expect(provider.lastPrompt).toBeUndefined(); + }); + + it("sends no prompt for non-English dictation by default", async () => { + delete process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT; + const { provider } = await startDictation({ language: "zh" }); + expect(provider.lastLanguage).toBe("zh"); + expect(provider.lastPrompt).toBeUndefined(); + }); + + it("passes an explicit prompt override through", async () => { + process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT = + " Vocabulary: Paseo, Dokploy, Kubernetes. "; + const { provider } = await startDictation(); + expect(provider.lastPrompt).toBe("Vocabulary: Paseo, Dokploy, Kubernetes."); + }); + + it.each(["", " "])("treats a blank override (%j) as no prompt", async (value) => { + process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT = value; + const { provider } = await startDictation(); + expect(provider.lastPrompt).toBeUndefined(); + }); +}); diff --git a/packages/server/src/server/dictation/dictation-stream-manager.ts b/packages/server/src/server/dictation/dictation-stream-manager.ts index b1b061501f..9c9c957265 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.ts @@ -38,6 +38,12 @@ function parseNonNegativeNumber(value: string | undefined): number | null { return parsed; } +/** Unset, empty, and whitespace-only overrides all mean "send no prompt". */ +function readTranscriptionPromptOverride(env: NodeJS.ProcessEnv): string | undefined { + const value = env.PASEO_DICTATION_TRANSCRIPTION_PROMPT?.trim(); + return value ? value : undefined; +} + function convertPCMToWavBuffer( pcmBuffer: Buffer, sampleRate: number, @@ -185,16 +191,16 @@ export class DictationStreamManager { return; } - const transcriptionPrompt = - process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT ?? - "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript."; + // Whisper uses the prompt as transcript context, not silence instructions. + // Keep context opt-in so it does not bias speech or introduce unspoken text. + const transcriptionPrompt = readTranscriptionPromptOverride(process.env); let stt: ReturnType; try { stt = sttProvider.createSession({ logger: this.logger.child({ dictationId }), language: this.language, - prompt: transcriptionPrompt, + ...(transcriptionPrompt ? { prompt: transcriptionPrompt } : {}), }); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/public-docs/voice.md b/public-docs/voice.md index c90e5196bc..214044556f 100644 --- a/public-docs/voice.md +++ b/public-docs/voice.md @@ -142,8 +142,19 @@ Keys and base URLs follow the fallback rule above. - `PASEO_VOICE_LOCAL_STT_MODEL`, `PASEO_VOICE_LOCAL_TTS_MODEL`, local voice STT/TTS model IDs - `PASEO_DICTATION_LANGUAGE`, dictation STT language - `PASEO_VOICE_LANGUAGE`, voice mode STT language; falls back to `PASEO_DICTATION_LANGUAGE` when unset +- `PASEO_DICTATION_TRANSCRIPTION_PROMPT`, optional dictation transcription prompt. Unset by default; empty or whitespace-only is treated as unset - `PASEO_VOICE_LOCAL_TTS_SPEAKER_ID`, `PASEO_VOICE_LOCAL_TTS_SPEED`, optional local voice TTS tuning +## Transcription Prompt + +Paseo sends no transcription prompt by default. Whisper-family models use `prompt` as transcript context, so instruction text can distort the transcript or bias its language. Omitting it does not prevent every hallucination on silence or noise. + +Set `PASEO_DICTATION_TRANSCRIPTION_PROMPT` to send a prompt explicitly, for example to bias toward project vocabulary. An empty or whitespace-only value means no prompt. This affects dictation only; voice mode never sends a prompt. + +```bash +PASEO_DICTATION_TRANSCRIPTION_PROMPT="Vocabulary: Paseo, TypeScript." +``` + ## Operational Notes Voice mode can launch and control agents. Treat voice prompts with the same care as direct agent instructions, especially when specifying working directories or destructive operations.