diff --git a/docs/data-model.md b/docs/data-model.md index fe90e475d..7c635ad9b 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -163,7 +163,7 @@ Single file, validated with `PersistedConfigSchema`. root?: string // optional root for new worktrees; defaults to $PASEO_HOME/worktrees }, providers: { - openai: { apiKey: string }, + openai: { voice: { apiKey: string, baseUrl: string } }, local: { modelsDir: string } }, agents: { @@ -193,6 +193,37 @@ All fields are optional with sensible defaults. Local speech model ids are intentionally narrow: STT uses `parakeet-tdt-0.6b-v2-int8`, TTS uses `kokoro-en-v0_19`, and turn detection uses the bundled Silero VAD model. +Set these to select OpenAI instead of local speech: + +| Env var | Applies to | +| ------------------------------ | ------------------------------- | +| `PASEO_VOICE_STT_PROVIDER` | Voice mode STT provider | +| `PASEO_DICTATION_STT_PROVIDER` | Composer dictation STT provider | +| `PASEO_VOICE_TTS_PROVIDER` | Voice mode TTS provider | + +OpenAI voice can be configured under `providers.openai`: + +```json +{ + "providers": { + "openai": { + "voice": { + "apiKey": "sk-...", + "baseUrl": "https://api.openai.com/v1" + } + } + } +} +``` + +`providers.openai.voice.apiKey` and `providers.openai.voice.baseUrl` apply only to Paseo OpenAI voice features. + +Paseo uses these paths under the configured OpenAI base URL: + +- dictation STT: `/v1/audio/transcriptions` +- voice mode STT: `/v1/audio/transcriptions` +- voice mode TTS: `/v1/audio/speech` + --- ## 3. Schedule diff --git a/package.json b/package.json index 726ad9f38..c89a76cb7 100644 --- a/package.json +++ b/package.json @@ -2,12 +2,11 @@ "name": "paseo", "version": "0.1.100", "private": true, - "description": "Paseo: voice-controlled development environment with OpenAI Realtime API", + "description": "Paseo: voice-controlled development environment for local AI coding agents", "keywords": [ "development", "mcp", "openai", - "realtime", "voice", "voice-assistant" ], diff --git a/packages/server/scripts/dictation-generate-baseline.ts b/packages/server/scripts/dictation-generate-baseline.ts index 833c60529..8cc69ddf8 100644 --- a/packages/server/scripts/dictation-generate-baseline.ts +++ b/packages/server/scripts/dictation-generate-baseline.ts @@ -32,9 +32,9 @@ async function main(): Promise { } const apiKey = requireEnv("OPENAI_API_KEY"); - const transcriptionModel = process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe"; + const transcriptionModel = process.env.STT_MODEL ?? "gpt-4o-transcribe"; const prompt = - process.env.OPENAI_REALTIME_DICTATION_TRANSCRIPTION_PROMPT ?? + process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT ?? "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript."; const openai = new OpenAI({ apiKey }); diff --git a/packages/server/src/server/persisted-config.test.ts b/packages/server/src/server/persisted-config.test.ts index fbbff8582..2269bf51b 100644 --- a/packages/server/src/server/persisted-config.test.ts +++ b/packages/server/src/server/persisted-config.test.ts @@ -75,6 +75,24 @@ describe("PersistedConfigSchema worktrees config", () => { }); }); +describe("PersistedConfigSchema provider credentials", () => { + test("accepts OpenAI voice credentials", () => { + const parsed = PersistedConfigSchema.parse({ + providers: { + openai: { + voice: { + apiKey: " voice-secret ", + baseUrl: " https://voice.example.com/v1 ", + }, + }, + }, + }); + + expect(parsed.providers?.openai?.voice?.apiKey).toBe("voice-secret"); + expect(parsed.providers?.openai?.voice?.baseUrl).toBe("https://voice.example.com/v1"); + }); +}); + describe("PersistedConfigSchema daemon append system prompt", () => { test("accepts optional append system prompt", () => { const parsed = PersistedConfigSchema.parse({ diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 65d8c8e8b..51fa73eb8 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -45,9 +45,18 @@ const LogConfigSchema = z }) .strict(); -const ProviderCredentialsSchema = z +const OpenAiVoiceProviderSchema = z + .object({ + apiKey: z.string().trim().min(1).optional(), + baseUrl: z.string().trim().min(1).optional(), + }) + .strict(); + +const OpenAiProviderSchema = z .object({ apiKey: z.string().min(1).optional(), + voice: OpenAiVoiceProviderSchema.optional(), + baseUrl: z.string().trim().min(1).optional(), }) .strict(); @@ -59,7 +68,7 @@ const LocalSpeechProviderSchema = z const ProvidersSchema = z .object({ - openai: ProviderCredentialsSchema.optional(), + openai: OpenAiProviderSchema.optional(), local: LocalSpeechProviderSchema.optional(), }) .strict(); diff --git a/packages/server/src/server/speech/providers/openai/config.test.ts b/packages/server/src/server/speech/providers/openai/config.test.ts index 10b04364f..a4eaffc52 100644 --- a/packages/server/src/server/speech/providers/openai/config.test.ts +++ b/packages/server/src/server/speech/providers/openai/config.test.ts @@ -43,4 +43,118 @@ describe("resolveOpenAiSpeechConfig", () => { expect(resolved?.stt?.apiKey).toBe("sk-test"); expect(resolved?.tts?.apiKey).toBe("sk-test"); }); + + test("uses nested voice config before env and non-voice fallbacks", () => { + const persisted = PersistedConfigSchema.parse({ + providers: { + openai: { + apiKey: "fallback-config-key", + voice: { + apiKey: "voice-config-key", + baseUrl: " https://voice.example.com/v1 ", + }, + baseUrl: "https://legacy-config.example.com/v1", + }, + }, + }); + const env = { + OPENAI_API_KEY: "env-key", + OPENAI_VOICE_API_KEY: "voice-env-key", + OPENAI_VOICE_BASE_URL: "https://voice-env.example.com/v1", + OPENAI_BASE_URL: "https://env.example.com/v1", + } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ + env, + persisted, + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + }); + + expect(resolved?.apiKey).toBe("voice-config-key"); + expect(resolved?.baseUrl).toBe("https://voice.example.com/v1"); + expect(resolved?.stt?.apiKey).toBe("voice-config-key"); + expect(resolved?.stt?.baseUrl).toBe("https://voice.example.com/v1"); + expect(resolved?.tts?.apiKey).toBe("voice-config-key"); + expect(resolved?.tts?.baseUrl).toBe("https://voice.example.com/v1"); + }); + + test("uses voice env config when nested voice config is unset", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + OPENAI_API_KEY: "sk-test", + OPENAI_VOICE_API_KEY: "voice-env-key", + OPENAI_VOICE_BASE_URL: " https://voice-env.example.com/v1 ", + OPENAI_BASE_URL: "https://env.example.com/v1", + } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ + env, + persisted, + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + }); + + expect(resolved?.apiKey).toBe("voice-env-key"); + expect(resolved?.stt?.apiKey).toBe("voice-env-key"); + expect(resolved?.tts?.apiKey).toBe("voice-env-key"); + expect(resolved?.baseUrl).toBe("https://voice-env.example.com/v1"); + expect(resolved?.stt?.baseUrl).toBe("https://voice-env.example.com/v1"); + expect(resolved?.tts?.baseUrl).toBe("https://voice-env.example.com/v1"); + }); + + test("falls back to non-voice OpenAI config", () => { + const persisted = PersistedConfigSchema.parse({ + providers: { + openai: { + apiKey: "fallback-config-key", + baseUrl: " https://legacy-config.example.com/v1 ", + }, + }, + }); + const env = { + OPENAI_API_KEY: "sk-test", + OPENAI_BASE_URL: " https://env.example.com/v1 ", + } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ + env, + persisted, + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + }); + + expect(resolved?.apiKey).toBe("fallback-config-key"); + expect(resolved?.baseUrl).toBe("https://legacy-config.example.com/v1"); + }); + + test("falls back to global OpenAI env config when voice-specific inputs are unset", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + OPENAI_API_KEY: "env-key", + OPENAI_BASE_URL: " https://env.example.com/v1 ", + } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ + env, + persisted, + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + }); + + expect(resolved?.apiKey).toBe("env-key"); + expect(resolved?.baseUrl).toBe("https://env.example.com/v1"); + }); }); diff --git a/packages/server/src/server/speech/providers/openai/config.ts b/packages/server/src/server/speech/providers/openai/config.ts index 103be8fac..6d3352c0d 100644 --- a/packages/server/src/server/speech/providers/openai/config.ts +++ b/packages/server/src/server/speech/providers/openai/config.ts @@ -5,14 +5,13 @@ import type { RequestedSpeechProviders } from "../../speech-types.js"; import type { STTConfig } from "./stt.js"; import type { TTSConfig } from "./tts.js"; -export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; export const DEFAULT_OPENAI_TTS_MODEL = "tts-1"; export interface OpenAiSpeechProviderConfig { apiKey?: string; + baseUrl?: string; stt?: Partial & { apiKey?: string }; tts?: Partial & { apiKey?: string }; - realtimeTranscriptionModel?: string; } const OpenAiTtsVoiceSchema = z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]); @@ -33,6 +32,7 @@ const OptionalTrimmedStringSchema = z const OpenAiSpeechResolutionSchema = z.object({ apiKey: OptionalTrimmedStringSchema, + baseUrl: OptionalTrimmedStringSchema, sttConfidenceThreshold: OptionalFiniteNumberSchema, sttModel: OptionalTrimmedStringSchema, ttsVoice: z.string().trim().toLowerCase().pipe(OpenAiTtsVoiceSchema).default("alloy"), @@ -42,9 +42,6 @@ const OpenAiSpeechResolutionSchema = z.object({ .toLowerCase() .pipe(OpenAiTtsModelSchema) .default(DEFAULT_OPENAI_TTS_MODEL), - realtimeTranscriptionModel: OptionalTrimmedStringSchema.default( - DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - ), }); function isOpenAiProviderActive(provider: { enabled?: boolean; provider: string }): boolean { @@ -83,11 +80,6 @@ function buildOpenAiSttInput(params: { pickIfOpenAi(providers.voiceStt, persisted.features?.voiceMode?.stt?.model), pickIfOpenAi(providers.dictationStt, persisted.features?.dictation?.stt?.model), ]), - realtimeTranscriptionModel: firstDefined([ - env.OPENAI_REALTIME_TRANSCRIPTION_MODEL, - pickIfOpenAi(providers.dictationStt, persisted.features?.dictation?.stt?.model), - DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - ]), }; } @@ -118,8 +110,16 @@ function buildOpenAiResolutionInput(params: { }): Record { return { apiKey: firstDefined([ - params.env.OPENAI_API_KEY, + params.persisted.providers?.openai?.voice?.apiKey, + params.env.OPENAI_VOICE_API_KEY, params.persisted.providers?.openai?.apiKey, + params.env.OPENAI_API_KEY, + ]), + baseUrl: firstDefined([ + params.persisted.providers?.openai?.voice?.baseUrl, + params.env.OPENAI_VOICE_BASE_URL, + params.persisted.providers?.openai?.baseUrl, + params.env.OPENAI_BASE_URL, ]), ...buildOpenAiSttInput(params), ...buildOpenAiTtsInput(params), @@ -139,8 +139,10 @@ export function resolveOpenAiSpeechConfig(params: { return { apiKey: parsed.apiKey, + ...(parsed.baseUrl ? { baseUrl: parsed.baseUrl } : {}), stt: { apiKey: parsed.apiKey, + ...(parsed.baseUrl ? { baseUrl: parsed.baseUrl } : {}), ...(parsed.sttConfidenceThreshold !== undefined ? { confidenceThreshold: parsed.sttConfidenceThreshold } : {}), @@ -148,10 +150,10 @@ export function resolveOpenAiSpeechConfig(params: { }, tts: { apiKey: parsed.apiKey, + ...(parsed.baseUrl ? { baseUrl: parsed.baseUrl } : {}), voice: parsed.ttsVoice, model: parsed.ttsModel, responseFormat: "pcm", }, - realtimeTranscriptionModel: parsed.realtimeTranscriptionModel, }; } diff --git a/packages/server/src/server/speech/providers/openai/realtime-transcription-session.ts b/packages/server/src/server/speech/providers/openai/realtime-transcription-session.ts deleted file mode 100644 index b8e8eb00c..000000000 --- a/packages/server/src/server/speech/providers/openai/realtime-transcription-session.ts +++ /dev/null @@ -1,264 +0,0 @@ -import type pino from "pino"; -import { WebSocket } from "ws"; -import { EventEmitter } from "node:events"; -import type { StreamingTranscriptionSession } from "../../speech-provider.js"; - -type OpenAITurnDetection = - | null - | { - type: "server_vad"; - create_response?: boolean; - threshold?: number; - prefix_padding_ms?: number; - silence_duration_ms?: number; - } - | { - type: "semantic_vad"; - create_response?: boolean; - eagerness?: "low" | "medium" | "high"; - }; - -type OpenAIClientEvent = - | { - type: "session.update"; - session: { - type: "transcription"; - audio: { - input: { - format: { type: "audio/pcm"; rate: 24000 }; - transcription: { - model: string; - language?: string; - prompt?: string; - }; - turn_detection: OpenAITurnDetection; - }; - }; - }; - } - | { type: "input_audio_buffer.append"; audio: string } - | { type: "input_audio_buffer.commit" } - | { type: "input_audio_buffer.clear" }; - -type OpenAIServerEvent = - | { type: "session.created" | "session.updated" } - | { - type: "input_audio_buffer.committed"; - item_id: string; - previous_item_id: string | null; - } - | { type: "input_audio_buffer.speech_started" } - | { type: "input_audio_buffer.speech_stopped" } - | { - type: "conversation.item.input_audio_transcription.delta"; - item_id: string; - delta: string; - } - | { - type: "conversation.item.input_audio_transcription.completed"; - item_id: string; - transcript: string; - } - | { type: "error"; error?: { message?: string } }; - -export class OpenAIRealtimeTranscriptionSession - extends EventEmitter - implements StreamingTranscriptionSession -{ - public readonly requiredSampleRate = 24000; - private readonly apiKey: string; - private readonly logger: pino.Logger; - private readonly transcriptionModel: string; - private readonly language?: string; - private readonly prompt?: string; - private readonly turnDetection: OpenAITurnDetection; - - private ws: WebSocket | null = null; - private ready: Promise | null = null; - private closing = false; - private partialByItemId = new Map(); - - constructor(params: { - apiKey: string; - logger: pino.Logger; - transcriptionModel: string; - language?: string; - prompt?: string; - turnDetection?: OpenAITurnDetection; - }) { - super(); - this.apiKey = params.apiKey; - this.logger = params.logger.child({ provider: "openai", component: "realtime-transcription" }); - this.transcriptionModel = params.transcriptionModel; - this.language = params.language; - this.prompt = params.prompt; - this.turnDetection = params.turnDetection ?? null; - } - - public async connect(): Promise { - if (this.ready) { - return this.ready; - } - - this.closing = false; - this.ready = new Promise((resolve, reject) => { - const url = - process.env.OPENAI_REALTIME_URL ?? "wss://api.openai.com/v1/realtime?intent=transcription"; - const ws = new WebSocket(url, { - headers: { - Authorization: `Bearer ${this.apiKey}`, - }, - }); - this.ws = ws; - - let resolved = false; - - const fail = (error: Error) => { - if (resolved) { - this.emit("error", error); - return; - } - resolved = true; - reject(error); - }; - - ws.on("open", () => { - this.logger.debug("OpenAI realtime transcription websocket connected"); - const update: OpenAIClientEvent = { - type: "session.update", - session: { - type: "transcription", - audio: { - input: { - format: { type: "audio/pcm", rate: 24000 }, - transcription: { - model: this.transcriptionModel, - ...(this.language ? { language: this.language } : {}), - ...(this.prompt ? { prompt: this.prompt } : {}), - }, - turn_detection: this.turnDetection, - }, - }, - }, - }; - ws.send(JSON.stringify(update)); - }); - - ws.on("message", (data) => { - const text = typeof data === "string" ? data : data.toString("utf-8"); - let parsed: unknown; - try { - parsed = JSON.parse(text); - } catch { - return; - } - - const event = parsed as OpenAIServerEvent; - if (event.type === "session.created" || event.type === "session.updated") { - if (!resolved) { - resolved = true; - resolve(); - } - return; - } - - if (event.type === "input_audio_buffer.committed") { - this.emit("committed", { - segmentId: event.item_id, - previousSegmentId: event.previous_item_id, - }); - return; - } - - if (event.type === "input_audio_buffer.speech_started") { - this.emit("speech_started"); - return; - } - - if (event.type === "input_audio_buffer.speech_stopped") { - this.emit("speech_stopped"); - return; - } - - if (event.type === "conversation.item.input_audio_transcription.delta") { - const replaceDelta = this.transcriptionModel === "whisper-1"; - const prev = this.partialByItemId.get(event.item_id) ?? ""; - const next = replaceDelta ? event.delta : prev + event.delta; - this.partialByItemId.set(event.item_id, next); - this.emit("transcript", { segmentId: event.item_id, transcript: next, isFinal: false }); - return; - } - - if (event.type === "conversation.item.input_audio_transcription.completed") { - this.partialByItemId.set(event.item_id, event.transcript); - this.emit("transcript", { - segmentId: event.item_id, - transcript: event.transcript, - isFinal: true, - }); - return; - } - - if (event.type === "error") { - const message = event.error?.message ?? "OpenAI realtime error"; - fail(new Error(message)); - } - }); - - ws.on("error", (err) => { - fail(err instanceof Error ? err : new Error(String(err))); - }); - - ws.on("close", () => { - this.logger.debug("OpenAI realtime websocket closed"); - if (this.closing) { - return; - } - if (!resolved) { - fail(new Error("OpenAI realtime websocket closed before ready")); - return; - } - fail(new Error("OpenAI realtime websocket closed")); - }); - }); - - return this.ready; - } - - public appendPcm16(pcm16le: Buffer): void { - if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { - throw new Error("OpenAI realtime websocket not connected"); - } - const base64Audio = pcm16le.toString("base64"); - const event: OpenAIClientEvent = { type: "input_audio_buffer.append", audio: base64Audio }; - this.ws.send(JSON.stringify(event)); - } - - public commit(): void { - if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { - throw new Error("OpenAI realtime websocket not connected"); - } - const event: OpenAIClientEvent = { type: "input_audio_buffer.commit" }; - this.ws.send(JSON.stringify(event)); - } - - public clear(): void { - if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { - return; - } - const event: OpenAIClientEvent = { type: "input_audio_buffer.clear" }; - this.ws.send(JSON.stringify(event)); - } - - public close(): void { - try { - this.closing = true; - this.ws?.close(); - } catch { - // no-op - } finally { - this.ws = null; - this.ready = null; - } - } -} diff --git a/packages/server/src/server/speech/providers/openai/runtime.test.ts b/packages/server/src/server/speech/providers/openai/runtime.test.ts new file mode 100644 index 000000000..e30ca0fb3 --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/runtime.test.ts @@ -0,0 +1,35 @@ +import pino from "pino"; +import { describe, expect, test } from "vitest"; + +import { initializeOpenAiSpeechServices } from "./runtime.js"; +import { OpenAISTT } from "./stt.js"; +import { OpenAITTS } from "./tts.js"; + +describe("initializeOpenAiSpeechServices", () => { + test("uses REST OpenAI STT for voice and dictation", () => { + const services = initializeOpenAiSpeechServices({ + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceTurnDetection: { provider: "local", explicit: false, enabled: false }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + openaiConfig: { + apiKey: "sk-test", + stt: { apiKey: "sk-test" }, + tts: { apiKey: "sk-test" }, + }, + existing: { + turnDetectionService: null, + sttService: null, + ttsService: null, + dictationSttService: null, + }, + logger: pino({ level: "silent" }), + }); + + expect(services.sttService).toBeInstanceOf(OpenAISTT); + expect(services.dictationSttService).toBeInstanceOf(OpenAISTT); + expect(services.ttsService).toBeInstanceOf(OpenAITTS); + }); +}); diff --git a/packages/server/src/server/speech/providers/openai/runtime.ts b/packages/server/src/server/speech/providers/openai/runtime.ts index dfd68563c..392e90265 100644 --- a/packages/server/src/server/speech/providers/openai/runtime.ts +++ b/packages/server/src/server/speech/providers/openai/runtime.ts @@ -3,12 +3,7 @@ import type { Logger } from "pino"; import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js"; import type { RequestedSpeechProviders } from "../../speech-types.js"; import type { TurnDetectionProvider } from "../../turn-detection-provider.js"; -import { - DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - DEFAULT_OPENAI_TTS_MODEL, - type OpenAiSpeechProviderConfig, -} from "./config.js"; -import { OpenAIRealtimeTranscriptionSession } from "./realtime-transcription-session.js"; +import { DEFAULT_OPENAI_TTS_MODEL, type OpenAiSpeechProviderConfig } from "./config.js"; import { OpenAISTT } from "./stt.js"; import { OpenAITTS } from "./tts.js"; @@ -126,25 +121,6 @@ function createOpenAiTts( ); } -function createOpenAiDictationService( - apiKey: string, - openaiConfig: OpenAiSpeechProviderConfig | undefined, -): SpeechToTextProvider { - return { - id: "openai", - createSession: ({ logger: sessionLogger, language, prompt }) => - new OpenAIRealtimeTranscriptionSession({ - apiKey, - logger: sessionLogger, - transcriptionModel: - openaiConfig?.realtimeTranscriptionModel ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, - ...(language ? { language } : {}), - ...(prompt ? { prompt } : {}), - turnDetection: null, - }), - }; -} - export function initializeOpenAiSpeechServices(params: { providers: RequestedSpeechProviders; openaiConfig: OpenAiSpeechProviderConfig | undefined; @@ -186,9 +162,10 @@ export function initializeOpenAiSpeechServices(params: { } if (needsOpenAiDictation && openAiCredentials.openaiDictationApiKey) { - dictationSttService = createOpenAiDictationService( + dictationSttService = createOpenAiStt( openAiCredentials.openaiDictationApiKey, openaiConfig, + logger, ); } } else if (needsAnyOpenAi) { diff --git a/packages/server/src/server/speech/providers/openai/stt.test.ts b/packages/server/src/server/speech/providers/openai/stt.test.ts new file mode 100644 index 000000000..72707be25 --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/stt.test.ts @@ -0,0 +1,90 @@ +import pino from "pino"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +const { openAiConstructorOptionsMock, transcriptionsCreateMock } = vi.hoisted(() => ({ + openAiConstructorOptionsMock: vi.fn(), + transcriptionsCreateMock: vi.fn(), +})); + +vi.mock("openai", () => ({ + OpenAI: vi.fn(function OpenAI(options: unknown) { + openAiConstructorOptionsMock(options); + return { + audio: { + transcriptions: { + create: transcriptionsCreateMock, + }, + }, + }; + }), +})); + +import { OpenAISTT } from "./stt.js"; + +describe("OpenAISTT", () => { + afterEach(() => { + openAiConstructorOptionsMock.mockReset(); + transcriptionsCreateMock.mockReset(); + }); + + test("passes configured baseUrl to the OpenAI client", () => { + const provider = new OpenAISTT( + { apiKey: "sk-test", baseUrl: "https://speech.example.com/v1" }, + pino({ level: "silent" }), + ); + + expect(provider.id).toBe("openai"); + expect(openAiConstructorOptionsMock).toHaveBeenCalledWith({ + apiKey: "sk-test", + baseURL: "https://speech.example.com/v1", + }); + }); + + test("passes transcription prompt to OpenAI REST STT", async () => { + transcriptionsCreateMock.mockImplementation( + async (request: { file: NodeJS.ReadableStream }) => { + await new Promise((resolve, reject) => { + request.file.once("error", reject); + request.file.once("end", resolve); + request.file.resume(); + }); + return { text: "hello" }; + }, + ); + + const provider = new OpenAISTT( + { apiKey: "sk-test", model: "gpt-4o-transcribe" }, + pino({ level: "silent" }), + ); + const session = provider.createSession({ + logger: pino({ level: "silent" }), + language: "en", + prompt: "Only transcribe the speaker.", + }); + + const transcript = new Promise((resolve, reject) => { + session.on("transcript", (event) => { + if (event.isFinal) { + resolve(event.transcript); + } + }); + session.on("error", (error) => { + reject(error instanceof Error ? error : new Error(String(error))); + }); + }); + + await session.connect(); + session.appendPcm16(Buffer.from([0, 0, 0, 0])); + session.commit(); + + await expect(transcript).resolves.toBe("hello"); + expect(transcriptionsCreateMock).toHaveBeenCalledWith( + expect.objectContaining({ + language: "en", + model: "gpt-4o-transcribe", + prompt: "Only transcribe the speaker.", + response_format: "json", + }), + ); + }); +}); diff --git a/packages/server/src/server/speech/providers/openai/stt.ts b/packages/server/src/server/speech/providers/openai/stt.ts index 6288e5521..6a143bac5 100644 --- a/packages/server/src/server/speech/providers/openai/stt.ts +++ b/packages/server/src/server/speech/providers/openai/stt.ts @@ -17,6 +17,7 @@ export type { LogprobToken, TranscriptionResult }; export interface STTConfig { apiKey: string; + baseUrl?: string; model?: "whisper-1" | "gpt-4o-transcribe" | "gpt-4o-mini-transcribe" | (string & {}); confidenceThreshold?: number; // Default: -3.0 } @@ -56,6 +57,7 @@ export class OpenAISTT implements SpeechToTextProvider { this.logger = parentLogger.child({ module: "agent", provider: "openai", component: "stt" }); this.openaiClient = new OpenAI({ apiKey: sttConfig.apiKey, + ...(sttConfig.baseUrl ? { baseURL: sttConfig.baseUrl } : {}), }); this.logger.info({ model: sttConfig.model || "whisper-1" }, "STT (OpenAI Whisper) initialized"); } @@ -138,7 +140,13 @@ export class OpenAISTT implements SpeechToTextProvider { } const wav = convertPCMToWavBuffer(pcm16); - const result = await transcribeAudio(wav, "audio/wav", params.language ?? "en", logger); + const result = await transcribeAudio( + wav, + "audio/wav", + params.language ?? "en", + logger, + params.prompt, + ); emitter.emit("transcript", { segmentId: committedId, @@ -178,6 +186,7 @@ export class OpenAISTT implements SpeechToTextProvider { format: string, language: string, logger: pino.Logger, + prompt?: string, ): Promise { const startTime = Date.now(); let tempFilePath: string | null = null; @@ -198,6 +207,7 @@ export class OpenAISTT implements SpeechToTextProvider { file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)), language, model: modelToUse, + ...(prompt ? { prompt } : {}), ...(supportsLogprobs ? { include: includeLogprobs } : {}), response_format: "json", }); diff --git a/packages/server/src/server/speech/providers/openai/tts.test.ts b/packages/server/src/server/speech/providers/openai/tts.test.ts new file mode 100644 index 000000000..4d3a20269 --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/tts.test.ts @@ -0,0 +1,42 @@ +import pino from "pino"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +const { openAiConstructorOptionsMock, speechCreateMock } = vi.hoisted(() => ({ + openAiConstructorOptionsMock: vi.fn(), + speechCreateMock: vi.fn(), +})); + +vi.mock("openai", () => ({ + OpenAI: vi.fn(function OpenAI(options: unknown) { + openAiConstructorOptionsMock(options); + return { + audio: { + speech: { + create: speechCreateMock, + }, + }, + }; + }), +})); + +import { OpenAITTS } from "./tts.js"; + +describe("OpenAITTS", () => { + afterEach(() => { + openAiConstructorOptionsMock.mockReset(); + speechCreateMock.mockReset(); + }); + + test("passes configured baseUrl to the OpenAI client", () => { + const provider = new OpenAITTS( + { apiKey: "sk-test", baseUrl: "https://speech.example.com/v1" }, + pino({ level: "silent" }), + ); + + expect(provider.getConfig().baseUrl).toBe("https://speech.example.com/v1"); + expect(openAiConstructorOptionsMock).toHaveBeenCalledWith({ + apiKey: "sk-test", + baseURL: "https://speech.example.com/v1", + }); + }); +}); diff --git a/packages/server/src/server/speech/providers/openai/tts.ts b/packages/server/src/server/speech/providers/openai/tts.ts index f7cc7dd39..6cc7e5c5b 100644 --- a/packages/server/src/server/speech/providers/openai/tts.ts +++ b/packages/server/src/server/speech/providers/openai/tts.ts @@ -7,6 +7,7 @@ export type { SpeechStreamResult }; export interface TTSConfig { apiKey: string; + baseUrl?: string; model?: "tts-1" | "tts-1-hd"; voice?: "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer"; responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm"; @@ -27,6 +28,7 @@ export class OpenAITTS implements TextToSpeechProvider { this.logger = parentLogger.child({ module: "agent", provider: "openai", component: "tts" }); this.openaiClient = new OpenAI({ apiKey: ttsConfig.apiKey, + ...(ttsConfig.baseUrl ? { baseURL: ttsConfig.baseUrl } : {}), }); this.logger.info( diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts index 3d93b3673..84b341382 100644 --- a/packages/server/src/server/speech/speech-config-resolver.test.ts +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -129,7 +129,7 @@ describe("resolveSpeechConfig", () => { dictation: "es", voice: "pt", }); - expect(result.openai?.apiKey).toBe("env-key"); + expect(result.openai?.apiKey).toBe("persisted-key"); expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); }); diff --git a/public-docs/voice.md b/public-docs/voice.md index bb6adbf1b..0869028ea 100644 --- a/public-docs/voice.md +++ b/public-docs/voice.md @@ -8,7 +8,7 @@ category: Configuration # Voice -Paseo has first-class voice support for dictation and realtime conversations with your coding environment. +Paseo has first-class voice support for dictation and voice mode conversations with your coding environment. ## Philosophy @@ -72,11 +72,11 @@ For multilingual local dictation, set the model to v3 — it auto-detects the la } ``` -The `language` field applies only to the OpenAI STT provider: set `features.dictation.stt.language` for dictation and `features.voiceMode.stt.language` for realtime voice. If voice language is omitted, Paseo uses the dictation language before falling back to `en`. It has no effect on the local Parakeet models. +The `language` field applies only to the OpenAI STT provider: set `features.dictation.stt.language` for dictation and `features.voiceMode.stt.language` for voice mode. If voice language is omitted, Paseo uses the dictation language before falling back to `en`. It has no effect on the local Parakeet models. -## OpenAI Speech Option +## OpenAI Voice Option -You can switch dictation, voice STT, and voice TTS to OpenAI by setting provider fields to `openai` and providing `OPENAI_API_KEY`. +You can switch dictation, voice STT, and voice TTS to OpenAI by setting provider fields to `openai` and providing OpenAI credentials. ```json { @@ -89,22 +89,35 @@ You can switch dictation, voice STT, and voice TTS to OpenAI by setting provider } }, "providers": { - "openai": { "apiKey": "..." } + "openai": { + "voice": { + "apiKey": "...", + "baseUrl": "https://api.openai.com/v1" + } + } } } ``` +`providers.openai.voice.apiKey` and `providers.openai.voice.baseUrl` configure only Paseo OpenAI voice traffic, without changing Codex or other OpenAI-backed tools. + +Paseo uses these paths under the configured OpenAI base URL: + +- dictation STT: `/v1/audio/transcriptions` +- voice mode STT: `/v1/audio/transcriptions` +- voice mode TTS: `/v1/audio/speech` + ## Environment Variables -- `OPENAI_API_KEY`, OpenAI speech credentials - `PASEO_VOICE_LLM_PROVIDER`, voice agent provider override +- `PASEO_DICTATION_STT_PROVIDER`, `PASEO_VOICE_STT_PROVIDER`, `PASEO_VOICE_TTS_PROVIDER`, speech provider selection (`local` or `openai`) - `PASEO_LOCAL_MODELS_DIR`, local model storage directory - `PASEO_DICTATION_LOCAL_STT_MODEL`, local dictation STT model ID - `PASEO_VOICE_LOCAL_STT_MODEL`, `PASEO_VOICE_LOCAL_TTS_MODEL`, local voice STT/TTS model IDs - `PASEO_DICTATION_LANGUAGE`, dictation STT language (OpenAI STT only; ignored by local Parakeet) -- `PASEO_VOICE_LANGUAGE`, realtime voice STT language; falls back to `PASEO_DICTATION_LANGUAGE` when unset (OpenAI STT only; ignored by local Parakeet) +- `PASEO_VOICE_LANGUAGE`, voice mode STT language; falls back to `PASEO_DICTATION_LANGUAGE` when unset (OpenAI STT only; ignored by local Parakeet) - `PASEO_VOICE_LOCAL_TTS_SPEAKER_ID`, `PASEO_VOICE_LOCAL_TTS_SPEED`, optional local voice TTS tuning ## Operational Notes -Realtime voice can launch and control agents. Treat voice prompts with the same care as direct agent instructions, especially when specifying working directories or destructive operations. +Voice mode can launch and control agents. Treat voice prompts with the same care as direct agent instructions, especially when specifying working directories or destructive operations.