From d791f79efeb675f6002e316b0a6c582793dcc7d4 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 10:38:38 +0700 Subject: [PATCH 1/4] Local streaming STT/TTS with Parakeet + Pocket TTS - Add provider-agnostic streaming speech interfaces - Local STT via Sherpa (Zipformer/Paraformer) + Parakeet v3 (offline) - Local TTS via Pocket TTS ONNX (onnxruntime-node) with chunked streaming - Auto-download models into PASEO_HOME and add Node download script - Add E2E download test that round-trips TTS->STT --- README.md | 31 + package-lock.json | 269 ++++++++- .../app/e2e/fixtures/recording.baseline.txt | 2 + .../app/src/hooks/use-audio-player.native.ts | 80 ++- packages/server/package.json | 6 + .../server/scripts/download-speech-models.ts | 42 ++ packages/server/scripts/list-speech-models.ts | 11 + .../providers/codex-app-server-agent.test.ts | 6 +- .../src/server/agent/stt-manager.test.ts | 42 ++ .../server/src/server/agent/stt-manager.ts | 6 +- .../server/src/server/agent/stt-openai.ts | 18 +- .../src/server/agent/tts-manager.test.ts | 47 ++ .../server/src/server/agent/tts-manager.ts | 6 +- .../server/src/server/agent/tts-openai.ts | 10 +- packages/server/src/server/bootstrap.ts | 337 ++++++++++- packages/server/src/server/config.ts | 115 ++++ .../src/server/daemon-client.e2e.test.ts | 101 +++- .../dictation-stream-manager.test.ts | 31 + .../dictation/dictation-stream-manager.ts | 33 +- .../server/src/server/persisted-config.ts | 33 +- packages/server/src/server/session.ts | 10 +- packages/server/src/server/speech/audio.ts | 113 ++++ .../server/speech/pocket/pocket-tts-onnx.ts | 559 ++++++++++++++++++ .../src/server/speech/sherpa/model-catalog.ts | 128 ++++ .../speech/sherpa/model-downloader.test.ts | 56 ++ .../server/speech/sherpa/model-downloader.ts | 190 ++++++ .../sherpa/sherpa-offline-recognizer.ts | 104 ++++ .../speech/sherpa/sherpa-online-recognizer.ts | 127 ++++ .../speech/sherpa/sherpa-onnx-loader.ts | 18 + .../speech/sherpa/sherpa-onnx-node-loader.ts | 53 ++ .../sherpa-parakeet-realtime-session.ts | 153 +++++ .../speech/sherpa/sherpa-parakeet-stt.ts | 80 +++ .../speech/sherpa/sherpa-realtime-session.ts | 127 ++++ .../src/server/speech/sherpa/sherpa-stt.ts | 95 +++ .../src/server/speech/sherpa/sherpa-tts.ts | 121 ++++ .../speech/sherpa/speech-download.e2e.test.ts | 264 +++++++++ .../src/server/speech/speech-provider.ts | 30 + .../src/server/test-utils/paseo-daemon.ts | 4 +- .../server/src/server/websocket-server.ts | 12 +- scripts/speech/download-sherpa-models.sh | 95 +++ 40 files changed, 3421 insertions(+), 144 deletions(-) create mode 100644 packages/app/e2e/fixtures/recording.baseline.txt create mode 100644 packages/server/scripts/download-speech-models.ts create mode 100644 packages/server/scripts/list-speech-models.ts create mode 100644 packages/server/src/server/agent/stt-manager.test.ts create mode 100644 packages/server/src/server/agent/tts-manager.test.ts create mode 100644 packages/server/src/server/speech/audio.ts create mode 100644 packages/server/src/server/speech/pocket/pocket-tts-onnx.ts create mode 100644 packages/server/src/server/speech/sherpa/model-catalog.ts create mode 100644 packages/server/src/server/speech/sherpa/model-downloader.test.ts create mode 100644 packages/server/src/server/speech/sherpa/model-downloader.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-stt.ts create mode 100644 packages/server/src/server/speech/sherpa/sherpa-tts.ts create mode 100644 packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts create mode 100644 packages/server/src/server/speech/speech-provider.ts create mode 100755 scripts/speech/download-sherpa-models.sh diff --git a/README.md b/README.md index 9773b5b2d..9a70f5d36 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,37 @@ npm install -g @getpaseo/cli && paseo Then open the app and connect to your daemon. +## Local speech (STT/TTS) + +Paseo can run dictation + voice mode STT/TTS fully locally via `sherpa-onnx`. + +When the daemon starts with a local speech provider selected, it will download any missing model files automatically (unless `PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=0`). + +```bash +npm run speech:download --workspace=@getpaseo/server +``` + +Optional configuration: + +- `PASEO_SHERPA_ONNX_MODELS_DIR` (defaults to `~/.paseo/models/sherpa-onnx`) +- `PASEO_SHERPA_ONNX_AUTO_DOWNLOAD` (`1` by default; set `0` to disable automatic downloads on daemon start) +- `PASEO_SHERPA_STT_PRESET` (`zipformer`, `paraformer`, or `parakeet` for NVIDIA Parakeet TDT v3) +- `PASEO_SHERPA_TTS_PRESET` (`pocket-tts` (Kyutai Pocket TTS), `kitten`, or `kokoro`) +- `PASEO_DICTATION_STT_PROVIDER`, `PASEO_VOICE_STT_PROVIDER`, `PASEO_VOICE_TTS_PROVIDER` (`sherpa` or `openai`) + +To see all supported local model IDs: + +```bash +npm run speech:models --workspace=@getpaseo/server +``` + +Optional: run an end-to-end test that downloads real models and exercises streaming STT + streaming TTS: + +```bash +PASEO_SPEECH_E2E_DOWNLOAD=1 PASEO_SPEECH_E2E_MODEL_SET=parakeet-pocket \ + npx vitest run --workspace=@getpaseo/server src/server/speech/sherpa/speech-download.e2e.test.ts +``` + ## Documentation See [paseo.sh/docs](https://paseo.sh/docs) for full documentation. diff --git a/package-lock.json b/package-lock.json index bedc32ec7..4ec5f45a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5621,6 +5621,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@sctg/sentencepiece-js": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.3.tgz", + "integrity": "sha512-iPnzR2HGjdQQG2SpTyPH3wnnpgQ2aS14B3I2jHjomkoaMprM3Sn+WOQblHwByetnvn3n14y3IwjwogVUHA2cVA==", + "license": "Apache-2.0", + "dependencies": { + "app-root-path": "^3.1.0", + "buffer": "^6.0.3" + } + }, "node_modules/@segment/loosely-validate-event": { "version": "2.0.0", "dev": true, @@ -5739,9 +5749,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "dev": true, "license": "MIT", "optional": true, @@ -6995,6 +7002,15 @@ "node": ">=0.4.0" } }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/agent-base": { "version": "6.0.2", "dev": true, @@ -7157,6 +7173,15 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/app-root-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz", + "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/arg": { "version": "5.0.2", "license": "MIT" @@ -7791,6 +7816,13 @@ "version": "1.0.0", "license": "ISC" }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/bplist-creator": { "version": "0.1.0", "license": "MIT", @@ -8879,6 +8911,12 @@ "node": ">=8" } }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, "node_modules/detect-node-es": { "version": "1.1.0", "license": "MIT" @@ -9741,6 +9779,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.25.11", "hasInstallScript": true, @@ -11752,6 +11796,62 @@ "version": "0.4.1", "license": "BSD-2-Clause" }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/global-agent/node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/global-agent/node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/global-agent/node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/global-dirs": { "version": "0.1.1", "license": "MIT", @@ -11775,7 +11875,6 @@ }, "node_modules/globalthis": { "version": "1.0.4", - "dev": true, "license": "MIT", "dependencies": { "define-properties": "^1.2.1", @@ -13075,6 +13174,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, "node_modules/json5": { "version": "2.2.3", "license": "MIT", @@ -13397,9 +13502,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -13560,6 +13662,18 @@ "version": "1.3.0", "license": "Apache-2.0" }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "license": "MIT", @@ -14755,6 +14869,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/onnxruntime-common": { + "version": "1.24.1", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.1.tgz", + "integrity": "sha512-UnV15u4p4XxoIV+jFP4hXPsW93s3QrwLSpi20HUDYHoTfI4z4sjzex3L4XDOxGGZJ/M/catrwAG2go958UQq0w==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.24.1", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.1.tgz", + "integrity": "sha512-Ex/oUXKdhDoxvlNxBT3oYtW0MH88yYpPlXQeVQUXpcJQmN24usd/8RCoPLN5kCHwDsiZ+nqsnjciyFRl423dQw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "global-agent": "^3.0.0", + "onnxruntime-common": "1.24.1" + } + }, "node_modules/open": { "version": "8.4.2", "license": "MIT", @@ -16785,6 +16922,29 @@ "node": "*" } }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/roarr/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/rollup": { "version": "4.52.5", "license": "MIT", @@ -16992,6 +17152,12 @@ "semver": "bin/semver.js" } }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, "node_modules/send": { "version": "0.19.1", "license": "MIT", @@ -17312,6 +17478,91 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sherpa-onnx": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx/-/sherpa-onnx-1.12.23.tgz", + "integrity": "sha512-QwgDwSIXx4DgbBE7h7VUXEVLO2vf7w3aJX3ng6WoTAcfB2NW7So4Zp4GtGemwgxZ6sfEh02JEVXUidejkX7V0A==", + "license": "Apache-2.0" + }, + "node_modules/sherpa-onnx-darwin-arm64": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.12.23.tgz", + "integrity": "sha512-zbjNUUH/IXhjRyRJ9mpcWVOGIVr31a/qXBPsfOYc7U8cgwcq33Vmj2OzoLYWQF6T+puqCAE4nMxFAxJvdZekhg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sherpa-onnx-darwin-x64": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-x64/-/sherpa-onnx-darwin-x64-1.12.23.tgz", + "integrity": "sha512-hgPhBG85bgULJo5xS5xvh1A9+D6GBPwhZ8QCvWvh7YP1kF9UZTF1r5hWBOzZiR2Ax0gQMbxVvw1t/NpR2N7HWA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sherpa-onnx-linux-x64": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-linux-x64/-/sherpa-onnx-linux-x64-1.12.23.tgz", + "integrity": "sha512-pUZIdDvPtyRXQDGo9R9MIBf2AFUzfgcGmutoulsEdH3hpK6JteR7Z/5pfrZIIqe/O99djAjEHK4AlwLHC2jiZw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/sherpa-onnx-node": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-node/-/sherpa-onnx-node-1.12.23.tgz", + "integrity": "sha512-09SRixVSjsajxeCV8Hy9R5J4IHPtw7vNgaIcEokdh/LpU7sY+e12z9uHHIMMMgNiInyGEH74wIwjLXms+W7qRA==", + "license": "Apache-2.0", + "optionalDependencies": { + "sherpa-onnx-darwin-arm64": "^1.12.23", + "sherpa-onnx-darwin-x64": "^1.12.23", + "sherpa-onnx-linux-arm64": "^1.12.23", + "sherpa-onnx-linux-x64": "^1.12.23", + "sherpa-onnx-win-ia32": "^1.12.23", + "sherpa-onnx-win-x64": "^1.12.23" + } + }, + "node_modules/sherpa-onnx-win-ia32": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-win-ia32/-/sherpa-onnx-win-ia32-1.12.23.tgz", + "integrity": "sha512-MyLsK7r6dd7paglyTgb8UHTXTEFqOzA91u6VDV64Lq8rDGuOFVYioxX7vlwmGe1A9o7VhuOPNaKcRjEPtVDhBQ==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/sherpa-onnx-win-x64": { + "version": "1.12.23", + "resolved": "https://registry.npmjs.org/sherpa-onnx-win-x64/-/sherpa-onnx-win-x64-1.12.23.tgz", + "integrity": "sha512-pdHEYMJiYy8+xzH2WkBVS4/hnRwqjY8FaWnjs0NBgQZnPmc/k4M+TAiauTOuFDNK4GPwFQnjwrCGx6jI9AOkOg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/side-channel": { "version": "1.1.0", "license": "MIT", @@ -20036,6 +20287,7 @@ "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "^1.1.12", "@openrouter/ai-sdk-provider": "^1.2.0", + "@sctg/sentencepiece-js": "^1.1.0", "@xterm/headless": "^6.0.0", "ai": "^5.0.76", "ajv": "^8.17.1", @@ -20044,11 +20296,14 @@ "express-basic-auth": "^1.2.1", "mnemonic-id": "^3.2.7", "node-pty": "^1.0.0", + "onnxruntime-node": "^1.23.0", "openai": "^4.20.0", "pino": "^10.2.0", "pino-pretty": "^13.1.3", "playwright": "^1.56.1", "qrcode": "^1.5.4", + "sherpa-onnx": "^1.12.23", + "sherpa-onnx-node": "^1.12.23", "strip-ansi": "^7.1.2", "tiny-invariant": "^1.3.3", "uuid": "^9.0.1", diff --git a/packages/app/e2e/fixtures/recording.baseline.txt b/packages/app/e2e/fixtures/recording.baseline.txt new file mode 100644 index 000000000..4724395dc --- /dev/null +++ b/packages/app/e2e/fixtures/recording.baseline.txt @@ -0,0 +1,2 @@ +This is a voice note. + diff --git a/packages/app/src/hooks/use-audio-player.native.ts b/packages/app/src/hooks/use-audio-player.native.ts index 4729c9b9c..5e7b94ba9 100644 --- a/packages/app/src/hooks/use-audio-player.native.ts +++ b/packages/app/src/hooks/use-audio-player.native.ts @@ -14,32 +14,60 @@ interface QueuedAudio { } /** - * Resample PCM16 audio from 24kHz to 16kHz - * OpenAI returns 24kHz, Speechmatics expects 16kHz + * Resample PCM16 audio between sample rates. + * Speechmatics expects 16kHz. */ -function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array { - // PCM16 = 2 bytes per sample - const samples24k = pcm24k.length / 2; - const samples16k = Math.floor((samples24k * 16000) / 24000); - - const pcm16k = new Uint8Array(samples16k * 2); - const ratio = 24000 / 16000; // 1.5 - - for (let i = 0; i < samples16k; i++) { - const srcIndex = Math.floor(i * ratio) * 2; - if (srcIndex + 1 < pcm24k.length) { - pcm16k[i * 2] = pcm24k[srcIndex]; - pcm16k[i * 2 + 1] = pcm24k[srcIndex + 1]; - } +function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array { + if (fromRate === toRate) { + return pcm; } - console.log("[AudioPlayer] Resampled PCM:", { - input24k: pcm24k.length, - output16k: pcm16k.length, - durationMs: samples16k / 16, - }); + const inputSamples = Math.floor(pcm.length / 2); + const outputSamples = Math.floor((inputSamples * toRate) / fromRate); + const out = new Uint8Array(outputSamples * 2); - return pcm16k; + const ratio = fromRate / toRate; + + const readInt16 = (sampleIndex: number): number => { + const i = sampleIndex * 2; + if (i + 1 >= pcm.length) { + return 0; + } + const lo = pcm[i]!; + const hi = pcm[i + 1]!; + let value = (hi << 8) | lo; + if (value & 0x8000) { + value = value - 0x10000; + } + return value; + }; + + const writeInt16 = (sampleIndex: number, value: number): void => { + const clamped = Math.max(-32768, Math.min(32767, Math.round(value))); + const i = sampleIndex * 2; + out[i] = clamped & 0xff; + out[i + 1] = (clamped >> 8) & 0xff; + }; + + for (let i = 0; i < outputSamples; i++) { + const srcPos = i * ratio; + const i0 = Math.floor(srcPos); + const frac = srcPos - i0; + const s0 = readInt16(i0); + const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1)); + writeInt16(i, s0 + (s1 - s0) * frac); + } + + return out; +} + +function parsePcmSampleRate(mimeType: string): number | null { + const match = /rate=(\d+)/i.exec(mimeType); + if (!match) { + return null; + } + const rate = Number(match[1]); + return Number.isFinite(rate) && rate > 0 ? rate : null; } export interface AudioPlayerOptions { @@ -204,12 +232,12 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { console.log("[AudioPlayer] Resuming playback engine..."); resumePlayback(); - // Get PCM data from blob (server now sends PCM format) + // Get PCM data from blob (server sends PCM16) const arrayBuffer = await audioData.arrayBuffer(); - let pcm24k = new Uint8Array(arrayBuffer); + const pcm = new Uint8Array(arrayBuffer); - // Resample from 24kHz (OpenAI) to 16kHz (Speechmatics) - const pcm16k = resamplePcm24kTo16k(pcm24k); + const inputRate = parsePcmSampleRate(audioData.type || "") ?? 24000; + const pcm16k = resamplePcm16(pcm, inputRate, 16000); // Calculate total duration const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample diff --git a/packages/server/package.json b/packages/server/package.json index 10b42ac31..88293bf86 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -14,6 +14,8 @@ "start": "NODE_ENV=production node dist/server/index.js", "typecheck": "tsc -p tsconfig.server.typecheck.json --noEmit", "generate:config-schema": "tsx scripts/generate-config-schema.ts", + "speech:models": "tsx scripts/list-speech-models.ts", + "speech:download": "tsx scripts/download-speech-models.ts", "test": "vitest run", "test:watch": "vitest", "test:ui": "vitest --ui", @@ -48,6 +50,10 @@ "pino-pretty": "^13.1.3", "playwright": "^1.56.1", "qrcode": "^1.5.4", + "@sctg/sentencepiece-js": "^1.1.0", + "onnxruntime-node": "^1.23.0", + "sherpa-onnx": "^1.12.23", + "sherpa-onnx-node": "^1.12.23", "strip-ansi": "^7.1.2", "tiny-invariant": "^1.3.3", "uuid": "^9.0.1", diff --git a/packages/server/scripts/download-speech-models.ts b/packages/server/scripts/download-speech-models.ts new file mode 100644 index 000000000..978cc3e85 --- /dev/null +++ b/packages/server/scripts/download-speech-models.ts @@ -0,0 +1,42 @@ +import { resolvePaseoHome } from "../src/server/paseo-home.js"; +import { createRootLogger } from "../src/server/logger.js"; +import { ensureSherpaOnnxModels } from "../src/server/speech/sherpa/model-downloader.js"; +import type { SherpaOnnxModelId } from "../src/server/speech/sherpa/model-catalog.js"; + +function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } { + const home = resolvePaseoHome(); + let modelsDir = process.env.PASEO_SHERPA_ONNX_MODELS_DIR || `${home}/models/sherpa-onnx`; + const modelIds: SherpaOnnxModelId[] = []; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--models-dir") { + modelsDir = argv[i + 1] ?? modelsDir; + i++; + continue; + } + if (arg === "--model") { + const id = argv[i + 1] as SherpaOnnxModelId | undefined; + if (!id) { + throw new Error("--model requires a value"); + } + modelIds.push(id); + i++; + continue; + } + } + + if (modelIds.length === 0) { + const stt = (process.env.PASEO_SHERPA_STT_PRESET || "zipformer-bilingual-zh-en-2023-02-20") as SherpaOnnxModelId; + const tts = (process.env.PASEO_SHERPA_TTS_PRESET || "pocket-tts-onnx-int8") as SherpaOnnxModelId; + modelIds.push(stt, tts); + } + + return { modelsDir, modelIds }; +} + +const logger = createRootLogger({ level: "info", format: "pretty" }); + +const { modelsDir, modelIds } = parseArgs(process.argv.slice(2)); +await ensureSherpaOnnxModels({ modelsDir, modelIds, autoDownload: true, logger }); +logger.info({ modelsDir, modelIds }, "Done downloading speech models"); diff --git a/packages/server/scripts/list-speech-models.ts b/packages/server/scripts/list-speech-models.ts new file mode 100644 index 000000000..d907ab4eb --- /dev/null +++ b/packages/server/scripts/list-speech-models.ts @@ -0,0 +1,11 @@ +import { listSherpaOnnxModels } from "../src/server/speech/sherpa/model-catalog.js"; + +const models = listSherpaOnnxModels() + .slice() + .sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id)); + +for (const m of models) { + // eslint-disable-next-line no-console + console.log(`${m.kind}\t${m.id}\t${m.description}`); +} + diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index 5070c1337..799bbc95e 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -169,7 +169,7 @@ describe("Codex app-server provider (integration)", () => { cleanup(); rmSync(cwd, { recursive: true, force: true }); } - }, 60000); + }, 120000); test.runIf(isCodexInstalled())("listCommands includes custom prompts and executeCommand runs them", async () => { const cleanup = useTempCodexSessionDir(); @@ -730,7 +730,9 @@ describe("Codex app-server provider (integration)", () => { if (captured) { expect(sawPermissionResolved).toBe(true); } - expect(readFileSync(targetPath, "utf8").trim()).toBe("ok"); + const text = await waitForFileToContainText(targetPath, "ok", { timeoutMs: 5000 }); + expect(timelineItems.some((item) => hasApplyPatchFile(item, "approval-test.txt"))).toBe(true); + expect(text?.trim()).toBe("ok"); } finally { cleanup(); rmSync(cwd, { recursive: true, force: true }); diff --git a/packages/server/src/server/agent/stt-manager.test.ts b/packages/server/src/server/agent/stt-manager.test.ts new file mode 100644 index 000000000..df88f3bea --- /dev/null +++ b/packages/server/src/server/agent/stt-manager.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import pino from "pino"; + +import { STTManager } from "./stt-manager.js"; +import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js"; + +class FakeStt implements SpeechToTextProvider { + constructor(private readonly result: TranscriptionResult) {} + + async transcribeAudio(): Promise { + return this.result; + } +} + +describe("STTManager", () => { + it("returns empty text for low-confidence transcriptions", async () => { + const manager = new STTManager( + "s1", + pino({ level: "silent" }), + new FakeStt({ text: "um", isLowConfidence: true, avgLogprob: -10 }) + ); + + const result = await manager.transcribe(Buffer.from("x"), "audio/wav", { label: "t" }); + expect(result.text).toBe(""); + expect(result.isLowConfidence).toBe(true); + expect(result.byteLength).toBe(1); + }); + + it("passes through normal transcriptions", async () => { + const manager = new STTManager( + "s1", + pino({ level: "silent" }), + new FakeStt({ text: "hello world", language: "en", isLowConfidence: false }) + ); + + const result = await manager.transcribe(Buffer.from("abc"), "audio/wav"); + expect(result.text).toBe("hello world"); + expect(result.language).toBe("en"); + expect(result.byteLength).toBe(3); + }); +}); + diff --git a/packages/server/src/server/agent/stt-manager.ts b/packages/server/src/server/agent/stt-manager.ts index 4abb6ea15..cb454dc3b 100644 --- a/packages/server/src/server/agent/stt-manager.ts +++ b/packages/server/src/server/agent/stt-manager.ts @@ -1,5 +1,5 @@ import type pino from "pino"; -import type { OpenAISTT, TranscriptionResult } from "./stt-openai.js"; +import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js"; import { maybePersistDebugAudio } from "./stt-debug.js"; interface TranscriptionMetadata { @@ -21,9 +21,9 @@ export interface SessionTranscriptionResult extends TranscriptionResult { export class STTManager { private readonly sessionId: string; private readonly logger: pino.Logger; - private readonly stt: OpenAISTT | null; + private readonly stt: SpeechToTextProvider | null; - constructor(sessionId: string, logger: pino.Logger, stt: OpenAISTT | null) { + constructor(sessionId: string, logger: pino.Logger, stt: SpeechToTextProvider | null) { this.sessionId = sessionId; this.logger = logger.child({ module: "agent", component: "stt-manager", sessionId }); this.stt = stt; diff --git a/packages/server/src/server/agent/stt-openai.ts b/packages/server/src/server/agent/stt-openai.ts index 5e70c6a81..4326f6c68 100644 --- a/packages/server/src/server/agent/stt-openai.ts +++ b/packages/server/src/server/agent/stt-openai.ts @@ -5,6 +5,9 @@ import { join } from "path"; import { tmpdir } from "os"; import { v4 } from "uuid"; import { inferAudioExtension } from "./audio-utils.js"; +import type { LogprobToken, TranscriptionResult } from "../speech/speech-provider.js"; + +export type { LogprobToken, TranscriptionResult }; export interface STTConfig { apiKey: string; @@ -12,21 +15,6 @@ export interface STTConfig { confidenceThreshold?: number; // Default: -3.0 } -export interface LogprobToken { - token: string; - logprob: number; - bytes?: number[]; -} - -export interface TranscriptionResult { - text: string; - language?: string; - duration?: number; - logprobs?: LogprobToken[]; - avgLogprob?: number; - isLowConfidence?: boolean; -} - function isObject(value: unknown): value is { [key: string]: unknown } { return typeof value === "object" && value !== null; } diff --git a/packages/server/src/server/agent/tts-manager.test.ts b/packages/server/src/server/agent/tts-manager.test.ts new file mode 100644 index 000000000..5c0dea3be --- /dev/null +++ b/packages/server/src/server/agent/tts-manager.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import pino from "pino"; +import { Readable } from "node:stream"; + +import { TTSManager } from "./tts-manager.js"; +import type { TextToSpeechProvider } from "../speech/speech-provider.js"; +import type { SessionOutboundMessage } from "../messages.js"; + +class FakeTts implements TextToSpeechProvider { + async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> { + return { + stream: Readable.from([Buffer.from("a"), Buffer.from("b")]), + format: "pcm;rate=24000", + }; + } +} + +describe("TTSManager", () => { + it("emits chunks and resolves once confirmed", async () => { + const manager = new TTSManager("s1", pino({ level: "silent" }), new FakeTts()); + const abort = new AbortController(); + const emitted: SessionOutboundMessage[] = []; + + const task = manager.generateAndWaitForPlayback( + "hello", + (msg) => { + emitted.push(msg); + if (msg.type === "audio_output") { + manager.confirmAudioPlayed(msg.payload.id); + } + }, + abort.signal, + true + ); + + await task; + + const audioMsgs = emitted.filter((m) => m.type === "audio_output"); + expect(audioMsgs).toHaveLength(2); + const groupId = (audioMsgs[0] as any).payload.groupId; + expect(groupId).toBeTruthy(); + expect((audioMsgs[0] as any).payload.chunkIndex).toBe(0); + expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1); + expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true); + }); +}); + diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/agent/tts-manager.ts index 2b0491a52..083c94096 100644 --- a/packages/server/src/server/agent/tts-manager.ts +++ b/packages/server/src/server/agent/tts-manager.ts @@ -1,6 +1,6 @@ import type pino from "pino"; import { v4 as uuidv4 } from "uuid"; -import type { OpenAITTS } from "./tts-openai.js"; +import type { TextToSpeechProvider } from "../speech/speech-provider.js"; import type { SessionOutboundMessage } from "../messages.js"; interface PendingPlayback { @@ -17,9 +17,9 @@ interface PendingPlayback { export class TTSManager { private pendingPlaybacks: Map = new Map(); private readonly logger: pino.Logger; - private readonly tts: OpenAITTS | null; + private readonly tts: TextToSpeechProvider | null; - constructor(sessionId: string, logger: pino.Logger, tts: OpenAITTS | null) { + constructor(sessionId: string, logger: pino.Logger, tts: TextToSpeechProvider | null) { this.logger = logger.child({ module: "agent", component: "tts-manager", sessionId }); this.tts = tts; } diff --git a/packages/server/src/server/agent/tts-openai.ts b/packages/server/src/server/agent/tts-openai.ts index efe4e4ca0..bd96a80b4 100644 --- a/packages/server/src/server/agent/tts-openai.ts +++ b/packages/server/src/server/agent/tts-openai.ts @@ -1,6 +1,9 @@ import type pino from "pino"; import OpenAI from "openai"; -import { Readable } from "stream"; +import { Readable } from "node:stream"; +import type { SpeechStreamResult } from "../speech/speech-provider.js"; + +export type { SpeechStreamResult }; export interface TTSConfig { apiKey: string; @@ -9,11 +12,6 @@ export interface TTSConfig { responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm"; } -export interface SpeechStreamResult { - stream: Readable; - format: string; -} - export class OpenAITTS { private readonly openaiClient: OpenAI; private readonly config: TTSConfig; diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 92f13cff7..b47deb61f 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -42,6 +42,18 @@ import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; import { OpenAISTT, type STTConfig } from "./agent/stt-openai.js"; import { OpenAITTS, type TTSConfig } from "./agent/tts-openai.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; +import type { RealtimeTranscriptionSessionFactory } from "./dictation/dictation-stream-manager.js"; +import { SherpaOnlineRecognizerEngine } from "./speech/sherpa/sherpa-online-recognizer.js"; +import { SherpaOfflineRecognizerEngine } from "./speech/sherpa/sherpa-offline-recognizer.js"; +import { SherpaOnnxSTT } from "./speech/sherpa/sherpa-stt.js"; +import { SherpaOnnxParakeetSTT } from "./speech/sherpa/sherpa-parakeet-stt.js"; +import { SherpaOnnxTTS } from "./speech/sherpa/sherpa-tts.js"; +import { SherpaRealtimeTranscriptionSession } from "./speech/sherpa/sherpa-realtime-session.js"; +import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/sherpa/sherpa-parakeet-realtime-session.js"; +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/sherpa/model-downloader.js"; +import type { SherpaOnnxModelId } from "./speech/sherpa/model-catalog.js"; +import { PocketTtsOnnxTTS } from "./speech/pocket/pocket-tts-onnx.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; import { attachAgentStoragePersistence } from "./persistence-hooks.js"; @@ -72,6 +84,26 @@ export type PaseoOpenAIConfig = { tts?: Partial & { apiKey?: string }; }; +export type PaseoSherpaOnnxConfig = { + modelsDir: string; + autoDownload?: boolean; + stt?: { + preset?: string; + }; + tts?: { + preset?: string; + speakerId?: number; + speed?: number; + }; +}; + +export type PaseoSpeechConfig = { + dictationSttProvider?: "openai" | "sherpa"; + voiceSttProvider?: "openai" | "sherpa"; + voiceTtsProvider?: "openai" | "sherpa"; + sherpaOnnx?: PaseoSherpaOnnxConfig; +}; + export type PaseoDaemonConfig = { listen: string; paseoHome: string; @@ -87,6 +119,7 @@ export type PaseoDaemonConfig = { relayPublicEndpoint?: string; appBaseUrl?: string; openai?: PaseoOpenAIConfig; + speech?: PaseoSpeechConfig; openrouterApiKey?: string | null; voiceLlmModel?: string | null; dictationFinalTimeoutMs?: number; @@ -382,42 +415,282 @@ export async function createPaseoDaemon( } - let sttService: OpenAISTT | null = null; - let ttsService: OpenAITTS | null = null; + let sttService: SpeechToTextProvider | null = null; + let ttsService: TextToSpeechProvider | null = null; + let dictationSessionFactory: RealtimeTranscriptionSessionFactory | undefined; + + let sherpaOnline: SherpaOnlineRecognizerEngine | null = null; + let sherpaOffline: SherpaOfflineRecognizerEngine | null = null; + let sherpaTts: TextToSpeechProvider | null = null; const openaiApiKey = config.openai?.apiKey; - if (openaiApiKey) { + const speechConfig = config.speech ?? null; + const sherpaConfig = speechConfig?.sherpaOnnx ?? null; + + const wantsSherpaDictation = (speechConfig?.dictationSttProvider ?? "openai") === "sherpa"; + const wantsSherpaVoiceStt = (speechConfig?.voiceSttProvider ?? "openai") === "sherpa"; + const wantsSherpaVoiceTts = (speechConfig?.voiceTtsProvider ?? "openai") === "sherpa"; + + if ((wantsSherpaDictation || wantsSherpaVoiceStt || wantsSherpaVoiceTts) && sherpaConfig) { + const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true); + let sttPreset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim(); + if ( + sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" && + sttPreset !== "paraformer-bilingual-zh-en" && + sttPreset !== "parakeet-tdt-0.6b-v3-int8" + ) { + logger.warn( + { sttPreset }, + "Unknown Sherpa STT preset; falling back to zipformer-bilingual-zh-en-2023-02-20" + ); + sttPreset = "zipformer-bilingual-zh-en-2023-02-20"; + } + + let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); + if ( + ttsPreset !== "kitten-nano-en-v0_1-fp16" && + ttsPreset !== "kokoro-en-v0_19" && + ttsPreset !== "pocket-tts-onnx-int8" + ) { + logger.warn( + { ttsPreset }, + "Unknown Sherpa TTS preset; falling back to kitten-nano-en-v0_1-fp16" + ); + ttsPreset = "kitten-nano-en-v0_1-fp16"; + } + + const modelIds: SherpaOnnxModelId[] = []; + if (wantsSherpaDictation || wantsSherpaVoiceStt) { + modelIds.push(sttPreset as SherpaOnnxModelId); + } + if (wantsSherpaVoiceTts) { + modelIds.push(ttsPreset as SherpaOnnxModelId); + } + + try { + await ensureSherpaOnnxModels({ + modelsDir: sherpaConfig.modelsDir, + modelIds, + autoDownload, + logger, + }); + } catch (err) { + logger.error( + { + err, + modelsDir: sherpaConfig.modelsDir, + autoDownload, + hint: + "Run: npm run dev --workspace=@getpaseo/server, then run: " + + "`tsx packages/server/scripts/download-speech-models.ts --models-dir --model `", + }, + "Failed to ensure local speech models" + ); + } + } + + if ((wantsSherpaDictation || wantsSherpaVoiceStt) && sherpaConfig) { + let preset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim(); + if ( + preset !== "zipformer-bilingual-zh-en-2023-02-20" && + preset !== "paraformer-bilingual-zh-en" && + preset !== "parakeet-tdt-0.6b-v3-int8" + ) { + logger.warn( + { preset }, + "Unknown Sherpa STT preset; falling back to zipformer-bilingual-zh-en-2023-02-20" + ); + preset = "zipformer-bilingual-zh-en-2023-02-20"; + } + const base = sherpaConfig.modelsDir; + + try { + if (preset === "parakeet-tdt-0.6b-v3-int8") { + const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8"); + sherpaOffline = new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + joiner: `${modelDir}/joiner.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger + ); + } else { + const model = + preset === "paraformer-bilingual-zh-en" + ? { + kind: "paraformer" as const, + encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`, + decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`, + tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`, + } + : { + kind: "transducer" as const, + encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`, + decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`, + joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`, + tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`, + modelType: "zipformer", + }; + + sherpaOnline = new SherpaOnlineRecognizerEngine( + { + model, + numThreads: 1, + debug: 0, + }, + logger + ); + } + } catch (err) { + logger.error( + { + err, + modelsDir: sherpaConfig.modelsDir, + preset, + hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, + }, + "Failed to initialize Sherpa STT (models missing or invalid)" + ); + sherpaOnline = null; + sherpaOffline = null; + } + } else if (wantsSherpaDictation || wantsSherpaVoiceStt) { + logger.warn( + { configured: Boolean(sherpaConfig) }, + "Sherpa STT selected but no sherpaOnnx config found; STT will be unavailable" + ); + } + + if (wantsSherpaVoiceTts && sherpaConfig) { + let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); + if ( + preset !== "kitten-nano-en-v0_1-fp16" && + preset !== "kokoro-en-v0_19" && + preset !== "pocket-tts-onnx-int8" + ) { + logger.warn( + { preset }, + "Unknown Sherpa TTS preset; falling back to kitten-nano-en-v0_1-fp16" + ); + preset = "kitten-nano-en-v0_1-fp16"; + } + try { + if (preset === "pocket-tts-onnx-int8") { + const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8"); + sherpaTts = await PocketTtsOnnxTTS.create( + { + modelDir, + precision: "int8", + targetChunkMs: 50, + }, + logger + ); + } else { + const modelDir = `${sherpaConfig.modelsDir}/${preset}`; + sherpaTts = new SherpaOnnxTTS( + { + preset: preset as any, + modelDir, + speakerId: sherpaConfig.tts?.speakerId, + speed: sherpaConfig.tts?.speed, + }, + logger + ); + } + } catch (err) { + logger.error( + { + err, + preset, + hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, + }, + "Failed to initialize Sherpa TTS (models missing or invalid)" + ); + sherpaTts = null; + } + } else if (wantsSherpaVoiceTts) { + logger.warn( + { configured: Boolean(sherpaConfig) }, + "Sherpa TTS selected but no sherpaOnnx config found; TTS will be unavailable" + ); + } + + if (wantsSherpaVoiceStt && sherpaOffline) { + sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger); + } else if (wantsSherpaVoiceStt && sherpaOnline) { + sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger); + } + + if (wantsSherpaVoiceTts && sherpaTts) { + ttsService = sherpaTts; + } + + if (wantsSherpaDictation && sherpaOnline) { + dictationSessionFactory = () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }); + } else if (wantsSherpaDictation && sherpaOffline) { + dictationSessionFactory = () => new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }); + } + + const voiceSttProvider = speechConfig?.voiceSttProvider ?? "openai"; + const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "openai"; + const dictationSttProvider = speechConfig?.dictationSttProvider ?? "openai"; + + const needsOpenAiStt = !sttService && voiceSttProvider === "openai"; + const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai"; + const needsOpenAiDictation = + dictationSttProvider === "openai" || (dictationSttProvider === "sherpa" && !dictationSessionFactory); + + const fallbackOpenAiStt = !sttService && voiceSttProvider === "sherpa" && Boolean(openaiApiKey); + const fallbackOpenAiTts = !ttsService && voiceTtsProvider === "sherpa" && Boolean(openaiApiKey); + + if ((needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) && openaiApiKey) { logger.info("OpenAI client initialized"); - const sttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; - if (sttApiKey) { - const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {}; - sttService = new OpenAISTT( - { - apiKey: sttApiKey, - ...sttConfig, - }, - logger - ); + if (fallbackOpenAiStt) { + logger.warn("Falling back to OpenAI STT because Sherpa STT is unavailable"); + } + if (needsOpenAiStt || fallbackOpenAiStt) { + const sttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; + if (sttApiKey) { + const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {}; + sttService = new OpenAISTT( + { + apiKey: sttApiKey, + ...sttConfig, + }, + logger + ); + } } - const ttsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; - if (ttsApiKey) { - const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {}; - ttsService = new OpenAITTS( - { - apiKey: ttsApiKey, - voice: "alloy", - model: "tts-1", - responseFormat: "pcm", - ...ttsConfig, - }, - logger - ); + if (fallbackOpenAiTts) { + logger.warn("Falling back to OpenAI TTS because Sherpa TTS is unavailable"); } - - } else { - logger.warn("OPENAI_API_KEY not set - LLM, STT, and TTS features will not work"); + if (needsOpenAiTts || fallbackOpenAiTts) { + const ttsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; + if (ttsApiKey) { + const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {}; + ttsService = new OpenAITTS( + { + apiKey: ttsApiKey, + voice: "alloy", + model: "tts-1", + responseFormat: "pcm", + ...ttsConfig, + }, + logger + ); + } + } + } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) { + logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation fallback is unavailable"); } const wsServer = new VoiceAssistantWebSocketServer( @@ -439,6 +712,7 @@ export async function createPaseoDaemon( { openaiApiKey: config.openai?.apiKey ?? null, finalTimeoutMs: config.dictationFinalTimeoutMs, + ...(dictationSessionFactory ? { sessionFactory: dictationSessionFactory } : {}), } ); @@ -533,6 +807,11 @@ export async function createPaseoDaemon( await agentStorage.flush().catch(() => undefined); await shutdownProviders(logger); terminalManager.killAll(); + if (sherpaTts && typeof (sherpaTts as any).free === "function") { + (sherpaTts as any).free(); + } + sherpaOnline?.free(); + sherpaOffline?.free(); await relayTransport?.stop().catch(() => undefined); await wsServer.close(); await new Promise((resolve) => { diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 2465e2b82..fca13a681 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -76,6 +76,51 @@ function parseOpenAIConfig( }; } +function parseSpeechProviderId(value: unknown): "openai" | "sherpa" | null { + if (typeof value !== "string") { + return null; + } + const normalized = value.trim().toLowerCase(); + if (!normalized) { + return null; + } + if (normalized === "openai") return "openai"; + if (normalized === "sherpa" || normalized === "sherpa-onnx" || normalized === "local") { + return "sherpa"; + } + return null; +} + +function normalizeSherpaSttPreset(value: string): string { + const raw = value.trim(); + const normalized = raw.toLowerCase(); + if (normalized === "zipformer" || normalized === "zipformer-bilingual") { + return "zipformer-bilingual-zh-en-2023-02-20"; + } + if (normalized === "paraformer") { + return "paraformer-bilingual-zh-en"; + } + if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") { + return "parakeet-tdt-0.6b-v3-int8"; + } + return raw; +} + +function normalizeSherpaTtsPreset(value: string): string { + const raw = value.trim(); + const normalized = raw.toLowerCase(); + if (normalized === "pocket" || normalized === "pocket-tts") { + return "pocket-tts-onnx-int8"; + } + if (normalized === "kitten") { + return "kitten-nano-en-v0_1-fp16"; + } + if (normalized === "kokoro") { + return "kokoro-en-v0_19"; + } + return raw; +} + export function loadConfig( paseoHome: string, options?: { @@ -137,6 +182,70 @@ export function loadConfig( voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice, }); + const dictationSttProvider = + parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ?? + parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ?? + "sherpa"; + + const voiceSttProvider = + parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ?? + parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ?? + "sherpa"; + + const voiceTtsProvider = + parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ?? + parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ?? + "sherpa"; + + const shouldConfigureSherpa = + dictationSttProvider === "sherpa" || + voiceSttProvider === "sherpa" || + voiceTtsProvider === "sherpa" || + typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" || + Boolean(persisted.providers?.sherpaOnnx); + + const sherpaModelsDir = + (env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() || + path.join(paseoHome, "models", "sherpa-onnx"); + + const sherpaOnnx = shouldConfigureSherpa + ? { + modelsDir: sherpaModelsDir, + autoDownload: + env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined + ? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1" + : persisted.providers?.sherpaOnnx?.autoDownload ?? + // In tests we should never hit the network unexpectedly. + Boolean(env.VITEST) === false, + stt: { + preset: normalizeSherpaSttPreset( + (env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() || + (persisted.features?.voiceMode?.stt?.preset ?? + persisted.features?.dictation?.stt?.preset)?.trim() || + "zipformer-bilingual-zh-en-2023-02-20" + ), + }, + tts: { + preset: normalizeSherpaTtsPreset( + (env.PASEO_SHERPA_TTS_PRESET ?? + persisted.providers?.sherpaOnnx?.tts?.preset ?? + persisted.features?.voiceMode?.tts?.preset)?.trim() || + (env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8") + ), + speakerId: + env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined + ? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10) + : persisted.providers?.sherpaOnnx?.tts?.speakerId ?? + persisted.features?.voiceMode?.tts?.speakerId, + speed: + env.PASEO_SHERPA_TTS_SPEED !== undefined + ? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED) + : persisted.providers?.sherpaOnnx?.tts?.speed ?? + persisted.features?.voiceMode?.tts?.speed, + }, + } + : undefined; + const openrouterApiKey = env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null; const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null; @@ -158,6 +267,12 @@ export function loadConfig( relayPublicEndpoint, appBaseUrl, openai, + speech: { + dictationSttProvider, + voiceSttProvider, + voiceTtsProvider, + ...(sherpaOnnx ? { sherpaOnnx } : {}), + }, openrouterApiKey, voiceLlmModel, }; diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index d612d10fa..5a253f513 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -1,6 +1,6 @@ import { describe, test, expect, beforeAll, afterAll, beforeEach } from "vitest"; import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { tmpdir, homedir } from "node:os"; import path from "node:path"; import { execSync } from "node:child_process"; @@ -13,12 +13,45 @@ import { getFullAccessConfig, getAskModeConfig } from "./daemon-e2e/agent-config import { chunkPcm16, parsePcm16MonoWav, - requireEnv, - transcribeBaselineOpenAI, wordSimilarity, } from "./test-utils/dictation-e2e.js"; -const hasOpenAICredentials = !!process.env.OPENAI_API_KEY; +const openaiApiKey = process.env.OPENAI_API_KEY ?? null; + +const sherpaModelsDir = + process.env.PASEO_SHERPA_ONNX_MODELS_DIR ?? + path.join(homedir(), ".paseo", "models", "sherpa-onnx"); + +function hasSherpaZipformerModels(modelsDir: string): boolean { + return ( + existsSync( + path.join( + modelsDir, + "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", + "encoder-epoch-99-avg-1.onnx" + ) + ) && + existsSync( + path.join( + modelsDir, + "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", + "tokens.txt" + ) + ) + ); +} + +function hasSherpaKittenModels(modelsDir: string): boolean { + return ( + existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "model.fp16.onnx")) && + existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "voices.bin")) && + existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "tokens.txt")) + ); +} + +const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir); +const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey); +const speechTest = hasAnySpeech ? test : test.skip; function tmpCwd(): string { return mkdtempSync(path.join(tmpdir(), "daemon-client-")); @@ -55,14 +88,27 @@ function waitForSignal( }); } -(hasOpenAICredentials ? describe : describe.skip)("daemon client E2E", () => { +describe("daemon client E2E", () => { let ctx: DaemonTestContext; beforeAll(async () => { - const openaiApiKey = process.env.OPENAI_API_KEY ?? ""; ctx = await createDaemonTestContext({ dictationFinalTimeoutMs: 5000, - openai: { apiKey: openaiApiKey }, + ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}), + speech: { + dictationSttProvider: "sherpa", + voiceSttProvider: "sherpa", + voiceTtsProvider: "sherpa", + sherpaOnnx: { + modelsDir: sherpaModelsDir, + stt: { + preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", + }, + tts: { + preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16", + }, + }, + }, }); }, 60000); @@ -569,11 +615,9 @@ function waitForSignal( 120000 ); - test( + speechTest( "voice mode buffers audio until isLast and emits transcription_result", async () => { - requireEnv("OPENAI_API_KEY"); - await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`); const transcription = waitForSignal(30_000, (resolve) => { @@ -675,11 +719,9 @@ function waitForSignal( 90_000 ); - test( - "streams dictation PCM and returns final transcript via OpenAI Realtime transcription", + speechTest( + "streams dictation PCM and returns final transcript", async () => { - requireEnv("OPENAI_API_KEY"); - const fixturePath = path.resolve( process.cwd(), "..", @@ -713,11 +755,9 @@ function waitForSignal( 30_000 ); - test( - "realtime dictation transcript is similar to baseline (OpenAI transcriptions API)", + speechTest( + "realtime dictation transcript is similar to baseline fixture", async () => { - const apiKey = requireEnv("OPENAI_API_KEY"); - const fixturePath = path.resolve( process.cwd(), "..", @@ -732,14 +772,17 @@ function waitForSignal( const dictationId = `dict-baseline-${Date.now()}`; const format = "audio/pcm;rate=16000;bits=16"; - const baseline = await transcribeBaselineOpenAI({ - apiKey, - wav, - model: process.env.STT_MODEL ?? "whisper-1", - prompt: - process.env.OPENAI_REALTIME_DICTATION_TRANSCRIPTION_PROMPT ?? - "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript.", - }); + const baselinePath = path.resolve( + process.cwd(), + "..", + "app", + "e2e", + "fixtures", + "recording.baseline.txt" + ); + const baseline = await import("node:fs/promises") + .then((fs) => fs.readFile(baselinePath, "utf-8")) + .then((text) => text.trim()); await ctx.client.startDictationStream(dictationId, format); @@ -755,16 +798,14 @@ function waitForSignal( const result = await ctx.client.finishDictationStream(dictationId, finalSeq); expect(result.dictationId).toBe(dictationId); - expect(wordSimilarity(result.text, baseline)).toBeGreaterThan(0.8); + expect(wordSimilarity(result.text, baseline)).toBeGreaterThan(0.6); }, 30_000 ); - test( + speechTest( "fails fast if dictation finishes without sending required chunks", async () => { - requireEnv("OPENAI_API_KEY"); - const dictationId = `dict-missing-chunks-${Date.now()}`; const format = "audio/pcm;rate=16000;bits=16"; diff --git a/packages/server/src/server/dictation/dictation-stream-manager.test.ts b/packages/server/src/server/dictation/dictation-stream-manager.test.ts index c0581942c..830739d74 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.test.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.test.ts @@ -151,3 +151,34 @@ describe("DictationStreamManager (semantic VAD grace fallback)", () => { expect(final?.payload.text).toBe("hi there"); }); }); + +describe("DictationStreamManager (provider-agnostic session factory)", () => { + it("can start with a custom session factory even without OPENAI_API_KEY", async () => { + const original = process.env.OPENAI_API_KEY; + delete process.env.OPENAI_API_KEY; + + try { + const session = new FakeRealtimeSession(); + const factory: RealtimeTranscriptionSessionFactory = () => session; + const emitted: Array<{ type: string; payload: any }> = []; + const manager = new DictationStreamManager({ + logger: pino({ level: "silent" }), + emit: (msg) => emitted.push(msg), + sessionId: "s1", + openaiApiKey: null, + sessionFactory: factory, + }); + + await manager.handleStart("d-local", "audio/pcm;rate=16000;bits=16"); + + expect(session.connected).toBe(true); + expect(emitted.find((msg) => msg.type === "dictation_stream_error")).toBeUndefined(); + } finally { + if (original !== undefined) { + process.env.OPENAI_API_KEY = original; + } else { + delete process.env.OPENAI_API_KEY; + } + } + }); +}); diff --git a/packages/server/src/server/dictation/dictation-stream-manager.ts b/packages/server/src/server/dictation/dictation-stream-manager.ts index 20a9581cf..ebad8281b 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.ts @@ -11,7 +11,7 @@ import { OpenAIRealtimeTranscriptionSession } from "../agent/openai-realtime-tra const PCM_CHANNELS = 1; const PCM_BITS_PER_SAMPLE = 16; -const DICTATION_PCM_OUTPUT_RATE = 24000; +const OPENAI_DICTATION_PCM_OUTPUT_RATE = 24000; const DEFAULT_DICTATION_FINAL_TIMEOUT_MS = 10000; const DICTATION_VAD_GRACE_TIMEOUT_MS = Number.parseInt( process.env.OPENAI_REALTIME_DICTATION_VAD_GRACE_TIMEOUT_MS ?? "2000", @@ -106,7 +106,7 @@ export type RealtimeTranscriptionSession = { }; export type RealtimeTranscriptionSessionFactory = (params: { - apiKey: string; + apiKey?: string | null; logger: pino.Logger; transcriptionModel: string; language?: string; @@ -149,6 +149,7 @@ type DictationStreamState = { inputFormat: string; openai: RealtimeTranscriptionSession; inputRate: number; + outputRate: number; resampler: Pcm16MonoResampler | null; debugAudioChunks: Buffer[]; debugRecordingPath: string | null; @@ -194,6 +195,7 @@ export class DictationStreamManager { private readonly openaiApiKey: string | null; private readonly finalTimeoutMs: number; private readonly createSession: RealtimeTranscriptionSessionFactory; + private readonly requiresApiKey: boolean; private readonly streams = new Map(); constructor(params: { @@ -209,10 +211,18 @@ export class DictationStreamManager { this.sessionId = params.sessionId; this.openaiApiKey = params.openaiApiKey ?? null; this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS; + this.requiresApiKey = !params.sessionFactory; this.createSession = params.sessionFactory ?? - ((factoryParams) => - new OpenAIRealtimeTranscriptionSession(factoryParams)); + ((factoryParams) => { + if (!factoryParams.apiKey) { + throw new Error("OPENAI_API_KEY not set"); + } + return new OpenAIRealtimeTranscriptionSession({ + ...factoryParams, + apiKey: factoryParams.apiKey, + }); + }); } public cleanupAll(): void { @@ -225,7 +235,7 @@ export class DictationStreamManager { this.cleanupDictationStream(dictationId); const apiKey = this.openaiApiKey ?? process.env.OPENAI_API_KEY; - if (!apiKey) { + if (this.requiresApiKey && !apiKey) { this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false); return; } @@ -238,7 +248,7 @@ export class DictationStreamManager { const turnDetection = parseDictationTurnDetection(); const openai = this.createSession({ - apiKey, + apiKey: apiKey ?? null, logger: this.logger.child({ dictationId }), transcriptionModel, language: "en", @@ -338,18 +348,21 @@ export class DictationStreamManager { this.logger ); + const outputRate = this.requiresApiKey ? OPENAI_DICTATION_PCM_OUTPUT_RATE : inputRate; + this.streams.set(dictationId, { dictationId, sessionId: this.sessionId, inputFormat: format, openai, inputRate, + outputRate, resampler: - inputRate === DICTATION_PCM_OUTPUT_RATE + inputRate === outputRate ? null : new Pcm16MonoResampler({ inputRate, - outputRate: DICTATION_PCM_OUTPUT_RATE, + outputRate, }), debugAudioChunks: [], debugRecordingPath: null, @@ -510,7 +523,7 @@ export class DictationStreamManager { const pcmBuffer = Buffer.concat(state.debugAudioChunks); const wavBuffer = convertPCMToWavBuffer( pcmBuffer, - DICTATION_PCM_OUTPUT_RATE, + state.outputRate, PCM_CHANNELS, PCM_BITS_PER_SAMPLE ); @@ -610,7 +623,7 @@ export class DictationStreamManager { } else { const silenceBytes = Math.max( 0, - Math.round((DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000) + Math.round((state.outputRate * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000) ); if (silenceBytes > 0) { this.logger.debug( diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index f463fcdb1..0c3ad2792 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -17,10 +17,32 @@ const ProviderCredentialsSchema = z }) .strict(); +const SherpaOnnxProviderSchema = z + .object({ + modelsDir: z.string().min(1).optional(), + autoDownload: z.boolean().optional(), + stt: z + .object({ + preset: z.string().min(1).optional(), + }) + .strict() + .optional(), + tts: z + .object({ + preset: z.string().min(1).optional(), + speakerId: z.number().int().optional(), + speed: z.number().optional(), + }) + .strict() + .optional(), + }) + .strict(); + const ProvidersSchema = z .object({ openai: ProviderCredentialsSchema.optional(), openrouter: ProviderCredentialsSchema.optional(), + sherpaOnnx: SherpaOnnxProviderSchema.optional(), }) .strict(); @@ -28,8 +50,9 @@ const FeatureDictationSchema = z .object({ stt: z .object({ - provider: z.enum(["openai"]).optional(), + provider: z.enum(["openai", "sherpa"]).optional(), model: z.string().min(1).optional(), + preset: z.string().min(1).optional(), confidenceThreshold: z.number().optional(), }) .strict() @@ -48,16 +71,20 @@ const FeatureVoiceModeSchema = z .optional(), stt: z .object({ - provider: z.enum(["openai"]).optional(), + provider: z.enum(["openai", "sherpa"]).optional(), model: z.string().min(1).optional(), + preset: z.string().min(1).optional(), }) .strict() .optional(), tts: z .object({ - provider: z.enum(["openai"]).optional(), + provider: z.enum(["openai", "sherpa"]).optional(), model: z.enum(["tts-1", "tts-1-hd"]).optional(), voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(), + preset: z.string().min(1).optional(), + speakerId: z.number().int().optional(), + speed: z.number().optional(), }) .strict() .optional(), diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 8c72c2ffe..d6a6885de 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -33,12 +33,12 @@ import { getSystemPrompt } from "./agent/system-prompt.js"; import { getAllTools } from "./agent/llm-openai.js"; import { TTSManager } from "./agent/tts-manager.js"; import { STTManager } from "./agent/stt-manager.js"; -import type { OpenAISTT } from "./agent/stt-openai.js"; -import type { OpenAITTS } from "./agent/tts-openai.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; import { maybePersistTtsDebugAudio } from "./agent/tts-debug.js"; import { isPaseoDictationDebugEnabled } from "./agent/recordings-debug.js"; import { DictationStreamManager, + type RealtimeTranscriptionSessionFactory, } from "./dictation/dictation-stream-manager.js"; import type { VoiceConversationStore } from "./voice-conversation-store.js"; import { @@ -325,8 +325,8 @@ export class Session { agentManager: AgentManager, agentStorage: AgentStorage, createAgentMcpTransport: AgentMcpTransportFactory, - stt: OpenAISTT | null, - tts: OpenAITTS | null, + stt: SpeechToTextProvider | null, + tts: TextToSpeechProvider | null, terminalManager: TerminalManager | null, voiceConversationStore: VoiceConversationStore, voice?: { @@ -336,6 +336,7 @@ export class Session { dictation?: { openaiApiKey?: string | null; finalTimeoutMs?: number; + sessionFactory?: RealtimeTranscriptionSessionFactory; } ) { this.clientId = clientId; @@ -368,6 +369,7 @@ export class Session { emit: (msg) => this.emit(msg as unknown as SessionOutboundMessage), openaiApiKey: dictation?.openaiApiKey ?? null, finalTimeoutMs: dictation?.finalTimeoutMs, + ...(dictation?.sessionFactory ? { sessionFactory: dictation.sessionFactory } : {}), }); // Initialize agent MCP client asynchronously diff --git a/packages/server/src/server/speech/audio.ts b/packages/server/src/server/speech/audio.ts new file mode 100644 index 000000000..6d1907ce0 --- /dev/null +++ b/packages/server/src/server/speech/audio.ts @@ -0,0 +1,113 @@ +export function parsePcm16MonoWav(buffer: Buffer): { sampleRate: number; pcm16: Buffer } { + if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") { + throw new Error("Invalid WAV header"); + } + + let offset = 12; + let fmt: + | { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } + | null = null; + let dataChunk: Buffer | null = null; + + while (offset + 8 <= buffer.length) { + const id = buffer.toString("ascii", offset, offset + 4); + const size = buffer.readUInt32LE(offset + 4); + const payloadStart = offset + 8; + const payloadEnd = payloadStart + size; + if (payloadEnd > buffer.length) { + break; + } + + if (id === "fmt ") { + const audioFormat = buffer.readUInt16LE(payloadStart); + const channels = buffer.readUInt16LE(payloadStart + 2); + const sampleRate = buffer.readUInt32LE(payloadStart + 4); + const bitsPerSample = buffer.readUInt16LE(payloadStart + 14); + fmt = { audioFormat, channels, sampleRate, bitsPerSample }; + } else if (id === "data") { + dataChunk = buffer.subarray(payloadStart, payloadEnd); + } + + offset = payloadEnd + (size % 2); + } + + if (!fmt || !dataChunk) { + throw new Error("Missing WAV fmt/data chunks"); + } + if (fmt.audioFormat !== 1) { + throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`); + } + if (fmt.channels !== 1 || fmt.bitsPerSample !== 16) { + throw new Error( + `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}` + ); + } + if (dataChunk.length % 2 !== 0) { + throw new Error("WAV PCM16 data length must be even"); + } + return { sampleRate: fmt.sampleRate, pcm16: dataChunk }; +} + +export function parsePcmRateFromFormat(format: string, fallback: number | null = null): number | null { + const match = /(?:^|[;,\s])rate\s*=\s*(\d+)(?:$|[;,\s])/i.exec(format); + if (!match) { + return fallback; + } + const rate = Number.parseInt(match[1]!, 10); + return Number.isFinite(rate) && rate > 0 ? rate : fallback; +} + +export function pcm16lePeakAbs(pcm16le: Buffer): number { + if (pcm16le.length === 0) { + return 0; + } + if (pcm16le.length % 2 !== 0) { + throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`); + } + const samples = new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2); + let peak = 0; + for (let i = 0; i < samples.length; i += 1) { + const v = samples[i]!; + const abs = v < 0 ? -v : v; + if (abs > peak) { + peak = abs; + if (peak >= 32767) { + break; + } + } + } + return peak; +} + +export function pcm16leToFloat32(pcm16le: Buffer, gain: number = 1): Float32Array { + if (pcm16le.length % 2 !== 0) { + throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`); + } + const int16 = new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2); + const out = new Float32Array(int16.length); + for (let i = 0; i < int16.length; i += 1) { + const v = (int16[i]! / 32768.0) * gain; + out[i] = v > 1 ? 1 : v < -1 ? -1 : v; + } + return out; +} + +export function float32ToPcm16le(samples: Float32Array): Buffer { + const out = new Int16Array(samples.length); + for (let i = 0; i < samples.length; i += 1) { + const clamped = Math.max(-1, Math.min(1, samples[i]!)); + out[i] = Math.round(clamped * 32767); + } + return Buffer.from(out.buffer, out.byteOffset, out.byteLength); +} + +export function chunkBuffer(buffer: Buffer, chunkBytes: number): Buffer[] { + if (chunkBytes <= 0) { + return [buffer]; + } + const out: Buffer[] = []; + for (let offset = 0; offset < buffer.length; offset += chunkBytes) { + out.push(buffer.subarray(offset, Math.min(buffer.length, offset + chunkBytes))); + } + return out; +} diff --git a/packages/server/src/server/speech/pocket/pocket-tts-onnx.ts b/packages/server/src/server/speech/pocket/pocket-tts-onnx.ts new file mode 100644 index 000000000..58fd16e96 --- /dev/null +++ b/packages/server/src/server/speech/pocket/pocket-tts-onnx.ts @@ -0,0 +1,559 @@ +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { Readable } from "node:stream"; +import type pino from "pino"; + +import type { SpeechStreamResult, TextToSpeechProvider } from "../speech-provider.js"; +import { chunkBuffer, float32ToPcm16le, parsePcm16MonoWav, pcm16leToFloat32 } from "../audio.js"; +import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; + +type OrtModule = typeof import("onnxruntime-node"); +type OrtSession = import("onnxruntime-node").InferenceSession; +type OrtTensor = import("onnxruntime-node").Tensor; + +type SentencePieceProcessor = { + encodeIds: (text: string) => number[]; + load?: (modelPath: string) => unknown; + Load?: (modelPath: string) => unknown; +}; + +function assertFileExists(filePath: string, label: string): void { + if (!existsSync(filePath)) { + throw new Error(`Missing ${label}: ${filePath}`); + } +} + +function product(dims: number[]): number { + let out = 1; + for (const d of dims) out *= d; + return out; +} + +function normalizeDims(dims: Array): number[] { + // ONNX metadata can contain dynamic dimensions as strings (e.g. "batch") or -1. + // For state tensors we want a valid minimal shape, so coerce unknown/invalid dims to 1. + // Preserve explicit 0 dims (some models use empty initial state buffers with shape [0]). + return dims.map((d) => { + if (typeof d === "number" && Number.isFinite(d)) { + if (d === 0) return 0; + if (d > 0) return d; + return 1; + } + return 1; + }); +} + +function getSessionInputMeta( + session: OrtSession, + inputName: string +): { type?: string; dims?: Array } | undefined { + const metaAny = (session as any).inputMetadata as unknown; + if (Array.isArray(metaAny)) { + const entry = metaAny.find((m) => m && typeof m === "object" && (m as any).name === inputName) as any; + if (!entry) return undefined; + return { type: entry.type, dims: entry.shape }; + } + + if (metaAny && typeof metaAny === "object" && inputName in (metaAny as any)) { + const entry = (metaAny as any)[inputName] as any; + return { type: entry?.type, dims: entry?.dimensions ?? entry?.shape }; + } + + return undefined; +} + +function toBigInt64(values: number[]): BigInt64Array { + const out = new BigInt64Array(values.length); + for (let i = 0; i < values.length; i += 1) { + out[i] = BigInt(values[i]!); + } + return out; +} + +function randn(): number { + // Box–Muller + let u = 0; + let v = 0; + while (u === 0) u = Math.random(); + while (v === 0) v = Math.random(); + return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v); +} + +function normalizeTextForPocket(text: string): string { + const trimmed = text.trim(); + if (!trimmed) { + throw new Error("Cannot synthesize empty text"); + } + let out = trimmed; + if (out.length > 0 && /[A-Za-z0-9]$/.test(out)) { + out = `${out}.`; + } + if (out.length > 0 && /[a-z]/.test(out[0]!)) { + out = out[0]!.toUpperCase() + out.slice(1); + } + return out; +} + +async function loadOrt(): Promise { + return (await import("onnxruntime-node")) as OrtModule; +} + +async function loadSentencePiece(tokenizerModelPath: string): Promise { + const mod = await import("@sctg/sentencepiece-js"); + + const Processor = + (mod as any).SentencePieceProcessor ?? + (mod as any).default?.SentencePieceProcessor ?? + (mod as any).default; + + if (!Processor) { + throw new Error("Failed to load SentencePiece processor from @sctg/sentencepiece-js"); + } + + const sp: SentencePieceProcessor = new Processor(); + + if (typeof sp.load === "function") { + await sp.load(tokenizerModelPath); + } else if (typeof sp.Load === "function") { + sp.Load(tokenizerModelPath); + } else { + throw new Error("SentencePiece processor does not expose load()/Load()"); + } + + return sp; +} + +function getOrtProviders(ort: OrtModule, device: "auto" | "cpu" | "cuda"): string[] { + // NOTE: onnxruntime-node uses backend names like "cpu"/"coreml"/"webgpu" (not "CPUExecutionProvider"). + if (device === "cpu") return ["cpu"]; + if (device === "cuda") return ["cuda", "cpu"]; + // auto + // CoreML EP does not support some dynamic/zero-length shapes used by Pocket TTS (e.g. [1, 0, 32]). + // Default to CPU to keep behavior predictable across platforms. + void ort; + return ["cpu"]; +} + +function createZeroTensorForInput(ort: OrtModule, session: OrtSession, inputName: string): OrtTensor { + const meta = getSessionInputMeta(session, inputName); + const dims = normalizeDims(meta?.dims ?? []); + if (dims.length === 0) { + throw new Error(`Missing input metadata shape for ${inputName}`); + } + + const type = (meta?.type ?? "float32").toLowerCase(); + const size = product(dims); + + if (type.includes("int64")) { + return new ort.Tensor("int64", new BigInt64Array(size), dims); + } + if (type.includes("bool")) { + return new ort.Tensor("bool", new Uint8Array(size), dims); + } + return new ort.Tensor("float32", new Float32Array(size), dims); +} + +function initState(session: OrtSession, ort: OrtModule): Record { + const out: Record = {}; + for (const name of (session as any).inputNames as string[]) { + if (name.startsWith("state_")) { + out[name] = createZeroTensorForInput(ort, session, name); + } + } + return out; +} + +function updateStateFromOutputs( + state: Record, + outputs: Record +): void { + for (const [name, tensor] of Object.entries(outputs)) { + if (!name.startsWith("out_state_")) continue; + const idx = Number.parseInt(name.replace("out_state_", ""), 10); + if (Number.isFinite(idx)) { + state[`state_${idx}`] = tensor; + } + } +} + +function tensorDataFloat32(t: OrtTensor): Float32Array { + const data = (t as any).data; + if (data instanceof Float32Array) return data; + if (Array.isArray(data)) return Float32Array.from(data as number[]); + throw new Error("Unexpected tensor data type (expected Float32Array)"); +} + +export type PocketTtsOnnxConfig = { + modelDir: string; + precision?: "int8" | "fp32"; + device?: "auto" | "cpu" | "cuda"; + temperature?: number; + lsdSteps?: number; + maxFrames?: number; + framesAfterEos?: number; + firstChunkFrames?: number; + maxChunkFrames?: number; + targetChunkMs?: number; + referenceAudioFile?: string; +}; + +class PocketTtsOnnxEngine { + static readonly SAMPLE_RATE = 24000; + static readonly SAMPLES_PER_FRAME = 1920; + + private readonly ort: OrtModule; + + private readonly temperature: number; + private readonly lsdSteps: number; + private readonly maxFrames: number; + private readonly framesAfterEos: number; + + private readonly firstChunkFrames: number; + private readonly maxChunkFrames: number; + + private readonly tokenizer: SentencePieceProcessor; + private readonly textConditioner: OrtSession; + private readonly flowLmMain: OrtSession; + private readonly flowLmFlow: OrtSession; + private readonly mimiDecoder: OrtSession; + + private readonly stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>; + private readonly voiceEmbeddings: OrtTensor; + + private constructor(args: { + ort: OrtModule; + temperature: number; + lsdSteps: number; + maxFrames: number; + framesAfterEos: number; + firstChunkFrames: number; + maxChunkFrames: number; + tokenizer: SentencePieceProcessor; + textConditioner: OrtSession; + flowLmMain: OrtSession; + flowLmFlow: OrtSession; + mimiDecoder: OrtSession; + stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>; + voiceEmbeddings: OrtTensor; + }) { + this.ort = args.ort; + this.temperature = args.temperature; + this.lsdSteps = args.lsdSteps; + this.maxFrames = args.maxFrames; + this.framesAfterEos = args.framesAfterEos; + this.firstChunkFrames = args.firstChunkFrames; + this.maxChunkFrames = args.maxChunkFrames; + this.tokenizer = args.tokenizer; + this.textConditioner = args.textConditioner; + this.flowLmMain = args.flowLmMain; + this.flowLmFlow = args.flowLmFlow; + this.mimiDecoder = args.mimiDecoder; + this.stBuffers = args.stBuffers; + this.voiceEmbeddings = args.voiceEmbeddings; + } + + static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise { + const log = logger.child({ module: "speech", provider: "pocket-tts", component: "onnx-engine" }); + + const modelDir = config.modelDir; + const onnxDir = `${modelDir}/onnx`; + const precision = config.precision ?? "int8"; + const device = config.device ?? "auto"; + const temperature = config.temperature ?? 0.7; + const lsdSteps = config.lsdSteps ?? 10; + const maxFrames = config.maxFrames ?? 500; + const framesAfterEos = config.framesAfterEos ?? 3; + const firstChunkFrames = config.firstChunkFrames ?? 2; + const maxChunkFrames = config.maxChunkFrames ?? 15; + + const tokenizerPath = `${modelDir}/tokenizer.model`; + const referenceAudioFile = config.referenceAudioFile ?? `${modelDir}/reference_sample.wav`; + + const flowMainFile = precision === "int8" ? "flow_lm_main_int8.onnx" : "flow_lm_main.onnx"; + const flowFlowFile = precision === "int8" ? "flow_lm_flow_int8.onnx" : "flow_lm_flow.onnx"; + const decoderFile = precision === "int8" ? "mimi_decoder_int8.onnx" : "mimi_decoder.onnx"; + + assertFileExists(`${onnxDir}/mimi_encoder.onnx`, "PocketTTS mimi_encoder"); + assertFileExists(`${onnxDir}/text_conditioner.onnx`, "PocketTTS text_conditioner"); + assertFileExists(`${onnxDir}/${flowMainFile}`, "PocketTTS flow_lm_main"); + assertFileExists(`${onnxDir}/${flowFlowFile}`, "PocketTTS flow_lm_flow"); + assertFileExists(`${onnxDir}/${decoderFile}`, "PocketTTS mimi_decoder"); + assertFileExists(tokenizerPath, "PocketTTS tokenizer.model"); + assertFileExists(referenceAudioFile, "PocketTTS reference_sample.wav"); + + const ort = await loadOrt(); + const providers = getOrtProviders(ort, device); + + const [tokenizer, mimiEncoder, textConditioner, flowLmMain, flowLmFlow, mimiDecoder] = await Promise.all([ + loadSentencePiece(tokenizerPath), + ort.InferenceSession.create(`${onnxDir}/mimi_encoder.onnx`, { executionProviders: providers }), + ort.InferenceSession.create(`${onnxDir}/text_conditioner.onnx`, { executionProviders: providers }), + ort.InferenceSession.create(`${onnxDir}/${flowMainFile}`, { executionProviders: providers }), + ort.InferenceSession.create(`${onnxDir}/${flowFlowFile}`, { executionProviders: providers }), + ort.InferenceSession.create(`${onnxDir}/${decoderFile}`, { executionProviders: providers }), + ]); + + // Precompute flow matching time-step buffers. + const stBuffers: Array<{ s: OrtTensor; t: OrtTensor }> = []; + for (let j = 0; j < lsdSteps; j += 1) { + const s = j / lsdSteps; + const t = s + 1.0 / lsdSteps; + stBuffers.push({ + s: new ort.Tensor("float32", new Float32Array([s]), [1, 1]), + t: new ort.Tensor("float32", new Float32Array([t]), [1, 1]), + }); + } + + // Precompute reference voice embeddings once. + const refWav = await readFile(referenceAudioFile); + const parsed = parsePcm16MonoWav(refWav); + let pcm16 = parsed.pcm16; + if (parsed.sampleRate !== PocketTtsOnnxEngine.SAMPLE_RATE) { + const resampler = new Pcm16MonoResampler({ + inputRate: parsed.sampleRate, + outputRate: PocketTtsOnnxEngine.SAMPLE_RATE, + }); + pcm16 = resampler.processChunk(pcm16); + } + const floatAudio = pcm16leToFloat32(pcm16); + const audioTensor = new ort.Tensor( + "float32", + floatAudio, + [1, 1, floatAudio.length] + ); + + const encoded = await mimiEncoder.run({ audio: audioTensor }); + const firstOutName = (mimiEncoder as any).outputNames?.[0] as string | undefined; + const voiceEmb = firstOutName ? (encoded as any)[firstOutName] : (Object.values(encoded)[0] as any); + if (!voiceEmb) { + throw new Error("PocketTTS mimi_encoder: missing output"); + } + + log.info( + { precision, device, providers, lsdSteps, temperature }, + "PocketTTS ONNX initialized" + ); + + return new PocketTtsOnnxEngine({ + ort, + temperature, + lsdSteps, + maxFrames, + framesAfterEos, + firstChunkFrames, + maxChunkFrames, + tokenizer, + textConditioner, + flowLmMain, + flowLmFlow, + mimiDecoder, + stBuffers, + voiceEmbeddings: voiceEmb, + }); + } + + private tokenize(text: string): OrtTensor { + const normalized = normalizeTextForPocket(text); + const ids = this.tokenizer.encodeIds(normalized); + const data = toBigInt64(ids ?? []); + return new this.ort.Tensor("int64", data, [1, data.length]); + } + + private async runTextConditioner(tokenIds: OrtTensor): Promise { + const out = await this.textConditioner.run({ token_ids: tokenIds } as any); + const firstOutName = (this.textConditioner as any).outputNames?.[0] as string | undefined; + const t = firstOutName ? (out as any)[firstOutName] : (Object.values(out)[0] as any); + if (!t) throw new Error("PocketTTS text_conditioner: missing output"); + return t; + } + + private async *runFlowLm(textEmbeddings: OrtTensor): AsyncGenerator { + const ort = this.ort; + const state = initState(this.flowLmMain, ort); + + const emptySeq = new ort.Tensor("float32", new Float32Array(0), [1, 0, 32]); + const emptyText = new ort.Tensor("float32", new Float32Array(0), [1, 0, 1024]); + + // Voice conditioning pass + const resVoice = await this.flowLmMain.run({ + sequence: emptySeq, + text_embeddings: this.voiceEmbeddings, + ...state, + } as any); + updateStateFromOutputs(state, resVoice as any); + + // Text conditioning pass + const resText = await this.flowLmMain.run({ + sequence: emptySeq, + text_embeddings: textEmbeddings, + ...state, + } as any); + updateStateFromOutputs(state, resText as any); + + // Autoregressive generation + const curr = new Float32Array(32); + curr.fill(Number.NaN); + let currTensor = new ort.Tensor("float32", curr, [1, 1, 32]); + + const dt = 1.0 / this.lsdSteps; + let eosStep: number | null = null; + + for (let step = 0; step < this.maxFrames; step += 1) { + const resStep = await this.flowLmMain.run({ + sequence: currTensor, + text_embeddings: emptyText, + ...state, + } as any); + + const outputNames = (this.flowLmMain as any).outputNames as string[] | undefined; + const conditioningName = outputNames?.[0] ?? Object.keys(resStep)[0]!; + const eosName = outputNames?.[1] ?? Object.keys(resStep)[1]!; + + const conditioning = (resStep as any)[conditioningName] as OrtTensor; + const eos = (resStep as any)[eosName] as OrtTensor; + if (!conditioning || !eos) { + throw new Error("PocketTTS flow_lm_main: missing conditioning/EOS outputs"); + } + updateStateFromOutputs(state, resStep as any); + + const eosData = tensorDataFloat32(eos); + if (eosData[0]! > -4.0 && eosStep === null) { + eosStep = step; + } + if (eosStep !== null && step >= eosStep + this.framesAfterEos) { + break; + } + + // Flow matching with external Euler loop. + const std = this.temperature > 0 ? Math.sqrt(this.temperature) : 0; + const x = new Float32Array(32); + if (std > 0) { + for (let i = 0; i < x.length; i += 1) { + x[i] = randn() * std; + } + } + + for (const st of this.stBuffers) { + const xTensor = new ort.Tensor("float32", x, [1, 32]); + const flowOut = await this.flowLmFlow.run({ + c: conditioning, + s: st.s, + t: st.t, + x: xTensor, + } as any); + const first = (this.flowLmFlow as any).outputNames?.[0] as string | undefined; + const flowTensor = first ? (flowOut as any)[first] : (Object.values(flowOut)[0] as any); + if (!flowTensor) throw new Error("PocketTTS flow_lm_flow: missing output"); + const delta = tensorDataFloat32(flowTensor); + for (let i = 0; i < x.length; i += 1) { + x[i] = x[i]! + delta[i]! * dt; + } + } + + yield x; + currTensor = new ort.Tensor("float32", x, [1, 1, 32]); + } + } + + private async decodeLatentsChunk( + frames: Float32Array[], + state: Record + ): Promise { + const ort = this.ort; + const frameCount = frames.length; + const flattened = new Float32Array(frameCount * 32); + for (let i = 0; i < frameCount; i += 1) { + flattened.set(frames[i]!, i * 32); + } + const latent = new ort.Tensor("float32", flattened, [1, frameCount, 32]); + + const out = await this.mimiDecoder.run({ latent, ...state } as any); + updateStateFromOutputs(state, out as any); + + const firstOutName = (this.mimiDecoder as any).outputNames?.[0] as string | undefined; + const audioTensor = firstOutName ? (out as any)[firstOutName] : (Object.values(out)[0] as any); + if (!audioTensor) { + throw new Error("PocketTTS mimi_decoder: missing audio output"); + } + return tensorDataFloat32(audioTensor); + } + + async *streamAudio(text: string): AsyncGenerator { + const tokenIds = this.tokenize(text); + const textEmb = await this.runTextConditioner(tokenIds); + + const decoderState = initState(this.mimiDecoder, this.ort); + + const generated: Float32Array[] = []; + let decodedFrames = 0; + + for await (const latent of this.runFlowLm(textEmb)) { + generated.push(latent); + const pending = generated.length - decodedFrames; + + let chunkSize = 0; + if (decodedFrames === 0) { + if (pending >= this.firstChunkFrames) { + chunkSize = this.firstChunkFrames; + } + } else if (pending >= this.maxChunkFrames) { + chunkSize = this.maxChunkFrames; + } + + if (chunkSize > 0) { + const audio = await this.decodeLatentsChunk( + generated.slice(decodedFrames, decodedFrames + chunkSize), + decoderState + ); + decodedFrames += chunkSize; + yield audio; + } + } + + if (decodedFrames < generated.length) { + const audio = await this.decodeLatentsChunk(generated.slice(decodedFrames), decoderState); + yield audio; + } + } +} + +export class PocketTtsOnnxTTS implements TextToSpeechProvider { + private readonly engine: PocketTtsOnnxEngine; + private readonly chunkMs: number; + private readonly logger: pino.Logger; + + private constructor(engine: PocketTtsOnnxEngine, logger: pino.Logger, chunkMs: number) { + this.engine = engine; + this.chunkMs = chunkMs; + this.logger = logger.child({ module: "speech", provider: "pocket-tts", component: "tts" }); + } + + static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise { + const engine = await PocketTtsOnnxEngine.create(config, logger); + const chunkMs = config.targetChunkMs ?? 50; + return new PocketTtsOnnxTTS(engine, logger, chunkMs); + } + + async synthesizeSpeech(text: string): Promise { + const start = Date.now(); + const sampleRate = PocketTtsOnnxEngine.SAMPLE_RATE; + + const pcmChunkBytes = Math.max(2, Math.round((sampleRate * this.chunkMs) / 1000) * 2); + + const iterable = (async function* (engine: PocketTtsOnnxEngine) { + for await (const floatChunk of engine.streamAudio(text)) { + const pcm = float32ToPcm16le(floatChunk); + for (const chunk of chunkBuffer(pcm, pcmChunkBytes)) { + yield chunk; + } + } + })(this.engine); + + this.logger.debug({ ms: Date.now() - start, textLength: text.length }, "PocketTTS stream ready"); + + return { + stream: Readable.from(iterable), + format: `pcm;rate=${sampleRate}`, + }; + } +} diff --git a/packages/server/src/server/speech/sherpa/model-catalog.ts b/packages/server/src/server/speech/sherpa/model-catalog.ts new file mode 100644 index 000000000..94c6d62bd --- /dev/null +++ b/packages/server/src/server/speech/sherpa/model-catalog.ts @@ -0,0 +1,128 @@ +export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts"; + +export type SherpaOnnxModelId = + | "zipformer-bilingual-zh-en-2023-02-20" + | "paraformer-bilingual-zh-en" + | "parakeet-tdt-0.6b-v3-int8" + | "kitten-nano-en-v0_1-fp16" + | "kokoro-en-v0_19" + | "pocket-tts-onnx-int8"; + +export type SherpaOnnxModelSpec = { + id: SherpaOnnxModelId; + kind: SherpaOnnxModelKind; + archiveUrl?: string; + downloadFiles?: Array<{ url: string; relPath: string }>; + extractedDir: string; + requiredFiles: string[]; + description: string; +}; + +export const SHERPA_ONNX_MODEL_CATALOG: Record = { + "zipformer-bilingual-zh-en-2023-02-20": { + id: "zipformer-bilingual-zh-en-2023-02-20", + kind: "stt-online", + archiveUrl: + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2", + extractedDir: "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", + requiredFiles: [ + "encoder-epoch-99-avg-1.onnx", + "decoder-epoch-99-avg-1.onnx", + "joiner-epoch-99-avg-1.onnx", + "tokens.txt", + ], + description: "Streaming Zipformer transducer (fast, good accuracy).", + }, + "paraformer-bilingual-zh-en": { + id: "paraformer-bilingual-zh-en", + kind: "stt-online", + archiveUrl: + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2", + extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en", + requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"], + description: "Streaming Paraformer (often strong accuracy; heavier).", + }, + "parakeet-tdt-0.6b-v3-int8": { + id: "parakeet-tdt-0.6b-v3-int8", + kind: "stt-offline", + archiveUrl: + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2", + extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8", + requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"], + description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).", + }, + "kitten-nano-en-v0_1-fp16": { + id: "kitten-nano-en-v0_1-fp16", + kind: "tts", + archiveUrl: + "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2", + extractedDir: "kitten-nano-en-v0_1-fp16", + requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], + description: "KittenTTS (small, fast English TTS).", + }, + "kokoro-en-v0_19": { + id: "kokoro-en-v0_19", + kind: "tts", + archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2", + extractedDir: "kokoro-en-v0_19", + requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], + description: "Kokoro TTS (higher quality; larger).", + }, + "pocket-tts-onnx-int8": { + id: "pocket-tts-onnx-int8", + kind: "tts", + extractedDir: "pocket-tts-onnx-int8", + downloadFiles: [ + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_encoder.onnx", + relPath: "onnx/mimi_encoder.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/text_conditioner.onnx", + relPath: "onnx/text_conditioner.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_main_int8.onnx", + relPath: "onnx/flow_lm_main_int8.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_flow_int8.onnx", + relPath: "onnx/flow_lm_flow_int8.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_decoder_int8.onnx", + relPath: "onnx/mimi_decoder_int8.onnx", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/tokenizer.model", + relPath: "tokenizer.model", + }, + { + url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/reference_sample.wav", + relPath: "reference_sample.wav", + }, + ], + requiredFiles: [ + "onnx/mimi_encoder.onnx", + "onnx/text_conditioner.onnx", + "onnx/flow_lm_main_int8.onnx", + "onnx/flow_lm_flow_int8.onnx", + "onnx/mimi_decoder_int8.onnx", + "tokenizer.model", + "reference_sample.wav", + ], + description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).", + }, +}; + +export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] { + return Object.values(SHERPA_ONNX_MODEL_CATALOG); +} + +export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec { + const spec = SHERPA_ONNX_MODEL_CATALOG[id]; + if (!spec) { + throw new Error(`Unknown sherpa-onnx model id: ${id}`); + } + return spec; +} diff --git a/packages/server/src/server/speech/sherpa/model-downloader.test.ts b/packages/server/src/server/speech/sherpa/model-downloader.test.ts new file mode 100644 index 000000000..ab898bfae --- /dev/null +++ b/packages/server/src/server/speech/sherpa/model-downloader.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import pino from "pino"; + +import { ensureSherpaOnnxModel, getSherpaOnnxModelDir } from "./model-downloader.js"; + +function makeTmpDir(): string { + return mkdtempSync(path.join(tmpdir(), "paseo-speech-models-")); +} + +const logger = pino({ level: "silent" }); + +describe("sherpa model downloader", () => { + test("getSherpaOnnxModelDir maps modelId to extractedDir", () => { + const modelsDir = "/tmp/models"; + expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8")).toContain( + "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8" + ); + expect(getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8")).toContain( + "pocket-tts-onnx-int8" + ); + }); + + test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => { + const modelsDir = makeTmpDir(); + const modelDir = getSherpaOnnxModelDir(modelsDir, "kitten-nano-en-v0_1-fp16"); + + mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true }); + writeFileSync(path.join(modelDir, "model.fp16.onnx"), "x"); + writeFileSync(path.join(modelDir, "voices.bin"), "x"); + writeFileSync(path.join(modelDir, "tokens.txt"), "x"); + + const out = await ensureSherpaOnnxModel({ + modelsDir, + modelId: "kitten-nano-en-v0_1-fp16", + autoDownload: false, + logger, + }); + + expect(out).toBe(modelDir); + }); + + test("ensureSherpaOnnxModel throws when missing and autoDownload is false", async () => { + const modelsDir = makeTmpDir(); + await expect( + ensureSherpaOnnxModel({ + modelsDir, + modelId: "zipformer-bilingual-zh-en-2023-02-20", + autoDownload: false, + logger, + }) + ).rejects.toThrow(/auto-download/i); + }); +}); diff --git a/packages/server/src/server/speech/sherpa/model-downloader.ts b/packages/server/src/server/speech/sherpa/model-downloader.ts new file mode 100644 index 000000000..9c434c4be --- /dev/null +++ b/packages/server/src/server/speech/sherpa/model-downloader.ts @@ -0,0 +1,190 @@ +import { createWriteStream } from "node:fs"; +import { mkdir, rename, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { spawn } from "node:child_process"; +import type pino from "pino"; + +import { getSherpaOnnxModelSpec, type SherpaOnnxModelId } from "./model-catalog.js"; + +export type EnsureSherpaOnnxModelOptions = { + modelsDir: string; + modelId: SherpaOnnxModelId; + autoDownload: boolean; + logger: pino.Logger; +}; + +export function getSherpaOnnxModelDir(modelsDir: string, modelId: SherpaOnnxModelId): string { + const spec = getSherpaOnnxModelSpec(modelId); + return path.join(modelsDir, spec.extractedDir); +} + +async function hasRequiredFiles(modelDir: string, requiredFiles: string[]): Promise { + for (const rel of requiredFiles) { + const abs = path.join(modelDir, rel); + try { + const s = await stat(abs); + if (s.isDirectory()) { + continue; + } + if (s.isFile() && s.size > 0) { + continue; + } + return false; + } catch { + return false; + } + } + return true; +} + +async function downloadToFile(url: string, outputPath: string, logger: pino.Logger): Promise { + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`); + } + if (!res.body) { + throw new Error(`Failed to download ${url}: missing response body`); + } + + const tmpPath = `${outputPath}.tmp-${Date.now()}`; + await mkdir(path.dirname(outputPath), { recursive: true }); + + const total = Number(res.headers.get("content-length") ?? "0"); + let downloaded = 0; + let lastLoggedBucket = -1; + + const nodeStream = Readable.fromWeb(res.body as any).on("data", (chunk: Buffer) => { + downloaded += chunk.length; + if (total > 0) { + const pct = Math.floor((downloaded / total) * 100); + const bucket = Math.min(100, Math.floor(pct / 10) * 10); + if (bucket >= 0 && bucket <= 100 && bucket !== lastLoggedBucket) { + lastLoggedBucket = bucket; + logger.info({ pct: bucket, downloaded, total }, "Downloading model artifact"); + } + } + }); + + await pipeline(nodeStream, createWriteStream(tmpPath)); + await rename(tmpPath, outputPath); +} + +async function extractTarArchive(archivePath: string, destDir: string): Promise { + await mkdir(destDir, { recursive: true }); + + await new Promise((resolve, reject) => { + const child = spawn("tar", ["xf", archivePath, "-C", destDir], { stdio: "inherit" }); + child.on("error", reject); + child.on("exit", (code) => { + if (code === 0) resolve(); + else reject(new Error(`tar exited with code ${code}`)); + }); + }); +} + +async function isNonEmptyFile(filePath: string): Promise { + try { + const s = await stat(filePath); + return s.isFile() && s.size > 0; + } catch { + return false; + } +} + +export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise { + const logger = options.logger.child({ + module: "speech", + provider: "sherpa-onnx", + component: "model-downloader", + modelId: options.modelId, + }); + + const spec = getSherpaOnnxModelSpec(options.modelId); + const modelDir = path.join(options.modelsDir, spec.extractedDir); + if (await hasRequiredFiles(modelDir, spec.requiredFiles)) { + return modelDir; + } + + if (!options.autoDownload) { + throw new Error( + `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` + + `Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.` + ); + } + + if (spec.archiveUrl) { + logger.info({ modelsDir: options.modelsDir, url: spec.archiveUrl }, "Model files missing; downloading"); + + const downloadsDir = path.join(options.modelsDir, ".downloads"); + const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname); + const archivePath = path.join(downloadsDir, archiveFilename); + + if (!(await isNonEmptyFile(archivePath))) { + await downloadToFile(spec.archiveUrl, archivePath, logger); + } else { + logger.info({ archivePath }, "Using cached archive"); + } + + await extractTarArchive(archivePath, options.modelsDir); + + if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) { + throw new Error( + `Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.` + ); + } + + try { + await rm(archivePath, { force: true }); + } catch { + // ignore + } + + logger.info({ modelDir }, "Model ready"); + return modelDir; + } + + if (spec.downloadFiles && spec.downloadFiles.length > 0) { + logger.info({ modelsDir: options.modelsDir, fileCount: spec.downloadFiles.length }, "Model files missing; downloading"); + await mkdir(modelDir, { recursive: true }); + + for (const file of spec.downloadFiles) { + const dst = path.join(modelDir, file.relPath); + if (await isNonEmptyFile(dst)) { + continue; + } + await downloadToFile(file.url, dst, logger); + } + + if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) { + throw new Error( + `Downloaded files for ${options.modelId}, but required files are still missing in ${modelDir}.` + ); + } + + logger.info({ modelDir }, "Model ready"); + return modelDir; + } + + throw new Error(`Model spec for ${options.modelId} has no archiveUrl or downloadFiles`); +} + +export async function ensureSherpaOnnxModels(options: { + modelsDir: string; + modelIds: SherpaOnnxModelId[]; + autoDownload: boolean; + logger: pino.Logger; +}): Promise> { + const uniq = Array.from(new Set(options.modelIds)); + const out: Partial> = {}; + for (const id of uniq) { + out[id] = await ensureSherpaOnnxModel({ + modelsDir: options.modelsDir, + modelId: id, + autoDownload: options.autoDownload, + logger: options.logger, + }); + } + return out as Record; +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts b/packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts new file mode 100644 index 000000000..810b8d165 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts @@ -0,0 +1,104 @@ +import { existsSync } from "node:fs"; +import type pino from "pino"; + +import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; + +function assertFileExists(filePath: string, label: string): void { + if (!existsSync(filePath)) { + throw new Error(`Missing ${label}: ${filePath}`); + } +} + +export type SherpaOfflineRecognizerModel = { + kind: "nemo_transducer"; + encoder: string; + decoder: string; + joiner: string; + tokens: string; +}; + +export type SherpaOfflineRecognizerConfig = { + model: SherpaOfflineRecognizerModel; + numThreads?: number; + provider?: "cpu"; + debug?: 0 | 1; + sampleRate?: number; + featureDim?: number; + decodingMethod?: "greedy_search"; + maxActivePaths?: number; +}; + +export class SherpaOfflineRecognizerEngine { + public readonly recognizer: any; + public readonly sampleRate: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) { + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" }); + + assertFileExists(config.model.encoder, "offline encoder"); + assertFileExists(config.model.decoder, "offline decoder"); + assertFileExists(config.model.joiner, "offline joiner"); + assertFileExists(config.model.tokens, "tokens"); + + const sherpa = loadSherpaOnnxNode(); + + const recognizerConfig = { + featConfig: { + sampleRate: config.sampleRate ?? 16000, + featureDim: config.featureDim ?? 80, + }, + modelConfig: { + transducer: { + encoder: config.model.encoder, + decoder: config.model.decoder, + joiner: config.model.joiner, + }, + tokens: config.model.tokens, + modelType: "nemo_transducer", + numThreads: config.numThreads ?? 1, + provider: config.provider ?? "cpu", + debug: config.debug ?? 0, + }, + decodingMethod: config.decodingMethod ?? "greedy_search", + maxActivePaths: config.maxActivePaths ?? 4, + }; + + this.recognizer = new sherpa.OfflineRecognizer(recognizerConfig); + const sr = this.recognizer?.config?.featConfig?.sampleRate; + this.sampleRate = typeof sr === "number" && Number.isFinite(sr) && sr > 0 + ? sr + : recognizerConfig.featConfig.sampleRate; + + this.logger.info( + { sampleRate: this.sampleRate, numThreads: recognizerConfig.modelConfig.numThreads }, + "Sherpa offline recognizer initialized" + ); + } + + createStream(): any { + return this.recognizer.createStream(); + } + + acceptWaveform(stream: any, sampleRate: number, samples: Float32Array): void { + if (!stream || typeof stream.acceptWaveform !== "function") { + throw new Error("Unexpected sherpa offline stream: missing acceptWaveform()"); + } + + // sherpa-onnx-node expects: acceptWaveform({ samples, sampleRate }) + // sherpa-onnx (WASM) expects: acceptWaveform(sampleRate, samples) + if (stream.acceptWaveform.length <= 1) { + stream.acceptWaveform({ samples, sampleRate }); + } else { + stream.acceptWaveform(sampleRate, samples); + } + } + + free(): void { + try { + this.recognizer?.free?.(); + } catch (err) { + this.logger.warn({ err }, "Failed to free sherpa offline recognizer"); + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts b/packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts new file mode 100644 index 000000000..c7c8c3b1b --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts @@ -0,0 +1,127 @@ +import { existsSync } from "node:fs"; +import type pino from "pino"; + +import { loadSherpaOnnx } from "./sherpa-onnx-loader.js"; + +export type SherpaOnlineRecognizerModel = + | { + kind: "transducer"; + encoder: string; + decoder: string; + joiner: string; + tokens: string; + modelType?: string; + } + | { + kind: "paraformer"; + encoder: string; + decoder: string; + tokens: string; + }; + +export type SherpaOnlineRecognizerConfig = { + model: SherpaOnlineRecognizerModel; + numThreads?: number; + provider?: "cpu"; + debug?: 0 | 1; + sampleRate?: number; + featureDim?: number; + decodingMethod?: "greedy_search"; + maxActivePaths?: number; + enableEndpoint?: 0 | 1; + rule1MinTrailingSilence?: number; + rule2MinTrailingSilence?: number; + rule3MinUtteranceLength?: number; +}; + +function assertFileExists(filePath: string, label: string): void { + if (!existsSync(filePath)) { + throw new Error(`Missing ${label}: ${filePath}`); + } +} + +export class SherpaOnlineRecognizerEngine { + public readonly recognizer: any; + public readonly sampleRate: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) { + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" }); + + const { model } = config; + if (model.kind === "transducer") { + assertFileExists(model.encoder, "transducer encoder"); + assertFileExists(model.decoder, "transducer decoder"); + assertFileExists(model.joiner, "transducer joiner"); + assertFileExists(model.tokens, "tokens"); + } else { + assertFileExists(model.encoder, "paraformer encoder"); + assertFileExists(model.decoder, "paraformer decoder"); + assertFileExists(model.tokens, "tokens"); + } + + const sherpa = loadSherpaOnnx(); + const modelConfig = + model.kind === "transducer" + ? { + transducer: { + encoder: model.encoder, + decoder: model.decoder, + joiner: model.joiner, + }, + tokens: model.tokens, + modelType: model.modelType ?? "zipformer", + } + : { + paraformer: { + encoder: model.encoder, + decoder: model.decoder, + }, + tokens: model.tokens, + }; + + const featConfig = { + sampleRate: config.sampleRate ?? 16000, + featureDim: config.featureDim ?? 80, + }; + + const recognizerConfig = { + featConfig, + modelConfig: { + ...modelConfig, + // NOTE: In the WASM-backed `sherpa-onnx` npm package, online recognizers + // error when `numThreads > 1`. Keep the default conservative. + numThreads: config.numThreads ?? 1, + provider: config.provider ?? "cpu", + debug: config.debug ?? 0, + }, + decodingMethod: config.decodingMethod ?? "greedy_search", + maxActivePaths: config.maxActivePaths ?? 4, + enableEndpoint: config.enableEndpoint ?? 0, + rule1MinTrailingSilence: config.rule1MinTrailingSilence ?? 2.4, + rule2MinTrailingSilence: config.rule2MinTrailingSilence ?? 1.2, + rule3MinUtteranceLength: config.rule3MinUtteranceLength ?? 20, + }; + + this.recognizer = sherpa.createOnlineRecognizer(recognizerConfig); + const sr = this.recognizer?.config?.featConfig?.sampleRate; + this.sampleRate = typeof sr === "number" && Number.isFinite(sr) && sr > 0 ? sr : featConfig.sampleRate; + + this.logger.info( + { sampleRate: this.sampleRate, modelKind: model.kind, numThreads: config.numThreads ?? 2 }, + "Sherpa online recognizer initialized" + ); + } + + createStream(): any { + return this.recognizer.createStream(); + } + + free(): void { + try { + this.recognizer?.free?.(); + } catch (err) { + this.logger.warn({ err }, "Failed to free sherpa recognizer"); + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts b/packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts new file mode 100644 index 000000000..80bf1abe2 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts @@ -0,0 +1,18 @@ +import { createRequire } from "node:module"; + +export type SherpaOnnxModule = { + createOnlineRecognizer: (config: any) => any; + createOfflineRecognizer: (config: any) => any; + createOfflineTts: (config: any) => any; +}; + +let cached: SherpaOnnxModule | null = null; + +export function loadSherpaOnnx(): SherpaOnnxModule { + if (cached) { + return cached; + } + const require = createRequire(import.meta.url); + cached = require("sherpa-onnx") as SherpaOnnxModule; + return cached; +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts b/packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts new file mode 100644 index 000000000..8318b8673 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts @@ -0,0 +1,53 @@ +import { createRequire } from "node:module"; +import path from "node:path"; + +export type SherpaOnnxNodeModule = { + OfflineRecognizer: new (config: any) => any; + OnlineRecognizer?: new (config: any) => any; + OfflineTts?: new (config: any) => any; +}; + +let cached: SherpaOnnxNodeModule | null = null; + +function platformArch(): string { + const platform = process.platform === "win32" ? "win" : process.platform; + return `${platform}-${process.arch}`; +} + +function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void { + const current = process.env[envKey] ?? ""; + const parts = current.split(":").filter(Boolean); + if (parts.includes(dir)) { + return; + } + process.env[envKey] = [dir, ...parts].join(":"); +} + +export function loadSherpaOnnxNode(): SherpaOnnxNodeModule { + if (cached) { + return cached; + } + + const require = createRequire(import.meta.url); + + // sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64) + // that contains the native addon + its shared libraries. On macOS/Linux we need to ensure + // the appropriate library path env var includes that directory before requiring the addon. + const arch = platformArch(); + const pkgName = `sherpa-onnx-${arch}`; + + try { + const pkgJson = require.resolve(`${pkgName}/package.json`); + const pkgDir = path.dirname(pkgJson); + if (process.platform === "darwin") { + prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir); + } else if (process.platform === "linux") { + prependLibraryPath("LD_LIBRARY_PATH", pkgDir); + } + } catch { + // Best effort - if the platform package isn't present, require() below will throw a useful error. + } + + cached = require("sherpa-onnx-node") as SherpaOnnxNodeModule; + return cached; +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts b/packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts new file mode 100644 index 000000000..6a912bd69 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts @@ -0,0 +1,153 @@ +import { EventEmitter } from "node:events"; +import { v4 as uuidv4 } from "uuid"; + +import type { RealtimeTranscriptionSession } from "../../dictation/dictation-stream-manager.js"; +import { pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; + +export class SherpaParakeetRealtimeTranscriptionSession + extends EventEmitter + implements RealtimeTranscriptionSession +{ + private readonly engine: SherpaOfflineRecognizerEngine; + private connected = false; + + private currentItemId: string | null = null; + private previousItemId: string | null = null; + private lastPartialText = ""; + + private pcm16 = Buffer.alloc(0); + private lastDecodeAt = 0; + private decoding = false; + private pendingDecode = false; + private readonly minDecodeIntervalMs: number; + + constructor(params: { engine: SherpaOfflineRecognizerEngine; minDecodeIntervalMs?: number }) { + super(); + this.engine = params.engine; + this.minDecodeIntervalMs = params.minDecodeIntervalMs ?? 350; + } + + async connect(): Promise { + if (this.connected) { + return; + } + this.currentItemId = uuidv4(); + this.connected = true; + } + + appendPcm16Base64(base64Audio: string): void { + if (!this.connected || !this.currentItemId) { + this.emit("error", new Error("Parakeet realtime session not connected")); + return; + } + + try { + const chunk = Buffer.from(base64Audio, "base64"); + this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]); + void this.maybeDecode(false); + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + } + + commit(): void { + if (!this.connected || !this.currentItemId) { + this.emit("error", new Error("Parakeet realtime session not connected")); + return; + } + + void (async () => { + try { + await this.maybeDecode(true); + const finalText = this.lastPartialText; + const itemId = this.currentItemId!; + const previousItemId = this.previousItemId; + + this.emit("committed", { itemId, previousItemId }); + this.emit("transcript", { itemId, transcript: finalText, isFinal: true }); + + this.previousItemId = itemId; + this.currentItemId = uuidv4(); + this.lastPartialText = ""; + this.pcm16 = Buffer.alloc(0); + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + })(); + } + + clear(): void { + if (!this.connected) { + return; + } + this.pcm16 = Buffer.alloc(0); + this.currentItemId = uuidv4(); + this.lastPartialText = ""; + } + + close(): void { + this.connected = false; + this.currentItemId = null; + this.pcm16 = Buffer.alloc(0); + } + + private async maybeDecode(force: boolean): Promise { + if (!this.connected || !this.currentItemId) { + return; + } + + const now = Date.now(); + if (!force && now - this.lastDecodeAt < this.minDecodeIntervalMs) { + return; + } + + if (this.decoding) { + this.pendingDecode = true; + return; + } + + this.decoding = true; + try { + const text = await this.decodeNow(); + this.lastDecodeAt = Date.now(); + if (text !== this.lastPartialText) { + this.lastPartialText = text; + this.emit("transcript", { itemId: this.currentItemId, transcript: text, isFinal: false }); + } + } finally { + this.decoding = false; + if (this.pendingDecode) { + this.pendingDecode = false; + await this.maybeDecode(true); + } + } + } + + private async decodeNow(): Promise { + if (this.pcm16.length === 0) { + return ""; + } + + const peak = pcm16lePeakAbs(this.pcm16); + const peakFloat = peak / 32768.0; + const targetPeak = 0.6; + const maxGain = 50; + const gain = peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1; + + const stream = this.engine.createStream(); + try { + const floatSamples = pcm16leToFloat32(this.pcm16, gain); + this.engine.acceptWaveform(stream, this.engine.sampleRate, floatSamples); + this.engine.recognizer.decode(stream); + const result = this.engine.recognizer.getResult(stream); + return String(result?.text ?? result ?? "").trim(); + } finally { + try { + stream.free?.(); + } catch { + // ignore + } + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts b/packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts new file mode 100644 index 000000000..d3b0c151f --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts @@ -0,0 +1,80 @@ +import type pino from "pino"; + +import type { SpeechToTextProvider, TranscriptionResult } from "../speech-provider.js"; +import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; +import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; + +export type SherpaParakeetSttConfig = { + engine: SherpaOfflineRecognizerEngine; + silencePeakThreshold?: number; +}; + +export class SherpaOnnxParakeetSTT implements SpeechToTextProvider { + private readonly engine: SherpaOfflineRecognizerEngine; + private readonly silencePeakThreshold: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) { + this.engine = config.engine; + this.silencePeakThreshold = config.silencePeakThreshold ?? 300; + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" }); + } + + async transcribeAudio(audioBuffer: Buffer, format: string): Promise { + const start = Date.now(); + + let inputRate: number; + let pcm16: Buffer; + + if (format.toLowerCase().includes("audio/wav")) { + const parsed = parsePcm16MonoWav(audioBuffer); + inputRate = parsed.sampleRate; + pcm16 = parsed.pcm16; + } else if (format.toLowerCase().includes("audio/pcm")) { + inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate; + pcm16 = audioBuffer; + } else { + throw new Error(`Unsupported audio format for sherpa Parakeet STT: ${format}`); + } + + const peak = pcm16lePeakAbs(pcm16); + if (peak < this.silencePeakThreshold) { + return { text: "", duration: Date.now() - start, isLowConfidence: true }; + } + + let pcmForModel = pcm16; + if (inputRate !== this.engine.sampleRate) { + const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate }); + pcmForModel = resampler.processChunk(pcm16); + inputRate = this.engine.sampleRate; + } + + const peakForModel = pcm16lePeakAbs(pcmForModel); + const peakFloat = peakForModel / 32768.0; + const targetPeak = 0.6; + const maxGain = 50; + const gain = + peakFloat > 0 && peakFloat < targetPeak + ? Math.min(maxGain, targetPeak / peakFloat) + : 1; + + const stream = this.engine.createStream(); + try { + const floatSamples = pcm16leToFloat32(pcmForModel, gain); + this.engine.acceptWaveform(stream, inputRate, floatSamples); + this.engine.recognizer.decode(stream); + const result = this.engine.recognizer.getResult(stream); + const text = String(result?.text ?? result ?? "").trim(); + const duration = Date.now() - start; + this.logger.debug({ duration, textLength: text.length }, "Parakeet transcription complete"); + return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) }; + } finally { + try { + stream.free?.(); + } catch { + // ignore + } + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts b/packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts new file mode 100644 index 000000000..dec4fc297 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts @@ -0,0 +1,127 @@ +import { EventEmitter } from "node:events"; +import { v4 as uuidv4 } from "uuid"; + +import type { RealtimeTranscriptionSession } from "../../dictation/dictation-stream-manager.js"; +import { pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; + +export class SherpaRealtimeTranscriptionSession + extends EventEmitter + implements RealtimeTranscriptionSession +{ + private readonly engine: SherpaOnlineRecognizerEngine; + private stream: any | null = null; + private connected = false; + + private currentItemId: string | null = null; + private previousItemId: string | null = null; + private lastPartialText = ""; + private readonly tailPaddingMs: number; + + constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) { + super(); + this.engine = params.engine; + this.tailPaddingMs = params.tailPaddingMs ?? 500; + } + + async connect(): Promise { + if (this.connected) { + return; + } + this.stream = this.engine.createStream(); + this.currentItemId = uuidv4(); + this.connected = true; + } + + appendPcm16Base64(base64Audio: string): void { + if (!this.connected || !this.stream || !this.currentItemId) { + this.emit("error", new Error("Sherpa realtime session not connected")); + return; + } + + try { + const pcm16 = Buffer.from(base64Audio, "base64"); + const peak = pcm16lePeakAbs(pcm16); + const peakFloat = peak / 32768.0; + const targetPeak = 0.6; + const maxGain = 50; + const gain = + peakFloat > 0 && peakFloat < targetPeak + ? Math.min(maxGain, targetPeak / peakFloat) + : 1; + const floatSamples = pcm16leToFloat32(pcm16, gain); + this.stream.acceptWaveform(this.engine.sampleRate, floatSamples); + + while (this.engine.recognizer.isReady(this.stream)) { + this.engine.recognizer.decode(this.stream); + } + + const text = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim(); + if (text !== this.lastPartialText) { + this.lastPartialText = text; + this.emit("transcript", { itemId: this.currentItemId, transcript: text, isFinal: false }); + } + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + } + + commit(): void { + if (!this.connected || !this.stream || !this.currentItemId) { + this.emit("error", new Error("Sherpa realtime session not connected")); + return; + } + + try { + const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000)); + if (padSamples > 0) { + this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples)); + } + + while (this.engine.recognizer.isReady(this.stream)) { + this.engine.recognizer.decode(this.stream); + } + + const finalText = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim(); + const itemId = this.currentItemId; + const previousItemId = this.previousItemId; + + this.emit("committed", { itemId, previousItemId }); + this.emit("transcript", { itemId, transcript: finalText, isFinal: true }); + + this.previousItemId = itemId; + this.currentItemId = uuidv4(); + this.lastPartialText = ""; + this.engine.recognizer.reset(this.stream); + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + } + + clear(): void { + if (!this.connected || !this.stream) { + return; + } + try { + this.engine.recognizer.reset(this.stream); + this.currentItemId = uuidv4(); + this.lastPartialText = ""; + } catch (err) { + this.emit("error", err instanceof Error ? err : new Error(String(err))); + } + } + + close(): void { + if (!this.stream) { + return; + } + try { + this.stream.free?.(); + } catch { + // ignore + } finally { + this.stream = null; + this.connected = false; + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-stt.ts b/packages/server/src/server/speech/sherpa/sherpa-stt.ts new file mode 100644 index 000000000..863b4270d --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-stt.ts @@ -0,0 +1,95 @@ +import type pino from "pino"; + +import type { SpeechToTextProvider, TranscriptionResult } from "../speech-provider.js"; +import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; +import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; + +export type SherpaSttConfig = { + engine: SherpaOnlineRecognizerEngine; + silencePeakThreshold?: number; + tailPaddingMs?: number; +}; + +export class SherpaOnnxSTT implements SpeechToTextProvider { + private readonly engine: SherpaOnlineRecognizerEngine; + private readonly silencePeakThreshold: number; + private readonly tailPaddingMs: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaSttConfig, logger: pino.Logger) { + this.engine = config.engine; + this.silencePeakThreshold = config.silencePeakThreshold ?? 300; + this.tailPaddingMs = config.tailPaddingMs ?? 500; + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" }); + } + + async transcribeAudio(audioBuffer: Buffer, format: string): Promise { + const start = Date.now(); + + let inputRate: number; + let pcm16: Buffer; + + if (format.toLowerCase().includes("audio/wav")) { + const parsed = parsePcm16MonoWav(audioBuffer); + inputRate = parsed.sampleRate; + pcm16 = parsed.pcm16; + } else if (format.toLowerCase().includes("audio/pcm")) { + inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate; + pcm16 = audioBuffer; + } else { + throw new Error(`Unsupported audio format for sherpa STT: ${format}`); + } + + const peak = pcm16lePeakAbs(pcm16); + if (peak < this.silencePeakThreshold) { + return { text: "", duration: Date.now() - start, isLowConfidence: true }; + } + + let pcmForModel = pcm16; + if (inputRate !== this.engine.sampleRate) { + const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate }); + pcmForModel = resampler.processChunk(pcm16); + inputRate = this.engine.sampleRate; + } + + const peakForModel = pcm16lePeakAbs(pcmForModel); + const peakFloat = peakForModel / 32768.0; + const targetPeak = 0.6; + const maxGain = 50; + const gain = + peakFloat > 0 && peakFloat < targetPeak + ? Math.min(maxGain, targetPeak / peakFloat) + : 1; + + const stream = this.engine.createStream(); + try { + const floatSamples = pcm16leToFloat32(pcmForModel, gain); + stream.acceptWaveform(inputRate, floatSamples); + + while (this.engine.recognizer.isReady(stream)) { + this.engine.recognizer.decode(stream); + } + + const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000)); + if (padSamples > 0) { + stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples)); + } + + while (this.engine.recognizer.isReady(stream)) { + this.engine.recognizer.decode(stream); + } + + const text = String(this.engine.recognizer.getResult(stream)?.text ?? "").trim(); + const duration = Date.now() - start; + this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete"); + return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) }; + } finally { + try { + stream.free?.(); + } catch { + // ignore + } + } + } +} diff --git a/packages/server/src/server/speech/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/sherpa/sherpa-tts.ts new file mode 100644 index 000000000..2b5482248 --- /dev/null +++ b/packages/server/src/server/speech/sherpa/sherpa-tts.ts @@ -0,0 +1,121 @@ +import type pino from "pino"; +import { Readable } from "node:stream"; +import { existsSync } from "node:fs"; + +import type { SpeechStreamResult, TextToSpeechProvider } from "../speech-provider.js"; +import { chunkBuffer, float32ToPcm16le } from "../audio.js"; +import { loadSherpaOnnx } from "./sherpa-onnx-loader.js"; + +export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16"; + +export type SherpaTtsConfig = { + preset: SherpaTtsPreset; + modelDir: string; + speakerId?: number; + speed?: number; + lengthScale?: number; + numThreads?: number; +}; + +function assertFileExists(filePath: string, label: string): void { + if (!existsSync(filePath)) { + throw new Error(`Missing ${label}: ${filePath}`); + } +} + +export class SherpaOnnxTTS implements TextToSpeechProvider { + private readonly tts: any; + private readonly speakerId: number; + private readonly speed: number; + private readonly logger: pino.Logger; + + constructor(config: SherpaTtsConfig, logger: pino.Logger) { + if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") { + throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`); + } + this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" }); + this.speakerId = config.speakerId ?? 0; + this.speed = config.speed ?? 1.0; + + const sherpa = loadSherpaOnnx(); + + const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx"; + const modelPath = `${config.modelDir}/${modelFile}`; + const voicesPath = `${config.modelDir}/voices.bin`; + const tokensPath = `${config.modelDir}/tokens.txt`; + const dataDir = `${config.modelDir}/espeak-ng-data`; + + assertFileExists(modelPath, "TTS model"); + assertFileExists(voicesPath, "TTS voices"); + assertFileExists(tokensPath, "TTS tokens"); + assertFileExists(dataDir, "TTS espeak-ng dataDir"); + + const modelConfigKey = + config.preset === "kokoro-en-v0_19" + ? "offlineTtsKokoroModelConfig" + : "offlineTtsKittenModelConfig"; + + const modelConfig = { + [modelConfigKey]: { + model: modelPath, + voices: voicesPath, + tokens: tokensPath, + dataDir, + lengthScale: config.lengthScale ?? 1.0, + }, + numThreads: config.numThreads ?? 2, + debug: 0, + provider: "cpu", + }; + + const offlineTtsConfig = { + offlineTtsModelConfig: modelConfig, + maxNumSentences: 1, + }; + + this.tts = sherpa.createOfflineTts(offlineTtsConfig); + this.logger.info({ preset: config.preset, modelDir: config.modelDir }, "Sherpa offline TTS initialized"); + } + + async synthesizeSpeech(text: string): Promise { + const trimmed = text.trim(); + if (!trimmed) { + throw new Error("Cannot synthesize empty text"); + } + + const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed }); + const samples: Float32Array | null = + audio && audio.samples instanceof Float32Array + ? audio.samples + : audio && Array.isArray(audio.samples) + ? Float32Array.from(audio.samples as number[]) + : null; + const sampleRate: number = + audio && typeof audio.sampleRate === "number" && Number.isFinite(audio.sampleRate) && audio.sampleRate > 0 + ? audio.sampleRate + : typeof this.tts.sampleRate === "number" + ? this.tts.sampleRate + : 24000; + + if (!samples) { + throw new Error("Unexpected sherpa TTS output: missing Float32 samples"); + } + + const pcm16 = float32ToPcm16le(samples); + const chunkBytes = Math.max(2, Math.round(sampleRate * 0.05) * 2); // ~50ms + const chunks = chunkBuffer(pcm16, chunkBytes); + + return { + stream: Readable.from(chunks), + format: `pcm;rate=${sampleRate}`, + }; + } + + free(): void { + try { + this.tts?.free?.(); + } catch { + // ignore + } + } +} diff --git a/packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts new file mode 100644 index 000000000..da828a29d --- /dev/null +++ b/packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, test } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import pino from "pino"; + +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; +import type { SherpaOnnxModelId } from "./model-catalog.js"; +import { createDaemonTestContext } from "../../test-utils/index.js"; +import { parsePcm16MonoWav, wordSimilarity } from "../../test-utils/dictation-e2e.js"; +import { SherpaOnnxTTS } from "./sherpa-tts.js"; +import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js"; +import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; +import { SherpaOnnxSTT } from "./sherpa-stt.js"; +import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; +import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js"; + +const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1"; +const downloadTest = RUN ? test : test.skip; + +type ModelSet = "zipformer-kitten" | "parakeet-pocket"; + +function getModelSet(): ModelSet { + const raw = (process.env.PASEO_SPEECH_E2E_MODEL_SET ?? "parakeet-pocket").trim().toLowerCase(); + if (raw === "zipformer-kitten" || raw === "zipformer") return "zipformer-kitten"; + if (raw === "parakeet-pocket" || raw === "parakeet") return "parakeet-pocket"; + throw new Error(`Unknown PASEO_SPEECH_E2E_MODEL_SET: ${raw}`); +} + +async function readFixtureWav(): Promise { + const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav"); + return import("node:fs/promises").then((fs) => fs.readFile(fixturePath)); +} + +async function readBaseline(): Promise { + const baselinePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.baseline.txt"); + return import("node:fs/promises").then((fs) => fs.readFile(baselinePath, "utf-8")).then((t) => t.trim()); +} + +async function readAllChunks(stream: NodeJS.ReadableStream): Promise<{ chunks: Buffer[]; combined: Buffer }> { + const chunks: Buffer[] = []; + for await (const chunk of stream as any) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return { chunks, combined: Buffer.concat(chunks) }; +} + +function waitForSignal( + timeoutMs: number, + setup: ( + resolve: (value: T) => void, + reject: (error: Error) => void + ) => () => void +): Promise { + return new Promise((resolve, reject) => { + let cleanup: (() => void) | null = null; + const timeout = setTimeout(() => { + cleanup?.(); + reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`)); + }, timeoutMs); + + cleanup = setup( + (value) => { + clearTimeout(timeout); + cleanup?.(); + resolve(value); + }, + (error) => { + clearTimeout(timeout); + cleanup?.(); + reject(error); + } + ); + }); +} + +function toAudioPcmFormat(format: string): string { + const trimmed = format.trim(); + const lower = trimmed.toLowerCase(); + if (lower.startsWith("audio/pcm")) { + return trimmed; + } + if (lower.startsWith("pcm")) { + const rest = trimmed.replace(/^pcm;?/i, ""); + return rest ? `audio/pcm;${rest}` : "audio/pcm"; + } + // Fall back to treating it as a suffix (e.g. "rate=24000") + return `audio/pcm;${trimmed}`; +} + +describe("speech models (download E2E)", () => { + downloadTest( + "downloads models and runs streaming STT + streaming TTS", + async () => { + const logger = pino({ level: "silent" }); + const set = getModelSet(); + + const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-")); + const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx"); + + const modelIds: SherpaOnnxModelId[] = + set === "parakeet-pocket" + ? ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"] + : ["zipformer-bilingual-zh-en-2023-02-20", "kitten-nano-en-v0_1-fp16"]; + + await ensureSherpaOnnxModels({ + modelsDir, + modelIds, + autoDownload: true, + logger, + }); + + const ctx = await createDaemonTestContext({ + paseoHomeRoot, + dictationFinalTimeoutMs: 8000, + speech: { + dictationSttProvider: "sherpa", + voiceSttProvider: "sherpa", + voiceTtsProvider: "sherpa", + sherpaOnnx: { + modelsDir, + autoDownload: false, + stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" }, + tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" }, + }, + }, + }); + + try { + const wav = await readFixtureWav(); + const { sampleRate, pcm16 } = parsePcm16MonoWav(wav); + expect(sampleRate).toBe(16000); + const format = "audio/pcm;rate=16000;bits=16"; + + // Streaming STT: dictation path (verifies websocket streaming + partials + final) + const dictationId = `dict-download-${Date.now()}`; + let partialCount = 0; + const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => { + if (message.type !== "dictation_stream_partial") return; + if (message.payload.dictationId !== dictationId) return; + partialCount += 1; + }); + + await ctx.client.startDictationStream(dictationId, format); + const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16 + let seq = 0; + for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { + const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); + ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format); + seq += 1; + } + const finalSeq = seq - 1; + const dictationFinal = await ctx.client.finishDictationStream(dictationId, finalSeq); + unsubscribe(); + + expect(dictationFinal.text.toLowerCase()).toContain("voice note"); + const baseline = await readBaseline(); + expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45); + expect(partialCount).toBeGreaterThan(0); + + // Voice-mode STT: chunked upload until isLast=true + const transcriptionPromise = waitForSignal(30000, (resolve, reject) => { + const offResult = ctx.client.on("transcription_result", (message) => { + if (message.type !== "transcription_result") return; + resolve(message.payload.text); + }); + const offError = ctx.client.on("activity_log", (message) => { + if (message.type !== "activity_log") return; + const payload = message.payload as { type?: unknown; content?: unknown }; + if (payload.type !== "error") return; + const content = typeof payload.content === "string" ? payload.content : null; + if (!content) return; + reject(new Error(content)); + }); + return () => { + offResult(); + offError(); + }; + }); + + await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`); + for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { + const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); + const isLast = offset + chunkBytes >= pcm16.length; + await ctx.client.sendVoiceAudioChunk(chunk.toString("base64"), format, isLast); + } + const voiceText = (await transcriptionPromise).trim().toLowerCase(); + if (voiceText.length > 0) { + expect(voiceText).toContain("voice note"); + } + await ctx.client.setVoiceConversation(false); + + // Streaming TTS: generate locally from downloaded model and validate chunking. + const ttsText = "This is a voice note."; + if (set === "parakeet-pocket") { + const modelDir = getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8"); + const tts = await PocketTtsOnnxTTS.create({ modelDir, precision: "int8", targetChunkMs: 50 }, logger); + const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); + const { chunks, combined } = await readAllChunks(stream); + + expect(ttsFormat).toMatch(/rate=\d+/); + expect(chunks.length).toBeGreaterThan(3); + expect(combined.byteLength).toBeGreaterThan(2000); + + // Round trip: TTS -> STT (offline parakeet) + const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8"); + const engine = new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${sttModelDir}/encoder.int8.onnx`, + decoder: `${sttModelDir}/decoder.int8.onnx`, + joiner: `${sttModelDir}/joiner.int8.onnx`, + tokens: `${sttModelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger + ); + const stt = new SherpaOnnxParakeetSTT({ engine }, logger); + const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat)); + engine.free(); + expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25); + } else { + const ttsModelDir = path.join(modelsDir, "kitten-nano-en-v0_1-fp16"); + const tts = new SherpaOnnxTTS({ preset: "kitten-nano-en-v0_1-fp16", modelDir: ttsModelDir }, logger); + const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); + const { chunks, combined } = await readAllChunks(stream); + tts.free(); + + expect(ttsFormat).toMatch(/rate=\d+/); + expect(chunks.length).toBeGreaterThan(3); + expect(combined.byteLength).toBeGreaterThan(2000); + + // Round trip: TTS -> STT (online zipformer, offline segment) + const sttModelDir = path.join(modelsDir, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"); + const engine = new SherpaOnlineRecognizerEngine( + { + model: { + kind: "transducer", + encoder: `${sttModelDir}/encoder-epoch-99-avg-1.onnx`, + decoder: `${sttModelDir}/decoder-epoch-99-avg-1.onnx`, + joiner: `${sttModelDir}/joiner-epoch-99-avg-1.onnx`, + tokens: `${sttModelDir}/tokens.txt`, + modelType: "zipformer", + }, + numThreads: 1, + debug: 0, + }, + logger + ); + const stt = new SherpaOnnxSTT({ engine }, logger); + const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat)); + engine.free(); + expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25); + } + } finally { + await ctx.cleanup(); + } + }, + 15 * 60_000 + ); +}); diff --git a/packages/server/src/server/speech/speech-provider.ts b/packages/server/src/server/speech/speech-provider.ts new file mode 100644 index 000000000..3ed103666 --- /dev/null +++ b/packages/server/src/server/speech/speech-provider.ts @@ -0,0 +1,30 @@ +import type { Readable } from "node:stream"; + +export interface LogprobToken { + token: string; + logprob: number; + bytes?: number[]; +} + +export interface TranscriptionResult { + text: string; + language?: string; + duration?: number; + logprobs?: LogprobToken[]; + avgLogprob?: number; + isLowConfidence?: boolean; +} + +export interface SpeechToTextProvider { + transcribeAudio(audioBuffer: Buffer, format: string): Promise; +} + +export interface SpeechStreamResult { + stream: Readable; + format: string; +} + +export interface TextToSpeechProvider { + synthesizeSpeech(text: string): Promise; +} + diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index 89028cde6..7024991b8 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { mkdir, mkdtemp, rm } from "node:fs/promises"; import pino from "pino"; -import { createPaseoDaemon, type PaseoDaemonConfig, type PaseoOpenAIConfig } from "../bootstrap.js"; +import { createPaseoDaemon, type PaseoDaemonConfig, type PaseoOpenAIConfig, type PaseoSpeechConfig } from "../bootstrap.js"; import type { AgentClient, AgentProvider } from "../agent/agent-sdk-types.js"; import { createTestAgentClients } from "./fake-agent-client.js"; @@ -20,6 +20,7 @@ type TestPaseoDaemonOptions = { staticDir?: string; cleanup?: boolean; openai?: PaseoOpenAIConfig; + speech?: PaseoSpeechConfig; dictationFinalTimeoutMs?: number; }; @@ -76,6 +77,7 @@ export async function createTestPaseoDaemon( relayEndpoint: options.relayEndpoint ?? "relay.paseo.sh:443", appBaseUrl: "https://app.paseo.sh", openai: options.openai, + speech: options.speech, openrouterApiKey: null, dictationFinalTimeoutMs: options.dictationFinalTimeoutMs, downloadTokenTtlMs: options.downloadTokenTtlMs, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index f3691c1fd..b7e84f4bf 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -6,8 +6,6 @@ import { hostname as getHostname } from "node:os"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; -import type { OpenAISTT } from "./agent/stt-openai.js"; -import type { OpenAITTS } from "./agent/tts-openai.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import type pino from "pino"; import { @@ -22,6 +20,8 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; import { VoiceConversationStore } from "./voice-conversation-store.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; +import type { RealtimeTranscriptionSessionFactory } from "./dictation/dictation-stream-manager.js"; export type AgentMcpTransportFactory = () => Promise; @@ -67,13 +67,14 @@ export class VoiceAssistantWebSocketServer { private readonly pushTokenStore: PushTokenStore; private readonly pushService: PushService; private readonly createAgentMcpTransport: AgentMcpTransportFactory; - private readonly stt: OpenAISTT | null; - private readonly tts: OpenAITTS | null; + private readonly stt: SpeechToTextProvider | null; + private readonly tts: TextToSpeechProvider | null; private readonly terminalManager: TerminalManager | null; private readonly voiceConversationStore: VoiceConversationStore; private readonly dictation: { openaiApiKey?: string | null; finalTimeoutMs?: number; + sessionFactory?: RealtimeTranscriptionSessionFactory; } | null; private readonly voice: { openrouterApiKey?: string | null; @@ -90,7 +91,7 @@ export class VoiceAssistantWebSocketServer { paseoHome: string, createAgentMcpTransport: AgentMcpTransportFactory, wsConfig: WebSocketServerConfig, - speech?: { stt: OpenAISTT | null; tts: OpenAITTS | null }, + speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null }, terminalManager?: TerminalManager | null, voice?: { openrouterApiKey?: string | null; @@ -99,6 +100,7 @@ export class VoiceAssistantWebSocketServer { dictation?: { openaiApiKey?: string | null; finalTimeoutMs?: number; + sessionFactory?: RealtimeTranscriptionSessionFactory; } ) { this.logger = logger.child({ module: "websocket-server" }); diff --git a/scripts/speech/download-sherpa-models.sh b/scripts/speech/download-sherpa-models.sh new file mode 100755 index 000000000..5a3ccdc5c --- /dev/null +++ b/scripts/speech/download-sherpa-models.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Download local speech models for Paseo (sherpa-onnx). + +Defaults: + - STT: sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + - TTS: kitten-nano-en-v0_1-fp16 + +Usage: + scripts/speech/download-sherpa-models.sh [--models-dir DIR] [--with-kokoro] [--with-paraformer] + +Preferred: + npm run speech:download --workspace=@getpaseo/server + +Notes: + - Models are downloaded from the sherpa-onnx GitHub releases. + - Pocket TTS is downloaded by the Node script (`npm run speech:download --workspace=@getpaseo/server`) + because it is a file-based HuggingFace model (not a single tarball). + - Set PASEO_SHERPA_ONNX_MODELS_DIR to override where the daemon looks. +EOF +} + +MODELS_DIR="" +WITH_KOKORO=0 +WITH_PARAFORMER=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --models-dir) + MODELS_DIR="${2:-}" + shift 2 + ;; + --with-kokoro) + WITH_KOKORO=1 + shift 1 + ;; + --with-paraformer) + WITH_PARAFORMER=1 + shift 1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown arg: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "${MODELS_DIR}" ]]; then + if [[ -n "${PASEO_SHERPA_ONNX_MODELS_DIR:-}" ]]; then + MODELS_DIR="${PASEO_SHERPA_ONNX_MODELS_DIR}" + elif [[ -n "${PASEO_HOME:-}" ]]; then + MODELS_DIR="${PASEO_HOME}/models/sherpa-onnx" + else + MODELS_DIR="${HOME}/.paseo/models/sherpa-onnx" + fi +fi + +mkdir -p "${MODELS_DIR}" +cd "${MODELS_DIR}" + +download_and_extract() { + local url="$1" + local filename + filename="$(basename "$url")" + + echo "Downloading ${filename}..." + curl -fsSL -O "${url}" + echo "Extracting ${filename}..." + tar xf "${filename}" + rm -f "${filename}" +} + +echo "NOTE: This script is deprecated. Prefer: npm run speech:download --workspace=@getpaseo/server" >&2 + +download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2" +download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2" + +if [[ "${WITH_PARAFORMER}" -eq 1 ]]; then + download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2" +fi + +if [[ "${WITH_KOKORO}" -eq 1 ]]; then + download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2" +fi + +echo "Done." +echo "Models dir: ${MODELS_DIR}" From 277080a004c00d68f0ceb7b50c4e58c1b1e50c8d Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 12:43:10 +0700 Subject: [PATCH 2/4] Refactor speech providers to stream-first local/openai architecture --- packages/cli/src/cli.ts | 4 + packages/cli/src/commands/speech/download.ts | 68 ++++ packages/cli/src/commands/speech/index.ts | 29 ++ packages/cli/src/commands/speech/models.ts | 72 ++++ .../server/scripts/download-speech-models.ts | 4 +- packages/server/scripts/list-speech-models.ts | 3 +- packages/server/src/client/daemon-client.ts | 53 +++ .../src/server/agent/stt-manager.test.ts | 56 ++- .../server/src/server/agent/stt-manager.ts | 122 +++++-- .../server/src/server/agent/stt-openai.ts | 139 ------- packages/server/src/server/bootstrap.ts | 103 ++++-- packages/server/src/server/config.ts | 20 +- .../src/server/daemon-client.e2e.test.ts | 6 +- .../dictation-stream-manager.test.ts | 86 ++--- .../dictation/dictation-stream-manager.ts | 338 ++++-------------- .../server/src/server/persisted-config.ts | 20 +- packages/server/src/server/session.ts | 131 ++++++- .../local}/pocket/pocket-tts-onnx.ts | 6 +- .../local}/sherpa/model-catalog.ts | 0 .../local}/sherpa/model-downloader.test.ts | 0 .../local}/sherpa/model-downloader.ts | 0 .../sherpa/sherpa-offline-recognizer.ts | 0 .../local}/sherpa/sherpa-online-recognizer.ts | 0 .../local}/sherpa/sherpa-onnx-loader.ts | 0 .../local}/sherpa/sherpa-onnx-node-loader.ts | 0 .../sherpa-parakeet-realtime-session.ts | 43 +-- .../local}/sherpa/sherpa-parakeet-stt.ts | 87 ++++- .../local}/sherpa/sherpa-realtime-session.ts | 45 +-- .../local}/sherpa/sherpa-stt.ts | 86 ++++- .../local}/sherpa/sherpa-tts.ts | 4 +- .../local}/sherpa/speech-download.e2e.test.ts | 10 +- .../openai/realtime-transcription-session.ts} | 18 +- .../src/server/speech/providers/openai/stt.ts | 269 ++++++++++++++ .../providers/openai/tts.ts} | 4 +- .../src/server/speech/speech-provider.ts | 48 ++- .../server/src/server/websocket-server.ts | 7 +- packages/server/src/shared/messages.ts | 49 +++ scripts/speech/download-sherpa-models.sh | 95 ----- 38 files changed, 1294 insertions(+), 731 deletions(-) create mode 100644 packages/cli/src/commands/speech/download.ts create mode 100644 packages/cli/src/commands/speech/index.ts create mode 100644 packages/cli/src/commands/speech/models.ts delete mode 100644 packages/server/src/server/agent/stt-openai.ts rename packages/server/src/server/speech/{ => providers/local}/pocket/pocket-tts-onnx.ts (99%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/model-catalog.ts (100%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/model-downloader.test.ts (100%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/model-downloader.ts (100%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-offline-recognizer.ts (100%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-online-recognizer.ts (100%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-onnx-loader.ts (100%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-onnx-node-loader.ts (100%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-parakeet-realtime-session.ts (73%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-parakeet-stt.ts (50%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-realtime-session.ts (70%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-stt.ts (55%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/sherpa-tts.ts (97%) rename packages/server/src/server/speech/{ => providers/local}/sherpa/speech-download.e2e.test.ts (97%) rename packages/server/src/server/{agent/openai-realtime-transcription.ts => speech/providers/openai/realtime-transcription-session.ts} (91%) create mode 100644 packages/server/src/server/speech/providers/openai/stt.ts rename packages/server/src/server/{agent/tts-openai.ts => speech/providers/openai/tts.ts} (93%) delete mode 100755 scripts/speech/download-sherpa-models.sh diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index cee6d72f6..b3222c7c4 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -3,6 +3,7 @@ import { createAgentCommand } from './commands/agent/index.js' import { createDaemonCommand } from './commands/daemon/index.js' import { createPermitCommand } from './commands/permit/index.js' import { createProviderCommand } from './commands/provider/index.js' +import { createSpeechCommand } from './commands/speech/index.js' import { createWorktreeCommand } from './commands/worktree/index.js' import { runLsCommand } from './commands/agent/ls.js' import { runRunCommand } from './commands/agent/run.js' @@ -134,6 +135,9 @@ export function createCli(): Command { // Provider commands program.addCommand(createProviderCommand()) + // Speech model commands + program.addCommand(createSpeechCommand()) + // Worktree commands program.addCommand(createWorktreeCommand()) diff --git a/packages/cli/src/commands/speech/download.ts b/packages/cli/src/commands/speech/download.ts new file mode 100644 index 000000000..c37b84b18 --- /dev/null +++ b/packages/cli/src/commands/speech/download.ts @@ -0,0 +1,68 @@ +import type { Command } from "commander"; +import type { + CommandError, + CommandOptions, + ListResult, + OutputSchema, +} from "../../output/index.js"; +import { connectToDaemon } from "../../utils/client.js"; + +interface SpeechDownloadRow { + modelId: string; + status: "downloaded"; +} + +const speechDownloadSchema: OutputSchema = { + idField: "modelId", + columns: [ + { header: "MODEL", field: "modelId", width: 36 }, + { header: "STATUS", field: "status", width: 12, color: () => "green" }, + ], +}; + +export type SpeechDownloadResult = ListResult; + +export interface SpeechDownloadOptions extends CommandOptions { + host?: string; + model?: string[]; +} + +export async function runSpeechDownloadCommand( + options: SpeechDownloadOptions, + _command: Command +): Promise { + const client = await connectToDaemon({ host: options.host }); + try { + const response = await client.downloadSpeechModels({ + modelIds: options.model && options.model.length > 0 ? options.model : undefined, + }); + if (response.error) { + const commandError: CommandError = { + code: "SPEECH_MODELS_DOWNLOAD_FAILED", + message: response.error, + }; + throw commandError; + } + + return { + type: "list", + data: response.downloadedModelIds.map((modelId) => ({ + modelId, + status: "downloaded" as const, + })), + schema: speechDownloadSchema, + }; + } catch (error) { + if (typeof error === "object" && error && "code" in error && "message" in error) { + throw error; + } + const message = error instanceof Error ? error.message : String(error); + const commandError: CommandError = { + code: "SPEECH_MODELS_DOWNLOAD_FAILED", + message: `Failed to download speech models: ${message}`, + }; + throw commandError; + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/cli/src/commands/speech/index.ts b/packages/cli/src/commands/speech/index.ts new file mode 100644 index 000000000..6661313a5 --- /dev/null +++ b/packages/cli/src/commands/speech/index.ts @@ -0,0 +1,29 @@ +import { Command } from "commander"; +import { withOutput } from "../../output/index.js"; +import { runSpeechModelsCommand } from "./models.js"; +import { runSpeechDownloadCommand } from "./download.js"; + +function collectMultiple(value: string, previous: string[]): string[] { + return previous.concat([value]); +} + +export function createSpeechCommand(): Command { + const speech = new Command("speech").description("Manage local speech models"); + + speech + .command("models") + .description("List local speech model download status") + .option("--json", "Output in JSON format") + .option("--host ", "Daemon host:port (default: localhost:6767)") + .action(withOutput(runSpeechModelsCommand)); + + speech + .command("download") + .description("Download local speech models") + .option("--model ", "Model ID to download (repeatable)", collectMultiple, []) + .option("--json", "Output in JSON format") + .option("--host ", "Daemon host:port (default: localhost:6767)") + .action(withOutput(runSpeechDownloadCommand)); + + return speech; +} diff --git a/packages/cli/src/commands/speech/models.ts b/packages/cli/src/commands/speech/models.ts new file mode 100644 index 000000000..eca33edd6 --- /dev/null +++ b/packages/cli/src/commands/speech/models.ts @@ -0,0 +1,72 @@ +import type { Command } from "commander"; +import type { + CommandError, + CommandOptions, + ListResult, + OutputSchema, +} from "../../output/index.js"; +import { connectToDaemon } from "../../utils/client.js"; + +interface SpeechModelListItem { + id: string; + kind: string; + status: "downloaded" | "missing"; + modelDir: string; + missingFiles: string; +} + +const speechModelsSchema: OutputSchema = { + idField: "id", + columns: [ + { header: "MODEL", field: "id", width: 36 }, + { header: "KIND", field: "kind", width: 12 }, + { + header: "STATUS", + field: "status", + width: 12, + color: (value) => (value === "downloaded" ? "green" : "yellow"), + }, + { header: "MODEL DIR", field: "modelDir", width: 44 }, + { header: "MISSING FILES", field: "missingFiles", width: 40 }, + ], +}; + +export type SpeechModelsResult = ListResult; + +export interface SpeechModelsOptions extends CommandOptions { + host?: string; +} + +export async function runSpeechModelsCommand( + options: SpeechModelsOptions, + _command: Command +): Promise { + const client = await connectToDaemon({ host: options.host }); + try { + const response = await client.listSpeechModels(); + const rows: SpeechModelListItem[] = response.models + .slice() + .sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id)) + .map((model) => ({ + id: model.id, + kind: model.kind, + status: model.isDownloaded ? "downloaded" : "missing", + modelDir: model.modelDir, + missingFiles: model.missingFiles?.join(", ") ?? "", + })); + return { + type: "list", + data: rows, + schema: speechModelsSchema, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const commandError: CommandError = { + code: "SPEECH_MODELS_LIST_FAILED", + message: `Failed to list speech models: ${message}`, + }; + throw commandError; + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/server/scripts/download-speech-models.ts b/packages/server/scripts/download-speech-models.ts index 978cc3e85..a531c5942 100644 --- a/packages/server/scripts/download-speech-models.ts +++ b/packages/server/scripts/download-speech-models.ts @@ -1,7 +1,7 @@ import { resolvePaseoHome } from "../src/server/paseo-home.js"; import { createRootLogger } from "../src/server/logger.js"; -import { ensureSherpaOnnxModels } from "../src/server/speech/sherpa/model-downloader.js"; -import type { SherpaOnnxModelId } from "../src/server/speech/sherpa/model-catalog.js"; +import { ensureSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-downloader.js"; +import type { SherpaOnnxModelId } from "../src/server/speech/providers/local/sherpa/model-catalog.js"; function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } { const home = resolvePaseoHome(); diff --git a/packages/server/scripts/list-speech-models.ts b/packages/server/scripts/list-speech-models.ts index d907ab4eb..8462dfc27 100644 --- a/packages/server/scripts/list-speech-models.ts +++ b/packages/server/scripts/list-speech-models.ts @@ -1,4 +1,4 @@ -import { listSherpaOnnxModels } from "../src/server/speech/sherpa/model-catalog.js"; +import { listSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-catalog.js"; const models = listSherpaOnnxModels() .slice() @@ -8,4 +8,3 @@ for (const m of models) { // eslint-disable-next-line no-console console.log(`${m.kind}\t${m.id}\t${m.description}`); } - diff --git a/packages/server/src/client/daemon-client.ts b/packages/server/src/client/daemon-client.ts index 9ced8cff3..8994e9407 100644 --- a/packages/server/src/client/daemon-client.ts +++ b/packages/server/src/client/daemon-client.ts @@ -36,6 +36,8 @@ import type { ExecuteCommandResponse, ListVoiceConversationsResponseMessage, ListProviderModelsResponseMessage, + SpeechModelsListResponse, + SpeechModelsDownloadResponse, ListTerminalsResponse, CreateTerminalResponse, SubscribeTerminalResponse, @@ -201,6 +203,8 @@ type PaseoWorktreeArchivePayload = PaseoWorktreeArchiveResponse["payload"]; type FileExplorerPayload = FileExplorerResponse["payload"]; type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"]; type ListProviderModelsPayload = ListProviderModelsResponseMessage["payload"]; +type SpeechModelsListPayload = SpeechModelsListResponse["payload"]; +type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"]; type ListCommandsPayload = ListCommandsResponse["payload"]; type ExecuteCommandPayload = ExecuteCommandResponse["payload"]; type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"]; @@ -2014,6 +2018,55 @@ export class DaemonClient { }); } + async listSpeechModels(requestId?: string): Promise { + const resolvedRequestId = this.createRequestId(requestId); + const message = SessionInboundMessageSchema.parse({ + type: "speech_models_list_request", + requestId: resolvedRequestId, + }); + return this.sendRequest({ + requestId: resolvedRequestId, + message, + timeout: 30000, + options: { skipQueue: true }, + select: (msg) => { + if (msg.type !== "speech_models_list_response") { + return null; + } + if (msg.payload.requestId !== resolvedRequestId) { + return null; + } + return msg.payload; + }, + }); + } + + async downloadSpeechModels( + options?: { modelIds?: string[]; requestId?: string } + ): Promise { + const resolvedRequestId = this.createRequestId(options?.requestId); + const message = SessionInboundMessageSchema.parse({ + type: "speech_models_download_request", + modelIds: options?.modelIds, + requestId: resolvedRequestId, + }); + return this.sendRequest({ + requestId: resolvedRequestId, + message, + timeout: 30 * 60 * 1000, + options: { skipQueue: true }, + select: (msg) => { + if (msg.type !== "speech_models_download_response") { + return null; + } + if (msg.payload.requestId !== resolvedRequestId) { + return null; + } + return msg.payload; + }, + }); + } + async listCommands( agentId: string, requestId?: string diff --git a/packages/server/src/server/agent/stt-manager.test.ts b/packages/server/src/server/agent/stt-manager.test.ts index df88f3bea..8b8f91a68 100644 --- a/packages/server/src/server/agent/stt-manager.test.ts +++ b/packages/server/src/server/agent/stt-manager.test.ts @@ -1,14 +1,53 @@ import { describe, expect, it } from "vitest"; import pino from "pino"; +import { EventEmitter } from "node:events"; import { STTManager } from "./stt-manager.js"; -import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js"; +import type { + SpeechToTextProvider, + StreamingTranscriptionSession, + TranscriptionResult, +} from "../speech/speech-provider.js"; class FakeStt implements SpeechToTextProvider { + public readonly id = "fake"; constructor(private readonly result: TranscriptionResult) {} - async transcribeAudio(): Promise { - return this.result; + createSession(_params: { + logger: any; + language?: string; + prompt?: string; + }): StreamingTranscriptionSession { + const emitter = new EventEmitter(); + const result = this.result; + let segmentId = "seg-1"; + let previousSegmentId: string | null = null; + + return { + requiredSampleRate: 24000, + async connect() {}, + appendPcm16() {}, + commit() { + (emitter as any).emit("committed", { segmentId, previousSegmentId }); + (emitter as any).emit("transcript", { + segmentId, + transcript: result.text, + isFinal: true, + language: result.language, + logprobs: result.logprobs, + avgLogprob: result.avgLogprob, + isLowConfidence: result.isLowConfidence, + }); + previousSegmentId = segmentId; + segmentId = "seg-2"; + }, + clear() {}, + close() {}, + on(event: any, handler: any) { + emitter.on(event, handler); + return undefined; + }, + }; } } @@ -20,10 +59,12 @@ describe("STTManager", () => { new FakeStt({ text: "um", isLowConfidence: true, avgLogprob: -10 }) ); - const result = await manager.transcribe(Buffer.from("x"), "audio/wav", { label: "t" }); + const result = await manager.transcribe(Buffer.alloc(2), "audio/pcm;rate=24000", { + label: "t", + }); expect(result.text).toBe(""); expect(result.isLowConfidence).toBe(true); - expect(result.byteLength).toBe(1); + expect(result.byteLength).toBe(2); }); it("passes through normal transcriptions", async () => { @@ -33,10 +74,9 @@ describe("STTManager", () => { new FakeStt({ text: "hello world", language: "en", isLowConfidence: false }) ); - const result = await manager.transcribe(Buffer.from("abc"), "audio/wav"); + const result = await manager.transcribe(Buffer.alloc(4), "audio/pcm;rate=24000"); expect(result.text).toBe("hello world"); expect(result.language).toBe("en"); - expect(result.byteLength).toBe(3); + expect(result.byteLength).toBe(4); }); }); - diff --git a/packages/server/src/server/agent/stt-manager.ts b/packages/server/src/server/agent/stt-manager.ts index cb454dc3b..6cdd35ea8 100644 --- a/packages/server/src/server/agent/stt-manager.ts +++ b/packages/server/src/server/agent/stt-manager.ts @@ -1,6 +1,8 @@ import type pino from "pino"; import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js"; import { maybePersistDebugAudio } from "./stt-debug.js"; +import { parsePcm16MonoWav, parsePcmRateFromFormat } from "../speech/audio.js"; +import { Pcm16MonoResampler } from "./pcm16-resampler.js"; interface TranscriptionMetadata { agentId?: string; @@ -63,36 +65,104 @@ export class STTManager { this.logger.warn({ err: error }, "Failed to persist debug audio"); } - const result = await this.stt.transcribeAudio(audio, format); + const session = this.stt.createSession({ + logger: this.logger.child({ component: "stt-session" }), + language: "en", + }); - // Filter out low-confidence transcriptions (non-speech sounds) - if (result.isLowConfidence) { - this.logger.debug( - { text: result.text, avgLogprob: result.avgLogprob }, - "Filtered low-confidence transcription (likely non-speech)" - ); - - // Return empty text to ignore this transcription - return { - ...result, - text: "", - byteLength: audio.length, - format, - debugRecordingPath: debugRecordingPath ?? undefined, - }; + let inputRate: number; + let pcm16: Buffer; + if (format.toLowerCase().includes("audio/wav")) { + const parsed = parsePcm16MonoWav(audio); + inputRate = parsed.sampleRate; + pcm16 = parsed.pcm16; + } else if (format.toLowerCase().includes("audio/pcm")) { + inputRate = + parsePcmRateFromFormat(format, session.requiredSampleRate) ?? + session.requiredSampleRate; + pcm16 = audio; + } else { + throw new Error(`Unsupported audio format for STT: ${format}`); } - this.logger.debug( - { text: result.text, avgLogprob: result.avgLogprob }, - "Transcription complete" - ); + let pcmForModel = pcm16; + if (inputRate !== session.requiredSampleRate) { + const resampler = new Pcm16MonoResampler({ + inputRate, + outputRate: session.requiredSampleRate, + }); + pcmForModel = resampler.processChunk(pcm16); + inputRate = session.requiredSampleRate; + } - return { - ...result, - debugRecordingPath: debugRecordingPath ?? undefined, - byteLength: audio.length, - format, - }; + try { + const startedAt = Date.now(); + const finalEventPromise = new Promise<{ + transcript: string; + language?: string; + logprobs?: TranscriptionResult["logprobs"]; + avgLogprob?: number; + isLowConfidence?: boolean; + }>((resolve, reject) => { + session.on("error", reject); + session.on("transcript", (payload) => { + if (!payload.isFinal) { + return; + } + resolve({ + transcript: payload.transcript, + language: payload.language, + logprobs: payload.logprobs, + avgLogprob: payload.avgLogprob, + isLowConfidence: payload.isLowConfidence, + }); + }); + }); + + await session.connect(); + session.appendPcm16(pcmForModel); + session.commit(); + const finalEvent = await finalEventPromise; + const result: TranscriptionResult = { + text: finalEvent.transcript, + language: finalEvent.language, + logprobs: finalEvent.logprobs, + avgLogprob: finalEvent.avgLogprob, + isLowConfidence: finalEvent.isLowConfidence, + duration: Date.now() - startedAt, + }; + + // Filter out low-confidence transcriptions (non-speech sounds) + if (result.isLowConfidence) { + this.logger.debug( + { text: result.text, avgLogprob: result.avgLogprob }, + "Filtered low-confidence transcription (likely non-speech)" + ); + + // Return empty text to ignore this transcription + return { + ...result, + text: "", + byteLength: audio.length, + format, + debugRecordingPath: debugRecordingPath ?? undefined, + }; + } + + this.logger.debug( + { text: result.text, avgLogprob: result.avgLogprob }, + "Transcription complete" + ); + + return { + ...result, + debugRecordingPath: debugRecordingPath ?? undefined, + byteLength: audio.length, + format, + }; + } finally { + session.close(); + } } /** diff --git a/packages/server/src/server/agent/stt-openai.ts b/packages/server/src/server/agent/stt-openai.ts deleted file mode 100644 index 4326f6c68..000000000 --- a/packages/server/src/server/agent/stt-openai.ts +++ /dev/null @@ -1,139 +0,0 @@ -import type pino from "pino"; -import OpenAI from "openai"; -import { writeFile, unlink } from "fs/promises"; -import { join } from "path"; -import { tmpdir } from "os"; -import { v4 } from "uuid"; -import { inferAudioExtension } from "./audio-utils.js"; -import type { LogprobToken, TranscriptionResult } from "../speech/speech-provider.js"; - -export type { LogprobToken, TranscriptionResult }; - -export interface STTConfig { - apiKey: string; - model?: "whisper-1" | "gpt-4o-transcribe" | "gpt-4o-mini-transcribe" | (string & {}); - confidenceThreshold?: number; // Default: -3.0 -} - -function isObject(value: unknown): value is { [key: string]: unknown } { - return typeof value === "object" && value !== null; -} - -function isLogprobToken(value: unknown): value is LogprobToken { - if (!isObject(value)) { - return false; - } - if (typeof value.token !== "string") { - return false; - } - if (typeof value.logprob !== "number") { - return false; - } - if (value.bytes === undefined) { - return true; - } - return Array.isArray(value.bytes) && value.bytes.every((entry) => typeof entry === "number"); -} - -function isLogprobTokenArray(value: unknown): value is LogprobToken[] { - return Array.isArray(value) && value.every((entry) => isLogprobToken(entry)); -} - -export class OpenAISTT { - private readonly openaiClient: OpenAI; - private readonly config: STTConfig; - private readonly logger: pino.Logger; - - constructor(sttConfig: STTConfig, parentLogger: pino.Logger) { - this.config = sttConfig; - this.logger = parentLogger.child({ module: "agent", provider: "openai", component: "stt" }); - this.openaiClient = new OpenAI({ - apiKey: sttConfig.apiKey, - }); - this.logger.info({ model: sttConfig.model || "whisper-1" }, "STT (OpenAI Whisper) initialized"); - } - - public async transcribeAudio(audioBuffer: Buffer, format: string): Promise { - const startTime = Date.now(); - let tempFilePath: string | null = null; - - try { - const ext = inferAudioExtension(format); - tempFilePath = join(tmpdir(), `audio-${v4()}.${ext}`); - await writeFile(tempFilePath, audioBuffer); - - this.logger.debug( - { tempFilePath, bytes: audioBuffer.length }, - "Transcribing audio file" - ); - - const modelToUse = this.config.model ?? "whisper-1"; - const supportsLogprobs = - modelToUse === "gpt-4o-transcribe" || modelToUse === "gpt-4o-mini-transcribe"; - const includeLogprobs: ["logprobs"] = ["logprobs"]; - - const response = await this.openaiClient.audio.transcriptions.create({ - file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)), - language: "en", - model: modelToUse, - ...(supportsLogprobs ? { include: includeLogprobs } : {}), - response_format: "json", - }); - - const duration = Date.now() - startTime; - const confidenceThreshold = this.config.confidenceThreshold ?? -3.0; - - let avgLogprob: number | undefined; - let isLowConfidence = false; - const logprobs = - supportsLogprobs && - isObject(response) && - isLogprobTokenArray(response.logprobs) - ? response.logprobs - : undefined; - - if (logprobs && logprobs.length > 0) { - const totalLogprob = logprobs.reduce((sum, token) => sum + token.logprob, 0); - avgLogprob = totalLogprob / logprobs.length; - isLowConfidence = avgLogprob < confidenceThreshold; - - if (isLowConfidence) { - this.logger.debug( - { - avgLogprob, - threshold: confidenceThreshold, - text: response.text, - tokenLogprobs: logprobs.map((t) => `${t.token}:${t.logprob.toFixed(2)}`).join(", "), - }, - "Low confidence transcription detected" - ); - } - } - - this.logger.debug({ duration, text: response.text, avgLogprob }, "Transcription complete"); - - return { - text: response.text, - duration: duration, - logprobs: logprobs, - avgLogprob: avgLogprob, - isLowConfidence: isLowConfidence, - language: - isObject(response) && typeof response.language === "string" - ? response.language - : undefined, - }; - } catch (error: any) { - this.logger.error({ err: error }, "Transcription error"); - throw new Error(`STT transcription failed: ${error.message}`); - } finally { - if (tempFilePath) { - try { - await unlink(tempFilePath); - } catch (cleanupError) { - this.logger.warn({ tempFilePath }, "Failed to clean up temp file"); - } - } - } - } -} diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index b47deb61f..5dcfd16d3 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -40,20 +40,20 @@ function parseListenString(listen: string): ListenTarget { import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; -import { OpenAISTT, type STTConfig } from "./agent/stt-openai.js"; -import { OpenAITTS, type TTSConfig } from "./agent/tts-openai.js"; +import { OpenAISTT, type STTConfig } from "./speech/providers/openai/stt.js"; +import { OpenAITTS, type TTSConfig } from "./speech/providers/openai/tts.js"; +import { OpenAIRealtimeTranscriptionSession } from "./speech/providers/openai/realtime-transcription-session.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; -import type { RealtimeTranscriptionSessionFactory } from "./dictation/dictation-stream-manager.js"; -import { SherpaOnlineRecognizerEngine } from "./speech/sherpa/sherpa-online-recognizer.js"; -import { SherpaOfflineRecognizerEngine } from "./speech/sherpa/sherpa-offline-recognizer.js"; -import { SherpaOnnxSTT } from "./speech/sherpa/sherpa-stt.js"; -import { SherpaOnnxParakeetSTT } from "./speech/sherpa/sherpa-parakeet-stt.js"; -import { SherpaOnnxTTS } from "./speech/sherpa/sherpa-tts.js"; -import { SherpaRealtimeTranscriptionSession } from "./speech/sherpa/sherpa-realtime-session.js"; -import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/sherpa/sherpa-parakeet-realtime-session.js"; -import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/sherpa/model-downloader.js"; -import type { SherpaOnnxModelId } from "./speech/sherpa/model-catalog.js"; -import { PocketTtsOnnxTTS } from "./speech/pocket/pocket-tts-onnx.js"; +import { SherpaOnlineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-online-recognizer.js"; +import { SherpaOfflineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-offline-recognizer.js"; +import { SherpaOnnxSTT } from "./speech/providers/local/sherpa/sherpa-stt.js"; +import { SherpaOnnxParakeetSTT } from "./speech/providers/local/sherpa/sherpa-parakeet-stt.js"; +import { SherpaOnnxTTS } from "./speech/providers/local/sherpa/sherpa-tts.js"; +import { SherpaRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-realtime-session.js"; +import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-parakeet-realtime-session.js"; +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/providers/local/sherpa/model-downloader.js"; +import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js"; +import { PocketTtsOnnxTTS } from "./speech/providers/local/pocket/pocket-tts-onnx.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; import { attachAgentStoragePersistence } from "./persistence-hooks.js"; @@ -98,9 +98,9 @@ export type PaseoSherpaOnnxConfig = { }; export type PaseoSpeechConfig = { - dictationSttProvider?: "openai" | "sherpa"; - voiceSttProvider?: "openai" | "sherpa"; - voiceTtsProvider?: "openai" | "sherpa"; + dictationSttProvider?: "openai" | "local"; + voiceSttProvider?: "openai" | "local"; + voiceTtsProvider?: "openai" | "local"; sherpaOnnx?: PaseoSherpaOnnxConfig; }; @@ -417,7 +417,7 @@ export async function createPaseoDaemon( let sttService: SpeechToTextProvider | null = null; let ttsService: TextToSpeechProvider | null = null; - let dictationSessionFactory: RealtimeTranscriptionSessionFactory | undefined; + let dictationSttService: SpeechToTextProvider | null = null; let sherpaOnline: SherpaOnlineRecognizerEngine | null = null; let sherpaOffline: SherpaOfflineRecognizerEngine | null = null; @@ -427,11 +427,11 @@ export async function createPaseoDaemon( const speechConfig = config.speech ?? null; const sherpaConfig = speechConfig?.sherpaOnnx ?? null; - const wantsSherpaDictation = (speechConfig?.dictationSttProvider ?? "openai") === "sherpa"; - const wantsSherpaVoiceStt = (speechConfig?.voiceSttProvider ?? "openai") === "sherpa"; - const wantsSherpaVoiceTts = (speechConfig?.voiceTtsProvider ?? "openai") === "sherpa"; + const wantsLocalDictation = (speechConfig?.dictationSttProvider ?? "openai") === "local"; + const wantsLocalVoiceStt = (speechConfig?.voiceSttProvider ?? "openai") === "local"; + const wantsLocalVoiceTts = (speechConfig?.voiceTtsProvider ?? "openai") === "local"; - if ((wantsSherpaDictation || wantsSherpaVoiceStt || wantsSherpaVoiceTts) && sherpaConfig) { + if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) { const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true); let sttPreset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim(); if ( @@ -460,10 +460,10 @@ export async function createPaseoDaemon( } const modelIds: SherpaOnnxModelId[] = []; - if (wantsSherpaDictation || wantsSherpaVoiceStt) { + if (wantsLocalDictation || wantsLocalVoiceStt) { modelIds.push(sttPreset as SherpaOnnxModelId); } - if (wantsSherpaVoiceTts) { + if (wantsLocalVoiceTts) { modelIds.push(ttsPreset as SherpaOnnxModelId); } @@ -489,7 +489,7 @@ export async function createPaseoDaemon( } } - if ((wantsSherpaDictation || wantsSherpaVoiceStt) && sherpaConfig) { + if ((wantsLocalDictation || wantsLocalVoiceStt) && sherpaConfig) { let preset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim(); if ( preset !== "zipformer-bilingual-zh-en-2023-02-20" && @@ -561,14 +561,14 @@ export async function createPaseoDaemon( sherpaOnline = null; sherpaOffline = null; } - } else if (wantsSherpaDictation || wantsSherpaVoiceStt) { + } else if (wantsLocalDictation || wantsLocalVoiceStt) { logger.warn( { configured: Boolean(sherpaConfig) }, "Sherpa STT selected but no sherpaOnnx config found; STT will be unavailable" ); } - if (wantsSherpaVoiceTts && sherpaConfig) { + if (wantsLocalVoiceTts && sherpaConfig) { let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); if ( preset !== "kitten-nano-en-v0_1-fp16" && @@ -615,27 +615,34 @@ export async function createPaseoDaemon( ); sherpaTts = null; } - } else if (wantsSherpaVoiceTts) { + } else if (wantsLocalVoiceTts) { logger.warn( { configured: Boolean(sherpaConfig) }, "Sherpa TTS selected but no sherpaOnnx config found; TTS will be unavailable" ); } - if (wantsSherpaVoiceStt && sherpaOffline) { + if (wantsLocalVoiceStt && sherpaOffline) { sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger); - } else if (wantsSherpaVoiceStt && sherpaOnline) { + } else if (wantsLocalVoiceStt && sherpaOnline) { sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger); } - if (wantsSherpaVoiceTts && sherpaTts) { + if (wantsLocalVoiceTts && sherpaTts) { ttsService = sherpaTts; } - if (wantsSherpaDictation && sherpaOnline) { - dictationSessionFactory = () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }); - } else if (wantsSherpaDictation && sherpaOffline) { - dictationSessionFactory = () => new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }); + if (wantsLocalDictation && sherpaOnline) { + dictationSttService = { + id: "local", + createSession: () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }), + }; + } else if (wantsLocalDictation && sherpaOffline) { + dictationSttService = { + id: "local", + createSession: () => + new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }), + }; } const voiceSttProvider = speechConfig?.voiceSttProvider ?? "openai"; @@ -645,10 +652,10 @@ export async function createPaseoDaemon( const needsOpenAiStt = !sttService && voiceSttProvider === "openai"; const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai"; const needsOpenAiDictation = - dictationSttProvider === "openai" || (dictationSttProvider === "sherpa" && !dictationSessionFactory); + dictationSttProvider === "openai" || (dictationSttProvider === "local" && !dictationSttService); - const fallbackOpenAiStt = !sttService && voiceSttProvider === "sherpa" && Boolean(openaiApiKey); - const fallbackOpenAiTts = !ttsService && voiceTtsProvider === "sherpa" && Boolean(openaiApiKey); + const fallbackOpenAiStt = !sttService && voiceSttProvider === "local" && Boolean(openaiApiKey); + const fallbackOpenAiTts = !ttsService && voiceTtsProvider === "local" && Boolean(openaiApiKey); if ((needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) && openaiApiKey) { logger.info("OpenAI client initialized"); @@ -689,6 +696,25 @@ export async function createPaseoDaemon( ); } } + + if (needsOpenAiDictation) { + const dictationApiKey = config.openai?.apiKey ?? openaiApiKey; + const transcriptionModel = + process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe"; + + dictationSttService = { + id: "openai", + createSession: ({ logger: sessionLogger, language, prompt }) => + new OpenAIRealtimeTranscriptionSession({ + apiKey: dictationApiKey, + logger: sessionLogger, + transcriptionModel, + ...(language ? { language } : {}), + ...(prompt ? { prompt } : {}), + turnDetection: null, + }), + }; + } } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) { logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation fallback is unavailable"); } @@ -710,9 +736,8 @@ export async function createPaseoDaemon( voiceLlmModel: config.voiceLlmModel ?? null, }, { - openaiApiKey: config.openai?.apiKey ?? null, finalTimeoutMs: config.dictationFinalTimeoutMs, - ...(dictationSessionFactory ? { sessionFactory: dictationSessionFactory } : {}), + stt: dictationSttService, } ); diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index fca13a681..84f4a10b4 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -1,8 +1,8 @@ import path from "node:path"; import type { PaseoDaemonConfig } from "./bootstrap.js"; -import type { STTConfig } from "./agent/stt-openai.js"; -import type { TTSConfig } from "./agent/tts-openai.js"; +import type { STTConfig } from "./speech/providers/openai/stt.js"; +import type { TTSConfig } from "./speech/providers/openai/tts.js"; import { loadPersistedConfig } from "./persisted-config.js"; import { mergeAllowedHosts, @@ -76,7 +76,7 @@ function parseOpenAIConfig( }; } -function parseSpeechProviderId(value: unknown): "openai" | "sherpa" | null { +function parseSpeechProviderId(value: unknown): "openai" | "local" | null { if (typeof value !== "string") { return null; } @@ -86,7 +86,7 @@ function parseSpeechProviderId(value: unknown): "openai" | "sherpa" | null { } if (normalized === "openai") return "openai"; if (normalized === "sherpa" || normalized === "sherpa-onnx" || normalized === "local") { - return "sherpa"; + return "local"; } return null; } @@ -185,22 +185,22 @@ export function loadConfig( const dictationSttProvider = parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ?? parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ?? - "sherpa"; + "local"; const voiceSttProvider = parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ?? parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ?? - "sherpa"; + "local"; const voiceTtsProvider = parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ?? parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ?? - "sherpa"; + "local"; const shouldConfigureSherpa = - dictationSttProvider === "sherpa" || - voiceSttProvider === "sherpa" || - voiceTtsProvider === "sherpa" || + dictationSttProvider === "local" || + voiceSttProvider === "local" || + voiceTtsProvider === "local" || typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" || Boolean(persisted.providers?.sherpaOnnx); diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 5a253f513..d1119f402 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -96,9 +96,9 @@ describe("daemon client E2E", () => { dictationFinalTimeoutMs: 5000, ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}), speech: { - dictationSttProvider: "sherpa", - voiceSttProvider: "sherpa", - voiceTtsProvider: "sherpa", + dictationSttProvider: "local", + voiceSttProvider: "local", + voiceTtsProvider: "local", sherpaOnnx: { modelsDir: sherpaModelsDir, stt: { diff --git a/packages/server/src/server/dictation/dictation-stream-manager.test.ts b/packages/server/src/server/dictation/dictation-stream-manager.test.ts index 830739d74..2e8ae0251 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.test.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.test.ts @@ -4,23 +4,26 @@ import pino from "pino"; import { DictationStreamManager, - type RealtimeTranscriptionSession, - type RealtimeTranscriptionSessionFactory, } from "./dictation-stream-manager.js"; +import type { + SpeechToTextProvider, + StreamingTranscriptionSession, +} from "../speech/speech-provider.js"; -class FakeRealtimeSession extends EventEmitter implements RealtimeTranscriptionSession { +class FakeRealtimeSession extends EventEmitter implements StreamingTranscriptionSession { connected = false; - appended: string[] = []; + appended: Buffer[] = []; commitCalls = 0; clearCalls = 0; closed = false; + requiredSampleRate = 24000; async connect(): Promise { this.connected = true; } - appendPcm16Base64(base64Audio: string): void { - this.appended.push(base64Audio); + appendPcm16(pcm16le: Buffer): void { + this.appended.push(pcm16le); } commit(): void { @@ -35,12 +38,12 @@ class FakeRealtimeSession extends EventEmitter implements RealtimeTranscriptionS this.closed = true; } - emitCommitted(itemId: string): void { - this.emit("committed", { itemId, previousItemId: null }); + emitCommitted(segmentId: string): void { + this.emit("committed", { segmentId, previousSegmentId: null }); } - emitTranscript(itemId: string, transcript: string, isFinal: boolean): void { - this.emit("transcript", { itemId, transcript, isFinal }); + emitTranscript(segmentId: string, transcript: string, isFinal: boolean): void { + this.emit("transcript", { segmentId, transcript, isFinal }); } emitError(message: string): void { @@ -48,6 +51,14 @@ class FakeRealtimeSession extends EventEmitter implements RealtimeTranscriptionS } } +class FakeSttProvider implements SpeechToTextProvider { + public readonly id = "fake"; + constructor(private readonly session: FakeRealtimeSession) {} + createSession(_params: { logger: any; language?: string; prompt?: string }): StreamingTranscriptionSession { + return this.session; + } +} + const buildPcmBase64 = (sampleValue: number, sampleCount: number): string => { const samples = new Int16Array(sampleCount); samples.fill(sampleValue); @@ -59,34 +70,29 @@ const tick = async (): Promise => { await Promise.resolve(); }; -describe("DictationStreamManager (semantic VAD grace fallback)", () => { +describe("DictationStreamManager (finish buffer-too-small tolerance)", () => { const env = { - turnDetection: process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION, dictationDebug: process.env.PASEO_DICTATION_DEBUG, }; beforeEach(() => { vi.useFakeTimers(); - process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION = "semantic_vad"; process.env.PASEO_DICTATION_DEBUG = "false"; }); afterEach(() => { vi.useRealTimers(); - process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION = env.turnDetection; process.env.PASEO_DICTATION_DEBUG = env.dictationDebug; }); it("treats buffer-too-small as benign and finalizes with existing transcripts", async () => { const session = new FakeRealtimeSession(); - const factory: RealtimeTranscriptionSessionFactory = () => session; const emitted: Array<{ type: string; payload: any }> = []; const manager = new DictationStreamManager({ logger: pino({ level: "silent" }), emit: (msg) => emitted.push(msg), sessionId: "s1", - openaiApiKey: "k", - sessionFactory: factory, + stt: new FakeSttProvider(session), finalTimeoutMs: 5000, }); @@ -98,14 +104,11 @@ describe("DictationStreamManager (semantic VAD grace fallback)", () => { format: "audio/pcm;rate=24000;bits=16", }); - session.emitTranscript("i1", "hello world", true); + session.emitTranscript("seg-1", "hello world", true); await manager.handleFinish("d1", 0); await tick(); - vi.advanceTimersByTime(2000); - await tick(); - session.emitError( "Error committing input audio buffer: buffer too small. Expected at least 100ms of audio, but buffer only has 0.00ms of audio." ); @@ -117,56 +120,21 @@ describe("DictationStreamManager (semantic VAD grace fallback)", () => { expect(final?.payload.text).toBe("hello world"); expect(session.closed).toBe(true); }); - - it("does not fallback-commit if committed event arrives during grace window", async () => { - const session = new FakeRealtimeSession(); - const factory: RealtimeTranscriptionSessionFactory = () => session; - const emitted: Array<{ type: string; payload: any }> = []; - const manager = new DictationStreamManager({ - logger: pino({ level: "silent" }), - emit: (msg) => emitted.push(msg), - sessionId: "s1", - openaiApiKey: "k", - sessionFactory: factory, - finalTimeoutMs: 5000, - }); - - await manager.handleStart("d1", "audio/pcm;rate=24000;bits=16"); - await manager.handleChunk({ - dictationId: "d1", - seq: 0, - audioBase64: buildPcmBase64(2000, 2400), - format: "audio/pcm;rate=24000;bits=16", - }); - - await manager.handleFinish("d1", 0); - session.emitCommitted("i1"); - session.emitTranscript("i1", "hi there", true); - - vi.advanceTimersByTime(2000); - await tick(); - - expect(session.commitCalls).toBe(0); - const final = emitted.find((msg) => msg.type === "dictation_stream_final"); - expect(final?.payload.text).toBe("hi there"); - }); }); -describe("DictationStreamManager (provider-agnostic session factory)", () => { - it("can start with a custom session factory even without OPENAI_API_KEY", async () => { +describe("DictationStreamManager (provider-agnostic provider)", () => { + it("does not require OPENAI_API_KEY", async () => { const original = process.env.OPENAI_API_KEY; delete process.env.OPENAI_API_KEY; try { const session = new FakeRealtimeSession(); - const factory: RealtimeTranscriptionSessionFactory = () => session; const emitted: Array<{ type: string; payload: any }> = []; const manager = new DictationStreamManager({ logger: pino({ level: "silent" }), emit: (msg) => emitted.push(msg), sessionId: "s1", - openaiApiKey: null, - sessionFactory: factory, + stt: new FakeSttProvider(session), }); await manager.handleStart("d-local", "audio/pcm;rate=16000;bits=16"); diff --git a/packages/server/src/server/dictation/dictation-stream-manager.ts b/packages/server/src/server/dictation/dictation-stream-manager.ts index ebad8281b..0a6f21b38 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.ts @@ -7,112 +7,19 @@ import { } from "../agent/dictation-debug.js"; import { isPaseoDictationDebugEnabled } from "../agent/recordings-debug.js"; import { Pcm16MonoResampler } from "../agent/pcm16-resampler.js"; -import { OpenAIRealtimeTranscriptionSession } from "../agent/openai-realtime-transcription.js"; +import type { + SpeechToTextProvider, + StreamingTranscriptionSession, +} from "../speech/speech-provider.js"; +import { parsePcmRateFromFormat, pcm16lePeakAbs } from "../speech/audio.js"; const PCM_CHANNELS = 1; const PCM_BITS_PER_SAMPLE = 16; -const OPENAI_DICTATION_PCM_OUTPUT_RATE = 24000; const DEFAULT_DICTATION_FINAL_TIMEOUT_MS = 10000; -const DICTATION_VAD_GRACE_TIMEOUT_MS = Number.parseInt( - process.env.OPENAI_REALTIME_DICTATION_VAD_GRACE_TIMEOUT_MS ?? "2000", - 10 -); const DICTATION_SILENCE_PEAK_THRESHOLD = Number.parseInt( - process.env.OPENAI_REALTIME_DICTATION_SILENCE_PEAK_THRESHOLD ?? "300", + process.env.PASEO_DICTATION_SILENCE_PEAK_THRESHOLD ?? "300", 10 ); -const DICTATION_TURN_DETECTION = ( - process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION ?? "semantic_vad" -).trim(); -const DICTATION_SEMANTIC_VAD_EAGERNESS = ( - process.env.OPENAI_REALTIME_DICTATION_SEMANTIC_VAD_EAGERNESS ?? "medium" -).trim(); -const DICTATION_FLUSH_SILENCE_MS = Number.parseInt( - process.env.OPENAI_REALTIME_DICTATION_FLUSH_SILENCE_MS ?? "800", - 10 -); - -type OpenAITurnDetection = - | null - | { - type: "server_vad"; - create_response: false; - threshold?: number; - prefix_padding_ms?: number; - silence_duration_ms?: number; - } - | { type: "semantic_vad"; create_response: false; eagerness?: "low" | "medium" | "high" }; - -function pcm16lePeakAbs(pcm16le: Buffer): number { - if (pcm16le.length === 0) { - return 0; - } - if (pcm16le.length % 2 !== 0) { - throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`); - } - const samples = new Int16Array( - pcm16le.buffer, - pcm16le.byteOffset, - pcm16le.byteLength / 2 - ); - let peak = 0; - for (let i = 0; i < samples.length; i += 1) { - const v = samples[i]!; - const abs = v < 0 ? -v : v; - if (abs > peak) { - peak = abs; - if (peak >= 32767) { - break; - } - } - } - return peak; -} - -function parseDictationTurnDetection(): OpenAITurnDetection { - if ( - !DICTATION_TURN_DETECTION || - DICTATION_TURN_DETECTION === "none" || - DICTATION_TURN_DETECTION === "null" - ) { - return null; - } - if (DICTATION_TURN_DETECTION === "server_vad") { - return { type: "server_vad", create_response: false }; - } - const eagerness = - DICTATION_SEMANTIC_VAD_EAGERNESS === "low" || - DICTATION_SEMANTIC_VAD_EAGERNESS === "high" - ? (DICTATION_SEMANTIC_VAD_EAGERNESS as "low" | "high") - : ("medium" as const); - return { type: "semantic_vad", create_response: false, eagerness }; -} - -export type RealtimeTranscriptionSession = { - connect(): Promise; - appendPcm16Base64(base64Audio: string): void; - commit(): void; - clear(): void; - close(): void; - on( - event: "committed", - handler: (payload: { itemId: string; previousItemId: string | null }) => void - ): unknown; - on( - event: "transcript", - handler: (payload: { itemId: string; transcript: string; isFinal: boolean }) => void - ): unknown; - on(event: "error", handler: (err: unknown) => void): unknown; -}; - -export type RealtimeTranscriptionSessionFactory = (params: { - apiKey?: string | null; - logger: pino.Logger; - transcriptionModel: string; - language?: string; - prompt?: string; - turnDetection: OpenAITurnDetection; -}) => RealtimeTranscriptionSession; function convertPCMToWavBuffer( pcmBuffer: Buffer, @@ -147,7 +54,7 @@ type DictationStreamState = { dictationId: string; sessionId: string; inputFormat: string; - openai: RealtimeTranscriptionSession; + stt: StreamingTranscriptionSession; inputRate: number; outputRate: number; resampler: Pcm16MonoResampler | null; @@ -159,17 +66,14 @@ type DictationStreamState = { ackSeq: number; bytesSinceCommit: number; peakSinceCommit: number; - committedItemIds: string[]; - transcriptsByItemId: Map; - finalTranscriptItemIds: Set; + committedSegmentIds: string[]; + transcriptsBySegmentId: Map; + finalTranscriptSegmentIds: Set; awaitingFinalCommit: boolean; - vadGraceTimeout: ReturnType | null; - fallbackCommitAttempted: boolean; finishRequested: boolean; finishSealed: boolean; finalSeq: number | null; finalTimeout: ReturnType | null; - isSemanticVad: boolean; }; export type DictationStreamOutboundMessage = @@ -192,37 +96,22 @@ export class DictationStreamManager { private readonly logger: pino.Logger; private readonly emit: (msg: DictationStreamOutboundMessage) => void; private readonly sessionId: string; - private readonly openaiApiKey: string | null; + private readonly stt: SpeechToTextProvider | null; private readonly finalTimeoutMs: number; - private readonly createSession: RealtimeTranscriptionSessionFactory; - private readonly requiresApiKey: boolean; private readonly streams = new Map(); constructor(params: { logger: pino.Logger; emit: (msg: DictationStreamOutboundMessage) => void; sessionId: string; - openaiApiKey?: string | null; + stt: SpeechToTextProvider | null; finalTimeoutMs?: number; - sessionFactory?: RealtimeTranscriptionSessionFactory; }) { this.logger = params.logger.child({ component: "dictation-stream-manager" }); this.emit = params.emit; this.sessionId = params.sessionId; - this.openaiApiKey = params.openaiApiKey ?? null; + this.stt = params.stt; this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS; - this.requiresApiKey = !params.sessionFactory; - this.createSession = - params.sessionFactory ?? - ((factoryParams) => { - if (!factoryParams.apiKey) { - throw new Error("OPENAI_API_KEY not set"); - } - return new OpenAIRealtimeTranscriptionSession({ - ...factoryParams, - apiKey: factoryParams.apiKey, - }); - }); } public cleanupAll(): void { @@ -234,39 +123,30 @@ export class DictationStreamManager { public async handleStart(dictationId: string, format: string): Promise { this.cleanupDictationStream(dictationId); - const apiKey = this.openaiApiKey ?? process.env.OPENAI_API_KEY; - if (this.requiresApiKey && !apiKey) { - this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false); + if (!this.stt) { + this.failDictationStream(dictationId, "Dictation STT not configured", false); return; } - const transcriptionModel = - process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe"; const transcriptionPrompt = - process.env.OPENAI_REALTIME_DICTATION_TRANSCRIPTION_PROMPT ?? + process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT ?? "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript."; - const turnDetection = parseDictationTurnDetection(); - const openai = this.createSession({ - apiKey: apiKey ?? null, + const stt = this.stt.createSession({ logger: this.logger.child({ dictationId }), - transcriptionModel, language: "en", prompt: transcriptionPrompt, - turnDetection, }); - openai.on("committed", ({ itemId }: { itemId: string }) => { + stt.on("committed", ({ segmentId }) => { const state = this.streams.get(dictationId); if (!state) { return; } - this.clearVadGraceTimeout(state); - state.committedItemIds.push(itemId); + state.committedSegmentIds.push(segmentId); state.bytesSinceCommit = 0; state.peakSinceCommit = 0; - // When finishing, we require at least one commit after finish if we flushed pending audio. if (state.finishRequested && state.awaitingFinalCommit) { state.awaitingFinalCommit = false; } @@ -274,52 +154,37 @@ export class DictationStreamManager { this.maybeFinalizeDictationStream(dictationId); }); - openai.on( - "transcript", - ({ - itemId, - transcript, - isFinal, - }: { - itemId: string; - transcript: string; - isFinal: boolean; - }) => { - const state = this.streams.get(dictationId); - if (!state) { - return; - } - state.transcriptsByItemId.set(itemId, transcript); - if (isFinal) { - state.finalTranscriptItemIds.add(itemId); - } - - // If we triggered a finish commit but OpenAI doesn't emit committed events (or they arrive late), - // allow final transcripts to unblock finalization. - if (state.finishRequested && state.awaitingFinalCommit && isFinal) { - this.clearVadGraceTimeout(state); - state.awaitingFinalCommit = false; - } - - const orderedIds = state.committedItemIds.includes(itemId) - ? state.committedItemIds - : [...state.committedItemIds, itemId]; - const partialText = orderedIds - .map((id) => state.transcriptsByItemId.get(id) ?? "") - .join(" ") - .trim(); - this.emitDictationPartial(dictationId, partialText); - - this.maybeSealDictationStreamFinish(dictationId); - this.maybeFinalizeDictationStream(dictationId); + stt.on("transcript", ({ segmentId, transcript, isFinal }) => { + const state = this.streams.get(dictationId); + if (!state) { + return; + } + state.transcriptsBySegmentId.set(segmentId, transcript); + if (isFinal) { + state.finalTranscriptSegmentIds.add(segmentId); } - ); - openai.on("error", (err) => { + if (state.finishRequested && state.awaitingFinalCommit && isFinal) { + state.awaitingFinalCommit = false; + } + + const orderedIds = state.committedSegmentIds.includes(segmentId) + ? state.committedSegmentIds + : [...state.committedSegmentIds, segmentId]; + const partialText = orderedIds + .map((id) => state.transcriptsBySegmentId.get(id) ?? "") + .join(" ") + .trim(); + this.emitDictationPartial(dictationId, partialText); + + this.maybeSealDictationStreamFinish(dictationId); + this.maybeFinalizeDictationStream(dictationId); + }); + + stt.on("error", (err) => { const message = err instanceof Error ? err.message : String(err); const state = this.streams.get(dictationId); if (state && state.finishRequested && isBufferTooSmallError(message)) { - this.clearVadGraceTimeout(state); if (state.awaitingFinalCommit) { state.awaitingFinalCommit = false; } @@ -329,14 +194,13 @@ export class DictationStreamManager { void this.failAndCleanupDictationStream(dictationId, message, true); }); - await openai.connect(); + await stt.connect(); - const rateMatch = /(?:^|[;,\s])rate\s*=\s*(\d+)(?:$|[;,\s])/i.exec(format); - const inputRate = rateMatch ? Number.parseInt(rateMatch[1]!, 10) : 16000; + const inputRate = parsePcmRateFromFormat(format, 16000) ?? 16000; if (!Number.isFinite(inputRate) || inputRate <= 0) { this.failDictationStream(dictationId, `Invalid dictation input rate in format: ${format}`, false); try { - openai.close(); + stt.close(); } catch { // no-op } @@ -348,13 +212,13 @@ export class DictationStreamManager { this.logger ); - const outputRate = this.requiresApiKey ? OPENAI_DICTATION_PCM_OUTPUT_RATE : inputRate; + const outputRate = stt.requiredSampleRate; this.streams.set(dictationId, { dictationId, sessionId: this.sessionId, inputFormat: format, - openai, + stt, inputRate, outputRate, resampler: @@ -372,17 +236,14 @@ export class DictationStreamManager { ackSeq: -1, bytesSinceCommit: 0, peakSinceCommit: 0, - committedItemIds: [], - transcriptsByItemId: new Map(), - finalTranscriptItemIds: new Set(), + committedSegmentIds: [], + transcriptsBySegmentId: new Map(), + finalTranscriptSegmentIds: new Set(), awaitingFinalCommit: false, - vadGraceTimeout: null, - fallbackCommitAttempted: false, finishRequested: false, finishSealed: false, finalSeq: null, finalTimeout: null, - isSemanticVad: turnDetection?.type === "semantic_vad", }); this.emitDictationAck(dictationId, -1); @@ -425,7 +286,7 @@ export class DictationStreamManager { const resampled = state.resampler ? state.resampler.processChunk(pcm16) : pcm16; if (resampled.length > 0) { - state.openai.appendPcm16Base64(resampled.toString("base64")); + state.stt.appendPcm16(resampled); state.debugAudioChunks.push(resampled); state.bytesSinceCommit += resampled.length; state.peakSinceCommit = Math.max(state.peakSinceCommit, pcm16lePeakAbs(resampled)); @@ -579,12 +440,11 @@ export class DictationStreamManager { if (!state) { return; } - this.clearVadGraceTimeout(state); if (state.finalTimeout) { clearTimeout(state.finalTimeout); } try { - state.openai.close(); + state.stt.close(); } catch { // no-op } @@ -616,37 +476,18 @@ export class DictationStreamManager { }, "Dictation finish: clearing silence-only tail (skip final commit)" ); - state.openai.clear(); + state.stt.clear(); state.bytesSinceCommit = 0; state.peakSinceCommit = 0; state.awaitingFinalCommit = false; } else { - const silenceBytes = Math.max( - 0, - Math.round((state.outputRate * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000) - ); - if (silenceBytes > 0) { - this.logger.debug( - { dictationId, silenceMs: DICTATION_FLUSH_SILENCE_MS, silenceBytes }, - "Dictation finish: appending silence tail for semantic VAD flush" - ); - const silence = Buffer.alloc(silenceBytes); - state.openai.appendPcm16Base64(silence.toString("base64")); - state.debugAudioChunks.push(silence); - state.bytesSinceCommit += silenceBytes; - } - state.awaitingFinalCommit = true; - if (state.isSemanticVad) { - this.startVadGraceTimeout(state); - } else { - try { - state.openai.commit(); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - void this.failAndCleanupDictationStream(dictationId, message, true); - return; - } + try { + state.stt.commit(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + void this.failAndCleanupDictationStream(dictationId, message, true); + return; } } } else { @@ -672,15 +513,15 @@ export class DictationStreamManager { return; } - const committedSet = new Set(state.committedItemIds); - const orderedItemIds: string[] = [...state.committedItemIds]; - for (const itemId of state.transcriptsByItemId.keys()) { - if (!committedSet.has(itemId)) { - orderedItemIds.push(itemId); + const committedSet = new Set(state.committedSegmentIds); + const orderedSegmentIds: string[] = [...state.committedSegmentIds]; + for (const segmentId of state.transcriptsBySegmentId.keys()) { + if (!committedSet.has(segmentId)) { + orderedSegmentIds.push(segmentId); } } - if (orderedItemIds.length === 0) { + if (orderedSegmentIds.length === 0) { void (async () => { const debugRecordingPath = await this.maybePersistDictationStreamAudio(dictationId); this.emit({ @@ -708,15 +549,15 @@ export class DictationStreamManager { return; } - const allTranscriptsReady = orderedItemIds.every((itemId) => - state.finalTranscriptItemIds.has(itemId) + const allTranscriptsReady = orderedSegmentIds.every((segmentId) => + state.finalTranscriptSegmentIds.has(segmentId) ); if (!allTranscriptsReady) { return; } - const orderedText = orderedItemIds - .map((itemId) => state.transcriptsByItemId.get(itemId) ?? "") + const orderedText = orderedSegmentIds + .map((segmentId) => state.transcriptsBySegmentId.get(segmentId) ?? "") .join(" ") .trim(); @@ -745,41 +586,6 @@ export class DictationStreamManager { this.cleanupDictationStream(dictationId); })(); } - - private startVadGraceTimeout(state: DictationStreamState): void { - if (state.vadGraceTimeout || DICTATION_VAD_GRACE_TIMEOUT_MS <= 0) { - return; - } - state.vadGraceTimeout = setTimeout(() => { - state.vadGraceTimeout = null; - if (!state.finishRequested || !state.awaitingFinalCommit) { - return; - } - if (state.bytesSinceCommit <= 0 || state.fallbackCommitAttempted) { - return; - } - state.fallbackCommitAttempted = true; - try { - state.openai.commit(); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (isBufferTooSmallError(message)) { - state.awaitingFinalCommit = false; - this.maybeFinalizeDictationStream(state.dictationId); - return; - } - void this.failAndCleanupDictationStream(state.dictationId, message, true); - } - }, DICTATION_VAD_GRACE_TIMEOUT_MS); - } - - private clearVadGraceTimeout(state: DictationStreamState): void { - if (!state.vadGraceTimeout) { - return; - } - clearTimeout(state.vadGraceTimeout); - state.vadGraceTimeout = null; - } } function isBufferTooSmallError(message: string): boolean { diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 0c3ad2792..59469bc3a 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -46,11 +46,25 @@ const ProvidersSchema = z }) .strict(); +const SpeechProviderIdSchema = z.preprocess( + (value) => { + if (typeof value !== "string") { + return value; + } + const normalized = value.trim().toLowerCase(); + if (normalized === "sherpa" || normalized === "sherpa-onnx") { + return "local"; + } + return normalized; + }, + z.enum(["openai", "local"]) +); + const FeatureDictationSchema = z .object({ stt: z .object({ - provider: z.enum(["openai", "sherpa"]).optional(), + provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), preset: z.string().min(1).optional(), confidenceThreshold: z.number().optional(), @@ -71,7 +85,7 @@ const FeatureVoiceModeSchema = z .optional(), stt: z .object({ - provider: z.enum(["openai", "sherpa"]).optional(), + provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), preset: z.string().min(1).optional(), }) @@ -79,7 +93,7 @@ const FeatureVoiceModeSchema = z .optional(), tts: z .object({ - provider: z.enum(["openai", "sherpa"]).optional(), + provider: SpeechProviderIdSchema.optional(), model: z.enum(["tts-1", "tts-1-hd"]).optional(), voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(), preset: z.string().min(1).optional(), diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index d6a6885de..ae5417017 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -38,7 +38,6 @@ import { maybePersistTtsDebugAudio } from "./agent/tts-debug.js"; import { isPaseoDictationDebugEnabled } from "./agent/recordings-debug.js"; import { DictationStreamManager, - type RealtimeTranscriptionSessionFactory, } from "./dictation/dictation-stream-manager.js"; import type { VoiceConversationStore } from "./voice-conversation-store.js"; import { @@ -106,6 +105,14 @@ import { } from "../utils/checkout-git.js"; import { getProjectIcon } from "../utils/project-icon.js"; import { expandTilde } from "../utils/path.js"; +import { + ensureSherpaOnnxModels, + getSherpaOnnxModelDir, +} from "./speech/providers/local/sherpa/model-downloader.js"; +import { + listSherpaOnnxModels, + type SherpaOnnxModelId, +} from "./speech/providers/local/sherpa/model-catalog.js"; import type pino from "pino"; const execAsync = promisify(exec); @@ -334,9 +341,8 @@ export class Session { voiceLlmModel?: string | null; }, dictation?: { - openaiApiKey?: string | null; finalTimeoutMs?: number; - sessionFactory?: RealtimeTranscriptionSessionFactory; + stt?: SpeechToTextProvider | null; } ) { this.clientId = clientId; @@ -367,9 +373,8 @@ export class Session { logger: this.sessionLogger, sessionId: this.sessionId, emit: (msg) => this.emit(msg as unknown as SessionOutboundMessage), - openaiApiKey: dictation?.openaiApiKey ?? null, + stt: dictation?.stt ?? null, finalTimeoutMs: dictation?.finalTimeoutMs, - ...(dictation?.sessionFactory ? { sessionFactory: dictation.sessionFactory } : {}), }); // Initialize agent MCP client asynchronously @@ -975,6 +980,14 @@ export class Session { await this.handleListProviderModelsRequest(msg); break; + case "speech_models_list_request": + await this.handleSpeechModelsListRequest(msg); + break; + + case "speech_models_download_request": + await this.handleSpeechModelsDownloadRequest(msg); + break; + case "clear_agent_attention": await this.handleClearAgentAttention(msg.agentId); break; @@ -1910,6 +1923,114 @@ export class Session { } } + private async handleSpeechModelsListRequest( + msg: Extract + ): Promise { + const modelsDir = + process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() || + join(this.paseoHome, "models", "sherpa-onnx"); + + const models = await Promise.all( + listSherpaOnnxModels().map(async (model) => { + const modelDir = getSherpaOnnxModelDir(modelsDir, model.id); + const missingFiles: string[] = []; + for (const rel of model.requiredFiles) { + const filePath = join(modelDir, rel); + try { + const fileStat = await stat(filePath); + if (fileStat.isDirectory()) { + continue; + } + if (!fileStat.isFile() || fileStat.size <= 0) { + missingFiles.push(rel); + } + } catch { + missingFiles.push(rel); + } + } + + return { + id: model.id, + kind: model.kind, + description: model.description, + modelDir, + isDownloaded: missingFiles.length === 0, + ...(missingFiles.length > 0 ? { missingFiles } : {}), + }; + }) + ); + + this.emit({ + type: "speech_models_list_response", + payload: { + modelsDir, + models, + requestId: msg.requestId, + }, + }); + } + + private async handleSpeechModelsDownloadRequest( + msg: Extract + ): Promise { + const modelsDir = + process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() || + join(this.paseoHome, "models", "sherpa-onnx"); + + const modelIdsRaw = + msg.modelIds && msg.modelIds.length > 0 + ? msg.modelIds + : [ + process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", + process.env.PASEO_SHERPA_TTS_PRESET ?? "pocket-tts-onnx-int8", + ]; + + const allModelIds = new Set(listSherpaOnnxModels().map((m) => m.id)); + const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as SherpaOnnxModelId)); + if (invalid.length > 0) { + this.emit({ + type: "speech_models_download_response", + payload: { + modelsDir, + downloadedModelIds: [], + error: `Unknown speech model id(s): ${invalid.join(", ")}`, + requestId: msg.requestId, + }, + }); + return; + } + + const modelIds = modelIdsRaw as SherpaOnnxModelId[]; + try { + await ensureSherpaOnnxModels({ + modelsDir, + modelIds, + autoDownload: true, + logger: this.sessionLogger, + }); + this.emit({ + type: "speech_models_download_response", + payload: { + modelsDir, + downloadedModelIds: modelIds, + error: null, + requestId: msg.requestId, + }, + }); + } catch (error) { + this.sessionLogger.error({ err: error, modelIds }, "Failed to download speech models"); + this.emit({ + type: "speech_models_download_response", + payload: { + modelsDir, + downloadedModelIds: [], + error: error instanceof Error ? error.message : String(error), + requestId: msg.requestId, + }, + }); + } + } + private normalizeGitOptions( gitOptions?: GitSetupOptions, legacyWorktreeName?: string diff --git a/packages/server/src/server/speech/pocket/pocket-tts-onnx.ts b/packages/server/src/server/speech/providers/local/pocket/pocket-tts-onnx.ts similarity index 99% rename from packages/server/src/server/speech/pocket/pocket-tts-onnx.ts rename to packages/server/src/server/speech/providers/local/pocket/pocket-tts-onnx.ts index 58fd16e96..847edf13a 100644 --- a/packages/server/src/server/speech/pocket/pocket-tts-onnx.ts +++ b/packages/server/src/server/speech/providers/local/pocket/pocket-tts-onnx.ts @@ -3,9 +3,9 @@ import { readFile } from "node:fs/promises"; import { Readable } from "node:stream"; import type pino from "pino"; -import type { SpeechStreamResult, TextToSpeechProvider } from "../speech-provider.js"; -import { chunkBuffer, float32ToPcm16le, parsePcm16MonoWav, pcm16leToFloat32 } from "../audio.js"; -import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; +import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js"; +import { chunkBuffer, float32ToPcm16le, parsePcm16MonoWav, pcm16leToFloat32 } from "../../../audio.js"; +import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js"; type OrtModule = typeof import("onnxruntime-node"); type OrtSession = import("onnxruntime-node").InferenceSession; diff --git a/packages/server/src/server/speech/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts similarity index 100% rename from packages/server/src/server/speech/sherpa/model-catalog.ts rename to packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts diff --git a/packages/server/src/server/speech/sherpa/model-downloader.test.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.test.ts similarity index 100% rename from packages/server/src/server/speech/sherpa/model-downloader.test.ts rename to packages/server/src/server/speech/providers/local/sherpa/model-downloader.test.ts diff --git a/packages/server/src/server/speech/sherpa/model-downloader.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts similarity index 100% rename from packages/server/src/server/speech/sherpa/model-downloader.ts rename to packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts diff --git a/packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts similarity index 100% rename from packages/server/src/server/speech/sherpa/sherpa-offline-recognizer.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts diff --git a/packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts similarity index 100% rename from packages/server/src/server/speech/sherpa/sherpa-online-recognizer.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts diff --git a/packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts similarity index 100% rename from packages/server/src/server/speech/sherpa/sherpa-onnx-loader.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts diff --git a/packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts similarity index 100% rename from packages/server/src/server/speech/sherpa/sherpa-onnx-node-loader.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts diff --git a/packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-realtime-session.ts similarity index 73% rename from packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-realtime-session.ts index 6a912bd69..52651543c 100644 --- a/packages/server/src/server/speech/sherpa/sherpa-parakeet-realtime-session.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-realtime-session.ts @@ -1,22 +1,23 @@ import { EventEmitter } from "node:events"; import { v4 as uuidv4 } from "uuid"; -import type { RealtimeTranscriptionSession } from "../../dictation/dictation-stream-manager.js"; -import { pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import type { StreamingTranscriptionSession } from "../../../speech-provider.js"; +import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; export class SherpaParakeetRealtimeTranscriptionSession extends EventEmitter - implements RealtimeTranscriptionSession + implements StreamingTranscriptionSession { private readonly engine: SherpaOfflineRecognizerEngine; private connected = false; - private currentItemId: string | null = null; - private previousItemId: string | null = null; + public readonly requiredSampleRate: number; + private currentSegmentId: string | null = null; + private previousSegmentId: string | null = null; private lastPartialText = ""; - private pcm16 = Buffer.alloc(0); + private pcm16: Buffer = Buffer.alloc(0); private lastDecodeAt = 0; private decoding = false; private pendingDecode = false; @@ -25,6 +26,7 @@ export class SherpaParakeetRealtimeTranscriptionSession constructor(params: { engine: SherpaOfflineRecognizerEngine; minDecodeIntervalMs?: number }) { super(); this.engine = params.engine; + this.requiredSampleRate = this.engine.sampleRate; this.minDecodeIntervalMs = params.minDecodeIntervalMs ?? 350; } @@ -32,18 +34,17 @@ export class SherpaParakeetRealtimeTranscriptionSession if (this.connected) { return; } - this.currentItemId = uuidv4(); + this.currentSegmentId = uuidv4(); this.connected = true; } - appendPcm16Base64(base64Audio: string): void { - if (!this.connected || !this.currentItemId) { + appendPcm16(chunk: Buffer): void { + if (!this.connected || !this.currentSegmentId) { this.emit("error", new Error("Parakeet realtime session not connected")); return; } try { - const chunk = Buffer.from(base64Audio, "base64"); this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]); void this.maybeDecode(false); } catch (err) { @@ -52,7 +53,7 @@ export class SherpaParakeetRealtimeTranscriptionSession } commit(): void { - if (!this.connected || !this.currentItemId) { + if (!this.connected || !this.currentSegmentId) { this.emit("error", new Error("Parakeet realtime session not connected")); return; } @@ -61,14 +62,14 @@ export class SherpaParakeetRealtimeTranscriptionSession try { await this.maybeDecode(true); const finalText = this.lastPartialText; - const itemId = this.currentItemId!; - const previousItemId = this.previousItemId; + const segmentId = this.currentSegmentId!; + const previousSegmentId = this.previousSegmentId; - this.emit("committed", { itemId, previousItemId }); - this.emit("transcript", { itemId, transcript: finalText, isFinal: true }); + this.emit("committed", { segmentId, previousSegmentId }); + this.emit("transcript", { segmentId, transcript: finalText, isFinal: true }); - this.previousItemId = itemId; - this.currentItemId = uuidv4(); + this.previousSegmentId = segmentId; + this.currentSegmentId = uuidv4(); this.lastPartialText = ""; this.pcm16 = Buffer.alloc(0); } catch (err) { @@ -82,18 +83,18 @@ export class SherpaParakeetRealtimeTranscriptionSession return; } this.pcm16 = Buffer.alloc(0); - this.currentItemId = uuidv4(); + this.currentSegmentId = uuidv4(); this.lastPartialText = ""; } close(): void { this.connected = false; - this.currentItemId = null; + this.currentSegmentId = null; this.pcm16 = Buffer.alloc(0); } private async maybeDecode(force: boolean): Promise { - if (!this.connected || !this.currentItemId) { + if (!this.connected || !this.currentSegmentId) { return; } @@ -113,7 +114,7 @@ export class SherpaParakeetRealtimeTranscriptionSession this.lastDecodeAt = Date.now(); if (text !== this.lastPartialText) { this.lastPartialText = text; - this.emit("transcript", { itemId: this.currentItemId, transcript: text, isFinal: false }); + this.emit("transcript", { segmentId: this.currentSegmentId, transcript: text, isFinal: false }); } } finally { this.decoding = false; diff --git a/packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts similarity index 50% rename from packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts index d3b0c151f..3b23fa8e1 100644 --- a/packages/server/src/server/speech/sherpa/sherpa-parakeet-stt.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts @@ -1,8 +1,14 @@ +import { EventEmitter } from "node:events"; +import { v4 as uuidv4 } from "uuid"; import type pino from "pino"; -import type { SpeechToTextProvider, TranscriptionResult } from "../speech-provider.js"; -import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; -import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import type { + SpeechToTextProvider, + StreamingTranscriptionSession, + TranscriptionResult, +} from "../../../speech-provider.js"; +import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js"; +import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; export type SherpaParakeetSttConfig = { @@ -14,6 +20,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider { private readonly engine: SherpaOfflineRecognizerEngine; private readonly silencePeakThreshold: number; private readonly logger: pino.Logger; + public readonly id = "local" as const; constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) { this.engine = config.engine; @@ -21,7 +28,79 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider { this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" }); } - async transcribeAudio(audioBuffer: Buffer, format: string): Promise { + public createSession(params: { + logger: pino.Logger; + language?: string; + prompt?: string; + }): StreamingTranscriptionSession { + const emitter = new EventEmitter(); + const logger = params.logger.child({ provider: "local", component: "parakeet-stt-session" }); + const requiredSampleRate = this.engine.sampleRate; + let connected = false; + let segmentId = uuidv4(); + let previousSegmentId: string | null = null; + let pcm16: Buffer = Buffer.alloc(0); + + return { + requiredSampleRate, + async connect() { + connected = true; + }, + appendPcm16(chunk: Buffer) { + if (!connected) { + (emitter as any).emit("error", new Error("STT session not connected")); + return; + } + pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]); + }, + commit: () => { + if (!connected) { + (emitter as any).emit("error", new Error("STT session not connected")); + return; + } + + const committedId = segmentId; + const prev = previousSegmentId; + (emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev }); + + void (async () => { + try { + const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`); + (emitter as any).emit("transcript", { + segmentId: committedId, + transcript: rt.text, + isFinal: true, + language: rt.language, + logprobs: rt.logprobs, + avgLogprob: rt.avgLogprob, + isLowConfidence: rt.isLowConfidence, + }); + } catch (err) { + (emitter as any).emit("error", err); + } finally { + previousSegmentId = committedId; + segmentId = uuidv4(); + pcm16 = Buffer.alloc(0); + logger.debug({ bytes: pcm16.length }, "Parakeet session reset"); + } + })(); + }, + clear() { + pcm16 = Buffer.alloc(0); + segmentId = uuidv4(); + }, + close() { + connected = false; + pcm16 = Buffer.alloc(0); + }, + on(event: any, handler: any) { + emitter.on(event, handler); + return undefined; + }, + }; + } + + public async transcribeAudio(audioBuffer: Buffer, format: string): Promise { const start = Date.now(); let inputRate: number; diff --git a/packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts similarity index 70% rename from packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts index dec4fc297..6c6a251f9 100644 --- a/packages/server/src/server/speech/sherpa/sherpa-realtime-session.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts @@ -1,26 +1,28 @@ import { EventEmitter } from "node:events"; import { v4 as uuidv4 } from "uuid"; -import type { RealtimeTranscriptionSession } from "../../dictation/dictation-stream-manager.js"; -import { pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import type { StreamingTranscriptionSession } from "../../../speech-provider.js"; +import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js"; import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; export class SherpaRealtimeTranscriptionSession extends EventEmitter - implements RealtimeTranscriptionSession + implements StreamingTranscriptionSession { private readonly engine: SherpaOnlineRecognizerEngine; private stream: any | null = null; private connected = false; - private currentItemId: string | null = null; - private previousItemId: string | null = null; + public readonly requiredSampleRate: number; + private currentSegmentId: string | null = null; + private previousSegmentId: string | null = null; private lastPartialText = ""; private readonly tailPaddingMs: number; constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) { super(); this.engine = params.engine; + this.requiredSampleRate = this.engine.sampleRate; this.tailPaddingMs = params.tailPaddingMs ?? 500; } @@ -29,19 +31,18 @@ export class SherpaRealtimeTranscriptionSession return; } this.stream = this.engine.createStream(); - this.currentItemId = uuidv4(); + this.currentSegmentId = uuidv4(); this.connected = true; } - appendPcm16Base64(base64Audio: string): void { - if (!this.connected || !this.stream || !this.currentItemId) { + appendPcm16(pcm16le: Buffer): void { + if (!this.connected || !this.stream || !this.currentSegmentId) { this.emit("error", new Error("Sherpa realtime session not connected")); return; } try { - const pcm16 = Buffer.from(base64Audio, "base64"); - const peak = pcm16lePeakAbs(pcm16); + const peak = pcm16lePeakAbs(pcm16le); const peakFloat = peak / 32768.0; const targetPeak = 0.6; const maxGain = 50; @@ -49,7 +50,7 @@ export class SherpaRealtimeTranscriptionSession peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1; - const floatSamples = pcm16leToFloat32(pcm16, gain); + const floatSamples = pcm16leToFloat32(pcm16le, gain); this.stream.acceptWaveform(this.engine.sampleRate, floatSamples); while (this.engine.recognizer.isReady(this.stream)) { @@ -59,7 +60,11 @@ export class SherpaRealtimeTranscriptionSession const text = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim(); if (text !== this.lastPartialText) { this.lastPartialText = text; - this.emit("transcript", { itemId: this.currentItemId, transcript: text, isFinal: false }); + this.emit("transcript", { + segmentId: this.currentSegmentId, + transcript: text, + isFinal: false, + }); } } catch (err) { this.emit("error", err instanceof Error ? err : new Error(String(err))); @@ -67,7 +72,7 @@ export class SherpaRealtimeTranscriptionSession } commit(): void { - if (!this.connected || !this.stream || !this.currentItemId) { + if (!this.connected || !this.stream || !this.currentSegmentId) { this.emit("error", new Error("Sherpa realtime session not connected")); return; } @@ -83,14 +88,14 @@ export class SherpaRealtimeTranscriptionSession } const finalText = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim(); - const itemId = this.currentItemId; - const previousItemId = this.previousItemId; + const segmentId = this.currentSegmentId; + const previousSegmentId = this.previousSegmentId; - this.emit("committed", { itemId, previousItemId }); - this.emit("transcript", { itemId, transcript: finalText, isFinal: true }); + this.emit("committed", { segmentId, previousSegmentId }); + this.emit("transcript", { segmentId, transcript: finalText, isFinal: true }); - this.previousItemId = itemId; - this.currentItemId = uuidv4(); + this.previousSegmentId = segmentId; + this.currentSegmentId = uuidv4(); this.lastPartialText = ""; this.engine.recognizer.reset(this.stream); } catch (err) { @@ -104,7 +109,7 @@ export class SherpaRealtimeTranscriptionSession } try { this.engine.recognizer.reset(this.stream); - this.currentItemId = uuidv4(); + this.currentSegmentId = uuidv4(); this.lastPartialText = ""; } catch (err) { this.emit("error", err instanceof Error ? err : new Error(String(err))); diff --git a/packages/server/src/server/speech/sherpa/sherpa-stt.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts similarity index 55% rename from packages/server/src/server/speech/sherpa/sherpa-stt.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts index 863b4270d..ff308f51d 100644 --- a/packages/server/src/server/speech/sherpa/sherpa-stt.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts @@ -1,8 +1,14 @@ +import { EventEmitter } from "node:events"; +import { v4 as uuidv4 } from "uuid"; import type pino from "pino"; -import type { SpeechToTextProvider, TranscriptionResult } from "../speech-provider.js"; -import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js"; -import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../audio.js"; +import type { + SpeechToTextProvider, + StreamingTranscriptionSession, + TranscriptionResult, +} from "../../../speech-provider.js"; +import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js"; +import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js"; import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; export type SherpaSttConfig = { @@ -16,6 +22,7 @@ export class SherpaOnnxSTT implements SpeechToTextProvider { private readonly silencePeakThreshold: number; private readonly tailPaddingMs: number; private readonly logger: pino.Logger; + public readonly id = "local" as const; constructor(config: SherpaSttConfig, logger: pino.Logger) { this.engine = config.engine; @@ -24,7 +31,78 @@ export class SherpaOnnxSTT implements SpeechToTextProvider { this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" }); } - async transcribeAudio(audioBuffer: Buffer, format: string): Promise { + public createSession(params: { + logger: pino.Logger; + language?: string; + prompt?: string; + }): StreamingTranscriptionSession { + const emitter = new EventEmitter(); + void params; + const requiredSampleRate = this.engine.sampleRate; + let connected = false; + let segmentId = uuidv4(); + let previousSegmentId: string | null = null; + let pcm16: Buffer = Buffer.alloc(0); + + return { + requiredSampleRate, + async connect() { + connected = true; + }, + appendPcm16(chunk: Buffer) { + if (!connected) { + (emitter as any).emit("error", new Error("STT session not connected")); + return; + } + pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]); + }, + commit: () => { + if (!connected) { + (emitter as any).emit("error", new Error("STT session not connected")); + return; + } + + const committedId = segmentId; + const prev = previousSegmentId; + (emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev }); + + void (async () => { + try { + const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`); + (emitter as any).emit("transcript", { + segmentId: committedId, + transcript: rt.text, + isFinal: true, + language: rt.language, + logprobs: rt.logprobs, + avgLogprob: rt.avgLogprob, + isLowConfidence: rt.isLowConfidence, + }); + } catch (err) { + (emitter as any).emit("error", err); + } finally { + previousSegmentId = committedId; + segmentId = uuidv4(); + pcm16 = Buffer.alloc(0); + } + })(); + }, + clear() { + pcm16 = Buffer.alloc(0); + segmentId = uuidv4(); + }, + close() { + connected = false; + pcm16 = Buffer.alloc(0); + }, + on(event: any, handler: any) { + emitter.on(event, handler); + return undefined; + }, + }; + } + + public async transcribeAudio(audioBuffer: Buffer, format: string): Promise { const start = Date.now(); let inputRate: number; diff --git a/packages/server/src/server/speech/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts similarity index 97% rename from packages/server/src/server/speech/sherpa/sherpa-tts.ts rename to packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts index 2b5482248..99d9dab6a 100644 --- a/packages/server/src/server/speech/sherpa/sherpa-tts.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts @@ -2,8 +2,8 @@ import type pino from "pino"; import { Readable } from "node:stream"; import { existsSync } from "node:fs"; -import type { SpeechStreamResult, TextToSpeechProvider } from "../speech-provider.js"; -import { chunkBuffer, float32ToPcm16le } from "../audio.js"; +import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js"; +import { chunkBuffer, float32ToPcm16le } from "../../../audio.js"; import { loadSherpaOnnx } from "./sherpa-onnx-loader.js"; export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16"; diff --git a/packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts similarity index 97% rename from packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts rename to packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index da828a29d..8d66efdc5 100644 --- a/packages/server/src/server/speech/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -6,8 +6,8 @@ import pino from "pino"; import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; import type { SherpaOnnxModelId } from "./model-catalog.js"; -import { createDaemonTestContext } from "../../test-utils/index.js"; -import { parsePcm16MonoWav, wordSimilarity } from "../../test-utils/dictation-e2e.js"; +import { createDaemonTestContext } from "../../../../test-utils/index.js"; +import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js"; import { SherpaOnnxTTS } from "./sherpa-tts.js"; import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js"; import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js"; @@ -114,9 +114,9 @@ describe("speech models (download E2E)", () => { paseoHomeRoot, dictationFinalTimeoutMs: 8000, speech: { - dictationSttProvider: "sherpa", - voiceSttProvider: "sherpa", - voiceTtsProvider: "sherpa", + dictationSttProvider: "local", + voiceSttProvider: "local", + voiceTtsProvider: "local", sherpaOnnx: { modelsDir, autoDownload: false, diff --git a/packages/server/src/server/agent/openai-realtime-transcription.ts b/packages/server/src/server/speech/providers/openai/realtime-transcription-session.ts similarity index 91% rename from packages/server/src/server/agent/openai-realtime-transcription.ts rename to packages/server/src/server/speech/providers/openai/realtime-transcription-session.ts index a7beaced8..c4854f256 100644 --- a/packages/server/src/server/agent/openai-realtime-transcription.ts +++ b/packages/server/src/server/speech/providers/openai/realtime-transcription-session.ts @@ -1,6 +1,7 @@ import type pino from "pino"; import WebSocket from "ws"; import { EventEmitter } from "node:events"; +import type { StreamingTranscriptionSession } from "../../speech-provider.js"; type OpenAITurnDetection = | null @@ -60,7 +61,11 @@ type OpenAIServerEvent = } | { type: "error"; error?: { message?: string } }; -export class OpenAIRealtimeTranscriptionSession extends EventEmitter { +export class OpenAIRealtimeTranscriptionSession + extends EventEmitter + implements StreamingTranscriptionSession +{ + public readonly requiredSampleRate = 24000; private readonly apiKey: string; private readonly logger: pino.Logger; private readonly transcriptionModel: string; @@ -161,8 +166,8 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter { if (event.type === "input_audio_buffer.committed") { this.emit("committed", { - itemId: event.item_id, - previousItemId: event.previous_item_id, + segmentId: event.item_id, + previousSegmentId: event.previous_item_id, }); return; } @@ -182,13 +187,13 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter { const prev = this.partialByItemId.get(event.item_id) ?? ""; const next = replaceDelta ? event.delta : prev + event.delta; this.partialByItemId.set(event.item_id, next); - this.emit("transcript", { itemId: event.item_id, transcript: next, isFinal: false }); + this.emit("transcript", { segmentId: event.item_id, transcript: next, isFinal: false }); return; } if (event.type === "conversation.item.input_audio_transcription.completed") { this.partialByItemId.set(event.item_id, event.transcript); - this.emit("transcript", { itemId: event.item_id, transcript: event.transcript, isFinal: true }); + this.emit("transcript", { segmentId: event.item_id, transcript: event.transcript, isFinal: true }); return; } @@ -218,10 +223,11 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter { return this.ready; } - public appendPcm16Base64(base64Audio: string): void { + public appendPcm16(pcm16le: Buffer): void { if (!this.ws || this.ws.readyState !== WebSocket.OPEN) { throw new Error("OpenAI realtime websocket not connected"); } + const base64Audio = pcm16le.toString("base64"); const event: OpenAIClientEvent = { type: "input_audio_buffer.append", audio: base64Audio }; this.ws.send(JSON.stringify(event)); } diff --git a/packages/server/src/server/speech/providers/openai/stt.ts b/packages/server/src/server/speech/providers/openai/stt.ts new file mode 100644 index 000000000..f9ffab062 --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/stt.ts @@ -0,0 +1,269 @@ +import { EventEmitter } from "node:events"; +import type pino from "pino"; +import OpenAI from "openai"; +import { writeFile, unlink } from "fs/promises"; +import { join } from "path"; +import { tmpdir } from "os"; +import { v4 } from "uuid"; +import { inferAudioExtension } from "../../../agent/audio-utils.js"; +import type { + LogprobToken, + SpeechToTextProvider, + StreamingTranscriptionSession, + TranscriptionResult, +} from "../../speech-provider.js"; + +export type { LogprobToken, TranscriptionResult }; + +export interface STTConfig { + apiKey: string; + model?: "whisper-1" | "gpt-4o-transcribe" | "gpt-4o-mini-transcribe" | (string & {}); + confidenceThreshold?: number; // Default: -3.0 +} + +function isObject(value: unknown): value is { [key: string]: unknown } { + return typeof value === "object" && value !== null; +} + +function isLogprobToken(value: unknown): value is LogprobToken { + if (!isObject(value)) { + return false; + } + if (typeof value.token !== "string") { + return false; + } + if (typeof value.logprob !== "number") { + return false; + } + if (value.bytes === undefined) { + return true; + } + return Array.isArray(value.bytes) && value.bytes.every((entry) => typeof entry === "number"); +} + +function isLogprobTokenArray(value: unknown): value is LogprobToken[] { + return Array.isArray(value) && value.every((entry) => isLogprobToken(entry)); +} + +export class OpenAISTT implements SpeechToTextProvider { + private readonly openaiClient: OpenAI; + private readonly config: STTConfig; + private readonly logger: pino.Logger; + public readonly id = "openai" as const; + + constructor(sttConfig: STTConfig, parentLogger: pino.Logger) { + this.config = sttConfig; + this.logger = parentLogger.child({ module: "agent", provider: "openai", component: "stt" }); + this.openaiClient = new OpenAI({ + apiKey: sttConfig.apiKey, + }); + this.logger.info({ model: sttConfig.model || "whisper-1" }, "STT (OpenAI Whisper) initialized"); + } + + public createSession(params: { + logger: pino.Logger; + language?: string; + prompt?: string; + }): StreamingTranscriptionSession { + const emitter = new EventEmitter(); + const logger = params.logger.child({ provider: "openai", component: "stt-session" }); + const requiredSampleRate = 24000; + + let connected = false; + let segmentId = v4(); + let previousSegmentId: string | null = null; + let pcm16: Buffer = Buffer.alloc(0); + const transcribeAudio = this.transcribeAudioInternal.bind(this); + + const convertPCMToWavBuffer = (pcmBuffer: Buffer): Buffer => { + const headerSize = 44; + const channels = 1; + const bitsPerSample = 16; + const sampleRate = requiredSampleRate; + const wavBuffer = Buffer.alloc(headerSize + pcmBuffer.length); + const byteRate = (sampleRate * channels * bitsPerSample) / 8; + const blockAlign = (channels * bitsPerSample) / 8; + + wavBuffer.write("RIFF", 0); + wavBuffer.writeUInt32LE(36 + pcmBuffer.length, 4); + wavBuffer.write("WAVE", 8); + wavBuffer.write("fmt ", 12); + wavBuffer.writeUInt32LE(16, 16); + wavBuffer.writeUInt16LE(1, 20); + wavBuffer.writeUInt16LE(channels, 22); + wavBuffer.writeUInt32LE(sampleRate, 24); + wavBuffer.writeUInt32LE(byteRate, 28); + wavBuffer.writeUInt16LE(blockAlign, 32); + wavBuffer.writeUInt16LE(bitsPerSample, 34); + wavBuffer.write("data", 36); + wavBuffer.writeUInt32LE(pcmBuffer.length, 40); + pcmBuffer.copy(wavBuffer, 44); + + return wavBuffer; + }; + + return { + requiredSampleRate, + async connect() { + connected = true; + }, + appendPcm16(chunk: Buffer) { + if (!connected) { + (emitter as any).emit("error", new Error("STT session not connected")); + return; + } + pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]); + }, + commit() { + if (!connected) { + (emitter as any).emit("error", new Error("STT session not connected")); + return; + } + + const committedId = segmentId; + const prev = previousSegmentId; + (emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev }); + + void (async () => { + try { + if (pcm16.length === 0) { + (emitter as any).emit("transcript", { + segmentId: committedId, + transcript: "", + isFinal: true, + language: params.language, + isLowConfidence: true, + }); + return; + } + + const wav = convertPCMToWavBuffer(pcm16); + const result = await transcribeAudio( + wav, + "audio/wav", + params.language ?? "en", + logger + ); + + (emitter as any).emit("transcript", { + segmentId: committedId, + transcript: result.text, + isFinal: true, + language: result.language, + logprobs: result.logprobs, + avgLogprob: result.avgLogprob, + isLowConfidence: result.isLowConfidence, + }); + } catch (err) { + (emitter as any).emit("error", err); + } finally { + previousSegmentId = committedId; + segmentId = v4(); + pcm16 = Buffer.alloc(0); + } + })(); + }, + clear() { + pcm16 = Buffer.alloc(0); + segmentId = v4(); + }, + close() { + connected = false; + pcm16 = Buffer.alloc(0); + }, + on(event: any, handler: any) { + emitter.on(event, handler); + return undefined; + }, + }; + } + + private async transcribeAudioInternal( + audioBuffer: Buffer, + format: string, + language: string, + logger: pino.Logger + ): Promise { + const startTime = Date.now(); + let tempFilePath: string | null = null; + + try { + const ext = inferAudioExtension(format); + tempFilePath = join(tmpdir(), `audio-${v4()}.${ext}`); + await writeFile(tempFilePath, audioBuffer); + + logger.debug( + { tempFilePath, bytes: audioBuffer.length }, + "Transcribing audio file" + ); + + const modelToUse = this.config.model ?? "whisper-1"; + const supportsLogprobs = + modelToUse === "gpt-4o-transcribe" || modelToUse === "gpt-4o-mini-transcribe"; + const includeLogprobs: ["logprobs"] = ["logprobs"]; + + const response = await this.openaiClient.audio.transcriptions.create({ + file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)), + language, + model: modelToUse, + ...(supportsLogprobs ? { include: includeLogprobs } : {}), + response_format: "json", + }); + + const duration = Date.now() - startTime; + const confidenceThreshold = this.config.confidenceThreshold ?? -3.0; + + let avgLogprob: number | undefined; + let isLowConfidence = false; + const logprobs = + supportsLogprobs && + isObject(response) && + isLogprobTokenArray(response.logprobs) + ? response.logprobs + : undefined; + + if (logprobs && logprobs.length > 0) { + const totalLogprob = logprobs.reduce((sum, token) => sum + token.logprob, 0); + avgLogprob = totalLogprob / logprobs.length; + isLowConfidence = avgLogprob < confidenceThreshold; + + if (isLowConfidence) { + logger.debug( + { + avgLogprob, + threshold: confidenceThreshold, + text: response.text, + tokenLogprobs: logprobs.map((t) => `${t.token}:${t.logprob.toFixed(2)}`).join(", "), + }, + "Low confidence transcription detected" + ); + } + } + + logger.debug({ duration, text: response.text, avgLogprob }, "Transcription complete"); + + return { + text: response.text, + duration: duration, + logprobs: logprobs, + avgLogprob: avgLogprob, + isLowConfidence: isLowConfidence, + language: + isObject(response) && typeof response.language === "string" + ? response.language + : undefined, + }; + } catch (error: any) { + logger.error({ err: error }, "Transcription error"); + throw new Error(`STT transcription failed: ${error.message}`); + } finally { + if (tempFilePath) { + try { + await unlink(tempFilePath); + } catch (cleanupError) { + logger.warn({ tempFilePath }, "Failed to clean up temp file"); + } + } + } + } +} diff --git a/packages/server/src/server/agent/tts-openai.ts b/packages/server/src/server/speech/providers/openai/tts.ts similarity index 93% rename from packages/server/src/server/agent/tts-openai.ts rename to packages/server/src/server/speech/providers/openai/tts.ts index bd96a80b4..7333f3126 100644 --- a/packages/server/src/server/agent/tts-openai.ts +++ b/packages/server/src/server/speech/providers/openai/tts.ts @@ -1,7 +1,7 @@ import type pino from "pino"; import OpenAI from "openai"; import { Readable } from "node:stream"; -import type { SpeechStreamResult } from "../speech/speech-provider.js"; +import type { SpeechStreamResult, TextToSpeechProvider } from "../../speech-provider.js"; export type { SpeechStreamResult }; @@ -12,7 +12,7 @@ export interface TTSConfig { responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm"; } -export class OpenAITTS { +export class OpenAITTS implements TextToSpeechProvider { private readonly openaiClient: OpenAI; private readonly config: TTSConfig; private readonly logger: pino.Logger; diff --git a/packages/server/src/server/speech/speech-provider.ts b/packages/server/src/server/speech/speech-provider.ts index 3ed103666..fb58ba910 100644 --- a/packages/server/src/server/speech/speech-provider.ts +++ b/packages/server/src/server/speech/speech-provider.ts @@ -1,3 +1,4 @@ +import type pino from "pino"; import type { Readable } from "node:stream"; export interface LogprobToken { @@ -15,8 +16,52 @@ export interface TranscriptionResult { isLowConfidence?: boolean; } +export interface StreamingTranscriptionCommittedEvent { + segmentId: string; + previousSegmentId: string | null; +} + +export interface StreamingTranscriptionEvent { + segmentId: string; + transcript: string; + isFinal: boolean; + language?: string; + logprobs?: LogprobToken[]; + avgLogprob?: number; + isLowConfidence?: boolean; +} + +export type StreamingTranscriptionSession = { + /** + * Required PCM16LE sample rate for `appendPcm16()`. + * Callers are responsible for resampling before appending. + */ + requiredSampleRate: number; + + connect(): Promise; + appendPcm16(pcm16le: Buffer): void; + commit(): void; + clear(): void; + close(): void; + + on( + event: "committed", + handler: (payload: StreamingTranscriptionCommittedEvent) => void + ): unknown; + on( + event: "transcript", + handler: (payload: StreamingTranscriptionEvent) => void + ): unknown; + on(event: "error", handler: (err: unknown) => void): unknown; +}; + export interface SpeechToTextProvider { - transcribeAudio(audioBuffer: Buffer, format: string): Promise; + id: "openai" | "local" | (string & {}); + createSession(params: { + logger: pino.Logger; + language?: string; + prompt?: string; + }): StreamingTranscriptionSession; } export interface SpeechStreamResult { @@ -27,4 +72,3 @@ export interface SpeechStreamResult { export interface TextToSpeechProvider { synthesizeSpeech(text: string): Promise; } - diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index b7e84f4bf..4d347694e 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -21,7 +21,6 @@ import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; import { VoiceConversationStore } from "./voice-conversation-store.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; -import type { RealtimeTranscriptionSessionFactory } from "./dictation/dictation-stream-manager.js"; export type AgentMcpTransportFactory = () => Promise; @@ -72,9 +71,8 @@ export class VoiceAssistantWebSocketServer { private readonly terminalManager: TerminalManager | null; private readonly voiceConversationStore: VoiceConversationStore; private readonly dictation: { - openaiApiKey?: string | null; finalTimeoutMs?: number; - sessionFactory?: RealtimeTranscriptionSessionFactory; + stt?: SpeechToTextProvider | null; } | null; private readonly voice: { openrouterApiKey?: string | null; @@ -98,9 +96,8 @@ export class VoiceAssistantWebSocketServer { voiceLlmModel?: string | null; }, dictation?: { - openaiApiKey?: string | null; finalTimeoutMs?: number; - sessionFactory?: RealtimeTranscriptionSessionFactory; + stt?: SpeechToTextProvider | null; } ) { this.logger = logger.child({ module: "websocket-server" }); diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index f53783f8b..a8f2a3d03 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -489,6 +489,17 @@ export const ListProviderModelsRequestMessageSchema = z.object({ requestId: z.string(), }); +export const SpeechModelsListRequestSchema = z.object({ + type: z.literal("speech_models_list_request"), + requestId: z.string(), +}); + +export const SpeechModelsDownloadRequestSchema = z.object({ + type: z.literal("speech_models_download_request"), + modelIds: z.array(z.string()).optional(), + requestId: z.string(), +}); + export const ResumeAgentRequestMessageSchema = z.object({ type: z.literal("resume_agent_request"), handle: AgentPersistenceHandleSchema, @@ -887,6 +898,8 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ DictationStreamCancelMessageSchema, CreateAgentRequestMessageSchema, ListProviderModelsRequestMessageSchema, + SpeechModelsListRequestSchema, + SpeechModelsDownloadRequestSchema, ResumeAgentRequestMessageSchema, RefreshAgentRequestMessageSchema, CancelAgentRequestMessageSchema, @@ -1518,6 +1531,34 @@ export const ListProviderModelsResponseMessageSchema = z.object({ }), }); +export const SpeechModelsListResponseSchema = z.object({ + type: z.literal("speech_models_list_response"), + payload: z.object({ + modelsDir: z.string(), + models: z.array( + z.object({ + id: z.string(), + kind: z.string(), + description: z.string(), + modelDir: z.string(), + isDownloaded: z.boolean(), + missingFiles: z.array(z.string()).optional(), + }) + ), + requestId: z.string(), + }), +}); + +export const SpeechModelsDownloadResponseSchema = z.object({ + type: z.literal("speech_models_download_response"), + payload: z.object({ + modelsDir: z.string(), + downloadedModelIds: z.array(z.string()), + error: z.string().nullable(), + requestId: z.string(), + }), +}); + const AgentSlashCommandSchema = z.object({ name: z.string(), description: z.string(), @@ -1673,6 +1714,8 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ ProjectIconResponseSchema, FileDownloadTokenResponseSchema, ListProviderModelsResponseMessageSchema, + SpeechModelsListResponseSchema, + SpeechModelsDownloadResponseSchema, ListCommandsResponseSchema, ExecuteCommandResponseSchema, ListTerminalsResponseSchema, @@ -1727,6 +1770,8 @@ export type AgentDeletedMessage = z.infer; export type ListProviderModelsResponseMessage = z.infer< typeof ListProviderModelsResponseMessageSchema >; +export type SpeechModelsListResponse = z.infer; +export type SpeechModelsDownloadResponse = z.infer; export type InitializeAgentResponseMessage = z.infer; // Type exports for payload types @@ -1747,6 +1792,10 @@ export type CreateAgentRequestMessage = z.infer; +export type SpeechModelsListRequestMessage = z.infer; +export type SpeechModelsDownloadRequestMessage = z.infer< + typeof SpeechModelsDownloadRequestSchema +>; export type ResumeAgentRequestMessage = z.infer; export type DeleteAgentRequestMessage = z.infer; export type InitializeAgentRequestMessage = z.infer; diff --git a/scripts/speech/download-sherpa-models.sh b/scripts/speech/download-sherpa-models.sh deleted file mode 100755 index 5a3ccdc5c..000000000 --- a/scripts/speech/download-sherpa-models.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'EOF' -Download local speech models for Paseo (sherpa-onnx). - -Defaults: - - STT: sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 - - TTS: kitten-nano-en-v0_1-fp16 - -Usage: - scripts/speech/download-sherpa-models.sh [--models-dir DIR] [--with-kokoro] [--with-paraformer] - -Preferred: - npm run speech:download --workspace=@getpaseo/server - -Notes: - - Models are downloaded from the sherpa-onnx GitHub releases. - - Pocket TTS is downloaded by the Node script (`npm run speech:download --workspace=@getpaseo/server`) - because it is a file-based HuggingFace model (not a single tarball). - - Set PASEO_SHERPA_ONNX_MODELS_DIR to override where the daemon looks. -EOF -} - -MODELS_DIR="" -WITH_KOKORO=0 -WITH_PARAFORMER=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --models-dir) - MODELS_DIR="${2:-}" - shift 2 - ;; - --with-kokoro) - WITH_KOKORO=1 - shift 1 - ;; - --with-paraformer) - WITH_PARAFORMER=1 - shift 1 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "Unknown arg: $1" >&2 - usage >&2 - exit 2 - ;; - esac -done - -if [[ -z "${MODELS_DIR}" ]]; then - if [[ -n "${PASEO_SHERPA_ONNX_MODELS_DIR:-}" ]]; then - MODELS_DIR="${PASEO_SHERPA_ONNX_MODELS_DIR}" - elif [[ -n "${PASEO_HOME:-}" ]]; then - MODELS_DIR="${PASEO_HOME}/models/sherpa-onnx" - else - MODELS_DIR="${HOME}/.paseo/models/sherpa-onnx" - fi -fi - -mkdir -p "${MODELS_DIR}" -cd "${MODELS_DIR}" - -download_and_extract() { - local url="$1" - local filename - filename="$(basename "$url")" - - echo "Downloading ${filename}..." - curl -fsSL -O "${url}" - echo "Extracting ${filename}..." - tar xf "${filename}" - rm -f "${filename}" -} - -echo "NOTE: This script is deprecated. Prefer: npm run speech:download --workspace=@getpaseo/server" >&2 - -download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2" -download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2" - -if [[ "${WITH_PARAFORMER}" -eq 1 ]]; then - download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2" -fi - -if [[ "${WITH_KOKORO}" -eq 1 ]]; then - download_and_extract "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2" -fi - -echo "Done." -echo "Models dir: ${MODELS_DIR}" From 1c6a1d1dd6d53362fb4f50f0075d2d84ed1edd4a Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 13:16:37 +0700 Subject: [PATCH 3/4] Remove unshipped sherpa provider compatibility aliases --- packages/server/src/server/config.ts | 4 +--- packages/server/src/server/persisted-config.ts | 6 +----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 84f4a10b4..adcc2e52f 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -85,9 +85,7 @@ function parseSpeechProviderId(value: unknown): "openai" | "local" | null { return null; } if (normalized === "openai") return "openai"; - if (normalized === "sherpa" || normalized === "sherpa-onnx" || normalized === "local") { - return "local"; - } + if (normalized === "local") return "local"; return null; } diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 59469bc3a..4b34cbbcd 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -51,11 +51,7 @@ const SpeechProviderIdSchema = z.preprocess( if (typeof value !== "string") { return value; } - const normalized = value.trim().toLowerCase(); - if (normalized === "sherpa" || normalized === "sherpa-onnx") { - return "local"; - } - return normalized; + return value.trim().toLowerCase(); }, z.enum(["openai", "local"]) ); From 4dc20f58d026a5acf9f74c48b668fcf3d3da5d55 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 13:33:26 +0700 Subject: [PATCH 4/4] Harden speech provider reconciliation and startup logging --- .../server/src/server/bootstrap.smoke.test.ts | 43 ++++++ packages/server/src/server/bootstrap.ts | 142 +++++++++++++++--- 2 files changed, 162 insertions(+), 23 deletions(-) diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 1d10a0016..65834a8b6 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -1,6 +1,12 @@ +import os from "node:os"; +import path from "node:path"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import pino from "pino"; import { describe, expect, test } from "vitest"; +import { createPaseoDaemon, type PaseoDaemonConfig } from "./bootstrap.js"; import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js"; +import { createTestAgentClients } from "./test-utils/fake-agent-client.js"; describe("paseo daemon bootstrap", () => { test("starts and serves health endpoint", async () => { @@ -22,4 +28,41 @@ describe("paseo daemon bootstrap", () => { await daemonHandle.close(); } }); + + test("fails fast when OpenAI speech provider is configured without credentials", async () => { + const paseoHomeRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-openai-config-")); + const paseoHome = path.join(paseoHomeRoot, ".paseo"); + const staticDir = await mkdtemp(path.join(os.tmpdir(), "paseo-static-")); + await mkdir(paseoHome, { recursive: true }); + + const config: PaseoDaemonConfig = { + listen: "127.0.0.1:0", + paseoHome, + corsAllowedOrigins: [], + allowedHosts: true, + mcpEnabled: false, + staticDir, + mcpDebug: false, + agentClients: createTestAgentClients(), + agentStoragePath: path.join(paseoHome, "agents"), + relayEnabled: false, + appBaseUrl: "https://app.paseo.sh", + openai: undefined, + speech: { + dictationSttProvider: "openai", + voiceSttProvider: "openai", + voiceTtsProvider: "openai", + }, + openrouterApiKey: null, + }; + + try { + await expect(createPaseoDaemon(config, pino({ level: "silent" }))).rejects.toThrow( + "Missing OpenAI credentials" + ); + } finally { + await rm(paseoHomeRoot, { recursive: true, force: true }); + await rm(staticDir, { recursive: true, force: true }); + } + }); }); diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 5dcfd16d3..b8343abfa 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -427,9 +427,68 @@ export async function createPaseoDaemon( const speechConfig = config.speech ?? null; const sherpaConfig = speechConfig?.sherpaOnnx ?? null; - const wantsLocalDictation = (speechConfig?.dictationSttProvider ?? "openai") === "local"; - const wantsLocalVoiceStt = (speechConfig?.voiceSttProvider ?? "openai") === "local"; - const wantsLocalVoiceTts = (speechConfig?.voiceTtsProvider ?? "openai") === "local"; + const voiceSttProvider = speechConfig?.voiceSttProvider ?? "local"; + const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "local"; + const dictationSttProvider = speechConfig?.dictationSttProvider ?? "local"; + + const wantsLocalDictation = dictationSttProvider === "local"; + const wantsLocalVoiceStt = voiceSttProvider === "local"; + const wantsLocalVoiceTts = voiceTtsProvider === "local"; + + const openaiSttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; + const openaiTtsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; + const openaiDictationApiKey = openaiApiKey; + + const missingOpenAiCredentialsFor: string[] = []; + if (voiceSttProvider === "openai" && !openaiSttApiKey) { + missingOpenAiCredentialsFor.push("voice.stt"); + } + if (voiceTtsProvider === "openai" && !openaiTtsApiKey) { + missingOpenAiCredentialsFor.push("voice.tts"); + } + if (dictationSttProvider === "openai" && !openaiDictationApiKey) { + missingOpenAiCredentialsFor.push("dictation.stt"); + } + + if (missingOpenAiCredentialsFor.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: dictationSttProvider, + voiceStt: voiceSttProvider, + voiceTts: voiceTtsProvider, + }, + missingOpenAiCredentialsFor, + }, + "Invalid speech configuration: OpenAI provider selected but credentials are missing" + ); + throw new Error( + `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` + ); + } + + logger.info( + { + requestedProviders: { + dictationStt: dictationSttProvider, + voiceStt: voiceSttProvider, + voiceTts: voiceTtsProvider, + }, + availability: { + openai: { + stt: Boolean(openaiSttApiKey), + tts: Boolean(openaiTtsApiKey), + dictationStt: Boolean(openaiDictationApiKey), + }, + local: { + configured: Boolean(sherpaConfig), + modelsDir: sherpaConfig?.modelsDir ?? null, + autoDownload: sherpaConfig?.autoDownload ?? null, + }, + }, + }, + "Speech provider reconciliation started" + ); if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) { const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true); @@ -468,6 +527,14 @@ export async function createPaseoDaemon( } try { + logger.info( + { + modelsDir: sherpaConfig.modelsDir, + modelIds, + autoDownload, + }, + "Ensuring local speech models" + ); await ensureSherpaOnnxModels({ modelsDir: sherpaConfig.modelsDir, modelIds, @@ -564,7 +631,7 @@ export async function createPaseoDaemon( } else if (wantsLocalDictation || wantsLocalVoiceStt) { logger.warn( { configured: Boolean(sherpaConfig) }, - "Sherpa STT selected but no sherpaOnnx config found; STT will be unavailable" + "Local STT selected but local provider config is missing; STT will be unavailable" ); } @@ -618,7 +685,7 @@ export async function createPaseoDaemon( } else if (wantsLocalVoiceTts) { logger.warn( { configured: Boolean(sherpaConfig) }, - "Sherpa TTS selected but no sherpaOnnx config found; TTS will be unavailable" + "Local TTS selected but local provider config is missing; TTS will be unavailable" ); } @@ -645,31 +712,29 @@ export async function createPaseoDaemon( }; } - const voiceSttProvider = speechConfig?.voiceSttProvider ?? "openai"; - const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "openai"; - const dictationSttProvider = speechConfig?.dictationSttProvider ?? "openai"; - const needsOpenAiStt = !sttService && voiceSttProvider === "openai"; const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai"; const needsOpenAiDictation = dictationSttProvider === "openai" || (dictationSttProvider === "local" && !dictationSttService); - const fallbackOpenAiStt = !sttService && voiceSttProvider === "local" && Boolean(openaiApiKey); - const fallbackOpenAiTts = !ttsService && voiceTtsProvider === "local" && Boolean(openaiApiKey); + const fallbackOpenAiStt = !sttService && voiceSttProvider === "local" && Boolean(openaiSttApiKey); + const fallbackOpenAiTts = !ttsService && voiceTtsProvider === "local" && Boolean(openaiTtsApiKey); - if ((needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) && openaiApiKey) { - logger.info("OpenAI client initialized"); + if ( + (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) && + (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey) + ) { + logger.info("OpenAI speech provider initialized"); if (fallbackOpenAiStt) { - logger.warn("Falling back to OpenAI STT because Sherpa STT is unavailable"); + logger.warn("Falling back to OpenAI STT because local STT is unavailable"); } if (needsOpenAiStt || fallbackOpenAiStt) { - const sttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; - if (sttApiKey) { + if (openaiSttApiKey) { const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {}; sttService = new OpenAISTT( { - apiKey: sttApiKey, + apiKey: openaiSttApiKey, ...sttConfig, }, logger @@ -678,15 +743,14 @@ export async function createPaseoDaemon( } if (fallbackOpenAiTts) { - logger.warn("Falling back to OpenAI TTS because Sherpa TTS is unavailable"); + logger.warn("Falling back to OpenAI TTS because local TTS is unavailable"); } if (needsOpenAiTts || fallbackOpenAiTts) { - const ttsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; - if (ttsApiKey) { + if (openaiTtsApiKey) { const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {}; ttsService = new OpenAITTS( { - apiKey: ttsApiKey, + apiKey: openaiTtsApiKey, voice: "alloy", model: "tts-1", responseFormat: "pcm", @@ -698,7 +762,6 @@ export async function createPaseoDaemon( } if (needsOpenAiDictation) { - const dictationApiKey = config.openai?.apiKey ?? openaiApiKey; const transcriptionModel = process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe"; @@ -706,7 +769,7 @@ export async function createPaseoDaemon( id: "openai", createSession: ({ logger: sessionLogger, language, prompt }) => new OpenAIRealtimeTranscriptionSession({ - apiKey: dictationApiKey, + apiKey: openaiDictationApiKey!, logger: sessionLogger, transcriptionModel, ...(language ? { language } : {}), @@ -719,6 +782,39 @@ export async function createPaseoDaemon( logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation fallback is unavailable"); } + const effectiveProviders = { + dictationStt: dictationSttService?.id ?? "unavailable", + voiceStt: sttService?.id ?? "unavailable", + voiceTts: !ttsService ? "unavailable" : ttsService === sherpaTts ? "local" : "openai", + }; + const unavailableFeatures = [ + !dictationSttService ? "dictation.stt" : null, + !sttService ? "voice.stt" : null, + !ttsService ? "voice.tts" : null, + ].filter((feature): feature is string => feature !== null); + + if (unavailableFeatures.length > 0) { + logger.warn( + { + requestedProviders: { + dictationStt: dictationSttProvider, + voiceStt: voiceSttProvider, + voiceTts: voiceTtsProvider, + }, + effectiveProviders, + unavailableFeatures, + }, + "Speech provider reconciliation completed with unavailable features" + ); + } else { + logger.info( + { + effectiveProviders, + }, + "Speech provider reconciliation completed" + ); + } + const wsServer = new VoiceAssistantWebSocketServer( httpServer, logger,