Merge pull request #19 from boudra/local-streaming-speech-tts

Local streaming STT/TTS (Parakeet v3 + Pocket TTS)
2026-07-29 12:01:31 +00:00 · 2026-02-06 13:45:40 +07:00
parent c343931fec 4d28499482
commit 8889f6f05c
48 changed files with 4586 additions and 615 deletions
--- a/README.md
+++ b/README.md
@@ -32,6 +32,37 @@ npm install -g @getpaseo/cli && paseo

 Then open the app and connect to your daemon.

+## Local speech (STT/TTS)
+
+Paseo can run dictation + voice mode STT/TTS fully locally via `sherpa-onnx`.
+
+When the daemon starts with a local speech provider selected, it will download any missing model files automatically (unless `PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=0`).
+
+```bash
+npm run speech:download --workspace=@getpaseo/server
+```
+
+Optional configuration:
+
+- `PASEO_SHERPA_ONNX_MODELS_DIR` (defaults to `~/.paseo/models/sherpa-onnx`)
+- `PASEO_SHERPA_ONNX_AUTO_DOWNLOAD` (`1` by default; set `0` to disable automatic downloads on daemon start)
+- `PASEO_SHERPA_STT_PRESET` (`zipformer`, `paraformer`, or `parakeet` for NVIDIA Parakeet TDT v3)
+- `PASEO_SHERPA_TTS_PRESET` (`pocket-tts` (Kyutai Pocket TTS), `kitten`, or `kokoro`)
+- `PASEO_DICTATION_STT_PROVIDER`, `PASEO_VOICE_STT_PROVIDER`, `PASEO_VOICE_TTS_PROVIDER` (`sherpa` or `openai`)
+
+To see all supported local model IDs:
+
+```bash
+npm run speech:models --workspace=@getpaseo/server
+```
+
+Optional: run an end-to-end test that downloads real models and exercises streaming STT + streaming TTS:
+
+```bash
+PASEO_SPEECH_E2E_DOWNLOAD=1 PASEO_SPEECH_E2E_MODEL_SET=parakeet-pocket \
+  npx vitest run --workspace=@getpaseo/server src/server/speech/sherpa/speech-download.e2e.test.ts
+```
+
 ## Documentation

 See [paseo.sh/docs](https://paseo.sh/docs) for full documentation.
--- a/package-lock.json
+++ b/package-lock.json
@@ -5621,6 +5621,16 @@
      "dev": true,
      "license": "MIT"
    },
+    "node_modules/@sctg/sentencepiece-js": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/@sctg/sentencepiece-js/-/sentencepiece-js-1.3.3.tgz",
+      "integrity": "sha512-iPnzR2HGjdQQG2SpTyPH3wnnpgQ2aS14B3I2jHjomkoaMprM3Sn+WOQblHwByetnvn3n14y3IwjwogVUHA2cVA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "app-root-path": "^3.1.0",
+        "buffer": "^6.0.3"
+      }
+    },
    "node_modules/@segment/loosely-validate-event": {
      "version": "2.0.0",
      "dev": true,
@@ -5739,9 +5749,6 @@
      "cpu": [
        "x64"
      ],
-      "libc": [
-        "glibc"
-      ],
      "dev": true,
      "license": "MIT",
      "optional": true,
@@ -6995,6 +7002,15 @@
        "node": ">=0.4.0"
      }
    },
+    "node_modules/adm-zip": {
+      "version": "0.5.16",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz",
+      "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=12.0"
+      }
+    },
    "node_modules/agent-base": {
      "version": "6.0.2",
      "dev": true,
@@ -7157,6 +7173,15 @@
        "url": "https://github.com/sponsors/jonschlinkert"
      }
    },
+    "node_modules/app-root-path": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-3.1.0.tgz",
+      "integrity": "sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 6.0.0"
+      }
+    },
    "node_modules/arg": {
      "version": "5.0.2",
      "license": "MIT"
@@ -7791,6 +7816,13 @@
      "version": "1.0.0",
      "license": "ISC"
    },
+    "node_modules/boolean": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz",
+      "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==",
+      "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
+      "license": "MIT"
+    },
    "node_modules/bplist-creator": {
      "version": "0.1.0",
      "license": "MIT",
@@ -8879,6 +8911,12 @@
        "node": ">=8"
      }
    },
+    "node_modules/detect-node": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz",
+      "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==",
+      "license": "MIT"
+    },
    "node_modules/detect-node-es": {
      "version": "1.1.0",
      "license": "MIT"
@@ -9741,6 +9779,12 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/es6-error": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz",
+      "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==",
+      "license": "MIT"
+    },
    "node_modules/esbuild": {
      "version": "0.25.11",
      "hasInstallScript": true,
@@ -11752,6 +11796,62 @@
      "version": "0.4.1",
      "license": "BSD-2-Clause"
    },
+    "node_modules/global-agent": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz",
+      "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "boolean": "^3.0.1",
+        "es6-error": "^4.1.1",
+        "matcher": "^3.0.0",
+        "roarr": "^2.15.3",
+        "semver": "^7.3.2",
+        "serialize-error": "^7.0.1"
+      },
+      "engines": {
+        "node": ">=10.0"
+      }
+    },
+    "node_modules/global-agent/node_modules/semver": {
+      "version": "7.7.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz",
+      "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==",
+      "license": "ISC",
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/global-agent/node_modules/serialize-error": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz",
+      "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==",
+      "license": "MIT",
+      "dependencies": {
+        "type-fest": "^0.13.1"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/global-agent/node_modules/type-fest": {
+      "version": "0.13.1",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",
+      "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==",
+      "license": "(MIT OR CC0-1.0)",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
    "node_modules/global-dirs": {
      "version": "0.1.1",
      "license": "MIT",
@@ -11775,7 +11875,6 @@
    },
    "node_modules/globalthis": {
      "version": "1.0.4",
-      "dev": true,
      "license": "MIT",
      "dependencies": {
        "define-properties": "^1.2.1",
@@ -13075,6 +13174,12 @@
      "dev": true,
      "license": "MIT"
    },
+    "node_modules/json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
+      "license": "ISC"
+    },
    "node_modules/json5": {
      "version": "2.2.3",
      "license": "MIT",
@@ -13397,9 +13502,6 @@
      "cpu": [
        "x64"
      ],
-      "libc": [
-        "glibc"
-      ],
      "license": "MPL-2.0",
      "optional": true,
      "os": [
@@ -13560,6 +13662,18 @@
      "version": "1.3.0",
      "license": "Apache-2.0"
    },
+    "node_modules/matcher": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
+      "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==",
+      "license": "MIT",
+      "dependencies": {
+        "escape-string-regexp": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
    "node_modules/math-intrinsics": {
      "version": "1.1.0",
      "license": "MIT",
@@ -14755,6 +14869,29 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/onnxruntime-common": {
+      "version": "1.24.1",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.1.tgz",
+      "integrity": "sha512-UnV15u4p4XxoIV+jFP4hXPsW93s3QrwLSpi20HUDYHoTfI4z4sjzex3L4XDOxGGZJ/M/catrwAG2go958UQq0w==",
+      "license": "MIT"
+    },
+    "node_modules/onnxruntime-node": {
+      "version": "1.24.1",
+      "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.1.tgz",
+      "integrity": "sha512-Ex/oUXKdhDoxvlNxBT3oYtW0MH88yYpPlXQeVQUXpcJQmN24usd/8RCoPLN5kCHwDsiZ+nqsnjciyFRl423dQw==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "os": [
+        "win32",
+        "darwin",
+        "linux"
+      ],
+      "dependencies": {
+        "adm-zip": "^0.5.16",
+        "global-agent": "^3.0.0",
+        "onnxruntime-common": "1.24.1"
+      }
+    },
    "node_modules/open": {
      "version": "8.4.2",
      "license": "MIT",
@@ -16785,6 +16922,29 @@
        "node": "*"
      }
    },
+    "node_modules/roarr": {
+      "version": "2.15.4",
+      "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
+      "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "boolean": "^3.0.1",
+        "detect-node": "^2.0.4",
+        "globalthis": "^1.0.1",
+        "json-stringify-safe": "^5.0.1",
+        "semver-compare": "^1.0.0",
+        "sprintf-js": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=8.0"
+      }
+    },
+    "node_modules/roarr/node_modules/sprintf-js": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
+      "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
+      "license": "BSD-3-Clause"
+    },
    "node_modules/rollup": {
      "version": "4.52.5",
      "license": "MIT",
@@ -16992,6 +17152,12 @@
        "semver": "bin/semver.js"
      }
    },
+    "node_modules/semver-compare": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
+      "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==",
+      "license": "MIT"
+    },
    "node_modules/send": {
      "version": "0.19.1",
      "license": "MIT",
@@ -17312,6 +17478,91 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/sherpa-onnx": {
+      "version": "1.12.23",
+      "resolved": "https://registry.npmjs.org/sherpa-onnx/-/sherpa-onnx-1.12.23.tgz",
+      "integrity": "sha512-QwgDwSIXx4DgbBE7h7VUXEVLO2vf7w3aJX3ng6WoTAcfB2NW7So4Zp4GtGemwgxZ6sfEh02JEVXUidejkX7V0A==",
+      "license": "Apache-2.0"
+    },
+    "node_modules/sherpa-onnx-darwin-arm64": {
+      "version": "1.12.23",
+      "resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-arm64/-/sherpa-onnx-darwin-arm64-1.12.23.tgz",
+      "integrity": "sha512-zbjNUUH/IXhjRyRJ9mpcWVOGIVr31a/qXBPsfOYc7U8cgwcq33Vmj2OzoLYWQF6T+puqCAE4nMxFAxJvdZekhg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/sherpa-onnx-darwin-x64": {
+      "version": "1.12.23",
+      "resolved": "https://registry.npmjs.org/sherpa-onnx-darwin-x64/-/sherpa-onnx-darwin-x64-1.12.23.tgz",
+      "integrity": "sha512-hgPhBG85bgULJo5xS5xvh1A9+D6GBPwhZ8QCvWvh7YP1kF9UZTF1r5hWBOzZiR2Ax0gQMbxVvw1t/NpR2N7HWA==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/sherpa-onnx-linux-x64": {
+      "version": "1.12.23",
+      "resolved": "https://registry.npmjs.org/sherpa-onnx-linux-x64/-/sherpa-onnx-linux-x64-1.12.23.tgz",
+      "integrity": "sha512-pUZIdDvPtyRXQDGo9R9MIBf2AFUzfgcGmutoulsEdH3hpK6JteR7Z/5pfrZIIqe/O99djAjEHK4AlwLHC2jiZw==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/sherpa-onnx-node": {
+      "version": "1.12.23",
+      "resolved": "https://registry.npmjs.org/sherpa-onnx-node/-/sherpa-onnx-node-1.12.23.tgz",
+      "integrity": "sha512-09SRixVSjsajxeCV8Hy9R5J4IHPtw7vNgaIcEokdh/LpU7sY+e12z9uHHIMMMgNiInyGEH74wIwjLXms+W7qRA==",
+      "license": "Apache-2.0",
+      "optionalDependencies": {
+        "sherpa-onnx-darwin-arm64": "^1.12.23",
+        "sherpa-onnx-darwin-x64": "^1.12.23",
+        "sherpa-onnx-linux-arm64": "^1.12.23",
+        "sherpa-onnx-linux-x64": "^1.12.23",
+        "sherpa-onnx-win-ia32": "^1.12.23",
+        "sherpa-onnx-win-x64": "^1.12.23"
+      }
+    },
+    "node_modules/sherpa-onnx-win-ia32": {
+      "version": "1.12.23",
+      "resolved": "https://registry.npmjs.org/sherpa-onnx-win-ia32/-/sherpa-onnx-win-ia32-1.12.23.tgz",
+      "integrity": "sha512-MyLsK7r6dd7paglyTgb8UHTXTEFqOzA91u6VDV64Lq8rDGuOFVYioxX7vlwmGe1A9o7VhuOPNaKcRjEPtVDhBQ==",
+      "cpu": [
+        "ia32"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
+    "node_modules/sherpa-onnx-win-x64": {
+      "version": "1.12.23",
+      "resolved": "https://registry.npmjs.org/sherpa-onnx-win-x64/-/sherpa-onnx-win-x64-1.12.23.tgz",
+      "integrity": "sha512-pdHEYMJiYy8+xzH2WkBVS4/hnRwqjY8FaWnjs0NBgQZnPmc/k4M+TAiauTOuFDNK4GPwFQnjwrCGx6jI9AOkOg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "Apache-2.0",
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
    "node_modules/side-channel": {
      "version": "1.1.0",
      "license": "MIT",
@@ -20036,6 +20287,7 @@
        "@modelcontextprotocol/sdk": "^1.20.1",
        "@opencode-ai/sdk": "^1.1.12",
        "@openrouter/ai-sdk-provider": "^1.2.0",
+        "@sctg/sentencepiece-js": "^1.1.0",
        "@xterm/headless": "^6.0.0",
        "ai": "^5.0.76",
        "ajv": "^8.17.1",
@@ -20044,11 +20296,14 @@
        "express-basic-auth": "^1.2.1",
        "mnemonic-id": "^3.2.7",
        "node-pty": "^1.0.0",
+        "onnxruntime-node": "^1.23.0",
        "openai": "^4.20.0",
        "pino": "^10.2.0",
        "pino-pretty": "^13.1.3",
        "playwright": "^1.56.1",
        "qrcode": "^1.5.4",
+        "sherpa-onnx": "^1.12.23",
+        "sherpa-onnx-node": "^1.12.23",
        "strip-ansi": "^7.1.2",
        "tiny-invariant": "^1.3.3",
        "uuid": "^9.0.1",
--- a/packages/app/e2e/fixtures/recording.baseline.txt
+++ b/packages/app/e2e/fixtures/recording.baseline.txt
@@ -0,0 +1,2 @@
+This is a voice note.
+
--- a/packages/app/src/hooks/use-audio-player.native.ts
+++ b/packages/app/src/hooks/use-audio-player.native.ts
@@ -14,32 +14,60 @@ interface QueuedAudio {
 }

 /**
- * Resample PCM16 audio from 24kHz to 16kHz
- * OpenAI returns 24kHz, Speechmatics expects 16kHz
+ * Resample PCM16 audio between sample rates.
+ * Speechmatics expects 16kHz.
 */
-function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
-  // PCM16 = 2 bytes per sample
-  const samples24k = pcm24k.length / 2;
-  const samples16k = Math.floor((samples24k * 16000) / 24000);
-
-  const pcm16k = new Uint8Array(samples16k * 2);
-  const ratio = 24000 / 16000; // 1.5
-
-  for (let i = 0; i < samples16k; i++) {
-    const srcIndex = Math.floor(i * ratio) * 2;
-    if (srcIndex + 1 < pcm24k.length) {
-      pcm16k[i * 2] = pcm24k[srcIndex];
-      pcm16k[i * 2 + 1] = pcm24k[srcIndex + 1];
-    }
+function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array {
+  if (fromRate === toRate) {
+    return pcm;
  }

-  console.log("[AudioPlayer] Resampled PCM:", {
-    input24k: pcm24k.length,
-    output16k: pcm16k.length,
-    durationMs: samples16k / 16,
-  });
+  const inputSamples = Math.floor(pcm.length / 2);
+  const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
+  const out = new Uint8Array(outputSamples * 2);

-  return pcm16k;
+  const ratio = fromRate / toRate;
+
+  const readInt16 = (sampleIndex: number): number => {
+    const i = sampleIndex * 2;
+    if (i + 1 >= pcm.length) {
+      return 0;
+    }
+    const lo = pcm[i]!;
+    const hi = pcm[i + 1]!;
+    let value = (hi << 8) | lo;
+    if (value & 0x8000) {
+      value = value - 0x10000;
+    }
+    return value;
+  };
+
+  const writeInt16 = (sampleIndex: number, value: number): void => {
+    const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
+    const i = sampleIndex * 2;
+    out[i] = clamped & 0xff;
+    out[i + 1] = (clamped >> 8) & 0xff;
+  };
+
+  for (let i = 0; i < outputSamples; i++) {
+    const srcPos = i * ratio;
+    const i0 = Math.floor(srcPos);
+    const frac = srcPos - i0;
+    const s0 = readInt16(i0);
+    const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
+    writeInt16(i, s0 + (s1 - s0) * frac);
+  }
+
+  return out;
+}
+
+function parsePcmSampleRate(mimeType: string): number | null {
+  const match = /rate=(\d+)/i.exec(mimeType);
+  if (!match) {
+    return null;
+  }
+  const rate = Number(match[1]);
+  return Number.isFinite(rate) && rate > 0 ? rate : null;
 }

 export interface AudioPlayerOptions {
@@ -204,12 +232,12 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
        console.log("[AudioPlayer] Resuming playback engine...");
        resumePlayback();

-        // Get PCM data from blob (server now sends PCM format)
+        // Get PCM data from blob (server sends PCM16)
        const arrayBuffer = await audioData.arrayBuffer();
-        let pcm24k = new Uint8Array(arrayBuffer);
+        const pcm = new Uint8Array(arrayBuffer);

-        // Resample from 24kHz (OpenAI) to 16kHz (Speechmatics)
-        const pcm16k = resamplePcm24kTo16k(pcm24k);
+        const inputRate = parsePcmSampleRate(audioData.type || "") ?? 24000;
+        const pcm16k = resamplePcm16(pcm, inputRate, 16000);

        // Calculate total duration
        const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample
--- a/packages/cli/src/cli.ts
+++ b/packages/cli/src/cli.ts
@@ -3,6 +3,7 @@ import { createAgentCommand } from './commands/agent/index.js'
 import { createDaemonCommand } from './commands/daemon/index.js'
 import { createPermitCommand } from './commands/permit/index.js'
 import { createProviderCommand } from './commands/provider/index.js'
+import { createSpeechCommand } from './commands/speech/index.js'
 import { createWorktreeCommand } from './commands/worktree/index.js'
 import { runLsCommand } from './commands/agent/ls.js'
 import { runRunCommand } from './commands/agent/run.js'
@@ -134,6 +135,9 @@ export function createCli(): Command {
  // Provider commands
  program.addCommand(createProviderCommand())

+  // Speech model commands
+  program.addCommand(createSpeechCommand())
+
  // Worktree commands
  program.addCommand(createWorktreeCommand())

--- a/packages/cli/src/commands/speech/download.ts
+++ b/packages/cli/src/commands/speech/download.ts
@@ -0,0 +1,68 @@
+import type { Command } from "commander";
+import type {
+  CommandError,
+  CommandOptions,
+  ListResult,
+  OutputSchema,
+} from "../../output/index.js";
+import { connectToDaemon } from "../../utils/client.js";
+
+interface SpeechDownloadRow {
+  modelId: string;
+  status: "downloaded";
+}
+
+const speechDownloadSchema: OutputSchema<SpeechDownloadRow> = {
+  idField: "modelId",
+  columns: [
+    { header: "MODEL", field: "modelId", width: 36 },
+    { header: "STATUS", field: "status", width: 12, color: () => "green" },
+  ],
+};
+
+export type SpeechDownloadResult = ListResult<SpeechDownloadRow>;
+
+export interface SpeechDownloadOptions extends CommandOptions {
+  host?: string;
+  model?: string[];
+}
+
+export async function runSpeechDownloadCommand(
+  options: SpeechDownloadOptions,
+  _command: Command
+): Promise<SpeechDownloadResult> {
+  const client = await connectToDaemon({ host: options.host });
+  try {
+    const response = await client.downloadSpeechModels({
+      modelIds: options.model && options.model.length > 0 ? options.model : undefined,
+    });
+    if (response.error) {
+      const commandError: CommandError = {
+        code: "SPEECH_MODELS_DOWNLOAD_FAILED",
+        message: response.error,
+      };
+      throw commandError;
+    }
+
+    return {
+      type: "list",
+      data: response.downloadedModelIds.map((modelId) => ({
+        modelId,
+        status: "downloaded" as const,
+      })),
+      schema: speechDownloadSchema,
+    };
+  } catch (error) {
+    if (typeof error === "object" && error && "code" in error && "message" in error) {
+      throw error;
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    const commandError: CommandError = {
+      code: "SPEECH_MODELS_DOWNLOAD_FAILED",
+      message: `Failed to download speech models: ${message}`,
+    };
+    throw commandError;
+  } finally {
+    await client.close().catch(() => {});
+  }
+}
--- a/packages/cli/src/commands/speech/index.ts
+++ b/packages/cli/src/commands/speech/index.ts
@@ -0,0 +1,29 @@
+import { Command } from "commander";
+import { withOutput } from "../../output/index.js";
+import { runSpeechModelsCommand } from "./models.js";
+import { runSpeechDownloadCommand } from "./download.js";
+
+function collectMultiple(value: string, previous: string[]): string[] {
+  return previous.concat([value]);
+}
+
+export function createSpeechCommand(): Command {
+  const speech = new Command("speech").description("Manage local speech models");
+
+  speech
+    .command("models")
+    .description("List local speech model download status")
+    .option("--json", "Output in JSON format")
+    .option("--host <host>", "Daemon host:port (default: localhost:6767)")
+    .action(withOutput(runSpeechModelsCommand));
+
+  speech
+    .command("download")
+    .description("Download local speech models")
+    .option("--model <id>", "Model ID to download (repeatable)", collectMultiple, [])
+    .option("--json", "Output in JSON format")
+    .option("--host <host>", "Daemon host:port (default: localhost:6767)")
+    .action(withOutput(runSpeechDownloadCommand));
+
+  return speech;
+}
--- a/packages/cli/src/commands/speech/models.ts
+++ b/packages/cli/src/commands/speech/models.ts
@@ -0,0 +1,72 @@
+import type { Command } from "commander";
+import type {
+  CommandError,
+  CommandOptions,
+  ListResult,
+  OutputSchema,
+} from "../../output/index.js";
+import { connectToDaemon } from "../../utils/client.js";
+
+interface SpeechModelListItem {
+  id: string;
+  kind: string;
+  status: "downloaded" | "missing";
+  modelDir: string;
+  missingFiles: string;
+}
+
+const speechModelsSchema: OutputSchema<SpeechModelListItem> = {
+  idField: "id",
+  columns: [
+    { header: "MODEL", field: "id", width: 36 },
+    { header: "KIND", field: "kind", width: 12 },
+    {
+      header: "STATUS",
+      field: "status",
+      width: 12,
+      color: (value) => (value === "downloaded" ? "green" : "yellow"),
+    },
+    { header: "MODEL DIR", field: "modelDir", width: 44 },
+    { header: "MISSING FILES", field: "missingFiles", width: 40 },
+  ],
+};
+
+export type SpeechModelsResult = ListResult<SpeechModelListItem>;
+
+export interface SpeechModelsOptions extends CommandOptions {
+  host?: string;
+}
+
+export async function runSpeechModelsCommand(
+  options: SpeechModelsOptions,
+  _command: Command
+): Promise<SpeechModelsResult> {
+  const client = await connectToDaemon({ host: options.host });
+  try {
+    const response = await client.listSpeechModels();
+    const rows: SpeechModelListItem[] = response.models
+      .slice()
+      .sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id))
+      .map((model) => ({
+        id: model.id,
+        kind: model.kind,
+        status: model.isDownloaded ? "downloaded" : "missing",
+        modelDir: model.modelDir,
+        missingFiles: model.missingFiles?.join(", ") ?? "",
+      }));
+    return {
+      type: "list",
+      data: rows,
+      schema: speechModelsSchema,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const commandError: CommandError = {
+      code: "SPEECH_MODELS_LIST_FAILED",
+      message: `Failed to list speech models: ${message}`,
+    };
+    throw commandError;
+  } finally {
+    await client.close().catch(() => {});
+  }
+}
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -14,6 +14,8 @@
    "start": "NODE_ENV=production node dist/server/index.js",
    "typecheck": "tsc -p tsconfig.server.typecheck.json --noEmit",
    "generate:config-schema": "tsx scripts/generate-config-schema.ts",
+    "speech:models": "tsx scripts/list-speech-models.ts",
+    "speech:download": "tsx scripts/download-speech-models.ts",
    "test": "vitest run",
    "test:watch": "vitest",
    "test:ui": "vitest --ui",
@@ -48,6 +50,10 @@
    "pino-pretty": "^13.1.3",
    "playwright": "^1.56.1",
    "qrcode": "^1.5.4",
+    "@sctg/sentencepiece-js": "^1.1.0",
+    "onnxruntime-node": "^1.23.0",
+    "sherpa-onnx": "^1.12.23",
+    "sherpa-onnx-node": "^1.12.23",
    "strip-ansi": "^7.1.2",
    "tiny-invariant": "^1.3.3",
    "uuid": "^9.0.1",
--- a/packages/server/scripts/download-speech-models.ts
+++ b/packages/server/scripts/download-speech-models.ts
@@ -0,0 +1,42 @@
+import { resolvePaseoHome } from "../src/server/paseo-home.js";
+import { createRootLogger } from "../src/server/logger.js";
+import { ensureSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-downloader.js";
+import type { SherpaOnnxModelId } from "../src/server/speech/providers/local/sherpa/model-catalog.js";
+
+function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } {
+  const home = resolvePaseoHome();
+  let modelsDir = process.env.PASEO_SHERPA_ONNX_MODELS_DIR || `${home}/models/sherpa-onnx`;
+  const modelIds: SherpaOnnxModelId[] = [];
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === "--models-dir") {
+      modelsDir = argv[i + 1] ?? modelsDir;
+      i++;
+      continue;
+    }
+    if (arg === "--model") {
+      const id = argv[i + 1] as SherpaOnnxModelId | undefined;
+      if (!id) {
+        throw new Error("--model requires a value");
+      }
+      modelIds.push(id);
+      i++;
+      continue;
+    }
+  }
+
+  if (modelIds.length === 0) {
+    const stt = (process.env.PASEO_SHERPA_STT_PRESET || "zipformer-bilingual-zh-en-2023-02-20") as SherpaOnnxModelId;
+    const tts = (process.env.PASEO_SHERPA_TTS_PRESET || "pocket-tts-onnx-int8") as SherpaOnnxModelId;
+    modelIds.push(stt, tts);
+  }
+
+  return { modelsDir, modelIds };
+}
+
+const logger = createRootLogger({ level: "info", format: "pretty" });
+
+const { modelsDir, modelIds } = parseArgs(process.argv.slice(2));
+await ensureSherpaOnnxModels({ modelsDir, modelIds, autoDownload: true, logger });
+logger.info({ modelsDir, modelIds }, "Done downloading speech models");
--- a/packages/server/scripts/list-speech-models.ts
+++ b/packages/server/scripts/list-speech-models.ts
@@ -0,0 +1,10 @@
+import { listSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-catalog.js";
+
+const models = listSherpaOnnxModels()
+  .slice()
+  .sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id));
+
+for (const m of models) {
+  // eslint-disable-next-line no-console
+  console.log(`${m.kind}\t${m.id}\t${m.description}`);
+}
--- a/packages/server/src/client/daemon-client.ts
+++ b/packages/server/src/client/daemon-client.ts
@@ -36,6 +36,8 @@ import type {
  ExecuteCommandResponse,
  ListVoiceConversationsResponseMessage,
  ListProviderModelsResponseMessage,
+  SpeechModelsListResponse,
+  SpeechModelsDownloadResponse,
  ListTerminalsResponse,
  CreateTerminalResponse,
  SubscribeTerminalResponse,
@@ -201,6 +203,8 @@ type PaseoWorktreeArchivePayload = PaseoWorktreeArchiveResponse["payload"];
 type FileExplorerPayload = FileExplorerResponse["payload"];
 type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"];
 type ListProviderModelsPayload = ListProviderModelsResponseMessage["payload"];
+type SpeechModelsListPayload = SpeechModelsListResponse["payload"];
+type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"];
 type ListCommandsPayload = ListCommandsResponse["payload"];
 type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
 type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
@@ -2014,6 +2018,55 @@ export class DaemonClient {
    });
  }

+  async listSpeechModels(requestId?: string): Promise<SpeechModelsListPayload> {
+    const resolvedRequestId = this.createRequestId(requestId);
+    const message = SessionInboundMessageSchema.parse({
+      type: "speech_models_list_request",
+      requestId: resolvedRequestId,
+    });
+    return this.sendRequest({
+      requestId: resolvedRequestId,
+      message,
+      timeout: 30000,
+      options: { skipQueue: true },
+      select: (msg) => {
+        if (msg.type !== "speech_models_list_response") {
+          return null;
+        }
+        if (msg.payload.requestId !== resolvedRequestId) {
+          return null;
+        }
+        return msg.payload;
+      },
+    });
+  }
+
+  async downloadSpeechModels(
+    options?: { modelIds?: string[]; requestId?: string }
+  ): Promise<SpeechModelsDownloadPayload> {
+    const resolvedRequestId = this.createRequestId(options?.requestId);
+    const message = SessionInboundMessageSchema.parse({
+      type: "speech_models_download_request",
+      modelIds: options?.modelIds,
+      requestId: resolvedRequestId,
+    });
+    return this.sendRequest({
+      requestId: resolvedRequestId,
+      message,
+      timeout: 30 * 60 * 1000,
+      options: { skipQueue: true },
+      select: (msg) => {
+        if (msg.type !== "speech_models_download_response") {
+          return null;
+        }
+        if (msg.payload.requestId !== resolvedRequestId) {
+          return null;
+        }
+        return msg.payload;
+      },
+    });
+  }
+
  async listCommands(
    agentId: string,
    requestId?: string
--- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts
+++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts
@@ -168,7 +168,7 @@ describe("Codex app-server provider (integration)", () => {
      cleanup();
      rmSync(cwd, { recursive: true, force: true });
    }
-  }, 60000);
+  }, 120000);

  test.runIf(isCodexInstalled())("listCommands includes custom prompts and executeCommand runs them", async () => {
    const cleanup = useTempCodexSessionDir();
--- a/packages/server/src/server/agent/stt-manager.test.ts
+++ b/packages/server/src/server/agent/stt-manager.test.ts
@@ -0,0 +1,82 @@
+import { describe, expect, it } from "vitest";
+import pino from "pino";
+import { EventEmitter } from "node:events";
+
+import { STTManager } from "./stt-manager.js";
+import type {
+  SpeechToTextProvider,
+  StreamingTranscriptionSession,
+  TranscriptionResult,
+} from "../speech/speech-provider.js";
+
+class FakeStt implements SpeechToTextProvider {
+  public readonly id = "fake";
+  constructor(private readonly result: TranscriptionResult) {}
+
+  createSession(_params: {
+    logger: any;
+    language?: string;
+    prompt?: string;
+  }): StreamingTranscriptionSession {
+    const emitter = new EventEmitter();
+    const result = this.result;
+    let segmentId = "seg-1";
+    let previousSegmentId: string | null = null;
+
+    return {
+      requiredSampleRate: 24000,
+      async connect() {},
+      appendPcm16() {},
+      commit() {
+        (emitter as any).emit("committed", { segmentId, previousSegmentId });
+        (emitter as any).emit("transcript", {
+          segmentId,
+          transcript: result.text,
+          isFinal: true,
+          language: result.language,
+          logprobs: result.logprobs,
+          avgLogprob: result.avgLogprob,
+          isLowConfidence: result.isLowConfidence,
+        });
+        previousSegmentId = segmentId;
+        segmentId = "seg-2";
+      },
+      clear() {},
+      close() {},
+      on(event: any, handler: any) {
+        emitter.on(event, handler);
+        return undefined;
+      },
+    };
+  }
+}
+
+describe("STTManager", () => {
+  it("returns empty text for low-confidence transcriptions", async () => {
+    const manager = new STTManager(
+      "s1",
+      pino({ level: "silent" }),
+      new FakeStt({ text: "um", isLowConfidence: true, avgLogprob: -10 })
+    );
+
+    const result = await manager.transcribe(Buffer.alloc(2), "audio/pcm;rate=24000", {
+      label: "t",
+    });
+    expect(result.text).toBe("");
+    expect(result.isLowConfidence).toBe(true);
+    expect(result.byteLength).toBe(2);
+  });
+
+  it("passes through normal transcriptions", async () => {
+    const manager = new STTManager(
+      "s1",
+      pino({ level: "silent" }),
+      new FakeStt({ text: "hello world", language: "en", isLowConfidence: false })
+    );
+
+    const result = await manager.transcribe(Buffer.alloc(4), "audio/pcm;rate=24000");
+    expect(result.text).toBe("hello world");
+    expect(result.language).toBe("en");
+    expect(result.byteLength).toBe(4);
+  });
+});
--- a/packages/server/src/server/agent/stt-manager.ts
+++ b/packages/server/src/server/agent/stt-manager.ts
@@ -1,6 +1,8 @@
 import type pino from "pino";
-import type { OpenAISTT, TranscriptionResult } from "./stt-openai.js";
+import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js";
 import { maybePersistDebugAudio } from "./stt-debug.js";
+import { parsePcm16MonoWav, parsePcmRateFromFormat } from "../speech/audio.js";
+import { Pcm16MonoResampler } from "./pcm16-resampler.js";

 interface TranscriptionMetadata {
  agentId?: string;
@@ -21,9 +23,9 @@ export interface SessionTranscriptionResult extends TranscriptionResult {
 export class STTManager {
  private readonly sessionId: string;
  private readonly logger: pino.Logger;
-  private readonly stt: OpenAISTT | null;
+  private readonly stt: SpeechToTextProvider | null;

-  constructor(sessionId: string, logger: pino.Logger, stt: OpenAISTT | null) {
+  constructor(sessionId: string, logger: pino.Logger, stt: SpeechToTextProvider | null) {
    this.sessionId = sessionId;
    this.logger = logger.child({ module: "agent", component: "stt-manager", sessionId });
    this.stt = stt;
@@ -63,36 +65,104 @@ export class STTManager {
      this.logger.warn({ err: error }, "Failed to persist debug audio");
    }

-    const result = await this.stt.transcribeAudio(audio, format);
+    const session = this.stt.createSession({
+      logger: this.logger.child({ component: "stt-session" }),
+      language: "en",
+    });

-    // Filter out low-confidence transcriptions (non-speech sounds)
-    if (result.isLowConfidence) {
-      this.logger.debug(
-        { text: result.text, avgLogprob: result.avgLogprob },
-        "Filtered low-confidence transcription (likely non-speech)"
-      );
-
-      // Return empty text to ignore this transcription
-      return {
-        ...result,
-        text: "",
-        byteLength: audio.length,
-        format,
-        debugRecordingPath: debugRecordingPath ?? undefined,
-      };
+    let inputRate: number;
+    let pcm16: Buffer;
+    if (format.toLowerCase().includes("audio/wav")) {
+      const parsed = parsePcm16MonoWav(audio);
+      inputRate = parsed.sampleRate;
+      pcm16 = parsed.pcm16;
+    } else if (format.toLowerCase().includes("audio/pcm")) {
+      inputRate =
+        parsePcmRateFromFormat(format, session.requiredSampleRate) ??
+        session.requiredSampleRate;
+      pcm16 = audio;
+    } else {
+      throw new Error(`Unsupported audio format for STT: ${format}`);
    }

-    this.logger.debug(
-      { text: result.text, avgLogprob: result.avgLogprob },
-      "Transcription complete"
-    );
+    let pcmForModel = pcm16;
+    if (inputRate !== session.requiredSampleRate) {
+      const resampler = new Pcm16MonoResampler({
+        inputRate,
+        outputRate: session.requiredSampleRate,
+      });
+      pcmForModel = resampler.processChunk(pcm16);
+      inputRate = session.requiredSampleRate;
+    }

-    return {
-      ...result,
-      debugRecordingPath: debugRecordingPath ?? undefined,
-      byteLength: audio.length,
-      format,
-    };
+    try {
+      const startedAt = Date.now();
+      const finalEventPromise = new Promise<{
+        transcript: string;
+        language?: string;
+        logprobs?: TranscriptionResult["logprobs"];
+        avgLogprob?: number;
+        isLowConfidence?: boolean;
+      }>((resolve, reject) => {
+        session.on("error", reject);
+        session.on("transcript", (payload) => {
+          if (!payload.isFinal) {
+            return;
+          }
+          resolve({
+            transcript: payload.transcript,
+            language: payload.language,
+            logprobs: payload.logprobs,
+            avgLogprob: payload.avgLogprob,
+            isLowConfidence: payload.isLowConfidence,
+          });
+        });
+      });
+
+      await session.connect();
+      session.appendPcm16(pcmForModel);
+      session.commit();
+      const finalEvent = await finalEventPromise;
+      const result: TranscriptionResult = {
+        text: finalEvent.transcript,
+        language: finalEvent.language,
+        logprobs: finalEvent.logprobs,
+        avgLogprob: finalEvent.avgLogprob,
+        isLowConfidence: finalEvent.isLowConfidence,
+        duration: Date.now() - startedAt,
+      };
+
+      // Filter out low-confidence transcriptions (non-speech sounds)
+      if (result.isLowConfidence) {
+        this.logger.debug(
+          { text: result.text, avgLogprob: result.avgLogprob },
+          "Filtered low-confidence transcription (likely non-speech)"
+        );
+
+        // Return empty text to ignore this transcription
+        return {
+          ...result,
+          text: "",
+          byteLength: audio.length,
+          format,
+          debugRecordingPath: debugRecordingPath ?? undefined,
+        };
+      }
+
+      this.logger.debug(
+        { text: result.text, avgLogprob: result.avgLogprob },
+        "Transcription complete"
+      );
+
+      return {
+        ...result,
+        debugRecordingPath: debugRecordingPath ?? undefined,
+        byteLength: audio.length,
+        format,
+      };
+    } finally {
+      session.close();
+    }
  }

  /**
--- a/packages/server/src/server/agent/stt-openai.ts
+++ b/packages/server/src/server/agent/stt-openai.ts
@@ -1,151 +0,0 @@
-import type pino from "pino";
-import OpenAI from "openai";
-import { writeFile, unlink } from "fs/promises";
-import { join } from "path";
-import { tmpdir } from "os";
-import { v4 } from "uuid";
-import { inferAudioExtension } from "./audio-utils.js";
-
-export interface STTConfig {
-  apiKey: string;
-  model?: "whisper-1" | "gpt-4o-transcribe" | "gpt-4o-mini-transcribe" | (string & {});
-  confidenceThreshold?: number; // Default: -3.0
-}
-
-export interface LogprobToken {
-  token: string;
-  logprob: number;
-  bytes?: number[];
-}
-
-export interface TranscriptionResult {
-  text: string;
-  language?: string;
-  duration?: number;
-  logprobs?: LogprobToken[];
-  avgLogprob?: number;
-  isLowConfidence?: boolean;
-}
-
-function isObject(value: unknown): value is { [key: string]: unknown } {
-  return typeof value === "object" && value !== null;
-}
-
-function isLogprobToken(value: unknown): value is LogprobToken {
-  if (!isObject(value)) {
-    return false;
-  }
-  if (typeof value.token !== "string") {
-    return false;
-  }
-  if (typeof value.logprob !== "number") {
-    return false;
-  }
-  if (value.bytes === undefined) {
-    return true;
-  }
-  return Array.isArray(value.bytes) && value.bytes.every((entry) => typeof entry === "number");
-}
-
-function isLogprobTokenArray(value: unknown): value is LogprobToken[] {
-  return Array.isArray(value) && value.every((entry) => isLogprobToken(entry));
-}
-
-export class OpenAISTT {
-  private readonly openaiClient: OpenAI;
-  private readonly config: STTConfig;
-  private readonly logger: pino.Logger;
-
-  constructor(sttConfig: STTConfig, parentLogger: pino.Logger) {
-    this.config = sttConfig;
-    this.logger = parentLogger.child({ module: "agent", provider: "openai", component: "stt" });
-    this.openaiClient = new OpenAI({
-      apiKey: sttConfig.apiKey,
-    });
-    this.logger.info({ model: sttConfig.model || "whisper-1" }, "STT (OpenAI Whisper) initialized");
-  }
-
-  public async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
-    const startTime = Date.now();
-    let tempFilePath: string | null = null;
-
-    try {
-      const ext = inferAudioExtension(format);
-      tempFilePath = join(tmpdir(), `audio-${v4()}.${ext}`);
-      await writeFile(tempFilePath, audioBuffer);
-
-      this.logger.debug(
-        { tempFilePath, bytes: audioBuffer.length },
-        "Transcribing audio file"
-      );
-
-      const modelToUse = this.config.model ?? "whisper-1";
-      const supportsLogprobs =
-        modelToUse === "gpt-4o-transcribe" || modelToUse === "gpt-4o-mini-transcribe";
-      const includeLogprobs: ["logprobs"] = ["logprobs"];
-
-      const response = await this.openaiClient.audio.transcriptions.create({
-        file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)),
-        language: "en",
-        model: modelToUse,
-        ...(supportsLogprobs ? { include: includeLogprobs } : {}),
-        response_format: "json",
-      });
-
-      const duration = Date.now() - startTime;
-      const confidenceThreshold = this.config.confidenceThreshold ?? -3.0;
-
-      let avgLogprob: number | undefined;
-      let isLowConfidence = false;
-      const logprobs =
-        supportsLogprobs &&
-        isObject(response) &&
-        isLogprobTokenArray(response.logprobs)
-          ? response.logprobs
-          : undefined;
-
-      if (logprobs && logprobs.length > 0) {
-        const totalLogprob = logprobs.reduce((sum, token) => sum + token.logprob, 0);
-        avgLogprob = totalLogprob / logprobs.length;
-        isLowConfidence = avgLogprob < confidenceThreshold;
-
-        if (isLowConfidence) {
-          this.logger.debug(
-            {
-              avgLogprob,
-              threshold: confidenceThreshold,
-              text: response.text,
-              tokenLogprobs: logprobs.map((t) => `${t.token}:${t.logprob.toFixed(2)}`).join(", "),
-            },
-            "Low confidence transcription detected"
-          );
-        }
-      }
-
-      this.logger.debug({ duration, text: response.text, avgLogprob }, "Transcription complete");
-
-      return {
-        text: response.text,
-        duration: duration,
-        logprobs: logprobs,
-        avgLogprob: avgLogprob,
-        isLowConfidence: isLowConfidence,
-        language:
-          isObject(response) && typeof response.language === "string"
-            ? response.language
-            : undefined,
-      };
-    } catch (error: any) {
-      this.logger.error({ err: error }, "Transcription error");
-      throw new Error(`STT transcription failed: ${error.message}`);
-    } finally {
-      if (tempFilePath) {
-        try {
-          await unlink(tempFilePath);
-        } catch (cleanupError) {
-          this.logger.warn({ tempFilePath }, "Failed to clean up temp file");
-        }
-      }
-    }
-  }
-}
--- a/packages/server/src/server/agent/tts-manager.test.ts
+++ b/packages/server/src/server/agent/tts-manager.test.ts
@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import pino from "pino";
+import { Readable } from "node:stream";
+
+import { TTSManager } from "./tts-manager.js";
+import type { TextToSpeechProvider } from "../speech/speech-provider.js";
+import type { SessionOutboundMessage } from "../messages.js";
+
+class FakeTts implements TextToSpeechProvider {
+  async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> {
+    return {
+      stream: Readable.from([Buffer.from("a"), Buffer.from("b")]),
+      format: "pcm;rate=24000",
+    };
+  }
+}
+
+describe("TTSManager", () => {
+  it("emits chunks and resolves once confirmed", async () => {
+    const manager = new TTSManager("s1", pino({ level: "silent" }), new FakeTts());
+    const abort = new AbortController();
+    const emitted: SessionOutboundMessage[] = [];
+
+    const task = manager.generateAndWaitForPlayback(
+      "hello",
+      (msg) => {
+        emitted.push(msg);
+        if (msg.type === "audio_output") {
+          manager.confirmAudioPlayed(msg.payload.id);
+        }
+      },
+      abort.signal,
+      true
+    );
+
+    await task;
+
+    const audioMsgs = emitted.filter((m) => m.type === "audio_output");
+    expect(audioMsgs).toHaveLength(2);
+    const groupId = (audioMsgs[0] as any).payload.groupId;
+    expect(groupId).toBeTruthy();
+    expect((audioMsgs[0] as any).payload.chunkIndex).toBe(0);
+    expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1);
+    expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true);
+  });
+});
+
--- a/packages/server/src/server/agent/tts-manager.ts
+++ b/packages/server/src/server/agent/tts-manager.ts
@@ -1,6 +1,6 @@
 import type pino from "pino";
 import { v4 as uuidv4 } from "uuid";
-import type { OpenAITTS } from "./tts-openai.js";
+import type { TextToSpeechProvider } from "../speech/speech-provider.js";
 import type { SessionOutboundMessage } from "../messages.js";

 interface PendingPlayback {
@@ -17,9 +17,9 @@ interface PendingPlayback {
 export class TTSManager {
  private pendingPlaybacks: Map<string, PendingPlayback> = new Map();
  private readonly logger: pino.Logger;
-  private readonly tts: OpenAITTS | null;
+  private readonly tts: TextToSpeechProvider | null;

-  constructor(sessionId: string, logger: pino.Logger, tts: OpenAITTS | null) {
+  constructor(sessionId: string, logger: pino.Logger, tts: TextToSpeechProvider | null) {
    this.logger = logger.child({ module: "agent", component: "tts-manager", sessionId });
    this.tts = tts;
  }
--- a/packages/server/src/server/bootstrap.smoke.test.ts
+++ b/packages/server/src/server/bootstrap.smoke.test.ts
@@ -1,6 +1,12 @@
+import os from "node:os";
+import path from "node:path";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import pino from "pino";
 import { describe, expect, test } from "vitest";

+import { createPaseoDaemon, type PaseoDaemonConfig } from "./bootstrap.js";
 import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js";
+import { createTestAgentClients } from "./test-utils/fake-agent-client.js";

 describe("paseo daemon bootstrap", () => {
  test("starts and serves health endpoint", async () => {
@@ -22,4 +28,41 @@ describe("paseo daemon bootstrap", () => {
      await daemonHandle.close();
    }
  });
+
+  test("fails fast when OpenAI speech provider is configured without credentials", async () => {
+    const paseoHomeRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-openai-config-"));
+    const paseoHome = path.join(paseoHomeRoot, ".paseo");
+    const staticDir = await mkdtemp(path.join(os.tmpdir(), "paseo-static-"));
+    await mkdir(paseoHome, { recursive: true });
+
+    const config: PaseoDaemonConfig = {
+      listen: "127.0.0.1:0",
+      paseoHome,
+      corsAllowedOrigins: [],
+      allowedHosts: true,
+      mcpEnabled: false,
+      staticDir,
+      mcpDebug: false,
+      agentClients: createTestAgentClients(),
+      agentStoragePath: path.join(paseoHome, "agents"),
+      relayEnabled: false,
+      appBaseUrl: "https://app.paseo.sh",
+      openai: undefined,
+      speech: {
+        dictationSttProvider: "openai",
+        voiceSttProvider: "openai",
+        voiceTtsProvider: "openai",
+      },
+      openrouterApiKey: null,
+    };
+
+    try {
+      await expect(createPaseoDaemon(config, pino({ level: "silent" }))).rejects.toThrow(
+        "Missing OpenAI credentials"
+      );
+    } finally {
+      await rm(paseoHomeRoot, { recursive: true, force: true });
+      await rm(staticDir, { recursive: true, force: true });
+    }
+  });
 });
--- a/packages/server/src/server/bootstrap.ts
+++ b/packages/server/src/server/bootstrap.ts
@@ -40,8 +40,20 @@ function parseListenString(listen: string): ListenTarget {

 import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
 import { DownloadTokenStore } from "./file-download/token-store.js";
-import { OpenAISTT, type STTConfig } from "./agent/stt-openai.js";
-import { OpenAITTS, type TTSConfig } from "./agent/tts-openai.js";
+import { OpenAISTT, type STTConfig } from "./speech/providers/openai/stt.js";
+import { OpenAITTS, type TTSConfig } from "./speech/providers/openai/tts.js";
+import { OpenAIRealtimeTranscriptionSession } from "./speech/providers/openai/realtime-transcription-session.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
+import { SherpaOnlineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-online-recognizer.js";
+import { SherpaOfflineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-offline-recognizer.js";
+import { SherpaOnnxSTT } from "./speech/providers/local/sherpa/sherpa-stt.js";
+import { SherpaOnnxParakeetSTT } from "./speech/providers/local/sherpa/sherpa-parakeet-stt.js";
+import { SherpaOnnxTTS } from "./speech/providers/local/sherpa/sherpa-tts.js";
+import { SherpaRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-realtime-session.js";
+import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-parakeet-realtime-session.js";
+import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/providers/local/sherpa/model-downloader.js";
+import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
+import { PocketTtsOnnxTTS } from "./speech/providers/local/pocket/pocket-tts-onnx.js";
 import { AgentManager } from "./agent/agent-manager.js";
 import { AgentStorage } from "./agent/agent-storage.js";
 import { attachAgentStoragePersistence } from "./persistence-hooks.js";
@@ -72,6 +84,26 @@ export type PaseoOpenAIConfig = {
  tts?: Partial<TTSConfig> & { apiKey?: string };
 };

+export type PaseoSherpaOnnxConfig = {
+  modelsDir: string;
+  autoDownload?: boolean;
+  stt?: {
+    preset?: string;
+  };
+  tts?: {
+    preset?: string;
+    speakerId?: number;
+    speed?: number;
+  };
+};
+
+export type PaseoSpeechConfig = {
+  dictationSttProvider?: "openai" | "local";
+  voiceSttProvider?: "openai" | "local";
+  voiceTtsProvider?: "openai" | "local";
+  sherpaOnnx?: PaseoSherpaOnnxConfig;
+};
+
 export type PaseoDaemonConfig = {
  listen: string;
  paseoHome: string;
@@ -87,6 +119,7 @@ export type PaseoDaemonConfig = {
  relayPublicEndpoint?: string;
  appBaseUrl?: string;
  openai?: PaseoOpenAIConfig;
+  speech?: PaseoSpeechConfig;
  openrouterApiKey?: string | null;
  voiceLlmModel?: string | null;
  dictationFinalTimeoutMs?: number;
@@ -382,42 +415,404 @@ export async function createPaseoDaemon(
  }


-  let sttService: OpenAISTT | null = null;
-  let ttsService: OpenAITTS | null = null;
+  let sttService: SpeechToTextProvider | null = null;
+  let ttsService: TextToSpeechProvider | null = null;
+  let dictationSttService: SpeechToTextProvider | null = null;
+
+  let sherpaOnline: SherpaOnlineRecognizerEngine | null = null;
+  let sherpaOffline: SherpaOfflineRecognizerEngine | null = null;
+  let sherpaTts: TextToSpeechProvider | null = null;

  const openaiApiKey = config.openai?.apiKey;
-  if (openaiApiKey) {
-    logger.info("OpenAI client initialized");
+  const speechConfig = config.speech ?? null;
+  const sherpaConfig = speechConfig?.sherpaOnnx ?? null;

-    const sttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey;
-    if (sttApiKey) {
-      const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {};
-      sttService = new OpenAISTT(
-        {
-          apiKey: sttApiKey,
-          ...sttConfig,
+  const voiceSttProvider = speechConfig?.voiceSttProvider ?? "local";
+  const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "local";
+  const dictationSttProvider = speechConfig?.dictationSttProvider ?? "local";
+
+  const wantsLocalDictation = dictationSttProvider === "local";
+  const wantsLocalVoiceStt = voiceSttProvider === "local";
+  const wantsLocalVoiceTts = voiceTtsProvider === "local";
+
+  const openaiSttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey;
+  const openaiTtsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey;
+  const openaiDictationApiKey = openaiApiKey;
+
+  const missingOpenAiCredentialsFor: string[] = [];
+  if (voiceSttProvider === "openai" && !openaiSttApiKey) {
+    missingOpenAiCredentialsFor.push("voice.stt");
+  }
+  if (voiceTtsProvider === "openai" && !openaiTtsApiKey) {
+    missingOpenAiCredentialsFor.push("voice.tts");
+  }
+  if (dictationSttProvider === "openai" && !openaiDictationApiKey) {
+    missingOpenAiCredentialsFor.push("dictation.stt");
+  }
+
+  if (missingOpenAiCredentialsFor.length > 0) {
+    logger.error(
+      {
+        requestedProviders: {
+          dictationStt: dictationSttProvider,
+          voiceStt: voiceSttProvider,
+          voiceTts: voiceTtsProvider,
        },
-        logger
+        missingOpenAiCredentialsFor,
+      },
+      "Invalid speech configuration: OpenAI provider selected but credentials are missing"
+    );
+    throw new Error(
+      `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
+    );
+  }
+
+  logger.info(
+    {
+      requestedProviders: {
+        dictationStt: dictationSttProvider,
+        voiceStt: voiceSttProvider,
+        voiceTts: voiceTtsProvider,
+      },
+      availability: {
+        openai: {
+          stt: Boolean(openaiSttApiKey),
+          tts: Boolean(openaiTtsApiKey),
+          dictationStt: Boolean(openaiDictationApiKey),
+        },
+        local: {
+          configured: Boolean(sherpaConfig),
+          modelsDir: sherpaConfig?.modelsDir ?? null,
+          autoDownload: sherpaConfig?.autoDownload ?? null,
+        },
+      },
+    },
+    "Speech provider reconciliation started"
+  );
+
+  if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) {
+    const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true);
+    let sttPreset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim();
+    if (
+      sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" &&
+      sttPreset !== "paraformer-bilingual-zh-en" &&
+      sttPreset !== "parakeet-tdt-0.6b-v3-int8"
+    ) {
+      logger.warn(
+        { sttPreset },
+        "Unknown Sherpa STT preset; falling back to zipformer-bilingual-zh-en-2023-02-20"
      );
+      sttPreset = "zipformer-bilingual-zh-en-2023-02-20";
    }

-    const ttsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey;
-    if (ttsApiKey) {
-      const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {};
-      ttsService = new OpenAITTS(
-        {
-          apiKey: ttsApiKey,
-          voice: "alloy",
-          model: "tts-1",
-          responseFormat: "pcm",
-          ...ttsConfig,
-        },
-        logger
+    let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
+    if (
+      ttsPreset !== "kitten-nano-en-v0_1-fp16" &&
+      ttsPreset !== "kokoro-en-v0_19" &&
+      ttsPreset !== "pocket-tts-onnx-int8"
+    ) {
+      logger.warn(
+        { ttsPreset },
+        "Unknown Sherpa TTS preset; falling back to kitten-nano-en-v0_1-fp16"
      );
+      ttsPreset = "kitten-nano-en-v0_1-fp16";
    }

+    const modelIds: SherpaOnnxModelId[] = [];
+    if (wantsLocalDictation || wantsLocalVoiceStt) {
+      modelIds.push(sttPreset as SherpaOnnxModelId);
+    }
+    if (wantsLocalVoiceTts) {
+      modelIds.push(ttsPreset as SherpaOnnxModelId);
+    }
+
+    try {
+      logger.info(
+        {
+          modelsDir: sherpaConfig.modelsDir,
+          modelIds,
+          autoDownload,
+        },
+        "Ensuring local speech models"
+      );
+      await ensureSherpaOnnxModels({
+        modelsDir: sherpaConfig.modelsDir,
+        modelIds,
+        autoDownload,
+        logger,
+      });
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          modelsDir: sherpaConfig.modelsDir,
+          autoDownload,
+          hint:
+            "Run: npm run dev --workspace=@getpaseo/server, then run: " +
+            "`tsx packages/server/scripts/download-speech-models.ts --models-dir <DIR> --model <MODEL_ID>`",
+        },
+        "Failed to ensure local speech models"
+      );
+    }
+  }
+
+  if ((wantsLocalDictation || wantsLocalVoiceStt) && sherpaConfig) {
+    let preset = (sherpaConfig.stt?.preset ?? "zipformer-bilingual-zh-en-2023-02-20").trim();
+    if (
+      preset !== "zipformer-bilingual-zh-en-2023-02-20" &&
+      preset !== "paraformer-bilingual-zh-en" &&
+      preset !== "parakeet-tdt-0.6b-v3-int8"
+    ) {
+      logger.warn(
+        { preset },
+        "Unknown Sherpa STT preset; falling back to zipformer-bilingual-zh-en-2023-02-20"
+      );
+      preset = "zipformer-bilingual-zh-en-2023-02-20";
+    }
+    const base = sherpaConfig.modelsDir;
+
+    try {
+      if (preset === "parakeet-tdt-0.6b-v3-int8") {
+        const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8");
+        sherpaOffline = new SherpaOfflineRecognizerEngine(
+          {
+            model: {
+              kind: "nemo_transducer",
+              encoder: `${modelDir}/encoder.int8.onnx`,
+              decoder: `${modelDir}/decoder.int8.onnx`,
+              joiner: `${modelDir}/joiner.int8.onnx`,
+              tokens: `${modelDir}/tokens.txt`,
+            },
+            numThreads: 2,
+            debug: 0,
+          },
+          logger
+        );
+      } else {
+        const model =
+          preset === "paraformer-bilingual-zh-en"
+            ? {
+                kind: "paraformer" as const,
+                encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`,
+                decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`,
+                tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`,
+              }
+            : {
+                kind: "transducer" as const,
+                encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`,
+                decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`,
+                joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`,
+                tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`,
+                modelType: "zipformer",
+              };
+
+        sherpaOnline = new SherpaOnlineRecognizerEngine(
+          {
+            model,
+            numThreads: 1,
+            debug: 0,
+          },
+          logger
+        );
+      }
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          modelsDir: sherpaConfig.modelsDir,
+          preset,
+          hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
+        },
+        "Failed to initialize Sherpa STT (models missing or invalid)"
+      );
+      sherpaOnline = null;
+      sherpaOffline = null;
+    }
+  } else if (wantsLocalDictation || wantsLocalVoiceStt) {
+    logger.warn(
+      { configured: Boolean(sherpaConfig) },
+      "Local STT selected but local provider config is missing; STT will be unavailable"
+    );
+  }
+
+  if (wantsLocalVoiceTts && sherpaConfig) {
+    let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
+    if (
+      preset !== "kitten-nano-en-v0_1-fp16" &&
+      preset !== "kokoro-en-v0_19" &&
+      preset !== "pocket-tts-onnx-int8"
+    ) {
+      logger.warn(
+        { preset },
+        "Unknown Sherpa TTS preset; falling back to kitten-nano-en-v0_1-fp16"
+      );
+      preset = "kitten-nano-en-v0_1-fp16";
+    }
+    try {
+      if (preset === "pocket-tts-onnx-int8") {
+        const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8");
+        sherpaTts = await PocketTtsOnnxTTS.create(
+          {
+            modelDir,
+            precision: "int8",
+            targetChunkMs: 50,
+          },
+          logger
+        );
+      } else {
+        const modelDir = `${sherpaConfig.modelsDir}/${preset}`;
+        sherpaTts = new SherpaOnnxTTS(
+          {
+            preset: preset as any,
+            modelDir,
+            speakerId: sherpaConfig.tts?.speakerId,
+            speed: sherpaConfig.tts?.speed,
+          },
+          logger
+        );
+      }
+    } catch (err) {
+      logger.error(
+        {
+          err,
+          preset,
+          hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
+        },
+        "Failed to initialize Sherpa TTS (models missing or invalid)"
+      );
+      sherpaTts = null;
+    }
+  } else if (wantsLocalVoiceTts) {
+    logger.warn(
+      { configured: Boolean(sherpaConfig) },
+      "Local TTS selected but local provider config is missing; TTS will be unavailable"
+    );
+  }
+
+  if (wantsLocalVoiceStt && sherpaOffline) {
+    sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger);
+  } else if (wantsLocalVoiceStt && sherpaOnline) {
+    sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger);
+  }
+
+  if (wantsLocalVoiceTts && sherpaTts) {
+    ttsService = sherpaTts;
+  }
+
+  if (wantsLocalDictation && sherpaOnline) {
+    dictationSttService = {
+      id: "local",
+      createSession: () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }),
+    };
+  } else if (wantsLocalDictation && sherpaOffline) {
+    dictationSttService = {
+      id: "local",
+      createSession: () =>
+        new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }),
+    };
+  }
+
+  const needsOpenAiStt = !sttService && voiceSttProvider === "openai";
+  const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai";
+  const needsOpenAiDictation =
+    dictationSttProvider === "openai" || (dictationSttProvider === "local" && !dictationSttService);
+
+  const fallbackOpenAiStt = !sttService && voiceSttProvider === "local" && Boolean(openaiSttApiKey);
+  const fallbackOpenAiTts = !ttsService && voiceTtsProvider === "local" && Boolean(openaiTtsApiKey);
+
+  if (
+    (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) &&
+    (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey)
+  ) {
+    logger.info("OpenAI speech provider initialized");
+
+    if (fallbackOpenAiStt) {
+      logger.warn("Falling back to OpenAI STT because local STT is unavailable");
+    }
+    if (needsOpenAiStt || fallbackOpenAiStt) {
+      if (openaiSttApiKey) {
+        const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {};
+        sttService = new OpenAISTT(
+          {
+            apiKey: openaiSttApiKey,
+            ...sttConfig,
+          },
+          logger
+        );
+      }
+    }
+
+    if (fallbackOpenAiTts) {
+      logger.warn("Falling back to OpenAI TTS because local TTS is unavailable");
+    }
+    if (needsOpenAiTts || fallbackOpenAiTts) {
+      if (openaiTtsApiKey) {
+        const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {};
+        ttsService = new OpenAITTS(
+          {
+            apiKey: openaiTtsApiKey,
+            voice: "alloy",
+            model: "tts-1",
+            responseFormat: "pcm",
+            ...ttsConfig,
+          },
+          logger
+        );
+      }
+    }
+
+    if (needsOpenAiDictation) {
+      const transcriptionModel =
+        process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
+
+      dictationSttService = {
+        id: "openai",
+        createSession: ({ logger: sessionLogger, language, prompt }) =>
+          new OpenAIRealtimeTranscriptionSession({
+            apiKey: openaiDictationApiKey!,
+            logger: sessionLogger,
+            transcriptionModel,
+            ...(language ? { language } : {}),
+            ...(prompt ? { prompt } : {}),
+            turnDetection: null,
+          }),
+      };
+    }
+  } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation || fallbackOpenAiStt || fallbackOpenAiTts) {
+    logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation fallback is unavailable");
+  }
+
+  const effectiveProviders = {
+    dictationStt: dictationSttService?.id ?? "unavailable",
+    voiceStt: sttService?.id ?? "unavailable",
+    voiceTts: !ttsService ? "unavailable" : ttsService === sherpaTts ? "local" : "openai",
+  };
+  const unavailableFeatures = [
+    !dictationSttService ? "dictation.stt" : null,
+    !sttService ? "voice.stt" : null,
+    !ttsService ? "voice.tts" : null,
+  ].filter((feature): feature is string => feature !== null);
+
+  if (unavailableFeatures.length > 0) {
+    logger.warn(
+      {
+        requestedProviders: {
+          dictationStt: dictationSttProvider,
+          voiceStt: voiceSttProvider,
+          voiceTts: voiceTtsProvider,
+        },
+        effectiveProviders,
+        unavailableFeatures,
+      },
+      "Speech provider reconciliation completed with unavailable features"
+    );
  } else {
-    logger.warn("OPENAI_API_KEY not set - LLM, STT, and TTS features will not work");
+    logger.info(
+      {
+        effectiveProviders,
+      },
+      "Speech provider reconciliation completed"
+    );
  }

  const wsServer = new VoiceAssistantWebSocketServer(
@@ -437,8 +832,8 @@ export async function createPaseoDaemon(
      voiceLlmModel: config.voiceLlmModel ?? null,
    },
    {
-      openaiApiKey: config.openai?.apiKey ?? null,
      finalTimeoutMs: config.dictationFinalTimeoutMs,
+      stt: dictationSttService,
    }
  );

@@ -533,6 +928,11 @@ export async function createPaseoDaemon(
    await agentStorage.flush().catch(() => undefined);
    await shutdownProviders(logger);
    terminalManager.killAll();
+    if (sherpaTts && typeof (sherpaTts as any).free === "function") {
+      (sherpaTts as any).free();
+    }
+    sherpaOnline?.free();
+    sherpaOffline?.free();
    await relayTransport?.stop().catch(() => undefined);
    await wsServer.close();
    await new Promise<void>((resolve) => {
--- a/packages/server/src/server/config.ts
+++ b/packages/server/src/server/config.ts
@@ -1,8 +1,8 @@
 import path from "node:path";

 import type { PaseoDaemonConfig } from "./bootstrap.js";
-import type { STTConfig } from "./agent/stt-openai.js";
-import type { TTSConfig } from "./agent/tts-openai.js";
+import type { STTConfig } from "./speech/providers/openai/stt.js";
+import type { TTSConfig } from "./speech/providers/openai/tts.js";
 import { loadPersistedConfig } from "./persisted-config.js";
 import {
  mergeAllowedHosts,
@@ -76,6 +76,49 @@ function parseOpenAIConfig(
  };
 }

+function parseSpeechProviderId(value: unknown): "openai" | "local" | null {
+  if (typeof value !== "string") {
+    return null;
+  }
+  const normalized = value.trim().toLowerCase();
+  if (!normalized) {
+    return null;
+  }
+  if (normalized === "openai") return "openai";
+  if (normalized === "local") return "local";
+  return null;
+}
+
+function normalizeSherpaSttPreset(value: string): string {
+  const raw = value.trim();
+  const normalized = raw.toLowerCase();
+  if (normalized === "zipformer" || normalized === "zipformer-bilingual") {
+    return "zipformer-bilingual-zh-en-2023-02-20";
+  }
+  if (normalized === "paraformer") {
+    return "paraformer-bilingual-zh-en";
+  }
+  if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") {
+    return "parakeet-tdt-0.6b-v3-int8";
+  }
+  return raw;
+}
+
+function normalizeSherpaTtsPreset(value: string): string {
+  const raw = value.trim();
+  const normalized = raw.toLowerCase();
+  if (normalized === "pocket" || normalized === "pocket-tts") {
+    return "pocket-tts-onnx-int8";
+  }
+  if (normalized === "kitten") {
+    return "kitten-nano-en-v0_1-fp16";
+  }
+  if (normalized === "kokoro") {
+    return "kokoro-en-v0_19";
+  }
+  return raw;
+}
+
 export function loadConfig(
  paseoHome: string,
  options?: {
@@ -137,6 +180,70 @@ export function loadConfig(
    voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice,
  });

+  const dictationSttProvider =
+    parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ??
+    parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ??
+    "local";
+
+  const voiceSttProvider =
+    parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ??
+    parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ??
+    "local";
+
+  const voiceTtsProvider =
+    parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ??
+    parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ??
+    "local";
+
+  const shouldConfigureSherpa =
+    dictationSttProvider === "local" ||
+    voiceSttProvider === "local" ||
+    voiceTtsProvider === "local" ||
+    typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" ||
+    Boolean(persisted.providers?.sherpaOnnx);
+
+  const sherpaModelsDir =
+    (env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() ||
+    path.join(paseoHome, "models", "sherpa-onnx");
+
+  const sherpaOnnx = shouldConfigureSherpa
+    ? {
+        modelsDir: sherpaModelsDir,
+        autoDownload:
+          env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined
+            ? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1"
+            : persisted.providers?.sherpaOnnx?.autoDownload ??
+              // In tests we should never hit the network unexpectedly.
+              Boolean(env.VITEST) === false,
+        stt: {
+          preset: normalizeSherpaSttPreset(
+            (env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() ||
+              (persisted.features?.voiceMode?.stt?.preset ??
+                persisted.features?.dictation?.stt?.preset)?.trim() ||
+              "zipformer-bilingual-zh-en-2023-02-20"
+          ),
+        },
+        tts: {
+          preset: normalizeSherpaTtsPreset(
+            (env.PASEO_SHERPA_TTS_PRESET ??
+              persisted.providers?.sherpaOnnx?.tts?.preset ??
+              persisted.features?.voiceMode?.tts?.preset)?.trim() ||
+              (env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8")
+          ),
+          speakerId:
+            env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined
+              ? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10)
+              : persisted.providers?.sherpaOnnx?.tts?.speakerId ??
+                persisted.features?.voiceMode?.tts?.speakerId,
+          speed:
+            env.PASEO_SHERPA_TTS_SPEED !== undefined
+              ? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED)
+              : persisted.providers?.sherpaOnnx?.tts?.speed ??
+                persisted.features?.voiceMode?.tts?.speed,
+        },
+      }
+    : undefined;
+
  const openrouterApiKey =
    env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null;
  const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null;
@@ -158,6 +265,12 @@ export function loadConfig(
    relayPublicEndpoint,
    appBaseUrl,
    openai,
+    speech: {
+      dictationSttProvider,
+      voiceSttProvider,
+      voiceTtsProvider,
+      ...(sherpaOnnx ? { sherpaOnnx } : {}),
+    },
    openrouterApiKey,
    voiceLlmModel,
  };
--- a/packages/server/src/server/daemon-client.e2e.test.ts
+++ b/packages/server/src/server/daemon-client.e2e.test.ts
@@ -1,6 +1,6 @@
 import { describe, test, expect, beforeAll, afterAll, beforeEach } from "vitest";
 import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs";
-import { tmpdir } from "node:os";
+import { tmpdir, homedir } from "node:os";
 import path from "node:path";
 import { execSync } from "node:child_process";

@@ -13,12 +13,45 @@ import { getFullAccessConfig, getAskModeConfig } from "./daemon-e2e/agent-config
 import {
  chunkPcm16,
  parsePcm16MonoWav,
-  requireEnv,
-  transcribeBaselineOpenAI,
  wordSimilarity,
 } from "./test-utils/dictation-e2e.js";

-const hasOpenAICredentials = !!process.env.OPENAI_API_KEY;
+const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
+
+const sherpaModelsDir =
+  process.env.PASEO_SHERPA_ONNX_MODELS_DIR ??
+  path.join(homedir(), ".paseo", "models", "sherpa-onnx");
+
+function hasSherpaZipformerModels(modelsDir: string): boolean {
+  return (
+    existsSync(
+      path.join(
+        modelsDir,
+        "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
+        "encoder-epoch-99-avg-1.onnx"
+      )
+    ) &&
+    existsSync(
+      path.join(
+        modelsDir,
+        "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
+        "tokens.txt"
+      )
+    )
+  );
+}
+
+function hasSherpaKittenModels(modelsDir: string): boolean {
+  return (
+    existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "model.fp16.onnx")) &&
+    existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "voices.bin")) &&
+    existsSync(path.join(modelsDir, "kitten-nano-en-v0_1-fp16", "tokens.txt"))
+  );
+}
+
+const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir);
+const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
+const speechTest = hasAnySpeech ? test : test.skip;

 function tmpCwd(): string {
  return mkdtempSync(path.join(tmpdir(), "daemon-client-"));
@@ -55,14 +88,27 @@ function waitForSignal<T>(
  });
 }

-(hasOpenAICredentials ? describe : describe.skip)("daemon client E2E", () => {
+describe("daemon client E2E", () => {
  let ctx: DaemonTestContext;

  beforeAll(async () => {
-    const openaiApiKey = process.env.OPENAI_API_KEY ?? "";
    ctx = await createDaemonTestContext({
      dictationFinalTimeoutMs: 5000,
-      openai: { apiKey: openaiApiKey },
+      ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}),
+      speech: {
+        dictationSttProvider: "local",
+        voiceSttProvider: "local",
+        voiceTtsProvider: "local",
+        sherpaOnnx: {
+          modelsDir: sherpaModelsDir,
+          stt: {
+            preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20",
+          },
+          tts: {
+            preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16",
+          },
+        },
+      },
    });
  }, 60000);

@@ -569,11 +615,9 @@ function waitForSignal<T>(
    120000
  );

-  test(
+  speechTest(
    "voice mode buffers audio until isLast and emits transcription_result",
    async () => {
-      requireEnv("OPENAI_API_KEY");
-
      await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`);

      const transcription = waitForSignal(30_000, (resolve) => {
@@ -675,11 +719,9 @@ function waitForSignal<T>(
    90_000
  );

-  test(
-    "streams dictation PCM and returns final transcript via OpenAI Realtime transcription",
+  speechTest(
+    "streams dictation PCM and returns final transcript",
    async () => {
-      requireEnv("OPENAI_API_KEY");
-
      const fixturePath = path.resolve(
        process.cwd(),
        "..",
@@ -713,11 +755,9 @@ function waitForSignal<T>(
    30_000
  );

-  test(
-    "realtime dictation transcript is similar to baseline (OpenAI transcriptions API)",
+  speechTest(
+    "realtime dictation transcript is similar to baseline fixture",
    async () => {
-      const apiKey = requireEnv("OPENAI_API_KEY");
-
      const fixturePath = path.resolve(
        process.cwd(),
        "..",
@@ -732,14 +772,17 @@ function waitForSignal<T>(
      const dictationId = `dict-baseline-${Date.now()}`;
      const format = "audio/pcm;rate=16000;bits=16";

-      const baseline = await transcribeBaselineOpenAI({
-        apiKey,
-        wav,
-        model: process.env.STT_MODEL ?? "whisper-1",
-        prompt:
-          process.env.OPENAI_REALTIME_DICTATION_TRANSCRIPTION_PROMPT ??
-          "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript.",
-      });
+      const baselinePath = path.resolve(
+        process.cwd(),
+        "..",
+        "app",
+        "e2e",
+        "fixtures",
+        "recording.baseline.txt"
+      );
+      const baseline = await import("node:fs/promises")
+        .then((fs) => fs.readFile(baselinePath, "utf-8"))
+        .then((text) => text.trim());

      await ctx.client.startDictationStream(dictationId, format);

@@ -755,16 +798,14 @@ function waitForSignal<T>(
      const result = await ctx.client.finishDictationStream(dictationId, finalSeq);

      expect(result.dictationId).toBe(dictationId);
-      expect(wordSimilarity(result.text, baseline)).toBeGreaterThan(0.8);
+      expect(wordSimilarity(result.text, baseline)).toBeGreaterThan(0.6);
    },
    30_000
  );

-  test(
+  speechTest(
    "fails fast if dictation finishes without sending required chunks",
    async () => {
-      requireEnv("OPENAI_API_KEY");
-
      const dictationId = `dict-missing-chunks-${Date.now()}`;
      const format = "audio/pcm;rate=16000;bits=16";

--- a/packages/server/src/server/dictation/dictation-stream-manager.test.ts
+++ b/packages/server/src/server/dictation/dictation-stream-manager.test.ts
@@ -4,23 +4,26 @@ import pino from "pino";

 import {
  DictationStreamManager,
-  type RealtimeTranscriptionSession,
-  type RealtimeTranscriptionSessionFactory,
 } from "./dictation-stream-manager.js";
+import type {
+  SpeechToTextProvider,
+  StreamingTranscriptionSession,
+} from "../speech/speech-provider.js";

-class FakeRealtimeSession extends EventEmitter implements RealtimeTranscriptionSession {
+class FakeRealtimeSession extends EventEmitter implements StreamingTranscriptionSession {
  connected = false;
-  appended: string[] = [];
+  appended: Buffer[] = [];
  commitCalls = 0;
  clearCalls = 0;
  closed = false;
+  requiredSampleRate = 24000;

  async connect(): Promise<void> {
    this.connected = true;
  }

-  appendPcm16Base64(base64Audio: string): void {
-    this.appended.push(base64Audio);
+  appendPcm16(pcm16le: Buffer): void {
+    this.appended.push(pcm16le);
  }

  commit(): void {
@@ -35,12 +38,12 @@ class FakeRealtimeSession extends EventEmitter implements RealtimeTranscriptionS
    this.closed = true;
  }

-  emitCommitted(itemId: string): void {
-    this.emit("committed", { itemId, previousItemId: null });
+  emitCommitted(segmentId: string): void {
+    this.emit("committed", { segmentId, previousSegmentId: null });
  }

-  emitTranscript(itemId: string, transcript: string, isFinal: boolean): void {
-    this.emit("transcript", { itemId, transcript, isFinal });
+  emitTranscript(segmentId: string, transcript: string, isFinal: boolean): void {
+    this.emit("transcript", { segmentId, transcript, isFinal });
  }

  emitError(message: string): void {
@@ -48,6 +51,14 @@ class FakeRealtimeSession extends EventEmitter implements RealtimeTranscriptionS
  }
 }

+class FakeSttProvider implements SpeechToTextProvider {
+  public readonly id = "fake";
+  constructor(private readonly session: FakeRealtimeSession) {}
+  createSession(_params: { logger: any; language?: string; prompt?: string }): StreamingTranscriptionSession {
+    return this.session;
+  }
+}
+
 const buildPcmBase64 = (sampleValue: number, sampleCount: number): string => {
  const samples = new Int16Array(sampleCount);
  samples.fill(sampleValue);
@@ -59,34 +70,29 @@ const tick = async (): Promise<void> => {
  await Promise.resolve();
 };

-describe("DictationStreamManager (semantic VAD grace fallback)", () => {
+describe("DictationStreamManager (finish buffer-too-small tolerance)", () => {
  const env = {
-    turnDetection: process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION,
    dictationDebug: process.env.PASEO_DICTATION_DEBUG,
  };

  beforeEach(() => {
    vi.useFakeTimers();
-    process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION = "semantic_vad";
    process.env.PASEO_DICTATION_DEBUG = "false";
  });

  afterEach(() => {
    vi.useRealTimers();
-    process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION = env.turnDetection;
    process.env.PASEO_DICTATION_DEBUG = env.dictationDebug;
  });

  it("treats buffer-too-small as benign and finalizes with existing transcripts", async () => {
    const session = new FakeRealtimeSession();
-    const factory: RealtimeTranscriptionSessionFactory = () => session;
    const emitted: Array<{ type: string; payload: any }> = [];
    const manager = new DictationStreamManager({
      logger: pino({ level: "silent" }),
      emit: (msg) => emitted.push(msg),
      sessionId: "s1",
-      openaiApiKey: "k",
-      sessionFactory: factory,
+      stt: new FakeSttProvider(session),
      finalTimeoutMs: 5000,
    });

@@ -98,14 +104,11 @@ describe("DictationStreamManager (semantic VAD grace fallback)", () => {
      format: "audio/pcm;rate=24000;bits=16",
    });

-    session.emitTranscript("i1", "hello world", true);
+    session.emitTranscript("seg-1", "hello world", true);

    await manager.handleFinish("d1", 0);
    await tick();

-    vi.advanceTimersByTime(2000);
-    await tick();
-
    session.emitError(
      "Error committing input audio buffer: buffer too small. Expected at least 100ms of audio, but buffer only has 0.00ms of audio."
    );
@@ -117,37 +120,33 @@ describe("DictationStreamManager (semantic VAD grace fallback)", () => {
    expect(final?.payload.text).toBe("hello world");
    expect(session.closed).toBe(true);
  });
+});

-  it("does not fallback-commit if committed event arrives during grace window", async () => {
-    const session = new FakeRealtimeSession();
-    const factory: RealtimeTranscriptionSessionFactory = () => session;
-    const emitted: Array<{ type: string; payload: any }> = [];
-    const manager = new DictationStreamManager({
-      logger: pino({ level: "silent" }),
-      emit: (msg) => emitted.push(msg),
-      sessionId: "s1",
-      openaiApiKey: "k",
-      sessionFactory: factory,
-      finalTimeoutMs: 5000,
-    });
+describe("DictationStreamManager (provider-agnostic provider)", () => {
+  it("does not require OPENAI_API_KEY", async () => {
+    const original = process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_API_KEY;

-    await manager.handleStart("d1", "audio/pcm;rate=24000;bits=16");
-    await manager.handleChunk({
-      dictationId: "d1",
-      seq: 0,
-      audioBase64: buildPcmBase64(2000, 2400),
-      format: "audio/pcm;rate=24000;bits=16",
-    });
+    try {
+      const session = new FakeRealtimeSession();
+      const emitted: Array<{ type: string; payload: any }> = [];
+      const manager = new DictationStreamManager({
+        logger: pino({ level: "silent" }),
+        emit: (msg) => emitted.push(msg),
+        sessionId: "s1",
+        stt: new FakeSttProvider(session),
+      });

-    await manager.handleFinish("d1", 0);
-    session.emitCommitted("i1");
-    session.emitTranscript("i1", "hi there", true);
+      await manager.handleStart("d-local", "audio/pcm;rate=16000;bits=16");

-    vi.advanceTimersByTime(2000);
-    await tick();
-
-    expect(session.commitCalls).toBe(0);
-    const final = emitted.find((msg) => msg.type === "dictation_stream_final");
-    expect(final?.payload.text).toBe("hi there");
+      expect(session.connected).toBe(true);
+      expect(emitted.find((msg) => msg.type === "dictation_stream_error")).toBeUndefined();
+    } finally {
+      if (original !== undefined) {
+        process.env.OPENAI_API_KEY = original;
+      } else {
+        delete process.env.OPENAI_API_KEY;
+      }
+    }
  });
 });
--- a/packages/server/src/server/dictation/dictation-stream-manager.ts
+++ b/packages/server/src/server/dictation/dictation-stream-manager.ts
@@ -7,112 +7,19 @@ import {
 } from "../agent/dictation-debug.js";
 import { isPaseoDictationDebugEnabled } from "../agent/recordings-debug.js";
 import { Pcm16MonoResampler } from "../agent/pcm16-resampler.js";
-import { OpenAIRealtimeTranscriptionSession } from "../agent/openai-realtime-transcription.js";
+import type {
+  SpeechToTextProvider,
+  StreamingTranscriptionSession,
+} from "../speech/speech-provider.js";
+import { parsePcmRateFromFormat, pcm16lePeakAbs } from "../speech/audio.js";

 const PCM_CHANNELS = 1;
 const PCM_BITS_PER_SAMPLE = 16;
-const DICTATION_PCM_OUTPUT_RATE = 24000;
 const DEFAULT_DICTATION_FINAL_TIMEOUT_MS = 10000;
-const DICTATION_VAD_GRACE_TIMEOUT_MS = Number.parseInt(
-  process.env.OPENAI_REALTIME_DICTATION_VAD_GRACE_TIMEOUT_MS ?? "2000",
-  10
-);
 const DICTATION_SILENCE_PEAK_THRESHOLD = Number.parseInt(
-  process.env.OPENAI_REALTIME_DICTATION_SILENCE_PEAK_THRESHOLD ?? "300",
+  process.env.PASEO_DICTATION_SILENCE_PEAK_THRESHOLD ?? "300",
  10
 );
-const DICTATION_TURN_DETECTION = (
-  process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION ?? "semantic_vad"
-).trim();
-const DICTATION_SEMANTIC_VAD_EAGERNESS = (
-  process.env.OPENAI_REALTIME_DICTATION_SEMANTIC_VAD_EAGERNESS ?? "medium"
-).trim();
-const DICTATION_FLUSH_SILENCE_MS = Number.parseInt(
-  process.env.OPENAI_REALTIME_DICTATION_FLUSH_SILENCE_MS ?? "800",
-  10
-);
-
-type OpenAITurnDetection =
-  | null
-  | {
-      type: "server_vad";
-      create_response: false;
-      threshold?: number;
-      prefix_padding_ms?: number;
-      silence_duration_ms?: number;
-    }
-  | { type: "semantic_vad"; create_response: false; eagerness?: "low" | "medium" | "high" };
-
-function pcm16lePeakAbs(pcm16le: Buffer): number {
-  if (pcm16le.length === 0) {
-    return 0;
-  }
-  if (pcm16le.length % 2 !== 0) {
-    throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
-  }
-  const samples = new Int16Array(
-    pcm16le.buffer,
-    pcm16le.byteOffset,
-    pcm16le.byteLength / 2
-  );
-  let peak = 0;
-  for (let i = 0; i < samples.length; i += 1) {
-    const v = samples[i]!;
-    const abs = v < 0 ? -v : v;
-    if (abs > peak) {
-      peak = abs;
-      if (peak >= 32767) {
-        break;
-      }
-    }
-  }
-  return peak;
-}
-
-function parseDictationTurnDetection(): OpenAITurnDetection {
-  if (
-    !DICTATION_TURN_DETECTION ||
-    DICTATION_TURN_DETECTION === "none" ||
-    DICTATION_TURN_DETECTION === "null"
-  ) {
-    return null;
-  }
-  if (DICTATION_TURN_DETECTION === "server_vad") {
-    return { type: "server_vad", create_response: false };
-  }
-  const eagerness =
-    DICTATION_SEMANTIC_VAD_EAGERNESS === "low" ||
-    DICTATION_SEMANTIC_VAD_EAGERNESS === "high"
-      ? (DICTATION_SEMANTIC_VAD_EAGERNESS as "low" | "high")
-      : ("medium" as const);
-  return { type: "semantic_vad", create_response: false, eagerness };
-}
-
-export type RealtimeTranscriptionSession = {
-  connect(): Promise<void>;
-  appendPcm16Base64(base64Audio: string): void;
-  commit(): void;
-  clear(): void;
-  close(): void;
-  on(
-    event: "committed",
-    handler: (payload: { itemId: string; previousItemId: string | null }) => void
-  ): unknown;
-  on(
-    event: "transcript",
-    handler: (payload: { itemId: string; transcript: string; isFinal: boolean }) => void
-  ): unknown;
-  on(event: "error", handler: (err: unknown) => void): unknown;
-};
-
-export type RealtimeTranscriptionSessionFactory = (params: {
-  apiKey: string;
-  logger: pino.Logger;
-  transcriptionModel: string;
-  language?: string;
-  prompt?: string;
-  turnDetection: OpenAITurnDetection;
-}) => RealtimeTranscriptionSession;

 function convertPCMToWavBuffer(
  pcmBuffer: Buffer,
@@ -147,8 +54,9 @@ type DictationStreamState = {
  dictationId: string;
  sessionId: string;
  inputFormat: string;
-  openai: RealtimeTranscriptionSession;
+  stt: StreamingTranscriptionSession;
  inputRate: number;
+  outputRate: number;
  resampler: Pcm16MonoResampler | null;
  debugAudioChunks: Buffer[];
  debugRecordingPath: string | null;
@@ -158,17 +66,14 @@ type DictationStreamState = {
  ackSeq: number;
  bytesSinceCommit: number;
  peakSinceCommit: number;
-  committedItemIds: string[];
-  transcriptsByItemId: Map<string, string>;
-  finalTranscriptItemIds: Set<string>;
+  committedSegmentIds: string[];
+  transcriptsBySegmentId: Map<string, string>;
+  finalTranscriptSegmentIds: Set<string>;
  awaitingFinalCommit: boolean;
-  vadGraceTimeout: ReturnType<typeof setTimeout> | null;
-  fallbackCommitAttempted: boolean;
  finishRequested: boolean;
  finishSealed: boolean;
  finalSeq: number | null;
  finalTimeout: ReturnType<typeof setTimeout> | null;
-  isSemanticVad: boolean;
 };

 export type DictationStreamOutboundMessage =
@@ -191,28 +96,22 @@ export class DictationStreamManager {
  private readonly logger: pino.Logger;
  private readonly emit: (msg: DictationStreamOutboundMessage) => void;
  private readonly sessionId: string;
-  private readonly openaiApiKey: string | null;
+  private readonly stt: SpeechToTextProvider | null;
  private readonly finalTimeoutMs: number;
-  private readonly createSession: RealtimeTranscriptionSessionFactory;
  private readonly streams = new Map<string, DictationStreamState>();

  constructor(params: {
    logger: pino.Logger;
    emit: (msg: DictationStreamOutboundMessage) => void;
    sessionId: string;
-    openaiApiKey?: string | null;
+    stt: SpeechToTextProvider | null;
    finalTimeoutMs?: number;
-    sessionFactory?: RealtimeTranscriptionSessionFactory;
  }) {
    this.logger = params.logger.child({ component: "dictation-stream-manager" });
    this.emit = params.emit;
    this.sessionId = params.sessionId;
-    this.openaiApiKey = params.openaiApiKey ?? null;
+    this.stt = params.stt;
    this.finalTimeoutMs = params.finalTimeoutMs ?? DEFAULT_DICTATION_FINAL_TIMEOUT_MS;
-    this.createSession =
-      params.sessionFactory ??
-      ((factoryParams) =>
-        new OpenAIRealtimeTranscriptionSession(factoryParams));
  }

  public cleanupAll(): void {
@@ -224,39 +123,30 @@ export class DictationStreamManager {
  public async handleStart(dictationId: string, format: string): Promise<void> {
    this.cleanupDictationStream(dictationId);

-    const apiKey = this.openaiApiKey ?? process.env.OPENAI_API_KEY;
-    if (!apiKey) {
-      this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false);
+    if (!this.stt) {
+      this.failDictationStream(dictationId, "Dictation STT not configured", false);
      return;
    }

-    const transcriptionModel =
-      process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
    const transcriptionPrompt =
-      process.env.OPENAI_REALTIME_DICTATION_TRANSCRIPTION_PROMPT ??
+      process.env.PASEO_DICTATION_TRANSCRIPTION_PROMPT ??
      "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript.";

-    const turnDetection = parseDictationTurnDetection();
-    const openai = this.createSession({
-      apiKey,
+    const stt = this.stt.createSession({
      logger: this.logger.child({ dictationId }),
-      transcriptionModel,
      language: "en",
      prompt: transcriptionPrompt,
-      turnDetection,
    });

-    openai.on("committed", ({ itemId }: { itemId: string }) => {
+    stt.on("committed", ({ segmentId }) => {
      const state = this.streams.get(dictationId);
      if (!state) {
        return;
      }
-      this.clearVadGraceTimeout(state);
-      state.committedItemIds.push(itemId);
+      state.committedSegmentIds.push(segmentId);
      state.bytesSinceCommit = 0;
      state.peakSinceCommit = 0;

-      // When finishing, we require at least one commit after finish if we flushed pending audio.
      if (state.finishRequested && state.awaitingFinalCommit) {
        state.awaitingFinalCommit = false;
      }
@@ -264,52 +154,37 @@ export class DictationStreamManager {
      this.maybeFinalizeDictationStream(dictationId);
    });

-    openai.on(
-      "transcript",
-      ({
-        itemId,
-        transcript,
-        isFinal,
-      }: {
-        itemId: string;
-        transcript: string;
-        isFinal: boolean;
-      }) => {
-        const state = this.streams.get(dictationId);
-        if (!state) {
-          return;
-        }
-        state.transcriptsByItemId.set(itemId, transcript);
-        if (isFinal) {
-          state.finalTranscriptItemIds.add(itemId);
-        }
-
-        // If we triggered a finish commit but OpenAI doesn't emit committed events (or they arrive late),
-        // allow final transcripts to unblock finalization.
-        if (state.finishRequested && state.awaitingFinalCommit && isFinal) {
-          this.clearVadGraceTimeout(state);
-          state.awaitingFinalCommit = false;
-        }
-
-        const orderedIds = state.committedItemIds.includes(itemId)
-          ? state.committedItemIds
-          : [...state.committedItemIds, itemId];
-        const partialText = orderedIds
-          .map((id) => state.transcriptsByItemId.get(id) ?? "")
-          .join(" ")
-          .trim();
-        this.emitDictationPartial(dictationId, partialText);
-
-        this.maybeSealDictationStreamFinish(dictationId);
-        this.maybeFinalizeDictationStream(dictationId);
+    stt.on("transcript", ({ segmentId, transcript, isFinal }) => {
+      const state = this.streams.get(dictationId);
+      if (!state) {
+        return;
+      }
+      state.transcriptsBySegmentId.set(segmentId, transcript);
+      if (isFinal) {
+        state.finalTranscriptSegmentIds.add(segmentId);
      }
-    );

-    openai.on("error", (err) => {
+      if (state.finishRequested && state.awaitingFinalCommit && isFinal) {
+        state.awaitingFinalCommit = false;
+      }
+
+      const orderedIds = state.committedSegmentIds.includes(segmentId)
+        ? state.committedSegmentIds
+        : [...state.committedSegmentIds, segmentId];
+      const partialText = orderedIds
+        .map((id) => state.transcriptsBySegmentId.get(id) ?? "")
+        .join(" ")
+        .trim();
+      this.emitDictationPartial(dictationId, partialText);
+
+      this.maybeSealDictationStreamFinish(dictationId);
+      this.maybeFinalizeDictationStream(dictationId);
+    });
+
+    stt.on("error", (err) => {
      const message = err instanceof Error ? err.message : String(err);
      const state = this.streams.get(dictationId);
      if (state && state.finishRequested && isBufferTooSmallError(message)) {
-        this.clearVadGraceTimeout(state);
        if (state.awaitingFinalCommit) {
          state.awaitingFinalCommit = false;
        }
@@ -319,14 +194,13 @@ export class DictationStreamManager {
      void this.failAndCleanupDictationStream(dictationId, message, true);
    });

-    await openai.connect();
+    await stt.connect();

-    const rateMatch = /(?:^|[;,\s])rate\s*=\s*(\d+)(?:$|[;,\s])/i.exec(format);
-    const inputRate = rateMatch ? Number.parseInt(rateMatch[1]!, 10) : 16000;
+    const inputRate = parsePcmRateFromFormat(format, 16000) ?? 16000;
    if (!Number.isFinite(inputRate) || inputRate <= 0) {
      this.failDictationStream(dictationId, `Invalid dictation input rate in format: ${format}`, false);
      try {
-        openai.close();
+        stt.close();
      } catch {
        // no-op
      }
@@ -338,18 +212,21 @@ export class DictationStreamManager {
      this.logger
    );

+    const outputRate = stt.requiredSampleRate;
+
    this.streams.set(dictationId, {
      dictationId,
      sessionId: this.sessionId,
      inputFormat: format,
-      openai,
+      stt,
      inputRate,
+      outputRate,
      resampler:
-        inputRate === DICTATION_PCM_OUTPUT_RATE
+        inputRate === outputRate
          ? null
          : new Pcm16MonoResampler({
              inputRate,
-              outputRate: DICTATION_PCM_OUTPUT_RATE,
+              outputRate,
            }),
      debugAudioChunks: [],
      debugRecordingPath: null,
@@ -359,17 +236,14 @@ export class DictationStreamManager {
      ackSeq: -1,
      bytesSinceCommit: 0,
      peakSinceCommit: 0,
-      committedItemIds: [],
-      transcriptsByItemId: new Map(),
-      finalTranscriptItemIds: new Set(),
+      committedSegmentIds: [],
+      transcriptsBySegmentId: new Map(),
+      finalTranscriptSegmentIds: new Set(),
      awaitingFinalCommit: false,
-      vadGraceTimeout: null,
-      fallbackCommitAttempted: false,
      finishRequested: false,
      finishSealed: false,
      finalSeq: null,
      finalTimeout: null,
-      isSemanticVad: turnDetection?.type === "semantic_vad",
    });

    this.emitDictationAck(dictationId, -1);
@@ -412,7 +286,7 @@ export class DictationStreamManager {

      const resampled = state.resampler ? state.resampler.processChunk(pcm16) : pcm16;
      if (resampled.length > 0) {
-        state.openai.appendPcm16Base64(resampled.toString("base64"));
+        state.stt.appendPcm16(resampled);
        state.debugAudioChunks.push(resampled);
        state.bytesSinceCommit += resampled.length;
        state.peakSinceCommit = Math.max(state.peakSinceCommit, pcm16lePeakAbs(resampled));
@@ -510,7 +384,7 @@ export class DictationStreamManager {
    const pcmBuffer = Buffer.concat(state.debugAudioChunks);
    const wavBuffer = convertPCMToWavBuffer(
      pcmBuffer,
-      DICTATION_PCM_OUTPUT_RATE,
+      state.outputRate,
      PCM_CHANNELS,
      PCM_BITS_PER_SAMPLE
    );
@@ -566,12 +440,11 @@ export class DictationStreamManager {
    if (!state) {
      return;
    }
-    this.clearVadGraceTimeout(state);
    if (state.finalTimeout) {
      clearTimeout(state.finalTimeout);
    }
    try {
-      state.openai.close();
+      state.stt.close();
    } catch {
      // no-op
    }
@@ -603,37 +476,18 @@ export class DictationStreamManager {
          },
          "Dictation finish: clearing silence-only tail (skip final commit)"
        );
-        state.openai.clear();
+        state.stt.clear();
        state.bytesSinceCommit = 0;
        state.peakSinceCommit = 0;
        state.awaitingFinalCommit = false;
      } else {
-        const silenceBytes = Math.max(
-          0,
-          Math.round((DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000)
-        );
-        if (silenceBytes > 0) {
-          this.logger.debug(
-            { dictationId, silenceMs: DICTATION_FLUSH_SILENCE_MS, silenceBytes },
-            "Dictation finish: appending silence tail for semantic VAD flush"
-          );
-          const silence = Buffer.alloc(silenceBytes);
-          state.openai.appendPcm16Base64(silence.toString("base64"));
-          state.debugAudioChunks.push(silence);
-          state.bytesSinceCommit += silenceBytes;
-        }
-
        state.awaitingFinalCommit = true;
-        if (state.isSemanticVad) {
-          this.startVadGraceTimeout(state);
-        } else {
-          try {
-            state.openai.commit();
-          } catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            void this.failAndCleanupDictationStream(dictationId, message, true);
-            return;
-          }
+        try {
+          state.stt.commit();
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          void this.failAndCleanupDictationStream(dictationId, message, true);
+          return;
        }
      }
    } else {
@@ -659,15 +513,15 @@ export class DictationStreamManager {
      return;
    }

-    const committedSet = new Set(state.committedItemIds);
-    const orderedItemIds: string[] = [...state.committedItemIds];
-    for (const itemId of state.transcriptsByItemId.keys()) {
-      if (!committedSet.has(itemId)) {
-        orderedItemIds.push(itemId);
+    const committedSet = new Set(state.committedSegmentIds);
+    const orderedSegmentIds: string[] = [...state.committedSegmentIds];
+    for (const segmentId of state.transcriptsBySegmentId.keys()) {
+      if (!committedSet.has(segmentId)) {
+        orderedSegmentIds.push(segmentId);
      }
    }

-    if (orderedItemIds.length === 0) {
+    if (orderedSegmentIds.length === 0) {
      void (async () => {
        const debugRecordingPath = await this.maybePersistDictationStreamAudio(dictationId);
        this.emit({
@@ -695,15 +549,15 @@ export class DictationStreamManager {
      return;
    }

-    const allTranscriptsReady = orderedItemIds.every((itemId) =>
-      state.finalTranscriptItemIds.has(itemId)
+    const allTranscriptsReady = orderedSegmentIds.every((segmentId) =>
+      state.finalTranscriptSegmentIds.has(segmentId)
    );
    if (!allTranscriptsReady) {
      return;
    }

-    const orderedText = orderedItemIds
-      .map((itemId) => state.transcriptsByItemId.get(itemId) ?? "")
+    const orderedText = orderedSegmentIds
+      .map((segmentId) => state.transcriptsBySegmentId.get(segmentId) ?? "")
      .join(" ")
      .trim();

@@ -732,41 +586,6 @@ export class DictationStreamManager {
      this.cleanupDictationStream(dictationId);
    })();
  }
-
-  private startVadGraceTimeout(state: DictationStreamState): void {
-    if (state.vadGraceTimeout || DICTATION_VAD_GRACE_TIMEOUT_MS <= 0) {
-      return;
-    }
-    state.vadGraceTimeout = setTimeout(() => {
-      state.vadGraceTimeout = null;
-      if (!state.finishRequested || !state.awaitingFinalCommit) {
-        return;
-      }
-      if (state.bytesSinceCommit <= 0 || state.fallbackCommitAttempted) {
-        return;
-      }
-      state.fallbackCommitAttempted = true;
-      try {
-        state.openai.commit();
-      } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        if (isBufferTooSmallError(message)) {
-          state.awaitingFinalCommit = false;
-          this.maybeFinalizeDictationStream(state.dictationId);
-          return;
-        }
-        void this.failAndCleanupDictationStream(state.dictationId, message, true);
-      }
-    }, DICTATION_VAD_GRACE_TIMEOUT_MS);
-  }
-
-  private clearVadGraceTimeout(state: DictationStreamState): void {
-    if (!state.vadGraceTimeout) {
-      return;
-    }
-    clearTimeout(state.vadGraceTimeout);
-    state.vadGraceTimeout = null;
-  }
 }

 function isBufferTooSmallError(message: string): boolean {
--- a/packages/server/src/server/persisted-config.ts
+++ b/packages/server/src/server/persisted-config.ts
@@ -17,19 +17,52 @@ const ProviderCredentialsSchema = z
  })
  .strict();

+const SherpaOnnxProviderSchema = z
+  .object({
+    modelsDir: z.string().min(1).optional(),
+    autoDownload: z.boolean().optional(),
+    stt: z
+      .object({
+        preset: z.string().min(1).optional(),
+      })
+      .strict()
+      .optional(),
+    tts: z
+      .object({
+        preset: z.string().min(1).optional(),
+        speakerId: z.number().int().optional(),
+        speed: z.number().optional(),
+      })
+      .strict()
+      .optional(),
+  })
+  .strict();
+
 const ProvidersSchema = z
  .object({
    openai: ProviderCredentialsSchema.optional(),
    openrouter: ProviderCredentialsSchema.optional(),
+    sherpaOnnx: SherpaOnnxProviderSchema.optional(),
  })
  .strict();

+const SpeechProviderIdSchema = z.preprocess(
+  (value) => {
+    if (typeof value !== "string") {
+      return value;
+    }
+    return value.trim().toLowerCase();
+  },
+  z.enum(["openai", "local"])
+);
+
 const FeatureDictationSchema = z
  .object({
    stt: z
      .object({
-        provider: z.enum(["openai"]).optional(),
+        provider: SpeechProviderIdSchema.optional(),
        model: z.string().min(1).optional(),
+        preset: z.string().min(1).optional(),
        confidenceThreshold: z.number().optional(),
      })
      .strict()
@@ -48,16 +81,20 @@ const FeatureVoiceModeSchema = z
      .optional(),
    stt: z
      .object({
-        provider: z.enum(["openai"]).optional(),
+        provider: SpeechProviderIdSchema.optional(),
        model: z.string().min(1).optional(),
+        preset: z.string().min(1).optional(),
      })
      .strict()
      .optional(),
    tts: z
      .object({
-        provider: z.enum(["openai"]).optional(),
+        provider: SpeechProviderIdSchema.optional(),
        model: z.enum(["tts-1", "tts-1-hd"]).optional(),
        voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(),
+        preset: z.string().min(1).optional(),
+        speakerId: z.number().int().optional(),
+        speed: z.number().optional(),
      })
      .strict()
      .optional(),
--- a/packages/server/src/server/session.ts
+++ b/packages/server/src/server/session.ts
@@ -33,8 +33,7 @@ import { getSystemPrompt } from "./agent/system-prompt.js";
 import { getAllTools } from "./agent/llm-openai.js";
 import { TTSManager } from "./agent/tts-manager.js";
 import { STTManager } from "./agent/stt-manager.js";
-import type { OpenAISTT } from "./agent/stt-openai.js";
-import type { OpenAITTS } from "./agent/tts-openai.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
 import { maybePersistTtsDebugAudio } from "./agent/tts-debug.js";
 import { isPaseoDictationDebugEnabled } from "./agent/recordings-debug.js";
 import {
@@ -106,6 +105,14 @@ import {
 } from "../utils/checkout-git.js";
 import { getProjectIcon } from "../utils/project-icon.js";
 import { expandTilde } from "../utils/path.js";
+import {
+  ensureSherpaOnnxModels,
+  getSherpaOnnxModelDir,
+} from "./speech/providers/local/sherpa/model-downloader.js";
+import {
+  listSherpaOnnxModels,
+  type SherpaOnnxModelId,
+} from "./speech/providers/local/sherpa/model-catalog.js";
 import type pino from "pino";

 const execAsync = promisify(exec);
@@ -325,8 +332,8 @@ export class Session {
    agentManager: AgentManager,
    agentStorage: AgentStorage,
    createAgentMcpTransport: AgentMcpTransportFactory,
-    stt: OpenAISTT | null,
-    tts: OpenAITTS | null,
+    stt: SpeechToTextProvider | null,
+    tts: TextToSpeechProvider | null,
    terminalManager: TerminalManager | null,
    voiceConversationStore: VoiceConversationStore,
    voice?: {
@@ -334,8 +341,8 @@ export class Session {
      voiceLlmModel?: string | null;
    },
    dictation?: {
-      openaiApiKey?: string | null;
      finalTimeoutMs?: number;
+      stt?: SpeechToTextProvider | null;
    }
  ) {
    this.clientId = clientId;
@@ -366,7 +373,7 @@ export class Session {
      logger: this.sessionLogger,
      sessionId: this.sessionId,
      emit: (msg) => this.emit(msg as unknown as SessionOutboundMessage),
-      openaiApiKey: dictation?.openaiApiKey ?? null,
+      stt: dictation?.stt ?? null,
      finalTimeoutMs: dictation?.finalTimeoutMs,
    });

@@ -991,6 +998,14 @@ export class Session {
          await this.handleListProviderModelsRequest(msg);
          break;

+        case "speech_models_list_request":
+          await this.handleSpeechModelsListRequest(msg);
+          break;
+
+        case "speech_models_download_request":
+          await this.handleSpeechModelsDownloadRequest(msg);
+          break;
+
        case "clear_agent_attention":
          await this.handleClearAgentAttention(msg.agentId);
          break;
@@ -1926,6 +1941,114 @@ export class Session {
    }
  }

+  private async handleSpeechModelsListRequest(
+    msg: Extract<SessionInboundMessage, { type: "speech_models_list_request" }>
+  ): Promise<void> {
+    const modelsDir =
+      process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() ||
+      join(this.paseoHome, "models", "sherpa-onnx");
+
+    const models = await Promise.all(
+      listSherpaOnnxModels().map(async (model) => {
+        const modelDir = getSherpaOnnxModelDir(modelsDir, model.id);
+        const missingFiles: string[] = [];
+        for (const rel of model.requiredFiles) {
+          const filePath = join(modelDir, rel);
+          try {
+            const fileStat = await stat(filePath);
+            if (fileStat.isDirectory()) {
+              continue;
+            }
+            if (!fileStat.isFile() || fileStat.size <= 0) {
+              missingFiles.push(rel);
+            }
+          } catch {
+            missingFiles.push(rel);
+          }
+        }
+
+        return {
+          id: model.id,
+          kind: model.kind,
+          description: model.description,
+          modelDir,
+          isDownloaded: missingFiles.length === 0,
+          ...(missingFiles.length > 0 ? { missingFiles } : {}),
+        };
+      })
+    );
+
+    this.emit({
+      type: "speech_models_list_response",
+      payload: {
+        modelsDir,
+        models,
+        requestId: msg.requestId,
+      },
+    });
+  }
+
+  private async handleSpeechModelsDownloadRequest(
+    msg: Extract<SessionInboundMessage, { type: "speech_models_download_request" }>
+  ): Promise<void> {
+    const modelsDir =
+      process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() ||
+      join(this.paseoHome, "models", "sherpa-onnx");
+
+    const modelIdsRaw =
+      msg.modelIds && msg.modelIds.length > 0
+        ? msg.modelIds
+        : [
+            process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20",
+            process.env.PASEO_SHERPA_TTS_PRESET ?? "pocket-tts-onnx-int8",
+          ];
+
+    const allModelIds = new Set(listSherpaOnnxModels().map((m) => m.id));
+    const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as SherpaOnnxModelId));
+    if (invalid.length > 0) {
+      this.emit({
+        type: "speech_models_download_response",
+        payload: {
+          modelsDir,
+          downloadedModelIds: [],
+          error: `Unknown speech model id(s): ${invalid.join(", ")}`,
+          requestId: msg.requestId,
+        },
+      });
+      return;
+    }
+
+    const modelIds = modelIdsRaw as SherpaOnnxModelId[];
+    try {
+      await ensureSherpaOnnxModels({
+        modelsDir,
+        modelIds,
+        autoDownload: true,
+        logger: this.sessionLogger,
+      });
+      this.emit({
+        type: "speech_models_download_response",
+        payload: {
+          modelsDir,
+          downloadedModelIds: modelIds,
+          error: null,
+          requestId: msg.requestId,
+        },
+      });
+    } catch (error) {
+      this.sessionLogger.error({ err: error, modelIds }, "Failed to download speech models");
+      this.emit({
+        type: "speech_models_download_response",
+        payload: {
+          modelsDir,
+          downloadedModelIds: [],
+          error: error instanceof Error ? error.message : String(error),
+          requestId: msg.requestId,
+        },
+      });
+    }
+  }
+
  private normalizeGitOptions(
    gitOptions?: GitSetupOptions,
    legacyWorktreeName?: string
--- a/packages/server/src/server/speech/audio.ts
+++ b/packages/server/src/server/speech/audio.ts
@@ -0,0 +1,113 @@
+export function parsePcm16MonoWav(buffer: Buffer): { sampleRate: number; pcm16: Buffer } {
+  if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
+    throw new Error("Invalid WAV header");
+  }
+
+  let offset = 12;
+  let fmt:
+    | { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number }
+    | null = null;
+  let dataChunk: Buffer | null = null;
+
+  while (offset + 8 <= buffer.length) {
+    const id = buffer.toString("ascii", offset, offset + 4);
+    const size = buffer.readUInt32LE(offset + 4);
+    const payloadStart = offset + 8;
+    const payloadEnd = payloadStart + size;
+    if (payloadEnd > buffer.length) {
+      break;
+    }
+
+    if (id === "fmt ") {
+      const audioFormat = buffer.readUInt16LE(payloadStart);
+      const channels = buffer.readUInt16LE(payloadStart + 2);
+      const sampleRate = buffer.readUInt32LE(payloadStart + 4);
+      const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
+      fmt = { audioFormat, channels, sampleRate, bitsPerSample };
+    } else if (id === "data") {
+      dataChunk = buffer.subarray(payloadStart, payloadEnd);
+    }
+
+    offset = payloadEnd + (size % 2);
+  }
+
+  if (!fmt || !dataChunk) {
+    throw new Error("Missing WAV fmt/data chunks");
+  }
+  if (fmt.audioFormat !== 1) {
+    throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
+  }
+  if (fmt.channels !== 1 || fmt.bitsPerSample !== 16) {
+    throw new Error(
+      `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
+    );
+  }
+  if (dataChunk.length % 2 !== 0) {
+    throw new Error("WAV PCM16 data length must be even");
+  }
+  return { sampleRate: fmt.sampleRate, pcm16: dataChunk };
+}
+
+export function parsePcmRateFromFormat(format: string, fallback: number | null = null): number | null {
+  const match = /(?:^|[;,\s])rate\s*=\s*(\d+)(?:$|[;,\s])/i.exec(format);
+  if (!match) {
+    return fallback;
+  }
+  const rate = Number.parseInt(match[1]!, 10);
+  return Number.isFinite(rate) && rate > 0 ? rate : fallback;
+}
+
+export function pcm16lePeakAbs(pcm16le: Buffer): number {
+  if (pcm16le.length === 0) {
+    return 0;
+  }
+  if (pcm16le.length % 2 !== 0) {
+    throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
+  }
+  const samples = new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2);
+  let peak = 0;
+  for (let i = 0; i < samples.length; i += 1) {
+    const v = samples[i]!;
+    const abs = v < 0 ? -v : v;
+    if (abs > peak) {
+      peak = abs;
+      if (peak >= 32767) {
+        break;
+      }
+    }
+  }
+  return peak;
+}
+
+export function pcm16leToFloat32(pcm16le: Buffer, gain: number = 1): Float32Array {
+  if (pcm16le.length % 2 !== 0) {
+    throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
+  }
+  const int16 = new Int16Array(pcm16le.buffer, pcm16le.byteOffset, pcm16le.byteLength / 2);
+  const out = new Float32Array(int16.length);
+  for (let i = 0; i < int16.length; i += 1) {
+    const v = (int16[i]! / 32768.0) * gain;
+    out[i] = v > 1 ? 1 : v < -1 ? -1 : v;
+  }
+  return out;
+}
+
+export function float32ToPcm16le(samples: Float32Array): Buffer {
+  const out = new Int16Array(samples.length);
+  for (let i = 0; i < samples.length; i += 1) {
+    const clamped = Math.max(-1, Math.min(1, samples[i]!));
+    out[i] = Math.round(clamped * 32767);
+  }
+  return Buffer.from(out.buffer, out.byteOffset, out.byteLength);
+}
+
+export function chunkBuffer(buffer: Buffer, chunkBytes: number): Buffer[] {
+  if (chunkBytes <= 0) {
+    return [buffer];
+  }
+  const out: Buffer[] = [];
+  for (let offset = 0; offset < buffer.length; offset += chunkBytes) {
+    out.push(buffer.subarray(offset, Math.min(buffer.length, offset + chunkBytes)));
+  }
+  return out;
+}
--- a/packages/server/src/server/speech/providers/local/pocket/pocket-tts-onnx.ts
+++ b/packages/server/src/server/speech/providers/local/pocket/pocket-tts-onnx.ts
@@ -0,0 +1,559 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { Readable } from "node:stream";
+import type pino from "pino";
+
+import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js";
+import { chunkBuffer, float32ToPcm16le, parsePcm16MonoWav, pcm16leToFloat32 } from "../../../audio.js";
+import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
+
+type OrtModule = typeof import("onnxruntime-node");
+type OrtSession = import("onnxruntime-node").InferenceSession;
+type OrtTensor = import("onnxruntime-node").Tensor;
+
+type SentencePieceProcessor = {
+  encodeIds: (text: string) => number[];
+  load?: (modelPath: string) => unknown;
+  Load?: (modelPath: string) => unknown;
+};
+
+function assertFileExists(filePath: string, label: string): void {
+  if (!existsSync(filePath)) {
+    throw new Error(`Missing ${label}: ${filePath}`);
+  }
+}
+
+function product(dims: number[]): number {
+  let out = 1;
+  for (const d of dims) out *= d;
+  return out;
+}
+
+function normalizeDims(dims: Array<number | string | null | undefined>): number[] {
+  // ONNX metadata can contain dynamic dimensions as strings (e.g. "batch") or -1.
+  // For state tensors we want a valid minimal shape, so coerce unknown/invalid dims to 1.
+  // Preserve explicit 0 dims (some models use empty initial state buffers with shape [0]).
+  return dims.map((d) => {
+    if (typeof d === "number" && Number.isFinite(d)) {
+      if (d === 0) return 0;
+      if (d > 0) return d;
+      return 1;
+    }
+    return 1;
+  });
+}
+
+function getSessionInputMeta(
+  session: OrtSession,
+  inputName: string
+): { type?: string; dims?: Array<number | string | null> } | undefined {
+  const metaAny = (session as any).inputMetadata as unknown;
+  if (Array.isArray(metaAny)) {
+    const entry = metaAny.find((m) => m && typeof m === "object" && (m as any).name === inputName) as any;
+    if (!entry) return undefined;
+    return { type: entry.type, dims: entry.shape };
+  }
+
+  if (metaAny && typeof metaAny === "object" && inputName in (metaAny as any)) {
+    const entry = (metaAny as any)[inputName] as any;
+    return { type: entry?.type, dims: entry?.dimensions ?? entry?.shape };
+  }
+
+  return undefined;
+}
+
+function toBigInt64(values: number[]): BigInt64Array {
+  const out = new BigInt64Array(values.length);
+  for (let i = 0; i < values.length; i += 1) {
+    out[i] = BigInt(values[i]!);
+  }
+  return out;
+}
+
+function randn(): number {
+  // Box–Muller
+  let u = 0;
+  let v = 0;
+  while (u === 0) u = Math.random();
+  while (v === 0) v = Math.random();
+  return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
+}
+
+function normalizeTextForPocket(text: string): string {
+  const trimmed = text.trim();
+  if (!trimmed) {
+    throw new Error("Cannot synthesize empty text");
+  }
+  let out = trimmed;
+  if (out.length > 0 && /[A-Za-z0-9]$/.test(out)) {
+    out = `${out}.`;
+  }
+  if (out.length > 0 && /[a-z]/.test(out[0]!)) {
+    out = out[0]!.toUpperCase() + out.slice(1);
+  }
+  return out;
+}
+
+async function loadOrt(): Promise<OrtModule> {
+  return (await import("onnxruntime-node")) as OrtModule;
+}
+
+async function loadSentencePiece(tokenizerModelPath: string): Promise<SentencePieceProcessor> {
+  const mod = await import("@sctg/sentencepiece-js");
+
+  const Processor =
+    (mod as any).SentencePieceProcessor ??
+    (mod as any).default?.SentencePieceProcessor ??
+    (mod as any).default;
+
+  if (!Processor) {
+    throw new Error("Failed to load SentencePiece processor from @sctg/sentencepiece-js");
+  }
+
+  const sp: SentencePieceProcessor = new Processor();
+
+  if (typeof sp.load === "function") {
+    await sp.load(tokenizerModelPath);
+  } else if (typeof sp.Load === "function") {
+    sp.Load(tokenizerModelPath);
+  } else {
+    throw new Error("SentencePiece processor does not expose load()/Load()");
+  }
+
+  return sp;
+}
+
+function getOrtProviders(ort: OrtModule, device: "auto" | "cpu" | "cuda"): string[] {
+  // NOTE: onnxruntime-node uses backend names like "cpu"/"coreml"/"webgpu" (not "CPUExecutionProvider").
+  if (device === "cpu") return ["cpu"];
+  if (device === "cuda") return ["cuda", "cpu"];
+  // auto
+  // CoreML EP does not support some dynamic/zero-length shapes used by Pocket TTS (e.g. [1, 0, 32]).
+  // Default to CPU to keep behavior predictable across platforms.
+  void ort;
+  return ["cpu"];
+}
+
+function createZeroTensorForInput(ort: OrtModule, session: OrtSession, inputName: string): OrtTensor {
+  const meta = getSessionInputMeta(session, inputName);
+  const dims = normalizeDims(meta?.dims ?? []);
+  if (dims.length === 0) {
+    throw new Error(`Missing input metadata shape for ${inputName}`);
+  }
+
+  const type = (meta?.type ?? "float32").toLowerCase();
+  const size = product(dims);
+
+  if (type.includes("int64")) {
+    return new ort.Tensor("int64", new BigInt64Array(size), dims);
+  }
+  if (type.includes("bool")) {
+    return new ort.Tensor("bool", new Uint8Array(size), dims);
+  }
+  return new ort.Tensor("float32", new Float32Array(size), dims);
+}
+
+function initState(session: OrtSession, ort: OrtModule): Record<string, OrtTensor> {
+  const out: Record<string, OrtTensor> = {};
+  for (const name of (session as any).inputNames as string[]) {
+    if (name.startsWith("state_")) {
+      out[name] = createZeroTensorForInput(ort, session, name);
+    }
+  }
+  return out;
+}
+
+function updateStateFromOutputs(
+  state: Record<string, OrtTensor>,
+  outputs: Record<string, OrtTensor>
+): void {
+  for (const [name, tensor] of Object.entries(outputs)) {
+    if (!name.startsWith("out_state_")) continue;
+    const idx = Number.parseInt(name.replace("out_state_", ""), 10);
+    if (Number.isFinite(idx)) {
+      state[`state_${idx}`] = tensor;
+    }
+  }
+}
+
+function tensorDataFloat32(t: OrtTensor): Float32Array {
+  const data = (t as any).data;
+  if (data instanceof Float32Array) return data;
+  if (Array.isArray(data)) return Float32Array.from(data as number[]);
+  throw new Error("Unexpected tensor data type (expected Float32Array)");
+}
+
+export type PocketTtsOnnxConfig = {
+  modelDir: string;
+  precision?: "int8" | "fp32";
+  device?: "auto" | "cpu" | "cuda";
+  temperature?: number;
+  lsdSteps?: number;
+  maxFrames?: number;
+  framesAfterEos?: number;
+  firstChunkFrames?: number;
+  maxChunkFrames?: number;
+  targetChunkMs?: number;
+  referenceAudioFile?: string;
+};
+
+class PocketTtsOnnxEngine {
+  static readonly SAMPLE_RATE = 24000;
+  static readonly SAMPLES_PER_FRAME = 1920;
+
+  private readonly ort: OrtModule;
+
+  private readonly temperature: number;
+  private readonly lsdSteps: number;
+  private readonly maxFrames: number;
+  private readonly framesAfterEos: number;
+
+  private readonly firstChunkFrames: number;
+  private readonly maxChunkFrames: number;
+
+  private readonly tokenizer: SentencePieceProcessor;
+  private readonly textConditioner: OrtSession;
+  private readonly flowLmMain: OrtSession;
+  private readonly flowLmFlow: OrtSession;
+  private readonly mimiDecoder: OrtSession;
+
+  private readonly stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
+  private readonly voiceEmbeddings: OrtTensor;
+
+  private constructor(args: {
+    ort: OrtModule;
+    temperature: number;
+    lsdSteps: number;
+    maxFrames: number;
+    framesAfterEos: number;
+    firstChunkFrames: number;
+    maxChunkFrames: number;
+    tokenizer: SentencePieceProcessor;
+    textConditioner: OrtSession;
+    flowLmMain: OrtSession;
+    flowLmFlow: OrtSession;
+    mimiDecoder: OrtSession;
+    stBuffers: Array<{ s: OrtTensor; t: OrtTensor }>;
+    voiceEmbeddings: OrtTensor;
+  }) {
+    this.ort = args.ort;
+    this.temperature = args.temperature;
+    this.lsdSteps = args.lsdSteps;
+    this.maxFrames = args.maxFrames;
+    this.framesAfterEos = args.framesAfterEos;
+    this.firstChunkFrames = args.firstChunkFrames;
+    this.maxChunkFrames = args.maxChunkFrames;
+    this.tokenizer = args.tokenizer;
+    this.textConditioner = args.textConditioner;
+    this.flowLmMain = args.flowLmMain;
+    this.flowLmFlow = args.flowLmFlow;
+    this.mimiDecoder = args.mimiDecoder;
+    this.stBuffers = args.stBuffers;
+    this.voiceEmbeddings = args.voiceEmbeddings;
+  }
+
+  static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise<PocketTtsOnnxEngine> {
+    const log = logger.child({ module: "speech", provider: "pocket-tts", component: "onnx-engine" });
+
+    const modelDir = config.modelDir;
+    const onnxDir = `${modelDir}/onnx`;
+    const precision = config.precision ?? "int8";
+    const device = config.device ?? "auto";
+    const temperature = config.temperature ?? 0.7;
+    const lsdSteps = config.lsdSteps ?? 10;
+    const maxFrames = config.maxFrames ?? 500;
+    const framesAfterEos = config.framesAfterEos ?? 3;
+    const firstChunkFrames = config.firstChunkFrames ?? 2;
+    const maxChunkFrames = config.maxChunkFrames ?? 15;
+
+    const tokenizerPath = `${modelDir}/tokenizer.model`;
+    const referenceAudioFile = config.referenceAudioFile ?? `${modelDir}/reference_sample.wav`;
+
+    const flowMainFile = precision === "int8" ? "flow_lm_main_int8.onnx" : "flow_lm_main.onnx";
+    const flowFlowFile = precision === "int8" ? "flow_lm_flow_int8.onnx" : "flow_lm_flow.onnx";
+    const decoderFile = precision === "int8" ? "mimi_decoder_int8.onnx" : "mimi_decoder.onnx";
+
+    assertFileExists(`${onnxDir}/mimi_encoder.onnx`, "PocketTTS mimi_encoder");
+    assertFileExists(`${onnxDir}/text_conditioner.onnx`, "PocketTTS text_conditioner");
+    assertFileExists(`${onnxDir}/${flowMainFile}`, "PocketTTS flow_lm_main");
+    assertFileExists(`${onnxDir}/${flowFlowFile}`, "PocketTTS flow_lm_flow");
+    assertFileExists(`${onnxDir}/${decoderFile}`, "PocketTTS mimi_decoder");
+    assertFileExists(tokenizerPath, "PocketTTS tokenizer.model");
+    assertFileExists(referenceAudioFile, "PocketTTS reference_sample.wav");
+
+    const ort = await loadOrt();
+    const providers = getOrtProviders(ort, device);
+
+    const [tokenizer, mimiEncoder, textConditioner, flowLmMain, flowLmFlow, mimiDecoder] = await Promise.all([
+      loadSentencePiece(tokenizerPath),
+      ort.InferenceSession.create(`${onnxDir}/mimi_encoder.onnx`, { executionProviders: providers }),
+      ort.InferenceSession.create(`${onnxDir}/text_conditioner.onnx`, { executionProviders: providers }),
+      ort.InferenceSession.create(`${onnxDir}/${flowMainFile}`, { executionProviders: providers }),
+      ort.InferenceSession.create(`${onnxDir}/${flowFlowFile}`, { executionProviders: providers }),
+      ort.InferenceSession.create(`${onnxDir}/${decoderFile}`, { executionProviders: providers }),
+    ]);
+
+    // Precompute flow matching time-step buffers.
+    const stBuffers: Array<{ s: OrtTensor; t: OrtTensor }> = [];
+    for (let j = 0; j < lsdSteps; j += 1) {
+      const s = j / lsdSteps;
+      const t = s + 1.0 / lsdSteps;
+      stBuffers.push({
+        s: new ort.Tensor("float32", new Float32Array([s]), [1, 1]),
+        t: new ort.Tensor("float32", new Float32Array([t]), [1, 1]),
+      });
+    }
+
+    // Precompute reference voice embeddings once.
+    const refWav = await readFile(referenceAudioFile);
+    const parsed = parsePcm16MonoWav(refWav);
+    let pcm16 = parsed.pcm16;
+    if (parsed.sampleRate !== PocketTtsOnnxEngine.SAMPLE_RATE) {
+      const resampler = new Pcm16MonoResampler({
+        inputRate: parsed.sampleRate,
+        outputRate: PocketTtsOnnxEngine.SAMPLE_RATE,
+      });
+      pcm16 = resampler.processChunk(pcm16);
+    }
+    const floatAudio = pcm16leToFloat32(pcm16);
+    const audioTensor = new ort.Tensor(
+      "float32",
+      floatAudio,
+      [1, 1, floatAudio.length]
+    );
+
+    const encoded = await mimiEncoder.run({ audio: audioTensor });
+    const firstOutName = (mimiEncoder as any).outputNames?.[0] as string | undefined;
+    const voiceEmb = firstOutName ? (encoded as any)[firstOutName] : (Object.values(encoded)[0] as any);
+    if (!voiceEmb) {
+      throw new Error("PocketTTS mimi_encoder: missing output");
+    }
+
+    log.info(
+      { precision, device, providers, lsdSteps, temperature },
+      "PocketTTS ONNX initialized"
+    );
+
+    return new PocketTtsOnnxEngine({
+      ort,
+      temperature,
+      lsdSteps,
+      maxFrames,
+      framesAfterEos,
+      firstChunkFrames,
+      maxChunkFrames,
+      tokenizer,
+      textConditioner,
+      flowLmMain,
+      flowLmFlow,
+      mimiDecoder,
+      stBuffers,
+      voiceEmbeddings: voiceEmb,
+    });
+  }
+
+  private tokenize(text: string): OrtTensor {
+    const normalized = normalizeTextForPocket(text);
+    const ids = this.tokenizer.encodeIds(normalized);
+    const data = toBigInt64(ids ?? []);
+    return new this.ort.Tensor("int64", data, [1, data.length]);
+  }
+
+  private async runTextConditioner(tokenIds: OrtTensor): Promise<OrtTensor> {
+    const out = await this.textConditioner.run({ token_ids: tokenIds } as any);
+    const firstOutName = (this.textConditioner as any).outputNames?.[0] as string | undefined;
+    const t = firstOutName ? (out as any)[firstOutName] : (Object.values(out)[0] as any);
+    if (!t) throw new Error("PocketTTS text_conditioner: missing output");
+    return t;
+  }
+
+  private async *runFlowLm(textEmbeddings: OrtTensor): AsyncGenerator<Float32Array> {
+    const ort = this.ort;
+    const state = initState(this.flowLmMain, ort);
+
+    const emptySeq = new ort.Tensor("float32", new Float32Array(0), [1, 0, 32]);
+    const emptyText = new ort.Tensor("float32", new Float32Array(0), [1, 0, 1024]);
+
+    // Voice conditioning pass
+    const resVoice = await this.flowLmMain.run({
+      sequence: emptySeq,
+      text_embeddings: this.voiceEmbeddings,
+      ...state,
+    } as any);
+    updateStateFromOutputs(state, resVoice as any);
+
+    // Text conditioning pass
+    const resText = await this.flowLmMain.run({
+      sequence: emptySeq,
+      text_embeddings: textEmbeddings,
+      ...state,
+    } as any);
+    updateStateFromOutputs(state, resText as any);
+
+    // Autoregressive generation
+    const curr = new Float32Array(32);
+    curr.fill(Number.NaN);
+    let currTensor = new ort.Tensor("float32", curr, [1, 1, 32]);
+
+    const dt = 1.0 / this.lsdSteps;
+    let eosStep: number | null = null;
+
+    for (let step = 0; step < this.maxFrames; step += 1) {
+      const resStep = await this.flowLmMain.run({
+        sequence: currTensor,
+        text_embeddings: emptyText,
+        ...state,
+      } as any);
+
+      const outputNames = (this.flowLmMain as any).outputNames as string[] | undefined;
+      const conditioningName = outputNames?.[0] ?? Object.keys(resStep)[0]!;
+      const eosName = outputNames?.[1] ?? Object.keys(resStep)[1]!;
+
+      const conditioning = (resStep as any)[conditioningName] as OrtTensor;
+      const eos = (resStep as any)[eosName] as OrtTensor;
+      if (!conditioning || !eos) {
+        throw new Error("PocketTTS flow_lm_main: missing conditioning/EOS outputs");
+      }
+      updateStateFromOutputs(state, resStep as any);
+
+      const eosData = tensorDataFloat32(eos);
+      if (eosData[0]! > -4.0 && eosStep === null) {
+        eosStep = step;
+      }
+      if (eosStep !== null && step >= eosStep + this.framesAfterEos) {
+        break;
+      }
+
+      // Flow matching with external Euler loop.
+      const std = this.temperature > 0 ? Math.sqrt(this.temperature) : 0;
+      const x = new Float32Array(32);
+      if (std > 0) {
+        for (let i = 0; i < x.length; i += 1) {
+          x[i] = randn() * std;
+        }
+      }
+
+      for (const st of this.stBuffers) {
+        const xTensor = new ort.Tensor("float32", x, [1, 32]);
+        const flowOut = await this.flowLmFlow.run({
+          c: conditioning,
+          s: st.s,
+          t: st.t,
+          x: xTensor,
+        } as any);
+        const first = (this.flowLmFlow as any).outputNames?.[0] as string | undefined;
+        const flowTensor = first ? (flowOut as any)[first] : (Object.values(flowOut)[0] as any);
+        if (!flowTensor) throw new Error("PocketTTS flow_lm_flow: missing output");
+        const delta = tensorDataFloat32(flowTensor);
+        for (let i = 0; i < x.length; i += 1) {
+          x[i] = x[i]! + delta[i]! * dt;
+        }
+      }
+
+      yield x;
+      currTensor = new ort.Tensor("float32", x, [1, 1, 32]);
+    }
+  }
+
+  private async decodeLatentsChunk(
+    frames: Float32Array[],
+    state: Record<string, OrtTensor>
+  ): Promise<Float32Array> {
+    const ort = this.ort;
+    const frameCount = frames.length;
+    const flattened = new Float32Array(frameCount * 32);
+    for (let i = 0; i < frameCount; i += 1) {
+      flattened.set(frames[i]!, i * 32);
+    }
+    const latent = new ort.Tensor("float32", flattened, [1, frameCount, 32]);
+
+    const out = await this.mimiDecoder.run({ latent, ...state } as any);
+    updateStateFromOutputs(state, out as any);
+
+    const firstOutName = (this.mimiDecoder as any).outputNames?.[0] as string | undefined;
+    const audioTensor = firstOutName ? (out as any)[firstOutName] : (Object.values(out)[0] as any);
+    if (!audioTensor) {
+      throw new Error("PocketTTS mimi_decoder: missing audio output");
+    }
+    return tensorDataFloat32(audioTensor);
+  }
+
+  async *streamAudio(text: string): AsyncGenerator<Float32Array> {
+    const tokenIds = this.tokenize(text);
+    const textEmb = await this.runTextConditioner(tokenIds);
+
+    const decoderState = initState(this.mimiDecoder, this.ort);
+
+    const generated: Float32Array[] = [];
+    let decodedFrames = 0;
+
+    for await (const latent of this.runFlowLm(textEmb)) {
+      generated.push(latent);
+      const pending = generated.length - decodedFrames;
+
+      let chunkSize = 0;
+      if (decodedFrames === 0) {
+        if (pending >= this.firstChunkFrames) {
+          chunkSize = this.firstChunkFrames;
+        }
+      } else if (pending >= this.maxChunkFrames) {
+        chunkSize = this.maxChunkFrames;
+      }
+
+      if (chunkSize > 0) {
+        const audio = await this.decodeLatentsChunk(
+          generated.slice(decodedFrames, decodedFrames + chunkSize),
+          decoderState
+        );
+        decodedFrames += chunkSize;
+        yield audio;
+      }
+    }
+
+    if (decodedFrames < generated.length) {
+      const audio = await this.decodeLatentsChunk(generated.slice(decodedFrames), decoderState);
+      yield audio;
+    }
+  }
+}
+
+export class PocketTtsOnnxTTS implements TextToSpeechProvider {
+  private readonly engine: PocketTtsOnnxEngine;
+  private readonly chunkMs: number;
+  private readonly logger: pino.Logger;
+
+  private constructor(engine: PocketTtsOnnxEngine, logger: pino.Logger, chunkMs: number) {
+    this.engine = engine;
+    this.chunkMs = chunkMs;
+    this.logger = logger.child({ module: "speech", provider: "pocket-tts", component: "tts" });
+  }
+
+  static async create(config: PocketTtsOnnxConfig, logger: pino.Logger): Promise<PocketTtsOnnxTTS> {
+    const engine = await PocketTtsOnnxEngine.create(config, logger);
+    const chunkMs = config.targetChunkMs ?? 50;
+    return new PocketTtsOnnxTTS(engine, logger, chunkMs);
+  }
+
+  async synthesizeSpeech(text: string): Promise<SpeechStreamResult> {
+    const start = Date.now();
+    const sampleRate = PocketTtsOnnxEngine.SAMPLE_RATE;
+
+    const pcmChunkBytes = Math.max(2, Math.round((sampleRate * this.chunkMs) / 1000) * 2);
+
+    const iterable = (async function* (engine: PocketTtsOnnxEngine) {
+      for await (const floatChunk of engine.streamAudio(text)) {
+        const pcm = float32ToPcm16le(floatChunk);
+        for (const chunk of chunkBuffer(pcm, pcmChunkBytes)) {
+          yield chunk;
+        }
+      }
+    })(this.engine);
+
+    this.logger.debug({ ms: Date.now() - start, textLength: text.length }, "PocketTTS stream ready");
+
+    return {
+      stream: Readable.from(iterable),
+      format: `pcm;rate=${sampleRate}`,
+    };
+  }
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts
@@ -0,0 +1,128 @@
+export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
+
+export type SherpaOnnxModelId =
+  | "zipformer-bilingual-zh-en-2023-02-20"
+  | "paraformer-bilingual-zh-en"
+  | "parakeet-tdt-0.6b-v3-int8"
+  | "kitten-nano-en-v0_1-fp16"
+  | "kokoro-en-v0_19"
+  | "pocket-tts-onnx-int8";
+
+export type SherpaOnnxModelSpec = {
+  id: SherpaOnnxModelId;
+  kind: SherpaOnnxModelKind;
+  archiveUrl?: string;
+  downloadFiles?: Array<{ url: string; relPath: string }>;
+  extractedDir: string;
+  requiredFiles: string[];
+  description: string;
+};
+
+export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxModelSpec> = {
+  "zipformer-bilingual-zh-en-2023-02-20": {
+    id: "zipformer-bilingual-zh-en-2023-02-20",
+    kind: "stt-online",
+    archiveUrl:
+      "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2",
+    extractedDir: "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20",
+    requiredFiles: [
+      "encoder-epoch-99-avg-1.onnx",
+      "decoder-epoch-99-avg-1.onnx",
+      "joiner-epoch-99-avg-1.onnx",
+      "tokens.txt",
+    ],
+    description: "Streaming Zipformer transducer (fast, good accuracy).",
+  },
+  "paraformer-bilingual-zh-en": {
+    id: "paraformer-bilingual-zh-en",
+    kind: "stt-online",
+    archiveUrl:
+      "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2",
+    extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en",
+    requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"],
+    description: "Streaming Paraformer (often strong accuracy; heavier).",
+  },
+  "parakeet-tdt-0.6b-v3-int8": {
+    id: "parakeet-tdt-0.6b-v3-int8",
+    kind: "stt-offline",
+    archiveUrl:
+      "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
+    extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
+    requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
+    description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).",
+  },
+  "kitten-nano-en-v0_1-fp16": {
+    id: "kitten-nano-en-v0_1-fp16",
+    kind: "tts",
+    archiveUrl:
+      "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2",
+    extractedDir: "kitten-nano-en-v0_1-fp16",
+    requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
+    description: "KittenTTS (small, fast English TTS).",
+  },
+  "kokoro-en-v0_19": {
+    id: "kokoro-en-v0_19",
+    kind: "tts",
+    archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2",
+    extractedDir: "kokoro-en-v0_19",
+    requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
+    description: "Kokoro TTS (higher quality; larger).",
+  },
+  "pocket-tts-onnx-int8": {
+    id: "pocket-tts-onnx-int8",
+    kind: "tts",
+    extractedDir: "pocket-tts-onnx-int8",
+    downloadFiles: [
+      {
+        url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_encoder.onnx",
+        relPath: "onnx/mimi_encoder.onnx",
+      },
+      {
+        url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/text_conditioner.onnx",
+        relPath: "onnx/text_conditioner.onnx",
+      },
+      {
+        url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_main_int8.onnx",
+        relPath: "onnx/flow_lm_main_int8.onnx",
+      },
+      {
+        url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/flow_lm_flow_int8.onnx",
+        relPath: "onnx/flow_lm_flow_int8.onnx",
+      },
+      {
+        url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/onnx/mimi_decoder_int8.onnx",
+        relPath: "onnx/mimi_decoder_int8.onnx",
+      },
+      {
+        url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/tokenizer.model",
+        relPath: "tokenizer.model",
+      },
+      {
+        url: "https://huggingface.co/KevinAHM/pocket-tts-onnx/resolve/main/reference_sample.wav",
+        relPath: "reference_sample.wav",
+      },
+    ],
+    requiredFiles: [
+      "onnx/mimi_encoder.onnx",
+      "onnx/text_conditioner.onnx",
+      "onnx/flow_lm_main_int8.onnx",
+      "onnx/flow_lm_flow_int8.onnx",
+      "onnx/mimi_decoder_int8.onnx",
+      "tokenizer.model",
+      "reference_sample.wav",
+    ],
+    description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
+  },
+};
+
+export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] {
+  return Object.values(SHERPA_ONNX_MODEL_CATALOG);
+}
+
+export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec {
+  const spec = SHERPA_ONNX_MODEL_CATALOG[id];
+  if (!spec) {
+    throw new Error(`Unknown sherpa-onnx model id: ${id}`);
+  }
+  return spec;
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.test.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.test.ts
@@ -0,0 +1,56 @@
+import { describe, expect, test } from "vitest";
+import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import pino from "pino";
+
+import { ensureSherpaOnnxModel, getSherpaOnnxModelDir } from "./model-downloader.js";
+
+function makeTmpDir(): string {
+  return mkdtempSync(path.join(tmpdir(), "paseo-speech-models-"));
+}
+
+const logger = pino({ level: "silent" });
+
+describe("sherpa model downloader", () => {
+  test("getSherpaOnnxModelDir maps modelId to extractedDir", () => {
+    const modelsDir = "/tmp/models";
+    expect(getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8")).toContain(
+      "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8"
+    );
+    expect(getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8")).toContain(
+      "pocket-tts-onnx-int8"
+    );
+  });
+
+  test("ensureSherpaOnnxModel succeeds without downloading when files exist", async () => {
+    const modelsDir = makeTmpDir();
+    const modelDir = getSherpaOnnxModelDir(modelsDir, "kitten-nano-en-v0_1-fp16");
+
+    mkdirSync(path.join(modelDir, "espeak-ng-data"), { recursive: true });
+    writeFileSync(path.join(modelDir, "model.fp16.onnx"), "x");
+    writeFileSync(path.join(modelDir, "voices.bin"), "x");
+    writeFileSync(path.join(modelDir, "tokens.txt"), "x");
+
+    const out = await ensureSherpaOnnxModel({
+      modelsDir,
+      modelId: "kitten-nano-en-v0_1-fp16",
+      autoDownload: false,
+      logger,
+    });
+
+    expect(out).toBe(modelDir);
+  });
+
+  test("ensureSherpaOnnxModel throws when missing and autoDownload is false", async () => {
+    const modelsDir = makeTmpDir();
+    await expect(
+      ensureSherpaOnnxModel({
+        modelsDir,
+        modelId: "zipformer-bilingual-zh-en-2023-02-20",
+        autoDownload: false,
+        logger,
+      })
+    ).rejects.toThrow(/auto-download/i);
+  });
+});
--- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts
@@ -0,0 +1,190 @@
+import { createWriteStream } from "node:fs";
+import { mkdir, rename, rm, stat } from "node:fs/promises";
+import path from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { spawn } from "node:child_process";
+import type pino from "pino";
+
+import { getSherpaOnnxModelSpec, type SherpaOnnxModelId } from "./model-catalog.js";
+
+export type EnsureSherpaOnnxModelOptions = {
+  modelsDir: string;
+  modelId: SherpaOnnxModelId;
+  autoDownload: boolean;
+  logger: pino.Logger;
+};
+
+export function getSherpaOnnxModelDir(modelsDir: string, modelId: SherpaOnnxModelId): string {
+  const spec = getSherpaOnnxModelSpec(modelId);
+  return path.join(modelsDir, spec.extractedDir);
+}
+
+async function hasRequiredFiles(modelDir: string, requiredFiles: string[]): Promise<boolean> {
+  for (const rel of requiredFiles) {
+    const abs = path.join(modelDir, rel);
+    try {
+      const s = await stat(abs);
+      if (s.isDirectory()) {
+        continue;
+      }
+      if (s.isFile() && s.size > 0) {
+        continue;
+      }
+      return false;
+    } catch {
+      return false;
+    }
+  }
+  return true;
+}
+
+async function downloadToFile(url: string, outputPath: string, logger: pino.Logger): Promise<void> {
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`);
+  }
+  if (!res.body) {
+    throw new Error(`Failed to download ${url}: missing response body`);
+  }
+
+  const tmpPath = `${outputPath}.tmp-${Date.now()}`;
+  await mkdir(path.dirname(outputPath), { recursive: true });
+
+  const total = Number(res.headers.get("content-length") ?? "0");
+  let downloaded = 0;
+  let lastLoggedBucket = -1;
+
+  const nodeStream = Readable.fromWeb(res.body as any).on("data", (chunk: Buffer) => {
+    downloaded += chunk.length;
+    if (total > 0) {
+      const pct = Math.floor((downloaded / total) * 100);
+      const bucket = Math.min(100, Math.floor(pct / 10) * 10);
+      if (bucket >= 0 && bucket <= 100 && bucket !== lastLoggedBucket) {
+        lastLoggedBucket = bucket;
+        logger.info({ pct: bucket, downloaded, total }, "Downloading model artifact");
+      }
+    }
+  });
+
+  await pipeline(nodeStream, createWriteStream(tmpPath));
+  await rename(tmpPath, outputPath);
+}
+
+async function extractTarArchive(archivePath: string, destDir: string): Promise<void> {
+  await mkdir(destDir, { recursive: true });
+
+  await new Promise<void>((resolve, reject) => {
+    const child = spawn("tar", ["xf", archivePath, "-C", destDir], { stdio: "inherit" });
+    child.on("error", reject);
+    child.on("exit", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`tar exited with code ${code}`));
+    });
+  });
+}
+
+async function isNonEmptyFile(filePath: string): Promise<boolean> {
+  try {
+    const s = await stat(filePath);
+    return s.isFile() && s.size > 0;
+  } catch {
+    return false;
+  }
+}
+
+export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise<string> {
+  const logger = options.logger.child({
+    module: "speech",
+    provider: "sherpa-onnx",
+    component: "model-downloader",
+    modelId: options.modelId,
+  });
+
+  const spec = getSherpaOnnxModelSpec(options.modelId);
+  const modelDir = path.join(options.modelsDir, spec.extractedDir);
+  if (await hasRequiredFiles(modelDir, spec.requiredFiles)) {
+    return modelDir;
+  }
+
+  if (!options.autoDownload) {
+    throw new Error(
+      `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` +
+        `Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.`
+    );
+  }
+
+  if (spec.archiveUrl) {
+    logger.info({ modelsDir: options.modelsDir, url: spec.archiveUrl }, "Model files missing; downloading");
+
+    const downloadsDir = path.join(options.modelsDir, ".downloads");
+    const archiveFilename = path.basename(new URL(spec.archiveUrl).pathname);
+    const archivePath = path.join(downloadsDir, archiveFilename);
+
+    if (!(await isNonEmptyFile(archivePath))) {
+      await downloadToFile(spec.archiveUrl, archivePath, logger);
+    } else {
+      logger.info({ archivePath }, "Using cached archive");
+    }
+
+    await extractTarArchive(archivePath, options.modelsDir);
+
+    if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
+      throw new Error(
+        `Downloaded and extracted ${archiveFilename}, but required files are still missing in ${modelDir}.`
+      );
+    }
+
+    try {
+      await rm(archivePath, { force: true });
+    } catch {
+      // ignore
+    }
+
+    logger.info({ modelDir }, "Model ready");
+    return modelDir;
+  }
+
+  if (spec.downloadFiles && spec.downloadFiles.length > 0) {
+    logger.info({ modelsDir: options.modelsDir, fileCount: spec.downloadFiles.length }, "Model files missing; downloading");
+    await mkdir(modelDir, { recursive: true });
+
+    for (const file of spec.downloadFiles) {
+      const dst = path.join(modelDir, file.relPath);
+      if (await isNonEmptyFile(dst)) {
+        continue;
+      }
+      await downloadToFile(file.url, dst, logger);
+    }
+
+    if (!(await hasRequiredFiles(modelDir, spec.requiredFiles))) {
+      throw new Error(
+        `Downloaded files for ${options.modelId}, but required files are still missing in ${modelDir}.`
+      );
+    }
+
+    logger.info({ modelDir }, "Model ready");
+    return modelDir;
+  }
+
+  throw new Error(`Model spec for ${options.modelId} has no archiveUrl or downloadFiles`);
+}
+
+export async function ensureSherpaOnnxModels(options: {
+  modelsDir: string;
+  modelIds: SherpaOnnxModelId[];
+  autoDownload: boolean;
+  logger: pino.Logger;
+}): Promise<Record<SherpaOnnxModelId, string>> {
+  const uniq = Array.from(new Set(options.modelIds));
+  const out: Partial<Record<SherpaOnnxModelId, string>> = {};
+  for (const id of uniq) {
+    out[id] = await ensureSherpaOnnxModel({
+      modelsDir: options.modelsDir,
+      modelId: id,
+      autoDownload: options.autoDownload,
+      logger: options.logger,
+    });
+  }
+  return out as Record<SherpaOnnxModelId, string>;
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts
@@ -0,0 +1,104 @@
+import { existsSync } from "node:fs";
+import type pino from "pino";
+
+import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
+
+function assertFileExists(filePath: string, label: string): void {
+  if (!existsSync(filePath)) {
+    throw new Error(`Missing ${label}: ${filePath}`);
+  }
+}
+
+export type SherpaOfflineRecognizerModel = {
+  kind: "nemo_transducer";
+  encoder: string;
+  decoder: string;
+  joiner: string;
+  tokens: string;
+};
+
+export type SherpaOfflineRecognizerConfig = {
+  model: SherpaOfflineRecognizerModel;
+  numThreads?: number;
+  provider?: "cpu";
+  debug?: 0 | 1;
+  sampleRate?: number;
+  featureDim?: number;
+  decodingMethod?: "greedy_search";
+  maxActivePaths?: number;
+};
+
+export class SherpaOfflineRecognizerEngine {
+  public readonly recognizer: any;
+  public readonly sampleRate: number;
+  private readonly logger: pino.Logger;
+
+  constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) {
+    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" });
+
+    assertFileExists(config.model.encoder, "offline encoder");
+    assertFileExists(config.model.decoder, "offline decoder");
+    assertFileExists(config.model.joiner, "offline joiner");
+    assertFileExists(config.model.tokens, "tokens");
+
+    const sherpa = loadSherpaOnnxNode();
+
+    const recognizerConfig = {
+      featConfig: {
+        sampleRate: config.sampleRate ?? 16000,
+        featureDim: config.featureDim ?? 80,
+      },
+      modelConfig: {
+        transducer: {
+          encoder: config.model.encoder,
+          decoder: config.model.decoder,
+          joiner: config.model.joiner,
+        },
+        tokens: config.model.tokens,
+        modelType: "nemo_transducer",
+        numThreads: config.numThreads ?? 1,
+        provider: config.provider ?? "cpu",
+        debug: config.debug ?? 0,
+      },
+      decodingMethod: config.decodingMethod ?? "greedy_search",
+      maxActivePaths: config.maxActivePaths ?? 4,
+    };
+
+    this.recognizer = new sherpa.OfflineRecognizer(recognizerConfig);
+    const sr = this.recognizer?.config?.featConfig?.sampleRate;
+    this.sampleRate = typeof sr === "number" && Number.isFinite(sr) && sr > 0
+      ? sr
+      : recognizerConfig.featConfig.sampleRate;
+
+    this.logger.info(
+      { sampleRate: this.sampleRate, numThreads: recognizerConfig.modelConfig.numThreads },
+      "Sherpa offline recognizer initialized"
+    );
+  }
+
+  createStream(): any {
+    return this.recognizer.createStream();
+  }
+
+  acceptWaveform(stream: any, sampleRate: number, samples: Float32Array): void {
+    if (!stream || typeof stream.acceptWaveform !== "function") {
+      throw new Error("Unexpected sherpa offline stream: missing acceptWaveform()");
+    }
+
+    // sherpa-onnx-node expects: acceptWaveform({ samples, sampleRate })
+    // sherpa-onnx (WASM) expects: acceptWaveform(sampleRate, samples)
+    if (stream.acceptWaveform.length <= 1) {
+      stream.acceptWaveform({ samples, sampleRate });
+    } else {
+      stream.acceptWaveform(sampleRate, samples);
+    }
+  }
+
+  free(): void {
+    try {
+      this.recognizer?.free?.();
+    } catch (err) {
+      this.logger.warn({ err }, "Failed to free sherpa offline recognizer");
+    }
+  }
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts
@@ -0,0 +1,127 @@
+import { existsSync } from "node:fs";
+import type pino from "pino";
+
+import { loadSherpaOnnx } from "./sherpa-onnx-loader.js";
+
+export type SherpaOnlineRecognizerModel =
+  | {
+      kind: "transducer";
+      encoder: string;
+      decoder: string;
+      joiner: string;
+      tokens: string;
+      modelType?: string;
+    }
+  | {
+      kind: "paraformer";
+      encoder: string;
+      decoder: string;
+      tokens: string;
+    };
+
+export type SherpaOnlineRecognizerConfig = {
+  model: SherpaOnlineRecognizerModel;
+  numThreads?: number;
+  provider?: "cpu";
+  debug?: 0 | 1;
+  sampleRate?: number;
+  featureDim?: number;
+  decodingMethod?: "greedy_search";
+  maxActivePaths?: number;
+  enableEndpoint?: 0 | 1;
+  rule1MinTrailingSilence?: number;
+  rule2MinTrailingSilence?: number;
+  rule3MinUtteranceLength?: number;
+};
+
+function assertFileExists(filePath: string, label: string): void {
+  if (!existsSync(filePath)) {
+    throw new Error(`Missing ${label}: ${filePath}`);
+  }
+}
+
+export class SherpaOnlineRecognizerEngine {
+  public readonly recognizer: any;
+  public readonly sampleRate: number;
+  private readonly logger: pino.Logger;
+
+  constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) {
+    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" });
+
+    const { model } = config;
+    if (model.kind === "transducer") {
+      assertFileExists(model.encoder, "transducer encoder");
+      assertFileExists(model.decoder, "transducer decoder");
+      assertFileExists(model.joiner, "transducer joiner");
+      assertFileExists(model.tokens, "tokens");
+    } else {
+      assertFileExists(model.encoder, "paraformer encoder");
+      assertFileExists(model.decoder, "paraformer decoder");
+      assertFileExists(model.tokens, "tokens");
+    }
+
+    const sherpa = loadSherpaOnnx();
+    const modelConfig =
+      model.kind === "transducer"
+        ? {
+            transducer: {
+              encoder: model.encoder,
+              decoder: model.decoder,
+              joiner: model.joiner,
+            },
+            tokens: model.tokens,
+            modelType: model.modelType ?? "zipformer",
+          }
+        : {
+            paraformer: {
+              encoder: model.encoder,
+              decoder: model.decoder,
+            },
+            tokens: model.tokens,
+          };
+
+    const featConfig = {
+      sampleRate: config.sampleRate ?? 16000,
+      featureDim: config.featureDim ?? 80,
+    };
+
+    const recognizerConfig = {
+      featConfig,
+      modelConfig: {
+        ...modelConfig,
+        // NOTE: In the WASM-backed `sherpa-onnx` npm package, online recognizers
+        // error when `numThreads > 1`. Keep the default conservative.
+        numThreads: config.numThreads ?? 1,
+        provider: config.provider ?? "cpu",
+        debug: config.debug ?? 0,
+      },
+      decodingMethod: config.decodingMethod ?? "greedy_search",
+      maxActivePaths: config.maxActivePaths ?? 4,
+      enableEndpoint: config.enableEndpoint ?? 0,
+      rule1MinTrailingSilence: config.rule1MinTrailingSilence ?? 2.4,
+      rule2MinTrailingSilence: config.rule2MinTrailingSilence ?? 1.2,
+      rule3MinUtteranceLength: config.rule3MinUtteranceLength ?? 20,
+    };
+
+    this.recognizer = sherpa.createOnlineRecognizer(recognizerConfig);
+    const sr = this.recognizer?.config?.featConfig?.sampleRate;
+    this.sampleRate = typeof sr === "number" && Number.isFinite(sr) && sr > 0 ? sr : featConfig.sampleRate;
+
+    this.logger.info(
+      { sampleRate: this.sampleRate, modelKind: model.kind, numThreads: config.numThreads ?? 2 },
+      "Sherpa online recognizer initialized"
+    );
+  }
+
+  createStream(): any {
+    return this.recognizer.createStream();
+  }
+
+  free(): void {
+    try {
+      this.recognizer?.free?.();
+    } catch (err) {
+      this.logger.warn({ err }, "Failed to free sherpa recognizer");
+    }
+  }
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts
@@ -0,0 +1,18 @@
+import { createRequire } from "node:module";
+
+export type SherpaOnnxModule = {
+  createOnlineRecognizer: (config: any) => any;
+  createOfflineRecognizer: (config: any) => any;
+  createOfflineTts: (config: any) => any;
+};
+
+let cached: SherpaOnnxModule | null = null;
+
+export function loadSherpaOnnx(): SherpaOnnxModule {
+  if (cached) {
+    return cached;
+  }
+  const require = createRequire(import.meta.url);
+  cached = require("sherpa-onnx") as SherpaOnnxModule;
+  return cached;
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts
@@ -0,0 +1,53 @@
+import { createRequire } from "node:module";
+import path from "node:path";
+
+export type SherpaOnnxNodeModule = {
+  OfflineRecognizer: new (config: any) => any;
+  OnlineRecognizer?: new (config: any) => any;
+  OfflineTts?: new (config: any) => any;
+};
+
+let cached: SherpaOnnxNodeModule | null = null;
+
+function platformArch(): string {
+  const platform = process.platform === "win32" ? "win" : process.platform;
+  return `${platform}-${process.arch}`;
+}
+
+function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void {
+  const current = process.env[envKey] ?? "";
+  const parts = current.split(":").filter(Boolean);
+  if (parts.includes(dir)) {
+    return;
+  }
+  process.env[envKey] = [dir, ...parts].join(":");
+}
+
+export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
+  if (cached) {
+    return cached;
+  }
+
+  const require = createRequire(import.meta.url);
+
+  // sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64)
+  // that contains the native addon + its shared libraries. On macOS/Linux we need to ensure
+  // the appropriate library path env var includes that directory before requiring the addon.
+  const arch = platformArch();
+  const pkgName = `sherpa-onnx-${arch}`;
+
+  try {
+    const pkgJson = require.resolve(`${pkgName}/package.json`);
+    const pkgDir = path.dirname(pkgJson);
+    if (process.platform === "darwin") {
+      prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir);
+    } else if (process.platform === "linux") {
+      prependLibraryPath("LD_LIBRARY_PATH", pkgDir);
+    }
+  } catch {
+    // Best effort - if the platform package isn't present, require() below will throw a useful error.
+  }
+
+  cached = require("sherpa-onnx-node") as SherpaOnnxNodeModule;
+  return cached;
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-realtime-session.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-realtime-session.ts
@@ -0,0 +1,154 @@
+import { EventEmitter } from "node:events";
+import { v4 as uuidv4 } from "uuid";
+
+import type { StreamingTranscriptionSession } from "../../../speech-provider.js";
+import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
+import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
+
+export class SherpaParakeetRealtimeTranscriptionSession
+  extends EventEmitter
+  implements StreamingTranscriptionSession
+{
+  private readonly engine: SherpaOfflineRecognizerEngine;
+  private connected = false;
+
+  public readonly requiredSampleRate: number;
+  private currentSegmentId: string | null = null;
+  private previousSegmentId: string | null = null;
+  private lastPartialText = "";
+
+  private pcm16: Buffer = Buffer.alloc(0);
+  private lastDecodeAt = 0;
+  private decoding = false;
+  private pendingDecode = false;
+  private readonly minDecodeIntervalMs: number;
+
+  constructor(params: { engine: SherpaOfflineRecognizerEngine; minDecodeIntervalMs?: number }) {
+    super();
+    this.engine = params.engine;
+    this.requiredSampleRate = this.engine.sampleRate;
+    this.minDecodeIntervalMs = params.minDecodeIntervalMs ?? 350;
+  }
+
+  async connect(): Promise<void> {
+    if (this.connected) {
+      return;
+    }
+    this.currentSegmentId = uuidv4();
+    this.connected = true;
+  }
+
+  appendPcm16(chunk: Buffer): void {
+    if (!this.connected || !this.currentSegmentId) {
+      this.emit("error", new Error("Parakeet realtime session not connected"));
+      return;
+    }
+
+    try {
+      this.pcm16 = this.pcm16.length === 0 ? chunk : Buffer.concat([this.pcm16, chunk]);
+      void this.maybeDecode(false);
+    } catch (err) {
+      this.emit("error", err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+
+  commit(): void {
+    if (!this.connected || !this.currentSegmentId) {
+      this.emit("error", new Error("Parakeet realtime session not connected"));
+      return;
+    }
+
+    void (async () => {
+      try {
+        await this.maybeDecode(true);
+        const finalText = this.lastPartialText;
+        const segmentId = this.currentSegmentId!;
+        const previousSegmentId = this.previousSegmentId;
+
+        this.emit("committed", { segmentId, previousSegmentId });
+        this.emit("transcript", { segmentId, transcript: finalText, isFinal: true });
+
+        this.previousSegmentId = segmentId;
+        this.currentSegmentId = uuidv4();
+        this.lastPartialText = "";
+        this.pcm16 = Buffer.alloc(0);
+      } catch (err) {
+        this.emit("error", err instanceof Error ? err : new Error(String(err)));
+      }
+    })();
+  }
+
+  clear(): void {
+    if (!this.connected) {
+      return;
+    }
+    this.pcm16 = Buffer.alloc(0);
+    this.currentSegmentId = uuidv4();
+    this.lastPartialText = "";
+  }
+
+  close(): void {
+    this.connected = false;
+    this.currentSegmentId = null;
+    this.pcm16 = Buffer.alloc(0);
+  }
+
+  private async maybeDecode(force: boolean): Promise<void> {
+    if (!this.connected || !this.currentSegmentId) {
+      return;
+    }
+
+    const now = Date.now();
+    if (!force && now - this.lastDecodeAt < this.minDecodeIntervalMs) {
+      return;
+    }
+
+    if (this.decoding) {
+      this.pendingDecode = true;
+      return;
+    }
+
+    this.decoding = true;
+    try {
+      const text = await this.decodeNow();
+      this.lastDecodeAt = Date.now();
+      if (text !== this.lastPartialText) {
+        this.lastPartialText = text;
+        this.emit("transcript", { segmentId: this.currentSegmentId, transcript: text, isFinal: false });
+      }
+    } finally {
+      this.decoding = false;
+      if (this.pendingDecode) {
+        this.pendingDecode = false;
+        await this.maybeDecode(true);
+      }
+    }
+  }
+
+  private async decodeNow(): Promise<string> {
+    if (this.pcm16.length === 0) {
+      return "";
+    }
+
+    const peak = pcm16lePeakAbs(this.pcm16);
+    const peakFloat = peak / 32768.0;
+    const targetPeak = 0.6;
+    const maxGain = 50;
+    const gain = peakFloat > 0 && peakFloat < targetPeak ? Math.min(maxGain, targetPeak / peakFloat) : 1;
+
+    const stream = this.engine.createStream();
+    try {
+      const floatSamples = pcm16leToFloat32(this.pcm16, gain);
+      this.engine.acceptWaveform(stream, this.engine.sampleRate, floatSamples);
+      this.engine.recognizer.decode(stream);
+      const result = this.engine.recognizer.getResult(stream);
+      return String(result?.text ?? result ?? "").trim();
+    } finally {
+      try {
+        stream.free?.();
+      } catch {
+        // ignore
+      }
+    }
+  }
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts
@@ -0,0 +1,159 @@
+import { EventEmitter } from "node:events";
+import { v4 as uuidv4 } from "uuid";
+import type pino from "pino";
+
+import type {
+  SpeechToTextProvider,
+  StreamingTranscriptionSession,
+  TranscriptionResult,
+} from "../../../speech-provider.js";
+import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
+import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
+import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
+
+export type SherpaParakeetSttConfig = {
+  engine: SherpaOfflineRecognizerEngine;
+  silencePeakThreshold?: number;
+};
+
+export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
+  private readonly engine: SherpaOfflineRecognizerEngine;
+  private readonly silencePeakThreshold: number;
+  private readonly logger: pino.Logger;
+  public readonly id = "local" as const;
+
+  constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) {
+    this.engine = config.engine;
+    this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
+    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" });
+  }
+
+  public createSession(params: {
+    logger: pino.Logger;
+    language?: string;
+    prompt?: string;
+  }): StreamingTranscriptionSession {
+    const emitter = new EventEmitter();
+    const logger = params.logger.child({ provider: "local", component: "parakeet-stt-session" });
+    const requiredSampleRate = this.engine.sampleRate;
+    let connected = false;
+    let segmentId = uuidv4();
+    let previousSegmentId: string | null = null;
+    let pcm16: Buffer = Buffer.alloc(0);
+
+    return {
+      requiredSampleRate,
+      async connect() {
+        connected = true;
+      },
+      appendPcm16(chunk: Buffer) {
+        if (!connected) {
+          (emitter as any).emit("error", new Error("STT session not connected"));
+          return;
+        }
+        pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]);
+      },
+      commit: () => {
+        if (!connected) {
+          (emitter as any).emit("error", new Error("STT session not connected"));
+          return;
+        }
+
+        const committedId = segmentId;
+        const prev = previousSegmentId;
+        (emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev });
+
+        void (async () => {
+          try {
+            const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`);
+            (emitter as any).emit("transcript", {
+              segmentId: committedId,
+              transcript: rt.text,
+              isFinal: true,
+              language: rt.language,
+              logprobs: rt.logprobs,
+              avgLogprob: rt.avgLogprob,
+              isLowConfidence: rt.isLowConfidence,
+            });
+          } catch (err) {
+            (emitter as any).emit("error", err);
+          } finally {
+            previousSegmentId = committedId;
+            segmentId = uuidv4();
+            pcm16 = Buffer.alloc(0);
+            logger.debug({ bytes: pcm16.length }, "Parakeet session reset");
+          }
+        })();
+      },
+      clear() {
+        pcm16 = Buffer.alloc(0);
+        segmentId = uuidv4();
+      },
+      close() {
+        connected = false;
+        pcm16 = Buffer.alloc(0);
+      },
+      on(event: any, handler: any) {
+        emitter.on(event, handler);
+        return undefined;
+      },
+    };
+  }
+
+  public async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
+    const start = Date.now();
+
+    let inputRate: number;
+    let pcm16: Buffer;
+
+    if (format.toLowerCase().includes("audio/wav")) {
+      const parsed = parsePcm16MonoWav(audioBuffer);
+      inputRate = parsed.sampleRate;
+      pcm16 = parsed.pcm16;
+    } else if (format.toLowerCase().includes("audio/pcm")) {
+      inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate;
+      pcm16 = audioBuffer;
+    } else {
+      throw new Error(`Unsupported audio format for sherpa Parakeet STT: ${format}`);
+    }
+
+    const peak = pcm16lePeakAbs(pcm16);
+    if (peak < this.silencePeakThreshold) {
+      return { text: "", duration: Date.now() - start, isLowConfidence: true };
+    }
+
+    let pcmForModel = pcm16;
+    if (inputRate !== this.engine.sampleRate) {
+      const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate });
+      pcmForModel = resampler.processChunk(pcm16);
+      inputRate = this.engine.sampleRate;
+    }
+
+    const peakForModel = pcm16lePeakAbs(pcmForModel);
+    const peakFloat = peakForModel / 32768.0;
+    const targetPeak = 0.6;
+    const maxGain = 50;
+    const gain =
+      peakFloat > 0 && peakFloat < targetPeak
+        ? Math.min(maxGain, targetPeak / peakFloat)
+        : 1;
+
+    const stream = this.engine.createStream();
+    try {
+      const floatSamples = pcm16leToFloat32(pcmForModel, gain);
+      this.engine.acceptWaveform(stream, inputRate, floatSamples);
+      this.engine.recognizer.decode(stream);
+      const result = this.engine.recognizer.getResult(stream);
+      const text = String(result?.text ?? result ?? "").trim();
+      const duration = Date.now() - start;
+      this.logger.debug({ duration, textLength: text.length }, "Parakeet transcription complete");
+      return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) };
+    } finally {
+      try {
+        stream.free?.();
+      } catch {
+        // ignore
+      }
+    }
+  }
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts
@@ -0,0 +1,132 @@
+import { EventEmitter } from "node:events";
+import { v4 as uuidv4 } from "uuid";
+
+import type { StreamingTranscriptionSession } from "../../../speech-provider.js";
+import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
+import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
+
+export class SherpaRealtimeTranscriptionSession
+  extends EventEmitter
+  implements StreamingTranscriptionSession
+{
+  private readonly engine: SherpaOnlineRecognizerEngine;
+  private stream: any | null = null;
+  private connected = false;
+
+  public readonly requiredSampleRate: number;
+  private currentSegmentId: string | null = null;
+  private previousSegmentId: string | null = null;
+  private lastPartialText = "";
+  private readonly tailPaddingMs: number;
+
+  constructor(params: { engine: SherpaOnlineRecognizerEngine; tailPaddingMs?: number }) {
+    super();
+    this.engine = params.engine;
+    this.requiredSampleRate = this.engine.sampleRate;
+    this.tailPaddingMs = params.tailPaddingMs ?? 500;
+  }
+
+  async connect(): Promise<void> {
+    if (this.connected) {
+      return;
+    }
+    this.stream = this.engine.createStream();
+    this.currentSegmentId = uuidv4();
+    this.connected = true;
+  }
+
+  appendPcm16(pcm16le: Buffer): void {
+    if (!this.connected || !this.stream || !this.currentSegmentId) {
+      this.emit("error", new Error("Sherpa realtime session not connected"));
+      return;
+    }
+
+    try {
+      const peak = pcm16lePeakAbs(pcm16le);
+      const peakFloat = peak / 32768.0;
+      const targetPeak = 0.6;
+      const maxGain = 50;
+      const gain =
+        peakFloat > 0 && peakFloat < targetPeak
+          ? Math.min(maxGain, targetPeak / peakFloat)
+          : 1;
+      const floatSamples = pcm16leToFloat32(pcm16le, gain);
+      this.stream.acceptWaveform(this.engine.sampleRate, floatSamples);
+
+      while (this.engine.recognizer.isReady(this.stream)) {
+        this.engine.recognizer.decode(this.stream);
+      }
+
+      const text = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim();
+      if (text !== this.lastPartialText) {
+        this.lastPartialText = text;
+        this.emit("transcript", {
+          segmentId: this.currentSegmentId,
+          transcript: text,
+          isFinal: false,
+        });
+      }
+    } catch (err) {
+      this.emit("error", err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+
+  commit(): void {
+    if (!this.connected || !this.stream || !this.currentSegmentId) {
+      this.emit("error", new Error("Sherpa realtime session not connected"));
+      return;
+    }
+
+    try {
+      const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000));
+      if (padSamples > 0) {
+        this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
+      }
+
+      while (this.engine.recognizer.isReady(this.stream)) {
+        this.engine.recognizer.decode(this.stream);
+      }
+
+      const finalText = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim();
+      const segmentId = this.currentSegmentId;
+      const previousSegmentId = this.previousSegmentId;
+
+      this.emit("committed", { segmentId, previousSegmentId });
+      this.emit("transcript", { segmentId, transcript: finalText, isFinal: true });
+
+      this.previousSegmentId = segmentId;
+      this.currentSegmentId = uuidv4();
+      this.lastPartialText = "";
+      this.engine.recognizer.reset(this.stream);
+    } catch (err) {
+      this.emit("error", err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+
+  clear(): void {
+    if (!this.connected || !this.stream) {
+      return;
+    }
+    try {
+      this.engine.recognizer.reset(this.stream);
+      this.currentSegmentId = uuidv4();
+      this.lastPartialText = "";
+    } catch (err) {
+      this.emit("error", err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+
+  close(): void {
+    if (!this.stream) {
+      return;
+    }
+    try {
+      this.stream.free?.();
+    } catch {
+      // ignore
+    } finally {
+      this.stream = null;
+      this.connected = false;
+    }
+  }
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts
@@ -0,0 +1,173 @@
+import { EventEmitter } from "node:events";
+import { v4 as uuidv4 } from "uuid";
+import type pino from "pino";
+
+import type {
+  SpeechToTextProvider,
+  StreamingTranscriptionSession,
+  TranscriptionResult,
+} from "../../../speech-provider.js";
+import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
+import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
+import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
+
+export type SherpaSttConfig = {
+  engine: SherpaOnlineRecognizerEngine;
+  silencePeakThreshold?: number;
+  tailPaddingMs?: number;
+};
+
+export class SherpaOnnxSTT implements SpeechToTextProvider {
+  private readonly engine: SherpaOnlineRecognizerEngine;
+  private readonly silencePeakThreshold: number;
+  private readonly tailPaddingMs: number;
+  private readonly logger: pino.Logger;
+  public readonly id = "local" as const;
+
+  constructor(config: SherpaSttConfig, logger: pino.Logger) {
+    this.engine = config.engine;
+    this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
+    this.tailPaddingMs = config.tailPaddingMs ?? 500;
+    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" });
+  }
+
+  public createSession(params: {
+    logger: pino.Logger;
+    language?: string;
+    prompt?: string;
+  }): StreamingTranscriptionSession {
+    const emitter = new EventEmitter();
+    void params;
+    const requiredSampleRate = this.engine.sampleRate;
+    let connected = false;
+    let segmentId = uuidv4();
+    let previousSegmentId: string | null = null;
+    let pcm16: Buffer = Buffer.alloc(0);
+
+    return {
+      requiredSampleRate,
+      async connect() {
+        connected = true;
+      },
+      appendPcm16(chunk: Buffer) {
+        if (!connected) {
+          (emitter as any).emit("error", new Error("STT session not connected"));
+          return;
+        }
+        pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]);
+      },
+      commit: () => {
+        if (!connected) {
+          (emitter as any).emit("error", new Error("STT session not connected"));
+          return;
+        }
+
+        const committedId = segmentId;
+        const prev = previousSegmentId;
+        (emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev });
+
+        void (async () => {
+          try {
+            const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`);
+            (emitter as any).emit("transcript", {
+              segmentId: committedId,
+              transcript: rt.text,
+              isFinal: true,
+              language: rt.language,
+              logprobs: rt.logprobs,
+              avgLogprob: rt.avgLogprob,
+              isLowConfidence: rt.isLowConfidence,
+            });
+          } catch (err) {
+            (emitter as any).emit("error", err);
+          } finally {
+            previousSegmentId = committedId;
+            segmentId = uuidv4();
+            pcm16 = Buffer.alloc(0);
+          }
+        })();
+      },
+      clear() {
+        pcm16 = Buffer.alloc(0);
+        segmentId = uuidv4();
+      },
+      close() {
+        connected = false;
+        pcm16 = Buffer.alloc(0);
+      },
+      on(event: any, handler: any) {
+        emitter.on(event, handler);
+        return undefined;
+      },
+    };
+  }
+
+  public async transcribeAudio(audioBuffer: Buffer, format: string): Promise<TranscriptionResult> {
+    const start = Date.now();
+
+    let inputRate: number;
+    let pcm16: Buffer;
+
+    if (format.toLowerCase().includes("audio/wav")) {
+      const parsed = parsePcm16MonoWav(audioBuffer);
+      inputRate = parsed.sampleRate;
+      pcm16 = parsed.pcm16;
+    } else if (format.toLowerCase().includes("audio/pcm")) {
+      inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate;
+      pcm16 = audioBuffer;
+    } else {
+      throw new Error(`Unsupported audio format for sherpa STT: ${format}`);
+    }
+
+    const peak = pcm16lePeakAbs(pcm16);
+    if (peak < this.silencePeakThreshold) {
+      return { text: "", duration: Date.now() - start, isLowConfidence: true };
+    }
+
+    let pcmForModel = pcm16;
+    if (inputRate !== this.engine.sampleRate) {
+      const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate });
+      pcmForModel = resampler.processChunk(pcm16);
+      inputRate = this.engine.sampleRate;
+    }
+
+    const peakForModel = pcm16lePeakAbs(pcmForModel);
+    const peakFloat = peakForModel / 32768.0;
+    const targetPeak = 0.6;
+    const maxGain = 50;
+    const gain =
+      peakFloat > 0 && peakFloat < targetPeak
+        ? Math.min(maxGain, targetPeak / peakFloat)
+        : 1;
+
+    const stream = this.engine.createStream();
+    try {
+      const floatSamples = pcm16leToFloat32(pcmForModel, gain);
+      stream.acceptWaveform(inputRate, floatSamples);
+
+      while (this.engine.recognizer.isReady(stream)) {
+        this.engine.recognizer.decode(stream);
+      }
+
+      const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000));
+      if (padSamples > 0) {
+        stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
+      }
+
+      while (this.engine.recognizer.isReady(stream)) {
+        this.engine.recognizer.decode(stream);
+      }
+
+      const text = String(this.engine.recognizer.getResult(stream)?.text ?? "").trim();
+      const duration = Date.now() - start;
+      this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete");
+      return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) };
+    } finally {
+      try {
+        stream.free?.();
+      } catch {
+        // ignore
+      }
+    }
+  }
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts
@@ -0,0 +1,121 @@
+import type pino from "pino";
+import { Readable } from "node:stream";
+import { existsSync } from "node:fs";
+
+import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js";
+import { chunkBuffer, float32ToPcm16le } from "../../../audio.js";
+import { loadSherpaOnnx } from "./sherpa-onnx-loader.js";
+
+export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
+
+export type SherpaTtsConfig = {
+  preset: SherpaTtsPreset;
+  modelDir: string;
+  speakerId?: number;
+  speed?: number;
+  lengthScale?: number;
+  numThreads?: number;
+};
+
+function assertFileExists(filePath: string, label: string): void {
+  if (!existsSync(filePath)) {
+    throw new Error(`Missing ${label}: ${filePath}`);
+  }
+}
+
+export class SherpaOnnxTTS implements TextToSpeechProvider {
+  private readonly tts: any;
+  private readonly speakerId: number;
+  private readonly speed: number;
+  private readonly logger: pino.Logger;
+
+  constructor(config: SherpaTtsConfig, logger: pino.Logger) {
+    if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
+      throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
+    }
+    this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" });
+    this.speakerId = config.speakerId ?? 0;
+    this.speed = config.speed ?? 1.0;
+
+    const sherpa = loadSherpaOnnx();
+
+    const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx";
+    const modelPath = `${config.modelDir}/${modelFile}`;
+    const voicesPath = `${config.modelDir}/voices.bin`;
+    const tokensPath = `${config.modelDir}/tokens.txt`;
+    const dataDir = `${config.modelDir}/espeak-ng-data`;
+
+    assertFileExists(modelPath, "TTS model");
+    assertFileExists(voicesPath, "TTS voices");
+    assertFileExists(tokensPath, "TTS tokens");
+    assertFileExists(dataDir, "TTS espeak-ng dataDir");
+
+    const modelConfigKey =
+      config.preset === "kokoro-en-v0_19"
+        ? "offlineTtsKokoroModelConfig"
+        : "offlineTtsKittenModelConfig";
+
+    const modelConfig = {
+      [modelConfigKey]: {
+        model: modelPath,
+        voices: voicesPath,
+        tokens: tokensPath,
+        dataDir,
+        lengthScale: config.lengthScale ?? 1.0,
+      },
+      numThreads: config.numThreads ?? 2,
+      debug: 0,
+      provider: "cpu",
+    };
+
+    const offlineTtsConfig = {
+      offlineTtsModelConfig: modelConfig,
+      maxNumSentences: 1,
+    };
+
+    this.tts = sherpa.createOfflineTts(offlineTtsConfig);
+    this.logger.info({ preset: config.preset, modelDir: config.modelDir }, "Sherpa offline TTS initialized");
+  }
+
+  async synthesizeSpeech(text: string): Promise<SpeechStreamResult> {
+    const trimmed = text.trim();
+    if (!trimmed) {
+      throw new Error("Cannot synthesize empty text");
+    }
+
+    const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed });
+    const samples: Float32Array | null =
+      audio && audio.samples instanceof Float32Array
+        ? audio.samples
+        : audio && Array.isArray(audio.samples)
+          ? Float32Array.from(audio.samples as number[])
+          : null;
+    const sampleRate: number =
+      audio && typeof audio.sampleRate === "number" && Number.isFinite(audio.sampleRate) && audio.sampleRate > 0
+        ? audio.sampleRate
+        : typeof this.tts.sampleRate === "number"
+          ? this.tts.sampleRate
+          : 24000;
+
+    if (!samples) {
+      throw new Error("Unexpected sherpa TTS output: missing Float32 samples");
+    }
+
+    const pcm16 = float32ToPcm16le(samples);
+    const chunkBytes = Math.max(2, Math.round(sampleRate * 0.05) * 2); // ~50ms
+    const chunks = chunkBuffer(pcm16, chunkBytes);
+
+    return {
+      stream: Readable.from(chunks),
+      format: `pcm;rate=${sampleRate}`,
+    };
+  }
+
+  free(): void {
+    try {
+      this.tts?.free?.();
+    } catch {
+      // ignore
+    }
+  }
+}
--- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts
+++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts
@@ -0,0 +1,264 @@
+import { describe, expect, test } from "vitest";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import pino from "pino";
+
+import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
+import type { SherpaOnnxModelId } from "./model-catalog.js";
+import { createDaemonTestContext } from "../../../../test-utils/index.js";
+import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js";
+import { SherpaOnnxTTS } from "./sherpa-tts.js";
+import { PocketTtsOnnxTTS } from "../pocket/pocket-tts-onnx.js";
+import { SherpaOnlineRecognizerEngine } from "./sherpa-online-recognizer.js";
+import { SherpaOnnxSTT } from "./sherpa-stt.js";
+import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js";
+import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js";
+
+const RUN = process.env.PASEO_SPEECH_E2E_DOWNLOAD === "1";
+const downloadTest = RUN ? test : test.skip;
+
+type ModelSet = "zipformer-kitten" | "parakeet-pocket";
+
+function getModelSet(): ModelSet {
+  const raw = (process.env.PASEO_SPEECH_E2E_MODEL_SET ?? "parakeet-pocket").trim().toLowerCase();
+  if (raw === "zipformer-kitten" || raw === "zipformer") return "zipformer-kitten";
+  if (raw === "parakeet-pocket" || raw === "parakeet") return "parakeet-pocket";
+  throw new Error(`Unknown PASEO_SPEECH_E2E_MODEL_SET: ${raw}`);
+}
+
+async function readFixtureWav(): Promise<Buffer> {
+  const fixturePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.wav");
+  return import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
+}
+
+async function readBaseline(): Promise<string> {
+  const baselinePath = path.resolve(process.cwd(), "..", "app", "e2e", "fixtures", "recording.baseline.txt");
+  return import("node:fs/promises").then((fs) => fs.readFile(baselinePath, "utf-8")).then((t) => t.trim());
+}
+
+async function readAllChunks(stream: NodeJS.ReadableStream): Promise<{ chunks: Buffer[]; combined: Buffer }> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream as any) {
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+  }
+  return { chunks, combined: Buffer.concat(chunks) };
+}
+
+function waitForSignal<T>(
+  timeoutMs: number,
+  setup: (
+    resolve: (value: T) => void,
+    reject: (error: Error) => void
+  ) => () => void
+): Promise<T> {
+  return new Promise((resolve, reject) => {
+    let cleanup: (() => void) | null = null;
+    const timeout = setTimeout(() => {
+      cleanup?.();
+      reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`));
+    }, timeoutMs);
+
+    cleanup = setup(
+      (value) => {
+        clearTimeout(timeout);
+        cleanup?.();
+        resolve(value);
+      },
+      (error) => {
+        clearTimeout(timeout);
+        cleanup?.();
+        reject(error);
+      }
+    );
+  });
+}
+
+function toAudioPcmFormat(format: string): string {
+  const trimmed = format.trim();
+  const lower = trimmed.toLowerCase();
+  if (lower.startsWith("audio/pcm")) {
+    return trimmed;
+  }
+  if (lower.startsWith("pcm")) {
+    const rest = trimmed.replace(/^pcm;?/i, "");
+    return rest ? `audio/pcm;${rest}` : "audio/pcm";
+  }
+  // Fall back to treating it as a suffix (e.g. "rate=24000")
+  return `audio/pcm;${trimmed}`;
+}
+
+describe("speech models (download E2E)", () => {
+  downloadTest(
+    "downloads models and runs streaming STT + streaming TTS",
+    async () => {
+      const logger = pino({ level: "silent" });
+      const set = getModelSet();
+
+      const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
+      const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx");
+
+      const modelIds: SherpaOnnxModelId[] =
+        set === "parakeet-pocket"
+          ? ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"]
+          : ["zipformer-bilingual-zh-en-2023-02-20", "kitten-nano-en-v0_1-fp16"];
+
+      await ensureSherpaOnnxModels({
+        modelsDir,
+        modelIds,
+        autoDownload: true,
+        logger,
+      });
+
+      const ctx = await createDaemonTestContext({
+        paseoHomeRoot,
+        dictationFinalTimeoutMs: 8000,
+        speech: {
+          dictationSttProvider: "local",
+          voiceSttProvider: "local",
+          voiceTtsProvider: "local",
+          sherpaOnnx: {
+            modelsDir,
+            autoDownload: false,
+            stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" },
+            tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" },
+          },
+        },
+      });
+
+      try {
+        const wav = await readFixtureWav();
+        const { sampleRate, pcm16 } = parsePcm16MonoWav(wav);
+        expect(sampleRate).toBe(16000);
+        const format = "audio/pcm;rate=16000;bits=16";
+
+        // Streaming STT: dictation path (verifies websocket streaming + partials + final)
+        const dictationId = `dict-download-${Date.now()}`;
+        let partialCount = 0;
+        const unsubscribe = ctx.client.on("dictation_stream_partial", (message) => {
+          if (message.type !== "dictation_stream_partial") return;
+          if (message.payload.dictationId !== dictationId) return;
+          partialCount += 1;
+        });
+
+        await ctx.client.startDictationStream(dictationId, format);
+        const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
+        let seq = 0;
+        for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
+          const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
+          ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
+          seq += 1;
+        }
+        const finalSeq = seq - 1;
+        const dictationFinal = await ctx.client.finishDictationStream(dictationId, finalSeq);
+        unsubscribe();
+
+        expect(dictationFinal.text.toLowerCase()).toContain("voice note");
+        const baseline = await readBaseline();
+        expect(wordSimilarity(dictationFinal.text, baseline)).toBeGreaterThan(0.45);
+        expect(partialCount).toBeGreaterThan(0);
+
+        // Voice-mode STT: chunked upload until isLast=true
+        const transcriptionPromise = waitForSignal<string>(30000, (resolve, reject) => {
+          const offResult = ctx.client.on("transcription_result", (message) => {
+            if (message.type !== "transcription_result") return;
+            resolve(message.payload.text);
+          });
+          const offError = ctx.client.on("activity_log", (message) => {
+            if (message.type !== "activity_log") return;
+            const payload = message.payload as { type?: unknown; content?: unknown };
+            if (payload.type !== "error") return;
+            const content = typeof payload.content === "string" ? payload.content : null;
+            if (!content) return;
+            reject(new Error(content));
+          });
+          return () => {
+            offResult();
+            offError();
+          };
+        });
+
+        await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`);
+        for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
+          const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
+          const isLast = offset + chunkBytes >= pcm16.length;
+          await ctx.client.sendVoiceAudioChunk(chunk.toString("base64"), format, isLast);
+        }
+        const voiceText = (await transcriptionPromise).trim().toLowerCase();
+        if (voiceText.length > 0) {
+          expect(voiceText).toContain("voice note");
+        }
+        await ctx.client.setVoiceConversation(false);
+
+        // Streaming TTS: generate locally from downloaded model and validate chunking.
+        const ttsText = "This is a voice note.";
+        if (set === "parakeet-pocket") {
+          const modelDir = getSherpaOnnxModelDir(modelsDir, "pocket-tts-onnx-int8");
+          const tts = await PocketTtsOnnxTTS.create({ modelDir, precision: "int8", targetChunkMs: 50 }, logger);
+          const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
+          const { chunks, combined } = await readAllChunks(stream);
+
+          expect(ttsFormat).toMatch(/rate=\d+/);
+          expect(chunks.length).toBeGreaterThan(3);
+          expect(combined.byteLength).toBeGreaterThan(2000);
+
+          // Round trip: TTS -> STT (offline parakeet)
+          const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v3-int8");
+          const engine = new SherpaOfflineRecognizerEngine(
+            {
+              model: {
+                kind: "nemo_transducer",
+                encoder: `${sttModelDir}/encoder.int8.onnx`,
+                decoder: `${sttModelDir}/decoder.int8.onnx`,
+                joiner: `${sttModelDir}/joiner.int8.onnx`,
+                tokens: `${sttModelDir}/tokens.txt`,
+              },
+              numThreads: 2,
+              debug: 0,
+            },
+            logger
+          );
+          const stt = new SherpaOnnxParakeetSTT({ engine }, logger);
+          const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
+          engine.free();
+          expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
+        } else {
+          const ttsModelDir = path.join(modelsDir, "kitten-nano-en-v0_1-fp16");
+          const tts = new SherpaOnnxTTS({ preset: "kitten-nano-en-v0_1-fp16", modelDir: ttsModelDir }, logger);
+          const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText);
+          const { chunks, combined } = await readAllChunks(stream);
+          tts.free();
+
+          expect(ttsFormat).toMatch(/rate=\d+/);
+          expect(chunks.length).toBeGreaterThan(3);
+          expect(combined.byteLength).toBeGreaterThan(2000);
+
+          // Round trip: TTS -> STT (online zipformer, offline segment)
+          const sttModelDir = path.join(modelsDir, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20");
+          const engine = new SherpaOnlineRecognizerEngine(
+            {
+              model: {
+                kind: "transducer",
+                encoder: `${sttModelDir}/encoder-epoch-99-avg-1.onnx`,
+                decoder: `${sttModelDir}/decoder-epoch-99-avg-1.onnx`,
+                joiner: `${sttModelDir}/joiner-epoch-99-avg-1.onnx`,
+                tokens: `${sttModelDir}/tokens.txt`,
+                modelType: "zipformer",
+              },
+              numThreads: 1,
+              debug: 0,
+            },
+            logger
+          );
+          const stt = new SherpaOnnxSTT({ engine }, logger);
+          const rt = await stt.transcribeAudio(combined, toAudioPcmFormat(ttsFormat));
+          engine.free();
+          expect(wordSimilarity(rt.text, ttsText)).toBeGreaterThan(0.25);
+        }
+      } finally {
+        await ctx.cleanup();
+      }
+    },
+    15 * 60_000
+  );
+});
--- a/packages/server/src/server/speech/providers/openai/realtime-transcription-session.ts
+++ b/packages/server/src/server/speech/providers/openai/realtime-transcription-session.ts
@@ -1,6 +1,7 @@
 import type pino from "pino";
 import WebSocket from "ws";
 import { EventEmitter } from "node:events";
+import type { StreamingTranscriptionSession } from "../../speech-provider.js";

 type OpenAITurnDetection =
  | null
@@ -60,7 +61,11 @@ type OpenAIServerEvent =
    }
  | { type: "error"; error?: { message?: string } };

-export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
+export class OpenAIRealtimeTranscriptionSession
+  extends EventEmitter
+  implements StreamingTranscriptionSession
+{
+  public readonly requiredSampleRate = 24000;
  private readonly apiKey: string;
  private readonly logger: pino.Logger;
  private readonly transcriptionModel: string;
@@ -161,8 +166,8 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter {

        if (event.type === "input_audio_buffer.committed") {
          this.emit("committed", {
-            itemId: event.item_id,
-            previousItemId: event.previous_item_id,
+            segmentId: event.item_id,
+            previousSegmentId: event.previous_item_id,
          });
          return;
        }
@@ -182,13 +187,13 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
          const prev = this.partialByItemId.get(event.item_id) ?? "";
          const next = replaceDelta ? event.delta : prev + event.delta;
          this.partialByItemId.set(event.item_id, next);
-          this.emit("transcript", { itemId: event.item_id, transcript: next, isFinal: false });
+          this.emit("transcript", { segmentId: event.item_id, transcript: next, isFinal: false });
          return;
        }

        if (event.type === "conversation.item.input_audio_transcription.completed") {
          this.partialByItemId.set(event.item_id, event.transcript);
-          this.emit("transcript", { itemId: event.item_id, transcript: event.transcript, isFinal: true });
+          this.emit("transcript", { segmentId: event.item_id, transcript: event.transcript, isFinal: true });
          return;
        }

@@ -218,10 +223,11 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
    return this.ready;
  }

-  public appendPcm16Base64(base64Audio: string): void {
+  public appendPcm16(pcm16le: Buffer): void {
    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
      throw new Error("OpenAI realtime websocket not connected");
    }
+    const base64Audio = pcm16le.toString("base64");
    const event: OpenAIClientEvent = { type: "input_audio_buffer.append", audio: base64Audio };
    this.ws.send(JSON.stringify(event));
  }
--- a/packages/server/src/server/speech/providers/openai/stt.ts
+++ b/packages/server/src/server/speech/providers/openai/stt.ts
@@ -0,0 +1,269 @@
+import { EventEmitter } from "node:events";
+import type pino from "pino";
+import OpenAI from "openai";
+import { writeFile, unlink } from "fs/promises";
+import { join } from "path";
+import { tmpdir } from "os";
+import { v4 } from "uuid";
+import { inferAudioExtension } from "../../../agent/audio-utils.js";
+import type {
+  LogprobToken,
+  SpeechToTextProvider,
+  StreamingTranscriptionSession,
+  TranscriptionResult,
+} from "../../speech-provider.js";
+
+export type { LogprobToken, TranscriptionResult };
+
+export interface STTConfig {
+  apiKey: string;
+  model?: "whisper-1" | "gpt-4o-transcribe" | "gpt-4o-mini-transcribe" | (string & {});
+  confidenceThreshold?: number; // Default: -3.0
+}
+
+function isObject(value: unknown): value is { [key: string]: unknown } {
+  return typeof value === "object" && value !== null;
+}
+
+function isLogprobToken(value: unknown): value is LogprobToken {
+  if (!isObject(value)) {
+    return false;
+  }
+  if (typeof value.token !== "string") {
+    return false;
+  }
+  if (typeof value.logprob !== "number") {
+    return false;
+  }
+  if (value.bytes === undefined) {
+    return true;
+  }
+  return Array.isArray(value.bytes) && value.bytes.every((entry) => typeof entry === "number");
+}
+
+function isLogprobTokenArray(value: unknown): value is LogprobToken[] {
+  return Array.isArray(value) && value.every((entry) => isLogprobToken(entry));
+}
+
+export class OpenAISTT implements SpeechToTextProvider {
+  private readonly openaiClient: OpenAI;
+  private readonly config: STTConfig;
+  private readonly logger: pino.Logger;
+  public readonly id = "openai" as const;
+
+  constructor(sttConfig: STTConfig, parentLogger: pino.Logger) {
+    this.config = sttConfig;
+    this.logger = parentLogger.child({ module: "agent", provider: "openai", component: "stt" });
+    this.openaiClient = new OpenAI({
+      apiKey: sttConfig.apiKey,
+    });
+    this.logger.info({ model: sttConfig.model || "whisper-1" }, "STT (OpenAI Whisper) initialized");
+  }
+
+  public createSession(params: {
+    logger: pino.Logger;
+    language?: string;
+    prompt?: string;
+  }): StreamingTranscriptionSession {
+    const emitter = new EventEmitter();
+    const logger = params.logger.child({ provider: "openai", component: "stt-session" });
+    const requiredSampleRate = 24000;
+
+    let connected = false;
+    let segmentId = v4();
+    let previousSegmentId: string | null = null;
+    let pcm16: Buffer = Buffer.alloc(0);
+    const transcribeAudio = this.transcribeAudioInternal.bind(this);
+
+    const convertPCMToWavBuffer = (pcmBuffer: Buffer): Buffer => {
+      const headerSize = 44;
+      const channels = 1;
+      const bitsPerSample = 16;
+      const sampleRate = requiredSampleRate;
+      const wavBuffer = Buffer.alloc(headerSize + pcmBuffer.length);
+      const byteRate = (sampleRate * channels * bitsPerSample) / 8;
+      const blockAlign = (channels * bitsPerSample) / 8;
+
+      wavBuffer.write("RIFF", 0);
+      wavBuffer.writeUInt32LE(36 + pcmBuffer.length, 4);
+      wavBuffer.write("WAVE", 8);
+      wavBuffer.write("fmt ", 12);
+      wavBuffer.writeUInt32LE(16, 16);
+      wavBuffer.writeUInt16LE(1, 20);
+      wavBuffer.writeUInt16LE(channels, 22);
+      wavBuffer.writeUInt32LE(sampleRate, 24);
+      wavBuffer.writeUInt32LE(byteRate, 28);
+      wavBuffer.writeUInt16LE(blockAlign, 32);
+      wavBuffer.writeUInt16LE(bitsPerSample, 34);
+      wavBuffer.write("data", 36);
+      wavBuffer.writeUInt32LE(pcmBuffer.length, 40);
+      pcmBuffer.copy(wavBuffer, 44);
+
+      return wavBuffer;
+    };
+
+    return {
+      requiredSampleRate,
+      async connect() {
+        connected = true;
+      },
+      appendPcm16(chunk: Buffer) {
+        if (!connected) {
+          (emitter as any).emit("error", new Error("STT session not connected"));
+          return;
+        }
+        pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]);
+      },
+      commit() {
+        if (!connected) {
+          (emitter as any).emit("error", new Error("STT session not connected"));
+          return;
+        }
+
+        const committedId = segmentId;
+        const prev = previousSegmentId;
+        (emitter as any).emit("committed", { segmentId: committedId, previousSegmentId: prev });
+
+        void (async () => {
+          try {
+            if (pcm16.length === 0) {
+              (emitter as any).emit("transcript", {
+                segmentId: committedId,
+                transcript: "",
+                isFinal: true,
+                language: params.language,
+                isLowConfidence: true,
+              });
+              return;
+            }
+
+            const wav = convertPCMToWavBuffer(pcm16);
+            const result = await transcribeAudio(
+              wav,
+              "audio/wav",
+              params.language ?? "en",
+              logger
+            );
+
+            (emitter as any).emit("transcript", {
+              segmentId: committedId,
+              transcript: result.text,
+              isFinal: true,
+              language: result.language,
+              logprobs: result.logprobs,
+              avgLogprob: result.avgLogprob,
+              isLowConfidence: result.isLowConfidence,
+            });
+          } catch (err) {
+            (emitter as any).emit("error", err);
+          } finally {
+            previousSegmentId = committedId;
+            segmentId = v4();
+            pcm16 = Buffer.alloc(0);
+          }
+        })();
+      },
+      clear() {
+        pcm16 = Buffer.alloc(0);
+        segmentId = v4();
+      },
+      close() {
+        connected = false;
+        pcm16 = Buffer.alloc(0);
+      },
+      on(event: any, handler: any) {
+        emitter.on(event, handler);
+        return undefined;
+      },
+    };
+  }
+
+  private async transcribeAudioInternal(
+    audioBuffer: Buffer,
+    format: string,
+    language: string,
+    logger: pino.Logger
+  ): Promise<TranscriptionResult> {
+    const startTime = Date.now();
+    let tempFilePath: string | null = null;
+
+    try {
+      const ext = inferAudioExtension(format);
+      tempFilePath = join(tmpdir(), `audio-${v4()}.${ext}`);
+      await writeFile(tempFilePath, audioBuffer);
+
+      logger.debug(
+        { tempFilePath, bytes: audioBuffer.length },
+        "Transcribing audio file"
+      );
+
+      const modelToUse = this.config.model ?? "whisper-1";
+      const supportsLogprobs =
+        modelToUse === "gpt-4o-transcribe" || modelToUse === "gpt-4o-mini-transcribe";
+      const includeLogprobs: ["logprobs"] = ["logprobs"];
+
+      const response = await this.openaiClient.audio.transcriptions.create({
+        file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)),
+        language,
+        model: modelToUse,
+        ...(supportsLogprobs ? { include: includeLogprobs } : {}),
+        response_format: "json",
+      });
+
+      const duration = Date.now() - startTime;
+      const confidenceThreshold = this.config.confidenceThreshold ?? -3.0;
+
+      let avgLogprob: number | undefined;
+      let isLowConfidence = false;
+      const logprobs =
+        supportsLogprobs &&
+        isObject(response) &&
+        isLogprobTokenArray(response.logprobs)
+          ? response.logprobs
+          : undefined;
+
+      if (logprobs && logprobs.length > 0) {
+        const totalLogprob = logprobs.reduce((sum, token) => sum + token.logprob, 0);
+        avgLogprob = totalLogprob / logprobs.length;
+        isLowConfidence = avgLogprob < confidenceThreshold;
+
+        if (isLowConfidence) {
+          logger.debug(
+            {
+              avgLogprob,
+              threshold: confidenceThreshold,
+              text: response.text,
+              tokenLogprobs: logprobs.map((t) => `${t.token}:${t.logprob.toFixed(2)}`).join(", "),
+            },
+            "Low confidence transcription detected"
+          );
+        }
+      }
+
+      logger.debug({ duration, text: response.text, avgLogprob }, "Transcription complete");
+
+      return {
+        text: response.text,
+        duration: duration,
+        logprobs: logprobs,
+        avgLogprob: avgLogprob,
+        isLowConfidence: isLowConfidence,
+        language:
+          isObject(response) && typeof response.language === "string"
+            ? response.language
+            : undefined,
+      };
+    } catch (error: any) {
+      logger.error({ err: error }, "Transcription error");
+      throw new Error(`STT transcription failed: ${error.message}`);
+    } finally {
+      if (tempFilePath) {
+        try {
+          await unlink(tempFilePath);
+        } catch (cleanupError) {
+          logger.warn({ tempFilePath }, "Failed to clean up temp file");
+        }
+      }
+    }
+  }
+}
--- a/packages/server/src/server/speech/providers/openai/tts.ts
+++ b/packages/server/src/server/speech/providers/openai/tts.ts
@@ -1,6 +1,9 @@
 import type pino from "pino";
 import OpenAI from "openai";
-import { Readable } from "stream";
+import { Readable } from "node:stream";
+import type { SpeechStreamResult, TextToSpeechProvider } from "../../speech-provider.js";
+
+export type { SpeechStreamResult };

 export interface TTSConfig {
  apiKey: string;
@@ -9,12 +12,7 @@ export interface TTSConfig {
  responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm";
 }

-export interface SpeechStreamResult {
-  stream: Readable;
-  format: string;
-}
-
-export class OpenAITTS {
+export class OpenAITTS implements TextToSpeechProvider {
  private readonly openaiClient: OpenAI;
  private readonly config: TTSConfig;
  private readonly logger: pino.Logger;
--- a/packages/server/src/server/speech/speech-provider.ts
+++ b/packages/server/src/server/speech/speech-provider.ts
@@ -0,0 +1,74 @@
+import type pino from "pino";
+import type { Readable } from "node:stream";
+
+export interface LogprobToken {
+  token: string;
+  logprob: number;
+  bytes?: number[];
+}
+
+export interface TranscriptionResult {
+  text: string;
+  language?: string;
+  duration?: number;
+  logprobs?: LogprobToken[];
+  avgLogprob?: number;
+  isLowConfidence?: boolean;
+}
+
+export interface StreamingTranscriptionCommittedEvent {
+  segmentId: string;
+  previousSegmentId: string | null;
+}
+
+export interface StreamingTranscriptionEvent {
+  segmentId: string;
+  transcript: string;
+  isFinal: boolean;
+  language?: string;
+  logprobs?: LogprobToken[];
+  avgLogprob?: number;
+  isLowConfidence?: boolean;
+}
+
+export type StreamingTranscriptionSession = {
+  /**
+   * Required PCM16LE sample rate for `appendPcm16()`.
+   * Callers are responsible for resampling before appending.
+   */
+  requiredSampleRate: number;
+
+  connect(): Promise<void>;
+  appendPcm16(pcm16le: Buffer): void;
+  commit(): void;
+  clear(): void;
+  close(): void;
+
+  on(
+    event: "committed",
+    handler: (payload: StreamingTranscriptionCommittedEvent) => void
+  ): unknown;
+  on(
+    event: "transcript",
+    handler: (payload: StreamingTranscriptionEvent) => void
+  ): unknown;
+  on(event: "error", handler: (err: unknown) => void): unknown;
+};
+
+export interface SpeechToTextProvider {
+  id: "openai" | "local" | (string & {});
+  createSession(params: {
+    logger: pino.Logger;
+    language?: string;
+    prompt?: string;
+  }): StreamingTranscriptionSession;
+}
+
+export interface SpeechStreamResult {
+  stream: Readable;
+  format: string;
+}
+
+export interface TextToSpeechProvider {
+  synthesizeSpeech(text: string): Promise<SpeechStreamResult>;
+}
--- a/packages/server/src/server/test-utils/paseo-daemon.ts
+++ b/packages/server/src/server/test-utils/paseo-daemon.ts
@@ -4,7 +4,7 @@ import path from "node:path";
 import { mkdir, mkdtemp, rm } from "node:fs/promises";

 import pino from "pino";
-import { createPaseoDaemon, type PaseoDaemonConfig, type PaseoOpenAIConfig } from "../bootstrap.js";
+import { createPaseoDaemon, type PaseoDaemonConfig, type PaseoOpenAIConfig, type PaseoSpeechConfig } from "../bootstrap.js";
 import type { AgentClient, AgentProvider } from "../agent/agent-sdk-types.js";
 import { createTestAgentClients } from "./fake-agent-client.js";

@@ -20,6 +20,7 @@ type TestPaseoDaemonOptions = {
  staticDir?: string;
  cleanup?: boolean;
  openai?: PaseoOpenAIConfig;
+  speech?: PaseoSpeechConfig;
  dictationFinalTimeoutMs?: number;
 };

@@ -76,6 +77,7 @@ export async function createTestPaseoDaemon(
      relayEndpoint: options.relayEndpoint ?? "relay.paseo.sh:443",
      appBaseUrl: "https://app.paseo.sh",
      openai: options.openai,
+      speech: options.speech,
      openrouterApiKey: null,
      dictationFinalTimeoutMs: options.dictationFinalTimeoutMs,
      downloadTokenTtlMs: options.downloadTokenTtlMs,
--- a/packages/server/src/server/websocket-server.ts
+++ b/packages/server/src/server/websocket-server.ts
@@ -6,8 +6,6 @@ import { hostname as getHostname } from "node:os";
 import type { AgentManager } from "./agent/agent-manager.js";
 import type { AgentStorage } from "./agent/agent-storage.js";
 import type { DownloadTokenStore } from "./file-download/token-store.js";
-import type { OpenAISTT } from "./agent/stt-openai.js";
-import type { OpenAITTS } from "./agent/tts-openai.js";
 import type { TerminalManager } from "../terminal/terminal-manager.js";
 import type pino from "pino";
 import {
@@ -22,6 +20,7 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js";
 import { PushTokenStore } from "./push/token-store.js";
 import { PushService } from "./push/push-service.js";
 import { VoiceConversationStore } from "./voice-conversation-store.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";

 export type AgentMcpTransportFactory = () => Promise<Transport>;

@@ -67,13 +66,13 @@ export class VoiceAssistantWebSocketServer {
  private readonly pushTokenStore: PushTokenStore;
  private readonly pushService: PushService;
  private readonly createAgentMcpTransport: AgentMcpTransportFactory;
-  private readonly stt: OpenAISTT | null;
-  private readonly tts: OpenAITTS | null;
+  private readonly stt: SpeechToTextProvider | null;
+  private readonly tts: TextToSpeechProvider | null;
  private readonly terminalManager: TerminalManager | null;
  private readonly voiceConversationStore: VoiceConversationStore;
  private readonly dictation: {
-    openaiApiKey?: string | null;
    finalTimeoutMs?: number;
+    stt?: SpeechToTextProvider | null;
  } | null;
  private readonly voice: {
    openrouterApiKey?: string | null;
@@ -90,15 +89,15 @@ export class VoiceAssistantWebSocketServer {
    paseoHome: string,
    createAgentMcpTransport: AgentMcpTransportFactory,
    wsConfig: WebSocketServerConfig,
-    speech?: { stt: OpenAISTT | null; tts: OpenAITTS | null },
+    speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
    terminalManager?: TerminalManager | null,
    voice?: {
      openrouterApiKey?: string | null;
      voiceLlmModel?: string | null;
    },
    dictation?: {
-      openaiApiKey?: string | null;
      finalTimeoutMs?: number;
+      stt?: SpeechToTextProvider | null;
    }
  ) {
    this.logger = logger.child({ module: "websocket-server" });
--- a/packages/server/src/shared/messages.ts
+++ b/packages/server/src/shared/messages.ts
@@ -489,6 +489,17 @@ export const ListProviderModelsRequestMessageSchema = z.object({
  requestId: z.string(),
 });

+export const SpeechModelsListRequestSchema = z.object({
+  type: z.literal("speech_models_list_request"),
+  requestId: z.string(),
+});
+
+export const SpeechModelsDownloadRequestSchema = z.object({
+  type: z.literal("speech_models_download_request"),
+  modelIds: z.array(z.string()).optional(),
+  requestId: z.string(),
+});
+
 export const ResumeAgentRequestMessageSchema = z.object({
  type: z.literal("resume_agent_request"),
  handle: AgentPersistenceHandleSchema,
@@ -887,6 +898,8 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
  DictationStreamCancelMessageSchema,
  CreateAgentRequestMessageSchema,
  ListProviderModelsRequestMessageSchema,
+  SpeechModelsListRequestSchema,
+  SpeechModelsDownloadRequestSchema,
  ResumeAgentRequestMessageSchema,
  RefreshAgentRequestMessageSchema,
  CancelAgentRequestMessageSchema,
@@ -1518,6 +1531,34 @@ export const ListProviderModelsResponseMessageSchema = z.object({
  }),
 });

+export const SpeechModelsListResponseSchema = z.object({
+  type: z.literal("speech_models_list_response"),
+  payload: z.object({
+    modelsDir: z.string(),
+    models: z.array(
+      z.object({
+        id: z.string(),
+        kind: z.string(),
+        description: z.string(),
+        modelDir: z.string(),
+        isDownloaded: z.boolean(),
+        missingFiles: z.array(z.string()).optional(),
+      })
+    ),
+    requestId: z.string(),
+  }),
+});
+
+export const SpeechModelsDownloadResponseSchema = z.object({
+  type: z.literal("speech_models_download_response"),
+  payload: z.object({
+    modelsDir: z.string(),
+    downloadedModelIds: z.array(z.string()),
+    error: z.string().nullable(),
+    requestId: z.string(),
+  }),
+});
+
 const AgentSlashCommandSchema = z.object({
  name: z.string(),
  description: z.string(),
@@ -1673,6 +1714,8 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
  ProjectIconResponseSchema,
  FileDownloadTokenResponseSchema,
  ListProviderModelsResponseMessageSchema,
+  SpeechModelsListResponseSchema,
+  SpeechModelsDownloadResponseSchema,
  ListCommandsResponseSchema,
  ExecuteCommandResponseSchema,
  ListTerminalsResponseSchema,
@@ -1727,6 +1770,8 @@ export type AgentDeletedMessage = z.infer<typeof AgentDeletedMessageSchema>;
 export type ListProviderModelsResponseMessage = z.infer<
  typeof ListProviderModelsResponseMessageSchema
 >;
+export type SpeechModelsListResponse = z.infer<typeof SpeechModelsListResponseSchema>;
+export type SpeechModelsDownloadResponse = z.infer<typeof SpeechModelsDownloadResponseSchema>;
 export type InitializeAgentResponseMessage = z.infer<typeof InitializeAgentResponseMessageSchema>;

 // Type exports for payload types
@@ -1747,6 +1792,10 @@ export type CreateAgentRequestMessage = z.infer<typeof CreateAgentRequestMessage
 export type ListProviderModelsRequestMessage = z.infer<
  typeof ListProviderModelsRequestMessageSchema
 >;
+export type SpeechModelsListRequestMessage = z.infer<typeof SpeechModelsListRequestSchema>;
+export type SpeechModelsDownloadRequestMessage = z.infer<
+  typeof SpeechModelsDownloadRequestSchema
+>;
 export type ResumeAgentRequestMessage = z.infer<typeof ResumeAgentRequestMessageSchema>;
 export type DeleteAgentRequestMessage = z.infer<typeof DeleteAgentRequestMessageSchema>;
 export type InitializeAgentRequestMessage = z.infer<typeof InitializeAgentRequestMessageSchema>;