puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-07-29 12:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

```

Browse Source 
chore: configure VAD assets and update Playwright MCP settings

- Add vite-plugin-static-copy to bundle VAD WASM files
- Configure VAD with adjusted speech detection thresholds
- Switch Playwright MCP to omit image responses
- Fix WebSocket abort controller handling for interruptions
- Add WebFetch permission for VAD documentation
```
This commit is contained in:
Mohamed Boudra
2025-10-20 12:47:09 +02:00
parent 14230c681f
commit 13b5fab456
6 changed files with 306 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

234
package-lock.json generated
View File

@@ -2326,6 +2326,33 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/anymatch": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
"integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==",
"dev": true,
"license": "ISC",
"dependencies": {
"normalize-path": "^3.0.0",
"picomatch": "^2.0.4"
},
"engines": {
"node": ">= 8"
}
},
"node_modules/anymatch/node_modules/picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8.6"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@@ -2358,6 +2385,19 @@
"baseline-browser-mapping": "dist/cli.js"
}
},
"node_modules/binary-extensions": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",
"integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/body-parser": {
"version": "1.20.3",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
@@ -2397,6 +2437,19 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/braces": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
"integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
"dev": true,
"license": "MIT",
"dependencies": {
"fill-range": "^7.1.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/browserslist": {
"version": "4.26.3",
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.26.3.tgz",
@@ -2570,6 +2623,31 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/chokidar": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz",
"integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==",
"dev": true,
"license": "MIT",
"dependencies": {
"anymatch": "~3.1.2",
"braces": "~3.0.2",
"glob-parent": "~5.1.2",
"is-binary-path": "~2.1.0",
"is-glob": "~4.0.1",
"normalize-path": "~3.0.0",
"readdirp": "~3.6.0"
},
"engines": {
"node": ">= 8.10.0"
},
"funding": {
"url": "https://paulmillr.com/funding/"
},
"optionalDependencies": {
"fsevents": "~2.3.2"
}
},
"node_modules/chownr": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
@@ -3198,6 +3276,19 @@
}
}
},
"node_modules/fill-range": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
"integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
"dev": true,
"license": "MIT",
"dependencies": {
"to-regex-range": "^5.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/finalhandler": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
@@ -3384,6 +3475,19 @@
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
}
},
"node_modules/glob-parent": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
"dev": true,
"license": "ISC",
"dependencies": {
"is-glob": "^4.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/gopd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
@@ -3583,6 +3687,19 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/is-binary-path": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
"integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
"dev": true,
"license": "MIT",
"dependencies": {
"binary-extensions": "^2.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/is-decimal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz",
@@ -3593,6 +3710,16 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/is-extglob": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
"integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -3603,6 +3730,19 @@
"node": ">=8"
}
},
"node_modules/is-glob": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
"integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
"dev": true,
"license": "MIT",
"dependencies": {
"is-extglob": "^2.1.1"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/is-hexadecimal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz",
@@ -3613,6 +3753,16 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/is-number": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.12.0"
}
},
"node_modules/is-plain-obj": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz",
@@ -4771,6 +4921,16 @@
"dev": true,
"license": "MIT"
},
"node_modules/normalize-path": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
"integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -4878,6 +5038,19 @@
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"license": "MIT"
},
"node_modules/p-map": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.3.tgz",
"integrity": "sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/parse-entities": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz",
@@ -5179,6 +5352,32 @@
"node": ">=0.10.0"
}
},
"node_modules/readdirp": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
"integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
"dev": true,
"license": "MIT",
"dependencies": {
"picomatch": "^2.2.1"
},
"engines": {
"node": ">=8.10.0"
}
},
"node_modules/readdirp/node_modules/picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8.6"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/remark-parse": {
"version": "11.0.0",
"resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz",
@@ -5661,6 +5860,19 @@
"url": "https://github.com/sponsors/SuperchupuDev"
}
},
"node_modules/to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
"integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"is-number": "^7.0.0"
},
"engines": {
"node": ">=8.0"
}
},
"node_modules/toidentifier": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
@@ -6021,6 +6233,25 @@
}
}
},
"node_modules/vite-plugin-static-copy": {
"version": "3.1.4",
"resolved": "https://registry.npmjs.org/vite-plugin-static-copy/-/vite-plugin-static-copy-3.1.4.tgz",
"integrity": "sha512-iCmr4GSw4eSnaB+G8zc2f4dxSuDjbkjwpuBLLGvQYR9IW7rnDzftnUjOH5p4RYR+d4GsiBqXRvzuFhs5bnzVyw==",
"dev": true,
"license": "MIT",
"dependencies": {
"chokidar": "^3.6.0",
"p-map": "^7.0.3",
"picocolors": "^1.1.1",
"tinyglobby": "^0.2.15"
},
"engines": {
"node": "^18.0.0 || >=20.0.0"
},
"peerDependencies": {
"vite": "^5.0.0 || ^6.0.0 || ^7.0.0"
}
},
"node_modules/web-streams-polyfill": {
"version": "4.0.0-beta.3",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
@@ -6257,7 +6488,8 @@
"react-dom": "^18.2.0",
"tsx": "^4.6.0",
"typescript": "^5.2.2",
"vite": "^7.1.10"
"vite": "^7.1.10",
"vite-plugin-static-copy": "^3.1.4"
}
},
"packages/voice-assistant/node_modules/@types/node": {

3
packages/voice-assistant/package.json
View File

@@ -46,6 +46,7 @@
"react-dom": "^18.2.0",
"tsx": "^4.6.0",
"typescript": "^5.2.2",
"vite": "^7.1.10"
"vite": "^7.1.10",
"vite-plugin-static-copy": "^3.1.4"
}
}

2
packages/voice-assistant/src/server/agent/playwright-mcp.ts
View File

@@ -15,7 +15,7 @@ export async function getPlaywrightMCPClient() {
const transport = new StdioClientTransport({
command: "npx",
args: ["@playwright/mcp", "--browser", "firefox"],
args: ["@playwright/mcp", "--image-responses", "omit"],
});
playwrightMCPClient = await experimental_createMCPClient({

31
packages/voice-assistant/src/server/websocket-server.ts
View File

@@ -127,17 +127,23 @@ export class VoiceAssistantWebSocketServer {
// Handle user message through orchestrator
const payload = message.payload as { message: string };
const conversationId = this.conversationIds.get(ws);
const abortController = this.abortControllers.get(ws);
if (!conversationId) {
console.error("[WS] No conversation found for client");
break;
}
if (!abortController) {
console.error("[WS] No abort controller found for client");
break;
// Abort any ongoing request
const oldController = this.abortControllers.get(ws);
if (oldController) {
oldController.abort();
}
// Create new abort controller for this request
const newController = new AbortController();
this.abortControllers.set(ws, newController);
if (this.messageHandler) {
await this.messageHandler(conversationId, payload.message, abortController.signal);
await this.messageHandler(conversationId, payload.message, newController.signal);
} else {
console.warn("[WS] No message handler registered");
}
@@ -258,13 +264,16 @@ export class VoiceAssistantWebSocketServer {
// Clear buffer
this.audioBuffers.delete(clientId);
// Process audio through handler (STT)
const abortController = this.abortControllers.get(ws);
if (!abortController) {
console.error("[WS] No abort controller found for client");
return;
// Abort any ongoing request
const oldAbortController = this.abortControllers.get(ws);
if (oldAbortController) {
oldAbortController.abort();
}
// Create new abort controller for this request
const newAbortController = new AbortController();
this.abortControllers.set(ws, newAbortController);
if (this.audioHandler) {
this.broadcastActivityLog({
id: Date.now().toString(),
@@ -273,7 +282,7 @@ export class VoiceAssistantWebSocketServer {
content: "Transcribing audio...",
});
const transcript = await this.audioHandler(conversationId, completeAudio, format, abortController.signal);
const transcript = await this.audioHandler(conversationId, completeAudio, format, newAbortController.signal);
// Send transcription result back to client
this.sendToClient(ws, {

19
packages/voice-assistant/src/ui/lib/audio-realtime.ts
View File

@@ -65,9 +65,19 @@ export function createRealtimeVAD(config: RealtimeVADConfig): RealtimeVAD {
},
positiveSpeechThreshold: config.positiveSpeechThreshold ?? 0.5,
negativeSpeechThreshold: config.negativeSpeechThreshold ?? 0.35,
redemptionMs: config.redemptionFrames ? config.redemptionFrames * 16 : 128,
preSpeechPadMs: config.preSpeechPadFrames ? config.preSpeechPadFrames * 16 : 16,
minSpeechMs: config.minSpeechFrames ? config.minSpeechFrames * 16 : 48,
// Increased redemptionMs to wait longer before deciding speech has ended (allows for natural pauses)
redemptionMs: config.redemptionFrames
? config.redemptionFrames * 16
: 2000,
// Padding before speech starts
preSpeechPadMs: config.preSpeechPadFrames
? config.preSpeechPadFrames * 16
: 300,
// Minimum speech duration to avoid capturing very short sounds
minSpeechMs: config.minSpeechFrames ? config.minSpeechFrames * 16 : 500,
ortConfig: (ort) => {
ort.env.wasm.wasmPaths = "/";
},
});
isActive = true;
@@ -75,8 +85,7 @@ export function createRealtimeVAD(config: RealtimeVADConfig): RealtimeVAD {
console.log("[RealtimeVAD] VAD created and started successfully");
} catch (error) {
console.error("[RealtimeVAD] Failed to start VAD:", error);
const err =
error instanceof Error ? error : new Error(String(error));
const err = error instanceof Error ? error : new Error(String(error));
config.onError?.(err);
throw err;
}

37
packages/voice-assistant/vite.config.ts
View File

@@ -1,10 +1,45 @@
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
import path from "path";
import { viteStaticCopy } from "vite-plugin-static-copy";
// https://vitejs.dev/config/
export default defineConfig({
plugins: [react()],
plugins: [
react(),
viteStaticCopy({
targets: [
{
src: path.resolve(__dirname, '../../node_modules/@ricky0123/vad-web/dist/vad.worklet.bundle.min.js'),
dest: './'
},
{
src: path.resolve(__dirname, '../../node_modules/@ricky0123/vad-web/dist/silero_vad_v5.onnx'),
dest: './'
},
{
src: path.resolve(__dirname, '../../node_modules/@ricky0123/vad-web/dist/silero_vad_legacy.onnx'),
dest: './'
},
{
src: path.resolve(__dirname, '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.mjs'),
dest: './'
},
{
src: path.resolve(__dirname, '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.wasm'),
dest: './'
},
{
src: path.resolve(__dirname, '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.mjs'),
dest: './'
},
{
src: path.resolve(__dirname, '../../node_modules/onnxruntime-web/dist/ort-wasm-simd-threaded.jsep.wasm'),
dest: './'
}
]
})
],
root: "src/ui",
build: {
outDir: path.resolve(__dirname, "dist/ui"),