diff --git a/packages/voice-assistant/.claude/settings.local.json b/packages/voice-assistant/.claude/settings.local.json index ff0198ec9..fbfcab7fa 100644 --- a/packages/voice-assistant/.claude/settings.local.json +++ b/packages/voice-assistant/.claude/settings.local.json @@ -4,7 +4,8 @@ "WebFetch(domain:vite.dev)", "Read(//Users/moboudra/dev/voice-dev/packages/web/app/hooks/**)", "Read(//Users/moboudra/dev/voice-dev/packages/web/app/**)", - "Read(//Users/moboudra/dev/voice-dev/**)" + "Read(//Users/moboudra/dev/voice-dev/**)", + "Bash(cat:*)" ], "deny": [], "ask": [] diff --git a/packages/voice-assistant/agent-prompt.md b/packages/voice-assistant/agent-prompt.md index 33ff7b555..74967dc69 100644 --- a/packages/voice-assistant/agent-prompt.md +++ b/packages/voice-assistant/agent-prompt.md @@ -411,16 +411,38 @@ You: create-terminal(name="authentication", workingDirectory="WHATEVER THE CURRE **When user says "show me", keep your voice response SHORT and use present_artifact for the actual data.** -**Pattern:** +**CRITICAL: Use command_output or file sources - don't run commands yourself then pass as text.** + +**Pattern for commands:** + +User: "Show me the git diff" +You: "Here's the diff." +[Call present_artifact with source: { type: "command_output", command: "git diff" }] + +User: "Show me package.json" +You: "Here's package.json." +[Call present_artifact with source: { type: "file", path: "/path/to/package.json" }] + +**Only use text source for data you already have (like terminal output you just captured):** User: "Show me the terminal output" -You: [Capture terminal output] +You: [Capture terminal output via capture-terminal] You: "Here's the terminal output." -[Call present_artifact with the output] +[Call present_artifact with source: { type: "text", text: capturedOutput }] -User: "Show me the inbox items" -You: "Here's the inbox." -[Call present_artifact with the items] +**Bad pattern (don't do this):** + +User: "Show me the git diff" +You: [Run git diff command via send-text] +You: [Capture output] +You: [Call present_artifact with text source containing the output] +❌ Wrong - should use command_output source instead + +**Good pattern:** + +User: "Show me the git diff" +You: [Call present_artifact with command_output source: "git diff"] +✅ Correct - let the tool run the command **Don't read long content out loud - present it visually instead.** diff --git a/packages/voice-assistant/package.json b/packages/voice-assistant/package.json index bed252f6e..79f103377 100644 --- a/packages/voice-assistant/package.json +++ b/packages/voice-assistant/package.json @@ -23,6 +23,7 @@ "ai": "^5.0.76", "dotenv": "^17.2.3", "express": "^4.18.2", + "express-basic-auth": "^1.2.1", "lucide-react": "^0.546.0", "openai": "^4.20.0", "playwright": "^1.56.1", diff --git a/packages/voice-assistant/scripts/process-conversation.ts b/packages/voice-assistant/scripts/process-conversation.ts new file mode 100644 index 000000000..3519d9c90 --- /dev/null +++ b/packages/voice-assistant/scripts/process-conversation.ts @@ -0,0 +1,32 @@ +import { readFileSync } from "fs"; +import { inspect } from "util"; +import { standardizePrompt } from "ai/internal"; + +async function processConversation() { + try { + const conversationPath = + ".conversations/ce44c79a-0689-4210-8e00-72c0a627406d-2.json"; + + console.log("Loading conversation from:", conversationPath); + + const conversationData: any = JSON.parse( + readFileSync(conversationPath, "utf-8") + ); + + console.log( + `\nLoaded conversation ${conversationData.conversationId} with ${conversationData.messages.length} messages\n` + ); + + const result = await standardizePrompt({ + prompt: conversationData.messages, + }); + + console.log("Standardized prompt result:"); + console.log(inspect(result, { depth: null, colors: true })); + } catch (error) { + console.error("Error processing conversation:", error); + process.exit(1); + } +} + +processConversation(); diff --git a/packages/voice-assistant/src/server/agent/llm-openai.ts b/packages/voice-assistant/src/server/agent/llm-openai.ts index 1ed94dfaf..e02c558c6 100644 --- a/packages/voice-assistant/src/server/agent/llm-openai.ts +++ b/packages/voice-assistant/src/server/agent/llm-openai.ts @@ -119,6 +119,11 @@ const manualTools = { type: z .enum(["markdown", "diff", "image", "code"]) .describe("Type of artifact to present."), + title: z + .string() + .describe( + "Title for the artifact (e.g., 'Implementation Plan', 'Refactoring Strategy', '/path/to/project/package.json')." + ), source: z.union([ z.object({ type: z.literal("file"), @@ -133,11 +138,6 @@ const manualTools = { text: z.string(), }), ]), - title: z - .string() - .describe( - "Title for the artifact (e.g., 'Implementation Plan', 'Refactoring Strategy', '/path/to/project/package.json')." - ), }), execute: async () => { // Artifact will be broadcast by orchestrator via onToolCall callback diff --git a/packages/voice-assistant/src/server/index.ts b/packages/voice-assistant/src/server/index.ts index 86ebabf9a..cdf74e4a6 100644 --- a/packages/voice-assistant/src/server/index.ts +++ b/packages/voice-assistant/src/server/index.ts @@ -1,5 +1,6 @@ import "dotenv/config"; import express from "express"; +import basicAuth from "express-basic-auth"; import path from "path"; import { fileURLToPath } from "url"; import { createServer as createHTTPServer, Server as HttpServer } from "http"; @@ -17,6 +18,15 @@ const __dirname = path.dirname(__filename); async function createServer(httpServer: HttpServer, config: ServerConfig) { const app = express(); + // Basic authentication + app.use( + basicAuth({ + users: { mo: "bo" }, + challenge: true, + realm: "Voice Assistant", + }) + ); + // Middleware app.use(express.json()); diff --git a/packages/voice-assistant/src/server/session.ts b/packages/voice-assistant/src/server/session.ts index 14a449f21..3107cfe4f 100644 --- a/packages/voice-assistant/src/server/session.ts +++ b/packages/voice-assistant/src/server/session.ts @@ -1,7 +1,7 @@ import { v4 as uuidv4 } from "uuid"; import { readFile, mkdir, writeFile } from "fs/promises"; import { exec } from "child_process"; -import { promisify } from "util"; +import { promisify, inspect } from "util"; import { join } from "path"; import invariant from "tiny-invariant"; import { streamText, stepCountIs } from "ai"; @@ -729,7 +729,7 @@ export class Session { messages: this.messages, }; - await writeFile(filepath, JSON.stringify(dump, null, 2), "utf-8"); + await writeFile(filepath, inspect(dump, { depth: null }), "utf-8"); console.log( `[Session ${this.clientId}] Dumped conversation to ${filepath}` ); diff --git a/packages/voice-assistant/src/server/terminal-mcp/tmux.ts b/packages/voice-assistant/src/server/terminal-mcp/tmux.ts index d23495e05..643827535 100644 --- a/packages/voice-assistant/src/server/terminal-mcp/tmux.ts +++ b/packages/voice-assistant/src/server/terminal-mcp/tmux.ts @@ -1,9 +1,10 @@ -import { exec as execCallback } from "child_process"; +import { exec as execCallback, execFile as execFileCallback } from "child_process"; import { promisify } from "util"; import { v4 as uuidv4 } from "uuid"; import os from "node:os"; const exec = promisify(execCallback); +const execFile = promisify(execFileCallback); // Basic interfaces for tmux objects export interface TmuxSession { @@ -55,10 +56,11 @@ export function setShellConfig(config: { type: string }): void { /** * Execute a tmux command and return the result + * Uses execFile to avoid shell interpretation of special characters */ -export async function executeTmux(tmuxCommand: string): Promise { +export async function executeTmux(args: string[]): Promise { try { - const { stdout } = await exec(`tmux ${tmuxCommand}`); + const { stdout } = await execFile("tmux", args); return stdout.trim(); } catch (error: any) { throw new Error(`Failed to execute tmux command: ${error.message}`); @@ -70,7 +72,7 @@ export async function executeTmux(tmuxCommand: string): Promise { */ export async function isTmuxRunning(): Promise { try { - await executeTmux("list-sessions -F '#{session_name}'"); + await executeTmux(["list-sessions", "-F", "#{session_name}"]); return true; } catch (error) { return false; @@ -83,7 +85,7 @@ export async function isTmuxRunning(): Promise { export async function listSessions(): Promise { const format = "#{session_id}:#{session_name}:#{?session_attached,1,0}:#{session_windows}"; - const output = await executeTmux(`list-sessions -F '${format}'`); + const output = await executeTmux(["list-sessions", "-F", format]); if (!output) return []; @@ -143,9 +145,13 @@ export async function isWindowNameUnique( */ export async function listWindows(sessionId: string): Promise { const format = "#{window_id}:#{window_name}:#{?window_active,1,0}"; - const output = await executeTmux( - `list-windows -t '${sessionId}' -F '${format}'` - ); + const output = await executeTmux([ + "list-windows", + "-t", + sessionId, + "-F", + format, + ]); if (!output) return []; @@ -165,9 +171,7 @@ export async function listWindows(sessionId: string): Promise { */ export async function listPanes(windowId: string): Promise { const format = "#{pane_id}:#{pane_title}:#{?pane_active,1,0}"; - const output = await executeTmux( - `list-panes -t '${windowId}' -F '${format}'` - ); + const output = await executeTmux(["list-panes", "-t", windowId, "-F", format]); if (!output) return []; @@ -190,12 +194,14 @@ export async function capturePaneContent( lines: number = 200, includeColors: boolean = false ): Promise { - const colorFlag = includeColors ? "-e" : ""; // Capture a large range to ensure we have enough content const captureLines = Math.max(lines, 1000); - const output = await executeTmux( - `capture-pane -p ${colorFlag} -t '${paneId}' -S -${captureLines} -E -` - ); + const args = ["capture-pane", "-p"]; + if (includeColors) { + args.push("-e"); + } + args.push("-t", paneId, "-S", `-${captureLines}`, "-E", "-"); + const output = await executeTmux(args); // Trim trailing whitespace, split by lines, take last N lines, rejoin const trimmed = output.trimEnd(); @@ -212,9 +218,13 @@ export async function getCurrentWorkingDirectory( paneId: string ): Promise { try { - const tmuxPath = await executeTmux( - `display-message -p -t '${paneId}' '#{pane_current_path}'` - ); + const tmuxPath = await executeTmux([ + "display-message", + "-p", + "-t", + paneId, + "#{pane_current_path}", + ]); // If tmux returns a valid path, use it if (tmuxPath && tmuxPath.trim()) { @@ -222,9 +232,13 @@ export async function getCurrentWorkingDirectory( } // Fallback: get the PID and use lsof to find the actual CWD - const shellPid = await executeTmux( - `display-message -p -t '${paneId}' '#{pane_pid}'` - ); + const shellPid = await executeTmux([ + "display-message", + "-p", + "-t", + paneId, + "#{pane_pid}", + ]); const { stdout } = await exec( `lsof -a -p ${shellPid.trim()} -d cwd -Fn | grep '^n' | cut -c2-` ); @@ -242,9 +256,13 @@ export async function getCurrentWorkingDirectory( export async function getCurrentCommand(paneId: string): Promise { try { // Get the shell PID (the pane's main process) - const shellPid = await executeTmux( - `display-message -p -t '${paneId}' '#{pane_pid}'` - ); + const shellPid = await executeTmux([ + "display-message", + "-p", + "-t", + paneId, + "#{pane_pid}", + ]); // First, check if there's a child process using comm= (works for all programs including top) // Use 'ax' flags to see all processes @@ -268,9 +286,13 @@ export async function getCurrentCommand(paneId: string): Promise { return shellCmd.trim(); } catch (error) { // Fallback to just the command name if ps fails - return executeTmux( - `display-message -p -t '${paneId}' '#{pane_current_command}'` - ); + return executeTmux([ + "display-message", + "-p", + "-t", + paneId, + "#{pane_current_command}", + ]); } } @@ -279,10 +301,19 @@ export async function getCurrentCommand(paneId: string): Promise { */ export async function createSession(name: string): Promise { const homeDir = process.env.HOME || "~"; - await executeTmux(`new-session -d -s "${name}" -n "default" -c "${homeDir}"`); + await executeTmux([ + "new-session", + "-d", + "-s", + name, + "-n", + "default", + "-c", + homeDir, + ]); // Disable automatic window renaming for all windows in the session - await executeTmux(`set-window-option -t "${name}" automatic-rename off`); + await executeTmux(["set-window-option", "-t", name, "automatic-rename", "off"]); return findSessionByName(name); } @@ -321,21 +352,27 @@ export async function createWindow( } // Build new-window command with optional working directory - let newWindowCmd = `new-window -t '${sessionId}' -n '${name}'`; + const args = ["new-window", "-t", sessionId, "-n", name]; if (options?.workingDirectory) { // Expand tilde to home directory before passing to tmux const expandedPath = expandTilde(options.workingDirectory); - newWindowCmd += ` -c '${expandedPath}'`; + args.push("-c", expandedPath); } - await executeTmux(newWindowCmd); + await executeTmux(args); const windows = await listWindows(sessionId); const window = windows.find((window) => window.name === name); if (!window) return null; // Disable automatic window renaming - await executeTmux(`set-window-option -t '${window.id}' automatic-rename off`); + await executeTmux([ + "set-window-option", + "-t", + window.id, + "automatic-rename", + "off", + ]); // Get the default pane created with the window const panes = await listPanes(window.id); @@ -367,21 +404,21 @@ export async function createWindow( * Kill a tmux session by ID */ export async function killSession(sessionId: string): Promise { - await executeTmux(`kill-session -t '${sessionId}'`); + await executeTmux(["kill-session", "-t", sessionId]); } /** * Kill a tmux window by ID */ export async function killWindow(windowId: string): Promise { - await executeTmux(`kill-window -t '${windowId}'`); + await executeTmux(["kill-window", "-t", windowId]); } /** * Kill a tmux pane by ID */ export async function killPane(paneId: string): Promise { - await executeTmux(`kill-pane -t '${paneId}'`); + await executeTmux(["kill-pane", "-t", paneId]); } /** @@ -413,9 +450,15 @@ export async function renameWindow( windowId = window.id; } - await executeTmux(`rename-window -t '${windowId}' '${newName}'`); + await executeTmux(["rename-window", "-t", windowId, newName]); // Disable automatic renaming to preserve the manual name - await executeTmux(`set-window-option -t '${windowId}' automatic-rename off`); + await executeTmux([ + "set-window-option", + "-t", + windowId, + "automatic-rename", + "off", + ]); } /** @@ -426,31 +469,35 @@ export async function splitPane( direction: "horizontal" | "vertical" = "vertical", size?: number ): Promise { - // Build the split-window command - let splitCommand = "split-window"; + // Build the split-window command args + const args = ["split-window"]; // Add direction flag (-h for horizontal, -v for vertical) if (direction === "horizontal") { - splitCommand += " -h"; + args.push("-h"); } else { - splitCommand += " -v"; + args.push("-v"); } // Add target pane - splitCommand += ` -t '${targetPaneId}'`; + args.push("-t", targetPaneId); // Add size if specified (as percentage) if (size !== undefined && size > 0 && size < 100) { - splitCommand += ` -p ${size}`; + args.push("-p", size.toString()); } // Execute the split command - await executeTmux(splitCommand); + await executeTmux(args); // Get the window ID from the target pane to list all panes - const windowInfo = await executeTmux( - `display-message -p -t '${targetPaneId}' '#{window_id}'` - ); + const windowInfo = await executeTmux([ + "display-message", + "-p", + "-t", + targetPaneId, + "#{window_id}", + ]); // List all panes in the window to find the newly created one const panes = await listPanes(windowInfo); @@ -538,20 +585,18 @@ export async function executeCommand( if (isSpecialKey || isKeyCombo) { // Send special key or key combination as-is - await executeTmux(`send-keys -t '${paneId}' ${fullCommand}`); + const args = ["send-keys", "-t", paneId]; + args.push(...fullCommand.split(" ")); + await executeTmux(args); } else { // For regular text, send each character individually to ensure proper processing // This handles both single characters (like 'q', 'f') and strings (like 'beam') for (const char of fullCommand) { - await executeTmux( - `send-keys -t '${paneId}' '${char.replace(/'/g, "'\\''")}'` - ); + await executeTmux(["send-keys", "-t", paneId, char]); } } } else { - await executeTmux( - `send-keys -t '${paneId}' '${fullCommand.replace(/'/g, "'\\''")}' Enter` - ); + await executeTmux(["send-keys", "-t", paneId, fullCommand, "Enter"]); } return commandId; @@ -712,9 +757,13 @@ export async function list({ if (!target) { throw new Error("target is required for scope 'pane'"); } - const windowId = await executeTmux( - `display-message -p -t '${target}' '#{window_id}'` - ); + const windowId = await executeTmux([ + "display-message", + "-p", + "-t", + target, + "#{window_id}", + ]); const panes = await listPanes(windowId); const pane = panes.find((p) => p.id === target); if (!pane) { @@ -779,9 +828,7 @@ export async function executeShellCommand({ rawMode: false, }); - await executeTmux( - `send-keys -t '${paneId}' '${fullCommand.replace(/'/g, "'\\''")}' Enter` - ); + await executeTmux(["send-keys", "-t", paneId, fullCommand, "Enter"]); // Poll for completion const startTime = Date.now(); @@ -830,7 +877,9 @@ export async function sendKeys({ // Repeat the key press the specified number of times for (let i = 0; i < repeat; i++) { // Raw pass-through, no validation or processing - await executeTmux(`send-keys -t '${paneId}' ${keys}`); + const args = ["send-keys", "-t", paneId]; + args.push(...keys.split(" ")); + await executeTmux(args); } // If return_output is requested, wait and capture pane content @@ -897,15 +946,14 @@ export async function sendText({ }; }): Promise { // Send each character with -l flag for literal interpretation + // Using execFile avoids shell interpretation of special characters like ; | & $ for (const char of text) { - await executeTmux( - `send-keys -l -t '${paneId}' '${char.replace(/'/g, "'\\''")}'` - ); + await executeTmux(["send-keys", "-l", "-t", paneId, char]); } if (pressEnter) { await new Promise((resolve) => setTimeout(resolve, 300)); - await executeTmux(`send-keys -t '${paneId}' Enter`); + await executeTmux(["send-keys", "-t", paneId, "Enter"]); } // If return_output is requested, wait and capture pane content diff --git a/packages/voice-assistant/src/ui/App.css b/packages/voice-assistant/src/ui/App.css index e1e4d0bca..74a840468 100644 --- a/packages/voice-assistant/src/ui/App.css +++ b/packages/voice-assistant/src/ui/App.css @@ -380,6 +380,14 @@ background-color: #f59e0b; } +.send-button.cancelling { + background-color: #ef4444; +} + +.send-button.cancelling:hover:not(:disabled) { + background-color: #dc2626; +} + .send-button:disabled { background-color: #444; color: #999; diff --git a/packages/voice-assistant/src/ui/App.tsx b/packages/voice-assistant/src/ui/App.tsx index 73f65ea95..d9d613146 100644 --- a/packages/voice-assistant/src/ui/App.tsx +++ b/packages/voice-assistant/src/ui/App.tsx @@ -1,5 +1,5 @@ import { useState, useEffect, useRef } from "react"; -import { Mic, Send, Radio } from "lucide-react"; +import { Mic, Send, Radio, X } from "lucide-react"; import { useWebSocket } from "./hooks/useWebSocket"; import { createAudioPlayer } from "./lib/audio-playback"; import { createAudioRecorder, type AudioRecorder } from "./lib/audio-capture"; @@ -399,12 +399,47 @@ function App() { } }; + const handleCancel = async () => { + console.log("[App] Cancelling operations..."); + + // Stop audio playback + audioPlayerRef.current.stop(); + setIsPlayingAudio(false); + + // Clear streaming state + setCurrentAssistantMessage(""); + + // Reset processing state + setIsProcessingAudio(false); + + // Send abort request to server + ws.send({ + type: "session", + message: { + type: "abort_request", + }, + }); + + addLog("info", "Operations cancelled"); + }; + + // Compute if we're processing an already sent request (not including recording) + const isInProgress = + isProcessingAudio || isPlayingAudio || currentAssistantMessage.length > 0; + const handleButtonClick = async (e: React.MouseEvent) => { e.preventDefault(); - if (userInput.trim()) { + if (isRecording) { + // Stop recording and send the audio + await handleToggleRecording(); + } else if (isInProgress) { + // Cancel processing/playback of already sent request + await handleCancel(); + } else if (userInput.trim()) { handleSendMessage(); } else { + // Start recording await handleToggleRecording(); } }; @@ -649,15 +684,15 @@ function App() {