diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index 5f86207c2..af387fbea 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -31,6 +31,8 @@ import { encodeImages } from "@/utils/encode-images"; import { useKeyboardNavStore } from "@/stores/keyboard-nav-store"; import { focusWithRetries } from "@/utils/web-focus"; import { useVoiceOptional } from "@/contexts/voice-context"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { Shortcut } from "@/components/ui/shortcut"; type QueuedMessage = { id: string; @@ -512,10 +514,23 @@ export function AgentInputArea({ }, [setUserInput] ); + const hasSendableContent = userInput.trim().length > 0 || selectedImages.length > 0; // Handle keyboard navigation for command autocomplete const handleCommandKeyPress = useCallback( (event: { key: string; preventDefault: () => void }) => { + if ( + event.key === "Escape" && + isAgentRunning && + !hasSendableContent && + !isCancellingAgent && + isConnected + ) { + event.preventDefault(); + handleCancelAgent(); + return true; + } + if (!showCommandAutocomplete || filteredCommands.length === 0) { return false; } @@ -558,54 +573,72 @@ export function AgentInputArea({ filteredCommands, commandSelectedIndex, handleCommandSelect, + hasSendableContent, + isAgentRunning, + isCancellingAgent, + isConnected, setUserInput, ] ); - const hasSendableContent = userInput.trim().length > 0 || selectedImages.length > 0; - const cancelButton = isAgentRunning && !hasSendableContent ? ( - - {isCancellingAgent ? ( - - ) : ( - - )} - + + + {isCancellingAgent ? ( + + ) : ( + + )} + + + + Interrupt + + + + ) : null; const rightContent = ( {!isVoiceModeForAgent ? ( - - {voice?.isVoiceSwitching ? ( - - ) : ( - - )} - + + + {voice?.isVoiceSwitching ? ( + + ) : ( + + )} + + + + Voice mode + + + + ) : null} {cancelButton} @@ -647,7 +680,7 @@ export function AgentInputArea({ onPress={() => handleSendQueuedNow(item.id)} style={[styles.queueActionButton, styles.queueSendButton]} > - + @@ -752,6 +785,19 @@ const styles = StyleSheet.create(((theme: Theme) => ({ backgroundColor: theme.colors.palette.green[600], borderColor: theme.colors.palette.green[800], }, + tooltipRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + tooltipText: { + fontSize: theme.fontSize.sm, + color: theme.colors.popoverForeground, + }, + tooltipShortcut: { + backgroundColor: theme.colors.surface3, + borderColor: theme.colors.borderAccent, + }, buttonDisabled: { opacity: 0.5, }, @@ -765,7 +811,7 @@ const styles = StyleSheet.create(((theme: Theme) => ({ justifyContent: "space-between", paddingHorizontal: theme.spacing[3], paddingVertical: theme.spacing[2], - backgroundColor: theme.colors.surface2, + backgroundColor: theme.colors.surface1, borderRadius: theme.borderRadius.lg, borderWidth: theme.borderWidth[1], borderColor: theme.colors.border, @@ -790,7 +836,7 @@ const styles = StyleSheet.create(((theme: Theme) => ({ backgroundColor: theme.colors.surface2, }, queueSendButton: { - backgroundColor: theme.colors.palette.blue[600], + backgroundColor: theme.colors.accent, }, sendErrorText: { color: theme.colors.palette.red[500], diff --git a/packages/app/src/components/agent-status-bar.tsx b/packages/app/src/components/agent-status-bar.tsx index 6eba51dd3..066f4f10b 100644 --- a/packages/app/src/components/agent-status-bar.tsx +++ b/packages/app/src/components/agent-status-bar.tsx @@ -377,14 +377,14 @@ const styles = StyleSheet.create((theme) => ({ modeBadge: { flexDirection: "row", alignItems: "center", - backgroundColor: theme.colors.surface2, + backgroundColor: "transparent", gap: theme.spacing[1], paddingHorizontal: theme.spacing[2], paddingVertical: theme.spacing[1], borderRadius: theme.borderRadius["2xl"], }, modeBadgeHovered: { - backgroundColor: theme.colors.surface1, + backgroundColor: theme.colors.surface2, }, modeBadgePressed: { backgroundColor: theme.colors.surface0, diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx index 6fac1fc2b..ea7463850 100644 --- a/packages/app/src/components/message-input.tsx +++ b/packages/app/src/components/message-input.tsx @@ -1,5 +1,6 @@ import { View, + Text, TextInput, Pressable, ActivityIndicator, @@ -31,6 +32,8 @@ import { RealtimeVoiceOverlay } from "./realtime-voice-overlay"; import type { DaemonClient } from "@server/client/daemon-client"; import { usePanelStore } from "@/stores/panel-store"; import { useVoiceOptional } from "@/contexts/voice-context"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { Shortcut } from "@/components/ui/shortcut"; export interface ImageAttachment { uri: string; @@ -217,6 +220,7 @@ export const MessageInput = forwardRef( const socketConnected = client?.isConnected ?? false; return socketConnected; }, [client]); + const isConnected = client?.isConnected ?? false; const { isRecording: isDictating, @@ -263,9 +267,39 @@ export const MessageInput = forwardRef( sendAfterTranscriptRef.current = false; }, [dictationStatus, isDictating, isDictationProcessing]); - // Cmd+D to start/submit dictation, Escape to cancel + // Cmd+D to start/submit dictation, Cmd+Shift+D toggles realtime voice, Escape cancels dictation useEffect(() => { if (!IS_WEB) return; + const toggleRealtimeVoice = () => { + if (!voice || !voiceServerId || !voiceAgentId || !isConnected || disabled) { + return; + } + if (voice.isVoiceSwitching) { + return; + } + if (voice.isVoiceModeForAgent(voiceServerId, voiceAgentId)) { + const tasks: Promise[] = []; + if (isAgentRunning && client) { + tasks.push(client.cancelAgent(voiceAgentId)); + } + tasks.push(voice.stopVoice()); + void Promise.allSettled(tasks).then((results) => { + results.forEach((result) => { + if (result.status === "rejected") { + console.error( + "[MessageInput] Failed to stop realtime voice", + result.reason + ); + } + }); + }); + return; + } + void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => { + console.error("[MessageInput] Failed to start realtime voice", error); + }); + }; + const resolveNativeInput = (): unknown => { const current = textInputRef.current as any; if (!current) return null; @@ -280,12 +314,32 @@ export const MessageInput = forwardRef( if (!isScreenFocused && !isInputFocusedRef.current && !isFromInput) { return; } + const isMod = event.metaKey || event.ctrlKey; + const isKeyD = event.code === "KeyD" || event.key.toLowerCase() === "d"; + + if (isMod && event.shiftKey && isKeyD && !event.repeat) { + event.preventDefault(); + toggleRealtimeVoice(); + return; + } + + if ( + isRealtimeVoiceForCurrentAgent && + !event.metaKey && + !event.ctrlKey && + !event.altKey && + !event.shiftKey && + (event.code === "Space" || event.key === " ") && + !event.repeat + ) { + event.preventDefault(); + voice?.toggleMute(); + return; + } + const dictating = isDictatingRef.current; // Cmd+D: start dictation or submit if already dictating - if ( - (event.metaKey || event.ctrlKey) && - (event.code === "KeyD" || event.key.toLowerCase() === "d") - ) { + if (isMod && isKeyD) { event.preventDefault(); if (dictating) { sendAfterTranscriptRef.current = true; @@ -303,7 +357,20 @@ export const MessageInput = forwardRef( } window.addEventListener("keydown", handleKeyDown); return () => window.removeEventListener("keydown", handleKeyDown); - }, [cancelDictation, confirmDictation, isScreenFocused, startDictation]); + }, [ + cancelDictation, + client, + confirmDictation, + disabled, + isAgentRunning, + isConnected, + isRealtimeVoiceForCurrentAgent, + isScreenFocused, + startDictation, + voiceAgentId, + voiceServerId, + voice, + ]); // Animate overlay useEffect(() => { @@ -381,6 +448,29 @@ export const MessageInput = forwardRef( }); }, [client, isAgentRunning, isRealtimeVoiceForCurrentAgent, voice, voiceAgentId]); + const handleToggleRealtimeVoiceShortcut = useCallback(() => { + if (!voice || !voiceServerId || !voiceAgentId || !isConnected || disabled) { + return; + } + if (voice.isVoiceSwitching) { + return; + } + if (voice.isVoiceModeForAgent(voiceServerId, voiceAgentId)) { + void handleStopRealtimeVoice(); + return; + } + void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => { + console.error("[MessageInput] Failed to start realtime voice", error); + }); + }, [ + disabled, + handleStopRealtimeVoice, + isConnected, + voice, + voiceAgentId, + voiceServerId, + ]); + const handleSendMessage = useCallback(() => { const trimmed = value.trim(); if (!trimmed && images.length === 0) return; @@ -514,6 +604,13 @@ export const MessageInput = forwardRef( return; } + // Cmd+Shift+D or Ctrl+Shift+D: toggle realtime voice mode + if ((metaKey || ctrlKey) && shiftKey && key === "d") { + event.preventDefault(); + handleToggleRealtimeVoiceShortcut(); + return; + } + // Cmd+D or Ctrl+D: start dictation or submit if already dictating if ((metaKey || ctrlKey) && key === "d") { event.preventDefault(); @@ -533,6 +630,18 @@ export const MessageInput = forwardRef( return; } + if ( + isRealtimeVoiceForCurrentAgent && + !metaKey && + !ctrlKey && + !shiftKey && + event.nativeEvent.key === " " + ) { + event.preventDefault(); + voice?.toggleMute(); + return; + } + if (event.nativeEvent.key !== "Enter") return; // Shift+Enter: add newline (default behavior, don't intercept) @@ -555,7 +664,6 @@ export const MessageInput = forwardRef( const hasImages = images.length > 0; const hasSendableContent = value.trim().length > 0 || hasImages; const shouldShowSendButton = hasSendableContent || isSubmitLoading; - const isConnected = client?.isConnected ?? false; return ( @@ -636,90 +744,130 @@ export const MessageInput = forwardRef( {/* Left: attachment button + leftContent slot */} {onPickImages && ( - - - + + + + + + Attach images + + )} {leftContent} {/* Right: voice button, contextual button (realtime/send/cancel) */} - - {isDictating ? ( - - ) : isRealtimeVoiceForCurrentAgent && voice?.isMuted ? ( - - ) : ( - - )} - + + + {isDictating ? ( + + ) : isRealtimeVoiceForCurrentAgent && voice?.isMuted ? ( + + ) : ( + + )} + + + + + {isRealtimeVoiceForCurrentAgent + ? voice?.isMuted + ? "Unmute voice" + : "Mute voice" + : "Dictation"} + + + + + {rightContent} {shouldShowSendButton && isAgentRunning && onQueue && ( - - - + + + + + + + Queue + + + + )} {shouldShowSendButton && ( - + - {isSubmitLoading ? ( - - ) : ( - - )} - + disabled + } + accessibilityLabel={isAgentRunning ? "Send and interrupt" : "Send message"} + accessibilityRole="button" + style={[ + styles.sendButton, + (!isConnected || + isSubmitDisabled || + isSubmitLoading || + disabled) && + styles.buttonDisabled, + ]} + > + {isSubmitLoading ? ( + + ) : ( + + )} + + + + Send + + + + )} @@ -775,7 +923,7 @@ const styles = StyleSheet.create(((theme: any) => ({ inputWrapper: { flexDirection: "column", gap: theme.spacing[3], - backgroundColor: theme.colors.surface2, + backgroundColor: theme.colors.surface1, borderWidth: theme.borderWidth[1], borderColor: theme.colors.borderAccent, borderRadius: theme.borderRadius["2xl"], @@ -898,6 +1046,19 @@ const styles = StyleSheet.create(((theme: any) => ({ alignItems: "center", justifyContent: "center", }, + tooltipRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + tooltipText: { + fontSize: theme.fontSize.sm, + color: theme.colors.popoverForeground, + }, + tooltipShortcut: { + backgroundColor: theme.colors.surface3, + borderColor: theme.colors.borderAccent, + }, buttonDisabled: { opacity: 0.5, }, diff --git a/packages/app/src/components/ui/tooltip.tsx b/packages/app/src/components/ui/tooltip.tsx index f4aa1c274..e73d680a5 100644 --- a/packages/app/src/components/ui/tooltip.tsx +++ b/packages/app/src/components/ui/tooltip.tsx @@ -463,9 +463,9 @@ const styles = StyleSheet.create((theme) => ({ content: { paddingVertical: theme.spacing[1], paddingHorizontal: theme.spacing[2], - borderRadius: theme.borderRadius.lg, + borderRadius: theme.borderRadius.xl, backgroundColor: theme.colors.popover, - borderWidth: theme.borderWidth[1], + borderWidth: theme.borderWidth[2], borderColor: theme.colors.border, shadowColor: "#000", shadowOpacity: 0.2, diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 246c1eb37..dfebb2e2c 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -68,6 +68,11 @@ import type { } from "./agent/agent-sdk-types.js"; import { AgentStorage, type StoredAgentRecord } from "./agent/agent-storage.js"; import { isValidAgentProvider, AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; +import { + buildVoiceAgentMcpServerConfig, + buildVoiceModeSystemPrompt, + stripVoiceModeSystemPrompt, +} from "./voice-config.js"; import { isVoicePermissionAllowed } from "./voice-permission-policy.js"; import { listDirectoryEntries, @@ -130,18 +135,6 @@ const MAX_AGENTS_PER_PROJECT = 5; * Uses Claude Haiku for speed and cost efficiency. */ const AUTO_GEN_MODEL = "haiku"; -const VOICE_AGENT_SYSTEM_INSTRUCTION = [ - "You are the Paseo voice assistant.", - "The user cannot see your chat messages or tool calls.", - "Always use the speak tool for all user-facing communication.", - "Before calling any non-speak tool, first call speak with a short acknowledgement of what you heard and what you will do next.", - "For long-running work, use speak to provide progress updates before and during execution.", - "Treat the user input as transcribed speech and confirm intent in your first speak response to catch transcription errors early.", - "If the transcription seems incomplete, cut off, or ambiguous, ask a clarifying question via speak before taking action.", - "Use concise plain language suitable for speech output.", - "Never use bash, file-edit, or web tools directly.", - "Only use the paseo MCP tools.", -].join(" "); function deriveRemoteProjectKey(remoteUrl: string | null): string | null { if (!remoteUrl) { @@ -222,25 +215,6 @@ function escapeXmlText(value: string): string { .replace(/>/g, ">"); } -export function buildVoiceAgentMcpServerConfig(params: { - command: string; - baseArgs: string[]; - socketPath: string; - env?: Record; -}): { - type: "stdio"; - command: string; - args: string[]; - env?: Record; -} { - return { - type: "stdio", - command: params.command, - args: [...params.baseArgs, "--socket", params.socketPath], - ...(params.env ? { env: params.env } : {}), - }; -} - type ProcessingPhase = "idle" | "transcribing" | "llm"; type NormalizedGitOptions = { @@ -1541,13 +1515,6 @@ export class Session { return JSON.parse(JSON.stringify(servers)) as Record; } - private buildVoiceModeSystemPrompt(existing?: string): string { - const chunks = [existing?.trim(), VOICE_AGENT_SYSTEM_INSTRUCTION].filter( - (value): value is string => Boolean(value && value.length > 0) - ); - return chunks.join("\n\n"); - } - private buildVoiceModeMcpServers( existing: Record | undefined, socketPath: string @@ -1579,12 +1546,12 @@ export class Session { this.registerVoiceBridgeForAgent(agentId); const baseConfig: VoiceModeBaseConfig = { - systemPrompt: existing.config.systemPrompt, + systemPrompt: stripVoiceModeSystemPrompt(existing.config.systemPrompt), mcpServers: this.cloneMcpServers(existing.config.mcpServers), }; this.voiceModeBaseConfig = baseConfig; const refreshOverrides: Partial = { - systemPrompt: this.buildVoiceModeSystemPrompt(baseConfig.systemPrompt), + systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, true), mcpServers: this.buildVoiceModeMcpServers(baseConfig.mcpServers, socketPath), }; @@ -1625,7 +1592,7 @@ export class Session { const baseConfig = this.voiceModeBaseConfig; try { await this.agentManager.refreshAgentFromPersistence(agentId, { - systemPrompt: baseConfig.systemPrompt, + systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, false), mcpServers: this.cloneMcpServers(baseConfig.mcpServers), }); } catch (error) { diff --git a/packages/server/src/server/session.voice-mcp-config.test.ts b/packages/server/src/server/session.voice-mcp-config.test.ts index ae69dd65f..39415524e 100644 --- a/packages/server/src/server/session.voice-mcp-config.test.ts +++ b/packages/server/src/server/session.voice-mcp-config.test.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from "vitest"; -import { buildVoiceAgentMcpServerConfig } from "./session.js"; +import { + buildVoiceAgentMcpServerConfig, + buildVoiceModeSystemPrompt, + stripVoiceModeSystemPrompt, +} from "./voice-config.js"; describe("voice MCP stdio config", () => { test("builds stdio MCP config for voice agent", () => { @@ -25,3 +29,48 @@ describe("voice MCP stdio config", () => { }); }); }); + +describe("voice mode prompt instructions", () => { + test("builds enabled voice instructions and preserves base prompt", () => { + const prompt = buildVoiceModeSystemPrompt("Base system prompt", true); + + expect(prompt).toContain("Base system prompt"); + expect(prompt).toContain(""); + expect(prompt).toContain("Paseo voice mode is now on."); + expect(prompt).toContain("Always use the speak tool for all user-facing communication."); + expect(prompt).toContain(""); + }); + + test("builds disabled voice instructions and supersedes previous voice block", () => { + const existing = [ + "Base system prompt", + "", + "legacy voice instruction", + "", + ].join("\n\n"); + + const prompt = buildVoiceModeSystemPrompt(existing, false); + + expect(prompt).toContain("Base system prompt"); + expect(prompt).toContain("Paseo voice mode is now off."); + expect(prompt).toContain("Ignore any earlier Paseo voice mode instructions in this thread."); + expect(prompt.match(//g)?.length ?? 0).toBe(1); + expect(prompt).not.toContain("legacy voice instruction"); + }); + + test("strips voice blocks from persisted prompt", () => { + const existing = [ + "Base system prompt", + "", + "legacy voice instruction", + "", + ].join("\n\n"); + + expect(stripVoiceModeSystemPrompt(existing)).toBe("Base system prompt"); + expect( + stripVoiceModeSystemPrompt( + ["", "legacy voice instruction", ""].join("\n\n") + ) + ).toBeUndefined(); + }); +}); diff --git a/packages/server/src/server/voice-config.ts b/packages/server/src/server/voice-config.ts new file mode 100644 index 000000000..af5bef30a --- /dev/null +++ b/packages/server/src/server/voice-config.ts @@ -0,0 +1,75 @@ +const VOICE_PROMPT_BLOCK_START = ""; +const VOICE_PROMPT_BLOCK_END = ""; + +const VOICE_AGENT_SYSTEM_INSTRUCTION = [ + "Paseo voice mode is now on.", + "You are the Paseo voice assistant.", + "The user cannot see your chat messages or tool calls.", + "Always use the speak tool for all user-facing communication.", + "Before calling any non-speak tool, first call speak with a short acknowledgement of what you heard and what you will do next.", + "For long-running work, use speak to provide progress updates before and during execution.", + "Treat the user input as transcribed speech.", + "If the user intent is clear, proceed without extra confirmation.", + "If the transcription seems incomplete, cut off, ambiguous, or may contain a non-obvious mistake or misspelling, ask a clarifying question via speak before taking action.", + "Use concise plain language suitable for speech output.", +].join(" "); + +const VOICE_AGENT_DISABLED_INSTRUCTION = [ + "Paseo voice mode is now off.", + "Ignore any earlier Paseo voice mode instructions in this thread.", +].join(" "); + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function makeVoicePromptBlockRegex(): RegExp { + return new RegExp( + `${escapeRegExp(VOICE_PROMPT_BLOCK_START)}[\\s\\S]*?${escapeRegExp( + VOICE_PROMPT_BLOCK_END + )}`, + "g" + ); +} + +export function stripVoiceModeSystemPrompt(existing?: string): string | undefined { + const trimmed = existing?.trim(); + if (!trimmed) { + return undefined; + } + const stripped = trimmed.replace(makeVoicePromptBlockRegex(), "").trim(); + return stripped.length > 0 ? stripped : undefined; +} + +export function buildVoiceModeSystemPrompt(existing: string | undefined, enabled: boolean): string { + const basePrompt = stripVoiceModeSystemPrompt(existing); + const voiceInstruction = enabled + ? VOICE_AGENT_SYSTEM_INSTRUCTION + : VOICE_AGENT_DISABLED_INSTRUCTION; + const voiceBlock = [VOICE_PROMPT_BLOCK_START, voiceInstruction, VOICE_PROMPT_BLOCK_END].join( + "\n" + ); + + return [basePrompt, voiceBlock] + .filter((entry): entry is string => typeof entry === "string" && entry.length > 0) + .join("\n\n"); +} + +export function buildVoiceAgentMcpServerConfig(params: { + command: string; + baseArgs: string[]; + socketPath: string; + env?: Record; +}): { + type: "stdio"; + command: string; + args: string[]; + env?: Record; +} { + return { + type: "stdio", + command: params.command, + args: [...params.baseArgs, "--socket", params.socketPath], + ...(params.env ? { env: params.env } : {}), + }; +}