diff --git a/CHANGELOG.md b/CHANGELOG.md index 22241d6dd..06a4959f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,38 @@ # Changelog +## 0.1.50 - 2026-04-07 + +### Added +- Context window meter — see how much of the context window your agent has used, with color thresholds at 70% and 90%. Works with Claude Code, Codex, and OpenCode. +- Open in editor — jump from any workspace straight into Cursor, VS Code, Zed, or your file manager. Paseo remembers your choice. +- Side-by-side diffs — toggle between unified and split-column diff views, with a whitespace visibility option. +- Spoken messages — when using voice mode, agent speech now appears as regular messages in the conversation instead of raw tool output. +- Plan actions — plan cards now show the actions your agent supports (e.g. "Implement", "Deny") instead of generic accept/reject buttons. +- Background git fetch — ahead/behind counts in the Changes pane stay up to date automatically. + +### Improved +- Workspaces load instantly on connect instead of waiting for a full sync. +- File explorer and diff pane remember which folders are expanded when you switch tabs. +- Closing a workspace tab is now instant. +- Settings shows a Refresh button for providers and displays error details inline. +- Reload agent moved away from the close button to prevent accidental taps. + +### Fixed +- Voice mode no longer drifts into false speech detection during long sessions. +- Garbled overlapping text on plan cards. +- Changes pane could show stale diffs when working with git worktrees. +- Restarting an agent quickly could crash the session. +- Copilot no longer pauses for permission prompts in autopilot mode. +- Connection and pairing dialogs now display correctly on tablets. +- Orchestration errors from agents are now surfaced instead of silently lost. +- Diff stats no longer reset to zero when reconnecting. + ## 0.1.49 - 2026-04-07 ### Fixed -- Models and providers now load reliably on first app connect instead of requiring a second status refresh. -- Model picker on running agents now only shows models from the agent's own provider, not every provider on the server. -- Model data is now prefetched consistently regardless of which screen you open first. -- Draft and running-agent flows now share the same provider data path, eliminating stale model lists from legacy fallbacks. +- Models and providers now load reliably on first connect instead of requiring a manual refresh. +- Model picker only shows models from the agent's own provider, not every provider on the server. +- Model lists stay consistent regardless of which screen you open first. ## 0.1.48 - 2026-04-05 diff --git a/docs/RELEASE.md b/docs/RELEASE.md index e8a46db4b..b20aaec68 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -129,6 +129,16 @@ No prefix (`v`), no extra text. The parser matches the first `## X.Y.Z` line to - **Only Claude should write changelog entries.** - If you are Codex and a stable release needs a changelog entry, launch a Claude agent with Paseo to draft it, then review and commit the result. +## Changelog voice + +The changelog is shown on the Paseo homepage. Write it for **end users**, not developers. + +- **Frame everything from the user's perspective.** Describe what changed in the app, not what changed in the code. Users care that "workspaces load instantly" — not that a component no longer remounts. +- **Never mention component names, internal modules, or implementation details.** No `WorkingIndicator`, no `accumulatedUsage`, no `reconcileAndEmitWorkspaceUpdates`. +- **Collapse internal iterations.** If a feature was added and then fixed within the same release, just list the feature as working. Users never saw the broken version. +- **Only list changes relative to the previous stable release.** The diff is `v(previous)..HEAD`. If something was introduced and fixed between those two tags, it never shipped — don't mention the fix. +- **Cut low-signal entries.** "Toolbar buttons have consistent sizing" is too granular. Combine small polish items or drop them. + ## Pre-release sanity check Before cutting any release (RC or stable), run a Codex review of the diff as a last line of defence against shipping bugs. diff --git a/nix/package.nix b/nix/package.nix index 8fc0b31b1..d8136dd59 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -42,7 +42,7 @@ buildNpmPackage rec { # To update: run `nix build` with lib.fakeHash, copy the `got:` hash. # CI auto-updates this when package-lock.json changes (see .github/workflows/). - npmDepsHash = "sha256-7ZBAwyC3CGqA9bK7bDW8YvlQyYhw1bsfSRd/LrdfA/E="; + npmDepsHash = "sha256-bkSOrHjU2rKSku8sOcZRIVKG/mdSOV/lXiYuqUzeXD8="; # Prevent onnxruntime-node's install script from running during automatic # npm rebuild (it tries to download from api.nuget.org, which fails in the sandbox). diff --git a/package-lock.json b/package-lock.json index 32509dbfb..10f08308a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "paseo", - "version": "0.1.49", + "version": "0.1.50", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "paseo", - "version": "0.1.49", + "version": "0.1.50", "hasInstallScript": true, "license": "AGPL-3.0-or-later", "workspaces": [ @@ -36221,16 +36221,16 @@ }, "packages/app": { "name": "@getpaseo/app", - "version": "0.1.49", + "version": "0.1.50", "dependencies": { "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", "@expo/vector-icons": "^15.0.2", "@floating-ui/react-native": "^0.10.7", - "@getpaseo/expo-two-way-audio": "0.1.49", - "@getpaseo/highlight": "0.1.49", - "@getpaseo/server": "0.1.49", + "@getpaseo/expo-two-way-audio": "0.1.50", + "@getpaseo/highlight": "0.1.50", + "@getpaseo/server": "0.1.50", "@gorhom/bottom-sheet": "^5.2.6", "@gorhom/portal": "^1.0.14", "@react-native-async-storage/async-storage": "2.2.0", @@ -36347,11 +36347,11 @@ }, "packages/cli": { "name": "@getpaseo/cli", - "version": "0.1.49", + "version": "0.1.50", "dependencies": { "@clack/prompts": "^1.0.0", - "@getpaseo/relay": "0.1.49", - "@getpaseo/server": "0.1.49", + "@getpaseo/relay": "0.1.50", + "@getpaseo/server": "0.1.50", "chalk": "^5.3.0", "commander": "^12.0.0", "mime-types": "^2.1.35", @@ -36392,11 +36392,11 @@ }, "packages/desktop": { "name": "@getpaseo/desktop", - "version": "0.1.49", + "version": "0.1.50", "license": "AGPL-3.0-or-later", "dependencies": { - "@getpaseo/cli": "0.1.49", - "@getpaseo/server": "0.1.49", + "@getpaseo/cli": "0.1.50", + "@getpaseo/server": "0.1.50", "electron-log": "^5.4.3", "electron-updater": "^6.6.2", "ws": "^8.14.2" @@ -36430,7 +36430,7 @@ }, "packages/expo-two-way-audio": { "name": "@getpaseo/expo-two-way-audio", - "version": "0.1.49", + "version": "0.1.50", "license": "MIT", "devDependencies": { "@biomejs/biome": "1.9.4", @@ -36631,7 +36631,7 @@ }, "packages/highlight": { "name": "@getpaseo/highlight", - "version": "0.1.49", + "version": "0.1.50", "dependencies": { "@lezer/common": "^1.5.0", "@lezer/cpp": "^1.1.5", @@ -36657,7 +36657,7 @@ }, "packages/relay": { "name": "@getpaseo/relay", - "version": "0.1.49", + "version": "0.1.50", "dependencies": { "base64-js": "^1.5.1", "tweetnacl": "^1.0.3", @@ -36673,14 +36673,14 @@ }, "packages/server": { "name": "@getpaseo/server", - "version": "0.1.49", + "version": "0.1.50", "dependencies": { "@agentclientprotocol/sdk": "^0.17.1", "@ai-sdk/openai": "2.0.52", "@anthropic-ai/claude-agent-sdk": "^0.2.11", "@deepgram/sdk": "^3.4.0", - "@getpaseo/highlight": "0.1.49", - "@getpaseo/relay": "0.1.49", + "@getpaseo/highlight": "0.1.50", + "@getpaseo/relay": "0.1.50", "@isaacs/ttlcache": "^2.1.4", "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "1.2.6", @@ -37083,7 +37083,7 @@ }, "packages/website": { "name": "@getpaseo/website", - "version": "0.1.49", + "version": "0.1.50", "dependencies": { "@cloudflare/vite-plugin": "^1.20.3", "@cloudflare/workers-types": "^4.20260114.0", diff --git a/package.json b/package.json index 15f2fa86d..1b90aca43 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "paseo", - "version": "0.1.49", + "version": "0.1.50", "private": true, "workspaces": [ "packages/expo-two-way-audio", diff --git a/packages/app/package.json b/packages/app/package.json index 3f4c48f1d..8689b9f57 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,7 +1,7 @@ { "name": "@getpaseo/app", "main": "index.ts", - "version": "0.1.49", + "version": "0.1.50", "private": true, "scripts": { "start": "expo start", @@ -31,9 +31,9 @@ "@dnd-kit/utilities": "^3.2.2", "@expo/vector-icons": "^15.0.2", "@floating-ui/react-native": "^0.10.7", - "@getpaseo/expo-two-way-audio": "0.1.49", - "@getpaseo/highlight": "0.1.49", - "@getpaseo/server": "0.1.49", + "@getpaseo/expo-two-way-audio": "0.1.50", + "@getpaseo/highlight": "0.1.50", + "@getpaseo/server": "0.1.50", "@gorhom/bottom-sheet": "^5.2.6", "@gorhom/portal": "^1.0.14", "@react-native-async-storage/async-storage": "2.2.0", diff --git a/packages/app/src/components/agent-stream-view.tsx b/packages/app/src/components/agent-stream-view.tsx index d38bddad5..298da47f4 100644 --- a/packages/app/src/components/agent-stream-view.tsx +++ b/packages/app/src/components/agent-stream-view.tsx @@ -26,6 +26,7 @@ import { Check, ChevronDown, X } from "lucide-react-native"; import { usePanelStore } from "@/stores/panel-store"; import { AssistantMessage, + SpeakMessage, UserMessage, ActivityLog, ToolCall, @@ -365,6 +366,21 @@ const AgentStreamViewComponent = forwardRef + ); + } + return ( + + {lineNumber != null ? String(lineNumber) : ""} + + + ); +} + +function DiffTextLine({ + line, + wrapLines, +}: { + line: DiffLine; + wrapLines: boolean; +}) { + return ( + + {line.tokens && line.type !== "header" ? ( + + ) : ( + + {line.content || " "} + + )} + + ); +} + +function SplitTextLine({ + line, + wrapLines, +}: { + line: SplitDiffDisplayLine | null; + wrapLines: boolean; +}) { + return ( + + {line?.tokens ? ( + + ) : ( + + {line?.content ?? ""} + + )} + + ); +} + function DiffLineView({ line, lineNumber, @@ -153,10 +243,7 @@ function DiffLineView({ @@ -190,26 +277,20 @@ function DiffLineView({ ); } -function SplitDiffCell({ +function SplitDiffLine({ line, gutterWidth, wrapLines, - showDivider = false, }: { line: SplitDiffDisplayLine | null; gutterWidth: number; wrapLines: boolean; - showDivider?: boolean; }) { return ( @@ -229,7 +310,7 @@ function SplitDiffCell({ + + {rows.map((row, i) => { + if (row.kind === "header") { + return ( + + {row.content} + + ); + } + return ( + + ); + })} + + + ); + } + + return ( + + + {rows.map((row, i) => { + if (row.kind === "header") { + return ; + } + const line = side === "left" ? row.left : row.right; + return ; + })} + + + 0 && { minWidth: scrollWidth }]}> + {rows.map((row, i) => { + if (row.kind === "header") { + return ( + + {row.content} + + ); + } + return ; + })} + + + + ); +} + const DiffFileHeader = memo(function DiffFileHeader({ file, isExpanded, @@ -368,85 +524,70 @@ function DiffFileBody({ } const gutterWidth = lineNumberGutterWidth(maxLineNo); - const linesContent = - layout === "split" - ? buildSplitDiffRows(file).map((row, rowIndex) => { - if (row.kind === "header") { - return ( - - {row.content} - - ); - } - - return ( - - - - - ); - }) - : file.hunks.map((hunk, hunkIndex) => { - let oldLineNo = hunk.oldStart; - let newLineNo = hunk.newStart; - return hunk.lines.map((line, lineIndex) => { - let lineNumber: number | null = null; - if (line.type === "remove") { - lineNumber = oldLineNo; - oldLineNo++; - } else if (line.type === "add") { - lineNumber = newLineNo; - newLineNo++; - } else if (line.type === "context") { - lineNumber = newLineNo; - oldLineNo++; - newLineNo++; - } - return ( - - ); - }); - }); - - const availableWidth = bodyWidth > 0 ? bodyWidth : scrollViewWidth; - const contentContainer = ( - 0 && - (layout === "split" - ? { width: availableWidth, minWidth: availableWidth, maxWidth: availableWidth } - : { minWidth: availableWidth }), - ]} - > - {linesContent} - - ); - - if (wrapLines) { - return {contentContainer}; + if (layout === "split") { + const rows = buildSplitDiffRows(file); + return ( + + + + + ); } + const computedLines: { line: DiffLine; lineNumber: number | null; key: string }[] = []; + for (const [hunkIndex, hunk] of file.hunks.entries()) { + let oldLineNo = hunk.oldStart; + let newLineNo = hunk.newStart; + for (const [lineIndex, line] of hunk.lines.entries()) { + let lineNumber: number | null = null; + if (line.type === "remove") { + lineNumber = oldLineNo; + oldLineNo++; + } else if (line.type === "add") { + lineNumber = newLineNo; + newLineNo++; + } else if (line.type === "context") { + lineNumber = newLineNo; + oldLineNo++; + newLineNo++; + } + computedLines.push({ line, lineNumber, key: `${hunkIndex}-${lineIndex}` }); + } + } + + if (wrapLines) { + return ( + + + {computedLines.map(({ line, lineNumber, key }) => ( + + ))} + + + ); + } + + const availableWidth = bodyWidth > 0 ? bodyWidth : scrollViewWidth; return ( - - {contentContainer} - + + + {computedLines.map(({ line, lineNumber, key }) => ( + + ))} + + + 0 && { minWidth: availableWidth }]}> + {computedLines.map(({ line, key }) => ( + + ))} + + + ); })()} @@ -1612,21 +1753,35 @@ const styles = StyleSheet.create((theme) => ({ borderTopColor: theme.colors.border, backgroundColor: theme.colors.surface1, }, + diffContentRow: { + flexDirection: "row", + alignItems: "stretch", + }, diffContentInner: { flexDirection: "column", }, linesContainer: { backgroundColor: theme.colors.surface1, }, - splitLinesContainer: { + gutterColumn: { backgroundColor: theme.colors.surface1, - minWidth: 760, + }, + gutterCell: { + borderRightWidth: theme.borderWidth[1], + borderRightColor: theme.colors.border, + justifyContent: "flex-start", + }, + textLineContainer: { + flexDirection: "row", + alignItems: "stretch", + paddingLeft: theme.spacing[2], }, splitRow: { flexDirection: "row", alignItems: "stretch", - width: "100%", - minWidth: "100%", + }, + splitColumnScroll: { + flex: 1, }, splitHeaderRow: { backgroundColor: theme.colors.surface2, @@ -1635,11 +1790,11 @@ const styles = StyleSheet.create((theme) => ({ splitCell: { flex: 1, flexBasis: 0, - minWidth: 0, + backgroundColor: theme.colors.surface2, + }, + splitCellRow: { flexDirection: "row", alignItems: "stretch", - overflow: "hidden", - backgroundColor: theme.colors.surface2, }, emptySplitCell: { backgroundColor: theme.colors.surfaceDiffEmpty, diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx index 3bc5eacb3..978c21745 100644 --- a/packages/app/src/components/message.tsx +++ b/packages/app/src/components/message.tsx @@ -39,6 +39,7 @@ import { Copy, TriangleAlertIcon, Scissors, + MicVocal, } from "lucide-react-native"; import { StyleSheet, useUnistyles, UnistylesRuntime } from "react-native-unistyles"; import Animated, { @@ -913,6 +914,65 @@ export const AssistantMessage = memo(function AssistantMessage({ ); }); +interface SpeakMessageProps { + message: string; + timestamp: number; + disableOuterSpacing?: boolean; +} + +const speakMessageStylesheet = StyleSheet.create((theme) => ({ + container: { + paddingHorizontal: theme.spacing[2], + paddingVertical: theme.spacing[3], + }, + containerSpacing: { + marginBottom: theme.spacing[4], + }, + header: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + marginBottom: theme.spacing[2], + }, + headerLabel: { + fontFamily: Fonts.sans, + fontSize: 12, + fontWeight: "500", + color: theme.colors.foregroundMuted, + }, + text: { + fontFamily: Fonts.sans, + fontSize: theme.fontSize.base, + lineHeight: 22, + color: theme.colors.foreground, + }, +})); + +export const SpeakMessage = memo(function SpeakMessage({ + message, + timestamp, + disableOuterSpacing, +}: SpeakMessageProps) { + const { theme } = useUnistyles(); + const resolvedDisableOuterSpacing = useDisableOuterSpacing(disableOuterSpacing); + + return ( + + + + Spoke + + {message} + + ); +}); + interface ActivityLogProps { type: "system" | "info" | "success" | "error" | "artifact"; message: string; diff --git a/packages/app/src/components/stream-strategy-native.tsx b/packages/app/src/components/stream-strategy-native.tsx index ac75da5b2..97664cd39 100644 --- a/packages/app/src/components/stream-strategy-native.tsx +++ b/packages/app/src/components/stream-strategy-native.tsx @@ -313,7 +313,7 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat keyExtractor={(item) => item.id} testID="agent-chat-scroll" nativeID="agent-chat-scroll-native-virtualized" - ListHeaderComponent={liveHeaderContent ? () => liveHeaderContent : undefined} + ListHeaderComponent={liveHeaderContent ?? undefined} contentContainerStyle={baseListContentContainerStyle} style={listStyle} onLayout={handleListLayout} diff --git a/packages/app/src/screens/settings-screen.tsx b/packages/app/src/screens/settings-screen.tsx index 3d043cd9d..465402c1f 100644 --- a/packages/app/src/screens/settings-screen.tsx +++ b/packages/app/src/screens/settings-screen.tsx @@ -476,17 +476,25 @@ function ProvidersSection({ routeServerId }: ProvidersSectionProps) { const entry = entries?.find((e) => e.provider === def.id); const status = entry?.status ?? "unavailable"; const ProviderIcon = getProviderIcon(def.id); + const providerError = + status === "error" && typeof entry?.error === "string" && entry.error.trim().length > 0 + ? entry.error.trim() + : null; return ( - - - {def.label} + + + + {def.label} + + {providerError ? ( + + {providerError} + + ) : null} { manager.destroy(); }); + test("refresh during an in-flight refresh is a no-op", async () => { + const fetchModels = deferred(); + const fetchModes = deferred(); + const { registry, handles } = createRegistry([ + createMockProvider({ + provider: "codex", + fetchModels: async () => fetchModels.promise, + fetchModes: async () => fetchModes.promise, + }), + ]); + const manager = new ProviderSnapshotManager(registry, createTestLogger()); + const changes: ProviderSnapshotEntry[][] = []; + manager.on("change", (entries) => changes.push(entries)); + + manager.refresh("/tmp/project"); + + expect(manager.getSnapshot("/tmp/project")).toEqual([ + { provider: "codex", status: "loading" }, + ]); + + manager.refresh("/tmp/project"); + manager.refresh("/tmp/project"); + manager.refresh("/tmp/project"); + + expect(changes).toHaveLength(1); + expect(handles.codex?.isAvailable).toHaveBeenCalledTimes(1); + + fetchModels.resolve([createModel("codex", "gpt-5.2")]); + fetchModes.resolve([createMode("auto")]); + + await vi.waitFor(() => { + expect(getProviderEntry(manager.getSnapshot("/tmp/project"), "codex")).toMatchObject({ + provider: "codex", + status: "ready", + models: [createModel("codex", "gpt-5.2")], + modes: [createMode("auto")], + }); + }); + + expect(handles.codex?.fetchModels).toHaveBeenCalledTimes(1); + expect(handles.codex?.fetchModes).toHaveBeenCalledTimes(1); + + manager.destroy(); + }); + test("multiple getSnapshot calls for same cwd do not trigger multiple warmUps", async () => { const codexModels = deferred(); const { registry, handles } = createRegistry([ diff --git a/packages/server/src/server/agent/provider-snapshot-manager.ts b/packages/server/src/server/agent/provider-snapshot-manager.ts index dded5a2b4..78b56ce70 100644 --- a/packages/server/src/server/agent/provider-snapshot-manager.ts +++ b/packages/server/src/server/agent/provider-snapshot-manager.ts @@ -32,8 +32,7 @@ export class ProviderSnapshotManager { const cwdKey = normalizeCwdKey(cwd); const entries = this.snapshots.get(cwdKey); if (!entries) { - const loadingEntries = this.createLoadingEntries(); - this.snapshots.set(cwdKey, loadingEntries); + const loadingEntries = this.resetSnapshotToLoading(cwdKey); void this.warmUp(cwd); return entriesToArray(loadingEntries); } @@ -42,7 +41,11 @@ export class ProviderSnapshotManager { refresh(cwd?: string): void { const cwdKey = normalizeCwdKey(cwd); - this.snapshots.set(cwdKey, this.createLoadingEntries()); + if (this.warmUps.has(cwdKey)) { + return; + } + this.resetSnapshotToLoading(cwdKey); + this.emitChange(cwdKey); void this.warmUp(cwd); } @@ -170,6 +173,15 @@ export class ProviderSnapshotManager { return created; } + private resetSnapshotToLoading(cwdKey: string): Map { + const snapshot = this.getOrCreateSnapshot(cwdKey); + snapshot.clear(); + for (const [provider, entry] of this.createLoadingEntries()) { + snapshot.set(provider, entry); + } + return snapshot; + } + private getProviderIds(): AgentProvider[] { return AGENT_PROVIDER_IDS.filter((provider) => this.providerRegistry[provider]); } diff --git a/packages/server/src/server/agent/providers/claude-agent.test.ts b/packages/server/src/server/agent/providers/claude-agent.test.ts index 68732b9e9..8da63128c 100644 --- a/packages/server/src/server/agent/providers/claude-agent.test.ts +++ b/packages/server/src/server/agent/providers/claude-agent.test.ts @@ -358,7 +358,7 @@ describe("ClaudeAgentSession context window usage", () => { }); } - test("convertUsage includes contextWindowMaxTokens and derives used tokens from result usage", async () => { + test("convertUsage includes contextWindowMaxTokens and derives used tokens from result usage as initial fallback", async () => { const session = await createSessionForTest(); const usage = session.convertUsage( @@ -388,7 +388,7 @@ describe("ClaudeAgentSession context window usage", () => { }); }); - test("contextWindowUsedTokens is derived from result usage when task_progress is missing", async () => { + test("contextWindowUsedTokens falls back to result usage when no task_progress was received", async () => { const session = await createSessionForTest(); const usage = session.convertUsage({ @@ -468,6 +468,124 @@ describe("ClaudeAgentSession context window usage", () => { }); }); + test("task_progress emits a usage_updated event", async () => { + const session = await createSessionForTest(); + + const events = session.translateMessageToEvents({ + type: "system", + subtype: "task_progress", + task_id: "task-1", + description: "Processing", + usage: { + total_tokens: 999, + tool_uses: 1, + duration_ms: 50, + }, + uuid: "task-progress-1", + session_id: "session-1", + }); + + expect(events).toContainEqual({ + type: "usage_updated", + provider: "claude", + usage: { + contextWindowUsedTokens: 999, + }, + }); + }); + + test("task_notification emits a usage_updated event", async () => { + const session = await createSessionForTest(); + + const events = session.translateMessageToEvents({ + type: "system", + subtype: "task_notification", + uuid: "task-note-1", + task_id: "task-1", + status: "running", + summary: "Background task still running", + usage: { + total_tokens: 777, + tool_uses: 1, + duration_ms: 50, + }, + session_id: "session-1", + } as any); + + expect(events).toContainEqual({ + type: "usage_updated", + provider: "claude", + usage: { + contextWindowUsedTokens: 777, + }, + }); + }); + + test("message_start stream events emit usage_updated with per-request usage", async () => { + const session = await createSessionForTest(); + + const events = session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_start", + message: { + usage: { + input_tokens: 100, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 30, + }, + }, + }, + session_id: "session-1", + } as any); + + expect(events).toContainEqual({ + type: "usage_updated", + provider: "claude", + usage: { + contextWindowUsedTokens: 150, + }, + }); + }); + + test("message_delta stream events update per-request usage", async () => { + const session = await createSessionForTest(); + + session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_start", + message: { + usage: { + input_tokens: 100, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 30, + }, + }, + }, + session_id: "session-1", + } as any); + + const events = session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_delta", + usage: { + output_tokens: 25, + }, + }, + session_id: "session-1", + } as any); + + expect(events).toContainEqual({ + type: "usage_updated", + provider: "claude", + usage: { + contextWindowUsedTokens: 175, + }, + }); + }); + test("task_progress usage takes priority over derived result usage", async () => { const session = await createSessionForTest(); @@ -508,7 +626,7 @@ describe("ClaudeAgentSession context window usage", () => { }); }); - test("contextWindowUsedTokens resets between turns and falls back to result usage", async () => { + test("contextWindowUsedTokens persists across turns from last task_progress", async () => { const queryFactory = createQueryFactoryForTurns([ [ { @@ -600,20 +718,23 @@ describe("ClaudeAgentSession context window usage", () => { contextWindowMaxTokens: 200_000, contextWindowUsedTokens: 999, }); + // Turn 2 has no task_progress, so contextWindowUsedTokens retains the + // last known value from turn 1 rather than deriving from accumulated + // result.usage (which would be incorrect — those are session-level totals). expect(secondTurn.usage).toEqual({ inputTokens: 11, cachedInputTokens: 6, outputTokens: 8, totalCostUsd: 0.1, contextWindowMaxTokens: 200_000, - contextWindowUsedTokens: 28, + contextWindowUsedTokens: 999, }); } finally { await session.close(); } }); - test("convertUsage derives used tokens even when modelUsage is missing", async () => { + test("convertUsage derives used tokens from result usage as fallback when task_progress is missing", async () => { const session = await createSessionForTest(); const usage = session.convertUsage({ @@ -635,4 +756,134 @@ describe("ClaudeAgentSession context window usage", () => { contextWindowUsedTokens: 22, }); }); + + test("convertUsage uses per-request stream usage when no task_progress is available", async () => { + const session = await createSessionForTest(); + + session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_start", + message: { + usage: { + input_tokens: 100, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 30, + }, + }, + }, + session_id: "session-1", + } as any); + session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_delta", + usage: { + output_tokens: 25, + }, + }, + session_id: "session-1", + } as any); + + const usage = session.convertUsage({ + type: "result", + subtype: "success", + usage: { + input_tokens: 10, + cache_read_input_tokens: 5, + output_tokens: 7, + }, + total_cost_usd: 0.12, + }); + + expect(usage).toEqual({ + inputTokens: 10, + cachedInputTokens: 5, + outputTokens: 7, + totalCostUsd: 0.12, + contextWindowUsedTokens: 175, + }); + }); + + test("per-request stream usage is not cumulative across API calls in a turn", async () => { + const session = await createSessionForTest(); + + session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_start", + message: { + usage: { + input_tokens: 100, + cache_creation_input_tokens: 20, + cache_read_input_tokens: 30, + }, + }, + }, + session_id: "session-1", + } as any); + session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_delta", + usage: { + output_tokens: 25, + }, + }, + session_id: "session-1", + } as any); + + const secondStartEvents = session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_start", + message: { + usage: { + input_tokens: 40, + cache_creation_input_tokens: 5, + cache_read_input_tokens: 10, + }, + }, + }, + session_id: "session-1", + } as any); + + expect(secondStartEvents).toContainEqual({ + type: "usage_updated", + provider: "claude", + usage: { + contextWindowUsedTokens: 55, + }, + }); + + session.translateMessageToEvents({ + type: "stream_event", + event: { + type: "message_delta", + usage: { + output_tokens: 7, + }, + }, + session_id: "session-1", + } as any); + + const usage = session.convertUsage({ + type: "result", + subtype: "success", + usage: { + input_tokens: 10, + cache_read_input_tokens: 5, + output_tokens: 7, + }, + total_cost_usd: 0.12, + }); + + expect(usage).toEqual({ + inputTokens: 10, + cachedInputTokens: 5, + outputTokens: 7, + totalCostUsd: 0.12, + contextWindowUsedTokens: 62, + }); + }); }); diff --git a/packages/server/src/server/agent/providers/claude-agent.ts b/packages/server/src/server/agent/providers/claude-agent.ts index fab2700eb..b8c742799 100644 --- a/packages/server/src/server/agent/providers/claude-agent.ts +++ b/packages/server/src/server/agent/providers/claude-agent.ts @@ -1224,16 +1224,67 @@ function extractContextWindowSize(modelUsage: unknown): number | undefined { return maxContextWindow; } -function readContextWindowUsedTokensFromTaskProgress( - message: SDKTaskProgressMessage, +function readUsageTotalTokens( + usage: unknown, ): number | undefined { - const totalTokens = message.usage?.total_tokens; + if (!usage || typeof usage !== "object") { + return undefined; + } + const totalTokens = (usage as { total_tokens?: unknown }).total_tokens; if (typeof totalTokens !== "number" || !Number.isFinite(totalTokens) || totalTokens < 0) { return undefined; } return totalTokens; } +function readContextWindowUsedTokensFromTaskProgress( + message: SDKTaskProgressMessage, +): number | undefined { + return readUsageTotalTokens(message.usage); +} + +function readUsageFromTaskNotification(message: { usage?: unknown }): number | undefined { + return readUsageTotalTokens(message.usage); +} + +function readStreamRequestInputTokens(event: Record): number | undefined { + const messageUsage = (event.message as { usage?: unknown } | undefined)?.usage; + if (!messageUsage || typeof messageUsage !== "object") { + return undefined; + } + const usage = messageUsage as { + input_tokens?: unknown; + cache_creation_input_tokens?: unknown; + cache_read_input_tokens?: unknown; + }; + const inputTokens = + typeof usage.input_tokens === "number" && Number.isFinite(usage.input_tokens) + ? usage.input_tokens + : undefined; + const cacheCreationInputTokens = + typeof usage.cache_creation_input_tokens === "number" && + Number.isFinite(usage.cache_creation_input_tokens) + ? usage.cache_creation_input_tokens + : 0; + const cacheReadInputTokens = + typeof usage.cache_read_input_tokens === "number" && + Number.isFinite(usage.cache_read_input_tokens) + ? usage.cache_read_input_tokens + : 0; + if (typeof inputTokens !== "number" || inputTokens < 0) { + return undefined; + } + return inputTokens + cacheCreationInputTokens + cacheReadInputTokens; +} + +function readStreamRequestOutputTokens(event: Record): number | undefined { + const outputTokens = (event.usage as { output_tokens?: unknown } | undefined)?.output_tokens; + if (typeof outputTokens !== "number" || !Number.isFinite(outputTokens) || outputTokens < 0) { + return undefined; + } + return outputTokens; +} + class ClaudeAgentSession implements AgentSession { readonly provider: "claude" = "claude"; readonly capabilities = CLAUDE_CAPABILITIES; @@ -1277,6 +1328,9 @@ class ClaudeAgentSession implements AgentSession { private lastForegroundPromptText: string | null = null; private foregroundHasVisibleActivity = false; private lastContextWindowUsedTokens: number | undefined; + private lastContextWindowMaxTokens: number | undefined; + private lastStreamRequestInputTokens: number | undefined; + private lastStreamRequestOutputTokens: number | undefined; private userMessageIds: string[] = []; private recentStderr = ""; private closed = false; @@ -1459,7 +1513,6 @@ class ClaudeAgentSession implements AgentSession { const sdkMessage = this.toSdkUserMessage(prompt); this.lastForegroundPromptText = this.extractPromptText(prompt); - this.lastContextWindowUsedTokens = undefined; const turnId = this.createTurnId("foreground"); this.activeForegroundTurnId = turnId; this.foregroundHasVisibleActivity = false; @@ -2303,7 +2356,6 @@ class ClaudeAgentSession implements AgentSession { this.notifySubscribers(event); this.activeForegroundTurnId = null; this.lastForegroundPromptText = null; - this.lastContextWindowUsedTokens = undefined; this.cancelCurrentTurn = null; this.syncTurnState("foreground turn terminal"); } @@ -2319,12 +2371,10 @@ class ClaudeAgentSession implements AgentSession { if (this.activeForegroundTurnId) { this.activeForegroundTurnId = null; this.lastForegroundPromptText = null; - this.lastContextWindowUsedTokens = undefined; this.cancelCurrentTurn = null; this.syncTurnState("foreground turn terminal"); } else if (this.autonomousTurn) { this.autonomousTurn = null; - this.lastContextWindowUsedTokens = undefined; this.syncTurnState("autonomous turn terminal"); } } @@ -2337,7 +2387,6 @@ class ClaudeAgentSession implements AgentSession { this.autonomousTurn = { id: this.createTurnId("autonomous"), }; - this.lastContextWindowUsedTokens = undefined; this.notifySubscribers({ type: "turn_started", provider: "claude" }); this.syncTurnState("autonomous turn started"); } @@ -2348,7 +2397,6 @@ class ClaudeAgentSession implements AgentSession { } this.notifySubscribers({ type: "turn_completed", provider: "claude" }); this.autonomousTurn = null; - this.lastContextWindowUsedTokens = undefined; this.syncTurnState("autonomous turn completed"); } @@ -2677,10 +2725,18 @@ class ClaudeAgentSession implements AgentSession { provider: "claude", }); } + const usage = readUsageFromTaskNotification(message); + if (typeof usage === "number") { + this.lastContextWindowUsedTokens = usage; + events.push(this.createUsageUpdatedEvent(usage)); + } } else if (message.subtype === "task_progress") { this.lastContextWindowUsedTokens = readContextWindowUsedTokensFromTaskProgress(message) ?? this.lastContextWindowUsedTokens; + if (typeof this.lastContextWindowUsedTokens === "number") { + events.push(this.createUsageUpdatedEvent(this.lastContextWindowUsedTokens)); + } } break; case "user": { @@ -2748,6 +2804,10 @@ class ClaudeAgentSession implements AgentSession { break; } case "stream_event": { + const usageUpdatedEvent = this.trackStreamEventUsage(message.event); + if (usageUpdatedEvent) { + events.push(usageUpdatedEvent); + } const timelineItems = this.mapPartialEvent(message.event, { suppressAssistantText: options?.suppressAssistantText ?? false, suppressReasoning: options?.suppressReasoning ?? false, @@ -2916,11 +2976,26 @@ class ClaudeAgentSession implements AgentSession { modelUsage ?? message.modelUsage, ); if (contextWindowMaxTokens !== undefined) { + this.lastContextWindowMaxTokens = contextWindowMaxTokens; usage.contextWindowMaxTokens = contextWindowMaxTokens; + } else if (this.lastContextWindowMaxTokens !== undefined) { + usage.contextWindowMaxTokens = this.lastContextWindowMaxTokens; } if (typeof this.lastContextWindowUsedTokens === "number") { + // task_progress.total_tokens is the accurate context window fill level. + // Prefer it over result.usage which contains accumulated session totals. usage.contextWindowUsedTokens = this.lastContextWindowUsedTokens; + } else if ( + typeof this.lastStreamRequestInputTokens === "number" && + typeof this.lastStreamRequestOutputTokens === "number" + ) { + usage.contextWindowUsedTokens = + this.lastStreamRequestInputTokens + this.lastStreamRequestOutputTokens; } else if (message.usage) { + // Fallback: derive from result.usage when no task_progress has been + // received yet. These values are accumulated across all API calls, but + // for the first turn they equal the per-call values so the estimate is + // reasonable. Once a task_progress arrives it takes over permanently. const usageWithCacheCreation = message.usage as typeof message.usage & { cache_creation_input_tokens?: number; }; @@ -2936,6 +3011,54 @@ class ClaudeAgentSession implements AgentSession { return usage; } + private createUsageUpdatedEvent(contextWindowUsedTokens: number): AgentStreamEvent { + const usage: AgentUsage = { + contextWindowUsedTokens, + }; + if (this.lastContextWindowMaxTokens !== undefined) { + usage.contextWindowMaxTokens = this.lastContextWindowMaxTokens; + } + return { + type: "usage_updated", + provider: "claude", + usage, + }; + } + + private trackStreamEventUsage(event: unknown): AgentStreamEvent | null { + if (!event || typeof event !== "object") { + return null; + } + const streamEvent = event as Record; + const eventType = readTrimmedString(streamEvent.type); + if (eventType === "message_start") { + const inputTokens = readStreamRequestInputTokens(streamEvent); + if (typeof inputTokens !== "number") { + return null; + } + this.lastStreamRequestInputTokens = inputTokens; + this.lastStreamRequestOutputTokens = 0; + } else if (eventType === "message_delta") { + const outputTokens = readStreamRequestOutputTokens(streamEvent); + if (typeof outputTokens !== "number") { + return null; + } + this.lastStreamRequestOutputTokens = outputTokens; + } else { + return null; + } + + if ( + typeof this.lastStreamRequestInputTokens !== "number" || + typeof this.lastStreamRequestOutputTokens !== "number" + ) { + return null; + } + return this.createUsageUpdatedEvent( + this.lastStreamRequestInputTokens + this.lastStreamRequestOutputTokens, + ); + } + private handlePermissionRequest: CanUseTool = async ( toolName, input, diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index 813a201c1..0cd41a189 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -577,6 +577,52 @@ describe("Codex app-server provider", () => { }); }); + test("emits usage_updated on token usage updates and keeps usage on turn completion", () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + (session as any).handleNotification("thread/tokenUsage/updated", { + tokenUsage: { + model_context_window: 200000, + last: { + total_tokens: 50000, + inputTokens: 30000, + cachedInputTokens: 5000, + outputTokens: 15000, + }, + }, + }); + (session as any).handleNotification("turn/completed", { + turn: { status: "completed", error: null }, + }); + + expect(events).toContainEqual({ + type: "usage_updated", + provider: "codex", + turnId: "test-turn", + usage: { + inputTokens: 30000, + cachedInputTokens: 5000, + outputTokens: 15000, + contextWindowMaxTokens: 200000, + contextWindowUsedTokens: 50000, + }, + }); + expect(events.at(-1)).toEqual({ + type: "turn_completed", + provider: "codex", + turnId: "test-turn", + usage: { + inputTokens: 30000, + cachedInputTokens: 5000, + outputTokens: 15000, + contextWindowMaxTokens: 200000, + contextWindowUsedTokens: 50000, + }, + }); + }); + test("approving a synthetic Codex plan permission disables plan and fast mode and starts implementation", async () => { const session = createSession({ featureValues: { plan_mode: true, fast_mode: true }, diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.ts index 141a05d96..35baca8d2 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.ts @@ -3506,6 +3506,13 @@ class CodexAppServerAgentSession implements AgentSession { if (parsed.kind === "token_usage_updated") { this.latestUsage = toAgentUsage(parsed.tokenUsage); + if (this.latestUsage) { + this.notifySubscribers({ + type: "usage_updated", + provider: CODEX_PROVIDER, + usage: this.latestUsage, + }); + } return; } diff --git a/packages/server/src/server/agent/providers/codex/tool-call-mapper.test.ts b/packages/server/src/server/agent/providers/codex/tool-call-mapper.test.ts index 65c4144d0..065934f9b 100644 --- a/packages/server/src/server/agent/providers/codex/tool-call-mapper.test.ts +++ b/packages/server/src/server/agent/providers/codex/tool-call-mapper.test.ts @@ -539,6 +539,26 @@ describe("codex tool-call mapper", () => { }); }); + it("normalizes codex paseo_voice.speak mcp calls and extracts spoken text", () => { + const item = mapCodexToolCallFromThreadItem({ + type: "mcpToolCall", + id: "codex-speak-thread-2", + status: "completed", + server: "paseo_voice", + tool: "speak", + arguments: { text: "Voice response from Codex via paseo_voice." }, + result: { ok: true }, + }); + + expect(item).toBeTruthy(); + expect(item?.name).toBe("speak"); + expect(item?.detail).toEqual({ + type: "unknown", + input: "Voice response from Codex via paseo_voice.", + output: null, + }); + }); + it("normalizes codex paseo speak rollout names and extracts spoken text", () => { const item = expectMapped( mapCodexRolloutToolCall({ diff --git a/packages/server/src/server/agent/providers/codex/tool-call-mapper.ts b/packages/server/src/server/agent/providers/codex/tool-call-mapper.ts index a5fd872f2..8b0657c05 100644 --- a/packages/server/src/server/agent/providers/codex/tool-call-mapper.ts +++ b/packages/server/src/server/agent/providers/codex/tool-call-mapper.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import type { ToolCallTimelineItem } from "../../agent-sdk-types.js"; import { extractCodexShellOutput, truncateDiffText } from "../tool-call-mapper-utils.js"; import { deriveCodexToolDetail, normalizeCodexFilePath } from "./tool-call-detail-parser.js"; +import { isSpeakToolName } from "../../tool-name-normalization.js"; type CodexMapperOptions = { cwd?: string | null }; @@ -67,7 +68,7 @@ const CodexEditToolNameSchema = z.union([ z.literal("apply_diff"), ]); const CodexSearchToolNameSchema = z.union([z.literal("search"), z.literal("web_search")]); -const CodexSpeakToolNameSchema = z.literal("paseo.speak"); +const CodexSpeakToolNameSchema = z.string().min(1).refine((name) => isSpeakToolName(name.trim())); const CodexToolKindSchema = z.enum([ "shell", diff --git a/packages/server/src/server/agent/providers/opencode-agent.ts b/packages/server/src/server/agent/providers/opencode-agent.ts index 7965c96f1..fcb93de7d 100644 --- a/packages/server/src/server/agent/providers/opencode-agent.ts +++ b/packages/server/src/server/agent/providers/opencode-agent.ts @@ -450,16 +450,16 @@ function mergeOpenCodeStepFinishUsage( const cost = readPositiveFiniteNumber(part.cost); if (inputTokens !== undefined) { - usage.inputTokens = (usage.inputTokens ?? 0) + inputTokens; + usage.inputTokens = inputTokens; } if (cacheReadTokens !== undefined) { - usage.cachedInputTokens = (usage.cachedInputTokens ?? 0) + cacheReadTokens; + usage.cachedInputTokens = cacheReadTokens; } if (outputTokens !== undefined) { - usage.outputTokens = (usage.outputTokens ?? 0) + outputTokens; + usage.outputTokens = outputTokens; } if (totalTokens > 0) { - usage.contextWindowUsedTokens = (usage.contextWindowUsedTokens ?? 0) + totalTokens; + usage.contextWindowUsedTokens = totalTokens; } if (cost !== undefined) { usage.totalCostUsd = (usage.totalCostUsd ?? 0) + cost; @@ -1144,6 +1144,13 @@ export function translateOpenCodeEvent( } } else if (partType === "step-finish") { mergeOpenCodeStepFinishUsage(state.accumulatedUsage, part); + if (hasNormalizedOpenCodeUsage(state.accumulatedUsage)) { + events.push({ + type: "usage_updated", + provider: "opencode", + usage: { ...state.accumulatedUsage }, + }); + } } break; } @@ -2041,6 +2048,17 @@ class OpenCodeAgentSession implements AgentSession { if (translatedEvent.type === "permission_requested") { this.pendingPermissions.set(translatedEvent.request.id, translatedEvent.request); } + if (translatedEvent.type === "turn_completed") { + if (hasNormalizedOpenCodeUsage(this.accumulatedUsage)) { + translatedEvent.usage = this.accumulatedUsage; + } + const contextWindowMaxTokens = + this.resolveSelectedModelContextWindowMaxTokens(); + this.accumulatedUsage = + contextWindowMaxTokens !== undefined + ? { contextWindowMaxTokens } + : {}; + } } return translated; diff --git a/packages/server/src/server/agent/providers/opencode/event-translator.test.ts b/packages/server/src/server/agent/providers/opencode/event-translator.test.ts index b68a4f009..72986fe28 100644 --- a/packages/server/src/server/agent/providers/opencode/event-translator.test.ts +++ b/packages/server/src/server/agent/providers/opencode/event-translator.test.ts @@ -255,6 +255,61 @@ describe("translateOpenCodeEvent", () => { ]); }); + it("emits usage_updated after step-finish parts", () => { + const state = createState(); + state.accumulatedUsage.contextWindowMaxTokens = 400_000; + + const events = translateOpenCodeEvent( + { + type: "message.part.updated", + properties: { + part: { + id: "step-finish-1", + sessionID: "session-1", + messageID: "message-usage-1", + type: "step-finish", + reason: "stop", + cost: 0.25, + tokens: { + total: 999_999, + input: 30_000, + output: 12_000, + reasoning: 10_000, + cache: { + read: 2_000, + write: 1_000, + }, + }, + }, + }, + }, + state, + ); + + expect(events).toEqual([ + { + type: "usage_updated", + provider: "opencode", + usage: { + contextWindowMaxTokens: 400_000, + contextWindowUsedTokens: 55_000, + cachedInputTokens: 2_000, + inputTokens: 30_000, + outputTokens: 12_000, + totalCostUsd: 0.25, + }, + }, + ]); + expect(state.accumulatedUsage).toEqual({ + contextWindowMaxTokens: 400_000, + contextWindowUsedTokens: 55_000, + cachedInputTokens: 2_000, + inputTokens: 30_000, + outputTokens: 12_000, + totalCostUsd: 0.25, + }); + }); + it("emits reasoning from message.part.delta events", () => { const state = createState(); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 1a2e4c845..dba2a9a86 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -120,6 +120,7 @@ import { buildVoiceAgentMcpServerConfig, buildVoiceModeSystemPrompt, stripVoiceModeSystemPrompt, + wrapSpokenInput, } from "./voice-config.js"; import { isVoicePermissionAllowed } from "./voice-permission-policy.js"; import { @@ -2712,6 +2713,7 @@ export class Session { messageId?: string, images?: Array<{ data: string; mimeType: string }>, runOptions?: AgentRunOptions, + options?: { spokenInput?: boolean }, ): Promise { this.sessionLogger.info( { agentId, textPreview: text.substring(0, 50), imageCount: images?.length ?? 0 }, @@ -2739,7 +2741,8 @@ export class Session { ); } - const prompt = this.buildAgentPrompt(text, images); + const promptText = options?.spokenInput ? wrapSpokenInput(text) : text; + const prompt = this.buildAgentPrompt(promptText, images); this.startAgentStream(agentId, prompt, runOptions); } @@ -7095,9 +7098,14 @@ export class Session { return; } - // Route voice utterances through the same send path as regular text input: - // interrupt-if-running, record message, then start a new stream. - await this.handleSendAgentMessage(agentId, result.text); + await this.handleSendAgentMessage( + agentId, + result.text, + undefined, + undefined, + undefined, + { spokenInput: true }, + ); await this.flushPendingAudioSegments("transcription complete"); } diff --git a/packages/server/src/server/session.workspace-git-watch.test.ts b/packages/server/src/server/session.workspace-git-watch.test.ts index ce12eb58c..e2dd5f54b 100644 --- a/packages/server/src/server/session.workspace-git-watch.test.ts +++ b/packages/server/src/server/session.workspace-git-watch.test.ts @@ -308,7 +308,8 @@ describe("workspace git watch targets", () => { workspaceDirectory: "/tmp/repo", }; - sessionAny.buildWorkspaceDescriptorMap = async () => new Map([["/tmp/repo", descriptor]]); + sessionAny.buildWorkspaceDescriptorMap = async () => + new Map([[descriptor.id, descriptor]]); await sessionAny.primeWorkspaceGitWatchFingerprints([descriptor]); @@ -554,7 +555,8 @@ describe("workspace git watch targets", () => { diffStat: { additions: 1, deletions: 0 }, }; - sessionAny.buildWorkspaceDescriptorMap = async () => new Map([["/tmp/repo", descriptor]]); + sessionAny.buildWorkspaceDescriptorMap = async () => + new Map([[descriptor.id, descriptor]]); await sessionAny.syncWorkspaceGitWatchTarget("/tmp/repo", { isGit: true }); sessionAny.primeWorkspaceGitWatchFingerprints([descriptor]); diff --git a/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts b/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts index c212d7204..87fe92b1f 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/silero-vad-session.ts @@ -8,10 +8,17 @@ import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; const DEFAULT_SAMPLE_RATE = 16000; const DEFAULT_BUFFER_SIZE_SECONDS = 60; const DEFAULT_SILERO_THRESHOLD = 0.5; -const DEFAULT_MIN_SILENCE_DURATION = 1.2; -const DEFAULT_MIN_SPEECH_DURATION = 0.2; const DEFAULT_WINDOW_SIZE = 512; +// Silero internal durations — kept low so isDetected() tracks actual sound. +// Our own state machine handles speech confirmation and end-of-speech detection. +const SILERO_MIN_SILENCE_DURATION = 0.2; +const SILERO_MIN_SPEECH_DURATION = 0.1; + +// Our boundary detection thresholds (in milliseconds). +const DEFAULT_CONFIRM_MS = 800; +const DEFAULT_SILENCE_MS = 1000; + type SherpaVadHandle = { acceptWaveform(samples: Float32Array): void; isDetected(): boolean; @@ -42,20 +49,30 @@ export interface SherpaSileroVadSessionConfig { modelPath?: string; sampleRate?: number; threshold?: number; - minSilenceDuration?: number; - minSpeechDuration?: number; windowSize?: number; bufferSizeInSeconds?: number; + confirmMs?: number; + silenceMs?: number; } +type VadPhase = + | { state: "idle" } + | { state: "confirming"; startedAt: number } + | { state: "speaking" } + | { state: "ending"; startedAt: number }; + export class SherpaSileroVadSession extends EventEmitter implements TurnDetectionSession { public readonly requiredSampleRate: number; private readonly vad: SherpaVadHandle; private readonly inputBuffer: SherpaCircularBufferHandle; private readonly windowSize: number; + private readonly msPerWindow: number; + private readonly confirmMs: number; + private readonly silenceMs: number; private connected = false; - private inSpeech = false; + private phase: VadPhase = { state: "idle" }; + private windowTimestamp = 0; private readonly logger; constructor(params: { @@ -67,16 +84,35 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio const config = params.config ?? {}; this.requiredSampleRate = config.sampleRate ?? DEFAULT_SAMPLE_RATE; this.windowSize = config.windowSize ?? DEFAULT_WINDOW_SIZE; + this.msPerWindow = (this.windowSize / this.requiredSampleRate) * 1000; + this.confirmMs = config.confirmMs ?? DEFAULT_CONFIRM_MS; + this.silenceMs = config.silenceMs ?? DEFAULT_SILENCE_MS; + + const threshold = config.threshold ?? DEFAULT_SILERO_THRESHOLD; + + this.logger.debug( + { + threshold, + sileroMinSilenceDuration: SILERO_MIN_SILENCE_DURATION, + sileroMinSpeechDuration: SILERO_MIN_SPEECH_DURATION, + confirmMs: this.confirmMs, + silenceMs: this.silenceMs, + windowSize: this.windowSize, + msPerWindow: this.msPerWindow, + sampleRate: this.requiredSampleRate, + }, + "[VAD] Initializing Silero VAD session", + ); const sherpa = loadSherpaOnnxNode() as unknown as SherpaVadModule; this.vad = new sherpa.Vad( { sileroVad: { model: config.modelPath ?? resolveBundledSileroVadModelPath(), - threshold: config.threshold ?? DEFAULT_SILERO_THRESHOLD, - minSilenceDuration: config.minSilenceDuration ?? DEFAULT_MIN_SILENCE_DURATION, - minSpeechDuration: config.minSpeechDuration ?? DEFAULT_MIN_SPEECH_DURATION, - windowSize: config.windowSize ?? DEFAULT_WINDOW_SIZE, + threshold, + minSilenceDuration: SILERO_MIN_SILENCE_DURATION, + minSpeechDuration: SILERO_MIN_SPEECH_DURATION, + windowSize: this.windowSize, }, sampleRate: this.requiredSampleRate, numThreads: 1, @@ -106,11 +142,14 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio try { const samples = pcm16leToFloat32(pcm16le, 1); this.inputBuffer.push(samples); + let windowsProcessed = 0; while (this.inputBuffer.size() > this.windowSize) { const window = this.inputBuffer.get(this.inputBuffer.head(), this.windowSize, false); this.inputBuffer.pop(this.windowSize); this.vad.acceptWaveform(window); - this.syncDetectionState(); + this.windowTimestamp += this.msPerWindow; + windowsProcessed++; + this.stepStateMachine(); } } catch (error) { this.emit("error", error instanceof Error ? error : new Error(String(error))); @@ -123,41 +162,111 @@ export class SherpaSileroVadSession extends EventEmitter implements TurnDetectio } try { + this.logger.debug( + { phase: this.phase.state }, + "[VAD] Flushing remaining audio", + ); this.vad.flush(); - this.syncDetectionState(); - if (this.inSpeech) { - this.inSpeech = false; + this.stepStateMachine(); + if (this.phase.state === "speaking" || this.phase.state === "ending") { + this.logger.debug("[VAD] Forcing speech_stopped after flush"); + this.phase = { state: "idle" }; this.emit("speech_stopped"); + } else if (this.phase.state === "confirming") { + this.logger.debug("[VAD] Discarding unconfirmed speech on flush"); + this.phase = { state: "idle" }; } } catch (error) { this.emit("error", error instanceof Error ? error : new Error(String(error))); } } - close(): void { + reset(): void { try { this.vad.reset(); this.inputBuffer.reset(); } catch { // ignore native cleanup failures } finally { - this.connected = false; - this.inSpeech = false; + this.phase = { state: "idle" }; } } - private syncDetectionState(): void { - const detected = this.vad.isDetected(); - if (detected && !this.inSpeech) { - this.inSpeech = true; - this.emit("speech_started"); - return; - } + close(): void { + this.reset(); + this.connected = false; + this.windowTimestamp = 0; + } - if (!detected && this.inSpeech && !this.vad.isEmpty()) { - this.logger.debug("Silero VAD marked end of speech"); - this.inSpeech = false; - this.emit("speech_stopped"); + private stepStateMachine(): void { + const detected = this.vad.isDetected(); + const now = this.windowTimestamp; + + switch (this.phase.state) { + case "idle": { + if (detected) { + this.logger.debug( + { now }, + "[VAD] idle → confirming (detection started)", + ); + this.phase = { state: "confirming", startedAt: now }; + } + break; + } + + case "confirming": { + if (!detected) { + const elapsed = now - this.phase.startedAt; + this.logger.debug( + { elapsed, confirmMs: this.confirmMs }, + "[VAD] confirming → idle (detection dropped before confirmation)", + ); + this.phase = { state: "idle" }; + break; + } + const elapsed = now - this.phase.startedAt; + if (elapsed >= this.confirmMs) { + this.logger.debug( + { elapsed, confirmMs: this.confirmMs }, + "[VAD] confirming → speaking (speech confirmed)", + ); + this.phase = { state: "speaking" }; + this.emit("speech_started"); + } + break; + } + + case "speaking": { + if (!detected) { + this.logger.debug( + { now }, + "[VAD] speaking → ending (silence started)", + ); + this.phase = { state: "ending", startedAt: now }; + } + break; + } + + case "ending": { + if (detected) { + this.logger.debug( + { elapsed: now - this.phase.startedAt }, + "[VAD] ending → speaking (speech resumed)", + ); + this.phase = { state: "speaking" }; + break; + } + const elapsed = now - this.phase.startedAt; + if (elapsed >= this.silenceMs) { + this.logger.debug( + { elapsed, silenceMs: this.silenceMs }, + "[VAD] ending → idle (speech stopped)", + ); + this.phase = { state: "idle" }; + this.emit("speech_stopped"); + } + break; + } } } } diff --git a/packages/server/src/server/speech/turn-detection-provider.ts b/packages/server/src/server/speech/turn-detection-provider.ts index 46fb6c386..d25501833 100644 --- a/packages/server/src/server/speech/turn-detection-provider.ts +++ b/packages/server/src/server/speech/turn-detection-provider.ts @@ -10,6 +10,7 @@ export type TurnDetectionSession = { connect(): Promise; appendPcm16(pcm16le: Buffer): void; flush(): void; + reset(): void; close(): void; on(event: "speech_started", handler: () => void): unknown; diff --git a/packages/server/src/server/voice-config.ts b/packages/server/src/server/voice-config.ts index 482e8197f..77b3a698a 100644 --- a/packages/server/src/server/voice-config.ts +++ b/packages/server/src/server/voice-config.ts @@ -53,6 +53,10 @@ export function buildVoiceModeSystemPrompt(existing: string | undefined, enabled .join("\n\n"); } +export function wrapSpokenInput(text: string): string { + return `\n${text}\n`; +} + export function buildVoiceAgentMcpServerConfig(params: { command: string; baseArgs: string[]; diff --git a/packages/server/src/server/voice/voice-turn-controller.test.ts b/packages/server/src/server/voice/voice-turn-controller.test.ts index 1a5446eb1..92901539b 100644 --- a/packages/server/src/server/voice/voice-turn-controller.test.ts +++ b/packages/server/src/server/voice/voice-turn-controller.test.ts @@ -19,6 +19,7 @@ class FakeTurnDetectionSession extends EventEmitter implements TurnDetectionSess } flush(): void {} + reset(): void {} close(): void {} } diff --git a/packages/server/src/server/voice/voice-turn-controller.ts b/packages/server/src/server/voice/voice-turn-controller.ts index b1eeac40e..3c2f0b76d 100644 --- a/packages/server/src/server/voice/voice-turn-controller.ts +++ b/packages/server/src/server/voice/voice-turn-controller.ts @@ -9,7 +9,7 @@ import { FixedDurationPcmRingBuffer } from "./fixed-duration-pcm-ring-buffer.js" const PCM_CHANNELS = 1; const PCM_BITS_PER_SAMPLE = 16; -const DEFAULT_PREFIX_DURATION_MS = 400; +const DEFAULT_PREFIX_DURATION_MS = 1000; type VoiceInputState = | { status: "idle" } @@ -134,6 +134,8 @@ export function createVoiceTurnController(params: { utteranceChunks = []; state = { status: "listening", rollingPrefixBytes: prefixBuffer.byteLength }; + detector.reset(); + await params.callbacks.onSpeechStopped(); params.logger.info( diff --git a/packages/website/package.json b/packages/website/package.json index 72a3e2419..e0deb6766 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/website", - "version": "0.1.49", + "version": "0.1.50", "private": true, "type": "module", "scripts": { diff --git a/packages/website/src/routes/__root.tsx b/packages/website/src/routes/__root.tsx index e887ac222..234ed8842 100644 --- a/packages/website/src/routes/__root.tsx +++ b/packages/website/src/routes/__root.tsx @@ -53,6 +53,12 @@ function RootDocument({ children }: Readonly<{ children: ReactNode }>) { +