From 5dc350034e803732d9406506bbd2970039205b3f Mon Sep 17 00:00:00 2001 From: "paseo-bot[bot]" <266920839+paseo-bot[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:05:12 +0000 Subject: [PATCH 01/32] Keep large Import Session lists responsive on Web (#7872) * Virtualize long Import Session Web lists while preserving native and short-list scrolling * fix(app): measure import list headers through the existing layout event Keep the virtualized import list usable in the existing component harness without introducing an observer mock. Import React for the unit JSX transform. Co-authored-by: royenheart --------- Co-authored-by: royenheart Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com> --- .../e2e/browser/import-session-flow.spec.ts | 137 ++++++++ .../app/e2e/support/helpers/import-session.ts | 34 +- .../src/components/import-session-list.tsx | 4 + .../components/import-session-list.types.ts | 13 + .../components/import-session-list.web.tsx | 141 ++++++++ .../src/components/import-session-sheet.tsx | 320 +++++++++++++----- 6 files changed, 555 insertions(+), 94 deletions(-) create mode 100644 packages/app/src/components/import-session-list.tsx create mode 100644 packages/app/src/components/import-session-list.types.ts create mode 100644 packages/app/src/components/import-session-list.web.tsx diff --git a/packages/app/e2e/browser/import-session-flow.spec.ts b/packages/app/e2e/browser/import-session-flow.spec.ts index c6d34f38f2..fe2f57c67a 100644 --- a/packages/app/e2e/browser/import-session-flow.spec.ts +++ b/packages/app/e2e/browser/import-session-flow.spec.ts @@ -229,6 +229,143 @@ test("captures the desktop import sheet and command-center entry", async ({ page }); }); +test("large import lists keep mounted rows bounded and remain scrollable", async ({ + page, +}, testInfo) => { + const fixtureDirectory = path.join(claudeConfigDirectory, "projects", "virtualization-fixture"); + await mkdir(fixtureDirectory, { recursive: true }); + try { + const newest = Date.now(); + for (let index = 0; index < 200; index += 1) { + const id = `virtualized-${String(index).padStart(3, "0")}`; + const file = path.join(fixtureDirectory, `${id}.jsonl`); + await writeFile( + file, + `${JSON.stringify({ + type: "user", + sessionId: id, + cwd: index % 2 === 0 ? scenario.projectRoot : scenario.worktreeDirectory, + message: { + role: "user", + content: `Virtualized session ${index}${index % 3 === 0 ? " with a longer preview that wraps onto a second line in a compact sheet" : ""}`, + }, + })}\n`, + ); + const timestamp = new Date(newest - index * 10); + await utimes(file, timestamp, timestamp); + } + for (const width of [1280, 390]) { + const flow = new ImportSessionFlow(page); + await flow.openWorkspace(scenario.project.workspaceId, { width, height: 844 }); + await flow.openGlobally(); + const rows = page.locator('[data-testid^="import-session-session-"]'); + const scrollToEnd = async (end = true) => { + // Scroll the body's real overflow element. Virtual rows need not exist yet. + await rows.first().evaluate((row, atEnd) => { + let element = row.parentElement; + while (element && !["auto", "scroll"].includes(getComputedStyle(element).overflowY)) { + element = element.parentElement; + } + if (!element) throw new Error("Import list has no scroll owner"); + element.scrollTop = atEnd ? element.scrollHeight : 0; + element.dispatchEvent(new Event("scroll")); + }, end); + }; + for (let pageIndex = 0; pageIndex < 3; pageIndex += 1) { + const loadMore = page.getByTestId("import-session-load-more"); + await expect(async () => { + await scrollToEnd(); + await expect(loadMore).toBeInViewport({ timeout: 500 }); + }).toPass({ timeout: 10_000 }); + await loadMore.click(); + if (pageIndex < 2) await expect(loadMore).toBeEnabled(); + } + await expect(async () => { + await scrollToEnd(); + await expect( + page.getByTestId("import-session-session-claude-virtualized-199"), + ).toBeInViewport({ timeout: 500 }); + }).toPass({ timeout: 10_000 }); + await expect.poll(() => rows.count()).toBeLessThan(100); + await page.screenshot({ path: testInfo.outputPath(`virtualized-${width}.png`) }); + if (width === 1280) { + const row = (index: number) => + page.getByTestId( + `import-session-session-claude-virtualized-${String(index).padStart(3, "0")}`, + ); + await expect(async () => { + await scrollToEnd(false); + await expect(row(0)).toBeInViewport({ timeout: 500 }); + }).toPass({ timeout: 10_000 }); + await row(0).focus(); + for (let index = 1; index <= 30; index += 1) { + await page.keyboard.press("Tab"); + await expect(row(index)).toBeFocused(); + } + for (let index = 29; index >= 15; index -= 1) { + await page.keyboard.press("Shift+Tab"); + await expect(row(index)).toBeFocused(); + } + const list = page.getByTestId("import-session-list"); + const scrollWithKey = async (event: "scroll" | "scrollend", key: string) => { + const listener = await list.evaluateHandle((node, eventName) => { + const promise = new Promise((resolve) => { + node.addEventListener(eventName, () => resolve(), { once: true }); + }); + return { promise }; + }, event); + try { + await page.keyboard.press(key); + await listener.evaluate((value) => value.promise); + } finally { + await listener.dispose(); + } + }; + const beforePageDown = await list.evaluate((node) => node.scrollTop); + await scrollWithKey("scroll", "PageDown"); + await expect + .poll(() => list.evaluate((node) => node.scrollTop)) + .toBeGreaterThan(beforePageDown); + await expect(row(15)).toBeFocused(); + await page.keyboard.press("End"); + await expect(row(15)).toBeFocused(); + await expect(row(199)).toBeInViewport(); + // Home must remain at the first row after the native page scroll ends. + await scrollWithKey("scrollend", "Home"); + await expect(row(0)).toBeInViewport(); + await expect(row(15)).toBeFocused(); + + await page.keyboard.press("End"); + await expect(row(199)).toBeInViewport(); + await row(199).focus(); + await scrollWithKey("scroll", "PageUp"); + await page.keyboard.press("Home"); + await expect(row(0)).toBeInViewport(); + await scrollWithKey("scrollend", "End"); + await expect(row(199)).toBeInViewport(); + await expect(row(199)).toBeFocused(); + await testInfo.attach("keyboard-scroll-owner", { + body: JSON.stringify( + await list.evaluate((node) => ({ + focusedRow: document.activeElement?.getAttribute("data-testid"), + scrollTop: node.scrollTop, + scrollHeight: node.scrollHeight, + clientHeight: node.clientHeight, + })), + ), + contentType: "application/json", + }); + await page.screenshot({ path: testInfo.outputPath("virtualized-keyboard-end.png") }); + await page.keyboard.press("Home"); + await expect(row(0)).toBeInViewport(); + } + await flow.close(); + } + } finally { + await rm(fixtureDirectory, { recursive: true, force: true }); + } +}); + async function capture(page: Page, testInfo: TestInfo, name: string): Promise { const outputPath = testInfo.outputPath(name); await page.screenshot({ path: outputPath, fullPage: true }); diff --git a/packages/app/e2e/support/helpers/import-session.ts b/packages/app/e2e/support/helpers/import-session.ts index 718b0d5965..9caabc5edd 100644 --- a/packages/app/e2e/support/helpers/import-session.ts +++ b/packages/app/e2e/support/helpers/import-session.ts @@ -27,7 +27,7 @@ export class ImportSessionFlow { await expectMobileAgentSidebarVisible(this.page); } await openGlobalNewWorkspaceComposer(this.page); - const button = this.page.getByTestId("new-workspace-import-session"); + const button = this.page.getByTestId("new-workspace-import-session").filter({ visible: true }); await expect(button).toHaveAccessibleName("Import session"); await expect(button).toBeInViewport(); } @@ -36,7 +36,7 @@ export class ImportSessionFlow { await this.openFromNewWorkspace(); } async openFromNewWorkspace() { - await this.page.getByTestId("new-workspace-import-session").click(); + await this.page.getByTestId("new-workspace-import-session").filter({ visible: true }).click(); await this.expectSheetReady(); } async openFromWorkspaceHeader() { @@ -86,7 +86,29 @@ export class ImportSessionFlow { expect((bounds?.x ?? width) + (bounds?.width ?? 1)).toBeLessThanOrEqual(width); } async revealSession(id: string) { - await this.page.getByTestId(rowTestId(id)).scrollIntoViewIfNeeded(); + const target = this.page.getByTestId(rowTestId(id)); + if ((await target.count()) === 0) { + await this.scrollList(false); + await expect(async () => { + if ((await target.count()) === 0) await this.scrollList(true); + await expect(target).toHaveCount(1, { timeout: 100 }); + }).toPass({ timeout: 10_000 }); + } + await target.scrollIntoViewIfNeeded(); + await expect(target).toBeInViewport(); + } + private async scrollList(forward: boolean) { + await this.page + .locator('[data-testid^="import-session-session-"]') + .first() + .evaluate((row, advance) => { + let element = row.parentElement; + while (element && !["auto", "scroll"].includes(getComputedStyle(element).overflowY)) + element = element.parentElement; + if (!element) throw new Error("Import list has no scroll owner"); + element.scrollTop = advance ? element.scrollTop + element.clientHeight * 0.8 : 0; + element.dispatchEvent(new Event("scroll")); + }, forward); } async search(query: string) { await this.page.getByTestId("import-session-search").fill(query); @@ -100,7 +122,8 @@ export class ImportSessionFlow { } async loadMore() { await this.page.getByTestId("import-session-load-more").click(); - await expect(this.page.getByText("Root session 20", { exact: true })).toBeVisible(); + await this.revealSession("fixture-root-20"); + await expect(this.page.getByText("Root session 20", { exact: true })).toBeInViewport(); await expect(this.page.getByTestId("import-session-load-more")).toHaveCount(0); } async retryProvider(provider: string, label: string) { @@ -111,6 +134,7 @@ export class ImportSessionFlow { await expect(this.page.getByRole("progressbar")).toHaveCount(0); } async importSession(id: string) { + await this.revealSession(id); await this.page.getByTestId(rowTestId(id)).click(); await expect(this.page.getByTestId("import-session-sheet")).toHaveCount(0, { timeout: 30_000, @@ -138,7 +162,7 @@ export class ImportSessionFlow { ); } async close() { - await this.page.keyboard.press("Escape"); + await this.page.getByRole("button", { name: "Close", exact: true }).last().click(); await expect(this.page.getByTestId("import-session-sheet")).toHaveCount(0); } async expectCommandCenterMatch() { diff --git a/packages/app/src/components/import-session-list.tsx b/packages/app/src/components/import-session-list.tsx new file mode 100644 index 0000000000..d2ff037651 --- /dev/null +++ b/packages/app/src/components/import-session-list.tsx @@ -0,0 +1,4 @@ +import type { ImportSessionListComponent } from "./import-session-list.types"; + +// Native keeps the sheet-owned ScrollView until its gestures and keyboard are verified. +export const ImportSessionList: ImportSessionListComponent | null = null; diff --git a/packages/app/src/components/import-session-list.types.ts b/packages/app/src/components/import-session-list.types.ts new file mode 100644 index 0000000000..e62b31bccb --- /dev/null +++ b/packages/app/src/components/import-session-list.types.ts @@ -0,0 +1,13 @@ +import type { ReactElement, ReactNode } from "react"; + +export interface ImportSessionListProps { + data: ReadonlyArray; + keyExtractor: (item: Item) => string; + renderItem: (item: Item) => ReactNode; + header: ReactElement | null; + footer: ReactElement | null; +} + +export type ImportSessionListComponent = ( + props: ImportSessionListProps, +) => ReactElement; diff --git a/packages/app/src/components/import-session-list.web.tsx b/packages/app/src/components/import-session-list.web.tsx new file mode 100644 index 0000000000..aa8f74cd7c --- /dev/null +++ b/packages/app/src/components/import-session-list.web.tsx @@ -0,0 +1,141 @@ +import React, { + useCallback, + useMemo, + useRef, + useState, + type CSSProperties, + type FocusEvent, + type KeyboardEvent, +} from "react"; +import { View, type LayoutChangeEvent } from "react-native"; +import { defaultRangeExtractor, useVirtualizer, type Range } from "@tanstack/react-virtual"; +import type { ImportSessionListProps } from "./import-session-list.types"; + +const SCROLLER_STYLE: CSSProperties = { + flex: 1, + minHeight: 0, + overflowY: "auto", + overflowX: "hidden", + overscrollBehavior: "contain", +}; +const CONTENT_STYLE: CSSProperties = { width: "100%", position: "relative" }; + +export function ImportSessionList({ + data, + keyExtractor, + renderItem, + header, + footer, +}: ImportSessionListProps) { + const scrollRef = useRef(null); + const [headerHeight, setHeaderHeight] = useState(0); + const [focusedKey, setFocusedKey] = useState(null); + const indexByKey = useMemo( + () => new Map(data.map((item, index) => [keyExtractor(item), index])), + [data, keyExtractor], + ); + const focusedIndex = focusedKey === null ? undefined : indexByKey.get(focusedKey); + const rangeExtractor = useCallback( + (range: Range) => { + const visible = defaultRangeExtractor(range); + if (focusedIndex === undefined) return visible; + // Keep focus and its Tab neighbours mounted when PageDown/End moves the window. + const focused = [focusedIndex - 1, focusedIndex, focusedIndex + 1].filter( + (index) => index >= 0 && index < data.length, + ); + return [...new Set([...visible, ...focused])].sort((left, right) => left - right); + }, + [data.length, focusedIndex], + ); + const handleFocus = useCallback((event: FocusEvent) => { + const row = event.target.closest("[data-session-key]"); + setFocusedKey(row?.dataset.sessionKey ?? null); + }, []); + const handleBlur = useCallback((event: FocusEvent) => { + if (!event.currentTarget.contains(event.relatedTarget)) setFocusedKey(null); + }, []); + const handleHeaderLayout = useCallback((event: LayoutChangeEvent) => { + setHeaderHeight(event.nativeEvent.layout.height); + }, []); + const getScrollElement = useCallback(() => scrollRef.current, []); + const getItemKey = useCallback( + (index: number) => keyExtractor(data[index]!), + [data, keyExtractor], + ); + const estimateSize = useCallback(() => 88, []); + const virtualizer = useVirtualizer({ + count: data.length, + getScrollElement, + getItemKey, + estimateSize, + scrollMargin: headerHeight, + overscan: 5, + rangeExtractor, + }); + const handleKeyDown = useCallback( + (event: KeyboardEvent) => { + const target = event.target; + if ( + event.defaultPrevented || + !(target instanceof HTMLElement) || + !event.currentTarget.contains(target) || + target.isContentEditable || + target.closest("input, textarea, select") + ) + return; + if (event.key === "End") { + event.preventDefault(); + // Reconcile the landing position as variable-height rows are measured. + virtualizer.scrollToIndex(data.length - 1, { align: "end" }); + } else if (event.key === "Home") { + event.preventDefault(); + virtualizer.scrollToOffset(0); + } else if (event.key === "PageDown" || event.key === "PageUp") { + event.preventDefault(); + // Native page-scroll animations can keep moving after Home or End. + const direction = event.key === "PageDown" ? 1 : -1; + virtualizer.scrollBy(direction * event.currentTarget.clientHeight); + } + }, + [data.length, virtualizer], + ); + const virtualRows = virtualizer.getVirtualItems(); + const totalSize = virtualizer.getTotalSize(); + const contentStyle = useMemo(() => ({ ...CONTENT_STYLE, height: totalSize }), [totalSize]); + const rowStyle = useCallback( + (offset: number): CSSProperties => ({ + ...CONTENT_STYLE, + position: "absolute", + top: 0, + transform: `translateY(${offset}px)`, + }), + [], + ); + + return ( +
+ {header} +
+ {virtualRows.map((row) => ( +
+ {renderItem(data[row.index]!)} +
+ ))} +
+
{footer}
+
+ ); +} diff --git a/packages/app/src/components/import-session-sheet.tsx b/packages/app/src/components/import-session-sheet.tsx index 3be15c0892..21296e54e0 100644 --- a/packages/app/src/components/import-session-sheet.tsx +++ b/packages/app/src/components/import-session-sheet.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import React, { type ReactNode, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Pressable, type PressableStateCallbackType, Text, View } from "react-native"; import { keepPreviousData, useMutation, useQueries, useQueryClient } from "@tanstack/react-query"; import { useTranslation } from "react-i18next"; @@ -11,6 +11,7 @@ import { ChevronDown, Inbox, Layers, RotateCw } from "lucide-react-native"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { AdaptiveModalSheet, type SheetHeader } from "@/components/adaptive-modal-sheet"; import { Button } from "@/components/ui/button"; +import { ImportSessionList } from "@/components/import-session-list"; import { LoadingSpinner } from "@/components/ui/loading-spinner"; import { Combobox, ComboboxItem, type ComboboxOption } from "@/components/ui/combobox"; import { useProviderIcon, useProviderIcons } from "@/components/provider-icons"; @@ -371,7 +372,33 @@ function ImportSessionActivityTime({ date }: { date: string }) { return {label}; } +function getSessionRowKey(entry: FetchRecentProviderSessionEntry): string { + return `${entry.providerId}:${entry.providerHandleId}`; +} + +function hasListHeader( + input: SheetStatusMessagesProps & { + showFilter: boolean; + hasProviderErrors: boolean; + }, +): boolean { + return ( + input.showFilter || + !input.isClientReady || + input.isSnapshotUnsupported || + input.hasNoImportableProviders || + (input.isLoadingSessions && !input.hasRows) || + input.importErrored || + input.hasProviderErrors + ); +} + function SessionRows({ + visible, + onClose, + header, + listHeader, + listFooter, serverId, entries, disabled, @@ -379,6 +406,11 @@ function SessionRows({ resolveFolder, onImportSession, }: { + visible: boolean; + onClose: () => void; + header: SheetHeader; + listHeader: ReactNode; + listFooter: ReactNode; serverId: string | null; entries: ReadonlyArray; disabled: boolean; @@ -386,20 +418,69 @@ function SessionRows({ resolveFolder: (entry: FetchRecentProviderSessionEntry) => string | null; onImportSession: (entry: FetchRecentProviderSessionEntry) => void; }) { + // Keep short results content-sized; only long lists need a bounded viewport. + const virtualized = ImportSessionList !== null && entries.length > 8; + const renderSessionRow = useCallback( + (entry: FetchRecentProviderSessionEntry) => ( + + ), + [serverId, disabled, importingSessionKey, resolveFolder, onImportSession], + ); + const renderItem = useCallback( + (item: FetchRecentProviderSessionEntry) => ( + {renderSessionRow(item)} + ), + [renderSessionRow], + ); + const virtualHeader = useMemo( + () => (listHeader ? {listHeader} : null), + [listHeader], + ); + const virtualFooter = useMemo( + () => {listFooter}, + [listFooter], + ); + return ( - - {entries.map((entry) => ( - - ))} - + + {ImportSessionList && virtualized ? ( + + + + ) : ( + <> + {listHeader} + {entries.length > 0 ? ( + {entries.map(renderSessionRow)} + ) : null} + {listFooter} + + )} + ); } @@ -726,85 +807,124 @@ export function ImportSessionSheet({ }); const showFilter = filterProviders.length > 1; const showLoadMore = hasMoreSessions(queries, pageLimit); - + const showListHeader = hasListHeader({ + showFilter, + isClientReady: Boolean(client), + isSnapshotUnsupported, + hasNoImportableProviders, + isLoadingSessions, + hasRows: visibleEntries.length > 0, + importErrored: importMutation.isError, + hasProviderErrors: providerErrorRows.length > 0, + }); + const listHeader = useMemo( + () => + showListHeader ? ( + + {showFilter ? ( + + + {selectedProvider === ALL_FILTER_VALUE ? ( + + ) : ( + + )} + + {selectedProviderLabel} + + + + + + ) : null} + 0} + importErrored={importMutation.isError} + /> + {providerErrorRows.length > 0 ? ( + + ) : null} + + ) : null, + [ + showListHeader, + showFilter, + filterTriggerStyle, + handleFilterOpen, + selectedProvider, + theme.colors.foregroundMuted, + SelectedProviderIcon, + selectedProviderLabel, + filterComboboxOptions, + handleFilterSelect, + renderFilterOption, + isFilterOpen, + client, + isSnapshotUnsupported, + hasNoImportableProviders, + isLoadingSessions, + visibleEntries.length, + importMutation.isError, + providerErrorRows, + handleRetryProvider, + ], + ); + const listFooter = useMemo( + () => ( + <> + {showLoadMore ? ( + + + + ) : null} + {showEmptyState ? : null} + + ), + [showLoadMore, handleLoadMore, isRefreshing, t, showEmptyState, emptyStateTitle], + ); return ( - - {showFilter ? ( - - - {selectedProvider === ALL_FILTER_VALUE ? ( - - ) : ( - - )} - - {selectedProviderLabel} - - - - - - ) : null} - 0} - importErrored={importMutation.isError} - /> - {providerErrorRows.length > 0 ? ( - - ) : null} - {visibleEntries.length > 0 ? ( - - ) : null} - {showLoadMore ? ( - - - - ) : null} - {showEmptyState ? : null} - + listHeader={listHeader} + listFooter={listFooter} + serverId={serverId} + entries={visibleEntries} + disabled={importMutation.isPending} + importingSessionKey={importingSessionKey} + resolveFolder={resolveFolder} + onImportSession={handleImportSession} + /> ); } @@ -853,9 +973,31 @@ const styles = StyleSheet.create((theme) => ({ fontSize: theme.fontSize.base, fontWeight: theme.fontWeight.medium, }, + listBody: { + gap: 0, + }, list: { + flex: 1, + minHeight: 0, + marginHorizontal: -theme.spacing[2], + }, + listHeader: { + marginHorizontal: theme.spacing[2], + paddingBottom: theme.spacing[4], + }, + listHeaderContent: { + gap: theme.spacing[4], + }, + smallList: { gap: theme.spacing[1], }, + listFooter: { + marginHorizontal: theme.spacing[2], + }, + listRow: { + marginHorizontal: theme.spacing[2], + paddingBottom: theme.spacing[1], + }, footer: { paddingTop: theme.spacing[2], alignItems: "center", From d0fd2e532059eeaec4c9e9e510fd65e1a3c6e1ed Mon Sep 17 00:00:00 2001 From: "paseo-bot[bot]" <266920839+paseo-bot[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:50:06 +0000 Subject: [PATCH 02/32] fix(client): close failed SSH tunnels without crashing callers (#7876) Route SSH stdin stream errors through the existing tunnel failure handler so pending writes cannot emit an unhandled EPIPE. Reproduce the peer abort using real OpenSSH in an isolated caller process. Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com> --- packages/client/src/node/ssh-tunnel.test.ts | 55 +++++++++++++++++++++ packages/client/src/node/ssh-tunnel.ts | 6 ++- 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 packages/client/src/node/ssh-tunnel.test.ts diff --git a/packages/client/src/node/ssh-tunnel.test.ts b/packages/client/src/node/ssh-tunnel.test.ts new file mode 100644 index 0000000000..6bdfe27fe4 --- /dev/null +++ b/packages/client/src/node/ssh-tunnel.test.ts @@ -0,0 +1,55 @@ +import { execFile, spawnSync } from "node:child_process"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const run = promisify(execFile); +const hasSsh = !spawnSync("ssh", ["-V"]).error; + +test.skipIf(!hasSsh)( + "an SSH peer abort during a pending write closes the tunnel without crashing its caller", + async () => { + // Isolate the caller so an unhandled stream error fails this assertion instead + // of crashing the test worker. Both the TCP peer and OpenSSH are real. + const { stdout } = await run( + process.execPath, + [ + "--import", + "tsx", + "--input-type=module", + "-e", + ` + import { createServer, createConnection } from 'node:net'; + import { once } from 'node:events'; + import { createSshTunnel } from ${JSON.stringify(new URL("./ssh-tunnel.ts", import.meta.url).href)}; + + const remote = createServer(socket => socket.destroy()); + remote.listen(0, '127.0.0.1'); + await once(remote, 'listening'); + const tunnel = await createSshTunnel({ + host: '127.0.0.1', sshPort: remote.address().port, daemonPort: 6767, + }); + const [host, port] = tunnel.endpoint.split(':'); + const client = createConnection({ host, port: Number(port) }); + client.on('error', () => undefined); + try { + const closed = new Promise(resolve => client.once('close', resolve)); + await once(client, 'connect'); + client.write(Buffer.alloc(8 * 1024 * 1024, 65)); + await closed; + console.log(JSON.stringify({ closed: client.destroyed, failure: tunnel.failureDetail() })); + } finally { + client.destroy(); + tunnel.close(); + await new Promise(resolve => remote.close(resolve)); + } + `, + ], + { timeout: 10_000 }, + ); + expect(JSON.parse(stdout)).toEqual({ + closed: true, + failure: expect.any(String), + }); + expect(JSON.parse(stdout).failure).not.toBe(""); + }, +); diff --git a/packages/client/src/node/ssh-tunnel.ts b/packages/client/src/node/ssh-tunnel.ts index 33900d0baa..a67c0a91c4 100644 --- a/packages/client/src/node/ssh-tunnel.ts +++ b/packages/client/src/node/ssh-tunnel.ts @@ -54,10 +54,12 @@ export function createSshTunnel(target: SshTransportTarget): Promise child.stderr.on("data", (chunk: Buffer | string) => { stderr = `${stderr}${chunk.toString()}`.slice(-SSH_STDERR_LIMIT); }); - child.on("error", (error) => { + const fail = (error: Error) => { failure = error.message; acceptedSocket.destroy(error); - }); + }; + child.on("error", fail); + child.stdin.on("error", fail); child.on("exit", (code, signal) => { if (code !== 0 || signal) failure = formatSshFailure(stderr, code, signal); acceptedSocket.destroy(failure ? new Error(failure) : undefined); From 8c9499dac61927b0814a2bc3b9646e7eb3534f7c Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 10:48:07 +0700 Subject: [PATCH 03/32] fix(app): dismiss file link tooltips when leaving their panel (#7842) Retained chats suspend while their tooltip portals remain mounted. Remove the surface before suspension and clear open state and pending hover timers so returning to the chat does not revive it. --- .../assistant-file-link-tooltip.spec.ts | 69 +++++++++++++++++++ packages/app/src/components/ui/tooltip.tsx | 14 +++- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 packages/app/e2e/browser/assistant-file-link-tooltip.spec.ts diff --git a/packages/app/e2e/browser/assistant-file-link-tooltip.spec.ts b/packages/app/e2e/browser/assistant-file-link-tooltip.spec.ts new file mode 100644 index 0000000000..69cccc52dd --- /dev/null +++ b/packages/app/e2e/browser/assistant-file-link-tooltip.spec.ts @@ -0,0 +1,69 @@ +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import { test, expect, type Page } from "../support/fixtures"; +import { openAgentRoute, seedMockAgentWorkspace } from "../support/helpers/mock-agent"; + +async function readFileLinkTooltip(page: Page): Promise { + const link = page.getByRole("link", { name: "the guide", exact: true }); + await link.hover(); + await expect(page.getByRole("tooltip")).toContainText("guide.md"); +} + +async function leaveFileLink(page: Page): Promise { + await page.mouse.move(5, 5); + await expect(page.getByRole("tooltip")).toHaveCount(0); +} + +async function openLinkedFile(page: Page): Promise { + await page.getByRole("link", { name: "the guide", exact: true }).click(); + await expect(page.getByTestId("markdown-preview-frame")).toContainText("Example guide"); + await leaveFileLink(page); +} + +async function returnToAgent(page: Page): Promise { + await page.getByRole("button", { name: "Tooltip regression", exact: true }).click(); + await expect(page.getByRole("link", { name: "the guide", exact: true })).toBeVisible(); + await leaveFileLink(page); +} + +async function switchToFileTab(page: Page): Promise { + await page.getByRole("button", { name: "guide.md", exact: true }).click(); + await expect(page.getByTestId("markdown-preview-frame")).toContainText("Example guide"); + await leaveFileLink(page); +} + +test("opening an assistant file link dismisses its tooltip over the file tab", async ({ + page, +}, testInfo) => { + testInfo.setTimeout(120_000); + const agent = await seedMockAgentWorkspace({ + repoPrefix: "tooltip-5811-", + title: "Tooltip regression", + featureValues: { + mockStreamingAssistantResponse: "Read [the guide](./guide.md).", + mockStreamingAssistantIntervalMs: 5, + }, + }); + try { + await writeFile( + path.join(agent.cwd, "guide.md"), + "# Example guide\n\n" + "Plain paragraph.\n\n".repeat(150), + ); + await openAgentRoute(page, agent); + await agent.client.sendAgentMessage(agent.agentId, "Show the file link."); + await agent.client.waitForFinish(agent.agentId, 30_000); + + await readFileLinkTooltip(page); + await leaveFileLink(page); + await readFileLinkTooltip(page); + await openLinkedFile(page); + await returnToAgent(page); + await readFileLinkTooltip(page); + await switchToFileTab(page); + await returnToAgent(page); + await readFileLinkTooltip(page); + await leaveFileLink(page); + } finally { + await agent.cleanup(); + } +}); diff --git a/packages/app/src/components/ui/tooltip.tsx b/packages/app/src/components/ui/tooltip.tsx index ae00768a34..697a80fdcf 100644 --- a/packages/app/src/components/ui/tooltip.tsx +++ b/packages/app/src/components/ui/tooltip.tsx @@ -27,6 +27,7 @@ import { FadeIn, FadeOut } from "react-native-reanimated"; import { StyleSheet } from "react-native-unistyles"; import { useIsCompactFormFactor } from "@/constants/layout"; import { FloatingSurface } from "@/components/ui/floating"; +import { useRetainedPanelActive } from "@/components/retained-panel"; import { isNative, isWeb } from "@/constants/platform"; import { getOverlayRoot, OVERLAY_Z } from "@/lib/overlay-root"; @@ -256,8 +257,15 @@ export function Tooltip({ }); const isCompact = useIsCompactFormFactor(); + const active = useRetainedPanelActive(); const opensOnPress = isNative || isCompact; - const enabled = opensOnPress ? enabledOnMobile : enabledOnDesktop; + const enabled = active && (opensOnPress ? enabledOnMobile : enabledOnDesktop); + + // Remove the portal in the inactive commit, before the retained chat freezes. + // Closing also prevents the tooltip from returning when its panel is shown again. + useEffect(() => { + if (!active && isOpen) setIsOpen(false); + }, [active, isOpen, setIsOpen]); const value = useMemo( () => ({ @@ -329,6 +337,10 @@ export function TooltipTrigger({ }; }, [clearOpenTimer]); + useEffect(() => { + if (!ctx.enabled) clearOpenTimer(); + }, [ctx.enabled, clearOpenTimer]); + const handleHoverIn = useCallback( (e?: unknown) => { if (isCallable(onHoverIn)) onHoverIn(e); From 4bbacfed4652e0d44677dc99f5b86836384cca2b Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 10:50:16 +0700 Subject: [PATCH 04/32] feat(app): fold finished agent turns behind a summary line (#7785) Long agent turns bury the answer under intermediate messages, thoughts and tool calls. A new Chat display preference (Full / Collapse tool calls / Collapse turns) replaces the tool call detail level. Collapse turns folds everything a finished turn did before its final message into one summary line under the prompt; opening it shows those rows inline behind the same indent rail that expanded tool call groups now use. Expanded turns stay real timeline rows so virtualization, chat find and scroll-to-message keep working; find and outline jumps open the turn that holds their target. Opening or closing a turn stops following output so the summary stays under the pointer. Collapse turns is the default for new installs only. Existing settings were materialized on first launch, so stored blobs without the new field map from the legacy tool call detail level and otherwise stay on Full. Co-authored-by: Ready <127541578+akaready@users.noreply.github.com> Co-authored-by: Waishnav <86405648+Waishnav@users.noreply.github.com> Co-authored-by: Shreyan Dey <90114886+Dey11@users.noreply.github.com> Co-authored-by: Sebastian Parfitt <7753735+Sebeb@users.noreply.github.com> --- packages/app/e2e/browser/changes-pane.spec.ts | 4 +- .../browser/tool-call-overview-sheet.spec.ts | 2 +- .../app/e2e/browser/tool-call-shimmer.spec.ts | 2 +- .../app/e2e/browser/turn-collapse.spec.ts | 67 +++++++++ packages/app/e2e/support/fixtures.ts | 5 + .../e2e/support/helpers/plugin-timeline.ts | 2 +- .../support/helpers/sidebar-nav-settings.ts | 5 +- .../chat-find/viewport.browser.test.ts | 1 + .../chat-outline/use-chat-outline.test.tsx | 3 + .../app/src/agent-stream/presentation.test.ts | 9 +- packages/app/src/agent-stream/presentation.ts | 80 ++++++++-- .../app/src/agent-stream/strategy-native.tsx | 1 + .../app/src/agent-stream/strategy-web.tsx | 9 +- packages/app/src/agent-stream/strategy.ts | 2 + .../agent-stream/turn-collapse/model.test.ts | 134 +++++++++++++++++ .../src/agent-stream/turn-collapse/model.ts | 122 +++++++++++++++ .../turn-collapse/summary-line.tsx | 48 ++++++ .../agent-stream/use-scroll-to-message.web.ts | 3 +- .../agent-stream/use-stream-history-window.ts | 3 +- packages/app/src/agent-stream/view.tsx | 140 ++++++++++++++++-- packages/app/src/components/indent-rail.tsx | 32 ++++ packages/app/src/components/message.tsx | 9 +- packages/app/src/hooks/use-settings/index.ts | 2 + .../src/hooks/use-settings/storage.test.ts | 37 ++++- .../app/src/hooks/use-settings/storage.ts | 21 ++- packages/app/src/i18n/resources/ar.ts | 17 ++- packages/app/src/i18n/resources/en.ts | 17 ++- packages/app/src/i18n/resources/es.ts | 17 ++- packages/app/src/i18n/resources/fr.ts | 17 ++- packages/app/src/i18n/resources/ja.ts | 17 ++- packages/app/src/i18n/resources/ko.ts | 17 ++- packages/app/src/i18n/resources/pt-BR.ts | 17 ++- packages/app/src/i18n/resources/ru.ts | 17 ++- packages/app/src/i18n/resources/zh-CN.ts | 17 ++- .../screens/settings/chat/chat-section.tsx | 28 ++-- .../tool-calls/detail-level/overview/label.ts | 45 ++++++ .../tool-calls/detail-level/overview/model.ts | 19 ++- .../tool-calls/detail-level/overview/view.tsx | 46 +----- .../src/tool-calls/detail-level/projection.ts | 4 +- packages/app/src/types/stream.ts | 6 + 40 files changed, 874 insertions(+), 170 deletions(-) create mode 100644 packages/app/e2e/browser/turn-collapse.spec.ts create mode 100644 packages/app/src/agent-stream/turn-collapse/model.test.ts create mode 100644 packages/app/src/agent-stream/turn-collapse/model.ts create mode 100644 packages/app/src/agent-stream/turn-collapse/summary-line.tsx create mode 100644 packages/app/src/components/indent-rail.tsx create mode 100644 packages/app/src/tool-calls/detail-level/overview/label.ts diff --git a/packages/app/e2e/browser/changes-pane.spec.ts b/packages/app/e2e/browser/changes-pane.spec.ts index 61173613ea..1ff6059460 100644 --- a/packages/app/e2e/browser/changes-pane.spec.ts +++ b/packages/app/e2e/browser/changes-pane.spec.ts @@ -1415,12 +1415,14 @@ test("collapsing and expanding restores the real horizontal scroll offset", asyn async function useCodeFont(page: Page, codeFontSize: number): Promise { await page.addInitScript( ({ settingsKey, fontSize }) => { - if (localStorage.getItem(settingsKey)) { + const settings = JSON.parse(localStorage.getItem(settingsKey) ?? "{}"); + if (settings.codeFontSize !== undefined) { return; } localStorage.setItem( settingsKey, JSON.stringify({ + ...settings, theme: "dark", sendBehavior: "interrupt", serviceUrlBehavior: "ask", diff --git a/packages/app/e2e/browser/tool-call-overview-sheet.spec.ts b/packages/app/e2e/browser/tool-call-overview-sheet.spec.ts index 927585b545..42831fad8a 100644 --- a/packages/app/e2e/browser/tool-call-overview-sheet.spec.ts +++ b/packages/app/e2e/browser/tool-call-overview-sheet.spec.ts @@ -143,7 +143,7 @@ async function configureOverviewToolCalls(page: Page): Promise { await page.addInitScript(() => { localStorage.setItem( "@paseo:app-settings", - JSON.stringify({ toolCallDetailLevel: "overview" }), + JSON.stringify({ chatDisplay: "collapseToolCalls" }), ); }); } diff --git a/packages/app/e2e/browser/tool-call-shimmer.spec.ts b/packages/app/e2e/browser/tool-call-shimmer.spec.ts index 71ef6555e6..40dde0aa99 100644 --- a/packages/app/e2e/browser/tool-call-shimmer.spec.ts +++ b/packages/app/e2e/browser/tool-call-shimmer.spec.ts @@ -161,7 +161,7 @@ test("measures an overview heading that becomes loading after its idle mount", a await page.addInitScript(() => { localStorage.setItem( "@paseo:app-settings", - JSON.stringify({ toolCallDetailLevel: "overview" }), + JSON.stringify({ chatDisplay: "collapseToolCalls" }), ); }); const agent = await seedMockAgentWorkspace({ diff --git a/packages/app/e2e/browser/turn-collapse.spec.ts b/packages/app/e2e/browser/turn-collapse.spec.ts new file mode 100644 index 0000000000..1deb050fa8 --- /dev/null +++ b/packages/app/e2e/browser/turn-collapse.spec.ts @@ -0,0 +1,67 @@ +import type { Page } from "@playwright/test"; +import { test, expect } from "../support/fixtures"; +import { expectComposerVisible } from "../support/helpers/composer"; +import { openAgentRoute, seedMockAgentWorkspace } from "../support/helpers/mock-agent"; + +type ChatDisplay = "full" | "collapseToolCalls" | "collapseTurns"; + +async function useChatDisplay(page: Page, chatDisplay: ChatDisplay): Promise { + await page.addInitScript((value) => { + localStorage.setItem("@paseo:app-settings", JSON.stringify({ chatDisplay: value })); + }, chatDisplay); +} + +async function seedAnsweredAgent(title: string) { + const agent = await seedMockAgentWorkspace({ + repoPrefix: "turn-collapse-", + title, + model: "ten-second-stream", + initialPrompt: "Why does the chat stop following new output?", + }); + await agent.client.waitForFinish(agent.agentId, 30_000); + await agent.client.sendAgentMessage(agent.agentId, "Make the fix and run the burst script."); + await agent.client.waitForFinish(agent.agentId, 30_000); + return agent; +} + +/** Chat rows between the latest turn's summary line and its reply, in document order. */ +async function rowsBetweenLatestSummaryAndReply(page: Page): Promise { + return page.evaluate(() => { + const rows = [...document.querySelectorAll("[data-message-id]")]; + const summaryRow = rows.findLastIndex((row) => row.querySelector("[data-testid=turn-summary]")); + const replyRow = rows.findLastIndex((row) => + row.querySelector("[data-testid=assistant-message]"), + ); + return replyRow - summaryRow - 1; + }); +} + +test.describe("Turn collapse", () => { + test("folds each answered turn behind a summary line that opens it", async ({ page }) => { + test.setTimeout(120_000); + await useChatDisplay(page, "collapseTurns"); + const agent = await seedAnsweredAgent("Turn collapse"); + try { + await openAgentRoute(page, agent); + await expectComposerVisible(page); + + const summary = page.getByTestId("turn-summary").last().getByRole("button"); + await expect(summary).toHaveText(/^Wrote \d+ messages?, edited 1 file/); + // Text chunks that stream back to back merge into one message, so the reply holds + // at least the stream's last chunk. + await expect(page.getByTestId("assistant-message").last()).toHaveText( + /\(end of synthetic stream\)$/, + ); + await expect.poll(() => rowsBetweenLatestSummaryAndReply(page)).toBe(0); + + await summary.click(); + await expect.poll(() => rowsBetweenLatestSummaryAndReply(page)).toBeGreaterThan(0); + await expect(summary).toBeInViewport(); + + await summary.click(); + await expect.poll(() => rowsBetweenLatestSummaryAndReply(page)).toBe(0); + } finally { + await agent.cleanup(); + } + }); +}); diff --git a/packages/app/e2e/support/fixtures.ts b/packages/app/e2e/support/fixtures.ts index c621f43fce..89cfb6937d 100644 --- a/packages/app/e2e/support/fixtures.ts +++ b/packages/app/e2e/support/fixtures.ts @@ -180,6 +180,11 @@ const test = daemonTest.extend<{ localStorage.setItem("@paseo:daemon-registry", JSON.stringify([daemon, ...extraHosts])); localStorage.removeItem("@paseo:settings"); localStorage.setItem("@paseo:create-agent-preferences", JSON.stringify(preferences)); + // New installs collapse finished turns; specs assert on the rows inside them, so they + // start from the full chat unless they seed their own settings. + if (!localStorage.getItem("@paseo:app-settings")) { + localStorage.setItem("@paseo:app-settings", JSON.stringify({ chatDisplay: "full" })); + } }, { daemon: testDaemon, diff --git a/packages/app/e2e/support/helpers/plugin-timeline.ts b/packages/app/e2e/support/helpers/plugin-timeline.ts index 145addc2ef..533b546022 100644 --- a/packages/app/e2e/support/helpers/plugin-timeline.ts +++ b/packages/app/e2e/support/helpers/plugin-timeline.ts @@ -116,7 +116,7 @@ export async function withTimelinePlugin( await page.addInitScript(() => { localStorage.setItem( "@paseo:app-settings", - JSON.stringify({ toolCallDetailLevel: "overview" }), + JSON.stringify({ chatDisplay: "collapseToolCalls" }), ); }); const agent = await seedMockAgentWorkspace({ diff --git a/packages/app/e2e/support/helpers/sidebar-nav-settings.ts b/packages/app/e2e/support/helpers/sidebar-nav-settings.ts index e3eff916e7..2037783ded 100644 --- a/packages/app/e2e/support/helpers/sidebar-nav-settings.ts +++ b/packages/app/e2e/support/helpers/sidebar-nav-settings.ts @@ -63,8 +63,9 @@ export async function seedSidebarFooterPreferences( ): Promise { await page.addInitScript( ({ key, sidebarFooterItems }) => { - if (localStorage.getItem(key)) return; - localStorage.setItem(key, JSON.stringify({ sidebarFooterItems })); + const settings = JSON.parse(localStorage.getItem(key) ?? "{}"); + if (settings.sidebarFooterItems) return; + localStorage.setItem(key, JSON.stringify({ ...settings, sidebarFooterItems })); }, { key: APP_SETTINGS_KEY, sidebarFooterItems: preferences }, ); diff --git a/packages/app/src/agent-stream/chat-find/viewport.browser.test.ts b/packages/app/src/agent-stream/chat-find/viewport.browser.test.ts index a7cf29994c..1b4b8d18e8 100644 --- a/packages/app/src/agent-stream/chat-find/viewport.browser.test.ts +++ b/packages/app/src/agent-stream/chat-find/viewport.browser.test.ts @@ -24,6 +24,7 @@ function viewport(root: HTMLElement, capture: Capture) { const handle: StreamViewportHandle = { scrollToBottom() {}, prepareForViewportChange() {}, + holdReadingPosition() {}, scrollToMessage(_messageId, occurrence) { if (occurrence) capture.occurrence = occurrence; }, diff --git a/packages/app/src/agent-stream/chat-outline/use-chat-outline.test.tsx b/packages/app/src/agent-stream/chat-outline/use-chat-outline.test.tsx index bd5a73ce94..890f6eaae7 100644 --- a/packages/app/src/agent-stream/chat-outline/use-chat-outline.test.tsx +++ b/packages/app/src/agent-stream/chat-outline/use-chat-outline.test.tsx @@ -265,6 +265,7 @@ describe("useChatOutline", () => { const viewport: StreamViewportHandle = { scrollToBottom: vi.fn(), prepareForViewportChange: vi.fn(), + holdReadingPosition: vi.fn(), scrollToMessage, }; const viewportRef = { current: viewport }; @@ -316,6 +317,7 @@ describe("useChatOutline", () => { current: { scrollToBottom: vi.fn(), prepareForViewportChange: vi.fn(), + holdReadingPosition: vi.fn(), scrollToMessage, }, }; @@ -371,6 +373,7 @@ describe("useChatOutline", () => { const viewport: StreamViewportHandle = { scrollToBottom: vi.fn(), prepareForViewportChange: vi.fn(), + holdReadingPosition: vi.fn(), scrollToMessage, }; const livePrompt = { diff --git a/packages/app/src/agent-stream/presentation.test.ts b/packages/app/src/agent-stream/presentation.test.ts index ef3a30e5c5..357b6c079b 100644 --- a/packages/app/src/agent-stream/presentation.test.ts +++ b/packages/app/src/agent-stream/presentation.test.ts @@ -4,6 +4,7 @@ import { transformTimelineItem, type TimelineItemTransform } from "@/plugins/tim import type { InstalledPlugin } from "@/plugins/types"; import { applyStreamEvent, + getStreamItemMessageId, hydrateStreamState, type StreamItem, type ToolCallItem, @@ -13,7 +14,7 @@ import type { AgentStreamEventPayload } from "@getpaseo/protocol/messages"; import { QueryClient } from "@tanstack/react-query"; import { describe, expect, it } from "vitest"; import { buildAgentStreamRenderModel } from "./model"; -import { createStreamPresentation, getStreamItemMessageId } from "./presentation"; +import { createStreamPresentation } from "./presentation"; const runtime = { paseo: {}, @@ -97,7 +98,11 @@ function pluginData(items: StreamItem[]): unknown[] { return items.flatMap((item) => (item.kind === "plugin" ? [item.data] : [])); } -const presentationOptions = { level: "overview" as const, isTurnActive: true }; +const presentationOptions = { + display: "collapseToolCalls" as const, + expandedTurnHostIds: new Set(), + isTurnActive: true, +}; function streamHarness(transform?: TimelineItemTransform) { let state: { tail: StreamItem[]; head: StreamItem[] } = { tail: [], head: [] }; diff --git a/packages/app/src/agent-stream/presentation.ts b/packages/app/src/agent-stream/presentation.ts index 4459486a7b..9e8e5b430e 100644 --- a/packages/app/src/agent-stream/presentation.ts +++ b/packages/app/src/agent-stream/presentation.ts @@ -1,3 +1,4 @@ +import type { ChatDisplay } from "@/hooks/use-settings"; import type { TimelineItemTransform } from "@/plugins/timeline/model"; import { projectPluginTimelineItems } from "@/plugins/timeline/projection"; import { @@ -5,23 +6,32 @@ import { projectToolCallDetailLevel, type PreparedToolCallHistory, type ToolCallDetailLevel, + type ToolCallDetailProjection, } from "@/tool-calls/detail-level/projection"; import type { StreamItem, UserMessageItem } from "@/types/stream"; +import { collapseTurns, type TurnCollapse } from "./turn-collapse/model"; interface PresentationInput { tail: StreamItem[]; head: StreamItem[]; transform: TimelineItemTransform | undefined; - level: ToolCallDetailLevel; + display: ChatDisplay; + /** User messages whose collapsed turn the reader opened. */ + expandedTurnHostIds: ReadonlySet; isTurnActive: boolean; } -/** Find and scrolling address a plugin row by the source it replaces. */ -export function getStreamItemMessageId(item: StreamItem): string { - if (item.kind === "plugin") return item.sourceId ?? item.id; - return item.id; +export interface StreamPresentation extends ToolCallDetailProjection { + turns: Omit; } +const NO_COLLAPSED_TURNS: StreamPresentation["turns"] = collapseTurns({ + tail: [], + isLatestTurnOpen: false, + expandedHostIds: new Set(), + getGroupedCalls: () => undefined, +}); + /** Source messages reach plugins before display projection. */ export function createStreamPresentation() { const userMessageCache = new WeakMap(); @@ -47,12 +57,47 @@ export function createStreamPresentation() { let preparedTail: StreamItem[] | undefined; let preparedLevel: ToolCallDetailLevel | undefined; let preparedHistory: PreparedToolCallHistory | null = null; + let collapsedSource: StreamItem[] | undefined; + let collapsedIsLatestTurnOpen = false; + let collapsedExpandedHostIds: ReadonlySet | undefined; + let collapsed: TurnCollapse | undefined; function presentRows(item: StreamItem): StreamItem[] { return [item.kind === "user_message" ? presentUserMessage(item) : item]; } - return (input: PresentationInput) => { + function collapse( + projection: ToolCallDetailProjection, + input: PresentationInput, + ): StreamPresentation { + if (input.display !== "collapseTurns") { + return { ...projection, turns: NO_COLLAPSED_TURNS }; + } + // Live text updates reach the head only, so the folded history is reused until the + // retained rows, the latest turn's liveness, or the reader's expanded turns change. + const isLatestTurnOpen = input.isTurnActive || projection.head.length > 0; + if ( + !collapsed || + collapsedSource !== projection.tail || + collapsedIsLatestTurnOpen !== isLatestTurnOpen || + collapsedExpandedHostIds !== input.expandedTurnHostIds + ) { + collapsed = collapseTurns({ + tail: projection.tail, + isLatestTurnOpen, + expandedHostIds: input.expandedTurnHostIds, + getGroupedCalls: (itemId) => projection.groupsByHostId.get(itemId)?.run.calls, + }); + collapsedSource = projection.tail; + collapsedIsLatestTurnOpen = isLatestTurnOpen; + collapsedExpandedHostIds = input.expandedTurnHostIds; + } + const { tail, ...turns } = collapsed; + return { ...projection, tail, turns }; + } + + return (input: PresentationInput): StreamPresentation => { + const level: ToolCallDetailLevel = input.display === "full" ? "detailed" : "overview"; // Retained history is not reprojected or regrouped on each live text update. if (historySource !== input.tail || historyTransform !== input.transform) { historyRows = projectPluginTimelineItems(input.tail, input.transform).flatMap( @@ -66,17 +111,20 @@ export function createStreamPresentation() { const blocks = presentRows(item); head.push(...blocks); } - if (preparedTail !== historyRows || preparedLevel !== input.level) { - preparedHistory = prepareToolCallHistory(input.level, historyRows); + if (preparedTail !== historyRows || preparedLevel !== level) { + preparedHistory = prepareToolCallHistory(level, historyRows); preparedTail = historyRows; - preparedLevel = input.level; + preparedLevel = level; } - return projectToolCallDetailLevel({ - level: input.level, - tail: historyRows, - head, - preparedHistory, - isTurnActive: input.isTurnActive, - }); + return collapse( + projectToolCallDetailLevel({ + level, + tail: historyRows, + head, + preparedHistory, + isTurnActive: input.isTurnActive, + }), + input, + ); }; } diff --git a/packages/app/src/agent-stream/strategy-native.tsx b/packages/app/src/agent-stream/strategy-native.tsx index e7568edfe8..7dc6e8d521 100644 --- a/packages/app/src/agent-stream/strategy-native.tsx +++ b/packages/app/src/agent-stream/strategy-native.tsx @@ -361,6 +361,7 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat bottomAnchorController.prepareForStickyViewportChange(); markNativeViewportSettling(); }, + holdReadingPosition: () => bottomAnchorController.detachByUser(), }; viewportRef.current = handle; return () => { diff --git a/packages/app/src/agent-stream/strategy-web.tsx b/packages/app/src/agent-stream/strategy-web.tsx index e57fc98e48..8e8bdf7680 100644 --- a/packages/app/src/agent-stream/strategy-web.tsx +++ b/packages/app/src/agent-stream/strategy-web.tsx @@ -20,7 +20,7 @@ import { withUnistyles } from "react-native-unistyles"; import { useRetainedPanelActive } from "@/components/retained-panel"; import { LoadingSpinner } from "@/components/ui/loading-spinner"; import { useStableEvent } from "@/hooks/use-stable-event"; -import type { StreamItem } from "@/types/stream"; +import { getStreamItemMessageId, type StreamItem } from "@/types/stream"; import type { Theme } from "@/styles/theme"; import { WEB_SCROLLBAR_SIZE_PX } from "@/styles/web-scrollbar"; import { DomOverlayScrollbar } from "@/components/ui/overlay-scrollbar/dom-overlay-scrollbar"; @@ -44,7 +44,6 @@ import { type HistoryStartSettleScheduler, } from "./history-start-settle-scheduler"; import { useChatFindSelectedMessageId } from "@/agent-stream/chat-find"; -import { getStreamItemMessageId } from "./presentation"; import { useScrollToMessage } from "./use-scroll-to-message.web"; interface CreateWebStreamStrategyInput { @@ -1174,6 +1173,11 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool } scheduleStickToBottom(); }, + holdReadingPosition: () => { + cancelPendingStickToBottom(); + setFollowOutput(false); + readingAnchor.scroll(scrollContainerRef.current?.scrollTop ?? 0); + }, scrollToMessage, }; viewportRef.current = handle; @@ -1186,6 +1190,7 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool }, [ cancelPendingStickToBottom, forceStickToBottom, + readingAnchor, scheduleStickToBottom, scrollToMessage, viewportRef, diff --git a/packages/app/src/agent-stream/strategy.ts b/packages/app/src/agent-stream/strategy.ts index 7604b48cdd..3d9df4d201 100644 --- a/packages/app/src/agent-stream/strategy.ts +++ b/packages/app/src/agent-stream/strategy.ts @@ -55,6 +55,8 @@ export interface StreamViewportHandle { focus?: () => void; scrollToBottom: (reason?: BottomAnchorLocalRequest["reason"]) => void; prepareForViewportChange: () => void; + /** Stop following output so the rows on screen stay put while rows below them open or close. */ + holdReadingPosition: () => void; scrollToMessage?: (messageId: string, occurrence?: ScrollToMessageOccurrence) => void; } diff --git a/packages/app/src/agent-stream/turn-collapse/model.test.ts b/packages/app/src/agent-stream/turn-collapse/model.test.ts new file mode 100644 index 0000000000..4535b14d9e --- /dev/null +++ b/packages/app/src/agent-stream/turn-collapse/model.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from "vitest"; +import type { StreamItem, ToolCallItem } from "@/types/stream"; +import { collapseTurns, type CollapseTurnsInput } from "./model"; + +const at = new Date("2026-01-01T00:00:00.000Z"); + +function user(id: string): StreamItem { + return { kind: "user_message", id, text: id, timestamp: at }; +} + +function reply(id: string): StreamItem { + return { kind: "assistant_message", id, text: id, timestamp: at }; +} + +function thought(id: string): StreamItem { + return { kind: "thought", id, text: id, timestamp: at, status: "ready" }; +} + +function shell(id: string): ToolCallItem { + return { + kind: "tool_call", + id, + timestamp: at, + payload: { + source: "agent", + data: { + provider: "claude", + callId: id, + name: "Bash", + status: "completed", + error: null, + detail: { type: "shell", command: "ls" }, + }, + }, + }; +} + +function collapse(tail: StreamItem[], options: Partial = {}) { + return collapseTurns({ + tail, + isLatestTurnOpen: false, + expandedHostIds: new Set(), + getGroupedCalls: () => undefined, + ...options, + }); +} + +const ids = (items: StreamItem[]) => items.map((item) => item.id); + +describe("collapseTurns", () => { + it("folds a finished turn's work into its user message and keeps the reply", () => { + const result = collapse([ + user("u1"), + thought("t1"), + reply("early"), + shell("c1"), + shell("c2"), + reply("final"), + ]); + + expect(ids(result.tail)).toEqual(["u1", "final"]); + expect(result.turnsByHostId.get("u1")).toEqual({ + hostId: "u1", + expanded: false, + summary: { + messageCount: 1, + tools: expect.objectContaining({ commandCount: 2 }), + }, + }); + expect(result.hostIdByFoldedMessageId.get("early")).toBe("u1"); + }); + + it("shows an expanded turn's work as nested rows between the prompt and the reply", () => { + const result = collapse([user("u1"), thought("t1"), shell("c1"), reply("final")], { + expandedHostIds: new Set(["u1"]), + }); + + expect(ids(result.tail)).toEqual(["u1", "t1", "c1", "final"]); + expect([...result.nestedItemIds]).toEqual(["t1", "c1"]); + expect(result.turnsByHostId.get("u1")?.expanded).toBe(true); + }); + + it("leaves the latest turn alone while it is still producing output", () => { + const tail = [user("u1"), shell("c1"), reply("r1"), user("u2"), shell("c2"), reply("r2")]; + + const result = collapse(tail, { isLatestTurnOpen: true }); + + expect(ids(result.tail)).toEqual(["u1", "r1", "u2", "c2", "r2"]); + expect([...result.turnsByHostId.keys()]).toEqual(["u1"]); + }); + + it("keeps rows after the reply, and turns with no work before the reply, as they are", () => { + const result = collapse([ + user("u1"), + shell("c1"), + reply("r1"), + shell("after"), + user("u2"), + reply("r2"), + ]); + + expect(ids(result.tail)).toEqual(["u1", "r1", "after", "u2", "r2"]); + expect([...result.turnsByHostId.keys()]).toEqual(["u1"]); + }); + + it("does not fold a turn that only thought before its reply", () => { + const tail = [user("u1"), thought("t1"), reply("r1")]; + + const result = collapse(tail); + + expect(result.tail).toBe(tail); + expect(result.turnsByHostId.size).toBe(0); + }); + + it("does not fold a turn without a reply or whose prompt is above the loaded history", () => { + const tail = [shell("c0"), reply("r0"), user("u1"), shell("c1")]; + + const result = collapse(tail); + + expect(result.tail).toBe(tail); + expect(result.turnsByHostId.size).toBe(0); + }); + + it("counts every call a tool-call group row stands for", () => { + const group = [shell("c1"), shell("c2"), shell("c3")]; + const result = collapse([user("u1"), group[0]!, reply("final")], { + getGroupedCalls: (itemId) => (itemId === "c1" ? group : undefined), + }); + + expect(result.turnsByHostId.get("u1")?.summary).toMatchObject({ + tools: { commandCount: 3 }, + }); + }); +}); diff --git a/packages/app/src/agent-stream/turn-collapse/model.ts b/packages/app/src/agent-stream/turn-collapse/model.ts new file mode 100644 index 0000000000..d9f532ad73 --- /dev/null +++ b/packages/app/src/agent-stream/turn-collapse/model.ts @@ -0,0 +1,122 @@ +import { getStreamItemMessageId, type StreamItem, type ToolCallItem } from "@/types/stream"; +import { summarizeToolCalls, type OverviewSummary } from "@/tool-calls/detail-level/overview/model"; + +export interface TurnSummary { + messageCount: number; + tools: OverviewSummary; +} + +export interface CollapsedTurn { + /** The user message the turn answers. Its row carries the turn's summary line. */ + hostId: string; + expanded: boolean; + summary: TurnSummary; +} + +export interface TurnCollapse { + tail: StreamItem[]; + turnsByHostId: ReadonlyMap; + /** Rows of expanded turns, which render inside the turn's indent rail. */ + nestedItemIds: ReadonlySet; + /** The turn a folded message belongs to, for surfaces that must reveal it. */ + hostIdByFoldedMessageId: ReadonlyMap; +} + +export interface CollapseTurnsInput { + tail: StreamItem[]; + /** The latest turn is still producing output, so it stays unfolded. */ + isLatestTurnOpen: boolean; + expandedHostIds: ReadonlySet; + /** The calls a tool-call group row stands for; undefined for a single call. */ + getGroupedCalls: (itemId: string) => readonly ToolCallItem[] | undefined; +} + +const EMPTY_TURNS: ReadonlyMap = new Map(); +const EMPTY_IDS: ReadonlySet = new Set(); +const EMPTY_HOSTS: ReadonlyMap = new Map(); + +function summarizeTurn( + items: readonly StreamItem[], + getGroupedCalls: CollapseTurnsInput["getGroupedCalls"], +): TurnSummary { + const calls: ToolCallItem[] = []; + let messageCount = 0; + for (const item of items) { + if (item.kind === "assistant_message") messageCount += 1; + else if (item.kind === "tool_call") calls.push(...(getGroupedCalls(item.id) ?? [item])); + } + return { messageCount, tools: summarizeToolCalls(calls) }; +} + +function hasWork(summary: TurnSummary): boolean { + return summary.messageCount > 0 || Object.values(summary.tools).some((count) => count > 0); +} + +function findLastAssistantIndex(tail: StreamItem[], start: number, end: number): number { + for (let index = end - 1; index >= start; index -= 1) { + if (tail[index]?.kind === "assistant_message") return index; + } + return -1; +} + +/** + * Folds each answered turn into its user message: everything the agent did before its last + * message collapses into one summary line, and that message stays in the chat. A turn folds + * only once it is complete, its user message is loaded, and it sent messages or called tools + * before its reply. Thoughts fold with the turn but are not counted. + */ +export function collapseTurns(input: CollapseTurnsInput): TurnCollapse { + const { tail } = input; + const output: StreamItem[] = []; + const turnsByHostId = new Map(); + const nestedItemIds = new Set(); + const hostIdByFoldedMessageId = new Map(); + + let runStart = 0; + const flushRun = (runEnd: number) => { + const host = tail[runStart - 1]; + const isClosed = runEnd < tail.length || !input.isLatestTurnOpen; + const replyIndex = findLastAssistantIndex(tail, runStart, runEnd); + if (host?.kind !== "user_message" || !isClosed || replyIndex <= runStart) { + output.push(...tail.slice(runStart, runEnd)); + return; + } + const folded = tail.slice(runStart, replyIndex); + const summary = summarizeTurn(folded, input.getGroupedCalls); + if (!hasWork(summary)) { + output.push(...tail.slice(runStart, runEnd)); + return; + } + const expanded = input.expandedHostIds.has(host.id); + turnsByHostId.set(host.id, { hostId: host.id, expanded, summary }); + for (const item of folded) { + hostIdByFoldedMessageId.set(getStreamItemMessageId(item), host.id); + if (expanded) nestedItemIds.add(item.id); + } + if (expanded) output.push(...folded); + output.push(...tail.slice(replyIndex, runEnd)); + }; + + for (let index = 0; index < tail.length; index += 1) { + if (tail[index]?.kind !== "user_message") continue; + flushRun(index); + output.push(tail[index]!); + runStart = index + 1; + } + flushRun(tail.length); + + if (turnsByHostId.size === 0) { + return { + tail, + turnsByHostId: EMPTY_TURNS, + nestedItemIds: EMPTY_IDS, + hostIdByFoldedMessageId: EMPTY_HOSTS, + }; + } + return { + tail: output.length === tail.length ? tail : output, + turnsByHostId, + nestedItemIds, + hostIdByFoldedMessageId, + }; +} diff --git a/packages/app/src/agent-stream/turn-collapse/summary-line.tsx b/packages/app/src/agent-stream/turn-collapse/summary-line.tsx new file mode 100644 index 0000000000..9f73f1ac22 --- /dev/null +++ b/packages/app/src/agent-stream/turn-collapse/summary-line.tsx @@ -0,0 +1,48 @@ +import { memo, useCallback, useMemo } from "react"; +import { useTranslation } from "react-i18next"; +import { Summary } from "lucide-react-native"; +import { ExpandableBadge } from "@/components/message"; +import { useOverviewSummaryLabel } from "@/tool-calls/detail-level/overview/label"; +import type { CollapsedTurn } from "./model"; + +/** + * The row a collapsed turn leaves under its user message, styled like a tool-call group. + * Pressing it opens the turn's rows beneath it. + */ +export const TurnSummaryLine = memo(function TurnSummaryLine({ + turn, + onExpandedChange, +}: { + turn: CollapsedTurn; + onExpandedChange: (hostId: string, expanded: boolean) => void; +}) { + const { t } = useTranslation(); + const { messageCount, tools } = turn.summary; + const messageParts = useMemo( + () => + messageCount > 0 + ? [ + t(`turnSummary.messages.${messageCount === 1 ? "one" : "other"}`, { + count: messageCount, + }), + ] + : [], + [messageCount, t], + ); + const label = useOverviewSummaryLabel(tools, messageParts); + const toggle = useCallback( + () => onExpandedChange(turn.hostId, !turn.expanded), + [onExpandedChange, turn.expanded, turn.hostId], + ); + + return ( + + ); +}); diff --git a/packages/app/src/agent-stream/use-scroll-to-message.web.ts b/packages/app/src/agent-stream/use-scroll-to-message.web.ts index 6de7a3296c..e593f1d323 100644 --- a/packages/app/src/agent-stream/use-scroll-to-message.web.ts +++ b/packages/app/src/agent-stream/use-scroll-to-message.web.ts @@ -1,8 +1,7 @@ import type React from "react"; import { useCallback, useEffect, useLayoutEffect, useMemo, useRef } from "react"; import type { Virtualizer } from "@tanstack/react-virtual"; -import type { StreamItem } from "@/types/stream"; -import { getStreamItemMessageId } from "./presentation"; +import { getStreamItemMessageId, type StreamItem } from "@/types/stream"; import { createPromptJumpSettleController, PROMPT_JUMP_TOP_INSET_PX } from "./prompt-jump-settle"; import type { ScrollToMessageOccurrence } from "./strategy"; diff --git a/packages/app/src/agent-stream/use-stream-history-window.ts b/packages/app/src/agent-stream/use-stream-history-window.ts index 9874158dcc..340ecb4da4 100644 --- a/packages/app/src/agent-stream/use-stream-history-window.ts +++ b/packages/app/src/agent-stream/use-stream-history-window.ts @@ -1,7 +1,6 @@ import { useCallback, useEffect, useMemo, useState } from "react"; -import type { StreamItem } from "@/types/stream"; +import { getStreamItemMessageId, type StreamItem } from "@/types/stream"; import { findMountedWindowStart, getMountedRecentStreamItems } from "./history-window"; -import { getStreamItemMessageId } from "./presentation"; // Callers reveal a message, not a row: an assistant message is several block rows and // the window has to open at the first of them. diff --git a/packages/app/src/agent-stream/view.tsx b/packages/app/src/agent-stream/view.tsx index d662a2f345..67e08a6f82 100644 --- a/packages/app/src/agent-stream/view.tsx +++ b/packages/app/src/agent-stream/view.tsx @@ -41,7 +41,7 @@ import { type InlinePathTarget, } from "@/components/message"; import { PlanCard } from "@/components/plan-card"; -import type { StreamItem } from "@/types/stream"; +import { getStreamItemMessageId, type StreamItem } from "@/types/stream"; import type { PendingMessageSubmission } from "@/composer/submission/model"; import type { TurnPresentation } from "@/timeline/turn-liveness"; import type { PendingPermission } from "@/types/shared"; @@ -62,8 +62,10 @@ import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import { ToolCallDetailsContent } from "@/components/tool-call-details"; import { QuestionFormCard } from "@/components/question-form-card"; import { ToolCallSheetProvider } from "@/components/tool-call-sheet"; -import { createStreamPresentation, getStreamItemMessageId } from "./presentation"; +import { createStreamPresentation } from "./presentation"; import { OverviewToolCallGroupView } from "@/tool-calls/detail-level/overview/view"; +import { IndentRail } from "@/components/indent-rail"; +import { TurnSummaryLine } from "./turn-collapse/summary-line"; import { type AgentStreamRenderModel, buildAgentStreamRenderModel } from "./model"; import { resolveStreamRenderStrategy } from "./strategy-resolver"; import { type StreamSegmentRenderers, type StreamViewportHandle } from "./strategy"; @@ -148,9 +150,16 @@ function renderPendingPermissionsNode(input: { ); } +/** + * Where a row sits in an opened turn: its summary line's row, a row inside the turn, or the + * turn's last row. The rail runs through the gaps between them so the turn reads as one block. + */ +type TurnIndent = "none" | "openedHost" | "nested" | "lastNested"; + function renderStreamItemWithTurnFooter(input: { content: ReactNode; layoutItem: StreamLayoutItem; + turnIndent: TurnIndent; strategy: TurnContentStrategy; supportsTimelineCursor: boolean; onForkAssistantTurn?: AssistantTurnForkHandler; @@ -171,7 +180,7 @@ function renderStreamItemWithTurnFooter(input: { /> ) : null; const content = ( - + {input.content} ); @@ -333,7 +342,7 @@ const AgentStreamViewComponent = forwardRef settings.autoExpandReasoning); - const toolCallDetailLevel = useSettings((settings) => settings.toolCallDetailLevel); + const chatDisplay = useSettings((settings) => settings.chatDisplay); const chatOutlineEnabled = useSettings((settings) => settings.chatOutlineEnabled); const contentMaxWidth = useSettings(resolveContentMaxWidth); const viewportRef = useRef(null); @@ -357,6 +366,7 @@ const AgentStreamViewComponent = forwardRef>( new Set(), ); + const [expandedTurnHostIds, setExpandedTurnHostIds] = useState>(new Set()); // Get serverId (fallback to agent's serverId if not provided) const resolvedServerId = serverId ?? context.serverId ?? ""; @@ -421,6 +431,7 @@ const AgentStreamViewComponent = forwardRef { + // A turn opens below its summary line; keep that line under the reader's pointer. + viewportRef.current?.holdReadingPosition(); + setExpandedTurnHostIds((previous) => { + if (previous.has(hostId) === expanded) return previous; + const next = new Set(previous); + if (expanded) { + next.add(hostId); + } else { + next.delete(hostId); + } + return next; + }); + }, []); + // Chat find reveals a match inside a collapsed turn by opening the turn first. + const revealLoadedMessage = useStableEvent((messageId: string): boolean => { + const hostId = presentation.turns.hostIdByFoldedMessageId.get(messageId); + if (hostId && !expandedTurnHostIds.has(hostId)) { + setTurnExpanded(hostId, true); + return true; + } + return revealLoadedHistory(messageId); + }); + // Read through stable events so a collapse rerun does not change the renderer identity; + // rows whose turn state changed are revised through `historyRowRevision`. + const getCollapsedTurn = useStableEvent((hostId: string) => + presentation.turns.turnsByHostId.get(hostId), + ); + const getTurnIndent = useStableEvent((layoutItem: StreamLayoutItem): TurnIndent => { + const { nestedItemIds } = presentation.turns; + if (nestedItemIds.has(layoutItem.item.id)) { + const below = layoutItem.belowItem; + return below && nestedItemIds.has(below.id) ? "nested" : "lastNested"; + } + return getCollapsedTurn(layoutItem.item.id)?.expanded ? "openedHost" : "none"; + }); + const setToolCallGroupExpanded = useCallback((groupId: string, expanded: boolean) => { setExpandedToolCallGroupIds((previous) => { const next = new Set(previous); @@ -665,7 +715,8 @@ const AgentStreamViewComponent = forwardRef) => { - return ( + const collapsedTurn = getCollapsedTurn(item.id); + const message = ( ); + if (!collapsedTurn) { + return message; + } + return ( + <> + {message} + + + ); }, - [context.capabilities, agentId, client, pendingClientMessageIds, resolvedServerId], + [ + context.capabilities, + agentId, + client, + getCollapsedTurn, + pendingClientMessageIds, + resolvedServerId, + setTurnExpanded, + ], ); const renderAssistantMessageItem = useCallback( @@ -893,12 +961,14 @@ const AgentStreamViewComponent = forwardRef ({ contentById: presentation.historyGroupUpdatesByHostId, - displayStateById: expandedToolCallGroupIds, + displayStateById: { + has: (id: string) => + expandedToolCallGroupIds.has(id) || + expandedTurnHostIds.has(id) || + nestedItemIds.has(id), + }, globalDisplayState: isMobile, }), - [expandedToolCallGroupIds, isMobile, presentation.historyGroupUpdatesByHostId], + [ + expandedToolCallGroupIds, + expandedTurnHostIds, + isMobile, + nestedItemIds, + presentation.historyGroupUpdatesByHostId, + ], ); const findItems = useMemo( @@ -1080,7 +1162,7 @@ const AgentStreamViewComponent = forwardRef @@ -1763,15 +1845,41 @@ const permissionStyles = StyleSheet.create((theme) => ({ })); interface StreamItemWrapperProps { - itemId: string; gapBelow: number; + turnIndent: TurnIndent; children: ReactNode; } -function StreamItemWrapper({ gapBelow, children }: StreamItemWrapperProps) { +function StreamItemWrapper({ gapBelow, turnIndent, children }: StreamItemWrapperProps) { const wrapperStyle = useMemo( - () => [stylesheet.streamItemWrapper, { marginBottom: gapBelow }], - [gapBelow], + () => [ + stylesheet.streamItemWrapper, + { marginBottom: turnIndent === "nested" || turnIndent === "openedHost" ? 0 : gapBelow }, + ], + [gapBelow, turnIndent], ); - return {children}; + const gapStyle = useMemo(() => ({ paddingBottom: gapBelow }), [gapBelow]); + switch (turnIndent) { + case "none": + return {children}; + case "openedHost": + return ( + + {children} + + + ); + case "nested": + return ( + + {children} + + ); + case "lastNested": + return ( + + {children} + + ); + } } diff --git a/packages/app/src/components/indent-rail.tsx b/packages/app/src/components/indent-rail.tsx new file mode 100644 index 0000000000..b54ac1ca9b --- /dev/null +++ b/packages/app/src/components/indent-rail.tsx @@ -0,0 +1,32 @@ +import type { ReactNode } from "react"; +import { View, type StyleProp, type ViewStyle } from "react-native"; +import { StyleSheet } from "react-native-unistyles"; + +/** + * Collapsed chat rows badge their summary with an icon whose center sits 7px into the content + * column (`ExpandableBadge` bleeds 13px left, then 1px border, 8px padding, 22px icon). The rail + * hangs from that center so opened rows read as children of the line that opened them. + */ +const RAIL_LEFT = 7; +const RAIL_WIDTH = 1; +export const INDENT_RAIL_CONTENT_OFFSET = 21; + +/** Indents opened chat rows under the summary that collapsed them, behind a vertical line. */ +export function IndentRail({ + children, + style, +}: { + children?: ReactNode; + style?: StyleProp; +}) { + return {children}; +} + +const styles = StyleSheet.create((theme) => ({ + rail: { + marginLeft: RAIL_LEFT, + paddingLeft: INDENT_RAIL_CONTENT_OFFSET - RAIL_LEFT - RAIL_WIDTH, + borderLeftWidth: RAIL_WIDTH, + borderLeftColor: theme.colors.border, + }, +})); diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx index d5a2244505..9430aa3a12 100644 --- a/packages/app/src/components/message.tsx +++ b/packages/app/src/components/message.tsx @@ -971,6 +971,10 @@ const expandableBadgeStylesheet = StyleSheet.create((theme) => ({ borderBottomLeftRadius: 0, borderBottomRightRadius: 0, }, + // Details that do not sit in a card leave the header as a lone highlighted pill. + pressableExpandedDetached: { + borderRadius: theme.borderRadius.full, + }, detailWrapperBorderless: { borderWidth: 0, }, @@ -2048,7 +2052,10 @@ export const ExpandableBadge = memo(function ExpandableBadge({ expandableBadgeStylesheet.pressable, isPressed && isInteractive ? expandableBadgeStylesheet.pressablePressed : null, isExpanded && expandableBadgeStylesheet.pressableExpanded, - isExpanded && !borderlessWhenExpanded && expandableBadgeStylesheet.pressableExpandedAttached, + isExpanded && + (borderlessWhenExpanded + ? expandableBadgeStylesheet.pressableExpandedDetached + : expandableBadgeStylesheet.pressableExpandedAttached), ], [borderlessWhenExpanded, isExpanded, isInteractive, isPressed], ); diff --git a/packages/app/src/hooks/use-settings/index.ts b/packages/app/src/hooks/use-settings/index.ts index 66a8e767b1..9f6f22fb23 100644 --- a/packages/app/src/hooks/use-settings/index.ts +++ b/packages/app/src/hooks/use-settings/index.ts @@ -42,6 +42,7 @@ import { saveAppSettings as saveAppSettingsPure, type AppSettings, type AppSettingsUpdate, + type ChatDisplay, type OpenInSidePanePreferences, type PullRequestOpenLocation, type DesktopSettingsBridge, @@ -84,6 +85,7 @@ export { export type { AppSettings, AppSettingsUpdate, + ChatDisplay, AppLanguage, OpenInSidePanePreferences, PullRequestOpenLocation, diff --git a/packages/app/src/hooks/use-settings/storage.test.ts b/packages/app/src/hooks/use-settings/storage.test.ts index dadb912a24..fe2ffdd2a0 100644 --- a/packages/app/src/hooks/use-settings/storage.test.ts +++ b/packages/app/src/hooks/use-settings/storage.test.ts @@ -185,6 +185,7 @@ describe("loadAppSettingsFromStorage", () => { expect(result).toEqual(DEFAULT_CLIENT_SETTINGS); expect(DEFAULT_CLIENT_SETTINGS.language).toBe("system"); + expect(DEFAULT_CLIENT_SETTINGS.chatDisplay).toBe("collapseTurns"); expect(JSON.parse(deps.storage.entries.get(APP_SETTINGS_KEY) ?? "null")).toEqual( DEFAULT_CLIENT_SETTINGS, ); @@ -454,6 +455,7 @@ describe("loadAppSettingsFromStorage", () => { ...DEFAULT_CLIENT_SETTINGS, theme: "dark", contentFontSize: DEFAULT_UI_BASE_FONT_SIZE, + chatDisplay: "full", }); expect(JSON.parse(deps.storage.entries.get(APP_SETTINGS_KEY) ?? "null")).toEqual({ manageBuiltInDaemon: false, @@ -592,6 +594,7 @@ describe("loadSettingsFromStorage", () => { ...DEFAULT_APP_SETTINGS, theme: "light", contentFontSize: DEFAULT_UI_BASE_FONT_SIZE, + chatDisplay: "full", }); }); @@ -636,6 +639,7 @@ describe("loadSettingsFromStorage", () => { ...DEFAULT_APP_SETTINGS, theme: "light", contentFontSize: DEFAULT_UI_BASE_FONT_SIZE, + chatDisplay: "full", manageBuiltInDaemon: false, releaseChannel: "beta", }); @@ -657,6 +661,7 @@ describe("loadSettingsFromStorage", () => { ...DEFAULT_APP_SETTINGS, theme: "light", contentFontSize: DEFAULT_UI_BASE_FONT_SIZE, + chatDisplay: "full", }); }); }); @@ -732,7 +737,7 @@ describe("saveAppSettings", () => { ...DEFAULT_CLIENT_SETTINGS, theme: "light", contentFontSize: DEFAULT_UI_BASE_FONT_SIZE, - toolCallDetailLevel: "overview", + chatDisplay: "collapseToolCalls", }); }); @@ -787,27 +792,45 @@ describe("appearance settings", () => { expect(result.contentFontSize).toBe(DEFAULT_UI_BASE_FONT_SIZE); expect(result.codeFontSize).toBe(DEFAULT_CODE_FONT_SIZE); expect(result.syntaxTheme).toBe("one"); - expect(result.toolCallDetailLevel).toBe("detailed"); + expect(result.chatDisplay).toBe("full"); }); - it("migrates the enabled compact tool call preference to overview", async () => { + it("migrates the enabled compact tool call preference to collapsed tool calls", async () => { const deps = makeDeps({ storage: createInMemoryKeyValueStorage({ [APP_SETTINGS_KEY]: JSON.stringify({ compactToolCalls: true }), }), }); - expect((await loadAppSettingsFromStorage(deps)).toolCallDetailLevel).toBe("overview"); + expect((await loadAppSettingsFromStorage(deps)).chatDisplay).toBe("collapseToolCalls"); }); - it("clears settings with an unrecognized tool call detail level", async () => { + it.each([ + ["overview", "collapseToolCalls"], + ["concise", "collapseToolCalls"], + ["detailed", "full"], + ["unknown", "full"], + ])("reads the %s tool call detail level as the %s chat display", async (legacy, display) => { const deps = makeDeps({ storage: createInMemoryKeyValueStorage({ - [APP_SETTINGS_KEY]: JSON.stringify({ toolCallDetailLevel: "unknown" }), + [APP_SETTINGS_KEY]: JSON.stringify({ toolCallDetailLevel: legacy }), }), }); - expect((await loadAppSettingsFromStorage(deps)).toolCallDetailLevel).toBe("detailed"); + expect((await loadAppSettingsFromStorage(deps)).chatDisplay).toBe(display); + }); + + it("keeps a stored chat display over the tool call detail level it replaced", async () => { + const deps = makeDeps({ + storage: createInMemoryKeyValueStorage({ + [APP_SETTINGS_KEY]: JSON.stringify({ + chatDisplay: "collapseTurns", + toolCallDetailLevel: "detailed", + }), + }), + }); + + expect((await loadAppSettingsFromStorage(deps)).chatDisplay).toBe("collapseTurns"); }); it("migrates a switched-off checks row item to the hidden checks display", async () => { diff --git a/packages/app/src/hooks/use-settings/storage.ts b/packages/app/src/hooks/use-settings/storage.ts index 4fe08bb6d5..3a5be63441 100644 --- a/packages/app/src/hooks/use-settings/storage.ts +++ b/packages/app/src/hooks/use-settings/storage.ts @@ -40,7 +40,8 @@ export type WorkspaceTitleSource = "title" | "branch"; export type PullRequestOpenLocation = "main" | "side" | "explorer"; /** What a sidebar workspace row shows in the space to the right of its title. */ export type SidebarWorkspaceTrailing = "diff" | "timestamp" | "none"; -export type ToolCallDetailLevel = "overview" | "detailed"; +/** How much of an agent's work the chat shows between the user's message and the reply. */ +export type ChatDisplay = "full" | "collapseToolCalls" | "collapseTurns"; const ThemePreferenceSchema = z.enum([ ...THEME_OPTIONS.map((option) => option.name), @@ -100,7 +101,7 @@ export interface AppSettings { /** How usage reads and which windows the sidebar summary shows. */ usage: UsagePreferences; autoExpandReasoning: boolean; - toolCallDetailLevel: ToolCallDetailLevel; + chatDisplay: ChatDisplay; chatOutlineEnabled: boolean; vimKeybindings: boolean; filePanelAssociations: Record; @@ -156,7 +157,7 @@ export const DEFAULT_CLIENT_SETTINGS: AppSettings = { sidebarFooterItems: [], usage: DEFAULT_USAGE_PREFERENCES, autoExpandReasoning: false, - toolCallDetailLevel: "detailed", + chatDisplay: "collapseTurns", chatOutlineEnabled: true, vimKeybindings: false, filePanelAssociations: {}, @@ -199,6 +200,7 @@ const SidebarRowItemsSchema = z type StoredAppSettingsFallback = AppSettings & { uiFontSize?: number; compactToolCalls?: boolean; + toolCallDetailLevel?: "overview" | "detailed"; manageBuiltInDaemon?: boolean; releaseChannel?: ReleaseChannel; needsWrite: boolean; @@ -251,11 +253,13 @@ const StoredAppSettingsSchema = z sidebarFooterItems: z.array(z.object({ key: z.string(), visible: z.boolean() })).catch([]), usage: UsagePreferencesSchema, autoExpandReasoning: z.boolean().catch(false), + chatDisplay: z.enum(["full", "collapseToolCalls", "collapseTurns"]).optional().catch("full"), + // COMPAT(toolCallDetailLevel): replaced by chatDisplay in v0.13, remove after 2027-04-24. toolCallDetailLevel: z .enum(["overview", "detailed"]) .or(z.literal("concise").transform(() => "overview" as const)) .optional() - .catch("detailed"), + .catch(undefined), // COMPAT(compactToolCalls): migrated in v0.1.105, remove after 2027-01-12. compactToolCalls: z.boolean().optional().catch(undefined), chatOutlineEnabled: z.boolean().catch(true), @@ -305,8 +309,10 @@ const StoredAppSettingsSchema = z (isChecksHiddenByLegacyRowItem(stored.sidebarRowItems) ? "none" : DEFAULT_SIDEBAR_CHECKS_DISPLAY); - const toolCallDetailLevel = - stored.toolCallDetailLevel ?? (stored.compactToolCalls ? "overview" : "detailed"); + const groupsToolCalls = stored.toolCallDetailLevel + ? stored.toolCallDetailLevel === "overview" + : stored.compactToolCalls === true; + const chatDisplay = stored.chatDisplay ?? (groupsToolCalls ? "collapseToolCalls" : "full"); return { ...stored, openInSidePane, @@ -321,7 +327,7 @@ const StoredAppSettingsSchema = z stored.sidebarRowItems.services ?? (stored.sidebarRowItems.scripts === false ? false : DEFAULT_SIDEBAR_ROW_ITEMS.services), }, - toolCallDetailLevel, + chatDisplay, needsWrite, }; }) @@ -449,6 +455,7 @@ export function normalizeAppSettings(value: unknown): AppSettings { manageBuiltInDaemon: _manageBuiltInDaemon, releaseChannel: _releaseChannel, compactToolCalls: _compactToolCalls, + toolCallDetailLevel: _toolCallDetailLevel, uiFontSize: _uiFontSize, ...settings } = StoredAppSettingsSchema.parse(value); diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index c765f3b835..3ff25adf40 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -2008,6 +2008,12 @@ export const ar: TranslationResources = { input: "مدخل", output: "الإخراج", }, + turnSummary: { + messages: { + one: "كتب {{count}} رسالة", + other: "كتب {{count}} رسائل", + }, + }, toolCallGroup: { editedFiles: { one: "حرّر {{count}} ملفًا", @@ -2312,12 +2318,13 @@ export const ar: TranslationResources = { label: "عرض التفكير دائماً", description: "إظهار تفكير الوكيل وخطوات الاستدلال بشكل كامل بشكل افتراضي", }, - toolCallDetail: { - label: "عرض استدعاءات الأدوات", - description: "كيفية ظهور استدعاءات الأدوات في المخطط الزمني", + chatDisplay: { + label: "عرض المحادثة", + description: "مقدار عمل الوكيل الذي تعرضه المحادثة", options: { - overview: "ملخص", - detailed: "التفاصيل الكاملة", + full: "كامل", + collapseToolCalls: "طيّ استدعاءات الأدوات", + collapseTurns: "طيّ الأدوار", }, }, language: { diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 351fb33ac3..1d0d1da97a 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -2033,6 +2033,12 @@ export const en = { input: "Input", output: "Output", }, + turnSummary: { + messages: { + one: "wrote {{count}} message", + other: "wrote {{count}} messages", + }, + }, toolCallGroup: { editedFiles: { one: "edited {{count}} file", @@ -2400,12 +2406,13 @@ export const en = { label: "Always expand reasoning", description: "Show agent thinking and chain-of-thought blocks fully expanded by default", }, - toolCallDetail: { - label: "Tool call display", - description: "How tool calls appear in the timeline", + chatDisplay: { + label: "Chat display", + description: "How much of an agent's work the chat shows", options: { - overview: "Summary", - detailed: "Full detail", + full: "Full", + collapseToolCalls: "Collapse tool calls", + collapseTurns: "Collapse turns", }, }, language: { diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index fd195ce599..feafcd225f 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -2056,6 +2056,12 @@ export const es: TranslationResources = { input: "Aporte", output: "Producción", }, + turnSummary: { + messages: { + one: "escribió {{count}} mensaje", + other: "escribió {{count}} mensajes", + }, + }, toolCallGroup: { editedFiles: { one: "editó {{count}} archivo", @@ -2328,12 +2334,13 @@ export const es: TranslationResources = { description: "Mostrar los bloques de pensamiento y razonamiento del agente totalmente expandidos de forma predeterminada", }, - toolCallDetail: { - label: "Visualización de llamadas a herramientas", - description: "Cómo aparecen las llamadas a herramientas en la cronología", + chatDisplay: { + label: "Visualización del chat", + description: "Cuánto del trabajo del agente muestra el chat", options: { - overview: "Resumen", - detailed: "Detalle completo", + full: "Completo", + collapseToolCalls: "Contraer llamadas a herramientas", + collapseTurns: "Contraer turnos", }, }, language: { diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 975d6a3736..6d8ee58400 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -2055,6 +2055,12 @@ export const fr: TranslationResources = { input: "Entrée", output: "Sortie", }, + turnSummary: { + messages: { + one: "{{count}} message écrit", + other: "{{count}} messages écrits", + }, + }, toolCallGroup: { editedFiles: { one: "{{count}} fichier modifié", @@ -2326,12 +2332,13 @@ export const fr: TranslationResources = { label: "Toujours afficher le raisonnement", description: "Afficher par défaut le raisonnement de l’agent entièrement déplié", }, - toolCallDetail: { - label: "Affichage des appels d’outils", - description: "Comment les appels d’outils apparaissent dans la chronologie", + chatDisplay: { + label: "Affichage du chat", + description: "La part du travail de l’agent affichée dans le chat", options: { - overview: "Résumé", - detailed: "Détails complets", + full: "Complet", + collapseToolCalls: "Réduire les appels d’outils", + collapseTurns: "Réduire les tours", }, }, language: { diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index 60a207e1e1..b8af6f91f6 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -2028,6 +2028,12 @@ export const ja: TranslationResources = { input: "入力", output: "出力", }, + turnSummary: { + messages: { + one: "{{count}}件のメッセージを送信", + other: "{{count}}件のメッセージを送信", + }, + }, toolCallGroup: { editedFiles: { one: "{{count}}個のファイルを編集", @@ -2296,12 +2302,13 @@ export const ja: TranslationResources = { label: "常に思考プロセスを展開", description: "デフォルトでAIのエージェント思考・推論ブロックを完全に展開して表示します", }, - toolCallDetail: { - label: "ツール呼び出しの表示", - description: "タイムラインでのツール呼び出しの表示方法", + chatDisplay: { + label: "チャットの表示", + description: "チャットに表示するエージェントの作業量", options: { - overview: "要約", - detailed: "すべての詳細", + full: "すべて", + collapseToolCalls: "ツール呼び出しを折りたたむ", + collapseTurns: "ターンを折りたたむ", }, }, language: { diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index 4773625327..1a9b36d9bd 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -2018,6 +2018,12 @@ export const ko: TranslationResources = { input: "입력", output: "출력", }, + turnSummary: { + messages: { + one: "{{count}} 메시지를 작성했습니다.", + other: "{{count}} 메시지를 작성했습니다.", + }, + }, toolCallGroup: { editedFiles: { one: "{{count}} 파일을 편집했습니다.", @@ -2287,12 +2293,13 @@ export const ko: TranslationResources = { label: "추론 항상 펼치기", description: "에이전트의 사고 및 추론 블록을 기본적으로 모두 펼쳐 표시합니다.", }, - toolCallDetail: { - label: "도구 호출 표시", - description: "타임라인에 도구 호출이 표시되는 방식", + chatDisplay: { + label: "채팅 표시", + description: "채팅에 표시할 에이전트 작업의 양", options: { - overview: "요약", - detailed: "전체 세부정보", + full: "전체", + collapseToolCalls: "도구 호출 접기", + collapseTurns: "턴 접기", }, }, language: { diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 766c6d98ed..0d7e6acb89 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -2041,6 +2041,12 @@ export const ptBR: TranslationResources = { input: "Entrada", output: "Saída", }, + turnSummary: { + messages: { + one: "escreveu {{count}} mensagem", + other: "escreveu {{count}} mensagens", + }, + }, toolCallGroup: { editedFiles: { one: "editou {{count}} arquivo", @@ -2311,12 +2317,13 @@ export const ptBR: TranslationResources = { description: "Mostrar os blocos de pensamento e raciocínio do agente totalmente expandidos por padrão", }, - toolCallDetail: { - label: "Exibição de chamadas de ferramentas", - description: "Como as chamadas de ferramentas aparecem na linha do tempo", + chatDisplay: { + label: "Exibição do chat", + description: "Quanto do trabalho do agente o chat mostra", options: { - overview: "Resumo", - detailed: "Detalhes completos", + full: "Completo", + collapseToolCalls: "Recolher chamadas de ferramentas", + collapseTurns: "Recolher turnos", }, }, language: { diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index 23f3d810ee..09d46fa075 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -2041,6 +2041,12 @@ export const ru: TranslationResources = { input: "Входные данные", output: "Выходные данные", }, + turnSummary: { + messages: { + one: "написано {{count}} сообщение", + other: "написаны сообщения ({{count}})", + }, + }, toolCallGroup: { editedFiles: { one: "изменён {{count}} файл", @@ -2313,12 +2319,13 @@ export const ru: TranslationResources = { description: "По умолчанию полностью разворачивать блоки размышлений и хода рассуждений агента", }, - toolCallDetail: { - label: "Отображение вызовов инструментов", - description: "Как вызовы инструментов отображаются на временной шкале", + chatDisplay: { + label: "Отображение чата", + description: "Какую часть работы агента показывает чат", options: { - overview: "Сводка", - detailed: "Все подробности", + full: "Полностью", + collapseToolCalls: "Сворачивать вызовы инструментов", + collapseTurns: "Сворачивать ходы", }, }, language: { diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index 43d8ceeef2..374bef6e88 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -1986,6 +1986,12 @@ export const zhCN: TranslationResources = { input: "输入", output: "输出", }, + turnSummary: { + messages: { + one: "写了 {{count}} 条消息", + other: "写了 {{count}} 条消息", + }, + }, toolCallGroup: { editedFiles: { one: "编辑了 {{count}} 个文件", @@ -2251,12 +2257,13 @@ export const zhCN: TranslationResources = { label: "始终展开推理过程", description: "默认情况下完全展开 AI 的思考和推理过程", }, - toolCallDetail: { - label: "工具调用显示", - description: "工具调用在时间线中的显示方式", + chatDisplay: { + label: "聊天显示", + description: "聊天中显示多少智能体的工作", options: { - overview: "摘要", - detailed: "完整详情", + full: "完整", + collapseToolCalls: "折叠工具调用", + collapseTurns: "折叠轮次", }, }, language: { diff --git a/packages/app/src/screens/settings/chat/chat-section.tsx b/packages/app/src/screens/settings/chat/chat-section.tsx index 8a81d30073..4f4254cb8e 100644 --- a/packages/app/src/screens/settings/chat/chat-section.tsx +++ b/packages/app/src/screens/settings/chat/chat-section.tsx @@ -8,29 +8,25 @@ import { SettingsSwitch, } from "@/components/settings"; import { isNative } from "@/constants/platform"; -import { useAppSettings, type AppSettings } from "@/hooks/use-settings"; +import { useAppSettings, type ChatDisplay } from "@/hooks/use-settings"; -const TOOL_CALL_DETAIL_LEVELS: readonly AppSettings["toolCallDetailLevel"][] = [ - "detailed", - "overview", -]; +const CHAT_DISPLAYS: readonly ChatDisplay[] = ["full", "collapseToolCalls", "collapseTurns"]; export function ChatSection() { const { t } = useTranslation(); const { settings, updateSettings } = useAppSettings(); - const toolCallDetailOptions = useMemo( + const chatDisplayOptions = useMemo( () => - TOOL_CALL_DETAIL_LEVELS.map((value) => ({ + CHAT_DISPLAYS.map((value) => ({ value, - label: t(`settings.general.toolCallDetail.options.${value}`), + label: t(`settings.general.chatDisplay.options.${value}`), })), [t], ); - const changeToolCallDetailLevel = useCallback( - (toolCallDetailLevel: AppSettings["toolCallDetailLevel"]) => - void updateSettings({ toolCallDetailLevel }), + const changeChatDisplay = useCallback( + (chatDisplay: ChatDisplay) => void updateSettings({ chatDisplay }), [updateSettings], ); const changeAutoExpandReasoning = useCallback( @@ -53,11 +49,11 @@ export function ChatSection() { onValueChange={changeAutoExpandReasoning} /> {isNative ? null : ( 2) { + joined = `${parts.slice(0, -1).join(", ")}, ${conjunction} ${parts.at(-1)}`; + } + const firstCharacter = joined[0]; + return firstCharacter ? `${firstCharacter.toLocaleUpperCase()}${joined.slice(1)}` : joined; +} + +/** + * One sentence of what a stretch of agent work did, such as "Edited 2 files and ran 3 commands". + * Leading parts come first, for summaries that count more than tool calls. + */ +export function useOverviewSummaryLabel( + summary: OverviewSummary, + leadingParts: readonly string[] = [], +): string { + const { t } = useTranslation(); + return useMemo(() => { + const parts = [...leadingParts]; + const entries = [ + [summary.editedFileCount, "toolCallGroup.editedFiles"], + [summary.commandCount, "toolCallGroup.commands"], + [summary.readFileCount, "toolCallGroup.readFiles"], + [summary.searchCount, "toolCallGroup.searches"], + [summary.otherToolCount, "toolCallGroup.otherTools"], + [summary.paseoCallCount, "toolCallGroup.paseoCalls"], + ] as const; + for (const [count, key] of entries) { + if (count > 0) { + parts.push(t(`${key}.${count === 1 ? "one" : "other"}`, { count })); + } + } + return joinSummaryParts(parts, t("toolCallGroup.and")); + }, [leadingParts, summary, t]); +} diff --git a/packages/app/src/tool-calls/detail-level/overview/model.ts b/packages/app/src/tool-calls/detail-level/overview/model.ts index f41790461c..0fb638c012 100644 --- a/packages/app/src/tool-calls/detail-level/overview/model.ts +++ b/packages/app/src/tool-calls/detail-level/overview/model.ts @@ -1,4 +1,5 @@ import { isPaseoToolName } from "@getpaseo/protocol/tool-name-normalization"; +import type { ToolCallItem } from "@/types/stream"; import { describeToolCall, type ToolCallRun } from "../grouping"; const DIRECT_PASEO_TOOL_PREFIX = "paseo_"; @@ -28,19 +29,18 @@ function isSearchCall(name: string): boolean { return DIRECT_SEARCH_TOOL_SUFFIX_PATTERN.test(name); } -export function buildOverviewGroup(run: ToolCallRun): OverviewToolCallGroup { +/** Counts tool calls by what they did, for any summary of a stretch of agent work. */ +export function summarizeToolCalls(calls: readonly ToolCallItem[]): OverviewSummary { const editedFiles = new Set(); const readFiles = new Set(); - let isLoading = false; let commandCount = 0; let searchCount = 0; let otherToolCount = 0; let paseoCallCount = 0; - for (const call of run.calls) { + for (const call of calls) { const descriptor = describeToolCall(call); const normalizedName = descriptor.name.trim().toLowerCase(); - isLoading ||= descriptor.status === "running" || descriptor.status === "executing"; if (isPaseoCall(descriptor.name, normalizedName)) { paseoCallCount += 1; } else if (descriptor.detail.type === "edit" || descriptor.detail.type === "write") { @@ -56,7 +56,7 @@ export function buildOverviewGroup(run: ToolCallRun): OverviewToolCallGroup { } } - const summary = { + return { editedFileCount: editedFiles.size, commandCount, readFileCount: readFiles.size, @@ -64,10 +64,17 @@ export function buildOverviewGroup(run: ToolCallRun): OverviewToolCallGroup { otherToolCount, paseoCallCount, }; +} + +export function buildOverviewGroup(run: ToolCallRun): OverviewToolCallGroup { + const isLoading = run.calls.some((call) => { + const status = describeToolCall(call).status; + return status === "running" || status === "executing"; + }); return { mode: "overview", run, isLoading, - summary, + summary: summarizeToolCalls(run.calls), }; } diff --git a/packages/app/src/tool-calls/detail-level/overview/view.tsx b/packages/app/src/tool-calls/detail-level/overview/view.tsx index 326cc172b2..b11e82e4f7 100644 --- a/packages/app/src/tool-calls/detail-level/overview/view.tsx +++ b/packages/app/src/tool-calls/detail-level/overview/view.tsx @@ -1,11 +1,12 @@ -import React, { memo, useCallback, useMemo, useRef, type ReactNode } from "react"; +import React, { memo, useCallback, useRef, type ReactNode } from "react"; import { ScrollView } from "react-native"; -import { useTranslation } from "react-i18next"; import { Wrench } from "lucide-react-native"; import { StyleSheet } from "react-native-unistyles"; +import { IndentRail } from "@/components/indent-rail"; import { ExpandableBadge } from "@/components/message"; import { useIsCompactFormFactor } from "@/constants/layout"; -import { type OverviewSummary, type OverviewToolCallGroup } from "./model"; +import { useOverviewSummaryLabel } from "./label"; +import type { OverviewToolCallGroup } from "./model"; import { OverviewToolCallGroupSheet } from "./sheet"; interface OverviewGroupProps { @@ -18,41 +19,6 @@ interface OverviewGroupProps { const TOOL_CALL_GROUP_MAX_HEIGHT = 400; -function joinSummaryParts(parts: string[], conjunction: string): string { - if (parts.length === 0) { - return ""; - } - let joined = parts[0] ?? ""; - if (parts.length === 2) { - joined = `${parts[0]} ${conjunction} ${parts[1]}`; - } else if (parts.length > 2) { - joined = `${parts.slice(0, -1).join(", ")}, ${conjunction} ${parts.at(-1)}`; - } - const firstCharacter = joined[0]; - return firstCharacter ? `${firstCharacter.toLocaleUpperCase()}${joined.slice(1)}` : joined; -} - -function useOverviewSummary(summary: OverviewSummary): string { - const { t } = useTranslation(); - return useMemo(() => { - const parts: string[] = []; - const entries = [ - [summary.editedFileCount, "toolCallGroup.editedFiles"], - [summary.commandCount, "toolCallGroup.commands"], - [summary.readFileCount, "toolCallGroup.readFiles"], - [summary.searchCount, "toolCallGroup.searches"], - [summary.otherToolCount, "toolCallGroup.otherTools"], - [summary.paseoCallCount, "toolCallGroup.paseoCalls"], - ] as const; - for (const [count, key] of entries) { - if (count > 0) { - parts.push(t(`${key}.${count === 1 ? "one" : "other"}`, { count })); - } - } - return joinSummaryParts(parts, t("toolCallGroup.and")); - }, [summary, t]); -} - export const OverviewToolCallGroupView = memo(function OverviewToolCallGroupView({ group, expanded, @@ -62,7 +28,7 @@ export const OverviewToolCallGroupView = memo(function OverviewToolCallGroupView }: OverviewGroupProps) { const scrollRef = useRef(null); const isCompact = useIsCompactFormFactor(); - const aggregateSummary = useOverviewSummary(group.summary); + const aggregateSummary = useOverviewSummaryLabel(group.summary); const scrollToLatest = useCallback(() => { scrollRef.current?.scrollToEnd({ animated: false }); }, []); @@ -82,7 +48,7 @@ export const OverviewToolCallGroupView = memo(function OverviewToolCallGroupView showsVerticalScrollIndicator onContentSizeChange={scrollToLatest} > - {children} + {children} ), [children, scrollToLatest], diff --git a/packages/app/src/tool-calls/detail-level/projection.ts b/packages/app/src/tool-calls/detail-level/projection.ts index 5dc1067914..9d57391b2e 100644 --- a/packages/app/src/tool-calls/detail-level/projection.ts +++ b/packages/app/src/tool-calls/detail-level/projection.ts @@ -1,5 +1,4 @@ import type { StreamItem } from "@/types/stream"; -import type { ToolCallDetailLevel } from "@/hooks/use-settings/storage"; import { groupLiveToolCalls, prepareGroupedHistory, @@ -8,7 +7,8 @@ import { } from "./grouping"; import { buildOverviewGroup, type OverviewToolCallGroup } from "./overview/model"; -export type { ToolCallDetailLevel } from "@/hooks/use-settings/storage"; +/** Whether consecutive tool calls show one by one or as one summary row. */ +export type ToolCallDetailLevel = "overview" | "detailed"; export type ToolCallDetailGroup = OverviewToolCallGroup; export interface PreparedToolCallHistory { diff --git a/packages/app/src/types/stream.ts b/packages/app/src/types/stream.ts index 9425cabbed..657d218b35 100644 --- a/packages/app/src/types/stream.ts +++ b/packages/app/src/types/stream.ts @@ -771,6 +771,12 @@ export type AgentToolCallItem = ToolCallItem & { payload: { source: "agent"; data: AgentToolCallData }; }; +/** Find and scrolling address a plugin row by the source it replaces. */ +export function getStreamItemMessageId(item: StreamItem): string { + if (item.kind === "plugin") return item.sourceId ?? item.id; + return item.id; +} + export function isAgentToolCallItem(item: StreamItem): item is AgentToolCallItem { return item.kind === "tool_call" && item.payload.source === "agent"; } From 2a890208ce0b8b9e522f8326c043c158e46396f0 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 10:51:19 +0700 Subject: [PATCH 05/32] feat(app): show a subagent's parent and a way back to it (#7784) A subagent pane now names its parent and takes you back to it. With a composer, a pill leads the rail above it; without one, a parent box takes the composer's place. Both open the parent through navigateToAgent. --- docs/agent-lifecycle.md | 2 +- .../browser/compact-workspace-sheet.spec.ts | 10 ++ .../browser/provider-subagents.real.spec.ts | 3 +- .../e2e/browser/subagent-origin-pane.spec.ts | 23 ++++ packages/app/src/composer/tracks.tsx | 6 +- packages/app/src/i18n/resources/ar.ts | 7 ++ packages/app/src/i18n/resources/en.ts | 7 ++ packages/app/src/i18n/resources/es.ts | 7 ++ packages/app/src/i18n/resources/fr.ts | 7 ++ packages/app/src/i18n/resources/ja.ts | 7 ++ packages/app/src/i18n/resources/ko.ts | 7 ++ packages/app/src/i18n/resources/pt-BR.ts | 7 ++ packages/app/src/i18n/resources/ru.ts | 7 ++ packages/app/src/i18n/resources/zh-CN.ts | 7 ++ packages/app/src/panels/agent-panel.tsx | 13 +- packages/app/src/panels/agent-tracks.tsx | 16 ++- .../src/panels/provider-subagent-panel.tsx | 47 +++---- packages/app/src/subagents/parent/callout.tsx | 116 ++++++++++++++++++ packages/app/src/subagents/parent/index.ts | 3 + packages/app/src/subagents/parent/pill.tsx | 79 ++++++++++++ .../src/subagents/parent/use-parent-agent.ts | 38 ++++++ 21 files changed, 392 insertions(+), 27 deletions(-) create mode 100644 packages/app/src/subagents/parent/callout.tsx create mode 100644 packages/app/src/subagents/parent/index.ts create mode 100644 packages/app/src/subagents/parent/pill.tsx create mode 100644 packages/app/src/subagents/parent/use-parent-agent.ts diff --git a/docs/agent-lifecycle.md b/docs/agent-lifecycle.md index 535a6dd712..e9ca2211c2 100644 --- a/docs/agent-lifecycle.md +++ b/docs/agent-lifecycle.md @@ -242,7 +242,7 @@ parentAgentId === thisAgent.id AND !archivedAt - **Provider subagents** are child executions owned by Claude, Codex, or OpenCode. They are not inserted into `AgentManager` as managed agents. Providers emit a separate descriptor and timeline stream through `agent.provider_subagents.*`; the client keeps that state outside the normal agent store and merges only the presentation rows into the track. A descriptor's optional `parentSubagentId` identifies its direct provider-subagent parent; an absent value identifies a direct child of the managed agent. -Clicking either kind opens a workspace tab. On compact layouts it opens in a [sheet](explorer-sidebar.md#sheets) over the subagents list instead; the sheet hides the composer and shows the agent's model and thinking as one line in its place (`PaneSubtitle`, the row provider subagents already use), and a Paseo subagent's sheet offers **Open as tab** (the full-screen glyph in the header) to get the composer back. A Paseo subagent tab is a normal interactive agent pane. A provider subagent tab is a read-only timeline pane with no composer, archive, detach, rewind, or fork actions. It shows its own direct children in a subagents track. Both panes use `AgentStreamView`, so message, reasoning, tool-call, and layout rendering stay identical. +Clicking either kind opens a workspace tab. On compact layouts it opens in a [sheet](explorer-sidebar.md#sheets) over the subagents list instead; the sheet hides the composer and shows the agent's model and thinking as one line in its place (`PaneSubtitle`, the row provider subagents already use), and a Paseo subagent's sheet offers **Open as tab** (the full-screen glyph in the header) to get the composer back. A Paseo subagent tab is a normal interactive agent pane. A provider subagent tab is a read-only timeline pane with no composer, archive, detach, rewind, or fork actions. It shows its own direct children in a subagents track. Every subagent pane links back to its parent (`packages/app/src/subagents/parent/`): a pill above the composer when there is one, a parent box in the composer's place when there is not. Both panes use `AgentStreamView`, so message, reasoning, tool-call, and layout rendering stay identical. Provider timelines use the same structural timeline item format but deliberately have a separate lifecycle and transport. A provider thread/session identifier is not a Paseo agent identifier, and closing its tab is always layout-only. diff --git a/packages/app/e2e/browser/compact-workspace-sheet.spec.ts b/packages/app/e2e/browser/compact-workspace-sheet.spec.ts index 990d490d9b..77d995eda7 100644 --- a/packages/app/e2e/browser/compact-workspace-sheet.spec.ts +++ b/packages/app/e2e/browser/compact-workspace-sheet.spec.ts @@ -99,6 +99,16 @@ test("compact opens files and subagents in a sheet, leaving the tabs for agents" await expectSubagentRowVisible(page, child.id); }); + await test.step("the parent box in a subagent's sheet closes it onto the parent", async () => { + await page.getByTestId(`subagents-track-row-${child.id}`).click(); + const childSheet = sheet(page, child.title); + await expectSheetAtSnap(page, child.title); + await expect(childSheet.getByTestId("composer-parent-agent-pill")).toHaveCount(0); + await childSheet.getByTestId("parent-agent-callout-open").click(); + await expect(childSheet).toHaveCount(0); + await expectOpenTabCount(page, 1); + }); + await test.step("Open as tab promotes the subagent to a full tab and ends the list", async () => { await page.getByTestId(`subagents-track-row-${child.id}`).click(); await expectSheetAtSnap(page, child.title); diff --git a/packages/app/e2e/browser/provider-subagents.real.spec.ts b/packages/app/e2e/browser/provider-subagents.real.spec.ts index 08d8a1afe1..b24ca9ba4e 100644 --- a/packages/app/e2e/browser/provider-subagents.real.spec.ts +++ b/packages/app/e2e/browser/provider-subagents.real.spec.ts @@ -136,7 +136,8 @@ test.describe("real provider subagent timelines", () => { panel.getByText("Start chatting with this agent...", { exact: true }), ).toHaveCount(0); - await page.getByTestId(`workspace-tab-agent_${handle.agentId}`).first().click(); + // A provider subagent has no composer; its callout is the way back to the parent. + await page.getByTestId("parent-agent-callout-open").click(); await expect( page.getByTestId("assistant-message").filter({ hasText: "ROOT_DONE" }).last(), ).toBeVisible({ timeout: 60_000 }); diff --git a/packages/app/e2e/browser/subagent-origin-pane.spec.ts b/packages/app/e2e/browser/subagent-origin-pane.spec.ts index 30b19e5099..545076ab06 100644 --- a/packages/app/e2e/browser/subagent-origin-pane.spec.ts +++ b/packages/app/e2e/browser/subagent-origin-pane.spec.ts @@ -125,6 +125,29 @@ test("opening an already-tabbed subagent reveals its left tab without a duplicat } }); +test("a subagent's parent pill returns to its parent", async ({ page }) => { + const workspace = await seedWorkspace({ repoPrefix: "subagent-parent-pill-" }); + try { + const pair = await seedParentWithSubagent(workspace, { + parentTitle: "Refactor auth flow", + childTitle: "Write session tests", + }); + await page.setViewportSize({ width: 1400, height: 900 }); + await openAgentRoute(page, { workspaceId: pair.workspaceId, agentId: pair.child.id }); + + const pill = page.getByTestId("composer-parent-agent-pill"); + await expect(pill).toHaveAccessibleName("Back to Refactor auth flow"); + await pill.click(); + + await expect(page.getByTestId(`workspace-tab-agent_${pair.parent.id}`)).toHaveAttribute( + "aria-selected", + "true", + ); + } finally { + await workspace.cleanup(); + } +}); + test("a preferred chat file link still opens beside its right-pane agent", async ({ page }) => { const target = "linked.ts:3"; const workspace = await seedMockAgentWorkspace({ diff --git a/packages/app/src/composer/tracks.tsx b/packages/app/src/composer/tracks.tsx index 3a4541d285..82bee6e3fa 100644 --- a/packages/app/src/composer/tracks.tsx +++ b/packages/app/src/composer/tracks.tsx @@ -276,7 +276,11 @@ export function ComposerTrackRow({ * knockout for marks that sit on top of an icon, and nothing sits under this one, so it comes off * too — leaving every mark's visible edge on the same rail whatever state it is in. */ -function ComposerTrackMark({ bucket }: { bucket: SidebarStateBucket | null }): ReactElement | null { +export function ComposerTrackMark({ + bucket, +}: { + bucket: SidebarStateBucket | null; +}): ReactElement | null { if (!bucket) { return null; } diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index 3ff25adf40..4fc0ff5277 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -1908,6 +1908,13 @@ export const ar: TranslationResources = { backdrop: "خلفية القائمة", }, subagents: { + parent: { + backToParent: "العودة إلى الوكيل الأصلي", + backTo: "العودة إلى {{label}}", + parent: "الأصل", + parentAgent: "الوكيل الأصلي", + openParent: "فتح الوكيل الأصلي", + }, title: "الوكلاء الفرعيون", pillLabelOne: "وكيل فرعي واحد", pillLabelMany: "{{count}} وكلاء فرعيين", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 1d0d1da97a..2e40707121 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -1933,6 +1933,13 @@ export const en = { backdrop: "Menu backdrop", }, subagents: { + parent: { + backToParent: "Back to parent", + backTo: "Back to {{label}}", + parent: "Parent", + parentAgent: "Parent agent", + openParent: "Open parent", + }, title: "Subagents", pillLabelOne: "1 subagent", pillLabelMany: "{{count}} subagents", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index feafcd225f..b62913d268 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -1956,6 +1956,13 @@ export const es: TranslationResources = { backdrop: "Fondo del menú", }, subagents: { + parent: { + backToParent: "Volver al padre", + backTo: "Volver a {{label}}", + parent: "Padre", + parentAgent: "Agente padre", + openParent: "Abrir padre", + }, title: "Subagentes", pillLabelOne: "1 subagente", pillLabelMany: "{{count}} subagentes", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 6d8ee58400..cc3742abad 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -1954,6 +1954,13 @@ export const fr: TranslationResources = { backdrop: "Arrière-plan du menu", }, subagents: { + parent: { + backToParent: "Retour au parent", + backTo: "Retour à {{label}}", + parent: "Parent", + parentAgent: "Agent parent", + openParent: "Ouvrir le parent", + }, title: "Sous-agents", pillLabelOne: "1 sous-agent", pillLabelMany: "{{count}} sous-agents", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index b8af6f91f6..67d46fe471 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -1927,6 +1927,13 @@ export const ja: TranslationResources = { backdrop: "メニューの背景", }, subagents: { + parent: { + backToParent: "親に戻る", + backTo: "{{label}} に戻る", + parent: "親", + parentAgent: "親エージェント", + openParent: "親を開く", + }, title: "サブエージェント", pillLabelOne: "サブエージェント 1 件", pillLabelMany: "サブエージェント {{count}} 件", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index 1a9b36d9bd..d95899ec35 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -1918,6 +1918,13 @@ export const ko: TranslationResources = { backdrop: "메뉴 배경", }, subagents: { + parent: { + backToParent: "상위 에이전트로 돌아가기", + backTo: "{{label}}(으)로 돌아가기", + parent: "상위", + parentAgent: "상위 에이전트", + openParent: "상위 에이전트 열기", + }, title: "하위 에이전트", pillLabelOne: "하위 에이전트 1개", pillLabelMany: "하위 에이전트 {{count}}개", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 0d7e6acb89..fd41468395 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -1941,6 +1941,13 @@ export const ptBR: TranslationResources = { backdrop: "Fundo do menu", }, subagents: { + parent: { + backToParent: "Voltar ao pai", + backTo: "Voltar para {{label}}", + parent: "Pai", + parentAgent: "Agente pai", + openParent: "Abrir pai", + }, title: "Subagentes", pillLabelOne: "1 subagente", pillLabelMany: "{{count}} subagentes", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index 09d46fa075..093fc60a51 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -1940,6 +1940,13 @@ export const ru: TranslationResources = { backdrop: "Фон меню", }, subagents: { + parent: { + backToParent: "Назад к родителю", + backTo: "Назад к {{label}}", + parent: "Родитель", + parentAgent: "Родительский агент", + openParent: "Открыть родителя", + }, title: "Субагенты", pillLabelOne: "1 субагент", pillLabelMany: "Субагентов: {{count}}", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index 374bef6e88..d899a10fe8 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -1886,6 +1886,13 @@ export const zhCN: TranslationResources = { backdrop: "菜单背景", }, subagents: { + parent: { + backToParent: "返回父代理", + backTo: "返回 {{label}}", + parent: "父代理", + parentAgent: "父代理", + openParent: "打开父代理", + }, title: "Subagent", pillLabelOne: "1 个 subagent", pillLabelMany: "{{count}} 个 subagent", diff --git a/packages/app/src/panels/agent-panel.tsx b/packages/app/src/panels/agent-panel.tsx index af14b94229..ea7cc68b37 100644 --- a/packages/app/src/panels/agent-panel.tsx +++ b/packages/app/src/panels/agent-panel.tsx @@ -80,6 +80,7 @@ import { } from "@/screens/agent/agent-ready-screen-bottom-anchor"; import { WorkspaceDraftAgentTab } from "@/composer/draft/workspace-tab"; import { AgentTracks, hasAgentTracks } from "@/panels/agent-tracks"; +import { ParentAgentCallout, useParentAgentId } from "@/subagents/parent"; import { useCreateFlowStore } from "@/stores/create-flow-store"; import { buildDraftStoreKey, generateDraftId } from "@/stores/draft-keys"; import { @@ -1109,7 +1110,11 @@ const ChatAgentReadyContent = memo(function ChatAgentReadyContent({ }); const hasPluginComposerPills = useHasPluginComposerPills(serverId, workspaceId, agentId); const hasActiveComposer = !agentState.archivedAt && !isArchivingCurrentAgent; + const parentAgentId = useParentAgentId(serverId, agentId); + // With a composer the parent is a pill above it; without one, the box takes the composer's place. + const parentPillAgentId = showsComposer ? parentAgentId : null; const hasVisibleAgentTracks = hasAgentTracks({ + parentAgentId: parentPillAgentId, subagentRows, tasks, archiveFinishedStatus: archiveFinishedSubagents.status, @@ -1161,6 +1166,9 @@ const ChatAgentReadyContent = memo(function ChatAgentReadyContent({ composerState, ], ); + const parentCallout = parentAgentId ? ( + + ) : null; const composerSection = showsComposer ? ( - ) : null; + ) : ( + parentCallout + ); const streamContent = ( {showsComposer ? null : } @@ -1203,6 +1213,7 @@ const ChatAgentReadyContent = memo(function ChatAgentReadyContent({ serverId={serverId} workspaceId={workspaceId} agentId={agentId} + parentAgentId={parentPillAgentId} subagentRows={subagentRows} tasks={tasks} archiveFinishedStatus={archiveFinishedSubagents.status} diff --git a/packages/app/src/panels/agent-tracks.tsx b/packages/app/src/panels/agent-tracks.tsx index 3fc42030a7..f7c724133a 100644 --- a/packages/app/src/panels/agent-tracks.tsx +++ b/packages/app/src/panels/agent-tracks.tsx @@ -16,6 +16,7 @@ import { useSubagentOpensInSheet, type SubagentRow, } from "@/subagents"; +import { ParentAgentPill } from "@/subagents/parent"; import { SubagentsTrack } from "@/subagents/track"; import type { TodoEntry } from "@/types/stream"; import { navigateToAgent } from "@/utils/navigate-to-agent"; @@ -34,6 +35,7 @@ export const AgentTracks = memo(function AgentTracks({ serverId, workspaceId, agentId, + parentAgentId, subagentRows, tasks, archiveFinishedStatus, @@ -43,6 +45,7 @@ export const AgentTracks = memo(function AgentTracks({ serverId: string; workspaceId: string; agentId: string; + parentAgentId: string | null; subagentRows: SubagentRow[]; tasks: TodoEntry[] | undefined; archiveFinishedStatus: ArchiveFinishedStatus; @@ -88,8 +91,12 @@ export const AgentTracks = memo(function AgentTracks({ [openSubagentTarget, serverId, workspaceId], ); const handleOpenProviderSubagent = useCallback( - (parentAgentId: string, subagentId: string) => { - openSubagentTarget({ kind: "provider_subagent", parentAgentId, subagentId }); + (ownerAgentId: string, subagentId: string) => { + openSubagentTarget({ + kind: "provider_subagent", + parentAgentId: ownerAgentId, + subagentId, + }); }, [openSubagentTarget], ); @@ -108,6 +115,7 @@ export const AgentTracks = memo(function AgentTracks({ if ( !hasWorkspaceDiffStat && !hasAgentTracks({ + parentAgentId, subagentRows, tasks, archiveFinishedStatus, @@ -119,6 +127,7 @@ export const AgentTracks = memo(function AgentTracks({ return ( + {parentAgentId ? : null} 0 || Boolean(tasks?.length) || archiveFinishedStatus.kind !== "idle" || diff --git a/packages/app/src/panels/provider-subagent-panel.tsx b/packages/app/src/panels/provider-subagent-panel.tsx index ecd4a082bf..e36f7880ca 100644 --- a/packages/app/src/panels/provider-subagent-panel.tsx +++ b/packages/app/src/panels/provider-subagent-panel.tsx @@ -16,6 +16,7 @@ import { usePaneContext } from "@/panels/pane-context"; import { PaneSubtitle } from "@/panels/pane-subtitle"; import { definePanel, type PanelDescriptor } from "@/panels/panel-registry"; import { useSessionStore } from "@/stores/session-store"; +import { ParentAgentCallout } from "@/subagents/parent"; import { type SubagentRow, useSubagentsForParent } from "@/subagents/select"; import { useSubagentOpensInSheet } from "@/subagents/opens-in-sheet"; import { SubagentsTrack } from "@/subagents/track"; @@ -253,33 +254,37 @@ function ProviderSubagentPanel() { return ( {subtitle ? : null} - - + + + + + ); } const styles = StyleSheet.create((theme) => ({ container: { flex: 1, minHeight: 0 }, + stream: { flex: 1, minHeight: 0 }, unsupported: { flex: 1, alignItems: "center", justifyContent: "center", padding: 24 }, unsupportedText: { color: theme.colors.foregroundMuted, textAlign: "center" }, })); diff --git a/packages/app/src/subagents/parent/callout.tsx b/packages/app/src/subagents/parent/callout.tsx new file mode 100644 index 0000000000..f76e1267e3 --- /dev/null +++ b/packages/app/src/subagents/parent/callout.tsx @@ -0,0 +1,116 @@ +import type { ReactElement } from "react"; +import { Text, View } from "react-native"; +import { useTranslation } from "react-i18next"; +import { CornerUpLeft } from "lucide-react-native"; +import { StyleSheet, withUnistyles } from "react-native-unistyles"; +import { Button } from "@/components/ui/button"; +import { ComposerTrackMark } from "@/composer/tracks"; +import { FOOTER_HEIGHT } from "@/constants/layout"; +import type { Theme } from "@/styles/theme"; +import { useParentAgent, type ParentAgent } from "./use-parent-agent"; + +function ParentIcon({ + icon: Icon, + size, + color = "", +}: { + icon: ParentAgent["icon"]; + size: number; + color?: string; +}) { + return ; +} + +const ThemedParentIcon = withUnistyles(ParentIcon); +const foregroundColorMapping = (theme: Theme) => ({ color: theme.colors.foreground }); + +/** Stands in the composer's place on a subagent shown without one: who the parent is, and the way back. */ +export function ParentAgentCallout({ + serverId, + parentAgentId, +}: { + serverId: string; + parentAgentId: string; +}): ReactElement { + const { t } = useTranslation(); + const parent = useParentAgent(serverId, parentAgentId); + + return ( + + + + + + + {t("subagents.parent.parent")} + + + {parent.label ?? t("subagents.parent.parentAgent")} + + + + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + rail: { + minHeight: FOOTER_HEIGHT, + alignItems: "center", + width: "100%", + padding: theme.spacing[4], + }, + callout: { + width: "100%", + maxWidth: theme.contentMaxWidth, + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[3], + backgroundColor: theme.colors.surface1, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.borderAccent, + borderRadius: theme.borderRadius["2xl"], + padding: theme.spacing[3], + paddingRight: { xs: theme.spacing[3], md: theme.spacing[4] }, + }, + avatar: { + width: 36, + height: 36, + borderRadius: theme.borderRadius.lg, + backgroundColor: theme.colors.surface2, + alignItems: "center", + justifyContent: "center", + }, + body: { + flex: 1, + minWidth: 0, + gap: theme.spacing[0.5], + }, + eyebrow: { + fontSize: theme.fontSize.sm, + color: theme.colors.foregroundMuted, + }, + titleRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + minWidth: 0, + }, + title: { + flexShrink: 1, + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.medium, + color: theme.colors.foreground, + }, +})); diff --git a/packages/app/src/subagents/parent/index.ts b/packages/app/src/subagents/parent/index.ts new file mode 100644 index 0000000000..daca7d3d16 --- /dev/null +++ b/packages/app/src/subagents/parent/index.ts @@ -0,0 +1,3 @@ +export { ParentAgentCallout } from "./callout"; +export { ParentAgentPill } from "./pill"; +export { useParentAgentId } from "./use-parent-agent"; diff --git a/packages/app/src/subagents/parent/pill.tsx b/packages/app/src/subagents/parent/pill.tsx new file mode 100644 index 0000000000..63f9407f8b --- /dev/null +++ b/packages/app/src/subagents/parent/pill.tsx @@ -0,0 +1,79 @@ +import { useCallback, useMemo, useState, type ReactElement } from "react"; +import { Text } from "react-native"; +import { useTranslation } from "react-i18next"; +import { ChevronLeft } from "lucide-react-native"; +import { StyleSheet, withUnistyles } from "react-native-unistyles"; +import { PressHighlight } from "@/components/ui/press-highlight"; +import { composerPillStyles } from "@/composer/pill-styles"; +import { ComposerTrackMark } from "@/composer/tracks"; +import type { Theme } from "@/styles/theme"; +import { useParentAgent } from "./use-parent-agent"; + +const ThemedChevronLeft = withUnistyles(ChevronLeft); +const mutedColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundMuted }); +const foregroundColorMapping = (theme: Theme) => ({ color: theme.colors.foreground }); + +/** + * The way back from a subagent to the agent that started it. It leads the pill rail and names + * the parent rather than saying "parent": in a stack of subagents the title is what tells you + * where you will land. + */ +export function ParentAgentPill({ + serverId, + parentAgentId, +}: { + serverId: string; + parentAgentId: string; +}): ReactElement { + const { t } = useTranslation(); + const parent = useParentAgent(serverId, parentAgentId); + const [isHovered, setIsHovered] = useState(false); + const handleHoverIn = useCallback(() => setIsHovered(true), []); + const handleHoverOut = useCallback(() => setIsHovered(false), []); + const bodyStyle = useMemo( + () => [composerPillStyles.body, styles.body, isHovered && composerPillStyles.bodyActive], + [isHovered], + ); + const labelStyle = useMemo( + () => [composerPillStyles.label, styles.label, isHovered && composerPillStyles.labelActive], + [isHovered], + ); + const label = parent.label ?? t("subagents.parent.backToParent"); + + return ( + + + + + {label} + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + // The chevron's box is wider than its stroke, so the pill pulls it toward the edge to keep the + // visible glyph on the same inset as other pills' leading marks. + body: { + flexShrink: 1, + minWidth: 0, + gap: theme.spacing[1.5], + paddingLeft: theme.spacing[2], + }, + label: { + flexShrink: 1, + maxWidth: { xs: 140, md: 240 }, + }, +})); diff --git a/packages/app/src/subagents/parent/use-parent-agent.ts b/packages/app/src/subagents/parent/use-parent-agent.ts new file mode 100644 index 0000000000..e3916e6bbe --- /dev/null +++ b/packages/app/src/subagents/parent/use-parent-agent.ts @@ -0,0 +1,38 @@ +import { useCallback } from "react"; +import { useAgentDescriptor } from "@/agent-presentation"; +import type { PanelDescriptor } from "@/panels/panel-registry"; +import { useSessionStore } from "@/stores/session-store"; +import type { SidebarStateBucket } from "@/utils/sidebar-agent-state"; +import { navigateToAgent } from "@/utils/navigate-to-agent"; + +export interface ParentAgent { + /** The parent's title, or null while it has none — callers fall back to a generic name. */ + label: string | null; + icon: PanelDescriptor["icon"]; + statusBucket: SidebarStateBucket | null; + open: () => void; +} + +/** The parent of a managed agent, or null for a root or detached agent. */ +export function useParentAgentId(serverId: string, agentId: string): string | null { + return useSessionStore((state) => { + const session = state.sessions[serverId]; + const agent = session?.agents.get(agentId) ?? session?.agentDetails.get(agentId); + return agent?.parentAgentId ?? null; + }); +} + +/** The parent as its own tab presents it, and the way back to it. */ +export function useParentAgent(serverId: string, parentAgentId: string): ParentAgent { + const descriptor = useAgentDescriptor({ kind: "agent", agentId: parentAgentId }, { serverId }); + // The parent may live in another workspace; navigateToAgent opens it wherever it is. + const open = useCallback(() => { + navigateToAgent({ serverId, agentId: parentAgentId }); + }, [parentAgentId, serverId]); + return { + label: descriptor.label || null, + icon: descriptor.icon, + statusBucket: descriptor.statusBucket, + open, + }; +} From 16fe07560f58ae0b35ccedaf40d04f426833b2b3 Mon Sep 17 00:00:00 2001 From: Alessio Re Date: Mon, 26 Oct 2026 04:51:44 +0100 Subject: [PATCH 06/32] fix(app): make question cards link-aware and selectable without accidental answers Render question bodies and option text with the shared Markdown renderer. Opening links or selecting question and description text leaves answers unchanged. Labels remain answer controls, and double-clicking a label advances only once. Add browser regression coverage and stabilize Markdown WASM loading in the test harness. Closes #3164. Co-authored-by: Alessio Re --- .../question-prompt-pagination.spec.ts | 212 +++++++++++++++++- .../app/src/components/markdown/renderer.tsx | 25 ++- .../app/src/components/markdown/styles.ts | 8 +- .../question-form-card.browser.test.tsx | 6 + .../app/src/components/question-form-card.tsx | 77 ++++--- packages/app/test-stubs/expo-clipboard.ts | 10 + .../app/test-stubs/lucide-react-native.ts | 5 + packages/app/vitest.config.ts | 19 ++ .../agent/providers/mock-load-test-agent.ts | 29 +++ 9 files changed, 348 insertions(+), 43 deletions(-) create mode 100644 packages/app/test-stubs/expo-clipboard.ts diff --git a/packages/app/e2e/browser/question-prompt-pagination.spec.ts b/packages/app/e2e/browser/question-prompt-pagination.spec.ts index b444a8cddf..90a87433ed 100644 --- a/packages/app/e2e/browser/question-prompt-pagination.spec.ts +++ b/packages/app/e2e/browser/question-prompt-pagination.spec.ts @@ -1,6 +1,6 @@ import { fork } from "node:child_process"; import path from "node:path"; -import { expect } from "@playwright/test"; +import { expect, type BrowserContext, type Locator, type Page } from "@playwright/test"; import { z } from "zod"; import { test } from "../support/fixtures"; import { expectVisibleAgentSurfacesIdle } from "../support/helpers/agent-stream"; @@ -34,8 +34,218 @@ const ROLLOUT_QUESTION = "Which rollout should we use?"; const SUCCESS_QUESTION = "What success criteria should we use?"; const REPO_URL_QUESTION = "What is the GitHub private repo URL to push to?"; const COMMIT_MESSAGE_QUESTION = "What should the first commit message be?"; +const LINKED_QUESTION = "Sign in at https://example.com/login, then follow the setup guide. Done?"; + +function selectedText(page: Page) { + return page.evaluate(() => window.getSelection()?.toString().trim()); +} + +const CHECKS_QUESTION = "Which checks should run?"; +const DONE_DESCRIPTION = "Logs are at https://example.com/logs if sign-in fails."; +const LINT_DESCRIPTION = "Rules: https://example.com/lint"; + +async function openLinkedQuestions(page: Page, context: BrowserContext) { + await context.route("https://example.com/**", (route) => + route.fulfill({ status: 200, body: "ok" }), + ); + const session = await seedMockAgentWorkspace({ + repoPrefix: "question-option-links-", + title: "Question option links e2e", + initialPrompt: "Emit synthetic questions with links.", + }); + await openAgentRoute(page, session); + await waitForQuestionPrompt(page, 120_000); + return { session, card: page.getByTestId("question-form-card").first() }; +} + +async function dragAcross(page: Page, target: Locator) { + const bounds = await target.boundingBox(); + if (!bounds) throw new Error("Text is not visible"); + const y = bounds.y + bounds.height / 2; + await page.mouse.move(bounds.x + 1, y); + await page.mouse.down(); + await page.mouse.move(bounds.x + bounds.width - 1, y, { steps: 12 }); + await page.mouse.up(); +} + +async function clickLinkAndExpectPage(context: BrowserContext, link: Locator, url: string) { + const popup = context.waitForEvent("page"); + await link.click(); + await expect.poll(async () => (await popup).url()).toBe(url); +} + +async function expectChecked(option: Locator, checked: boolean) { + await expect(option).toHaveAttribute("aria-checked", String(checked)); +} + +test.describe("Question option text", () => { + test("links in an option label or description open without answering", async ({ + page, + context, + }) => { + test.setTimeout(180_000); + const { session, card } = await openLinkedQuestions(page, context); + try { + const done = card.getByRole("radio", { name: "Done" }); + const notYet = card.getByRole("radio", { name: "Not yet" }); + + await clickLinkAndExpectPage( + context, + notYet.getByRole("link", { name: "status" }), + "https://example.com/status", + ); + await clickLinkAndExpectPage( + context, + done.getByRole("link", { name: "https://example.com/logs" }), + "https://example.com/logs", + ); + + await expectCurrentQuestion(page, { index: 1, total: 2, question: LINKED_QUESTION }); + await expectChecked(done, false); + await expectChecked(notYet, false); + } finally { + await session.cleanup(); + } + }); + + test("selecting description text never answers, and the label still does", async ({ + page, + context, + }) => { + test.setTimeout(180_000); + const { session, card } = await openLinkedQuestions(page, context); + try { + const done = card.getByRole("radio", { name: "Done" }); + const description = done.getByTestId("question-form-option-description"); + + await dragAcross(page, description); + await expect.poll(() => selectedText(page)).toBe(DONE_DESCRIPTION); + await expectChecked(done, false); + + await description.getByText("Logs are at", { exact: false }).dblclick({ + position: { x: 4, y: 4 }, + }); + await expect.poll(() => selectedText(page)).toBe("Logs"); + await expectCurrentQuestion(page, { index: 1, total: 2, question: LINKED_QUESTION }); + await expectChecked(done, false); + + await done.getByText("Done", { exact: true }).click(); + await expectCurrentQuestion(page, { index: 2, total: 2, question: CHECKS_QUESTION }); + await openQuestion(page, { index: 1, total: 2 }); + await expectChecked(done, true); + } finally { + await session.cleanup(); + } + }); + + test("double-clicking a single-select label answers once", async ({ page, context }) => { + test.setTimeout(180_000); + const { session, card } = await openLinkedQuestions(page, context); + try { + await card.getByRole("radio", { name: "Done" }).getByText("Done", { exact: true }).dblclick(); + + await expectCurrentQuestion(page, { index: 2, total: 2, question: CHECKS_QUESTION }); + await expectChecked(card.getByRole("checkbox", { name: "Lint" }), false); + await expectChecked(card.getByRole("checkbox", { name: "Typecheck" }), false); + await openQuestion(page, { index: 1, total: 2 }); + await expectChecked(card.getByRole("radio", { name: "Done" }), true); + } finally { + await session.cleanup(); + } + }); + + test("multi-select description links and selection leave the option unchanged", async ({ + page, + context, + }) => { + test.setTimeout(180_000); + const { session, card } = await openLinkedQuestions(page, context); + try { + await card.getByRole("radio", { name: "Done" }).getByText("Done", { exact: true }).click(); + await expectCurrentQuestion(page, { index: 2, total: 2, question: CHECKS_QUESTION }); + + const lint = card.getByRole("checkbox", { name: "Lint" }); + const typecheck = card.getByRole("checkbox", { name: "Typecheck" }); + const description = lint.getByTestId("question-form-option-description"); + + await clickLinkAndExpectPage( + context, + description.getByRole("link", { name: "https://example.com/lint" }), + "https://example.com/lint", + ); + await expectChecked(lint, false); + + await dragAcross(page, description); + await expect.poll(() => selectedText(page)).toBe(LINT_DESCRIPTION); + await expectChecked(lint, false); + + await description.getByText("Rules:", { exact: false }).dblclick({ + position: { x: 4, y: 4 }, + }); + await expect.poll(() => selectedText(page)).toBe("Rules"); + await expectChecked(lint, false); + + await typecheck.getByText("Typecheck", { exact: true }).dblclick(); + await expectChecked(typecheck, true); + await lint.getByText("Lint", { exact: true }).click(); + await expectChecked(lint, true); + } finally { + await session.cleanup(); + } + }); +}); test.describe("Question prompt pagination", () => { + test("question links open without answering, and the question text is selectable", async ({ + page, + context, + }) => { + test.setTimeout(180_000); + await context.route("https://example.com/**", (route) => + route.fulfill({ status: 200, body: "ok" }), + ); + + const session = await seedMockAgentWorkspace({ + repoPrefix: "question-links-", + title: "Question links e2e", + initialPrompt: "Emit synthetic questions with links.", + }); + + try { + await openAgentRoute(page, session); + await waitForQuestionPrompt(page, 120_000); + + const card = page.getByTestId("question-form-card").first(); + const question = card.getByTestId("question-form-current-question"); + await expect(question).toHaveText(LINKED_QUESTION); + await expect(question.getByRole("link", { name: "https://example.com/login" })).toBeVisible(); + await expect(question.getByRole("link", { name: "the setup guide" })).toBeVisible(); + + const bounds = await question.boundingBox(); + if (!bounds) throw new Error("Question text is not visible"); + const y = bounds.y + bounds.height / 2; + await page.mouse.move(bounds.x + 1, y); + await page.mouse.down(); + await page.mouse.move(bounds.x + bounds.width - 1, y, { steps: 12 }); + await page.mouse.up(); + await expect.poll(() => selectedText(page)).toBe(LINKED_QUESTION); + + const popup = context.waitForEvent("page"); + await question.getByRole("link", { name: "the setup guide" }).click(); + await expect.poll(async () => (await popup).url()).toBe("https://example.com/setup"); + + await expect(page.getByTestId("question-form-card")).toHaveCount(1); + for (const option of ["Done", "Not yet"]) { + await expect(card.getByRole("radio", { name: option })).toHaveAttribute( + "aria-checked", + "false", + ); + } + } finally { + await session.cleanup(); + } + }); + test("shows one question at a time with numbered navigation", async ({ page }) => { test.setTimeout(180_000); diff --git a/packages/app/src/components/markdown/renderer.tsx b/packages/app/src/components/markdown/renderer.tsx index 6d79beb01f..8e51495369 100644 --- a/packages/app/src/components/markdown/renderer.tsx +++ b/packages/app/src/components/markdown/renderer.tsx @@ -23,14 +23,29 @@ const ThemedMarkdown = withUnistyles(EnrichedMarkdownText); const MD4C_FLAGS = { latexMath: false, hardSoftBreaks: true, admonitions: true }; // Reading the breakpoint here makes Unistyles restyle the text when the layout crosses it. -const markdownStyle = (compact: boolean) => (theme: Theme, rt: { breakpoint?: string }) => - createMarkdownStyles(theme, { compact, blockGap: resolveDensity(rt, density.blockGap) }); +const markdownStyle = + (compact: boolean, muted = false) => + (theme: Theme, rt: { breakpoint?: string }) => + createMarkdownStyles(theme, { + compact, + muted, + blockGap: resolveDensity(rt, density.blockGap), + }); const regularStyle = markdownStyle(false); const compactStyle = markdownStyle(true); +const compactMutedStyle = markdownStyle(true, true); + +function pickMarkdownStyle(compact: boolean, muted: boolean) { + if (!compact) return regularStyle; + return muted ? compactMutedStyle : compactStyle; +} export interface MarkdownRendererProps extends MarkdownImageContext { text: string; compact?: boolean; + /** Muted text colour. Compact surfaces only. */ + muted?: boolean; + selectable?: boolean; streaming?: boolean; onLinkPress?: (url: string) => unknown; } @@ -56,6 +71,8 @@ export const MarkdownRenderer = memo(function MarkdownRenderer(props: MarkdownRe const MarkdownPart = memo(function MarkdownPart({ text, compact = false, + muted = false, + selectable = true, streaming = false, onLinkPress, ...imageContext @@ -98,9 +115,9 @@ const MarkdownPart = memo(function MarkdownPart({ ({ diff --git a/packages/app/src/components/question-form-card.browser.test.tsx b/packages/app/src/components/question-form-card.browser.test.tsx index 83120931d1..0b2a5df25d 100644 --- a/packages/app/src/components/question-form-card.browser.test.tsx +++ b/packages/app/src/components/question-form-card.browser.test.tsx @@ -10,6 +10,12 @@ import { QuestionFormCard } from "./question-form-card"; // Load translations so controls expose their real accessible names. void testI18n; +// The Markdown engine does not load in this harness; question and option text are covered by +// e2e/browser/question-prompt-pagination.spec.ts. Plain text keeps these tests on the answers. +vi.mock("@/components/markdown/renderer", () => ({ + MarkdownRenderer: ({ text }: { text: string }) => {text}, +})); + // App sources compile against the classic JSX runtime, which expects React on the global. beforeEach(() => vi.stubGlobal("React", React)); diff --git a/packages/app/src/components/question-form-card.tsx b/packages/app/src/components/question-form-card.tsx index 7014de89d1..8ff81963d1 100644 --- a/packages/app/src/components/question-form-card.tsx +++ b/packages/app/src/components/question-form-card.tsx @@ -1,7 +1,13 @@ import { Alert } from "@/components/ui/alert"; import { LoadingSpinner } from "@/components/ui/loading-spinner"; import { useState, useCallback, useMemo, useRef, type RefObject } from "react"; -import { View, Text, Pressable, type PressableStateCallbackType } from "react-native"; +import { + View, + Text, + Pressable, + type GestureResponderEvent, + type PressableStateCallbackType, +} from "react-native"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { useIsCompactFormFactor } from "@/constants/layout"; import { Check, X } from "lucide-react-native"; @@ -12,6 +18,7 @@ import { isWeb } from "@/constants/platform"; import { EditingTextInput as TextInput } from "@/components/ui/text-input"; import type { EditingTextInputHandle } from "@/components/ui/text-input/types"; import { ScrollableCodeSurface } from "@/components/ui/scrollable-code-surface"; +import { MarkdownRenderer } from "@/components/markdown/renderer"; import { areQuestionsAnswered, buildQuestionFormAnswers, @@ -36,6 +43,20 @@ const IS_WEB = isWeb; // drag to the preview's nested scroll, so lines past a cap would be unreachable. const OPTION_PREVIEW_MAX_HEIGHT = IS_WEB ? 240 : undefined; +// Web clicks that must not answer: a link in the label, anything in the description, and the +// second click of a double-click, which would otherwise land on the next question's options. +function isNonAnsweringClick(event: GestureResponderEvent, description: View | null): boolean { + if (!IS_WEB) return false; + const click = event.nativeEvent as unknown as MouseEvent; + if (click.detail > 1) return true; + const target = click.target as Element | null; + const descriptionElement = description as unknown as Element | null; + return Boolean(target?.closest("a") || descriptionElement?.contains(target)); +} + +// The description is selectable text, so a touch on it belongs to the text, not the option. +const claimResponder = () => true; + function getQuestionInputPlaceholder({ question, answerPlaceholder, @@ -71,9 +92,14 @@ function QuestionOptionRow({ }: QuestionOptionRowProps) { const { theme } = useUnistyles(); - const handlePress = useCallback(() => { - onToggle(qIndex, optIndex, multiSelect); - }, [onToggle, qIndex, optIndex, multiSelect]); + const descriptionRef = useRef(null); + const handlePress = useCallback( + (event: GestureResponderEvent) => { + if (isNonAnsweringClick(event, descriptionRef.current)) return; + onToggle(qIndex, optIndex, multiSelect); + }, + [onToggle, qIndex, optIndex, multiSelect], + ); const pressableStyle = useCallback( ({ pressed, hovered }: PressableStateCallbackType & { hovered?: boolean }) => [ @@ -86,17 +112,6 @@ function QuestionOptionRow({ [isSelected, theme.colors.surface2], ); - const optionLabelStyle = useMemo( - () => [ - styles.optionLabel, - { color: isSelected ? theme.colors.foreground : theme.colors.foregroundMuted }, - ], - [isSelected, theme.colors.foreground, theme.colors.foregroundMuted], - ); - const optionDescriptionStyle = useMemo( - () => [styles.optionDescription, { color: theme.colors.foregroundMuted }], - [theme.colors.foregroundMuted], - ); const accessibilityState = useMemo(() => ({ checked: isSelected }), [isSelected]); // Static left-side control: square for multi-select, circle for single-select. @@ -135,9 +150,15 @@ function QuestionOptionRow({ {isSelected && !multiSelect ? : null} - {option.label} + {option.description ? ( - {option.description} + + + ) : null} {option.preview ? ( // Shown for every option at once so they can be compared before picking. @@ -522,10 +543,6 @@ export function QuestionFormCard({ ], [theme.colors.surface1, theme.colors.border], ); - const questionTextStyle = useMemo( - () => [styles.questionText, { color: theme.colors.foreground }], - [theme.colors.foreground], - ); // Single-select radios need a group; checkboxes are valid standalone. const optionsGroupAccessibility = useMemo( () => @@ -570,9 +587,9 @@ export function QuestionFormCard({ onSelect={handleSelectQuestion} /> - - {activeQuestion?.question} - + + + {activeQuestion ? ( @@ -675,9 +692,6 @@ const styles = StyleSheet.create((theme) => ({ }, questionText: { flex: 1, - fontSize: theme.fontSize.base, - fontWeight: theme.fontWeight.normal, - lineHeight: 22, }, optionsWrap: { gap: theme.spacing[1], @@ -723,15 +737,6 @@ const styles = StyleSheet.create((theme) => ({ flex: 1, gap: theme.spacing[1], }, - optionLabel: { - fontSize: theme.fontSize.base, - fontWeight: theme.fontWeight.normal, - lineHeight: 22, - }, - optionDescription: { - fontSize: theme.fontSize.base, - lineHeight: 20, - }, optionPreviewText: { lineHeight: 20, ...(isWeb ? { whiteSpace: "pre-wrap", overflowWrap: "break-word" } : null), diff --git a/packages/app/test-stubs/expo-clipboard.ts b/packages/app/test-stubs/expo-clipboard.ts new file mode 100644 index 0000000000..9663320efb --- /dev/null +++ b/packages/app/test-stubs/expo-clipboard.ts @@ -0,0 +1,10 @@ +// The package ships JSX in .js files, which the dependency optimizer cannot parse. +export async function setStringAsync(): Promise { + return true; +} +export async function getStringAsync(): Promise { + return ""; +} +export async function hasStringAsync(): Promise { + return false; +} diff --git a/packages/app/test-stubs/lucide-react-native.ts b/packages/app/test-stubs/lucide-react-native.ts index 200048ad53..a85976e1bf 100644 --- a/packages/app/test-stubs/lucide-react-native.ts +++ b/packages/app/test-stubs/lucide-react-native.ts @@ -111,3 +111,8 @@ export const Upload = StubIcon; export const Wrench = StubIcon; export const X = StubIcon; export const XCircle = StubIcon; +export const Code = StubIcon; +export const Maximize2 = StubIcon; +export const Workflow = StubIcon; +export const ZoomIn = StubIcon; +export const ZoomOut = StubIcon; diff --git a/packages/app/vitest.config.ts b/packages/app/vitest.config.ts index 2d7149442e..2b56646b2a 100644 --- a/packages/app/vitest.config.ts +++ b/packages/app/vitest.config.ts @@ -19,6 +19,20 @@ const resolvePackageEntry = (packageName: string) => { export default defineConfig({ plugins: [ + { + name: "markdown-wasm-commonjs-export", + transform(source, id) { + // The ESM package contains a CommonJS WASM factory. Metro handles the + // interop; Vite needs an ESM export instead of the CommonJS trailer. + if (!id.endsWith("/lib/module/web/wasm/md4c.js")) return; + const trailer = source.lastIndexOf("\nif (typeof exports === 'object'"); + if (trailer < 0) throw new Error("Markdown WASM CommonJS trailer was not found"); + return { + code: `${source.slice(0, trailer)}\nexport default createMd4cModule;`, + map: null, + }; + }, + }, { name: "compile-usage-components", enforce: "pre", @@ -151,6 +165,7 @@ export default defineConfig({ "react-native-gesture-handler > invariant", ], exclude: [ + "react-native-enriched-markdown", "react-native-reanimated", "react-native-gesture-handler", "react-native-keyboard-controller", @@ -254,6 +269,10 @@ export default defineConfig({ find: /^react-native-reanimated\/scripts\/validate-worklets-version$/, replacement: path.resolve(__dirname, "test-stubs/reanimated-validate-worklets-version.ts"), }, + { + find: /^expo-clipboard$/, + replacement: path.resolve(__dirname, "test-stubs/expo-clipboard.ts"), + }, { find: /^expo-linking$/, replacement: path.resolve(__dirname, "test-stubs/expo-linking.ts"), diff --git a/packages/server/src/server/agent/providers/mock-load-test-agent.ts b/packages/server/src/server/agent/providers/mock-load-test-agent.ts index b7b301540a..7055030dc0 100644 --- a/packages/server/src/server/agent/providers/mock-load-test-agent.ts +++ b/packages/server/src/server/agent/providers/mock-load-test-agent.ts @@ -295,6 +295,35 @@ function parseMockQuestionPrompt(prompt: AgentPromptInput): MockQuestionPromptRe return null; } + if (/with\s+links/i.test(text)) { + return { + questions: [ + { + question: + "Sign in at https://example.com/login, then follow [the setup guide](https://example.com/setup). Done?", + header: "signIn", + options: [ + { + label: "Done", + description: "Logs are at https://example.com/logs if sign-in fails.", + }, + { label: "Not yet, see [status](https://example.com/status)" }, + ], + multiSelect: false, + }, + { + question: "Which checks should run?", + header: "checks", + options: [ + { label: "Lint", description: "Rules: https://example.com/lint" }, + { label: "Typecheck" }, + ], + multiSelect: true, + }, + ], + }; + } + if (/free[-\s]?write|freeform|text[-\s]?only/i.test(text)) { return { questions: [ From 6f07b0de489932e1c1dbe3e42f243fc054335700 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 11:18:47 +0700 Subject: [PATCH 07/32] feat(app): recall sent messages with Up and Down in an empty composer (#7710) Fixing a prompt after interrupting it, or retrying one after a provider error, meant scrolling up and copying it out of the chat. Up in an empty agent composer now fills the last message sent to that agent; Up walks older, Down walks newer and past the newest back to empty. History is read from the loaded timeline on key press, not kept in a separate store, so it follows rewinds, reloads and other devices and costs nothing while a turn streams. Recall only starts from an empty composer, so it never discards a half-written draft. While a recalled message is unedited the autocomplete stays hidden, otherwise a recalled `/command` would open the picker and Enter would pick an option instead of sending. --- .../app/e2e/browser/composer-recall.spec.ts | 81 +++++++++++++ packages/app/src/composer/index.tsx | 34 ++++-- packages/app/src/composer/input/input.tsx | 8 +- packages/app/src/composer/recall.test.ts | 109 ++++++++++++++++++ packages/app/src/composer/recall.ts | 68 +++++++++++ packages/app/src/stores/session-store.test.ts | 20 ++++ packages/app/src/stores/session-store.ts | 20 ++++ 7 files changed, 331 insertions(+), 9 deletions(-) create mode 100644 packages/app/e2e/browser/composer-recall.spec.ts create mode 100644 packages/app/src/composer/recall.test.ts create mode 100644 packages/app/src/composer/recall.ts diff --git a/packages/app/e2e/browser/composer-recall.spec.ts b/packages/app/e2e/browser/composer-recall.spec.ts new file mode 100644 index 0000000000..368d23409c --- /dev/null +++ b/packages/app/e2e/browser/composer-recall.spec.ts @@ -0,0 +1,81 @@ +import { expect, test, type Page } from "../support/fixtures"; +import { + composerLocator, + expectComposerDraft, + expectComposerVisible, + submitMessage, + typeIntoFocusedComposer, +} from "../support/helpers/composer"; +import { openAgentRoute, seedMockAgentWorkspace } from "../support/helpers/mock-agent"; + +function userMessages(page: Page) { + return page.getByTestId("user-message"); +} + +async function sendAndWaitForMessage(page: Page, text: string, expectedCount: number) { + await submitMessage(page, text); + await expect(userMessages(page)).toHaveCount(expectedCount, { timeout: 30_000 }); + await expectComposerDraft(page, ""); +} + +async function pressInComposer(page: Page, key: string) { + await composerLocator(page).press(key); +} + +test.describe("Composer recall", () => { + test("Up and Down walk sent messages from an empty composer", async ({ page }) => { + const session = await seedMockAgentWorkspace({ + repoPrefix: "composer-recall-", + title: "Composer recall", + }); + try { + await openAgentRoute(page, session); + await expectComposerVisible(page); + await sendAndWaitForMessage(page, "fix the auth bug", 1); + await sendAndWaitForMessage(page, "add tests", 2); + + await pressInComposer(page, "ArrowUp"); + await expectComposerDraft(page, "add tests"); + await pressInComposer(page, "ArrowUp"); + await expectComposerDraft(page, "fix the auth bug"); + await pressInComposer(page, "ArrowDown"); + await expectComposerDraft(page, "add tests"); + await pressInComposer(page, "ArrowDown"); + await expectComposerDraft(page, ""); + + await pressInComposer(page, "ArrowUp"); + await typeIntoFocusedComposer(page, " for login"); + await expectComposerDraft(page, "add tests for login"); + await pressInComposer(page, "ArrowUp"); + await expectComposerDraft(page, "add tests for login"); + + await pressInComposer(page, "Enter"); + await expect(userMessages(page)).toHaveCount(3, { timeout: 30_000 }); + await expect(userMessages(page).last()).toContainText("add tests for login"); + } finally { + await session.cleanup(); + } + }); + + test("a recalled slash message does not open the command picker", async ({ page }) => { + const session = await seedMockAgentWorkspace({ + repoPrefix: "composer-recall-slash-", + title: "Composer recall slash", + }); + try { + await openAgentRoute(page, session); + await expectComposerVisible(page); + await sendAndWaitForMessage(page, "/tdd", 1); + + const popover = page.getByTestId("composer-autocomplete-popover").filter({ visible: true }); + await pressInComposer(page, "ArrowUp"); + await expectComposerDraft(page, "/tdd"); + await expect(popover).toHaveCount(0); + + await typeIntoFocusedComposer(page, "x"); + await expect(popover.first()).toBeVisible({ timeout: 30_000 }); + } finally { + await session.cleanup(); + } + }); +}); diff --git a/packages/app/src/composer/index.tsx b/packages/app/src/composer/index.tsx index ed9ccca4f7..ba85de2516 100644 --- a/packages/app/src/composer/index.tsx +++ b/packages/app/src/composer/index.tsx @@ -55,7 +55,11 @@ import { } from "@/composer/agent-controls"; import { ContextWindowMeter } from "@/components/context-window-meter"; import { useImageAttachmentPicker } from "@/hooks/use-image-attachment-picker"; -import { selectAgentTurnPresentation, useSessionStore } from "@/stores/session-store"; +import { + selectAgentSentMessages, + selectAgentTurnPresentation, + useSessionStore, +} from "@/stores/session-store"; import { useFilePicker } from "@/hooks/use-file-picker"; import { useFileDrop } from "@/components/file-drop/use-file-drop"; import type { DroppedItem } from "@/components/file-drop/types"; @@ -118,6 +122,7 @@ import { useKeyboardActionHandler } from "@/hooks/use-keyboard-action-handler"; import type { KeyboardActionDefinition } from "@/keyboard/keyboard-action-dispatcher"; import type { MessageInputKeyboardActionKind } from "@/keyboard/actions"; import { submitAgentInput } from "@/composer/submit"; +import { ComposerRecall } from "@/composer/recall"; import { createMessageSubmissionWriter } from "@/composer/submission/writer"; import { ComposerKeyboardScopeProvider, useComposerKeyboardScope } from "@/composer/keyboard-scope"; import { useAppSettings } from "@/hooks/use-settings"; @@ -995,6 +1000,7 @@ interface ComposerAutocompleteHandle { function ComposerAutocompleteBinding({ text, cursor, + recall, configuration, inputRef, anchorRef, @@ -1003,6 +1009,7 @@ function ComposerAutocompleteBinding({ }: { text: ComposerTextSource; cursor: StoreApi; + recall: ComposerRecall; configuration: Omit< Parameters[0], "userInput" | "cursorIndex" | "onAutocompleteApplied" @@ -1020,9 +1027,11 @@ function ComposerAutocompleteBinding({ cursorIndex: Math.min(cursorIndex, userInput.length), onAutocompleteApplied: () => inputRef.current?.focus(), }); + // A recalled message is sent as is; its leading `/command` must not open the picker. + const isShown = show && !recall.isShowing(userInput); const handleKeyPress = autocomplete.onKeyPress; - useImperativeHandle(ref, () => ({ onKeyPress: (event) => show && handleKeyPress(event) }), [ - show, + useImperativeHandle(ref, () => ({ onKeyPress: (event) => isShown && handleKeyPress(event) }), [ + isShown, handleKeyPress, ]); const selectOption = autocomplete.onSelectOption; @@ -1032,7 +1041,7 @@ function ComposerAutocompleteBinding({ ); return ( () => cursorPublication.cancel(), [cursorPublication]); const autocompleteRef = useRef(null); + const recall = useMemo(() => new ComposerRecall(), []); const [isProcessing, setIsProcessing] = useState(false); const [pendingFiles, setPendingFiles] = useState([]); const nextPendingFileId = useRef(0); @@ -1981,10 +1991,19 @@ function ComposerContentImpl({ const hasSendableContent = hasText || selectedAttachments.length > 0; - // Handle keyboard navigation for command autocomplete. + // An open autocomplete owns the arrows; otherwise they may recall sent messages. const handleCommandKeyPress = useCallback( - (event: ComposerKeyPressEvent) => autocompleteRef.current?.onKeyPress(event) ?? false, - [], + (event: ComposerKeyPressEvent) => { + if (autocompleteRef.current?.onKeyPress(event)) return true; + const recalledText = recall.handleKeyPress(event, () => + selectAgentSentMessages(useSessionStore.getState().sessions[serverId], agentId), + ); + if (recalledText === null) return false; + event.preventDefault(); + replaceUserInput(recalledText); + return true; + }, + [agentId, recall, replaceUserInput, serverId], ); const cancelButtonStyle = useMemo( @@ -2423,6 +2442,7 @@ function ComposerContentImpl({ void; input: ComposerInputSnapshot; } @@ -210,6 +212,7 @@ type WebTextInputKeyPressEvent = NativeSyntheticEvent< metaKey?: boolean; ctrlKey?: boolean; shiftKey?: boolean; + altKey?: boolean; // Web-only: present on DOM KeyboardEvent during IME composition (CJK input). isComposing?: boolean; keyCode?: number; @@ -411,17 +414,18 @@ function handleDesktopKeyPressImpl( ): void { if (isImeComposingKeyboardEvent(event.nativeEvent)) return; + const { shiftKey, metaKey, ctrlKey, altKey } = event.nativeEvent; + if (ctx.onKeyPressCallback) { const handled = ctx.onKeyPressCallback({ key: event.nativeEvent.key, + hasModifier: Boolean(shiftKey || metaKey || ctrlKey || altKey), preventDefault: () => event.preventDefault(), input: ctx.input, }); if (handled) return; } - const { shiftKey, metaKey, ctrlKey } = event.nativeEvent; - if (event.nativeEvent.key !== "Enter") return; if (!ctx.submitOnEnter) return; if (shiftKey) return; diff --git a/packages/app/src/composer/recall.test.ts b/packages/app/src/composer/recall.test.ts new file mode 100644 index 0000000000..a1c6b618fc --- /dev/null +++ b/packages/app/src/composer/recall.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; +import type { ComposerKeyPressEvent } from "@/composer/input/input"; +import { ComposerRecall } from "@/composer/recall"; + +const SENT = ["add tests", "fix the auth bug"]; + +function press( + key: string, + text: string, + options: { caret?: number; selectionEnd?: number; hasModifier?: boolean } = {}, +): ComposerKeyPressEvent { + const caret = options.caret ?? text.length; + return { + key, + hasModifier: options.hasModifier ?? false, + preventDefault: () => undefined, + input: { text, selection: { start: caret, end: options.selectionEnd ?? caret } }, + }; +} + +/** Drives recall the way the composer does: each result becomes the composer text, caret at the end. */ +function composerWithRecall(sent: readonly string[] = SENT) { + const recall = new ComposerRecall(); + let text = ""; + return { + recall, + get text() { + return text; + }, + type(next: string) { + text = next; + }, + press(key: string, options: { caret?: number; hasModifier?: boolean } = {}) { + const result = recall.handleKeyPress(press(key, text, options), () => sent); + if (result !== null) text = result; + return result; + }, + }; +} + +describe("ComposerRecall", () => { + it("walks back through sent messages and forward to an empty composer", () => { + const composer = composerWithRecall(); + + expect(composer.press("ArrowUp")).toBe("add tests"); + expect(composer.press("ArrowUp")).toBe("fix the auth bug"); + expect(composer.press("ArrowDown")).toBe("add tests"); + expect(composer.press("ArrowDown")).toBe(""); + expect(composer.recall.isShowing(composer.text)).toBe(false); + }); + + it("leaves the caret to the input past the oldest message", () => { + const composer = composerWithRecall(); + composer.press("ArrowUp"); + composer.press("ArrowUp"); + + expect(composer.press("ArrowUp")).toBeNull(); + expect(composer.text).toBe("fix the auth bug"); + }); + + it("only starts from an empty composer", () => { + const composer = composerWithRecall(); + composer.type("half written"); + + expect(composer.press("ArrowUp")).toBeNull(); + expect(composer.press("ArrowDown")).toBeNull(); + }); + + it("ignores Down from an empty composer and Up with no history", () => { + expect(composerWithRecall().press("ArrowDown")).toBeNull(); + expect(composerWithRecall([]).press("ArrowUp")).toBeNull(); + }); + + it("hands the arrows back to the caret once the recalled message is edited", () => { + const composer = composerWithRecall(); + composer.press("ArrowUp"); + composer.type("add tests for login"); + + expect(composer.recall.isShowing(composer.text)).toBe(false); + expect(composer.press("ArrowUp")).toBeNull(); + expect(composer.press("ArrowDown")).toBeNull(); + }); + + it("moves between lines of a multi-line message before changing message", () => { + const composer = composerWithRecall(["first line\nsecond line", "older"]); + composer.press("ArrowUp"); + + expect(composer.press("ArrowUp", { caret: "first line\nsec".length })).toBeNull(); + expect(composer.press("ArrowUp", { caret: 3 })).toBe("older"); + expect(composer.press("ArrowDown")).toBe("first line\nsecond line"); + expect(composer.press("ArrowDown", { caret: 3 })).toBeNull(); + expect(composer.press("ArrowDown")).toBe(""); + }); + + it("ignores modified arrows, other keys, and a selection", () => { + const composer = composerWithRecall(); + + expect(composer.press("ArrowUp", { hasModifier: true })).toBeNull(); + expect(composer.press("Enter")).toBeNull(); + composer.press("ArrowUp"); + const recall = composer.recall; + expect( + recall.handleKeyPress( + press("ArrowUp", "add tests", { caret: 0, selectionEnd: 3 }), + () => SENT, + ), + ).toBeNull(); + }); +}); diff --git a/packages/app/src/composer/recall.ts b/packages/app/src/composer/recall.ts new file mode 100644 index 0000000000..24984d39e1 --- /dev/null +++ b/packages/app/src/composer/recall.ts @@ -0,0 +1,68 @@ +import type { ComposerKeyPressEvent } from "@/composer/input/input"; + +interface RecalledMessage { + index: number; + text: string; +} + +/** + * Walks the messages already sent to an agent from an empty composer, like a + * shell's Up/Down history. A message stays recalled only while the composer + * still holds its exact text; the first edit hands the arrows back to the caret. + */ +export class ComposerRecall { + private recalled: RecalledMessage | null = null; + + /** True while the composer shows a recalled message the user has not edited. */ + isShowing(text: string): boolean { + return this.recalled !== null && this.recalled.text === text; + } + + /** + * Returns the text the composer should show for this key press, or null to + * leave the key to the input. `readSentMessages` is newest first and is only + * read for keys recall acts on. + */ + handleKeyPress( + event: ComposerKeyPressEvent, + readSentMessages: () => readonly string[], + ): string | null { + if (event.key !== "ArrowUp" && event.key !== "ArrowDown") return null; + if (event.hasModifier) return null; + const { text, selection } = event.input; + if (selection.start !== selection.end) return null; + + const recalled = this.isShowing(text) ? this.recalled : null; + if (!recalled) { + this.recalled = null; + if (event.key !== "ArrowUp" || text.length > 0) return null; + return this.show(readSentMessages(), 0); + } + + if (event.key === "ArrowUp") { + if (!isCaretOnFirstLine(text, selection.start)) return null; + return this.show(readSentMessages(), recalled.index + 1); + } + if (!isCaretOnLastLine(text, selection.start)) return null; + if (recalled.index === 0) { + this.recalled = null; + return ""; + } + return this.show(readSentMessages(), recalled.index - 1); + } + + private show(messages: readonly string[], index: number): string | null { + const text = messages[index]; + if (text === undefined) return null; + this.recalled = { index, text }; + return text; + } +} + +function isCaretOnFirstLine(text: string, caret: number): boolean { + return !text.slice(0, caret).includes("\n"); +} + +function isCaretOnLastLine(text: string, caret: number): boolean { + return !text.slice(caret).includes("\n"); +} diff --git a/packages/app/src/stores/session-store.test.ts b/packages/app/src/stores/session-store.test.ts index 86ea81afc5..fb38605220 100644 --- a/packages/app/src/stores/session-store.test.ts +++ b/packages/app/src/stores/session-store.test.ts @@ -5,6 +5,7 @@ import type { WorkspaceDescriptorPayload } from "@getpaseo/protocol/messages"; import { normalizeWorkspaceDescriptor, + selectAgentSentMessages, selectAgentTurnPresentation, selectAgentTimelineState, useSessionStore, @@ -235,6 +236,25 @@ describe("agent timeline state", () => { ).toEqual({ status: "synced", items: [], range: null, older: "none", newer: "none" }); }); + it("lists sent messages newest first, including the live head", () => { + initializeTestSession(); + const sent = (id: string, text: string) => ({ ...submittedMessage(id), text }); + useSessionStore.getState().applyAgentTimelineResponseState("test-server", "agent-1", { + items: [sent("m1", "fix the auth bug"), sent("m2", "add tests"), sent("m3", "add tests")], + head: [sent("m4", " "), sent("m5", "refactor the wrong module")], + range: { epoch: "epoch-1", startSeq: 1, endSeq: 5 }, + older: "none", + newer: false, + synchronized: true, + acknowledgedClientMessageIds: [], + }); + + expect( + selectAgentSentMessages(useSessionStore.getState().sessions["test-server"], "agent-1"), + ).toEqual(["refactor the wrong module", "add tests", "fix the auth bug"]); + expect(selectAgentSentMessages(undefined, "agent-1")).toEqual([]); + }); + it("preserves older availability when an applied response leaves it unchanged", () => { initializeTestSession(); const store = useSessionStore.getState(); diff --git a/packages/app/src/stores/session-store.ts b/packages/app/src/stores/session-store.ts index aed06eef11..911573762f 100644 --- a/packages/app/src/stores/session-store.ts +++ b/packages/app/src/stores/session-store.ts @@ -332,6 +332,26 @@ export function selectAgentTimelineState( return items.length > 0 ? { status: "painted", items } : { status: "cold" }; } +/** Text of the loaded user messages, newest first, with back-to-back repeats collapsed. */ +export function selectAgentSentMessages( + session: SessionState | undefined, + agentId: string, +): string[] { + if (!session) return []; + const items = [ + ...(session.agentStreamTail.get(agentId) ?? []), + ...(session.agentStreamHead.get(agentId) ?? []), + ]; + const messages: string[] = []; + for (let index = items.length - 1; index >= 0; index -= 1) { + const item = items[index]; + if (item.kind !== "user_message" || item.text.trim().length === 0) continue; + if (messages[messages.length - 1] === item.text) continue; + messages.push(item.text); + } + return messages; +} + export function selectAgentTurnPresentation( session: SessionState | undefined, agentId: string, From 6521739947b30a3d0de950c4186310703998e2fd Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 11:22:07 +0700 Subject: [PATCH 08/32] fix(markdown): preserve inline code formatting and heading sizes (#7889) Scale inline code with the enclosing block using theme tokens, and preserve its size and font through bold and links. Use contextual package patches so clean installs reproduce the verified native source. --- .../e2e/mobile/markdown-formatting/README.md | 25 + .../e2e/mobile/markdown-formatting/native.tsx | 52 ++ packages/app/package.json | 2 +- .../src/components/markdown/renderer.test.ts | 15 + .../app/src/components/markdown/styles.ts | 2 +- ...own+1.2.0-nightly-20261014-766fc0e67.patch | 508 ++++++++++++++++-- 6 files changed, 557 insertions(+), 47 deletions(-) create mode 100644 packages/app/e2e/mobile/markdown-formatting/README.md create mode 100644 packages/app/e2e/mobile/markdown-formatting/native.tsx diff --git a/packages/app/e2e/mobile/markdown-formatting/README.md b/packages/app/e2e/mobile/markdown-formatting/README.md new file mode 100644 index 0000000000..c2ab0ddc65 --- /dev/null +++ b/packages/app/e2e/mobile/markdown-formatting/README.md @@ -0,0 +1,25 @@ +# Inline code formatting + +This fixture renders the production `MarkdownRenderer` with plain, bold, italic, +combined emphasis, strikethrough and linked code, plus bold code in headings, +lists and quotes. Toggle regular/compact text. Inline code uses the theme’s +code-to-body size ratio at every heading level; fenced code keeps the code-size +token. Bold and links preserve the resulting size. Check the glyphs, not the background height: native backgrounds use +line metrics. + +Use a development client rebuilt from this checkout. Follow the temporary entry +point and Metro workflow in [status-ring](../status-ring/README.md), importing +`MarkdownFormattingRegression` from `./e2e/mobile/markdown-formatting/native`. +Open the same Metro URL in a browser to compare web. Restore `index.ts` afterward. + +The package patch contains `InlineCodeFormattingTest`, which exercises actual +Android spans in Robolectric native graphics mode, covering draw and measurement, +span order, code inside paragraph/heading sizes, bold/italic composition, and +links preserving code and emphasized prose fonts. +It runs in the existing `test:android` CI job. To run only this regression locally, +from `packages/app/android` after Expo prebuild: + +```sh +./gradlew :react-native-enriched-markdown:testDebugUnitTest \ + --tests '*InlineCodeFormattingTest' --no-daemon --max-workers=2 +``` diff --git a/packages/app/e2e/mobile/markdown-formatting/native.tsx b/packages/app/e2e/mobile/markdown-formatting/native.tsx new file mode 100644 index 0000000000..d5d9a64aa0 --- /dev/null +++ b/packages/app/e2e/mobile/markdown-formatting/native.tsx @@ -0,0 +1,52 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { useCallback, useState } from "react"; +import { Button, ScrollView, Text } from "react-native"; +import { StyleSheet } from "react-native-unistyles"; +import { GestureHandlerRootView } from "react-native-gesture-handler"; +import { MarkdownRenderer } from "@/components/markdown/renderer"; + +const styles = StyleSheet.create((theme) => ({ + root: { flex: 1 }, + scroll: { backgroundColor: theme.colors.surface0 }, + content: { padding: 32, paddingTop: 60 }, + title: { color: theme.colors.foreground }, +})); + +const queryClient = new QueryClient(); + +const source = [ + "Plain: `MMMMiiii`", + "Bold: **`MMMMiiii`**", + "Italic: *`MMMMiiii`*", + "Both: ***`MMMMiiii`***", + "Strike: ~~`MMMMiiii`~~", + "Link: [`MMMMiiii`](https://example.com)", + "# Heading **`MMMMiiii`**", + "## Heading *`MMMMiiii`*", + "### Heading [`MMMMiiii`](https://example.com)", + "#### Heading ~~`MMMMiiii`~~", + "##### Heading ***`MMMMiiii`***", + "###### Heading `MMMMiiii`", + "- List **`MMMMiiii`**", + "> Quote **`MMMMiiii`**", + "Normal **bold** and *italic* prose.", +].join("\n\n"); + +export default function MarkdownFormattingRegression() { + const [compact, setCompact] = useState(false); + const toggleCompact = useCallback(() => setCompact((value) => !value), []); + return ( + + + + Inline code formatting regression + diff --git a/packages/app/src/components/settings/row-density.browser.test.tsx b/packages/app/src/components/settings/row-density.browser.test.tsx new file mode 100644 index 0000000000..d2742e3172 --- /dev/null +++ b/packages/app/src/components/settings/row-density.browser.test.tsx @@ -0,0 +1,77 @@ +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, describe, expect, it } from "vitest"; +import { Button } from "@/components/ui/button"; +import { SettingsCard, SettingsCollapsibleRow, SettingsRow } from "@/components/settings"; + +let root: Root | null = null; +let container: HTMLDivElement | null = null; + +afterEach(() => { + act(() => root?.unmount()); + container?.remove(); + root = null; + container = null; +}); + +function mountRows() { + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + act(() => + root?.render( + + + + + + + + + + + , + ), + ); +} + +function element(testID: string): HTMLElement { + const found = document.querySelector(`[data-testid="${testID}"]`); + if (!(found instanceof HTMLElement)) throw new Error(`Nothing rendered as ${testID}`); + return found; +} + +function heightOf(testID: string): number { + return element(testID).getBoundingClientRect().height; +} + +function titleFontSize(rowTestID: string, label: string): number { + const title = [...element(rowTestID).querySelectorAll("div")].find( + (node) => node.textContent === label && node.children.length === 0, + ); + if (!title) throw new Error(`No title ${label}`); + return Number.parseFloat(getComputedStyle(title).fontSize); +} + +describe("settings row density", () => { + it("lines up nested rows with and without a button, tighter than card rows", () => { + mountRows(); + + expect(heightOf("nested-with-button")).toBe(heightOf("nested-without-button")); + expect(heightOf("nested-with-button")).toBeLessThan(heightOf("card-row")); + expect(heightOf("group-toggle")).toBeGreaterThan(heightOf("nested-without-button")); + }); + + it("sizes nested text and controls one step below card rows", () => { + mountRows(); + + expect(heightOf("nested-button")).toBeLessThan(heightOf("card-button")); + expect(titleFontSize("nested-with-button", "Nested with a button")).toBeLessThan( + titleFontSize("card-row", "Card row"), + ); + }); +}); diff --git a/packages/app/src/components/settings/row-density.tsx b/packages/app/src/components/settings/row-density.tsx new file mode 100644 index 0000000000..d4bcdce33e --- /dev/null +++ b/packages/app/src/components/settings/row-density.tsx @@ -0,0 +1,57 @@ +import { createContext, useContext, useMemo, type ReactNode } from "react"; +import type { StyleProp, TextStyle, ViewStyle } from "react-native"; +import { StyleSheet } from "react-native-unistyles"; +import { CONTROL_HEIGHTS } from "@/components/ui/control-geometry"; +import type { ContainedControlSize } from "@/components/ui/control-size"; +import { settingsStyles } from "@/styles/settings"; + +// Rows revealed under a collapsible row belong to it, so they sit one step smaller than the rows +// of the card: tighter padding, smaller text, smaller controls. The container decides; a row +// never knows on its own that it is nested. +const NestedRowsContext = createContext(false); + +export function NestedSettingsRows({ children }: { children: ReactNode }) { + return {children}; +} + +export interface SettingsRowLayout { + row: StyleProp; + title: StyleProp; + hint: StyleProp; + /** The size of the controls in the row's trailing slot. */ + controlSize: ContainedControlSize; +} + +/** How a row lays out where it is rendered: in a card, or nested under a collapsible row. */ +export function useSettingsRowLayout(): SettingsRowLayout { + const nested = useContext(NestedRowsContext); + return useMemo( + () => + nested + ? { + row: [settingsStyles.row, styles.nestedRow], + title: [settingsStyles.rowTitle, styles.nestedText], + hint: [settingsStyles.rowHint, styles.nestedText], + controlSize: "xs", + } + : { + row: settingsStyles.row, + title: settingsStyles.rowTitle, + hint: settingsStyles.rowHint, + controlSize: "sm", + }, + [nested], + ); +} + +const styles = StyleSheet.create((theme) => ({ + // Every nested row is as tall as one holding an `xs` control, so rows with and without a + // trailing button line up. + nestedRow: { + paddingVertical: theme.spacing[2], + minHeight: CONTROL_HEIGHTS.tight + theme.spacing[2] * 2, + }, + nestedText: { + fontSize: theme.fontSize.sm, + }, +})); diff --git a/packages/app/src/components/settings/row-header.tsx b/packages/app/src/components/settings/row-header.tsx new file mode 100644 index 0000000000..9c3c415c34 --- /dev/null +++ b/packages/app/src/components/settings/row-header.tsx @@ -0,0 +1,113 @@ +import { ChevronRight } from "lucide-react-native"; +import { useCallback, useMemo } from "react"; +import { Text, View, type AccessibilityState, type PressableStateCallbackType } from "react-native"; +import { StyleSheet, withUnistyles } from "react-native-unistyles"; +import { mutedIconColorMapping } from "@/components/ui/icon-color"; +import { PressHighlight } from "@/components/ui/press-highlight"; +import { settingsStyles } from "@/styles/settings"; +import { ICON_SIZE, type Theme } from "@/styles/theme"; +import { useSettingsRowLayout } from "./row-density"; + +const ThemedChevron = withUnistyles(ChevronRight); +const hoveredIconColorMapping = (theme: Theme) => ({ color: theme.colors.foreground }); + +type HoverState = PressableStateCallbackType & { hovered?: boolean }; + +export interface SettingsRowHeaderProps { + label: string; + /** A short status shown before the chevron. */ + summary?: string; + onPress: () => void; + disabled?: boolean; + /** Turns the chevron down, for a row whose children are revealed in place. */ + expanded?: boolean; + accessibilityState?: AccessibilityState; + /** Props React Native Web does not derive from `accessibilityState`, such as `aria-expanded`. */ + webAccessibilityProps?: Record | null; + testID?: string; +} + +/** + * The pressable line shared by settings rows that lead somewhere: a label, an optional summary and + * a trailing chevron. A collapsible row turns the chevron down; a row that opens a sheet keeps it + * pointing right. + */ +export function SettingsRowHeader({ + label, + summary, + onPress, + disabled, + expanded = false, + accessibilityState, + webAccessibilityProps, + testID, +}: SettingsRowHeaderProps) { + const layout = useSettingsRowLayout(); + const rowStyle = useCallback( + ({ hovered, pressed }: HoverState) => [ + layout.row, + !disabled && hovered && styles.hovered, + !disabled && pressed && styles.pressed, + ], + [disabled, layout], + ); + const chevronStyle = useMemo( + () => [styles.chevron, expanded && styles.chevronExpanded], + [expanded], + ); + return ( + + {({ hovered }: HoverState) => ( + <> + + {label} + + {summary ? ( + + {summary} + + ) : null} + {disabled ? null : ( + + + + )} + + )} + + ); +} + +const styles = StyleSheet.create((theme) => ({ + summary: { + flexShrink: 1, + color: theme.colors.foregroundMuted, + fontSize: theme.fontSize.sm, + marginRight: theme.spacing[2], + }, + hovered: { + backgroundColor: theme.colors.surface2, + }, + pressed: { + backgroundColor: theme.colors.surface3, + }, + // Optical: the chevron's ink sits about 4px inside its box whether it points right or down; this + // puts the ink on the card's trailing rail, where the switch and the reports' refresh buttons end. + chevron: { marginRight: -theme.spacing[1] }, + chevronExpanded: { + transform: [{ rotate: "90deg" }], + }, +})); diff --git a/packages/app/src/components/ui/button.tsx b/packages/app/src/components/ui/button.tsx index 707ce60648..abcaf240d6 100644 --- a/packages/app/src/components/ui/button.tsx +++ b/packages/app/src/components/ui/button.tsx @@ -1,4 +1,5 @@ import { LoadingSpinner } from "@/components/ui/loading-spinner"; +import { useContainedControlSize } from "@/components/ui/control-size"; import { default as React, useCallback, @@ -162,7 +163,7 @@ const styles = StyleSheet.create((theme) => { export function Button({ children, variant = "secondary", - size = "md", + size: sizeProp, leftIcon, trailing, style, @@ -185,6 +186,8 @@ export function Button({ } >) { const [hovered, setHovered] = useState(false); + const containedSize = useContainedControlSize(); + const size = sizeProp ?? containedSize ?? "md"; const isDisabled = disabled || loading; let variantStyle: ViewStyle; diff --git a/packages/app/src/components/ui/control-geometry.test.ts b/packages/app/src/components/ui/control-geometry.test.ts index 0b0675596f..3544d9f940 100644 --- a/packages/app/src/components/ui/control-geometry.test.ts +++ b/packages/app/src/components/ui/control-geometry.test.ts @@ -81,6 +81,8 @@ describe("control geometry", () => { it("keeps field text sizing tied to control size", () => { const geometry = createControlGeometry(theme); + expect(geometry.fieldTextXs.fontSize).toBe(12); + expect(geometry.fieldTextXs.lineHeight).toBe(17); expect(geometry.fieldTextSm.fontSize).toBe(14); expect(geometry.fieldTextSm.lineHeight).toBe(20); expect(geometry.fieldTextMd.fontSize).toBe(14); @@ -95,6 +97,13 @@ describe("control geometry", () => { const geometry = createControlGeometry(theme); const borderWidth = theme.borderWidth[1]; + expect(geometry.fieldControlXs.minHeight).toBe(28); + expect( + geometry.fieldTextXs.lineHeight + + geometry.fieldControlXs.paddingVertical * 2 + + borderWidth * 2, + ).toBe(geometry.fieldControlXs.minHeight); + expect(geometry.fieldControlSm.minHeight).toBe(32); expect(geometry.fieldControlSm.paddingVertical).toBe(5); expect( diff --git a/packages/app/src/components/ui/control-geometry.ts b/packages/app/src/components/ui/control-geometry.ts index d56200887e..0a41f80039 100644 --- a/packages/app/src/components/ui/control-geometry.ts +++ b/packages/app/src/components/ui/control-geometry.ts @@ -2,7 +2,7 @@ import type { StyleProp, ViewStyle } from "react-native"; import { ICON_SIZE, type Theme } from "@/styles/theme"; export type ButtonControlSize = "xs" | "sm" | "md" | "lg"; -export type FieldControlSize = "sm" | "md"; +export type FieldControlSize = "xs" | "sm" | "md"; export type SegmentedControlSize = "xs" | "sm" | "md"; export type ControlInteractionPhase = "rest" | "hover" | "active"; @@ -188,8 +188,19 @@ export function resolveControlInteractionStyles( export function createControlGeometry(theme: Theme) { const controlBorderWidth = theme.borderWidth[1]; + const fieldTextXsLineHeight = fieldLineHeight(theme.fontSize.sm); const fieldTextSmLineHeight = fieldLineHeight(theme.fontSize.base); const fieldTextMdLineHeight = fieldLineHeight(theme.fontSize.base); + const fieldControlXs = { + minHeight: CONTROL_HEIGHTS.tight, + paddingHorizontal: theme.spacing[2], + paddingVertical: fieldVerticalPadding( + CONTROL_HEIGHTS.tight, + fieldTextXsLineHeight, + controlBorderWidth, + ), + borderRadius: theme.borderRadius.md, + }; const fieldControlSm = { minHeight: CONTROL_HEIGHTS.compact, paddingHorizontal: theme.spacing[3], @@ -210,6 +221,10 @@ export function createControlGeometry(theme: Theme) { ), borderRadius: theme.borderRadius.lg, }; + const fieldTextXs = { + fontSize: theme.fontSize.sm, + lineHeight: fieldTextXsLineHeight, + }; const fieldTextSm = { fontSize: theme.fontSize.base, lineHeight: fieldTextSmLineHeight, @@ -252,6 +267,10 @@ export function createControlGeometry(theme: Theme) { fontSize: theme.fontSize.sm, }, alert, + formTextInputXs: { + ...fieldControlXs, + ...fieldTextXs, + }, formTextInputSm: { ...fieldControlSm, ...fieldTextSm, @@ -264,8 +283,10 @@ export function createControlGeometry(theme: Theme) { ...fieldControlMd, ...fieldTextMd, }, + fieldControlXs, fieldControlSm, fieldControlMd, + fieldTextXs, fieldTextSm, fieldTextMd, controlRest: { diff --git a/packages/app/src/components/ui/control-size.tsx b/packages/app/src/components/ui/control-size.tsx new file mode 100644 index 0000000000..c6bcbbea78 --- /dev/null +++ b/packages/app/src/components/ui/control-size.tsx @@ -0,0 +1,23 @@ +import { createContext, useContext, type ReactNode } from "react"; + +/** The control sizes a container can ask of the controls placed in it. */ +export type ContainedControlSize = "xs" | "sm"; + +// A row decides how big its controls are, so a button or field placed in it needs no size of +// its own. A size passed to the control still wins. +const ControlSizeContext = createContext(null); + +export function ControlSizeProvider({ + size, + children, +}: { + size: ContainedControlSize; + children: ReactNode; +}) { + return {children}; +} + +/** The size the surrounding container asks for, or null outside one. */ +export function useContainedControlSize(): ContainedControlSize | null { + return useContext(ControlSizeContext); +} diff --git a/packages/app/src/components/ui/dropdown-trigger.tsx b/packages/app/src/components/ui/dropdown-trigger.tsx index 2fc1eb26ac..a276e40652 100644 --- a/packages/app/src/components/ui/dropdown-trigger.tsx +++ b/packages/app/src/components/ui/dropdown-trigger.tsx @@ -9,6 +9,7 @@ import { } from "@/components/ui/control-geometry"; import type { Theme } from "@/styles/theme"; import { DropdownMenuTrigger, type DropdownMenuTriggerProps } from "@/components/ui/dropdown-menu"; +import { useContainedControlSize } from "@/components/ui/control-size"; const ThemedChevronDown = withUnistyles(ChevronDown); @@ -32,10 +33,12 @@ interface DropdownTriggerProps extends Omit; +const FIELD_TEXT = { xs: "inputXs", sm: "inputSm", md: "inputMd" } as const satisfies Record< + FieldControlSize, + string +>; interface FieldProps { label: string; @@ -159,11 +169,13 @@ function assignTextInputRef( } export const FormTextInput = forwardRef( - function FormTextInput({ size = "md", style, onFocus, onBlur, editable, ...props }, ref) { + function FormTextInput({ size: sizeProp, style, onFocus, onBlur, editable, ...props }, ref) { const [focused, setFocused] = useState(false); + const containedSize = useContainedControlSize(); + const size = sizeProp ?? containedSize ?? "md"; const isDisabled = editable === false; - const chromeSizeStyle = size === "sm" ? formInputStyles.chromeSm : formInputStyles.chromeMd; - const inputSizeStyle = size === "sm" ? formInputStyles.inputSm : formInputStyles.inputMd; + const chromeSizeStyle = formInputStyles[FIELD_CHROME[size]]; + const inputSizeStyle = formInputStyles[FIELD_TEXT[size]]; const splitStyle = useMemo(() => splitFormTextInputStyle(style), [style]); const setInputRef = useCallback( (node: EditingTextInputHandle | null) => { @@ -262,6 +274,9 @@ const formInputStyles = StyleSheet.create((theme) => { chrome: { backgroundColor: theme.colors.surface2, }, + chromeXs: { + ...geometry.fieldControlXs, + }, chromeSm: { ...geometry.fieldControlSm, }, @@ -289,6 +304,9 @@ const formInputStyles = StyleSheet.create((theme) => { outlineColor: "transparent", outlineWidth: 0, }, + inputXs: { + ...geometry.fieldTextXs, + }, inputSm: { ...geometry.fieldTextSm, }, diff --git a/packages/app/src/components/ui/icon-color.ts b/packages/app/src/components/ui/icon-color.ts index 1093064423..f7891dabc4 100644 --- a/packages/app/src/components/ui/icon-color.ts +++ b/packages/app/src/components/ui/icon-color.ts @@ -1,5 +1,9 @@ import type { Theme } from "@/styles/theme"; +export const foregroundIconColorMapping = (theme: Theme) => ({ + color: theme.colors.foreground, +}); + export const mutedIconColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundMuted, }); diff --git a/packages/app/src/components/ui/segmented-control.tsx b/packages/app/src/components/ui/segmented-control.tsx index e0bbdf3f7b..d828c26c4d 100644 --- a/packages/app/src/components/ui/segmented-control.tsx +++ b/packages/app/src/components/ui/segmented-control.tsx @@ -7,6 +7,7 @@ import { segmentedIconSize, type SegmentedControlSize, } from "@/components/ui/control-geometry"; +import { useContainedControlSize } from "@/components/ui/control-size"; import type { Theme } from "@/styles/theme"; type SegmentedControlIconRenderer = (props: { color: string; size: number }) => ReactNode; @@ -48,11 +49,13 @@ export function SegmentedControl({ options, value, onValueChange, - size = "md", + size: sizeProp, hideLabels = false, style, testID, }: SegmentedControlProps) { + const containedSize = useContainedControlSize(); + const size = sizeProp ?? containedSize ?? "md"; const sizeStyles = { xs: { container: styles.containerXs, segment: styles.segmentXs, label: styles.labelXs }, sm: { container: styles.containerSm, segment: styles.segmentSm, label: styles.labelSm }, diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index da49bcec67..108fad41d4 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -2260,6 +2260,7 @@ export const ar: TranslationResources = { connections: "اتصالات", agents: "Agents", metadata: "البيانات الوصفية", + voice: "الصوت", workspaces: "Workspaces", providers: "مقدمي الخدمات", usage: "الاستخدام", @@ -2282,6 +2283,69 @@ export const ar: TranslationResources = { docs: "الوثائق", saveError: "تعذر تحديث إنشاء البيانات الوصفية", }, + voice: { + offline: "اتصل بهذا المضيف لضبط الصوت.", + updateHost: "حدّث المضيف لضبط الصوت.", + saveError: "تعذّر تحديث إعدادات الصوت", + provider: "المزوّد", + providers: { + local: "محلي", + openai: "OpenAI", + }, + voice: "الصوت", + save: "حفظ", + retry: "إعادة المحاولة", + cancel: "إلغاء", + model: "النموذج", + language: "اللغة", + languagePlaceholder: "اكتشاف تلقائي", + setBy: "محدد بواسطة {{name}}", + unknownProvider: "حدّث التطبيق لإعداد {{provider}}", + sites: { + dictationStt: "تحويل الكلام إلى نص", + voiceModeStt: "الاستماع", + voiceModeTts: "التحدث", + }, + dictation: { + title: "الإملاء", + enabled: "الإملاء", + enabledHint: "تحدّث في مربع الرسالة بدلًا من الكتابة", + }, + voiceMode: { + title: "وضع الصوت", + info: "يحوّل وضع الصوت كلامك إلى نص ويقرأ ردود الوكيل بصوت عالٍ.", + enabled: "وضع الصوت", + enabledHint: "تحدّث مع وكلائك دون استخدام يديك", + }, + openai: { + apiKey: "مفتاح API", + apiKeyHint: "يُحفظ على المضيف ولا يُعرض مرة أخرى.", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "محفوظ", + missingKey: "مفتاح API مفقود", + baseUrl: "عنوان URL الأساسي", + baseUrlHint: "للخوادم المتوافقة مع OpenAI. اتركه فارغًا لاستخدام OpenAI.", + }, + models: { + loading: "جارٍ تحميل النماذج...", + loadError: "تعذّر تحميل النماذج المحلية", + none: "لا توجد نماذج متاحة", + downloaded: "منزّل", + inUse: "قيد الاستخدام", + recommended: "موصى به", + download: "تنزيل", + downloadAction: "تنزيل {{label}}", + downloading: "جارٍ التنزيل...", + downloadingPercent: "جارٍ التنزيل {{percent}}%", + extracting: "جارٍ الاستخراج...", + delete: "حذف", + deleteAction: "حذف {{label}}", + deleteTitle: "حذف {{label}}؟", + deleteMessage: "سيؤدي هذا إلى تحرير {{size}}. يمكنك تنزيله مرة أخرى لاحقًا.", + languages: "لغات {{label}}", + moreLanguages: "{{languages}} و{{count}} أخرى", + }, + }, general: { title: "عام", sending: "الإرسال", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 57768f0ac0..16bddd9cf9 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -2287,6 +2287,7 @@ export const en = { connections: "Connections", agents: "Agents", metadata: "Metadata", + voice: "Voice", workspaces: "Workspaces", providers: "Providers", usage: "Usage", @@ -2371,6 +2372,69 @@ export const en = { docs: "Docs", saveError: "Unable to update metadata generation", }, + voice: { + offline: "Connect to this host to configure voice.", + updateHost: "Update the host to configure voice.", + saveError: "Unable to update voice settings", + provider: "Provider", + providers: { + local: "Local", + openai: "OpenAI", + }, + voice: "Voice", + save: "Save", + retry: "Retry", + cancel: "Cancel", + model: "Model", + language: "Language", + languagePlaceholder: "Auto-detect", + setBy: "Set by {{name}}", + unknownProvider: "Update the app to configure {{provider}}", + sites: { + dictationStt: "Speech-to-text", + voiceModeStt: "Listening", + voiceModeTts: "Speaking", + }, + dictation: { + title: "Dictation", + enabled: "Dictation", + enabledHint: "Speak into the message box instead of typing", + }, + voiceMode: { + title: "Voice mode", + info: "Voice mode transcribes what you say and reads agent replies aloud.", + enabled: "Voice mode", + enabledHint: "Talk with your agents hands-free", + }, + openai: { + apiKey: "API key", + apiKeyHint: "Saved on the host and never shown again.", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "Saved", + missingKey: "API key missing", + baseUrl: "Base URL", + baseUrlHint: "For OpenAI-compatible servers. Leave empty to use OpenAI.", + }, + models: { + loading: "Loading models...", + loadError: "Couldn't load local models", + none: "No models available", + downloaded: "Downloaded", + inUse: "In use", + recommended: "Recommended", + download: "Download", + downloadAction: "Download {{label}}", + downloading: "Downloading...", + downloadingPercent: "Downloading {{percent}}%", + extracting: "Extracting...", + delete: "Delete", + deleteAction: "Delete {{label}}", + deleteTitle: "Delete {{label}}?", + deleteMessage: "This frees {{size}}. You can download it again later.", + languages: "{{label}} languages", + moreLanguages: "{{languages}} and {{count}} more", + }, + }, general: { title: "General", sending: "Sending", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 51839eaf16..2226583781 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -2272,6 +2272,7 @@ export const es: TranslationResources = { connections: "Conexiones", agents: "Agents", metadata: "Metadatos", + voice: "Voz", workspaces: "Workspaces", providers: "Proveedores", usage: "Uso", @@ -2294,6 +2295,69 @@ export const es: TranslationResources = { docs: "Documentación", saveError: "No se pudo actualizar la generación de metadatos", }, + voice: { + offline: "Conéctate a este host para configurar la voz.", + updateHost: "Actualiza el host para configurar la voz.", + saveError: "No se pudo actualizar la configuración de voz", + provider: "Proveedor", + providers: { + local: "Local", + openai: "OpenAI", + }, + voice: "Voz", + save: "Guardar", + retry: "Reintentar", + cancel: "Cancelar", + model: "Modelo", + language: "Idioma", + languagePlaceholder: "Detección automática", + setBy: "Definido por {{name}}", + unknownProvider: "Actualiza la app para configurar {{provider}}", + sites: { + dictationStt: "Voz a texto", + voiceModeStt: "Escucha", + voiceModeTts: "Habla", + }, + dictation: { + title: "Dictado", + enabled: "Dictado", + enabledHint: "Habla en el cuadro de mensaje en lugar de escribir", + }, + voiceMode: { + title: "Modo de voz", + info: "El modo de voz transcribe lo que dices y lee en voz alta las respuestas del agente.", + enabled: "Modo de voz", + enabledHint: "Habla con tus agentes sin usar las manos", + }, + openai: { + apiKey: "Clave de API", + apiKeyHint: "Se guarda en el host y no se vuelve a mostrar.", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "Guardada", + missingKey: "Falta la clave de API", + baseUrl: "URL base", + baseUrlHint: "Para servidores compatibles con OpenAI. Déjala vacía para usar OpenAI.", + }, + models: { + loading: "Cargando modelos...", + loadError: "No se pudieron cargar los modelos locales", + none: "No hay modelos disponibles", + downloaded: "Descargado", + inUse: "En uso", + recommended: "Recomendado", + download: "Descargar", + downloadAction: "Descargar {{label}}", + downloading: "Descargando...", + downloadingPercent: "Descargando {{percent}} %", + extracting: "Extrayendo...", + delete: "Eliminar", + deleteAction: "Eliminar {{label}}", + deleteTitle: "¿Eliminar {{label}}?", + deleteMessage: "Esto libera {{size}}. Puedes volver a descargarlo más tarde.", + languages: "Idiomas de {{label}}", + moreLanguages: "{{languages}} y {{count}} más", + }, + }, general: { title: "General", sending: "Envío", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 4f7c9be2aa..65d72d405b 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -2272,6 +2272,7 @@ export const fr: TranslationResources = { connections: "Connexions", agents: "Agents", metadata: "Métadonnées", + voice: "Voix", workspaces: "Espaces de travail", providers: "Fournisseurs", usage: "Utilisation", @@ -2294,6 +2295,69 @@ export const fr: TranslationResources = { docs: "Documentation", saveError: "Impossible de mettre à jour la génération de métadonnées", }, + voice: { + offline: "Connectez-vous à cet hôte pour configurer la voix.", + updateHost: "Mettez à jour l'hôte pour configurer la voix.", + saveError: "Impossible de mettre à jour les réglages vocaux", + provider: "Fournisseur", + providers: { + local: "Local", + openai: "OpenAI", + }, + voice: "Voix", + save: "Enregistrer", + retry: "Réessayer", + cancel: "Annuler", + model: "Modèle", + language: "Langue", + languagePlaceholder: "Détection automatique", + setBy: "Défini par {{name}}", + unknownProvider: "Mettez à jour l'app pour configurer {{provider}}", + sites: { + dictationStt: "Transcription", + voiceModeStt: "Écoute", + voiceModeTts: "Parole", + }, + dictation: { + title: "Dictée", + enabled: "Dictée", + enabledHint: "Parlez dans la zone de message au lieu de taper", + }, + voiceMode: { + title: "Mode vocal", + info: "Le mode vocal transcrit ce que vous dites et lit à voix haute les réponses de l'agent.", + enabled: "Mode vocal", + enabledHint: "Parlez à vos agents sans les mains", + }, + openai: { + apiKey: "Clé d'API", + apiKeyHint: "Enregistrée sur l'hôte, jamais réaffichée.", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "Enregistrée", + missingKey: "Clé d'API manquante", + baseUrl: "URL de base", + baseUrlHint: "Pour les serveurs compatibles OpenAI. Laissez vide pour utiliser OpenAI.", + }, + models: { + loading: "Chargement des modèles...", + loadError: "Impossible de charger les modèles locaux", + none: "Aucun modèle disponible", + downloaded: "Téléchargé", + inUse: "Utilisé", + recommended: "Recommandé", + download: "Télécharger", + downloadAction: "Télécharger {{label}}", + downloading: "Téléchargement...", + downloadingPercent: "Téléchargement {{percent}} %", + extracting: "Extraction...", + delete: "Supprimer", + deleteAction: "Supprimer {{label}}", + deleteTitle: "Supprimer {{label}} ?", + deleteMessage: "Cela libère {{size}}. Vous pourrez le retélécharger plus tard.", + languages: "Langues de {{label}}", + moreLanguages: "{{languages}} et {{count}} autres", + }, + }, general: { title: "Général", sending: "Envoi", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index c0f22ffde5..5829bf4fa3 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -2245,6 +2245,7 @@ export const ja: TranslationResources = { connections: "接続", agents: "エージェント", metadata: "メタデータ", + voice: "音声", workspaces: "ワークスペース", providers: "プロバイダー", usage: "使用状況", @@ -2267,6 +2268,69 @@ export const ja: TranslationResources = { docs: "ドキュメント", saveError: "メタデータ生成を更新できません", }, + voice: { + offline: "音声を設定するには、このホストに接続してください。", + updateHost: "音声を設定するには、ホストを更新してください。", + saveError: "音声設定を更新できませんでした", + provider: "プロバイダー", + providers: { + local: "ローカル", + openai: "OpenAI", + }, + voice: "ボイス", + save: "保存", + retry: "再試行", + cancel: "キャンセル", + model: "モデル", + language: "言語", + languagePlaceholder: "自動検出", + setBy: "{{name}} で設定", + unknownProvider: "{{provider}} を設定するにはアプリを更新してください", + sites: { + dictationStt: "音声入力", + voiceModeStt: "聞き取り", + voiceModeTts: "読み上げ", + }, + dictation: { + title: "音声入力", + enabled: "音声入力", + enabledHint: "入力する代わりにメッセージ欄に話しかけます", + }, + voiceMode: { + title: "ボイスモード", + info: "ボイスモードは話した内容を文字起こしし、エージェントの返答を読み上げます。", + enabled: "ボイスモード", + enabledHint: "ハンズフリーでエージェントと会話します", + }, + openai: { + apiKey: "API キー", + apiKeyHint: "ホストに保存され、再表示されません。", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "保存済み", + missingKey: "API キーがありません", + baseUrl: "ベース URL", + baseUrlHint: "OpenAI 互換サーバー用です。OpenAI を使う場合は空欄のままにします。", + }, + models: { + loading: "モデルを読み込み中...", + loadError: "ローカルモデルを読み込めませんでした", + none: "利用できるモデルがありません", + downloaded: "ダウンロード済み", + inUse: "使用中", + recommended: "おすすめ", + download: "ダウンロード", + downloadAction: "{{label}} をダウンロード", + downloading: "ダウンロード中...", + downloadingPercent: "ダウンロード中 {{percent}}%", + extracting: "展開中...", + delete: "削除", + deleteAction: "{{label}} を削除", + deleteTitle: "{{label}} を削除しますか？", + deleteMessage: "{{size}} が解放されます。後で再ダウンロードできます。", + languages: "{{label}} の対応言語", + moreLanguages: "{{languages}} ほか {{count}} 言語", + }, + }, general: { title: "一般", sending: "送信", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index df571aae4d..36a27fc84e 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -2234,6 +2234,7 @@ export const ko: TranslationResources = { connections: "연결", agents: "에이전트", metadata: "메타데이터", + voice: "음성", workspaces: "워크스페이스", providers: "프로바이더", usage: "사용량", @@ -2256,6 +2257,69 @@ export const ko: TranslationResources = { docs: "문서", saveError: "메타데이터 생성을 업데이트할 수 없습니다", }, + voice: { + offline: "음성을 설정하려면 이 호스트에 연결하세요.", + updateHost: "음성을 설정하려면 호스트를 업데이트하세요.", + saveError: "음성 설정을 업데이트할 수 없습니다", + provider: "제공자", + providers: { + local: "로컬", + openai: "OpenAI", + }, + voice: "목소리", + save: "저장", + retry: "다시 시도", + cancel: "취소", + model: "모델", + language: "언어", + languagePlaceholder: "자동 감지", + setBy: "{{name}}에서 설정됨", + unknownProvider: "{{provider}}을(를) 설정하려면 앱을 업데이트하세요", + sites: { + dictationStt: "음성 텍스트 변환", + voiceModeStt: "듣기", + voiceModeTts: "말하기", + }, + dictation: { + title: "받아쓰기", + enabled: "받아쓰기", + enabledHint: "입력하는 대신 메시지 상자에 말하세요", + }, + voiceMode: { + title: "음성 모드", + info: "음성 모드는 말한 내용을 텍스트로 옮기고 에이전트의 답변을 소리 내어 읽어 줍니다.", + enabled: "음성 모드", + enabledHint: "손을 쓰지 않고 에이전트와 대화하세요", + }, + openai: { + apiKey: "API 키", + apiKeyHint: "호스트에 저장되며 다시 표시되지 않습니다.", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "저장됨", + missingKey: "API 키 없음", + baseUrl: "기본 URL", + baseUrlHint: "OpenAI 호환 서버용입니다. OpenAI를 사용하려면 비워 두세요.", + }, + models: { + loading: "모델을 불러오는 중...", + loadError: "로컬 모델을 불러올 수 없습니다", + none: "사용 가능한 모델 없음", + downloaded: "다운로드됨", + inUse: "사용 중", + recommended: "추천", + download: "다운로드", + downloadAction: "{{label}} 다운로드", + downloading: "다운로드 중...", + downloadingPercent: "다운로드 중 {{percent}}%", + extracting: "압축 해제 중...", + delete: "삭제", + deleteAction: "{{label}} 삭제", + deleteTitle: "{{label}}을(를) 삭제할까요?", + deleteMessage: "{{size}}가 확보됩니다. 나중에 다시 다운로드할 수 있습니다.", + languages: "{{label}} 지원 언어", + moreLanguages: "{{languages}} 외 {{count}}개", + }, + }, general: { title: "일반", sending: "전송", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 6ba6ec46bb..5a83b4345c 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -2257,6 +2257,7 @@ export const ptBR: TranslationResources = { connections: "Conexões", agents: "Agentes", metadata: "Metadados", + voice: "Voz", workspaces: "Workspaces", providers: "Provedores", usage: "Uso", @@ -2279,6 +2280,69 @@ export const ptBR: TranslationResources = { docs: "Documentação", saveError: "Não foi possível atualizar a geração de metadados", }, + voice: { + offline: "Conecte-se a este host para configurar a voz.", + updateHost: "Atualize o host para configurar a voz.", + saveError: "Não foi possível atualizar as configurações de voz", + provider: "Provedor", + providers: { + local: "Local", + openai: "OpenAI", + }, + voice: "Voz", + save: "Salvar", + retry: "Tentar novamente", + cancel: "Cancelar", + model: "Modelo", + language: "Idioma", + languagePlaceholder: "Detecção automática", + setBy: "Definido por {{name}}", + unknownProvider: "Atualize o app para configurar {{provider}}", + sites: { + dictationStt: "Fala para texto", + voiceModeStt: "Escuta", + voiceModeTts: "Fala", + }, + dictation: { + title: "Ditado", + enabled: "Ditado", + enabledHint: "Fale na caixa de mensagem em vez de digitar", + }, + voiceMode: { + title: "Modo de voz", + info: "O modo de voz transcreve o que você diz e lê em voz alta as respostas do agente.", + enabled: "Modo de voz", + enabledHint: "Converse com seus agentes sem usar as mãos", + }, + openai: { + apiKey: "Chave de API", + apiKeyHint: "Salva no host e nunca mostrada novamente.", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "Salva", + missingKey: "Chave de API ausente", + baseUrl: "URL base", + baseUrlHint: "Para servidores compatíveis com a OpenAI. Deixe vazio para usar a OpenAI.", + }, + models: { + loading: "Carregando modelos...", + loadError: "Não foi possível carregar os modelos locais", + none: "Nenhum modelo disponível", + downloaded: "Baixado", + inUse: "Em uso", + recommended: "Recomendado", + download: "Baixar", + downloadAction: "Baixar {{label}}", + downloading: "Baixando...", + downloadingPercent: "Baixando {{percent}}%", + extracting: "Extraindo...", + delete: "Excluir", + deleteAction: "Excluir {{label}}", + deleteTitle: "Excluir {{label}}?", + deleteMessage: "Isso libera {{size}}. Você pode baixá-lo de novo depois.", + languages: "Idiomas do {{label}}", + moreLanguages: "{{languages}} e mais {{count}}", + }, + }, general: { title: "Geral", sending: "Envio", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index f8f9969bf9..135d5fe2f9 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -2257,6 +2257,7 @@ export const ru: TranslationResources = { connections: "Подключения", agents: "Агенты", metadata: "Метаданные", + voice: "Голос", workspaces: "Рабочие пространства", providers: "Провайдеры", usage: "Использование", @@ -2280,6 +2281,70 @@ export const ru: TranslationResources = { docs: "Документация", saveError: "Не удалось обновить настройки генерации метаданных", }, + voice: { + offline: "Подключитесь к этому хосту, чтобы настроить голос.", + updateHost: "Обновите хост, чтобы настроить голос.", + saveError: "Не удалось обновить голосовые настройки", + provider: "Провайдер", + providers: { + local: "Локально", + openai: "OpenAI", + }, + voice: "Голос", + save: "Сохранить", + retry: "Повторить", + cancel: "Отмена", + model: "Модель", + language: "Язык", + languagePlaceholder: "Автоопределение", + setBy: "Задано через {{name}}", + unknownProvider: "Обновите приложение, чтобы настроить {{provider}}", + sites: { + dictationStt: "Распознавание речи", + voiceModeStt: "Прослушивание", + voiceModeTts: "Озвучивание", + }, + dictation: { + title: "Диктовка", + enabled: "Диктовка", + enabledHint: "Говорите в поле сообщения вместо набора текста", + }, + voiceMode: { + title: "Голосовой режим", + info: "Голосовой режим расшифровывает вашу речь и зачитывает ответы агента вслух.", + enabled: "Голосовой режим", + enabledHint: "Общайтесь с агентами без рук", + }, + openai: { + apiKey: "API-ключ", + apiKeyHint: "Хранится на хосте и больше не показывается.", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "Сохранён", + missingKey: "Нет API-ключа", + baseUrl: "Базовый URL", + baseUrlHint: + "Для серверов, совместимых с OpenAI. Оставьте пустым, чтобы использовать OpenAI.", + }, + models: { + loading: "Загрузка моделей...", + loadError: "Не удалось загрузить локальные модели", + none: "Нет доступных моделей", + downloaded: "Скачана", + inUse: "Используется", + recommended: "Рекомендуется", + download: "Скачать", + downloadAction: "Скачать {{label}}", + downloading: "Скачивание...", + downloadingPercent: "Скачивание {{percent}}%", + extracting: "Распаковка...", + delete: "Удалить", + deleteAction: "Удалить {{label}}", + deleteTitle: "Удалить {{label}}?", + deleteMessage: "Освободится {{size}}. Позже её можно скачать снова.", + languages: "Языки {{label}}", + moreLanguages: "{{languages}} и ещё {{count}}", + }, + }, general: { title: "Основные", sending: "Отправка", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index 9954bdb61b..78b0f06442 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -2202,6 +2202,7 @@ export const zhCN: TranslationResources = { connections: "连接", agents: "Agents", metadata: "元数据", + voice: "语音", workspaces: "Workspaces", providers: "Providers", usage: "使用情况", @@ -2223,6 +2224,69 @@ export const zhCN: TranslationResources = { docs: "文档", saveError: "无法更新元数据生成设置", }, + voice: { + offline: "连接到此主机以配置语音。", + updateHost: "请更新主机以配置语音。", + saveError: "无法更新语音设置", + provider: "提供方", + providers: { + local: "本地", + openai: "OpenAI", + }, + voice: "音色", + save: "保存", + retry: "重试", + cancel: "取消", + model: "模型", + language: "语言", + languagePlaceholder: "自动检测", + setBy: "由 {{name}} 设置", + unknownProvider: "更新应用以配置 {{provider}}", + sites: { + dictationStt: "语音转文字", + voiceModeStt: "聆听", + voiceModeTts: "朗读", + }, + dictation: { + title: "听写", + enabled: "听写", + enabledHint: "对着消息框说话，无需打字", + }, + voiceMode: { + title: "语音模式", + info: "语音模式会转写你说的话，并朗读智能体的回复。", + enabled: "语音模式", + enabledHint: "免手动与智能体对话", + }, + openai: { + apiKey: "API 密钥", + apiKeyHint: "保存在主机上，不会再次显示。", + keyPlaceholder: "sk-...", + savedKeyPlaceholder: "已保存", + missingKey: "缺少 API 密钥", + baseUrl: "基础 URL", + baseUrlHint: "用于兼容 OpenAI 的服务器。留空则使用 OpenAI。", + }, + models: { + loading: "正在加载模型...", + loadError: "无法加载本地模型", + none: "没有可用的模型", + downloaded: "已下载", + inUse: "使用中", + recommended: "推荐", + download: "下载", + downloadAction: "下载 {{label}}", + downloading: "正在下载...", + downloadingPercent: "正在下载 {{percent}}%", + extracting: "正在解压...", + delete: "删除", + deleteAction: "删除 {{label}}", + deleteTitle: "删除 {{label}}？", + deleteMessage: "这将释放 {{size}}。之后可以重新下载。", + languages: "{{label}} 支持的语言", + moreLanguages: "{{languages}} 等另外 {{count}} 种", + }, + }, general: { title: "通用", sending: "发送", diff --git a/packages/app/src/screens/settings-screen.tsx b/packages/app/src/screens/settings-screen.tsx index 10d87cdcba..b45b745212 100644 --- a/packages/app/src/screens/settings-screen.tsx +++ b/packages/app/src/screens/settings-screen.tsx @@ -37,6 +37,7 @@ import { Files, Smartphone, Sparkles, + Mic, Blocks, Globe, PanelLeft, @@ -118,6 +119,7 @@ import { PluginSettingsContent } from "@/plugins/settings"; import { useInstalledPlugins } from "@/plugins/registry"; import { HostPluginsPage } from "@/screens/settings/plugins-page"; import { MetadataGenerationPage } from "@/screens/settings/metadata-generation-page"; +import { HostVoicePage } from "@/screens/settings/voice/voice-page"; import ProjectsScreen from "@/screens/projects-screen"; import ProjectSettingsScreen from "@/screens/project-settings-screen"; import { SETTINGS_DESKTOP_SIDEBAR_WIDTH, useIsCompactFormFactor } from "@/constants/layout"; @@ -240,6 +242,7 @@ const HOST_SECTION_ITEMS: HostSectionItem[] = [ { id: "pair-device", labelKey: "openProject.tiles.pairDevice.title", icon: Smartphone }, { id: "agents", labelKey: "settings.hostSections.agents", icon: Bot }, { id: "metadata", labelKey: "settings.hostSections.metadata", icon: Sparkles }, + { id: "voice", labelKey: "settings.hostSections.voice", icon: Mic }, { id: "workspaces", labelKey: "settings.hostSections.workspaces", icon: FolderGit2 }, { id: "providers", labelKey: "settings.hostSections.providers", icon: Boxes }, { id: "usage", labelKey: "settings.hostSections.usage", icon: Gauge }, @@ -262,6 +265,8 @@ function renderHostSettingsContent( return ; case "metadata": return ; + case "voice": + return ; case "workspaces": return ; case "providers": diff --git a/packages/app/src/screens/settings/voice/speech-provider-form.test.ts b/packages/app/src/screens/settings/voice/speech-provider-form.test.ts new file mode 100644 index 0000000000..db88fc9bdd --- /dev/null +++ b/packages/app/src/screens/settings/voice/speech-provider-form.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import type { SpeechSettingsPatch } from "@getpaseo/protocol/speech/rpc-schemas"; +import { openSpeechProviderForm, type SpeechProviderFormSnapshot } from "./speech-provider-form"; + +function open(overrides: Partial = {}) { + return openSpeechProviderForm({ + siteId: "voiceMode.stt", + site: { provider: "local" }, + envOverrides: {}, + ...overrides, + }); +} + +async function savedPatch( + model: ReturnType, +): Promise { + let sent: SpeechSettingsPatch | undefined; + const result = await model.save(async (patch) => { + sent = patch; + }); + expect(result).toEqual({ status: "saved" }); + return sent; +} + +describe("speech provider form", () => { + it("edits and clears local STT language from the saved snapshot", async () => { + const form = open({ site: { provider: "local", language: "fr" } }); + expect(form.getState()).toMatchObject({ language: "fr", canSave: false }); + form.setLanguage(" de "); + expect(await savedPatch(form)).toEqual({ + voiceMode: { stt: { provider: "local", language: "de" } }, + }); + const clear = open({ site: { provider: "local", language: "fr" } }); + clear.setLanguage(""); + expect(await savedPatch(clear)).toEqual({ + voiceMode: { stt: { provider: "local", language: null } }, + }); + }); + + it("keeps STT language when switching either provider", async () => { + for (const provider of ["local", "openai"] as const) { + const form = open({ + site: + provider === "local" + ? { provider, language: "fr" } + : { provider, hasApiKey: false, language: "fr" }, + }); + form.setProvider(provider === "local" ? "openai" : "local"); + expect(await savedPatch(form)).toEqual({ + voiceMode: { stt: { provider: provider === "local" ? "openai" : "local", language: "fr" } }, + }); + } + }); + + it("keeps env-controlled language read-only after switching providers", async () => { + const form = open({ + site: { provider: "local", language: "fr" }, + envOverrides: { "features.voiceMode.stt.language": "PASEO_VOICE_LANGUAGE" }, + }); + form.setProvider("openai"); + expect(form.getState().fields.language).toEqual({ + envVar: "PASEO_VOICE_LANGUAGE", + readOnly: true, + }); + expect(await savedPatch(form)).toEqual({ voiceMode: { stt: { provider: "openai" } } }); + }); + + it("switches a site to OpenAI with a key and base URL", async () => { + const form = open(); + expect(form.getState().canSave).toBe(false); + + form.setProvider("openai"); + form.setApiKey(" sk-test "); + form.setBaseUrl("http://127.0.0.1:9999/v1"); + + expect(await savedPatch(form)).toEqual({ + voiceMode: { + stt: { provider: "openai", apiKey: "sk-test", baseUrl: "http://127.0.0.1:9999/v1" }, + }, + }); + }); + + it("re-saves OpenAI settings without touching the saved key", async () => { + const form = open({ + siteId: "dictation.stt", + site: { provider: "openai", hasApiKey: true, baseUrl: "https://a.example/v1" }, + }); + expect(form.getState()).toMatchObject({ apiKey: "", hasSavedApiKey: true, canSave: false }); + + form.setOpenAiModel("gpt-4o-transcribe"); + form.setBaseUrl(""); + + expect(await savedPatch(form)).toEqual({ + dictation: { stt: { provider: "openai", model: "gpt-4o-transcribe", baseUrl: null } }, + }); + }); + + it("selects a local model", async () => { + const form = open({ + siteId: "dictation.stt", + site: { provider: "local", model: "whisper-tiny" }, + }); + form.setLocalModel("whisper-tiny"); + expect(form.getState().canSave).toBe(false); + + form.setLocalModel("whisper-base"); + + expect(await savedPatch(form)).toEqual({ + dictation: { stt: { provider: "local", model: "whisper-base" } }, + }); + }); + + it("locks fields an env var decides and keeps fallback fields editable", async () => { + const form = open({ + site: { provider: "openai", hasApiKey: false }, + envOverrides: { + "features.voiceMode.stt.provider": "PASEO_VOICE_STT_PROVIDER", + "features.voiceMode.stt.model": "STT_MODEL", + "features.voiceMode.stt.apiKey": "OPENAI_API_KEY", + }, + }); + expect(form.getState().fields).toMatchObject({ + provider: { envVar: "PASEO_VOICE_STT_PROVIDER", readOnly: true }, + model: { envVar: "STT_MODEL", readOnly: true }, + apiKey: { envVar: "OPENAI_API_KEY", readOnly: false }, + baseUrl: { envVar: null, readOnly: false }, + }); + + form.setProvider("local"); + form.setOpenAiModel("gpt-4o-transcribe"); + form.setApiKey("sk-saved"); + + expect(form.getState().provider).toBe("openai"); + expect(await savedPatch(form)).toEqual({ + voiceMode: { stt: { provider: "openai", apiKey: "sk-saved" } }, + }); + }); + + it("does not apply the saved provider's env vars to another provider", () => { + const form = open({ + envOverrides: { "features.voiceMode.stt.model": "PASEO_VOICE_LOCAL_STT_MODEL" }, + }); + expect(form.getState().fields.model.readOnly).toBe(true); + + form.setProvider("openai"); + + expect(form.getState().fields.model).toEqual({ envVar: null, readOnly: false }); + }); + + it("keeps the input and allows a retry after a failed save", async () => { + const form = open({ siteId: "voiceMode.tts" }); + form.setProvider("openai"); + form.setApiKey("sk-test"); + form.setVoice("nova"); + const failure = new Error("disk full"); + + const result = await form.save(async () => { + expect(form.getState()).toMatchObject({ saving: true, canSave: false }); + throw failure; + }); + + expect(result).toEqual({ status: "failed", error: failure }); + expect(form.getState()).toMatchObject({ + provider: "openai", + apiKey: "sk-test", + voice: "nova", + saving: false, + canSave: true, + }); + expect(await savedPatch(form)).toEqual({ + voiceMode: { tts: { provider: "openai", apiKey: "sk-test", voice: "nova" } }, + }); + }); +}); diff --git a/packages/app/src/screens/settings/voice/speech-provider-form.ts b/packages/app/src/screens/settings/voice/speech-provider-form.ts new file mode 100644 index 0000000000..b8230104e5 --- /dev/null +++ b/packages/app/src/screens/settings/voice/speech-provider-form.ts @@ -0,0 +1,240 @@ +import type { SpeechSettingsPatch } from "@getpaseo/protocol/speech/rpc-schemas"; +import { + SPEECH_SITE_KIND, + buildSpeechSitePatch, + resolveOpenAiModel, + resolveOpenAiVoice, + speechSiteFieldPath, + type SpeechModelKind, + type SpeechProvider, + type SpeechSite, + type SpeechSiteId, + type SpeechSitePatch, +} from "./voice-settings"; + +// The provider sheet's form: one speech site, its provider and that provider's fields. +// Plain TypeScript so the save rules are testable without rendering (docs/forms.md). + +export interface SpeechProviderFormSnapshot { + siteId: SpeechSiteId; + site: Extract; + envOverrides: Record; +} + +export type SpeechProviderField = + | "provider" + | "model" + | "apiKey" + | "baseUrl" + | "language" + | "voice"; + +// An env var set for one of these is only a fallback: a value saved in the sheet wins, +// so the field stays editable. Every other field is decided by its env var. +const FALLBACK_FIELDS: ReadonlySet = new Set(["apiKey", "baseUrl"]); + +export interface SpeechProviderFieldSource { + /** The env var that decides this field's effective value, when one does. */ + envVar: string | null; + readOnly: boolean; +} + +export interface SpeechProviderFormState { + kind: SpeechModelKind; + provider: SpeechProvider; + /** The local model the user picked in this sheet; null keeps the saved or default model. */ + localModel: string | null; + savedLocalModel: string | undefined; + apiKey: string; + hasSavedApiKey: boolean; + baseUrl: string; + openAiModel: string; + language: string; + voice: string; + fields: Record; + saving: boolean; + canSave: boolean; +} + +export type SpeechProviderSaveResult = + | { status: "saved" } + | { status: "skipped" } + | { status: "failed"; error: unknown }; + +export interface SpeechProviderFormModel { + getState: () => SpeechProviderFormState; + subscribe: (listener: () => void) => () => void; + close: () => void; + setProvider: (provider: SpeechProvider) => void; + setLocalModel: (modelId: string) => void; + setApiKey: (value: string) => void; + setBaseUrl: (value: string) => void; + setOpenAiModel: (value: string) => void; + setLanguage: (value: string) => void; + setVoice: (value: string) => void; + /** Sends the changed fields through `save`; the inputs survive a failed save. */ + save: ( + save: (patch: SpeechSettingsPatch) => Promise, + ) => Promise; +} + +interface OpenAiValues { + baseUrl: string; + openAiModel: string; + voice: string; +} + +export function openSpeechProviderForm( + snapshot: SpeechProviderFormSnapshot, +): SpeechProviderFormModel { + const { siteId, site, envOverrides } = snapshot; + const kind = SPEECH_SITE_KIND[siteId]; + const savedOpenAi = site.provider === "openai" ? site : undefined; + const initialOpenAi: OpenAiValues = { + baseUrl: savedOpenAi?.baseUrl ?? "", + openAiModel: resolveOpenAiModel(kind, savedOpenAi?.model), + voice: resolveOpenAiVoice(savedOpenAi?.voice), + }; + const initialLanguage = site.language ?? ""; + // The daemon reports env vars for the provider that is in effect, so they only describe the + // provider-specific fields while that provider is selected. Language applies to either STT provider. + const savedFields = buildFieldSources(siteId, envOverrides); + const switchedFields = { + ...buildFieldSources(siteId, {}), + provider: savedFields.provider, + language: savedFields.language, + }; + const listeners = new Set<() => void>(); + let closed = false; + + let state: SpeechProviderFormState = derive({ + kind, + provider: site.provider, + localModel: null, + savedLocalModel: site.provider === "local" ? site.model : undefined, + apiKey: "", + hasSavedApiKey: savedOpenAi?.hasApiKey ?? false, + ...initialOpenAi, + language: initialLanguage, + fields: savedFields, + saving: false, + canSave: false, + }); + + function derive(next: SpeechProviderFormState): SpeechProviderFormState { + return { + ...next, + fields: next.provider === site.provider ? savedFields : switchedFields, + canSave: !next.saving && isChanged(next), + }; + } + + function isChanged(next: SpeechProviderFormState): boolean { + if (next.provider !== site.provider) return true; + if (kind === "stt" && next.language.trim() !== initialLanguage) return true; + if (next.provider === "local") { + return next.localModel !== null && next.localModel !== next.savedLocalModel; + } + return next.apiKey.trim().length > 0 || changedOpenAiFields(next).length > 0; + } + + function changedOpenAiFields(next: SpeechProviderFormState): (keyof OpenAiValues)[] { + const keys: (keyof OpenAiValues)[] = + kind === "stt" ? ["baseUrl", "openAiModel"] : ["baseUrl", "openAiModel", "voice"]; + return keys.filter((key) => next[key].trim() !== initialOpenAi[key]); + } + + function buildSitePatch(next: SpeechProviderFormState): SpeechSitePatch { + const language = next.language.trim(); + const languagePatch: Record = + kind === "stt" && + !next.fields.language.readOnly && + (language !== initialLanguage || (next.provider !== site.provider && language.length > 0)) + ? { language: language || null } + : {}; + if (next.provider === "local") { + return { + provider: "local", + ...languagePatch, + ...(next.localModel !== null && !next.fields.model.readOnly + ? { model: next.localModel } + : {}), + }; + } + const patch: Record = { provider: "openai", ...languagePatch }; + const apiKey = next.apiKey.trim(); + if (apiKey) patch.apiKey = apiKey; + for (const key of changedOpenAiFields(next)) { + const field = key === "openAiModel" ? "model" : key; + if (next.fields[field].readOnly) continue; + patch[field] = next[key].trim() || null; + } + return patch as SpeechSitePatch; + } + + function publish(next: SpeechProviderFormState): void { + if (closed) return; + state = derive(next); + for (const listener of listeners) listener(); + } + + function edit(change: Partial): void { + if (state.saving) return; + publish({ ...state, ...change }); + } + + return { + getState: () => state, + subscribe(listener) { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + close() { + closed = true; + listeners.clear(); + }, + setProvider: (provider) => { + if (!state.fields.provider.readOnly) edit({ provider }); + }, + setLocalModel: (localModel) => { + if (!state.fields.model.readOnly) edit({ localModel }); + }, + setApiKey: (apiKey) => edit({ apiKey }), + setBaseUrl: (baseUrl) => edit({ baseUrl }), + setOpenAiModel: (openAiModel) => edit({ openAiModel }), + setLanguage: (language) => edit({ language }), + setVoice: (voice) => edit({ voice }), + async save(save) { + if (!state.canSave) return { status: "skipped" }; + const patch = buildSpeechSitePatch(siteId, buildSitePatch(state)); + publish({ ...state, saving: true }); + try { + await save(patch); + return { status: "saved" }; + } catch (error) { + publish({ ...state, saving: false }); + return { status: "failed", error }; + } + }, + }; +} + +function buildFieldSources( + siteId: SpeechSiteId, + envOverrides: Record, +): Record { + const source = (field: SpeechProviderField): SpeechProviderFieldSource => { + const envVar = envOverrides[speechSiteFieldPath(siteId, field)] ?? null; + return { envVar, readOnly: envVar !== null && !FALLBACK_FIELDS.has(field) }; + }; + return { + provider: source("provider"), + model: source("model"), + apiKey: source("apiKey"), + baseUrl: source("baseUrl"), + language: source("language"), + voice: source("voice"), + }; +} diff --git a/packages/app/src/screens/settings/voice/speech-provider-sheet.tsx b/packages/app/src/screens/settings/voice/speech-provider-sheet.tsx new file mode 100644 index 0000000000..7c842053b3 --- /dev/null +++ b/packages/app/src/screens/settings/voice/speech-provider-sheet.tsx @@ -0,0 +1,585 @@ +import { + useCallback, + useEffect, + useMemo, + useState, + useSyncExternalStore, + type ReactNode, +} from "react"; +import { Text, View, type PressableStateCallbackType } from "react-native"; +import { useTranslation } from "react-i18next"; +import type { TFunction } from "i18next"; +import { StyleSheet, withUnistyles } from "react-native-unistyles"; +import { Check, Download, Languages, Trash2 } from "lucide-react-native"; +import type { LocalSpeechModel, SpeechSettingsPatch } from "@getpaseo/protocol/speech/rpc-schemas"; +import { AdaptiveModalSheet, type SheetHeader } from "@/components/adaptive-modal-sheet"; +import { + SettingsCard, + SettingsInfoTip, + SettingsInput, + SettingsRow, + SettingsSelect, +} from "@/components/settings"; +import { Button } from "@/components/ui/button"; +import { buttonControlHeight } from "@/components/ui/control-geometry"; +import { useContainedControlSize } from "@/components/ui/control-size"; +import { foregroundIconColorMapping } from "@/components/ui/icon-color"; +import { PressHighlight } from "@/components/ui/press-highlight"; +import { LoadingSpinner } from "@/components/ui/loading-spinner"; +import { SegmentedControl } from "@/components/ui/segmented-control"; +import { StatusBadge } from "@/components/ui/status-badge"; +import { useIsCompactFormFactor } from "@/constants/layout"; +import { useToast } from "@/contexts/toast-api-context"; +import { ICON_SIZE } from "@/styles/theme"; +import { confirmDialog } from "@/utils/confirm-dialog"; +import type { LocalSpeechModels } from "./use-local-speech-models"; +import { + openSpeechProviderForm, + type SpeechProviderFieldSource, + type SpeechProviderFormModel, + type SpeechProviderFormSnapshot, + type SpeechProviderFormState, +} from "./speech-provider-form"; +import { + SPEECH_PROVIDERS, + buildOpenAiModelOptions, + buildOpenAiVoiceOptions, + formatLanguages, + formatModelSize, + getLocalModelState, + resolveLocalModel, + type LocalModelState, + type SpeechProvider, +} from "./voice-settings"; + +const ThemedCheck = withUnistyles(Check); + +export interface SpeechProviderSheetProps extends SpeechProviderFormSnapshot { + title: string; + localModels: LocalSpeechModels; + visible: boolean; + onClose: () => void; + onDismiss: () => void; + save: (patch: SpeechSettingsPatch) => Promise; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +// The form opens from the props of the first render; later config changes do not reset input. +function useSpeechProviderForm(snapshot: SpeechProviderFormSnapshot): SpeechProviderFormModel { + const [model] = useState(() => openSpeechProviderForm(snapshot)); + useEffect(() => () => model.close(), [model]); + return model; +} + +/** One speech site's provider and that provider's settings. Mount it fresh for each open. */ +export function SpeechProviderSheet({ + title, + siteId, + site, + envOverrides, + localModels, + visible, + onClose, + onDismiss, + save, +}: SpeechProviderSheetProps) { + const { t } = useTranslation(); + const toast = useToast(); + const model = useSpeechProviderForm({ siteId, site, envOverrides }); + const state = useSyncExternalStore(model.subscribe, model.getState, model.getState); + + const submit = useCallback(() => { + void model.save(save).then((result) => { + if (result.status === "saved") onClose(); + if (result.status === "failed") { + toast.error(`${t("settings.voice.saveError")}: ${errorMessage(result.error)}`); + } + return undefined; + }); + }, [model, onClose, save, t, toast]); + + const header = useMemo(() => ({ title }), [title]); + const footer = useMemo( + () => ( + + + + + ), + [onClose, state.canSave, state.saving, submit, t], + ); + + return ( + + + + + {state.provider === "local" ? ( + <> + {state.kind === "stt" ? ( + + + + ) : null} + + + ) : ( + + )} + + ); +} + +interface FormProps { + state: SpeechProviderFormState; + model: SpeechProviderFormModel; +} + +function setByHint(t: TFunction, source: SpeechProviderFieldSource): string | undefined { + return source.envVar ? t("settings.voice.setBy", { name: source.envVar }) : undefined; +} + +function ProviderRow({ state, model }: FormProps) { + const { t } = useTranslation(); + const source = state.fields.provider; + const options = useMemo( + () => + SPEECH_PROVIDERS.map((provider) => ({ + value: provider, + label: t(`settings.voice.providers.${provider}`), + disabled: source.readOnly, + })), + [source.readOnly, t], + ); + return ( + + + options={options} + value={state.provider} + onValueChange={model.setProvider} + testID="voice-provider-picker" + /> + + ); +} + +function LanguageField({ state, model }: FormProps) { + const { t } = useTranslation(); + return ( + + ); +} + +// --------------------------------------------------------------------------- +// OpenAI +// --------------------------------------------------------------------------- + +function OpenAiFields({ state, model }: FormProps) { + const { t } = useTranslation(); + const modelOptions = useMemo( + () => buildOpenAiModelOptions(state.kind, state.openAiModel), + [state.kind, state.openAiModel], + ); + const voiceOptions = useMemo(() => buildOpenAiVoiceOptions(state.voice), [state.voice]); + const apiKeyInfo = useMemo( + () => ( + + ), + [t], + ); + const baseUrlInfo = useMemo( + () => ( + + ), + [t], + ); + + return ( + + + + + {state.kind === "stt" ? ( + + ) : ( + + )} + + ); +} + +// --------------------------------------------------------------------------- +// Local: the catalog, with the selected model checked +// --------------------------------------------------------------------------- + +function LocalModelList({ + state, + model, + localModels, +}: FormProps & { localModels: LocalSpeechModels }) { + const { t } = useTranslation(); + const { models, isLoading, error, refresh, download, remove } = localModels; + const kindModels = useMemo( + () => models.filter((entry) => entry.kind === state.kind), + [models, state.kind], + ); + const selectedId = + state.localModel ?? + resolveLocalModel({ kind: state.kind, model: state.savedLocalModel, localModels: models })?.id; + const source = state.fields.model; + + if (kindModels.length === 0) { + if (error) { + return ( + + + + + + ); + } + return ( + + + + ); + } + return ( + + {kindModels.map((entry) => ( + + ))} + + ); +} + +function progressLabel( + t: TFunction, + state: Extract, +): string { + if (state.kind === "extracting") return t("settings.voice.models.extracting"); + if (state.percent === null) return t("settings.voice.models.downloading"); + return t("settings.voice.models.downloadingPercent", { percent: state.percent }); +} + +type HoverState = PressableStateCallbackType & { hovered?: boolean }; + +function LocalModelRow({ + model, + selected, + selectionLocked, + hint, + onSelect, + onDownload, + onDelete, +}: { + model: LocalSpeechModel; + selected: boolean; + selectionLocked: boolean; + /** Why the selection is fixed, shown on the selected row. */ + hint: string | undefined; + onSelect: (modelId: string) => void; + onDownload: (modelId: string) => Promise; + onDelete: (modelId: string) => Promise; +}) { + const { t, i18n } = useTranslation(); + const state = getLocalModelState(model); + const [actionError, setActionError] = useState(null); + + const select = useCallback(() => onSelect(model.id), [model.id, onSelect]); + const download = useCallback(() => { + setActionError(null); + onDownload(model.id).catch((error: unknown) => setActionError(errorMessage(error))); + }, [model.id, onDownload]); + const remove = useCallback(() => { + void confirmDialog({ + title: t("settings.voice.models.deleteTitle", { label: model.label }), + message: t("settings.voice.models.deleteMessage", { + size: formatModelSize(model.downloadSizeBytes), + }), + confirmLabel: t("settings.voice.models.delete"), + destructive: true, + }) + .then((confirmed) => { + if (!confirmed) return undefined; + setActionError(null); + return onDelete(model.id); + }) + .catch((error: unknown) => setActionError(errorMessage(error))); + }, [model.downloadSizeBytes, model.id, model.label, onDelete, t]); + + const languages = formatLanguages({ + languages: model.languages, + locale: i18n.language, + formatMore: (listed, count) => + t("settings.voice.models.moreLanguages", { languages: listed, count }), + }); + const details = useMemo( + () => ( + + + + {formatModelSize(model.downloadSizeBytes)} + {state.kind === "installed" ? ( + + ) : null} + {hint ? {hint} : null} + + ), + [ + hint, + languages, + model.description, + model.downloadSizeBytes, + model.id, + model.inUse, + model.label, + state.kind, + t, + ], + ); + + const recommended = useMemo( + () => + model.isDefault ? ( + + ) : undefined, + [model.isDefault, t], + ); + + // Phone rows have no room beside the label for a labelled button, so actions show their icon. + const compact = useIsCompactFormFactor(); + const downloadLabel = + state.kind === "failed" ? t("settings.voice.retry") : t("settings.voice.models.download"); + let control = null; + if (state.kind === "downloading" || state.kind === "extracting") { + control = ( + + + {progressLabel(t, state)} + + ); + } else if (state.kind === "installed") { + control = model.inUse ? null : ( + + ); + } else { + control = ( + + ); + } + + const rowStyle = useCallback( + ({ hovered, pressed }: HoverState) => [ + !selectionLocked && hovered && styles.rowHovered, + !selectionLocked && pressed && styles.rowPressed, + ], + [selectionLocked], + ); + const accessibilityState = useMemo( + () => ({ checked: selected, disabled: selectionLocked }), + [selected, selectionLocked], + ); + + return ( + + + + {control} + + {selected ? ( + + ) : null} + + + + + ); +} + +// Every catalog row is as tall as one holding a button, so a row does not shrink when its +// download finishes and the button goes away. +function TrailingControls({ children }: { children: ReactNode }) { + const size = useContainedControlSize() ?? "sm"; + const style = useMemo(() => [styles.trailing, { minHeight: buttonControlHeight[size] }], [size]); + return {children}; +} + +const styles = StyleSheet.create((theme) => ({ + footer: { + flex: 1, + flexDirection: "row", + gap: theme.spacing[3], + }, + footerButton: { + flex: 1, + }, + rowHovered: { + backgroundColor: theme.colors.surface2, + }, + rowPressed: { + backgroundColor: theme.colors.surface3, + }, + // The model's details sit on the hint line, so every catalog row has the same two lines. + details: { + flexDirection: "row", + flexWrap: "wrap", + alignItems: "center", + gap: theme.spacing[2], + marginTop: theme.spacing[1], + }, + metaText: { + color: theme.colors.foregroundMuted, + fontSize: theme.fontSize.sm, + fontVariant: ["tabular-nums"], + }, + trailing: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + checkSlot: { + width: ICON_SIZE.sm, + alignItems: "center", + }, + progress: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + spinnerColor: { + color: theme.colors.foregroundMuted, + }, +})); diff --git a/packages/app/src/screens/settings/voice/use-local-speech-models.ts b/packages/app/src/screens/settings/voice/use-local-speech-models.ts new file mode 100644 index 0000000000..b68a7b8135 --- /dev/null +++ b/packages/app/src/screens/settings/voice/use-local-speech-models.ts @@ -0,0 +1,71 @@ +import { useCallback } from "react"; +import { useTranslation } from "react-i18next"; +import type { LocalSpeechModel } from "@getpaseo/protocol/speech/rpc-schemas"; +import { useFetchQuery } from "@/data/query"; +import { useHostRuntimeClient, useHostRuntimeIsConnected } from "@/runtime/host-runtime"; +import { isLocalModelBusy } from "./voice-settings"; + +const DOWNLOAD_POLL_INTERVAL_MS = 1000; + +const localSpeechModelsQueryKey = (serverId: string) => ["speech-local-models", serverId] as const; + +export interface LocalSpeechModels { + models: LocalSpeechModel[]; + isLoading: boolean; + error: Error | null; + refresh: () => void; + download: (modelId: string) => Promise; + remove: (modelId: string) => Promise; +} + +// The daemon reports download progress through the list, so the list polls while any +// model is downloading or extracting. +export function useLocalSpeechModels(serverId: string): LocalSpeechModels { + const { t } = useTranslation(); + const client = useHostRuntimeClient(serverId); + const isConnected = useHostRuntimeIsConnected(serverId); + const query = useFetchQuery({ + queryKey: localSpeechModelsQueryKey(serverId), + enabled: Boolean(client && isConnected), + dataShape: "list", + staleTimeMs: 0, + queryFn: () => { + if (!client) throw new Error(t("workspace.terminal.hostDisconnected")); + return client.listLocalSpeechModels(); + }, + refetchInterval: (current) => + current.state.data?.some(isLocalModelBusy) ? DOWNLOAD_POLL_INTERVAL_MS : false, + }); + const { refetch } = query; + + const refresh = useCallback(() => { + void refetch(); + }, [refetch]); + + const download = useCallback( + async (modelId: string) => { + if (!client) return; + await client.downloadLocalSpeechModel(modelId); + await refetch(); + }, + [client, refetch], + ); + + const remove = useCallback( + async (modelId: string) => { + if (!client) return; + await client.deleteLocalSpeechModel(modelId); + await refetch(); + }, + [client, refetch], + ); + + return { + models: query.data ?? [], + isLoading: query.isLoading, + error: query.error, + refresh, + download, + remove, + }; +} diff --git a/packages/app/src/screens/settings/voice/voice-page.tsx b/packages/app/src/screens/settings/voice/voice-page.tsx new file mode 100644 index 0000000000..b8b6402b76 --- /dev/null +++ b/packages/app/src/screens/settings/voice/voice-page.tsx @@ -0,0 +1,252 @@ +import { useCallback, useState } from "react"; +import { View } from "react-native"; +import { useTranslation } from "react-i18next"; +import type { TFunction } from "i18next"; +import { StyleSheet } from "react-native-unistyles"; +import type { SpeechSettingsPatch } from "@getpaseo/protocol/speech/rpc-schemas"; +import { + SettingsCard, + SettingsDetailRow, + SettingsRow, + SettingsSection, +} from "@/components/settings"; +import { Alert } from "@/components/ui/alert"; +import { LoadingSpinner } from "@/components/ui/loading-spinner"; +import { Switch } from "@/components/ui/switch"; +import { useToast } from "@/contexts/toast-api-context"; +import { useDaemonConfig } from "@/hooks/use-daemon-config"; +import { useHostFeature } from "@/runtime/host-features"; +import { useHostRuntimeIsConnected } from "@/runtime/host-runtime"; +import { SpeechProviderSheet } from "./speech-provider-sheet"; +import { useLocalSpeechModels } from "./use-local-speech-models"; +import { + getSpeechSite, + summarizeSpeechSite, + type SpeechSiteId, + type SpeechSiteSummary, +} from "./voice-settings"; + +type SavePatch = (patch: SpeechSettingsPatch) => Promise; +const EMPTY_ENV_OVERRIDES: Record = {}; + +// The sheet mounts fresh for every open (docs/forms.md); `openCount` keys each mount. +interface OpenSheet { + siteId: SpeechSiteId; + visible: boolean; + openCount: number; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function HostVoicePage({ serverId }: { serverId: string }) { + const { t } = useTranslation(); + const toast = useToast(); + const isConnected = useHostRuntimeIsConnected(serverId); + const supported = useHostFeature(serverId, "speechSettings"); + const { config, patchConfig } = useDaemonConfig(serverId); + const localModels = useLocalSpeechModels(serverId); + const { refresh: refreshModels } = localModels; + const [sheet, setSheet] = useState(null); + + // Selecting a model changes which models are in use, so the catalog is re-read after a save. + const save = useCallback( + async (patch) => { + await patchConfig({ features: patch }); + refreshModels(); + }, + [patchConfig, refreshModels], + ); + const saveSwitch = useCallback( + (patch: SpeechSettingsPatch) => { + save(patch).catch((error: unknown) => { + toast.error(`${t("settings.voice.saveError")}: ${errorMessage(error)}`); + }); + }, + [save, t, toast], + ); + const setDictationEnabled = useCallback( + (enabled: boolean) => saveSwitch({ dictation: { enabled } }), + [saveSwitch], + ); + const setVoiceModeEnabled = useCallback( + (enabled: boolean) => saveSwitch({ voiceMode: { enabled } }), + [saveSwitch], + ); + const openSheet = useCallback((siteId: SpeechSiteId) => { + setSheet((current) => ({ + siteId, + visible: true, + openCount: (current?.openCount ?? 0) + 1, + })); + }, []); + const closeSheet = useCallback(() => { + setSheet((current) => (current ? { ...current, visible: false } : null)); + }, []); + const dismissSheet = useCallback(() => setSheet(null), []); + + if (!isConnected) { + return ; + } + if (!supported) { + return ; + } + const features = config?.features; + if (!features) { + return ( + + + + ); + } + const envOverrides = config?.envOverrides ?? EMPTY_ENV_OVERRIDES; + const siteRow = (siteId: SpeechSiteId, label: string) => ( + + ); + const sheetSite = sheet ? getSpeechSite(features, sheet.siteId) : null; + + return ( + + + + + {siteRow("dictation.stt", t("settings.voice.sites.dictationStt"))} + + + + + + {siteRow("voiceMode.stt", t("settings.voice.sites.voiceModeStt"))} + {siteRow("voiceMode.tts", t("settings.voice.sites.voiceModeTts"))} + + + {sheet && sheetSite && sheetSite.provider !== "unknown" ? ( + + ) : null} + + ); +} + +function siteLabel(t: TFunction, siteId: SpeechSiteId): string { + switch (siteId) { + case "dictation.stt": + return t("settings.voice.sites.dictationStt"); + case "voiceMode.stt": + return t("settings.voice.sites.voiceModeStt"); + case "voiceMode.tts": + return t("settings.voice.sites.voiceModeTts"); + } +} + +function FeatureSwitchRow({ + label, + hint, + envVar, + value, + onChange, +}: { + label: string; + hint: string; + envVar: string | undefined; + value: boolean; + onChange: (value: boolean) => void; +}) { + const { t } = useTranslation(); + return ( + + + + ); +} + +function SpeechSiteRow({ + siteId, + label, + summary, + onOpen, +}: { + siteId: SpeechSiteId; + label: string; + summary: SpeechSiteSummary; + onOpen: (siteId: SpeechSiteId) => void; +}) { + const { t } = useTranslation(); + const open = useCallback(() => onOpen(siteId), [onOpen, siteId]); + return ( + + ); +} + +function summaryLabel(t: TFunction, summary: SpeechSiteSummary): string { + switch (summary.kind) { + case "local": + return summary.model + ? `${t("settings.voice.providers.local")} · ${summary.model}` + : t("settings.voice.providers.local"); + case "openai": + return `${t("settings.voice.providers.openai")} · ${ + summary.missingKey ? t("settings.voice.openai.missingKey") : summary.model + }`; + case "unknown": + return t("settings.voice.unknownProvider", { provider: summary.provider }); + } +} + +const styles = StyleSheet.create((theme) => ({ + loading: { + paddingVertical: theme.spacing[8], + alignItems: "center", + }, + spinnerColor: { + color: theme.colors.foregroundMuted, + }, +})); diff --git a/packages/app/src/screens/settings/voice/voice-settings.test.ts b/packages/app/src/screens/settings/voice/voice-settings.test.ts new file mode 100644 index 0000000000..2541c4892c --- /dev/null +++ b/packages/app/src/screens/settings/voice/voice-settings.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it } from "vitest"; +import type { LocalSpeechModel } from "@getpaseo/protocol/speech/rpc-schemas"; +import { + buildOpenAiModelOptions, + getSpeechSite, + formatLanguages, + formatModelSize, + getLocalModelState, + summarizeSpeechSite, +} from "./voice-settings"; + +function model(overrides: Partial): LocalSpeechModel { + return { + id: "parakeet-tdt-0.6b-v2-int8", + kind: "stt", + label: "Parakeet v2", + description: "Accurate English transcription.", + languages: ["en"], + downloadSizeBytes: 482_468_385, + isDefault: true, + inUse: true, + status: "installed", + ...overrides, + }; +} + +const catalog = [ + model({}), + model({ + id: "parakeet-tdt-110m-en-int8", + label: "Parakeet Small", + isDefault: false, + status: "not_installed", + inUse: false, + }), + model({ id: "kokoro-en-v0_19", kind: "tts", label: "Kokoro", inUse: false }), +]; + +describe("voice settings", () => { + it("keeps local STT language from the wire when opening a provider sheet", () => { + const features = { + dictation: { enabled: true, stt: { provider: "local", language: "fr" } }, + voiceMode: { + enabled: false, + stt: { provider: "local", language: "de" }, + tts: { provider: "local" }, + }, + }; + expect(getSpeechSite(features, "dictation.stt")).toMatchObject({ + provider: "local", + language: "fr", + }); + expect(getSpeechSite(features, "voiceMode.stt")).toMatchObject({ + provider: "local", + language: "de", + }); + }); + + it("summarizes a local site by its model, falling back to the catalog default", () => { + const summary = (modelId?: string) => + summarizeSpeechSite({ + siteId: "dictation.stt", + site: { provider: "local", model: modelId }, + envOverrides: {}, + localModels: catalog, + }); + expect(summary()).toEqual({ kind: "local", model: "Parakeet v2" }); + expect(summary("parakeet-tdt-110m-en-int8")).toEqual({ + kind: "local", + model: "Parakeet Small", + }); + }); + + it("flags an OpenAI site with no saved key and no env key", () => { + const summary = (envOverrides: Record) => + summarizeSpeechSite({ + siteId: "voiceMode.tts", + site: { provider: "openai", hasApiKey: false }, + envOverrides, + localModels: catalog, + }); + expect(summary({})).toEqual({ kind: "openai", model: "tts-1", missingKey: true }); + expect(summary({ "features.voiceMode.tts.apiKey": "OPENAI_API_KEY" })).toEqual({ + kind: "openai", + model: "tts-1", + missingKey: false, + }); + }); + + it("keeps a custom OpenAI-compatible model id selectable", () => { + expect(buildOpenAiModelOptions("stt", "my-whisper").at(-1)).toEqual({ + value: "my-whisper", + label: "my-whisper", + }); + }); + + it("reports download progress as a percentage", () => { + expect( + getLocalModelState(model({ status: "downloading", receivedBytes: 241, totalBytes: 482 })), + ).toEqual({ kind: "downloading", percent: 50 }); + expect(getLocalModelState(model({ status: "downloading" }))).toEqual({ + kind: "downloading", + percent: null, + }); + }); + + it("formats download sizes", () => { + expect(formatModelSize(482_468_385)).toBe("482 MB"); + expect(formatModelSize(1_250_000_000)).toBe("1.3 GB"); + }); + + it("names a short language list and counts the rest of a long one", () => { + const formatMore = (languages: string, count: number) => `${languages} and ${count} more`; + expect(formatLanguages({ languages: ["en", "de"], locale: "en", formatMore })).toBe( + "English, German", + ); + const whisper = ["en", "zh", "de", "es", "ru", "ko", "fr", "ja", "pt", "tr", "pl", "ca"]; + expect(formatLanguages({ languages: whisper, locale: "en", formatMore })).toBe( + "English, Chinese, German, Spanish, Russian, Korean, French, Japanese, Portuguese, Turkish and 2 more", + ); + }); +}); diff --git a/packages/app/src/screens/settings/voice/voice-settings.ts b/packages/app/src/screens/settings/voice/voice-settings.ts new file mode 100644 index 0000000000..79c1062b49 --- /dev/null +++ b/packages/app/src/screens/settings/voice/voice-settings.ts @@ -0,0 +1,240 @@ +import type { + LocalSpeechModel, + SpeechSettings, + SpeechSettingsPatch, +} from "@getpaseo/protocol/speech/rpc-schemas"; + +// What the Voice page needs to know about speech providers and models, kept free of React +// so the option lists and model states are testable on their own. + +export type SpeechProvider = "local" | "openai"; +export type SpeechModelKind = "stt" | "tts"; + +export const SPEECH_PROVIDERS: readonly SpeechProvider[] = ["local", "openai"]; + +// The three places that use speech. Each carries its own provider settings in config.json +// under `features.`. +export type SpeechSiteId = "dictation.stt" | "voiceMode.stt" | "voiceMode.tts"; + +export const SPEECH_SITE_KIND: Record = { + "dictation.stt": "stt", + "voiceMode.stt": "stt", + "voiceMode.tts": "tts", +}; + +/** The config path the daemon names in `envOverrides` for a field of a site. */ +export function speechSiteFieldPath(siteId: SpeechSiteId, field: string): string { + return `features.${siteId}.${field}`; +} + +/** One site's settings as the daemon reports them, narrowed to what the app can edit. */ +export type SpeechSite = + | { provider: "local"; model?: string; language?: string } + | { + provider: "openai"; + hasApiKey: boolean; + baseUrl?: string; + model?: string; + language?: string; + voice?: string; + } + | { provider: "unknown"; id: string }; + +type SpeechSiteSettings = SpeechSettings["dictation"]["stt"] | SpeechSettings["voiceMode"]["tts"]; + +export function getSpeechSite(features: SpeechSettings, siteId: SpeechSiteId): SpeechSite { + switch (siteId) { + case "dictation.stt": + return toSpeechSite(features.dictation.stt); + case "voiceMode.stt": + return toSpeechSite(features.voiceMode.stt); + case "voiceMode.tts": + return toSpeechSite(features.voiceMode.tts); + } +} + +function toSpeechSite(settings: SpeechSiteSettings): SpeechSite { + const text = (value: unknown) => (typeof value === "string" && value ? value : undefined); + if (settings.provider === "local") + return { provider: "local", model: text(settings.model), language: text(settings.language) }; + if (settings.provider === "openai") { + return { + provider: "openai", + hasApiKey: settings.hasApiKey === true, + baseUrl: text(settings.baseUrl), + model: text(settings.model), + language: text(settings.language), + voice: text(settings.voice), + }; + } + return { provider: "unknown", id: settings.provider }; +} + +export type SpeechSitePatch = NonNullable< + | NonNullable["stt"] + | NonNullable["tts"] +>; + +export function buildSpeechSitePatch( + siteId: SpeechSiteId, + patch: SpeechSitePatch, +): SpeechSettingsPatch { + switch (siteId) { + case "dictation.stt": + return { dictation: { stt: patch } }; + case "voiceMode.stt": + return { voiceMode: { stt: patch } }; + case "voiceMode.tts": + return { voiceMode: { tts: patch } }; + } +} + +// OpenAI accepts free-form ids so self-hosted servers can use their own; these are the +// ones the page offers. A saved id outside the list is still shown as an option. +const OPENAI_MODELS: Record = { + stt: { + ids: ["whisper-1", "gpt-4o-transcribe", "gpt-4o-mini-transcribe"], + defaultId: "whisper-1", + }, + tts: { ids: ["tts-1", "tts-1-hd", "gpt-4o-mini-tts"], defaultId: "tts-1" }, +}; + +const OPENAI_VOICES = ["alloy", "ash", "coral", "echo", "fable", "nova", "onyx", "sage", "shimmer"]; +const DEFAULT_OPENAI_VOICE = "alloy"; + +export interface SpeechModelOption { + value: string; + label: string; +} + +export function resolveOpenAiModel(kind: SpeechModelKind, model: string | undefined): string { + return model ?? OPENAI_MODELS[kind].defaultId; +} + +export function resolveLocalModel(input: { + kind: SpeechModelKind; + model: string | undefined; + localModels: readonly LocalSpeechModel[]; +}): LocalSpeechModel | undefined { + const kindModels = input.localModels.filter((model) => model.kind === input.kind); + if (input.model) return kindModels.find((model) => model.id === input.model); + return kindModels.find((model) => model.isDefault); +} + +export function buildOpenAiModelOptions( + kind: SpeechModelKind, + selected: string, +): SpeechModelOption[] { + return withSelected( + OPENAI_MODELS[kind].ids.map((id) => ({ value: id, label: id })), + selected, + ); +} + +export function resolveOpenAiVoice(voice: string | undefined): string { + return voice ?? DEFAULT_OPENAI_VOICE; +} + +export function buildOpenAiVoiceOptions(selected: string): SpeechModelOption[] { + return withSelected( + OPENAI_VOICES.map((voice) => ({ value: voice, label: voice })), + selected, + ); +} + +function withSelected(options: SpeechModelOption[], selected: string): SpeechModelOption[] { + if (!selected || options.some((option) => option.value === selected)) return options; + return [...options, { value: selected, label: selected }]; +} + +// What a page row says about a site before it is opened. +export type SpeechSiteSummary = + | { kind: "local"; model: string | null } + | { kind: "openai"; model: string; missingKey: boolean } + | { kind: "unknown"; provider: string }; + +export function summarizeSpeechSite(input: { + siteId: SpeechSiteId; + site: SpeechSite; + envOverrides: Record; + localModels: readonly LocalSpeechModel[]; +}): SpeechSiteSummary { + const { site, siteId } = input; + const kind = SPEECH_SITE_KIND[siteId]; + switch (site.provider) { + case "local": { + const model = resolveLocalModel({ kind, model: site.model, localModels: input.localModels }); + return { kind: "local", model: model?.label ?? site.model ?? null }; + } + case "openai": + return { + kind: "openai", + model: resolveOpenAiModel(kind, site.model), + missingKey: !site.hasApiKey && !input.envOverrides[speechSiteFieldPath(siteId, "apiKey")], + }; + case "unknown": + return { kind: "unknown", provider: site.id }; + } +} + +export type LocalModelState = + | { kind: "installed" } + | { kind: "not_installed" } + | { kind: "downloading"; percent: number | null } + | { kind: "extracting" } + | { kind: "failed"; error: string }; + +export function getLocalModelState(model: LocalSpeechModel): LocalModelState { + switch (model.status) { + case "installed": + return { kind: "installed" }; + case "downloading": { + const total = model.totalBytes ?? 0; + const received = model.receivedBytes ?? 0; + return { + kind: "downloading", + percent: total > 0 ? Math.min(100, Math.floor((received / total) * 100)) : null, + }; + } + case "extracting": + return { kind: "extracting" }; + case "failed": + return { kind: "failed", error: model.error ?? "" }; + default: + return { kind: "not_installed" }; + } +} + +export function isLocalModelBusy(model: LocalSpeechModel): boolean { + return model.status === "downloading" || model.status === "extracting"; +} + +export function formatModelSize(bytes: number): string { + const megabytes = bytes / 1_000_000; + if (megabytes >= 1000) return `${(megabytes / 1000).toFixed(1)} GB`; + return `${Math.round(megabytes)} MB`; +} + +// Multilingual models list close to a hundred languages; past this many the tooltip names +// the first ones and counts the rest. +const MAX_LISTED_LANGUAGES = 10; + +export function formatLanguages(input: { + languages: readonly string[]; + locale: string; + formatMore: (languages: string, count: number) => string; +}): string { + const names = languageNames(input.locale); + const listed = input.languages + .slice(0, MAX_LISTED_LANGUAGES) + .map((code) => names?.of(code) ?? code.toUpperCase()) + .join(", "); + const hidden = input.languages.length - MAX_LISTED_LANGUAGES; + return hidden > 0 ? input.formatMore(listed, hidden) : listed; +} + +// Hermes ships without Intl.DisplayNames; the language code is the fallback label there. +function languageNames(locale: string): Intl.DisplayNames | null { + if (typeof Intl.DisplayNames !== "function") return null; + return new Intl.DisplayNames([locale], { type: "language" }); +} diff --git a/packages/app/src/utils/host-routes.ts b/packages/app/src/utils/host-routes.ts index cb828f9e5a..c5913366d2 100644 --- a/packages/app/src/utils/host-routes.ts +++ b/packages/app/src/utils/host-routes.ts @@ -518,6 +518,7 @@ export const HOST_SECTION_SLUGS = [ "pair-device", "agents", "metadata", + "voice", "workspaces", "providers", "usage", diff --git a/packages/app/vitest.config.ts b/packages/app/vitest.config.ts index 2b56646b2a..3254fc5e0c 100644 --- a/packages/app/vitest.config.ts +++ b/packages/app/vitest.config.ts @@ -161,6 +161,7 @@ export default defineConfig({ "i18next", "@marijn/find-cluster-break", "tiny-invariant", + "react/jsx-dev-runtime", "react-native-gesture-handler > hoist-non-react-statics", "react-native-gesture-handler > invariant", ], @@ -171,6 +172,8 @@ export default defineConfig({ "react-native-keyboard-controller", ], }, + // babel-preset-expo compiles JSX with the automatic runtime, so app modules don't import React. + esbuild: { jsx: "automatic" }, // The globals a React Native bundler defines, which esbuild is no longer there to supply for // the package excluded above. define: { diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index 524df56704..59d928283f 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -19,6 +19,7 @@ import { CreationClient } from "./creation/index.js"; import type { CreationSnapshot } from "@getpaseo/protocol/messages"; import type { z } from "zod"; import type { SessionEventSubscription } from "@getpaseo/protocol/messages"; +import type { LocalSpeechModel } from "@getpaseo/protocol/speech/rpc-schemas"; import { CLIENT_CAPS, type ClientCapability } from "@getpaseo/protocol/client-capabilities"; import type { AgentAttentionNotificationPayload } from "@getpaseo/protocol/agent-attention-notification"; import { MAX_WEBSOCKET_MESSAGE_BYTES } from "@getpaseo/protocol/websocket-message-limit"; @@ -5547,6 +5548,34 @@ export class DaemonClient { }); } + async listLocalSpeechModels(requestId?: string): Promise { + const payload = await this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "speech.local_models.list.request" }, + responseType: "speech.local_models.list.response", + }); + if (payload.error) throw new Error(payload.error); + return payload.models; + } + + async downloadLocalSpeechModel(modelId: string, requestId?: string): Promise { + const payload = await this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "speech.local_models.download.request", modelId }, + responseType: "speech.local_models.download.response", + }); + if (payload.error) throw new Error(payload.error); + } + + async deleteLocalSpeechModel(modelId: string, requestId?: string): Promise { + const payload = await this.sendCorrelatedSessionRequest({ + requestId, + message: { type: "speech.local_models.delete.request", modelId }, + responseType: "speech.local_models.delete.response", + }); + if (payload.error) throw new Error(payload.error); + } + sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void { this.sendSessionMessageStrict(response); } diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index bc158a6322..3034598f1e 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -82,6 +82,16 @@ import { ScheduleRunOnceResponseSchema, ScheduleUpdateResponseSchema, } from "./schedule/rpc-schemas.js"; +import { + SpeechLocalModelsDeleteRequestSchema, + SpeechLocalModelsDeleteResponseSchema, + SpeechLocalModelsDownloadRequestSchema, + SpeechLocalModelsDownloadResponseSchema, + SpeechLocalModelsListRequestSchema, + SpeechLocalModelsListResponseSchema, + SpeechSettingsPatchSchema, + SpeechSettingsSchema, +} from "./speech/rpc-schemas.js"; import { LoopRunRequestSchema, LoopListRequestSchema, @@ -233,6 +243,9 @@ export const MutableDaemonConfigSchema = z skills: z.object({ selection: AgentSkillSelectionSchema.optional() }).strict().optional(), pluginsEnabled: z.boolean().optional(), plugins: z.record(PluginIdSchema, PluginSourceSchema).optional(), + // COMPAT(speechSettings): added in v0.12.0, optional because older daemons omit it. + features: SpeechSettingsSchema.optional(), + envOverrides: z.record(z.string(), z.string()).optional(), }) .passthrough(); @@ -254,6 +267,7 @@ export const MutableDaemonConfigPatchSchema = z agentProfiles: z.array(AgentProfileSchema).optional(), pluginsEnabled: z.boolean().optional(), plugins: z.record(PluginIdSchema, PluginSourceSchema).optional(), + features: SpeechSettingsPatchSchema.optional(), }) .partial() .passthrough(); @@ -3440,6 +3454,9 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ ChatWaitRequestSchema, ScheduleCreateRequestSchema, ScheduleListRequestSchema, + SpeechLocalModelsListRequestSchema, + SpeechLocalModelsDownloadRequestSchema, + SpeechLocalModelsDeleteRequestSchema, ScheduleInspectRequestSchema, ScheduleLogsRequestSchema, SchedulePauseRequestSchema, @@ -3674,6 +3691,8 @@ export const ServerInfoStatusPayloadSchema = z daemonStatusRpc: z.boolean().optional(), // COMPAT(daemonConfigReload): added in v0.4.0, remove gate after 2027-02-14. daemonConfigReload: z.boolean().optional(), + // COMPAT(speechSettings): added in v0.12.0, remove gate after 2027-04-18. + speechSettings: z.boolean().optional(), // COMPAT(relayConfig): added in v0.2.6, remove gate after 2027-01-31. relayConfig: z.boolean().optional(), // COMPAT(pushTokenRevocation): added in v0.3.2, remove gate after 2027-02-10. @@ -7152,6 +7171,9 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ ChatWaitResponseSchema, ScheduleCreateResponseSchema, ScheduleListResponseSchema, + SpeechLocalModelsListResponseSchema, + SpeechLocalModelsDownloadResponseSchema, + SpeechLocalModelsDeleteResponseSchema, ScheduleInspectResponseSchema, ScheduleLogsResponseSchema, SchedulePauseResponseSchema, diff --git a/packages/protocol/src/messages.wire-compat.test.ts b/packages/protocol/src/messages.wire-compat.test.ts index d3b3ce26df..d6b2bc02ff 100644 --- a/packages/protocol/src/messages.wire-compat.test.ts +++ b/packages/protocol/src/messages.wire-compat.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from "vitest"; import { z } from "zod"; import { + MutableDaemonConfigSchema, + MutableDaemonConfigPatchSchema, UsageReportEntrySchema, AgentSnapshotPayloadSchema, AgentTimelineItemPayloadSchema, @@ -525,3 +527,50 @@ test("released setup clients parse empty combined logs and retain command output }).detail.log, ).toBe("old daemon combined output"); }); + +test("local STT language survives wire reads and patches", () => { + const patch = { + features: { + dictation: { stt: { provider: "local", language: "fr" } }, + voiceMode: { stt: { provider: "local", language: null } }, + }, + }; + expect(MutableDaemonConfigPatchSchema.parse(patch)).toEqual(patch); +}); + +test("speech config reads remain compatible with clients before features and future providers", () => { + const oldSchema = MutableDaemonConfigSchema.omit({ features: true, envOverrides: true }); + const oldRead = oldSchema.parse({ + providers: {}, + relay: { enabled: false }, + mcp: { enabled: true, injectIntoAgents: true }, + }); + const read = { + ...oldRead, + features: { + dictation: { enabled: true, stt: { provider: "local", language: "fr" } }, + voiceMode: { + enabled: false, + stt: { provider: "future", model: "future-model", region: "eu" }, + tts: { provider: "openai", hasApiKey: false }, + }, + }, + envOverrides: { "features.voiceMode.stt.provider": "PASEO_VOICE_STT_PROVIDER" }, + }; + expect(oldSchema.parse(read)).toEqual(read); + expect(MutableDaemonConfigSchema.parse(oldRead)).toEqual(oldRead); + expect(MutableDaemonConfigSchema.parse(read).features?.dictation.stt).toEqual( + read.features.dictation.stt, + ); + expect(MutableDaemonConfigSchema.parse(read).features?.voiceMode.stt).toEqual( + read.features.voiceMode.stt, + ); + const oldPatchSchema = MutableDaemonConfigPatchSchema.omit({ features: true }); + const oldPatch = oldPatchSchema.parse({ relay: { enabled: true } }); + expect(MutableDaemonConfigPatchSchema.parse(oldPatch)).toEqual(oldPatch); + expect( + MutableDaemonConfigPatchSchema.parse({ + features: { voiceMode: { stt: { provider: "future", region: "eu" } } }, + }).features?.voiceMode?.stt, + ).toEqual({ provider: "future", region: "eu" }); +}); diff --git a/packages/protocol/src/speech/rpc-schemas.ts b/packages/protocol/src/speech/rpc-schemas.ts new file mode 100644 index 0000000000..437560c7e8 --- /dev/null +++ b/packages/protocol/src/speech/rpc-schemas.ts @@ -0,0 +1,202 @@ +import { z } from "zod"; + +// Per-site settings mirror config.json. Unknown providers remain parseable by older +// clients; the feature UI decides whether it knows how to edit them. +const ModelFields = { model: z.string().optional() }; +const OpenAiReadFields = { + ...ModelFields, + hasApiKey: z.boolean(), + baseUrl: z.string().optional(), +}; +const SttReadFields = { + language: z.string().optional(), + confidenceThreshold: z.number().optional(), +}; +const LocalTtsFields = { speakerId: z.number().int().optional(), speed: z.number().optional() }; + +export const SpeechSttSettingsSchema = z.union([ + z + .object({ provider: z.literal("local"), ...ModelFields, language: z.string().optional() }) + .passthrough(), + z.object({ provider: z.literal("openai"), ...OpenAiReadFields, ...SttReadFields }).passthrough(), + z.object({ provider: z.string(), ...ModelFields }).passthrough(), +]); +export const SpeechTtsSettingsSchema = z.union([ + z.object({ provider: z.literal("local"), ...ModelFields, ...LocalTtsFields }).passthrough(), + z + .object({ provider: z.literal("openai"), ...OpenAiReadFields, voice: z.string().optional() }) + .passthrough(), + z.object({ provider: z.string(), ...ModelFields }).passthrough(), +]); + +export const SpeechSettingsSchema = z + .object({ + dictation: z.object({ enabled: z.boolean(), stt: SpeechSttSettingsSchema }).passthrough(), + voiceMode: z + .object({ + enabled: z.boolean(), + stt: SpeechSttSettingsSchema, + tts: SpeechTtsSettingsSchema, + llm: z + .object({ provider: z.string().optional(), model: z.string().optional() }) + .passthrough() + .optional(), + turnDetection: z.object({ provider: z.string().optional() }).passthrough().optional(), + }) + .passthrough(), + }) + .passthrough(); + +const ModelPatchFields = { model: z.string().nullable().optional() }; +const OpenAiPatchFields = { + ...ModelPatchFields, + apiKey: z.string().nullable().optional(), + baseUrl: z.string().nullable().optional(), +}; +const SttPatchFields = { + language: z.string().nullable().optional(), + confidenceThreshold: z.number().nullable().optional(), +}; +const LocalTtsPatchFields = { + speakerId: z.number().int().nullable().optional(), + speed: z.number().nullable().optional(), +}; +const SpeechSttSettingsPatchSchema = z.union([ + z + .object({ + provider: z.literal("local"), + ...ModelPatchFields, + language: z.string().nullable().optional(), + }) + .passthrough(), + z + .object({ provider: z.literal("openai"), ...OpenAiPatchFields, ...SttPatchFields }) + .passthrough(), + z.object({ provider: z.string().nullable() }).passthrough(), + z + .object({ provider: z.undefined().optional(), ...OpenAiPatchFields, ...SttPatchFields }) + .passthrough(), +]); +const SpeechTtsSettingsPatchSchema = z.union([ + z + .object({ provider: z.literal("local"), ...ModelPatchFields, ...LocalTtsPatchFields }) + .passthrough(), + z + .object({ + provider: z.literal("openai"), + ...OpenAiPatchFields, + voice: z.string().nullable().optional(), + }) + .passthrough(), + z.object({ provider: z.string().nullable() }).passthrough(), + z + .object({ + provider: z.undefined().optional(), + ...OpenAiPatchFields, + ...LocalTtsPatchFields, + voice: z.string().nullable().optional(), + }) + .passthrough(), +]); +export const SpeechSettingsPatchSchema = z + .object({ + dictation: z + .object({ + enabled: z.boolean().nullable().optional(), + stt: SpeechSttSettingsPatchSchema.optional(), + }) + .passthrough() + .optional(), + voiceMode: z + .object({ + enabled: z.boolean().nullable().optional(), + stt: SpeechSttSettingsPatchSchema.optional(), + tts: SpeechTtsSettingsPatchSchema.optional(), + llm: z + .object({ + provider: z.string().nullable().optional(), + model: z.string().nullable().optional(), + }) + .passthrough() + .optional(), + turnDetection: z + .object({ provider: z.string().nullable().optional() }) + .passthrough() + .optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(); + +export type SpeechSettings = z.infer; +export type SpeechSettingsPatch = z.infer; + +// --------------------------------------------------------------------------- +// Local speech models: catalog, install state, download, delete. +// --------------------------------------------------------------------------- + +export const LocalSpeechModelSchema = z + .object({ + id: z.string(), + // "stt" or "tts". + kind: z.string(), + label: z.string(), + description: z.string(), + languages: z.array(z.string()), + downloadSizeBytes: z.number(), + isDefault: z.boolean(), + // Selected by an enabled feature; the daemon keeps it installed. + inUse: z.boolean(), + // "installed" | "not_installed" | "downloading" | "extracting" | "failed". + status: z.string(), + receivedBytes: z.number().optional(), + totalBytes: z.number().optional(), + error: z.string().optional(), + }) + .passthrough(); + +export type LocalSpeechModel = z.infer; + +export const SpeechLocalModelsListRequestSchema = z.object({ + type: z.literal("speech.local_models.list.request"), + requestId: z.string(), +}); + +export const SpeechLocalModelsDownloadRequestSchema = z.object({ + type: z.literal("speech.local_models.download.request"), + requestId: z.string(), + modelId: z.string(), +}); + +export const SpeechLocalModelsDeleteRequestSchema = z.object({ + type: z.literal("speech.local_models.delete.request"), + requestId: z.string(), + modelId: z.string(), +}); + +export const SpeechLocalModelsListResponseSchema = z.object({ + type: z.literal("speech.local_models.list.response"), + payload: z.object({ + requestId: z.string(), + models: z.array(LocalSpeechModelSchema), + error: z.string().nullable(), + }), +}); + +// Resolves once the download starts; poll the list for progress. +export const SpeechLocalModelsDownloadResponseSchema = z.object({ + type: z.literal("speech.local_models.download.response"), + payload: z.object({ + requestId: z.string(), + error: z.string().nullable(), + }), +}); + +export const SpeechLocalModelsDeleteResponseSchema = z.object({ + type: z.literal("speech.local_models.delete.response"), + payload: z.object({ + requestId: z.string(), + error: z.string().nullable(), + }), +}); diff --git a/packages/server/src/server/agent/stt-manager.test.ts b/packages/server/src/server/agent/stt-manager.test.ts index e3f6fb7d9c..5b1c036bc7 100644 --- a/packages/server/src/server/agent/stt-manager.test.ts +++ b/packages/server/src/server/agent/stt-manager.test.ts @@ -1,9 +1,12 @@ +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; import pino from "pino"; import { EventEmitter } from "node:events"; import { STTManager } from "./stt-manager.js"; -import { PersistedConfigSchema } from "../persisted-config.js"; +import { loadPersistedConfig } from "../persisted-config.js"; import { resolveSpeechConfig } from "../speech/speech-config-resolver.js"; import type { SpeechToTextProvider, @@ -97,12 +100,18 @@ class SequencedFakeStt implements SpeechToTextProvider { describe("STTManager", () => { function resolveVoiceLanguage(params: { env?: NodeJS.ProcessEnv; persisted?: unknown }): string { - const result = resolveSpeechConfig({ - paseoHome: "/tmp/paseo-home", - env: params.env ?? ({} as NodeJS.ProcessEnv), - persisted: PersistedConfigSchema.parse(params.persisted ?? {}), - }); - return result.speech.sttLanguages.voice; + const home = mkdtempSync(path.join(tmpdir(), "stt-language-")); + try { + writeFileSync(path.join(home, "config.json"), JSON.stringify(params.persisted ?? {})); + const result = resolveSpeechConfig({ + paseoHome: home, + env: params.env ?? {}, + persisted: loadPersistedConfig(home), + }); + return result.speech.sttLanguages.voice; + } finally { + rmSync(home, { recursive: true, force: true }); + } } async function transcribeWithResolvedVoiceLanguage(params: { diff --git a/packages/server/src/server/authorization/operation-permissions.ts b/packages/server/src/server/authorization/operation-permissions.ts index 80599a74a3..f7a5b17857 100644 --- a/packages/server/src/server/authorization/operation-permissions.ts +++ b/packages/server/src/server/authorization/operation-permissions.ts @@ -182,6 +182,9 @@ const INBOUND_PERMISSION = { set_agent_model_request: "workspace.write", set_agent_thinking_request: "workspace.write", set_daemon_config_request: "daemon.manage", + "speech.local_models.list.request": "daemon.read", + "speech.local_models.download.request": "daemon.manage", + "speech.local_models.delete.request": "daemon.manage", set_voice_mode: "workspace.write", shutdown_server_request: "daemon.manage", start_workspace_script_request: "workspace.write", @@ -417,6 +420,9 @@ const OUTBOUND_PERMISSION = { set_agent_model_response: "workspace.write", set_agent_thinking_response: "workspace.write", set_daemon_config_response: "daemon.manage", + "speech.local_models.list.response": "daemon.read", + "speech.local_models.download.response": "daemon.manage", + "speech.local_models.delete.response": "daemon.manage", set_voice_mode_response: "workspace.write", start_workspace_script_response: "workspace.write", stash_list_response: "workspace.read", diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 9419694043..2eeccfb653 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -138,6 +138,7 @@ import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/confi import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js"; import type { RequestedSpeechProviders } from "./speech/speech-types.js"; import { createSpeechService } from "./speech/speech-runtime.js"; +import { createSpeechSettingsOwner } from "./speech/speech-settings.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; import { attachAgentStoragePersistence } from "./persistence-hooks.js"; @@ -379,6 +380,8 @@ export interface PaseoSpeechSttLanguages { export interface PaseoSpeechConfig { providers: RequestedSpeechProviders; + // Where local models live, whether or not a feature currently uses the local provider. + localModelsDir: string; sttLanguages?: PaseoSpeechSttLanguages; local?: PaseoLocalSpeechConfig; } @@ -555,6 +558,10 @@ function createInitialRelayConfig(config: PaseoDaemonConfig): MutableDaemonConfi }; } +function resolveReloadEnv(config: PaseoDaemonConfig): NodeJS.ProcessEnv { + return config.configReload?.env ?? process.env; +} + function createInitialMutableDaemonConfig(config: PaseoDaemonConfig): MutableDaemonConfig { const providers = config.providerOverrides ?? {}; @@ -615,13 +622,24 @@ export async function createPaseoDaemon( const elapsed = () => `${(performance.now() - bootstrapStart).toFixed(0)}ms`; const daemonVersion = config.daemonVersion ?? resolveDaemonVersion(import.meta.url); const initialMutableConfig = createInitialMutableDaemonConfig(config); + const speechService = createSpeechService({ + logger, + openaiConfig: config.openai, + speechConfig: config.speech, + }); + logger.info({ elapsed: elapsed() }, "Speech service created"); const daemonConfigStore = new DaemonConfigStore(config.paseoHome, initialMutableConfig, logger, { relayEnabledMutable: config.relayEnabledMutable ?? true, startupPersisted: config.configReload?.startupPersisted, + speech: createSpeechSettingsOwner({ + paseoHome: config.paseoHome, + env: resolveReloadEnv(config), + speechService, + }), reloadSource: { resolve: (persisted) => { const reloaded = resolveConfigFromPersisted(config.paseoHome, persisted, { - env: config.configReload?.env ?? process.env, + env: resolveReloadEnv(config), cli: config.configReload?.cli, relayEnabledFallback: config.configReload?.relayEnabledFallback, }); @@ -1677,13 +1695,6 @@ export async function createPaseoDaemon( logger.info({ route: agentMcpRoute, enabled: mcpEnabled }, "Agent MCP route mounted"); } - const speechService = createSpeechService({ - logger, - openaiConfig: config.openai, - speechConfig: config.speech, - }); - logger.info({ elapsed: elapsed() }, "Speech service created"); - logger.info({ elapsed: elapsed() }, "Bootstrap complete, ready to start listening"); const start = async () => { diff --git a/packages/server/src/server/config.test.ts b/packages/server/src/server/config.test.ts index 864ad14100..e8c5f2faa0 100644 --- a/packages/server/src/server/config.test.ts +++ b/packages/server/src/server/config.test.ts @@ -115,9 +115,13 @@ describe("server config", () => { name: "OpenAI speech providers", providers: { dictation: "openai", voiceStt: "openai", voiceTts: "openai" }, expected: [ + "features.dictation.stt.apiKey", "features.dictation.stt.confidenceThreshold", "features.dictation.stt.model", + "features.voiceMode.stt.apiKey", + "features.voiceMode.stt.confidenceThreshold", "features.voiceMode.stt.model", + "features.voiceMode.tts.apiKey", "features.voiceMode.tts.model", "features.voiceMode.tts.voice", ], @@ -126,8 +130,9 @@ describe("server config", () => { name: "mixed local and OpenAI speech providers", providers: { dictation: "local", voiceStt: "openai", voiceTts: "local" }, expected: [ - "features.dictation.stt.confidenceThreshold", "features.dictation.stt.model", + "features.voiceMode.stt.apiKey", + "features.voiceMode.stt.confidenceThreshold", "features.voiceMode.stt.model", "features.voiceMode.tts.model", ], diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 8187ce03c6..74a8037d07 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -22,7 +22,7 @@ import { ProviderOverrideSchema } from "./agent/provider-launch-config.js"; import { AgentProviderSchema } from "@getpaseo/protocol/provider-manifest"; import { hashDaemonPassword } from "./auth.js"; import { resolveSpeechConfig } from "./speech/speech-config-resolver.js"; -import type { RequestedSpeechProviders } from "./speech/speech-types.js"; +import { readSpeechSettings } from "./speech/speech-settings.js"; import { mergeHostnames, parseHostnamesEnv, type HostnamesConfig } from "./hostnames.js"; import { resolveGitProcessPolicy } from "../utils/git-process-scheduler.js"; @@ -614,7 +614,7 @@ export function resolveConfigFromPersisted( persisted.agents?.providers as Record | undefined, ); - const overrideControlledPaths = resolveOverrideControlledPaths(env, cli, speech.providers); + const overrideControlledPaths = resolveOverrideControlledPaths(env, cli, persisted); return { listen, @@ -692,13 +692,13 @@ function parsePositiveGitOverride(value: string | undefined): boolean { function resolveOverrideControlledPaths( env: NodeJS.ProcessEnv, cli: CliConfigOverrides | undefined, - speechProviders: RequestedSpeechProviders, + persisted: PersistedConfig, ): string[] { return Array.from( new Set([ ...resolveDaemonOverrideControlledPaths(env, cli), ...resolveLogOverrideControlledPaths(env), - ...resolveSpeechOverrideControlledPaths(env, speechProviders), + ...resolveSpeechOverrideControlledPaths(env, persisted), ]), ).sort(); } @@ -807,67 +807,12 @@ function resolveLogOverrideControlledPaths(env: NodeJS.ProcessEnv): string[] { return paths; } -function isEnabledSpeechProvider( - provider: RequestedSpeechProviders[keyof RequestedSpeechProviders], - expected: "local" | "openai", -): boolean { - return provider.enabled !== false && provider.provider === expected; -} - function resolveSpeechOverrideControlledPaths( env: NodeJS.ProcessEnv, - providers: RequestedSpeechProviders, + persisted: PersistedConfig, ): string[] { - const paths: string[] = []; - const add = (envName: string, ...configPaths: string[]) => { - if (env[envName] !== undefined) paths.push(...configPaths); - }; - - add("PASEO_DICTATION_ENABLED", "features.dictation.enabled"); - add("PASEO_DICTATION_STT_PROVIDER", "features.dictation.stt.provider"); - if ( - env.PASEO_DICTATION_LOCAL_STT_MODEL !== undefined && - isEnabledSpeechProvider(providers.dictationStt, "local") - ) { - paths.push("features.dictation.stt.model"); - } - add("PASEO_DICTATION_LANGUAGE", "features.dictation.stt.language"); - add("PASEO_VOICE_MODE_ENABLED", "features.voiceMode.enabled"); - add("PASEO_VOICE_LLM_PROVIDER", "features.voiceMode.llm.provider"); - add("PASEO_VOICE_STT_PROVIDER", "features.voiceMode.stt.provider"); - if ( - env.PASEO_VOICE_LOCAL_STT_MODEL !== undefined && - isEnabledSpeechProvider(providers.voiceStt, "local") - ) { - paths.push("features.voiceMode.stt.model"); - } - add("PASEO_VOICE_LANGUAGE", "features.voiceMode.stt.language"); - add("PASEO_VOICE_TURN_DETECTION_PROVIDER", "features.voiceMode.turnDetection.provider"); - add("PASEO_VOICE_TTS_PROVIDER", "features.voiceMode.tts.provider"); - if ( - env.PASEO_VOICE_LOCAL_TTS_MODEL !== undefined && - isEnabledSpeechProvider(providers.voiceTts, "local") - ) { - paths.push("features.voiceMode.tts.model"); - } - add("PASEO_VOICE_LOCAL_TTS_SPEAKER_ID", "features.voiceMode.tts.speakerId"); - add("PASEO_VOICE_LOCAL_TTS_SPEED", "features.voiceMode.tts.speed"); - add("PASEO_LOCAL_MODELS_DIR", "providers.local.modelsDir"); - const openAiDictationStt = isEnabledSpeechProvider(providers.dictationStt, "openai"); - const openAiVoiceStt = isEnabledSpeechProvider(providers.voiceStt, "openai"); - if (env.STT_CONFIDENCE_THRESHOLD !== undefined && (openAiDictationStt || openAiVoiceStt)) { - paths.push("features.dictation.stt.confidenceThreshold"); - } - if (env.STT_MODEL !== undefined) { - if (openAiDictationStt) paths.push("features.dictation.stt.model"); - if (openAiVoiceStt) paths.push("features.voiceMode.stt.model"); - } - if (isEnabledSpeechProvider(providers.voiceTts, "openai")) { - add("TTS_MODEL", "features.voiceMode.tts.model"); - add("TTS_VOICE", "features.voiceMode.tts.voice"); - } - if (env.PASEO_DICTATION_LANGUAGE !== undefined && env.PASEO_VOICE_LANGUAGE === undefined) { - paths.push("features.voiceMode.stt.language"); - } + const paths = Object.keys(readSpeechSettings(persisted, env).envOverrides); + if (env.PASEO_VOICE_LLM_PROVIDER !== undefined) paths.push("features.voiceMode.llm.provider"); + if (env.PASEO_LOCAL_MODELS_DIR !== undefined) paths.push("providers.local.modelsDir"); return paths; } diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index b6518d773d..a442dfe6a9 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -1318,9 +1318,9 @@ test("emits disabled voice capability reasons on fresh daemon startup", async () expect(voice).toBeTruthy(); expect(voice?.dictation.enabled).toBe(false); - expect(voice?.dictation.reason).toBe("Dictation is disabled in daemon config."); + expect(voice?.dictation.reason).toBe("Dictation is off. Turn it on in Settings > Voice."); expect(voice?.voice.enabled).toBe(false); - expect(voice?.voice.reason).toBe("Realtime voice is disabled in daemon config."); + expect(voice?.voice.reason).toBe("Voice mode is off. Turn it on in Settings > Voice."); } finally { await client.close().catch(() => undefined); await isolatedCtx.cleanup(); diff --git a/packages/server/src/server/daemon-config-store.test.ts b/packages/server/src/server/daemon-config-store.test.ts index 095ae3b14b..f4c1a02de9 100644 --- a/packages/server/src/server/daemon-config-store.test.ts +++ b/packages/server/src/server/daemon-config-store.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, test } from "vitest"; @@ -6,6 +6,7 @@ import { afterEach, describe, expect, test } from "vitest"; import { DaemonConfigStore, applyMutableProviderConfigToOverrides } from "./daemon-config-store.js"; import { loadConfig } from "./config.js"; import { loadPersistedConfig } from "./persisted-config.js"; +import { readSpeechSettings } from "./speech/speech-settings.js"; import type { PersistedConfig } from "./persisted-config.js"; import type { MutableDaemonConfig } from "@getpaseo/protocol/messages"; @@ -1284,3 +1285,239 @@ describe("DaemonConfigStore reload", () => { }); }); }); + +describe("DaemonConfigStore speech settings", () => { + const tempDirs: string[] = []; + + afterEach(() => { + for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true }); + }); + + function createSpeechStore(initialPersisted?: unknown, env: NodeJS.ProcessEnv = {}) { + const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-speech-")); + tempDirs.push(paseoHome); + if (initialPersisted) { + writeFileSync( + path.join(paseoHome, "config.json"), + `${JSON.stringify(initialPersisted, null, 2)}\n`, + ); + } + const persisted = loadPersistedConfig(paseoHome); + const applied: PersistedConfig[] = []; + const speech = { + read: (next: PersistedConfig) => readSpeechSettings(next, env), + apply: (next: PersistedConfig) => applied.push(next), + }; + const store = new DaemonConfigStore(paseoHome, reloadableConfig(persisted), undefined, { + speech, + reloadSource: { + resolve: (next) => ({ mutable: reloadableConfig(next), overrideControlledPaths: [] }), + }, + }); + return { paseoHome, store, applied, persisted }; + } + + test("reads, patches, and clears local STT language without losing it on other saves", () => { + const { paseoHome, store } = createSpeechStore({ + features: { dictation: { stt: { language: "fr" } } }, + }); + expect(store.get().features?.dictation.stt).toEqual({ provider: "local", language: "fr" }); + store.patch({ features: { voiceMode: { enabled: true } } }); + expect(loadPersistedConfig(paseoHome).features?.dictation?.stt).toEqual({ + provider: "local", + language: "fr", + }); + const patched = store.patch({ + features: { dictation: { stt: { provider: "local", language: "de" } } }, + }); + expect(patched.features?.dictation.stt).toEqual({ provider: "local", language: "de" }); + store.patch({ features: { dictation: { stt: { language: null } } } }); + expect(loadPersistedConfig(paseoHome).features?.dictation?.stt).toEqual({ provider: "local" }); + }); + + test("reports language env overrides for both local STT sites", () => { + const { store } = createSpeechStore( + {}, + { PASEO_DICTATION_LANGUAGE: "fr", PASEO_VOICE_LANGUAGE: "de" }, + ); + expect(store.get().envOverrides).toMatchObject({ + "features.dictation.stt.language": "PASEO_DICTATION_LANGUAGE", + "features.voiceMode.stt.language": "PASEO_VOICE_LANGUAGE", + }); + }); + + test("keeps voice mode off until it is turned on", () => { + const { paseoHome, store } = createSpeechStore(); + + expect(store.get().features?.dictation.enabled).toBe(true); + expect(store.get().features?.voiceMode.enabled).toBe(false); + + const result = store.patch({ features: { voiceMode: { enabled: true } } }); + + expect(result.features?.voiceMode.enabled).toBe(true); + expect(loadPersistedConfig(paseoHome).features?.voiceMode?.enabled).toBe(true); + }); + + test("stores the OpenAI key without ever returning it", () => { + const { paseoHome, store, applied } = createSpeechStore(); + + const result = store.patch({ + features: { dictation: { stt: { provider: "openai", apiKey: " sk-test " } } }, + }); + + expect(result.features?.dictation.stt).toEqual({ provider: "openai", hasApiKey: true }); + expect(JSON.stringify(result)).not.toContain("sk-test"); + expect(loadPersistedConfig(paseoHome).features?.dictation?.stt).toMatchObject({ + apiKey: "sk-test", + }); + expect(applied).toHaveLength(1); + }); + + test("applies a replaced key even though the visible settings do not change", () => { + const { store, applied } = createSpeechStore({ + version: 1, + features: { dictation: { stt: { provider: "openai", apiKey: "sk-old" } } }, + }); + + store.patch({ features: { dictation: { stt: { apiKey: "sk-new" } } } }); + + expect(applied.map((next) => next.features?.dictation?.stt)).toEqual([ + { provider: "openai", apiKey: "sk-new" }, + ]); + }); + + test("switching provider clears the previous provider's model", () => { + const { paseoHome, store } = createSpeechStore({ + version: 1, + features: { dictation: { stt: { provider: "local", model: "parakeet-tdt-0.6b-v3-int8" } } }, + }); + + const result = store.patch({ features: { dictation: { stt: { provider: "openai" } } } }); + + expect(result.features?.dictation.stt).toEqual({ provider: "openai", hasApiKey: false }); + expect(loadPersistedConfig(paseoHome).features?.dictation?.stt).toEqual({ + provider: "openai", + }); + }); + + test("rejects a local model id outside the catalog without persisting it", () => { + const { paseoHome, store, applied } = createSpeechStore(); + + expect(() => + store.patch({ features: { voiceMode: { tts: { provider: "local", model: "nope" } } } }), + ).toThrow("Unknown local voice model: nope"); + expect(loadPersistedConfig(paseoHome).features?.voiceMode).toBeUndefined(); + expect(applied).toEqual([]); + }); + + test("reload applies hand-edited speech settings without a restart", () => { + const { paseoHome, store, applied, persisted } = createSpeechStore(); + writeFileSync( + path.join(paseoHome, "config.json"), + `${JSON.stringify({ ...persisted, features: { voiceMode: { enabled: false } } }, null, 2)}\n`, + ); + + const result = store.reload(); + + expect(result.appliedPaths).toContain("features.voiceMode"); + expect(result.restartRequiredPaths).toEqual([]); + expect(store.get().features?.voiceMode.enabled).toBe(false); + expect(applied).toHaveLength(1); + }); + test("merges same-provider fields and replaces the whole variant when provider changes", () => { + const { paseoHome, store } = createSpeechStore(); + store.patch({ + features: { + voiceMode: { + stt: { provider: "openai", apiKey: "secret", baseUrl: "https://site/v1", model: "first" }, + }, + }, + }); + store.patch({ features: { voiceMode: { stt: { model: "second", baseUrl: null } } } }); + expect(loadPersistedConfig(paseoHome).features?.voiceMode?.stt).toEqual({ + provider: "openai", + apiKey: "secret", + model: "second", + }); + store.patch({ features: { voiceMode: { stt: { provider: "openai", model: "third" } } } }); + expect(store.get().features?.voiceMode.stt).toEqual({ + provider: "openai", + model: "third", + hasApiKey: true, + }); + store.patch({ features: { voiceMode: { stt: { apiKey: "new-secret" } } } }); + store.patch({ features: { voiceMode: { stt: { provider: "local" } } } }); + store.patch({ features: { voiceMode: { stt: { provider: "openai" } } } }); + expect(store.get().features?.voiceMode.stt).toEqual({ provider: "openai", hasApiKey: false }); + }); + + test("keeps a saved OpenAI key when a full variant patch omits the write-only key", () => { + const { paseoHome, store } = createSpeechStore({ + features: { voiceMode: { stt: { provider: "openai", apiKey: "saved-secret" } } }, + }); + const result = store.patch({ + features: { voiceMode: { stt: { provider: "openai", model: "whisper-1" } } }, + }); + expect(result.features?.voiceMode.stt).toEqual({ + provider: "openai", + model: "whisper-1", + hasApiKey: true, + }); + expect(loadPersistedConfig(paseoHome).features?.voiceMode?.stt).toEqual({ + provider: "openai", + apiKey: "saved-secret", + model: "whisper-1", + }); + expect(JSON.stringify(result)).not.toContain("saved-secret"); + }); + + test("loads a legacy host, redacts its key and writes the normalized config after any save", () => { + const { paseoHome, store } = createSpeechStore({ + providers: { openai: { apiKey: "legacy-secret", stt: { baseUrl: "https://old/v1" } } }, + features: { voiceMode: { enabled: true, stt: { provider: "openai" } } }, + }); + expect(store.get().features?.voiceMode.stt).toEqual({ + provider: "openai", + baseUrl: "https://old/v1", + hasApiKey: true, + }); + expect(JSON.stringify(store.get())).not.toContain("legacy-secret"); + store.patch({ autoArchiveAfterMerge: true }); + const disk = JSON.parse(readFileSync(path.join(paseoHome, "config.json"), "utf8")); + expect(disk.providers).not.toHaveProperty("openai"); + expect(disk.features.voiceMode.stt.apiKey).toBe("legacy-secret"); + }); + + test("reports env overrides and removes fallback provenance after saving a key", () => { + const { store } = createSpeechStore( + { features: { voiceMode: { stt: { provider: "openai", model: "configured" } } } }, + { + PASEO_VOICE_STT_PROVIDER: "local", + OPENAI_API_KEY: "env-secret", + }, + ); + expect(store.get().features?.voiceMode.stt.provider).toBe("local"); + expect(store.get().envOverrides).toEqual({ + "features.voiceMode.stt.provider": "PASEO_VOICE_STT_PROVIDER", + }); + const other = createSpeechStore( + { features: { voiceMode: { stt: { provider: "openai" } } } }, + { + OPENAI_API_KEY: "env-secret", + OPENAI_STT_BASE_URL: "https://env/v1", + STT_MODEL: "override", + }, + ).store; + expect(other.get().envOverrides).toEqual({ + "features.voiceMode.stt.apiKey": "OPENAI_API_KEY", + "features.voiceMode.stt.baseUrl": "OPENAI_STT_BASE_URL", + "features.voiceMode.stt.model": "STT_MODEL", + }); + expect(other.get().features?.voiceMode.stt.hasApiKey).toBe(false); + other.patch({ + features: { voiceMode: { stt: { apiKey: "saved-secret", baseUrl: "https://saved/v1" } } }, + }); + expect(other.get().envOverrides).toEqual({ "features.voiceMode.stt.model": "STT_MODEL" }); + expect(JSON.stringify(other.get())).not.toContain("secret"); + }); +}); diff --git a/packages/server/src/server/daemon-config-store.ts b/packages/server/src/server/daemon-config-store.ts index 2241193e50..51957e15f6 100644 --- a/packages/server/src/server/daemon-config-store.ts +++ b/packages/server/src/server/daemon-config-store.ts @@ -9,6 +9,8 @@ import { MutableDaemonConfigPatchSchema, } from "@getpaseo/protocol/messages"; import type { AgentSkillSelection } from "@getpaseo/protocol/messages"; +import type { SpeechSettingsPatch } from "@getpaseo/protocol/speech/rpc-schemas"; +import { mergeSpeechSettingsPatch, type SpeechSettingsOwner } from "./speech/speech-settings.js"; export type { MutableDaemonConfig, MutableDaemonConfigPatch } from "@getpaseo/protocol/messages"; @@ -32,8 +34,11 @@ interface SupportedMutableConfigPatch { skills?: MutableDaemonConfig["skills"]; pluginsEnabled?: boolean; plugins?: MutableDaemonConfig["plugins"]; + features?: SpeechSettingsPatch; } +const SPEECH_PERSISTED_PATHS = ["features.dictation", "features.voiceMode"]; + interface LoggerLike { child(bindings: Record): LoggerLike; info(...args: unknown[]): void; @@ -296,6 +301,7 @@ function pickSupportedPatchFields(patch: MutableDaemonConfigPatch): SupportedMut ...(patch.agentProfiles !== undefined ? { agentProfiles: patch.agentProfiles } : {}), ...(patch.pluginsEnabled !== undefined ? { pluginsEnabled: patch.pluginsEnabled } : {}), ...(patch.plugins !== undefined ? { plugins: patch.plugins } : {}), + ...(patch.features !== undefined ? { features: patch.features } : {}), }; } @@ -338,6 +344,7 @@ export class DaemonConfigStore { private readonly relayEnabledMutable: boolean; private readonly reloadSource: DaemonConfigReloadSource | undefined; private readonly startupPersisted: PersistedConfig; + private readonly speechOwner: SpeechSettingsOwner | undefined; private lastKnownPersisted: PersistedConfig; constructor( @@ -348,18 +355,21 @@ export class DaemonConfigStore { relayEnabledMutable?: boolean; reloadSource?: DaemonConfigReloadSource; startupPersisted?: PersistedConfig; + speech?: SpeechSettingsOwner; } = {}, ) { this.paseoHome = paseoHome; this.logger = getLogger(logger); + this.relayEnabledMutable = options.relayEnabledMutable ?? true; + this.reloadSource = options.reloadSource; + this.speechOwner = options.speech; + this.startupPersisted = options.startupPersisted ?? loadPersistedConfig(paseoHome, this.logger); + this.lastKnownPersisted = this.startupPersisted; this.current = MutableDaemonConfigSchema.parse({ ...initial, relay: initial.relay ?? { enabled: true }, + ...this.readSpeech(this.startupPersisted), }); - this.relayEnabledMutable = options.relayEnabledMutable ?? true; - this.reloadSource = options.reloadSource; - this.startupPersisted = options.startupPersisted ?? loadPersistedConfig(paseoHome, this.logger); - this.lastKnownPersisted = this.startupPersisted; } public get(): MutableDaemonConfig { @@ -381,9 +391,11 @@ export class DaemonConfigStore { "Relay is controlled by a daemon launch override. Remove PASEO_RELAY_ENABLED or the relay CLI flag before changing it here.", ); } - const { removeProviders = [], ...configPatch } = parsedPatch; + const { removeProviders = [], ...patch } = parsedPatch; + const { features: speechPatch, ...configPatch } = patch; const removedProviders = Array.from(new Set(removeProviders)); const merged = deepMerge(this.current, configPatch); + if (speechPatch !== undefined) Object.assign(merged, this.readPatchedSpeech(speechPatch)); if (parsedPatch.skills?.selection !== undefined) { merged.skills = { selection: parsedPatch.skills.selection }; } @@ -397,30 +409,51 @@ export class DaemonConfigStore { const configChanged = !isEqualValue(this.current, next); - if (!configChanged && removedProviders.length === 0) { + // A replaced API key leaves the speech view unchanged, so a speech patch always persists. + if (!configChanged && removedProviders.length === 0 && speechPatch === undefined) { return this.current; } const { previous: persistedBeforePatch, knownNext } = this.persistConfig( - configPatch, + patch, removedProviders, ); - if (!configChanged) { - this.lastKnownPersisted = knownNext; - return this.current; - } - - try { - this.applyReplacement(next, { removedProviders }); - this.lastKnownPersisted = knownNext; - } catch (error) { - savePersistedConfig(this.paseoHome, persistedBeforePatch, this.logger); - throw error; + const speechChanged = this.speechPersistedChanged(this.lastKnownPersisted, knownNext); + if (configChanged) { + try { + this.applyReplacement(next, { removedProviders }); + } catch (error) { + savePersistedConfig(this.paseoHome, persistedBeforePatch, this.logger); + throw error; + } } + this.lastKnownPersisted = knownNext; + if (speechChanged) this.speechOwner?.apply(knownNext); return this.current; } + private readSpeech( + persisted: PersistedConfig, + ): Pick { + return this.speechOwner ? this.speechOwner.read(persisted) : {}; + } + + private readPatchedSpeech( + speechPatch: SpeechSettingsPatch, + ): Pick { + if (!this.speechOwner) { + throw new Error("Speech settings are unavailable for this daemon instance"); + } + return this.speechOwner.read(mergeSpeechSettingsPatch(this.lastKnownPersisted, speechPatch)); + } + + private speechPersistedChanged(previous: PersistedConfig, next: PersistedConfig): boolean { + return diffPaths(previous, next).some((path) => + SPEECH_PERSISTED_PATHS.some((owner) => pathBelongsTo(path, owner)), + ); + } + public reload(): DaemonConfigReloadResult { if (!this.reloadSource) { throw new Error("Daemon config reload is unavailable for this daemon instance"); @@ -433,6 +466,7 @@ export class DaemonConfigStore { const desired = MutableDaemonConfigSchema.parse({ ...resolved.mutable, plugins: this.current.plugins, + ...this.readSpeech(persisted), }); const changedSinceLastApply = diffPaths(this.lastKnownPersisted, persisted); const overrideControlledPaths = compactOwnedPaths( @@ -441,7 +475,7 @@ export class DaemonConfigStore { ), resolved.overrideControlledPaths, ); - const appliedPaths = RELOADABLE_PATHS.filter((persistedPath) => { + const appliedPaths: string[] = RELOADABLE_PATHS.filter((persistedPath) => { if (resolved.overrideControlledPaths.some((owner) => pathBelongsTo(persistedPath, owner))) { return false; } @@ -458,6 +492,8 @@ export class DaemonConfigStore { diffPaths(this.startupPersisted, persisted).filter((path) => { if (path === "$schema" || path === "version") return false; if (RELOADABLE_PATHS.some((owner) => pathBelongsTo(path, owner))) return false; + if (this.speechOwner && SPEECH_PERSISTED_PATHS.some((owner) => pathBelongsTo(path, owner))) + return false; return !resolved.overrideControlledPaths.some((owner) => pathBelongsTo(path, owner)); }), [], @@ -466,8 +502,17 @@ export class DaemonConfigStore { const removedProviders = Object.keys(this.current.providers).filter( (provider) => !(provider in desired.providers), ); + const speechChanged = this.speechPersistedChanged(this.lastKnownPersisted, persisted); this.applyReplacement(desired, { removedProviders }); this.lastKnownPersisted = persisted; + if (speechChanged && this.speechOwner) { + this.speechOwner.apply(persisted); + appliedPaths.push( + ...SPEECH_PERSISTED_PATHS.filter((owner) => + changedSinceLastApply.some((path) => pathBelongsTo(path, owner)), + ), + ); + } return { appliedPaths: [...appliedPaths].sort(), @@ -604,8 +649,11 @@ function mergeMutablePatchIntoPersistedConfig(params: { const { persisted, patch, removeProviders, persistRelayEnabled } = params; const daemon = mergeMutableDaemonPatch(persisted.daemon, patch, persistRelayEnabled); const agents = mergeMutableAgentPatch(persisted.agents, patch, removeProviders); + const withSpeech = patch.features + ? mergeSpeechSettingsPatch(persisted, patch.features) + : persisted; return { - ...persisted, + ...withSpeech, ...(patch.pluginsEnabled !== undefined ? { pluginsEnabled: patch.pluginsEnabled } : {}), ...(patch.plugins !== undefined ? { plugins: patch.plugins } : {}), ...(daemon ? { daemon } : { daemon: undefined }), diff --git a/packages/server/src/server/dictation/dictation-stream-manager.test.ts b/packages/server/src/server/dictation/dictation-stream-manager.test.ts index 975d6677b8..1d258553ca 100644 --- a/packages/server/src/server/dictation/dictation-stream-manager.test.ts +++ b/packages/server/src/server/dictation/dictation-stream-manager.test.ts @@ -1,9 +1,12 @@ +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import { EventEmitter } from "node:events"; import pino from "pino"; import { DictationStreamManager } from "./dictation-stream-manager.js"; -import { PersistedConfigSchema } from "../persisted-config.js"; +import { loadPersistedConfig } from "../persisted-config.js"; import { resolveSpeechConfig } from "../speech/speech-config-resolver.js"; import type { SpeechToTextProvider, @@ -131,12 +134,18 @@ describe("DictationStreamManager (provider-agnostic provider)", () => { env?: NodeJS.ProcessEnv; persisted?: unknown; }): string { - const result = resolveSpeechConfig({ - paseoHome: "/tmp/paseo-home", - env: params.env ?? ({} as NodeJS.ProcessEnv), - persisted: PersistedConfigSchema.parse(params.persisted ?? {}), - }); - return result.speech.sttLanguages.dictation; + const home = mkdtempSync(path.join(tmpdir(), "stt-language-")); + try { + writeFileSync(path.join(home, "config.json"), JSON.stringify(params.persisted ?? {})); + const result = resolveSpeechConfig({ + paseoHome: home, + env: params.env ?? {}, + persisted: loadPersistedConfig(home), + }); + return result.speech.sttLanguages.dictation; + } finally { + rmSync(home, { recursive: true, force: true }); + } } async function startWithResolvedDictationLanguage(params: { diff --git a/packages/server/src/server/persisted-config-edit.test.ts b/packages/server/src/server/persisted-config-edit.test.ts index 4b4bf4c0db..c838601255 100644 --- a/packages/server/src/server/persisted-config-edit.test.ts +++ b/packages/server/src/server/persisted-config-edit.test.ts @@ -43,3 +43,19 @@ test("configuration edits validate before writing and preserve unrelated setting await rm(root, { recursive: true, force: true }); } }); + +test("configuration paths edit fields inside speech provider variants", async () => { + const home = await mkdtemp(path.join(tmpdir(), "paseo-speech-edit-")); + try { + editPersistedConfig(home, "features.voiceMode.stt", { value: { provider: "openai" } }); + editPersistedConfig(home, "features.voiceMode.stt.apiKey", { value: "site-secret" }); + const config = readPersistedConfig(home); + expect(getPersistedConfigValue(config, "features.voiceMode.stt.apiKey")).toBe("site-secret"); + editPersistedConfig(home, "features.voiceMode.stt.apiKey", { unset: true }); + expect( + getPersistedConfigValue(readPersistedConfig(home), "features.voiceMode.stt.apiKey"), + ).toBeUndefined(); + } finally { + await rm(home, { recursive: true, force: true }); + } +}); diff --git a/packages/server/src/server/persisted-config.test.ts b/packages/server/src/server/persisted-config.test.ts index 6b23a6f365..019bcbb0f8 100644 --- a/packages/server/src/server/persisted-config.test.ts +++ b/packages/server/src/server/persisted-config.test.ts @@ -1,3 +1,4 @@ +import { resolveSpeechConfig } from "./speech/speech-config-resolver.js"; import { chmodSync, lstatSync, @@ -149,26 +150,33 @@ describe("PersistedConfigSchema worktrees config", () => { }); describe("PersistedConfigSchema provider credentials", () => { - test("accepts separate OpenAI STT and TTS credentials", () => { + test("accepts separate inline STT and TTS credentials", () => { const parsed = PersistedConfigSchema.parse({ - providers: { - openai: { + features: { + voiceMode: { stt: { + provider: "openai", apiKey: " stt-secret ", baseUrl: " https://stt.example.com/v1 ", }, tts: { + provider: "openai", apiKey: " tts-secret ", baseUrl: " https://tts.example.com/v1 ", }, }, }, }); - - expect(parsed.providers?.openai?.stt?.apiKey).toBe("stt-secret"); - expect(parsed.providers?.openai?.stt?.baseUrl).toBe("https://stt.example.com/v1"); - expect(parsed.providers?.openai?.tts?.apiKey).toBe("tts-secret"); - expect(parsed.providers?.openai?.tts?.baseUrl).toBe("https://tts.example.com/v1"); + expect(parsed.features?.voiceMode?.stt).toEqual({ + provider: "openai", + apiKey: "stt-secret", + baseUrl: "https://stt.example.com/v1", + }); + expect(parsed.features?.voiceMode?.tts).toEqual({ + provider: "openai", + apiKey: "tts-secret", + baseUrl: "https://tts.example.com/v1", + }); }); }); @@ -660,19 +668,21 @@ describe("PersistedConfigSchema voice mode config", () => { features: { dictation: { stt: { + provider: "openai", language: " fr ", }, }, voiceMode: { stt: { + provider: "openai", language: " de ", }, }, }, }); - expect(parsed.features?.dictation?.stt?.language).toBe("fr"); - expect(parsed.features?.voiceMode?.stt?.language).toBe("de"); + expect(parsed.features?.dictation?.stt).toMatchObject({ language: "fr" }); + expect(parsed.features?.voiceMode?.stt).toMatchObject({ language: "de" }); }); test("accepts a voice and model from a self-hosted OpenAI-compatible server", () => { @@ -685,7 +695,7 @@ describe("PersistedConfigSchema voice mode config", () => { }); expect(parsed.features?.voiceMode?.tts?.model).toBe("kokoro-v1:Q8_0"); - expect(parsed.features?.voiceMode?.tts?.voice).toBe("aiden"); + expect(parsed.features?.voiceMode?.tts).toMatchObject({ voice: "aiden" }); }); }); @@ -754,10 +764,7 @@ describe("loadPersistedConfig", () => { const config = loadPersistedConfig(home); - expect(config.providers?.openai?.apiKey).toBe("global-key"); - expect((config.providers?.openai as Record)?.voice).toBeUndefined(); - expect(config.providers?.openai?.stt).toBeUndefined(); - expect(config.providers?.openai?.tts).toBeUndefined(); + expect(config.providers).not.toHaveProperty("openai"); } finally { rmSync(home, { recursive: true, force: true }); } @@ -852,11 +859,7 @@ describe.skipIf(process.platform === "win32")("persisted config file permissions const home = path.join(parent, "home"); try { savePersistedConfig(home, { - providers: { - openai: { - apiKey: "secret", - }, - }, + features: { dictation: { stt: { provider: "openai", apiKey: "secret" } } }, }); expect(modeOf(home)).toBe(PRIVATE_DIRECTORY_MODE); @@ -898,3 +901,173 @@ describe.skipIf(process.platform === "win32")( }); }, ); + +describe("speech provider config normalization", () => { + test.each([undefined, "local"])( + "preserves local STT language on load and save (provider: %s)", + (provider) => { + const home = createTempHome(); + try { + writeFileSync( + path.join(home, "config.json"), + JSON.stringify({ + features: { dictation: { stt: { provider, language: "fr" } } }, + }), + ); + const loaded = loadPersistedConfig(home); + expect(loaded.features?.dictation?.stt).toEqual({ provider: "local", language: "fr" }); + expect( + resolveSpeechConfig({ paseoHome: home, env: {}, persisted: loaded }).speech.sttLanguages + .dictation, + ).toBe("fr"); + savePersistedConfig(home, loaded); + expect(loadPersistedConfig(home).features?.dictation?.stt).toEqual({ + provider: "local", + language: "fr", + }); + expect( + JSON.parse(readFileSync(path.join(home, "config.json"), "utf8")).features.dictation.stt, + ).toEqual({ provider: "local", language: "fr" }); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }, + ); + + test("loads and resolves legacy fields ignored by each site's provider", () => { + const home = mkdtempSync(path.join(tmpdir(), "speech-unused-options-")); + try { + const legacy = { + features: { + dictation: { stt: { provider: "local", language: "fr", confidenceThreshold: -1 } }, + voiceMode: { + enabled: true, + tts: { provider: "local", voice: "alloy", speakerId: 3, speed: 1.2 }, + }, + }, + }; + writeFileSync(path.join(home, "config.json"), JSON.stringify(legacy)); + const local = loadPersistedConfig(home); + expect(local.features?.dictation?.stt).toEqual({ provider: "local", language: "fr" }); + expect(local.features?.voiceMode?.tts).toEqual({ + provider: "local", + speakerId: 3, + speed: 1.2, + }); + expect( + resolveSpeechConfig({ paseoHome: home, env: {}, persisted: local }).speech.local?.models, + ).toMatchObject({ voiceTtsSpeakerId: 3, voiceTtsSpeed: 1.2 }); + writeFileSync( + path.join(home, "config.json"), + JSON.stringify({ + features: { + voiceMode: { + enabled: true, + tts: { provider: "openai", apiKey: "key", voice: "alloy", speakerId: 3, speed: 1.2 }, + }, + }, + }), + ); + const openai = loadPersistedConfig(home); + expect(openai.features?.voiceMode?.tts).toEqual({ + provider: "openai", + apiKey: "key", + voice: "alloy", + }); + expect( + resolveSpeechConfig({ paseoHome: home, env: {}, persisted: openai }).openai?.tts, + ).toMatchObject({ apiKey: "key", voice: "alloy" }); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test("warns with the path, never the discarded legacy key", () => { + const home = mkdtempSync(path.join(tmpdir(), "speech-warning-")); + try { + writeFileSync( + path.join(home, "config.json"), + JSON.stringify({ + providers: { openai: { apiKey: "discarded-secret" } }, + features: { voiceMode: { enabled: true, tts: { provider: "local" } } }, + }), + ); + const warnings: unknown[][] = []; + const logger = { + child: () => logger, + info: () => {}, + warn: (...args: unknown[]) => { + warnings.push(args); + }, + }; + const config = loadPersistedConfig(home, logger); + expect(config.providers).not.toHaveProperty("openai"); + expect(warnings).toHaveLength(1); + expect(JSON.stringify(warnings)).toContain("providers.openai.apiKey"); + expect(JSON.stringify(warnings)).toContain("no use site in config.json selects OpenAI"); + const resolved = resolveSpeechConfig({ + paseoHome: home, + persisted: config, + env: { PASEO_VOICE_TTS_PROVIDER: "openai" }, + }); + expect(resolved.speech.providers.voiceTts.provider).toBe("openai"); + expect(resolved.openai).toBeUndefined(); + expect(JSON.stringify(warnings)).not.toContain("discarded-secret"); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); + + test("folds legacy endpoint credentials into OpenAI sites on load and saves inline", () => { + const home = mkdtempSync(path.join(tmpdir(), "speech-normalize-")); + try { + writeFileSync( + path.join(home, "config.json"), + JSON.stringify({ + providers: { + openai: { + apiKey: "global-key", + baseUrl: "https://global/v1", + stt: { apiKey: "stt-key" }, + }, + }, + features: { + dictation: { stt: { model: "whisper-base", language: "en" } }, + voiceMode: { + enabled: true, + stt: { provider: "OPENAI", baseUrl: "https://site/v1" }, + tts: { provider: "openai" }, + }, + }, + }), + ); + const config = loadPersistedConfig(home); + expect(config.features?.dictation?.stt).toEqual({ + provider: "local", + model: "whisper-base", + language: "en", + }); + expect(config.features?.voiceMode?.stt).toEqual({ + provider: "openai", + apiKey: "stt-key", + baseUrl: "https://site/v1", + }); + expect(config.features?.voiceMode?.tts).toEqual({ + provider: "openai", + apiKey: "global-key", + baseUrl: "https://global/v1", + }); + expect(config.providers).not.toHaveProperty("openai"); + const resolved = resolveSpeechConfig({ paseoHome: home, env: {}, persisted: config }); + expect(resolved.openai?.stt?.apiKey).toBe("stt-key"); + expect(resolved.speech.providers.voiceStt).toMatchObject({ + provider: "openai", + enabled: true, + }); + savePersistedConfig(home, config); + expect(JSON.parse(readFileSync(path.join(home, "config.json"), "utf8"))).toEqual(config); + } finally { + rmSync(home, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 8b99a1ac45..c6c3f9c7e1 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -49,22 +49,6 @@ const LogConfigSchema = z }) .strict(); -const OpenAiSpeechEndpointSchema = z - .object({ - apiKey: z.string().trim().min(1).optional(), - baseUrl: z.string().trim().min(1).optional(), - }) - .strict(); - -const OpenAiProviderSchema = z - .object({ - apiKey: z.string().min(1).optional(), - baseUrl: z.string().trim().min(1).optional(), - stt: OpenAiSpeechEndpointSchema.optional(), - tts: OpenAiSpeechEndpointSchema.optional(), - }) - .strict(); - const LocalSpeechProviderSchema = z .object({ modelsDir: z.string().min(1).optional(), @@ -73,7 +57,6 @@ const LocalSpeechProviderSchema = z const ProvidersSchema = z .object({ - openai: OpenAiProviderSchema.optional(), local: LocalSpeechProviderSchema.optional(), }) .strict(); @@ -101,18 +84,54 @@ const SpeechProviderIdSchema = z .toLowerCase() .pipe(z.enum(["openai", "local"])); +const SpeechModelSchema = z.string().min(1).optional(); +const OpenAiConnectionFields = { + apiKey: z.string().trim().min(1).optional(), + baseUrl: z.string().trim().min(1).optional(), +}; + +export const SpeechSttConfigSchema = z.discriminatedUnion("provider", [ + z + .object({ + provider: z.literal("local"), + model: SpeechModelSchema, + language: z.string().trim().min(1).optional(), + }) + .strict(), + z + .object({ + provider: z.literal("openai"), + ...OpenAiConnectionFields, + model: SpeechModelSchema, + language: z.string().trim().min(1).optional(), + confidenceThreshold: z.number().optional(), + }) + .strict(), +]); + +export const SpeechTtsConfigSchema = z.discriminatedUnion("provider", [ + z + .object({ + provider: z.literal("local"), + model: SpeechModelSchema, + speakerId: z.number().int().optional(), + speed: z.number().optional(), + }) + .strict(), + z + .object({ + provider: z.literal("openai"), + ...OpenAiConnectionFields, + model: SpeechModelSchema, + voice: z.string().trim().min(1).optional(), + }) + .strict(), +]); + const FeatureDictationSchema = z .object({ enabled: z.boolean().optional(), - stt: z - .object({ - provider: SpeechProviderIdSchema.optional(), - model: z.string().min(1).optional(), - language: z.string().trim().min(1).optional(), - confidenceThreshold: z.number().optional(), - }) - .strict() - .optional(), + stt: SpeechSttConfigSchema.optional(), }) .strict(); @@ -126,31 +145,14 @@ const FeatureVoiceModeSchema = z }) .strict() .optional(), - stt: z - .object({ - provider: SpeechProviderIdSchema.optional(), - model: z.string().min(1).optional(), - language: z.string().trim().min(1).optional(), - }) - .strict() - .optional(), + stt: SpeechSttConfigSchema.optional(), turnDetection: z .object({ provider: SpeechProviderIdSchema.optional(), }) .strict() .optional(), - tts: z - .object({ - provider: SpeechProviderIdSchema.optional(), - model: z.string().min(1).optional(), - // Free-form: self-hosted OpenAI-compatible servers expose their own voice names. - voice: z.string().trim().min(1).optional(), - speakerId: z.number().int().optional(), - speed: z.number().optional(), - }) - .strict() - .optional(), + tts: SpeechTtsConfigSchema.optional(), }) .strict(); @@ -387,6 +389,7 @@ const DEFAULT_PERSISTED_CONFIG = PersistedConfigSchema.parse({ interface LoggerLike { child(bindings: Record): LoggerLike; info(...args: unknown[]): void; + warn?(...args: unknown[]): void; } function getConfigPath(paseoHome: string): string { @@ -397,45 +400,96 @@ function getLogger(logger: LoggerLike | undefined): LoggerLike | undefined { return logger?.child({ module: "config" }); } -// Removed config fields are stripped before parsing so the strict schema does not -// reject a config written by an older release. The stripped values are discarded, -// not migrated — there is no back-compat for the removed `providers.openai.voice` -// block (use `providers.openai.stt` / `providers.openai.tts`). -function stripRemovedConfigFields(parsed: unknown): unknown { - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - return parsed; +function isConfigRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function configRecord(value: unknown): Record | undefined { + return isConfigRecord(value) ? value : undefined; +} + +// COMPAT(speechProviderConfig): added in v0.12.0, remove after 2027-04-26. +// Fold shared credentials once at the file boundary; consumers see only the strict +// per-site shape. Never include credential values in diagnostics. +function normalizeSpeechProviderConfig(parsed: unknown, logger?: LoggerLike): unknown { + if (!isConfigRecord(parsed)) return parsed; + const root = structuredClone(parsed); + const providers = configRecord(root.providers); + const legacy = configRecord(providers?.openai); + const features = configRecord(root.features); + const foldedKeys = new Set(); + for (const [feature, kind] of [ + ["dictation", "stt"], + ["voiceMode", "stt"], + ["voiceMode", "tts"], + ] as const) { + const site = configRecord(configRecord(features?.[feature])?.[kind]); + if (site) normalizeSpeechSite(site, kind, legacy, foldedKeys); } - - const root = { ...(parsed as Record) }; - const providers = root.providers; - if (!providers || typeof providers !== "object" || Array.isArray(providers)) { - return root; + if (legacy) warnDiscardedSpeechKeys(legacy, foldedKeys, logger); + if (providers) { + delete providers.openai; + const local = configRecord(providers.local); + if (local) delete local.autoDownload; } - - const providersRecord = { ...(providers as Record) }; - - const local = providersRecord.local; - if (local && typeof local === "object" && !Array.isArray(local)) { - const localRecord = { ...(local as Record) }; - delete localRecord.autoDownload; - providersRecord.local = localRecord; - } - - const openai = providersRecord.openai; - if (openai && typeof openai === "object" && !Array.isArray(openai)) { - const openaiRecord = { ...(openai as Record) }; - // COMPAT(openaiVoiceConfig): added 2026-06-30, remove after 2026-12-30. - // Drop a `providers.openai.voice` block left by an older release so the strict - // schema doesn't reject it. The value is discarded, not migrated — there is no - // back-compat; configure `providers.openai.stt` / `providers.openai.tts` instead. - delete openaiRecord.voice; - providersRecord.openai = openaiRecord; - } - - root.providers = providersRecord; return root; } +function normalizeSpeechSite( + site: Record, + kind: "stt" | "tts", + legacy: Record | undefined, + foldedKeys: Set, +) { + site.provider = + typeof site.provider === "string" + ? site.provider.trim().toLowerCase() + : (site.provider ?? "local"); + if (kind === "stt" && site.provider === "local") { + delete site.confidenceThreshold; + } + if (kind === "tts") { + if (site.provider === "local") delete site.voice; + if (site.provider === "openai") { + delete site.speakerId; + delete site.speed; + } + } + if (site.provider !== "openai" || !legacy) return; + const endpoint = configRecord(legacy[kind]); + for (const field of ["apiKey", "baseUrl"] as const) { + if (site[field] !== undefined) continue; + if (endpoint?.[field] !== undefined) { + site[field] = endpoint[field]; + foldedKeys.add(`providers.openai.${kind}.${field}`); + } else if (legacy[field] !== undefined) { + site[field] = legacy[field]; + foldedKeys.add(`providers.openai.${field}`); + } + } +} + +function warnDiscardedSpeechKeys( + legacy: Record, + foldedKeys: Set, + logger?: LoggerLike, +) { + const keys = [ + ["providers.openai.apiKey", legacy.apiKey], + ...["stt", "tts"].map((kind) => [ + `providers.openai.${kind}.apiKey`, + configRecord(legacy[kind])?.apiKey, + ]), + ] as const; + for (const [keyPath, key] of keys) { + if (key && typeof keyPath === "string" && !foldedKeys.has(keyPath)) { + const message = `Dropped ${keyPath}: no use site in config.json selects OpenAI; re-enter it when selecting OpenAI.`; + if (logger?.warn) logger.warn(message); + else console.warn(message); + } + } +} + export function loadPersistedConfig(paseoHome: string, logger?: LoggerLike): PersistedConfig { const log = getLogger(logger); const configPath = getConfigPath(paseoHome); @@ -464,7 +518,7 @@ export function loadPersistedConfig(paseoHome: string, logger?: LoggerLike): Per }); } - const config = parseConfigFile(configPath, raw); + const config = parseConfigFile(configPath, raw, log); log?.info(`Loaded from ${configPath}`); return config; } @@ -486,7 +540,7 @@ export function readPersistedConfig( return parseConfigFile(configPath, raw); } -function parseConfigFile(configPath: string, raw: string): PersistedConfig { +function parseConfigFile(configPath: string, raw: string, logger?: LoggerLike): PersistedConfig { let parsed: unknown; try { parsed = parseConfigText(raw); @@ -497,7 +551,7 @@ function parseConfigFile(configPath: string, raw: string): PersistedConfig { }); } - const result = PersistedConfigSchema.safeParse(stripRemovedConfigFields(parsed)); + const result = PersistedConfigSchema.safeParse(normalizeSpeechProviderConfig(parsed, logger)); if (!result.success) { const issues = result.error.issues .map((i) => ` - ${i.path.join(".")}: ${i.message}`) @@ -525,6 +579,12 @@ function configPathParts(field: string): string[] { schema instanceof z.ZodPipe ) schema = (schema instanceof z.ZodPipe ? schema.in : schema.unwrap()) as z.ZodType; + if (schema instanceof z.ZodUnion) { + const variant = schema.options.find( + (option) => option instanceof z.ZodObject && Object.hasOwn(option.shape, part), + ); + if (variant instanceof z.ZodObject) schema = variant; + } if (!(schema instanceof z.ZodObject) || !Object.hasOwn(schema.shape, part)) { throw new Error( `Unknown configuration path: ${field}. Replace its containing object with JSON for dynamic keys.`, diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 9a2b2c0820..f573b1dd95 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -182,6 +182,10 @@ import { import { ScheduleSession } from "./session/schedule/schedule-session.js"; import { ProviderCatalogSession } from "./session/provider/provider-catalog-session.js"; import { UsageSession } from "./session/usage/usage-session.js"; +import { + SpeechModelsSession, + type SpeechModelsSessionOptions, +} from "./session/speech/speech-models-session.js"; import { WorkspaceFilesSession } from "./session/files/workspace-files-session.js"; import { AgentConfigSession } from "./session/agent-config/agent-config-session.js"; import { ProjectConfigSession } from "./session/project-config/project-config-session.js"; @@ -528,6 +532,7 @@ export interface SessionOptions { stt: Resolvable; sttLanguage?: string; tts: Resolvable; + speechModels?: SpeechModelsSessionOptions["runtime"]; terminalManager: TerminalManager | null; providerSnapshotManager: ProviderSnapshotManager; hubExecutionAgents?: HubExecutionAgents; @@ -811,6 +816,7 @@ export class Session { private readonly scheduleSession: ScheduleSession; private readonly providerCatalogSession: ProviderCatalogSession; private readonly usageSession: UsageSession; + private readonly speechModelsSession: SpeechModelsSession; private readonly workspaceFilesSession: WorkspaceFilesSession; private readonly agentConfigSession: AgentConfigSession; private readonly projectConfigSession: ProjectConfigSession; @@ -861,6 +867,7 @@ export class Session { stt, sttLanguage, tts, + speechModels, terminalManager, providerSnapshotManager, serviceProxy, @@ -1037,6 +1044,10 @@ export class Session { runtime: pluginRuntime, logger: this.sessionLogger, }); + this.speechModelsSession = new SpeechModelsSession({ + emit: (msg) => this.emit(msg), + runtime: speechModels, + }); this.agentConfigSession = new AgentConfigSession({ host: { emit: (msg) => this.emit(msg), @@ -2421,8 +2432,7 @@ export class Session { this.dispatchOrchestrationSkillsMessage(msg) ?? this.dispatchPluginDirectoryMessage(msg) ?? this.dispatchPluginMessage(msg) ?? - this.dispatchTerminalMessage(msg) ?? - this.dispatchScheduleMessage(msg) ?? + this.dispatchHostServiceMessage(msg) ?? this.dispatchMiscMessage(msg); if (promise) await promise; } @@ -2438,6 +2448,14 @@ export class Session { ); } + private dispatchHostServiceMessage(msg: SessionInboundMessage): Promise | undefined { + return ( + this.dispatchTerminalMessage(msg) ?? + this.dispatchScheduleMessage(msg) ?? + this.speechModelsSession.dispatch(msg) + ); + } + private dispatchWorkspaceLifecycleMessage(msg: SessionInboundMessage): Promise | undefined { return ( this.dispatchWorkspaceStateMessage(msg) ?? diff --git a/packages/server/src/server/session/speech/speech-models-session.test.ts b/packages/server/src/server/session/speech/speech-models-session.test.ts new file mode 100644 index 0000000000..9624475ad4 --- /dev/null +++ b/packages/server/src/server/session/speech/speech-models-session.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "vitest"; + +import type { SessionOutboundMessage } from "../../messages.js"; +import { + listLocalSpeechModels, + type LocalSpeechModelId, +} from "../../speech/providers/local/models.js"; +import { + LocalSpeechModelInUseError, + type LocalSpeechModelStatus, +} from "../../speech/speech-runtime.js"; +import { SpeechModelsSession } from "./speech-models-session.js"; + +const [firstModel] = listLocalSpeechModels(); + +function createSession(models: LocalSpeechModelStatus[], inUse: LocalSpeechModelId[] = []) { + const emitted: SessionOutboundMessage[] = []; + const downloads: LocalSpeechModelId[] = []; + const session = new SpeechModelsSession({ + emit: (message) => emitted.push(message), + runtime: { + listLocalModels: async () => models, + downloadLocalModel: (modelId) => { + downloads.push(modelId); + }, + deleteLocalModel: async (modelId) => { + if (inUse.includes(modelId)) throw new LocalSpeechModelInUseError(modelId); + }, + }, + }); + return { session, emitted, downloads }; +} + +describe("SpeechModelsSession", () => { + test("lists catalog details and download progress before anything is installed", async () => { + const { session, emitted } = createSession([ + { + spec: firstModel, + state: { status: "downloading", receivedBytes: 10, totalBytes: 40 }, + inUse: true, + }, + ]); + + await session.dispatch({ type: "speech.local_models.list.request", requestId: "r1" }); + + expect(emitted).toEqual([ + { + type: "speech.local_models.list.response", + payload: { + requestId: "r1", + error: null, + models: [ + { + id: firstModel.id, + kind: "stt", + label: firstModel.label, + description: firstModel.description, + languages: [...firstModel.languages], + downloadSizeBytes: firstModel.downloadSizeBytes, + isDefault: true, + inUse: true, + status: "downloading", + receivedBytes: 10, + totalBytes: 40, + }, + ], + }, + }, + ]); + }); + + test("rejects a model id outside the catalog without starting a download", () => { + const { session, emitted, downloads } = createSession([]); + + void session.dispatch({ + type: "speech.local_models.download.request", + requestId: "r2", + modelId: "../../etc", + }); + + expect(downloads).toEqual([]); + expect(emitted).toEqual([ + { + type: "speech.local_models.download.response", + payload: { requestId: "r2", error: "Unknown local speech model: ../../etc" }, + }, + ]); + }); + + test("reports why an in-use model cannot be deleted", async () => { + const { session, emitted } = createSession([], [firstModel.id]); + + await session.dispatch({ + type: "speech.local_models.delete.request", + requestId: "r3", + modelId: firstModel.id, + }); + + expect(emitted).toEqual([ + { + type: "speech.local_models.delete.response", + payload: { requestId: "r3", error: new LocalSpeechModelInUseError(firstModel.id).message }, + }, + ]); + }); +}); diff --git a/packages/server/src/server/session/speech/speech-models-session.ts b/packages/server/src/server/session/speech/speech-models-session.ts new file mode 100644 index 0000000000..d987623a8d --- /dev/null +++ b/packages/server/src/server/session/speech/speech-models-session.ts @@ -0,0 +1,124 @@ +import type { LocalSpeechModel } from "@getpaseo/protocol/speech/rpc-schemas"; + +import type { SessionInboundMessage, SessionOutboundMessage } from "../../messages.js"; +import { LocalSpeechModelIdSchema } from "../../speech/providers/local/models.js"; +import type { LocalSpeechModelStatus, SpeechService } from "../../speech/speech-runtime.js"; + +type SpeechModelsRuntime = Pick< + SpeechService, + "listLocalModels" | "downloadLocalModel" | "deleteLocalModel" +>; + +export interface SpeechModelsSessionOptions { + emit(message: SessionOutboundMessage): void; + // Absent when the daemon runs without speech. + runtime?: SpeechModelsRuntime | null; +} + +export class SpeechModelsSession { + constructor(private readonly options: SpeechModelsSessionOptions) {} + + // Handles the speech.local_models.* requests; returns undefined for any other message. + dispatch(msg: SessionInboundMessage): Promise | undefined { + switch (msg.type) { + case "speech.local_models.list.request": + return this.handleList(msg); + case "speech.local_models.download.request": + this.handleDownload(msg); + return Promise.resolve(); + case "speech.local_models.delete.request": + return this.handleDelete(msg); + default: + return undefined; + } + } + + private async handleList( + msg: Extract, + ): Promise { + try { + const models = await this.requireRuntime().listLocalModels(); + this.options.emit({ + type: "speech.local_models.list.response", + payload: { requestId: msg.requestId, models: models.map(toWireModel), error: null }, + }); + } catch (error) { + this.options.emit({ + type: "speech.local_models.list.response", + payload: { requestId: msg.requestId, models: [], error: describeError(error) }, + }); + } + } + + private handleDownload( + msg: Extract, + ): void { + try { + this.requireRuntime().downloadLocalModel(parseModelId(msg.modelId)); + this.options.emit({ + type: "speech.local_models.download.response", + payload: { requestId: msg.requestId, error: null }, + }); + } catch (error) { + this.options.emit({ + type: "speech.local_models.download.response", + payload: { requestId: msg.requestId, error: describeError(error) }, + }); + } + } + + private async handleDelete( + msg: Extract, + ): Promise { + try { + await this.requireRuntime().deleteLocalModel(parseModelId(msg.modelId)); + this.options.emit({ + type: "speech.local_models.delete.response", + payload: { requestId: msg.requestId, error: null }, + }); + } catch (error) { + this.options.emit({ + type: "speech.local_models.delete.response", + payload: { requestId: msg.requestId, error: describeError(error) }, + }); + } + } + + private requireRuntime(): SpeechModelsRuntime { + if (!this.options.runtime) { + throw new Error("Speech is unavailable on this daemon"); + } + return this.options.runtime; + } +} + +function parseModelId(modelId: string) { + const parsed = LocalSpeechModelIdSchema.safeParse(modelId); + if (!parsed.success) { + throw new Error(`Unknown local speech model: ${modelId}`); + } + return parsed.data; +} + +function toWireModel(model: LocalSpeechModelStatus): LocalSpeechModel { + const { spec, state, inUse } = model; + return { + id: spec.id, + kind: spec.kind === "tts" ? "tts" : "stt", + label: spec.label, + description: spec.description, + languages: [...spec.languages], + downloadSizeBytes: spec.downloadSizeBytes, + isDefault: spec.defaultFor !== undefined, + inUse, + status: state.status, + ...(state.status === "downloading" + ? { receivedBytes: state.receivedBytes, totalBytes: state.totalBytes } + : {}), + ...(state.status === "failed" ? { error: state.error } : {}), + }; +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts index 8e7d500dc2..dd52537796 100644 --- a/packages/server/src/server/speech/providers/local/config.ts +++ b/packages/server/src/server/speech/providers/local/config.ts @@ -29,6 +29,7 @@ export interface LocalSpeechProviderConfig { export interface ResolvedLocalSpeechConfig { local: LocalSpeechProviderConfig | undefined; + modelsDir: string; sttLanguages: LocalSpeechSttLanguageConfig; } @@ -68,12 +69,12 @@ const LocalSpeechResolutionSchema = z.object({ function persistedLocalFeatureModel( provider: RequestedSpeechProviders[keyof RequestedSpeechProviders]["provider"], enabled: boolean | undefined, - model: string | undefined, + site: { provider: string; model?: string } | undefined, ): string | undefined { if (provider !== "local" || enabled === false) { return undefined; } - return model; + return site?.provider === "local" ? site.model : undefined; } function shouldIncludeLocalProviderConfig(params: { @@ -143,6 +144,8 @@ function buildLocalSpeechResolutionInput(params: { includeProviderConfig: boolean; }): Record { const { paseoHome, env, persisted, providers, includeProviderConfig } = params; + const tts = persisted.features?.voiceMode?.tts; + const localTts = tts?.provider === "local" ? tts : undefined; return { includeProviderConfig, modelsDir: firstDefinedValue([ @@ -155,7 +158,7 @@ function buildLocalSpeechResolutionInput(params: { persistedLocalFeatureModel( providers.dictationStt.provider, providers.dictationStt.enabled, - persisted.features?.dictation?.stt?.model, + persisted.features?.dictation?.stt, ), DEFAULT_LOCAL_STT_MODEL, ]), @@ -164,7 +167,7 @@ function buildLocalSpeechResolutionInput(params: { persistedLocalFeatureModel( providers.voiceStt.provider, providers.voiceStt.enabled, - persisted.features?.voiceMode?.stt?.model, + persisted.features?.voiceMode?.stt, ), DEFAULT_LOCAL_STT_MODEL, ]), @@ -173,18 +176,18 @@ function buildLocalSpeechResolutionInput(params: { persistedLocalFeatureModel( providers.voiceTts.provider, providers.voiceTts.enabled, - persisted.features?.voiceMode?.tts?.model, + persisted.features?.voiceMode?.tts, ), DEFAULT_LOCAL_TTS_MODEL, ]), ...buildLocalSpeechLanguageResolutionInput({ env, persisted }), voiceLocalTtsSpeakerId: firstDefinedValue([ env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID, - persisted.features?.voiceMode?.tts?.speakerId, + localTts?.speakerId, ]), voiceLocalTtsSpeed: firstDefinedValue([ env.PASEO_VOICE_LOCAL_TTS_SPEED, - persisted.features?.voiceMode?.tts?.speed, + localTts?.speed, ]), }; } @@ -200,11 +203,10 @@ export function resolveLocalSpeechConfig(params: { buildLocalSpeechResolutionInput({ ...params, includeProviderConfig }), ); - const resolvedVoiceTtsSpeakerId = - parsed.voiceLocalTtsSpeakerId ?? - (parsed.voiceLocalTtsModel === "kokoro-en-v0_19" ? 0 : undefined); + const resolvedVoiceTtsSpeakerId = parsed.voiceLocalTtsSpeakerId ?? 0; return { + modelsDir: parsed.modelsDir, sttLanguages: { dictation: parsed.dictationLanguage, voice: parsed.voiceLanguage, diff --git a/packages/server/src/server/speech/providers/local/model-library.ts b/packages/server/src/server/speech/providers/local/model-library.ts new file mode 100644 index 0000000000..4ef19e3fa9 --- /dev/null +++ b/packages/server/src/server/speech/providers/local/model-library.ts @@ -0,0 +1,137 @@ +import type { Logger } from "pino"; + +import { + deleteSherpaOnnxModel, + ensureSherpaOnnxModel, + isSherpaOnnxModelInstalled, +} from "./sherpa/model-downloader.js"; +import { + listLocalSpeechModels, + type LocalSpeechModelId, + type LocalSpeechModelSpec, +} from "./models.js"; + +export type LocalSpeechModelInstallState = + | { status: "installed" } + | { status: "not_installed" } + | { status: "downloading"; receivedBytes: number; totalBytes: number } + | { status: "extracting" } + | { status: "failed"; error: string }; + +export interface LocalSpeechModelEntry { + spec: LocalSpeechModelSpec; + state: LocalSpeechModelInstallState; +} + +type InFlightState = Extract< + LocalSpeechModelInstallState, + { status: "downloading" | "extracting" } +>; + +interface InFlightDownload { + promise: Promise; + state: InFlightState; +} + +export class LocalSpeechModelBusyError extends Error { + constructor(public readonly modelId: LocalSpeechModelId) { + super(`${modelId} is downloading. Wait for the download to finish before deleting it.`); + this.name = "LocalSpeechModelBusyError"; + } +} + +// Owns the models directory: which catalog models are installed, which are downloading, +// and the last failure per model. Concurrent downloads of one model share a single transfer. +export class LocalSpeechModelLibrary { + private readonly inFlight = new Map(); + private readonly failures = new Map(); + private readonly abortController = new AbortController(); + private readonly logger: Logger; + + constructor( + private readonly modelsDir: string, + logger: Logger, + ) { + this.logger = logger.child({ module: "speech", component: "local-model-library" }); + } + + getModelsDir(): string { + return this.modelsDir; + } + + async list(): Promise { + return Promise.all( + listLocalSpeechModels().map(async (spec) => ({ spec, state: await this.getState(spec.id) })), + ); + } + + async listMissing(modelIds: LocalSpeechModelId[]): Promise { + const installed = await Promise.all( + modelIds.map((modelId) => isSherpaOnnxModelInstalled(this.modelsDir, modelId)), + ); + return modelIds.filter((_modelId, index) => !installed[index]); + } + + download(modelId: LocalSpeechModelId): Promise { + const existing = this.inFlight.get(modelId); + if (existing) { + return existing.promise; + } + this.failures.delete(modelId); + const entry: InFlightDownload = { + state: { status: "downloading", receivedBytes: 0, totalBytes: 0 }, + promise: Promise.resolve(""), + }; + entry.promise = ensureSherpaOnnxModel({ + modelsDir: this.modelsDir, + modelId, + logger: this.logger, + signal: this.abortController.signal, + onProgress: (progress) => { + entry.state = + progress.phase === "downloading" + ? { + status: "downloading", + receivedBytes: progress.receivedBytes, + totalBytes: progress.totalBytes, + } + : { status: "extracting" }; + }, + }) + .catch((error: unknown) => { + this.failures.set(modelId, error instanceof Error ? error.message : String(error)); + throw error; + }) + .finally(() => { + this.inFlight.delete(modelId); + }); + this.inFlight.set(modelId, entry); + return entry.promise; + } + + async delete(modelId: LocalSpeechModelId): Promise { + if (this.inFlight.has(modelId)) { + throw new LocalSpeechModelBusyError(modelId); + } + this.failures.delete(modelId); + await deleteSherpaOnnxModel(this.modelsDir, modelId); + this.logger.info({ modelId }, "Deleted local speech model"); + } + + async stop(): Promise { + this.abortController.abort(); + await Promise.allSettled(Array.from(this.inFlight.values(), (entry) => entry.promise)); + } + + private async getState(modelId: LocalSpeechModelId): Promise { + const inFlight = this.inFlight.get(modelId); + if (inFlight) { + return inFlight.state; + } + if (await isSherpaOnnxModelInstalled(this.modelsDir, modelId)) { + return { status: "installed" }; + } + const failure = this.failures.get(modelId); + return failure ? { status: "failed", error: failure } : { status: "not_installed" }; + } +} diff --git a/packages/server/src/server/speech/providers/local/models.test.ts b/packages/server/src/server/speech/providers/local/models.test.ts index e40cc25384..be010dcae9 100644 --- a/packages/server/src/server/speech/providers/local/models.test.ts +++ b/packages/server/src/server/speech/providers/local/models.test.ts @@ -13,12 +13,8 @@ import path from "node:path"; import pino from "pino"; import { describe, expect, onTestFinished, test } from "vitest"; -import { - ensureLocalSpeechModels, - getLocalSpeechModelDir, - listLocalSpeechModels, - listMissingLocalSpeechModels, -} from "./models.js"; +import { LocalSpeechModelLibrary } from "./model-library.js"; +import { getLocalSpeechModelDir, listLocalSpeechModels } from "./models.js"; const logger = pino({ level: "silent" }); const modelId = "parakeet-tdt-0.6b-v2-int8"; @@ -55,16 +51,31 @@ describe("local speech models", () => { const encoder = path.join(modelDir, spec.requiredFiles[0]); truncateSync(encoder, 1_000); - expect(await listMissingLocalSpeechModels({ modelsDir, modelIds: [modelId] })).toEqual([ - modelId, - ]); + const library = new LocalSpeechModelLibrary(modelsDir, logger); + expect(await library.listMissing([modelId])).toEqual([modelId]); - await ensureLocalSpeechModels({ modelsDir, modelIds: [modelId], logger }); + await library.download(modelId); expect(readFileSync(encoder, "utf8")).toBe( `complete ${spec.requiredFiles[0]}`.padEnd(64_000, "."), ); expect(existsSync(archive)).toBe(false); - expect(await listMissingLocalSpeechModels({ modelsDir, modelIds: [modelId] })).toEqual([]); + expect(await library.listMissing([modelId])).toEqual([]); + }); + + test("lists install state and deletes an installed model", async () => { + const { modelsDir } = setupModelsDirWithArchive(); + const library = new LocalSpeechModelLibrary(modelsDir, logger); + const stateOf = async () => + (await library.list()).find((entry) => entry.spec.id === modelId)?.state; + + expect(await stateOf()).toEqual({ status: "not_installed" }); + await library.download(modelId); + expect(await stateOf()).toEqual({ status: "installed" }); + + await library.delete(modelId); + + expect(await stateOf()).toEqual({ status: "not_installed" }); + expect(existsSync(getLocalSpeechModelDir(modelsDir, modelId))).toBe(false); }); }); diff --git a/packages/server/src/server/speech/providers/local/models.ts b/packages/server/src/server/speech/providers/local/models.ts index a884be86d7..7d4c7f56b5 100644 --- a/packages/server/src/server/speech/providers/local/models.ts +++ b/packages/server/src/server/speech/providers/local/models.ts @@ -1,11 +1,8 @@ -import { - ensureSherpaOnnxModels, - getSherpaOnnxModelDir, - isSherpaOnnxModelInstalled, -} from "./sherpa/model-downloader.js"; +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./sherpa/model-downloader.js"; import { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, + LocalSpeechModelIdSchema, LocalSttModelIdSchema, LocalTtsModelIdSchema, listSherpaOnnxModels, @@ -17,6 +14,7 @@ import { export { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, + LocalSpeechModelIdSchema, LocalSttModelIdSchema, LocalTtsModelIdSchema, type LocalSpeechModelId, @@ -34,16 +32,6 @@ export function getLocalSpeechModelDir(modelsDir: string, modelId: LocalSpeechMo return getSherpaOnnxModelDir(modelsDir, modelId); } -export async function listMissingLocalSpeechModels(options: { - modelsDir: string; - modelIds: LocalSpeechModelId[]; -}): Promise { - const installed = await Promise.all( - options.modelIds.map((modelId) => isSherpaOnnxModelInstalled(options.modelsDir, modelId)), - ); - return options.modelIds.filter((_modelId, index) => !installed[index]); -} - export async function ensureLocalSpeechModels(options: { modelsDir: string; modelIds: LocalSpeechModelId[]; diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 06d838c49a..b142f52dd6 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -4,43 +4,467 @@ export type SherpaOnnxModelKind = "stt-offline" | "tts"; type DefaultModelRole = "stt" | "tts"; -interface SherpaOnnxCatalogEntry { - kind: SherpaOnnxModelKind; +// How sherpa-onnx loads a model, with file names relative to the extracted directory. +export type SttEngine = + | { family: "transducer"; encoder: string; decoder: string; joiner: string; tokens: string } + | { family: "whisper"; encoder: string; decoder: string; tokens: string } + | { family: "sense_voice"; model: string; tokens: string } + | { family: "dolphin"; model: string; tokens: string } + | { family: "nemo_ctc"; model: string; tokens: string }; + +export type TtsEngine = + | { + family: "kokoro"; + model: string; + voices: string; + tokens: string; + dataDir: string; + // Multi-language Kokoro models read pronunciations from these instead of espeak alone. + lexicons?: readonly string[]; + dictDir?: string; + ruleFsts?: readonly string[]; + } + | { family: "kitten"; model: string; voices: string; tokens: string; dataDir: string } + | { family: "vits"; model: string; tokens: string; dataDir: string }; + +interface SherpaOnnxCatalogEntryBase { + label: string; + // One user-facing sentence shown before download. + description: string; + // BCP 47 language tags the model transcribes or speaks. + languages: readonly string[]; + // Exact size of the release archive. The downloader rejects any other size. + downloadSizeBytes: number; archiveUrl: string; extractedDir: string; - requiredFiles: string[]; - description: string; defaultFor?: DefaultModelRole; } +type SherpaOnnxCatalogEntry = + | (SherpaOnnxCatalogEntryBase & { kind: "stt-offline"; engine: SttEngine }) + | (SherpaOnnxCatalogEntryBase & { kind: "tts"; engine: TtsEngine }); + +// Every sherpa-onnx release archive extracts to a directory named after the archive. +function releaseArchive(tag: "asr-models" | "tts-models", name: string) { + return { + archiveUrl: `https://github.com/k2-fsa/sherpa-onnx/releases/download/${tag}/${name}.tar.bz2`, + extractedDir: name, + }; +} + +const PARAKEET_INT8_ENGINE = { + family: "transducer", + encoder: "encoder.int8.onnx", + decoder: "decoder.int8.onnx", + joiner: "joiner.int8.onnx", + tokens: "tokens.txt", +} as const satisfies SttEngine; + +function whisperEngine(size: string): SttEngine { + return { + family: "whisper", + encoder: `${size}-encoder.int8.onnx`, + decoder: `${size}-decoder.int8.onnx`, + tokens: `${size}-tokens.txt`, + }; +} + +function piperEngine(voice: string): TtsEngine { + return { + family: "vits", + model: `${voice}.onnx`, + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + }; +} + +// Parakeet TDT v3 transcribes these 25 European languages and detects which one is spoken. +const PARAKEET_V3_LANGUAGES = [ + "bg", + "hr", + "cs", + "da", + "nl", + "en", + "et", + "fi", + "fr", + "de", + "el", + "hu", + "it", + "lv", + "lt", + "mt", + "pl", + "pt", + "ro", + "sk", + "sl", + "es", + "sv", + "ru", + "uk", +] as const; + +// The 99 languages multilingual Whisper models transcribe, detecting which one is spoken. +const WHISPER_LANGUAGES = [ + "en", + "zh", + "de", + "es", + "ru", + "ko", + "fr", + "ja", + "pt", + "tr", + "pl", + "ca", + "nl", + "ar", + "sv", + "it", + "id", + "hi", + "fi", + "vi", + "he", + "uk", + "el", + "ms", + "cs", + "ro", + "da", + "hu", + "ta", + "no", + "th", + "ur", + "hr", + "bg", + "lt", + "la", + "mi", + "ml", + "cy", + "sk", + "te", + "fa", + "lv", + "bn", + "sr", + "az", + "sl", + "kn", + "et", + "mk", + "br", + "eu", + "is", + "hy", + "ne", + "mn", + "bs", + "kk", + "sq", + "sw", + "gl", + "mr", + "pa", + "si", + "km", + "sn", + "yo", + "so", + "af", + "oc", + "ka", + "be", + "tg", + "sd", + "gu", + "am", + "yi", + "lo", + "uz", + "fo", + "ht", + "ps", + "tk", + "nn", + "mt", + "sa", + "lb", + "my", + "bo", + "tl", + "mg", + "as", + "tt", + "haw", + "ln", + "ha", + "ba", + "jv", + "su", +] as const; + +// Dolphin's 40 Eastern languages. It also covers 22 Chinese dialects under "zh". +const DOLPHIN_LANGUAGES = [ + "zh", + "ja", + "th", + "ru", + "ko", + "id", + "vi", + "yue", + "hi", + "ur", + "ms", + "uz", + "ar", + "fa", + "bn", + "ta", + "te", + "ug", + "gu", + "my", + "tl", + "kk", + "or", + "ne", + "mn", + "km", + "jv", + "lo", + "si", + "fil", + "ps", + "pa", + "kab", + "ba", + "ks", + "tg", + "su", + "mr", + "ky", + "az", +] as const; + export const SHERPA_ONNX_MODEL_CATALOG = { "parakeet-tdt-0.6b-v2-int8": { kind: "stt-offline", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2", - extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", - requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"], - description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).", + label: "Parakeet v2", + description: "Accurate English transcription that runs well on a laptop CPU.", + languages: ["en"], + downloadSizeBytes: 482_468_385, + ...releaseArchive("asr-models", "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8"), + engine: PARAKEET_INT8_ENGINE, defaultFor: "stt", }, + "parakeet-tdt-110m-en-int8": { + kind: "stt-offline", + label: "Parakeet Small", + description: "Small, fast English model with punctuation. A quick download for any laptop.", + languages: ["en"], + downloadSizeBytes: 108_035_095, + ...releaseArchive("asr-models", "sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8"), + engine: PARAKEET_INT8_ENGINE, + }, "parakeet-tdt-0.6b-v3-int8": { kind: "stt-offline", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2", - extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8", - requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"], - description: - "NVIDIA Parakeet TDT v3 (offline NeMo transducer, 25 European languages, auto-detected).", + label: "Parakeet v3", + description: "Parakeet for 25 European languages, detecting which one you speak.", + languages: PARAKEET_V3_LANGUAGES, + downloadSizeBytes: 487_170_055, + ...releaseArchive("asr-models", "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8"), + engine: PARAKEET_INT8_ENGINE, + }, + "whisper-tiny-en": { + kind: "stt-offline", + label: "Whisper Tiny (English)", + description: "Small English Whisper model. Quick to download, less accurate than Parakeet.", + languages: ["en"], + downloadSizeBytes: 118_071_777, + ...releaseArchive("asr-models", "sherpa-onnx-whisper-tiny.en"), + engine: whisperEngine("tiny.en"), + }, + "whisper-base": { + kind: "stt-offline", + label: "Whisper Base", + description: "Compact Whisper model for 99 languages, detecting which one you speak.", + languages: WHISPER_LANGUAGES, + downloadSizeBytes: 207_557_382, + ...releaseArchive("asr-models", "sherpa-onnx-whisper-base"), + engine: whisperEngine("base"), + }, + "whisper-turbo": { + kind: "stt-offline", + label: "Whisper Turbo", + description: "Whisper's most accurate model for 99 languages and Cantonese. Slower on a CPU.", + languages: [...WHISPER_LANGUAGES, "yue"], + downloadSizeBytes: 563_790_207, + ...releaseArchive("asr-models", "sherpa-onnx-whisper-turbo"), + engine: whisperEngine("turbo"), + }, + "sense-voice-int8": { + kind: "stt-offline", + label: "SenseVoice", + description: "Chinese, Cantonese, English, Japanese and Korean, detecting which one you speak.", + languages: ["zh", "yue", "en", "ja", "ko"], + downloadSizeBytes: 165_783_878, + ...releaseArchive("asr-models", "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09"), + engine: { family: "sense_voice", model: "model.int8.onnx", tokens: "tokens.txt" }, + }, + "dolphin-base-int8": { + kind: "stt-offline", + label: "Dolphin", + description: "Small model for 40 Asian and Middle Eastern languages and Chinese dialects.", + languages: DOLPHIN_LANGUAGES, + downloadSizeBytes: 80_671_385, + ...releaseArchive("asr-models", "sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02"), + engine: { family: "dolphin", model: "model.int8.onnx", tokens: "tokens.txt" }, + }, + "giga-am-v3-ru": { + kind: "stt-offline", + label: "GigaAM v3 (Russian)", + description: "Russian transcription with punctuation.", + languages: ["ru"], + downloadSizeBytes: 170_197_019, + ...releaseArchive( + "asr-models", + "sherpa-onnx-nemo-transducer-punct-giga-am-v3-russian-2025-12-16", + ), + engine: { + family: "transducer", + encoder: "encoder.int8.onnx", + decoder: "decoder.onnx", + joiner: "joiner.onnx", + tokens: "tokens.txt", + }, + }, + "parakeet-tdt-ctc-0.6b-ja-int8": { + kind: "stt-offline", + label: "Parakeet (Japanese)", + description: "Japanese transcription with punctuation.", + languages: ["ja"], + downloadSizeBytes: 489_389_564, + ...releaseArchive("asr-models", "sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8"), + engine: { family: "nemo_ctc", model: "model.int8.onnx", tokens: "tokens.txt" }, }, "kokoro-en-v0_19": { kind: "tts", - archiveUrl: - "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2", - extractedDir: "kokoro-en-v0_19", - requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"], - description: "Kokoro TTS (higher quality; larger).", + label: "Kokoro (English)", + description: "Natural-sounding English voice for voice mode replies.", + languages: ["en"], + downloadSizeBytes: 319_625_534, + ...releaseArchive("tts-models", "kokoro-en-v0_19"), + engine: { + family: "kokoro", + model: "model.onnx", + voices: "voices.bin", + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + }, defaultFor: "tts", }, + "kokoro-int8-en-v0_19": { + kind: "tts", + label: "Kokoro Lite (English)", + description: "The same Kokoro English voices, quantized to a third of the download.", + languages: ["en"], + downloadSizeBytes: 103_248_205, + ...releaseArchive("tts-models", "kokoro-int8-en-v0_19"), + engine: { + family: "kokoro", + model: "model.int8.onnx", + voices: "voices.bin", + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + }, + }, + "kokoro-int8-multi-lang-v1_1": { + kind: "tts", + label: "Kokoro (Chinese and English)", + description: "Kokoro voices that speak both Mandarin Chinese and English.", + languages: ["zh", "en"], + downloadSizeBytes: 147_031_220, + ...releaseArchive("tts-models", "kokoro-int8-multi-lang-v1_1"), + engine: { + family: "kokoro", + model: "model.int8.onnx", + voices: "voices.bin", + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + lexicons: ["lexicon-us-en.txt", "lexicon-zh.txt"], + dictDir: "dict", + ruleFsts: ["date-zh.fst", "phone-zh.fst", "number-zh.fst"], + }, + }, + // Kitten v0.8 needs a newer sherpa-onnx than we pin; v0.2 loads on 1.12.28. + "kitten-nano-en-v0_2-fp16": { + kind: "tts", + label: "Kitten Nano (English)", + description: "Tiny, very fast English voice.", + languages: ["en"], + downloadSizeBytes: 26_586_708, + ...releaseArchive("tts-models", "kitten-nano-en-v0_2-fp16"), + engine: { + family: "kitten", + model: "model.fp16.onnx", + voices: "voices.bin", + tokens: "tokens.txt", + dataDir: "espeak-ng-data", + }, + }, + "piper-en-us-libritts-r": { + kind: "tts", + label: "Piper LibriTTS (US English)", + description: "Lightweight American English voice.", + languages: ["en-US"], + downloadSizeBytes: 23_398_348, + ...releaseArchive("tts-models", "vits-piper-en_US-libritts_r-medium-int8"), + engine: piperEngine("en_US-libritts_r-medium"), + }, + "piper-en-gb-alba": { + kind: "tts", + label: "Piper Alba (British English)", + description: "Lightweight British English voice.", + languages: ["en-GB"], + downloadSizeBytes: 21_104_326, + ...releaseArchive("tts-models", "vits-piper-en_GB-alba-medium-int8"), + engine: piperEngine("en_GB-alba-medium"), + }, + "piper-de-thorsten": { + kind: "tts", + label: "Piper Thorsten (German)", + description: "Lightweight German voice.", + languages: ["de"], + downloadSizeBytes: 20_949_833, + ...releaseArchive("tts-models", "vits-piper-de_DE-thorsten-medium-int8"), + engine: piperEngine("de_DE-thorsten-medium"), + }, + "piper-fr-siwis": { + kind: "tts", + label: "Piper Siwis (French)", + description: "Lightweight French voice.", + languages: ["fr"], + downloadSizeBytes: 20_914_888, + ...releaseArchive("tts-models", "vits-piper-fr_FR-siwis-medium-int8"), + engine: piperEngine("fr_FR-siwis-medium"), + }, + "piper-es-davefx": { + kind: "tts", + label: "Piper Davefx (Spanish)", + description: "Lightweight Spanish voice.", + languages: ["es"], + downloadSizeBytes: 21_171_632, + ...releaseArchive("tts-models", "vits-piper-es_ES-davefx-medium-int8"), + engine: piperEngine("es_ES-davefx-medium"), + }, } as const satisfies Record; export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG; @@ -97,24 +521,44 @@ function createModelIdSchema(modelIds: readonly T[]): z.ZodTyp .transform((value) => value as T); } +export const LocalSpeechModelIdSchema = createModelIdSchema(ALL_MODEL_IDS); export const LocalSttModelIdSchema = createModelIdSchema(LOCAL_STT_MODEL_IDS); export const LocalTtsModelIdSchema = createModelIdSchema(LOCAL_TTS_MODEL_IDS); export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & { id: SherpaOnnxModelId; + // Every file and directory that must exist for the model to count as installed. + requiredFiles: string[]; }; export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] { - return ALL_MODEL_IDS.map((id) => Object.assign({ id }, SHERPA_ONNX_MODEL_CATALOG[id])); + return ALL_MODEL_IDS.map(getSherpaOnnxModelSpec); } export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec { - const spec = SHERPA_ONNX_MODEL_CATALOG[id]; + const spec: SherpaOnnxCatalogEntry | undefined = SHERPA_ONNX_MODEL_CATALOG[id]; if (!spec) { throw new Error(`Unknown local speech model id: ${id}`); } return { id, ...spec, + requiredFiles: engineFiles(spec.engine), }; } + +// Every path an engine names, so the downloader can verify an install. +export function engineFiles(engine: SttEngine | TtsEngine): string[] { + return Object.entries(engine).flatMap(([key, value]) => { + if (key === "family") return []; + return typeof value === "string" ? [value] : (value as readonly string[]); + }); +} + +export function getLocalSttEngine(id: LocalSttModelId): SttEngine { + return SHERPA_ONNX_MODEL_CATALOG[id].engine; +} + +export function getLocalTtsEngine(id: LocalTtsModelId): TtsEngine { + return SHERPA_ONNX_MODEL_CATALOG[id].engine; +} diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts index cdd286389f..b7ba511224 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts @@ -8,11 +8,16 @@ import type pino from "pino"; import { getSherpaOnnxModelSpec, type SherpaOnnxModelId } from "./model-catalog.js"; import { spawnProcess } from "../../../../../utils/spawn.js"; +export type SherpaOnnxModelInstallProgress = + | { phase: "downloading"; receivedBytes: number; totalBytes: number } + | { phase: "extracting" }; + export interface EnsureSherpaOnnxModelOptions { modelsDir: string; modelId: SherpaOnnxModelId; logger: pino.Logger; signal?: AbortSignal; + onProgress?: (progress: SherpaOnnxModelInstallProgress) => void; } export function getSherpaOnnxModelDir(modelsDir: string, modelId: SherpaOnnxModelId): string { @@ -41,7 +46,9 @@ async function hasRequiredFiles(modelDir: string, requiredFiles: string[]): Prom interface DownloadToFileOptions { url: string; outputPath: string; + expectedBytes: number; signal?: AbortSignal; + onProgress?: (receivedBytes: number, totalBytes: number) => void; } async function downloadToFile(options: DownloadToFileOptions): Promise { @@ -60,6 +67,13 @@ async function downloadToFile(options: DownloadToFileOptions): Promise { // The fetch ReadableStream type is slightly different from what Readable.fromWeb expects // eslint-disable-next-line @typescript-eslint/no-explicit-any const nodeStream = Readable.fromWeb(res.body as any); + const contentLength = Number(res.headers.get("content-length")); + const totalBytes = contentLength > 0 ? contentLength : options.expectedBytes; + let receivedBytes = 0; + nodeStream.on("data", (chunk: Buffer) => { + receivedBytes += chunk.length; + options.onProgress?.(receivedBytes, totalBytes); + }); try { await pipeline(nodeStream, createWriteStream(tmpPath), { signal: options.signal }); @@ -157,9 +171,13 @@ export async function ensureSherpaOnnxModel( await downloadToFile({ url: spec.archiveUrl, outputPath: archivePath, + expectedBytes: spec.downloadSizeBytes, signal: options.signal, + onProgress: (receivedBytes, totalBytes) => + options.onProgress?.({ phase: "downloading", receivedBytes, totalBytes }), }); } + options.onProgress?.({ phase: "extracting" }); logger.info( { @@ -213,6 +231,16 @@ export async function ensureSherpaOnnxModel( } } +export async function deleteSherpaOnnxModel( + modelsDir: string, + modelId: SherpaOnnxModelId, +): Promise { + const { modelDir, archivePath, extractionDir } = getSherpaOnnxModelPaths(modelsDir, modelId); + await rm(modelDir, { recursive: true, force: true }); + await rm(archivePath, { force: true }); + await rm(extractionDir, { recursive: true, force: true }); +} + export async function ensureSherpaOnnxModels(options: { modelsDir: string; modelIds: SherpaOnnxModelId[]; diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts index cfde73addf..5137f51810 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts @@ -1,6 +1,8 @@ import { existsSync } from "node:fs"; +import path from "node:path"; import type pino from "pino"; +import { engineFiles, type SttEngine } from "./model-catalog.js"; import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; function assertFileExists(filePath: string, label: string): void { @@ -9,16 +11,43 @@ function assertFileExists(filePath: string, label: string): void { } } -export interface SherpaOfflineRecognizerModel { - kind: "nemo_transducer"; - encoder: string; - decoder: string; - joiner: string; - tokens: string; +// Each engine family takes its own block under sherpa's modelConfig. +function buildModelConfig(modelDir: string, engine: SttEngine): Record { + const file = (name: string) => path.join(modelDir, name); + switch (engine.family) { + case "transducer": + return { + transducer: { + encoder: file(engine.encoder), + decoder: file(engine.decoder), + joiner: file(engine.joiner), + }, + modelType: "nemo_transducer", + }; + case "whisper": + // An empty language makes Whisper detect the spoken language. + return { + whisper: { + encoder: file(engine.encoder), + decoder: file(engine.decoder), + language: "", + task: "transcribe", + }, + }; + case "sense_voice": + return { + senseVoice: { model: file(engine.model), language: "", useInverseTextNormalization: 1 }, + }; + case "dolphin": + return { dolphin: { model: file(engine.model) } }; + case "nemo_ctc": + return { nemoCtc: { model: file(engine.model) } }; + } } export interface SherpaOfflineRecognizerConfig { - model: SherpaOfflineRecognizerModel; + modelDir: string; + engine: SttEngine; numThreads?: number; provider?: "cpu"; debug?: 0 | 1; @@ -54,10 +83,9 @@ export class SherpaOfflineRecognizerEngine { component: "offline-recognizer", }); - assertFileExists(config.model.encoder, "offline encoder"); - assertFileExists(config.model.decoder, "offline decoder"); - assertFileExists(config.model.joiner, "offline joiner"); - assertFileExists(config.model.tokens, "tokens"); + for (const name of engineFiles(config.engine)) { + assertFileExists(path.join(config.modelDir, name), `${config.engine.family} model file`); + } const sherpa = loadSherpaOnnxNode(); @@ -67,13 +95,8 @@ export class SherpaOfflineRecognizerEngine { featureDim: config.featureDim ?? 80, }, modelConfig: { - transducer: { - encoder: config.model.encoder, - decoder: config.model.decoder, - joiner: config.model.joiner, - }, - tokens: config.model.tokens, - modelType: "nemo_transducer", + ...buildModelConfig(config.modelDir, config.engine), + tokens: path.join(config.modelDir, config.engine.tokens), numThreads: config.numThreads ?? 1, provider: config.provider ?? "cpu", debug: config.debug ?? 0, diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts index 06d4a5b41d..0ae51f86ee 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import pino from "pino"; +import { engineFiles, getLocalTtsEngine, type LocalTtsModelId } from "./model-catalog.js"; import { SherpaOnnxTTS } from "./sherpa-tts.js"; const directories: string[] = []; @@ -11,17 +12,28 @@ afterEach(() => { rmSync(directory, { recursive: true, force: true }); }); -function createTts(numThreads?: number, beforeGenerate?: () => Promise) { +function createTts( + numThreads?: number, + beforeGenerate?: () => Promise, + modelId: LocalTtsModelId = "kokoro-en-v0_19", +) { const modelDir = mkdtempSync(join(tmpdir(), "paseo-tts-")); directories.push(modelDir); - for (const name of ["model.onnx", "voices.bin", "tokens.txt"]) - writeFileSync(join(modelDir, name), ""); - mkdirSync(join(modelDir, "espeak-ng-data")); + // Data directories have no extension; everything else is a model file. + for (const name of engineFiles(getLocalTtsEngine(modelId))) { + if (name.includes(".")) writeFileSync(join(modelDir, name), ""); + else mkdirSync(join(modelDir, name)); + } let nativeConfig: unknown; const requests: unknown[] = []; const released: boolean[] = []; const tts = new SherpaOnnxTTS( - { preset: "kokoro-en-v0_19", modelDir, numThreads }, + { + modelId, + modelDir, + engine: getLocalTtsEngine(modelId), + numThreads, + }, pino({ level: "silent" }), () => ({ OfflineTts: class { @@ -138,4 +150,31 @@ describe("SherpaOnnxTTS", () => { expect(nativeConfig).not.toHaveProperty("provider"); }, ); + + it("loads Piper voices through the vits engine", () => { + const { nativeConfig } = createTts(undefined, undefined, "piper-de-thorsten"); + expect(nativeConfig).toMatchObject({ + model: { + vits: { + model: expect.stringMatching(/de_DE-thorsten-medium\.onnx$/), + tokens: expect.stringMatching(/tokens\.txt$/), + dataDir: expect.stringMatching(/espeak-ng-data$/), + }, + }, + }); + expect(nativeConfig).not.toHaveProperty("model.kokoro"); + }); + + it("gives multi-language Kokoro its lexicons, dictionary and number rules", () => { + const { nativeConfig } = createTts(undefined, undefined, "kokoro-int8-multi-lang-v1_1"); + expect(nativeConfig).toMatchObject({ + model: { + kokoro: { + lexicon: expect.stringMatching(/lexicon-us-en\.txt,.*lexicon-zh\.txt$/), + dictDir: expect.stringMatching(/dict$/), + }, + }, + ruleFsts: expect.stringMatching(/date-zh\.fst,.*phone-zh\.fst,.*number-zh\.fst$/), + }); + }); }); diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts index a078f81f51..12c86b8777 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts @@ -1,16 +1,17 @@ import type pino from "pino"; import { Readable } from "node:stream"; import { existsSync } from "node:fs"; +import path from "node:path"; import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js"; import { chunkBuffer, float32ToPcm16le } from "../../../audio.js"; import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js"; - -export type SherpaTtsPreset = "kokoro-en-v0_19"; +import { engineFiles, type LocalTtsModelId, type TtsEngine } from "./model-catalog.js"; export interface SherpaTtsConfig { - preset: SherpaTtsPreset; + modelId: LocalTtsModelId; modelDir: string; + engine: TtsEngine; speakerId?: number; speed?: number; lengthScale?: number; @@ -23,6 +24,53 @@ function assertFileExists(filePath: string, label: string): void { } } +// Each engine family takes its own block under sherpa's TTS model config. +function buildTtsModelConfig( + modelDir: string, + engine: TtsEngine, + lengthScale: number, +): Record { + const file = (name: string) => path.join(modelDir, name); + switch (engine.family) { + case "kokoro": + return { + kokoro: { + model: file(engine.model), + voices: file(engine.voices), + tokens: file(engine.tokens), + dataDir: file(engine.dataDir), + lexicon: (engine.lexicons ?? []).map(file).join(","), + dictDir: engine.dictDir ? file(engine.dictDir) : "", + lengthScale, + }, + }; + case "kitten": + return { + kitten: { + model: file(engine.model), + voices: file(engine.voices), + tokens: file(engine.tokens), + dataDir: file(engine.dataDir), + lengthScale, + }, + }; + case "vits": + return { + vits: { + model: file(engine.model), + tokens: file(engine.tokens), + dataDir: file(engine.dataDir), + lengthScale, + }, + }; + } +} + +function ruleFsts(modelDir: string, engine: TtsEngine): string { + if (engine.family !== "kokoro" || !engine.ruleFsts) return ""; + return engine.ruleFsts.map((name) => path.join(modelDir, name)).join(","); +} + interface SherpaOfflineTtsNative { sampleRate?: number; generateAsync: (args: { @@ -59,39 +107,28 @@ export class SherpaOnnxTTS implements TextToSpeechProvider { throw new Error("sherpa-onnx-node OfflineTts is unavailable"); } - const modelPath = `${config.modelDir}/model.onnx`; - const voicesPath = `${config.modelDir}/voices.bin`; - const tokensPath = `${config.modelDir}/tokens.txt`; - const dataDir = `${config.modelDir}/espeak-ng-data`; - - assertFileExists(modelPath, "TTS model"); - assertFileExists(voicesPath, "TTS voices"); - assertFileExists(tokensPath, "TTS tokens"); - assertFileExists(dataDir, "TTS espeak-ng dataDir"); + for (const name of engineFiles(config.engine)) { + assertFileExists(path.join(config.modelDir, name), `${config.engine.family} TTS file`); + } const modelConfig = { // The native parser reads these under model, despite the upstream JS typedef. numThreads: config.numThreads ?? 2, provider: "cpu", - kokoro: { - model: modelPath, - voices: voicesPath, - tokens: tokensPath, - dataDir, - lengthScale: config.lengthScale ?? 1.0, - }, + ...buildTtsModelConfig(config.modelDir, config.engine, config.lengthScale ?? 1.0), }; const offlineTtsConfig = { model: modelConfig, maxNumSentences: 1, + ruleFsts: ruleFsts(config.modelDir, config.engine), }; this.tts = new ( sherpa as unknown as { OfflineTts: new (config: unknown) => SherpaOfflineTtsNative } ).OfflineTts(offlineTtsConfig); this.logger.info( - { preset: config.preset, modelDir: config.modelDir }, + { modelId: config.modelId, modelDir: config.modelDir }, "Sherpa offline TTS initialized", ); } diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index ab2eefef55..43ab8af01a 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -7,6 +7,7 @@ import pino from "pino"; import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; import { createDaemonTestContext } from "../../../../test-utils/index.js"; import { parsePcm16MonoWav, wordSimilarity } from "../../../../test-utils/dictation-e2e.js"; +import { getLocalSttEngine, getLocalTtsEngine } from "./model-catalog.js"; import { SherpaOnnxTTS } from "./sherpa-tts.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa-offline-recognizer.js"; import { SherpaOnnxParakeetSTT } from "./sherpa-parakeet-stt.js"; @@ -192,7 +193,12 @@ downloadTest( const ttsText = "This is a voice note."; const ttsModelDir = getSherpaOnnxModelDir(modelsDir, "kokoro-en-v0_19"); const tts = new SherpaOnnxTTS( - { preset: "kokoro-en-v0_19", modelDir: ttsModelDir, speakerId: 0 }, + { + modelId: "kokoro-en-v0_19", + modelDir: ttsModelDir, + engine: getLocalTtsEngine("kokoro-en-v0_19"), + speakerId: 0, + }, logger, ); const { stream, format: ttsFormat } = await tts.synthesizeSpeech(ttsText); @@ -206,13 +212,8 @@ downloadTest( const sttModelDir = getSherpaOnnxModelDir(modelsDir, "parakeet-tdt-0.6b-v2-int8"); const engine = new SherpaOfflineRecognizerEngine( { - model: { - kind: "nemo_transducer", - encoder: `${sttModelDir}/encoder.int8.onnx`, - decoder: `${sttModelDir}/decoder.int8.onnx`, - joiner: `${sttModelDir}/joiner.int8.onnx`, - tokens: `${sttModelDir}/tokens.txt`, - }, + modelDir: sttModelDir, + engine: getLocalSttEngine("parakeet-tdt-0.6b-v2-int8"), numThreads: 2, debug: 0, }, diff --git a/packages/server/src/server/speech/providers/local/worker-process.ts b/packages/server/src/server/speech/providers/local/worker-process.ts index 957ee6aaa2..2afa3d336a 100644 --- a/packages/server/src/server/speech/providers/local/worker-process.ts +++ b/packages/server/src/server/speech/providers/local/worker-process.ts @@ -3,6 +3,7 @@ import pino from "pino"; import type { StreamingTranscriptionSession } from "../../speech-provider.js"; import type { TurnDetectionSession } from "../../turn-detection-provider.js"; import { getLocalSpeechModelDir, type LocalSttModelId, type LocalTtsModelId } from "./models.js"; +import { getLocalSttEngine, getLocalTtsEngine } from "./sherpa/model-catalog.js"; import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js"; import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js"; import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js"; @@ -85,13 +86,8 @@ function getSttEngine( const modelDir = getLocalSpeechModelDir(config.modelsDir, modelId); const created = new SherpaOfflineRecognizerEngine( { - model: { - kind: "nemo_transducer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - joiner: `${modelDir}/joiner.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, + modelDir, + engine: getLocalSttEngine(modelId), numThreads: 2, debug: 0, }, @@ -125,8 +121,9 @@ function getTtsProvider(config: LocalSpeechWorkerConfig): SherpaOnnxTTS { const modelDir = getLocalSpeechModelDir(config.modelsDir, ttsModelId(config)); const created = new SherpaOnnxTTS( { - preset: ttsModelId(config), + modelId: ttsModelId(config), modelDir, + engine: getLocalTtsEngine(ttsModelId(config)), speakerId: config.voiceTtsSpeakerId, speed: config.voiceTtsSpeed, }, diff --git a/packages/server/src/server/speech/providers/openai/config.test.ts b/packages/server/src/server/speech/providers/openai/config.test.ts index 1f3433a347..dd465dd2f9 100644 --- a/packages/server/src/server/speech/providers/openai/config.test.ts +++ b/packages/server/src/server/speech/providers/openai/config.test.ts @@ -1,5 +1,4 @@ import { describe, expect, test } from "vitest"; - import { PersistedConfigSchema } from "../../../persisted-config.js"; import { resolveOpenAiSpeechConfig } from "./config.js"; import type { RequestedSpeechProviders } from "../../speech-types.js"; @@ -10,233 +9,139 @@ const ALL_OPENAI: RequestedSpeechProviders = { voiceStt: { provider: "openai", explicit: true }, voiceTts: { provider: "openai", explicit: true }, }; +function resolve(input: unknown, env: NodeJS.ProcessEnv = {}) { + return resolveOpenAiSpeechConfig({ + persisted: PersistedConfigSchema.parse(input), + env, + providers: ALL_OPENAI, + }); +} describe("resolveOpenAiSpeechConfig", () => { - test("treats empty OPENAI_API_KEY as unset", () => { - const persisted = PersistedConfigSchema.parse({}); - const env = { - OPENAI_API_KEY: "", - } as NodeJS.ProcessEnv; - - const resolved = resolveOpenAiSpeechConfig({ - env, - persisted, - providers: { - ...ALL_OPENAI, - dictationStt: { provider: "local", explicit: false }, - voiceStt: { provider: "local", explicit: false }, - voiceTts: { provider: "local", explicit: false }, + test("treats empty credentials as unset", () => { + expect(resolve({}, { OPENAI_API_KEY: "", OPENAI_STT_API_KEY: " " })).toBeUndefined(); + }); + test("uses endpoint env before global env and ignores empty fallbacks", () => { + const result = resolve( + {}, + { + OPENAI_API_KEY: " global ", + OPENAI_BASE_URL: " https://global/v1 ", + OPENAI_STT_API_KEY: " stt ", + OPENAI_STT_BASE_URL: " https://stt/v1 ", + OPENAI_TTS_API_KEY: " ", + OPENAI_TTS_BASE_URL: "", }, + ); + expect(result?.dictationStt).toEqual({ apiKey: "stt", baseUrl: "https://stt/v1" }); + expect(result?.stt).toEqual({ apiKey: "stt", baseUrl: "https://stt/v1" }); + expect(result?.tts).toEqual({ + apiKey: "global", + baseUrl: "https://global/v1", + model: "tts-1", + voice: "alloy", + responseFormat: "pcm", }); - - expect(resolved).toBeUndefined(); }); - - test("applies trimmed OPENAI_API_KEY to both STT and TTS", () => { - const persisted = PersistedConfigSchema.parse({}); - const env = { - OPENAI_API_KEY: " sk-test ", - } as NodeJS.ProcessEnv; - - const resolved = resolveOpenAiSpeechConfig({ env, persisted, providers: ALL_OPENAI }); - - expect(resolved?.stt?.apiKey).toBe("sk-test"); - expect(resolved?.tts?.apiKey).toBe("sk-test"); - }); - - test("resolves distinct endpoints for STT and TTS", () => { - const persisted = PersistedConfigSchema.parse({ - providers: { - openai: { - stt: { - apiKey: "stt-key", - baseUrl: " https://stt.example.com/v1 ", + test("each site wins over endpoint and global env credentials and URL", () => { + const result = resolve( + { + features: { + dictation: { + stt: { + provider: "openai", + apiKey: "dictation", + baseUrl: "https://dictation/v1", + model: "dictation-model", + confidenceThreshold: -1, + }, }, - tts: { - apiKey: "tts-key", - baseUrl: " https://tts.example.com/v1 ", + voiceMode: { + stt: { + provider: "openai", + apiKey: "voice", + baseUrl: "https://voice/v1", + model: "voice-model", + confidenceThreshold: -2, + }, + tts: { + provider: "openai", + apiKey: "tts", + baseUrl: "https://tts/v1", + model: "kokoro-v1:Q8_0", + voice: "Aiden", + }, }, }, }, + { + OPENAI_API_KEY: "global", + OPENAI_BASE_URL: "https://global/v1", + OPENAI_STT_API_KEY: "endpoint", + OPENAI_STT_BASE_URL: "https://endpoint/v1", + OPENAI_TTS_API_KEY: "endpoint", + OPENAI_TTS_BASE_URL: "https://endpoint/v1", + }, + ); + expect(result?.dictationStt).toEqual({ + apiKey: "dictation", + baseUrl: "https://dictation/v1", + model: "dictation-model", + confidenceThreshold: -1, }); - - const resolved = resolveOpenAiSpeechConfig({ - env: {} as NodeJS.ProcessEnv, - persisted, - providers: ALL_OPENAI, + expect(result?.stt).toEqual({ + apiKey: "voice", + baseUrl: "https://voice/v1", + model: "voice-model", + confidenceThreshold: -2, + }); + expect(result?.tts).toEqual({ + apiKey: "tts", + baseUrl: "https://tts/v1", + model: "kokoro-v1:Q8_0", + voice: "Aiden", + responseFormat: "pcm", }); - - expect(resolved?.stt?.apiKey).toBe("stt-key"); - expect(resolved?.stt?.baseUrl).toBe("https://stt.example.com/v1"); - expect(resolved?.tts?.apiKey).toBe("tts-key"); - expect(resolved?.tts?.baseUrl).toBe("https://tts.example.com/v1"); }); - - test("prefers nested STT/TTS config over env and global fallbacks", () => { - const persisted = PersistedConfigSchema.parse({ - providers: { - openai: { - apiKey: "fallback-config-key", - baseUrl: "https://global-config.example.com/v1", - stt: { apiKey: "stt-config-key", baseUrl: " https://stt.example.com/v1 " }, - tts: { apiKey: "tts-config-key", baseUrl: " https://tts.example.com/v1 " }, + test("env overrides model, voice and threshold at every OpenAI site", () => { + const result = resolve( + { + features: { + dictation: { stt: { provider: "openai", model: "dictation", confidenceThreshold: -1 } }, + voiceMode: { + stt: { provider: "openai", model: "voice", confidenceThreshold: -2 }, + tts: { provider: "openai", model: "configured", voice: "Aiden" }, + }, }, }, - }); - const env = { - OPENAI_API_KEY: "env-key", - OPENAI_STT_API_KEY: "stt-env-key", - OPENAI_STT_BASE_URL: "https://stt-env.example.com/v1", - OPENAI_TTS_API_KEY: "tts-env-key", - OPENAI_TTS_BASE_URL: "https://tts-env.example.com/v1", - OPENAI_BASE_URL: "https://env.example.com/v1", - } as NodeJS.ProcessEnv; - - const resolved = resolveOpenAiSpeechConfig({ env, persisted, providers: ALL_OPENAI }); - - expect(resolved?.stt?.apiKey).toBe("stt-config-key"); - expect(resolved?.stt?.baseUrl).toBe("https://stt.example.com/v1"); - expect(resolved?.tts?.apiKey).toBe("tts-config-key"); - expect(resolved?.tts?.baseUrl).toBe("https://tts.example.com/v1"); - }); - - test("uses STT/TTS env config when nested config is unset", () => { - const persisted = PersistedConfigSchema.parse({}); - const env = { - OPENAI_API_KEY: "sk-test", - OPENAI_STT_API_KEY: "stt-env-key", - OPENAI_STT_BASE_URL: " https://stt-env.example.com/v1 ", - OPENAI_TTS_API_KEY: "tts-env-key", - OPENAI_TTS_BASE_URL: " https://tts-env.example.com/v1 ", - OPENAI_BASE_URL: "https://env.example.com/v1", - } as NodeJS.ProcessEnv; - - const resolved = resolveOpenAiSpeechConfig({ env, persisted, providers: ALL_OPENAI }); - - expect(resolved?.stt?.apiKey).toBe("stt-env-key"); - expect(resolved?.stt?.baseUrl).toBe("https://stt-env.example.com/v1"); - expect(resolved?.tts?.apiKey).toBe("tts-env-key"); - expect(resolved?.tts?.baseUrl).toBe("https://tts-env.example.com/v1"); - }); - - test("falls back to global OpenAI config for both STT and TTS", () => { - const persisted = PersistedConfigSchema.parse({ - providers: { - openai: { - apiKey: "fallback-config-key", - baseUrl: " https://global-config.example.com/v1 ", - }, + { + OPENAI_API_KEY: "key", + STT_MODEL: "override", + TTS_MODEL: "tts-override", + TTS_VOICE: "Nova", + STT_CONFIDENCE_THRESHOLD: "-3", }, + ); + expect(result?.dictationStt).toEqual({ + apiKey: "key", + model: "override", + confidenceThreshold: -3, }); - const env = {} as NodeJS.ProcessEnv; - - const resolved = resolveOpenAiSpeechConfig({ env, persisted, providers: ALL_OPENAI }); - - expect(resolved?.stt?.apiKey).toBe("fallback-config-key"); - expect(resolved?.stt?.baseUrl).toBe("https://global-config.example.com/v1"); - expect(resolved?.tts?.apiKey).toBe("fallback-config-key"); - expect(resolved?.tts?.baseUrl).toBe("https://global-config.example.com/v1"); + expect(result?.stt).toEqual({ apiKey: "key", model: "override", confidenceThreshold: -3 }); + expect(result?.tts).toMatchObject({ model: "tts-override", voice: "Nova" }); }); - - test("falls back to global OpenAI env config when feature inputs are unset", () => { - const persisted = PersistedConfigSchema.parse({}); - const env = { - OPENAI_API_KEY: "env-key", - OPENAI_BASE_URL: " https://env.example.com/v1 ", - } as NodeJS.ProcessEnv; - - const resolved = resolveOpenAiSpeechConfig({ env, persisted, providers: ALL_OPENAI }); - - expect(resolved?.stt?.apiKey).toBe("env-key"); - expect(resolved?.stt?.baseUrl).toBe("https://env.example.com/v1"); - expect(resolved?.tts?.apiKey).toBe("env-key"); - expect(resolved?.tts?.baseUrl).toBe("https://env.example.com/v1"); + test("only configures endpoints with credentials and ignores unused TTS options", () => { + const result = resolve( + { features: { dictation: { stt: { provider: "openai", apiKey: "dictation" } } } }, + { TTS_VOICE: "arbitrary", TTS_MODEL: "arbitrary" }, + ); + expect(result).toEqual({ dictationStt: { apiKey: "dictation" } }); }); - - test("ignores empty endpoint env vars and falls back to OPENAI_API_KEY", () => { - const persisted = PersistedConfigSchema.parse({}); - const env = { - OPENAI_API_KEY: "global-key", - OPENAI_STT_API_KEY: "", - OPENAI_STT_BASE_URL: " ", - OPENAI_TTS_API_KEY: "", - OPENAI_TTS_BASE_URL: "", - } as NodeJS.ProcessEnv; - - const resolved = resolveOpenAiSpeechConfig({ env, persisted, providers: ALL_OPENAI }); - - expect(resolved?.stt?.apiKey).toBe("global-key"); - expect(resolved?.tts?.apiKey).toBe("global-key"); - }); - - test("omits TTS when only an STT key is configured", () => { - const persisted = PersistedConfigSchema.parse({ - providers: { - openai: { - stt: { apiKey: "stt-only-key" }, - }, - }, + test("does not borrow another site's key", () => { + const result = resolve({ + features: { voiceMode: { stt: { provider: "openai", apiKey: "voice-only" } } }, }); - - const resolved = resolveOpenAiSpeechConfig({ - env: {} as NodeJS.ProcessEnv, - persisted, - providers: ALL_OPENAI, - }); - - expect(resolved?.stt?.apiKey).toBe("stt-only-key"); - expect(resolved?.tts).toBeUndefined(); - }); - - test("resolves STT even when an unused TTS env var is invalid", () => { - const persisted = PersistedConfigSchema.parse({ - providers: { - openai: { - stt: { apiKey: "stt-only-key" }, - }, - }, - }); - const env = { TTS_VOICE: "not-a-real-voice", TTS_MODEL: "bogus-model" } as NodeJS.ProcessEnv; - - const resolved = resolveOpenAiSpeechConfig({ env, persisted, providers: ALL_OPENAI }); - - expect(resolved?.stt?.apiKey).toBe("stt-only-key"); - expect(resolved?.tts).toBeUndefined(); - }); - - test("sends the model and voice an OpenAI-compatible server serves, as written", () => { - const persisted = PersistedConfigSchema.parse({ - providers: { - openai: { tts: { apiKey: "tts-key", baseUrl: "https://tts.example.com/v1" } }, - }, - features: { - voiceMode: { tts: { provider: "openai", model: "kokoro-v1:Q8_0", voice: "Aiden" } }, - }, - }); - - const resolved = resolveOpenAiSpeechConfig({ - env: {} as NodeJS.ProcessEnv, - persisted, - providers: ALL_OPENAI, - }); - - expect(resolved?.tts?.model).toBe("kokoro-v1:Q8_0"); - expect(resolved?.tts?.voice).toBe("Aiden"); - }); - - test("accepts OpenAI's own gpt-4o-mini-tts model", () => { - const persisted = PersistedConfigSchema.parse({ - providers: { openai: { tts: { apiKey: "tts-key" } } }, - features: { voiceMode: { tts: { provider: "openai", model: "gpt-4o-mini-tts" } } }, - }); - - const resolved = resolveOpenAiSpeechConfig({ - env: {} as NodeJS.ProcessEnv, - persisted, - providers: ALL_OPENAI, - }); - - expect(resolved?.tts?.model).toBe("gpt-4o-mini-tts"); - expect(resolved?.tts?.voice).toBe("alloy"); + expect(result?.dictationStt).toBeUndefined(); + expect(result?.stt).toEqual({ apiKey: "voice-only" }); }); }); diff --git a/packages/server/src/server/speech/providers/openai/config.ts b/packages/server/src/server/speech/providers/openai/config.ts index efb25bffaf..37813571f4 100644 --- a/packages/server/src/server/speech/providers/openai/config.ts +++ b/packages/server/src/server/speech/providers/openai/config.ts @@ -6,147 +6,61 @@ import type { STTConfig } from "./stt.js"; import type { TTSConfig } from "./tts.js"; export const DEFAULT_OPENAI_TTS_MODEL = "tts-1"; +export type OpenAiSttFeature = "dictation" | "voiceMode"; export interface OpenAiSpeechProviderConfig { - stt?: Partial & { apiKey?: string }; - tts?: Partial & { apiKey?: string }; + dictationStt?: Partial; + stt?: Partial; + tts?: Partial; } -const NumberLikeSchema = z.union([z.number(), z.string().trim().min(1)]); +const OptionalFiniteNumberSchema = z + .union([z.number(), z.string().trim().min(1)]) + .pipe(z.coerce.number().finite()) + .optional(); -const OptionalFiniteNumberSchema = NumberLikeSchema.pipe( - z.coerce.number().finite(), -).optional(); - -const OptionalTrimmedStringSchema = z - .string() - .trim() - .optional() - .transform((value) => (value && value.length > 0 ? value : undefined)); - -// Endpoint credentials only — plain trimmed strings, so this never throws on a -// malformed value. The STT/TTS option groups parse separately and only for the -// endpoint that is actually configured, so a stale env var for an unused endpoint -// (e.g. a leftover TTS_VOICE in an STT-only setup) can't break the other one. -const OpenAiEndpointKeysSchema = z.object({ - sttApiKey: OptionalTrimmedStringSchema, - sttBaseUrl: OptionalTrimmedStringSchema, - ttsApiKey: OptionalTrimmedStringSchema, - ttsBaseUrl: OptionalTrimmedStringSchema, -}); - -const OpenAiSttOptionsSchema = z.object({ - sttConfidenceThreshold: OptionalFiniteNumberSchema, - sttModel: OptionalTrimmedStringSchema, -}); - -// Voice and model are free-form so a self-hosted server can use its own ids; the -// OpenAI catalog is only the default. Both are sent as written, because the -// server owns those ids and their case (rosters use ids like kokoro-v1:Q8_0). -const OpenAiTtsOptionsSchema = z.object({ - ttsVoice: z.string().trim().default("alloy"), - ttsModel: z.string().trim().default(DEFAULT_OPENAI_TTS_MODEL), -}); - -function isOpenAiProviderActive(provider: { enabled?: boolean; provider: string }): boolean { - return provider.enabled !== false && provider.provider === "openai"; +function firstString(...values: (string | undefined)[]): string | undefined { + return values.map((value) => value?.trim()).find((value) => Boolean(value)); } -function pickIfOpenAi( - provider: { enabled?: boolean; provider: string }, - value: T | undefined, -): T | undefined { - return isOpenAiProviderActive(provider) ? value : undefined; -} +type Features = NonNullable; +type SttSite = NonNullable["stt"]>; +type TtsSite = NonNullable["tts"]>; -function firstDefined(values: Array): T | undefined { - for (const value of values) { - if (value === undefined || value === null) { - continue; - } - // Empty/whitespace env vars (e.g. a copied .env.example with OPENAI_STT_API_KEY=) - // must not shadow a later fallback such as OPENAI_API_KEY. - if (typeof value === "string" && value.trim().length === 0) { - continue; - } - return value; - } - return undefined; -} - -function buildOpenAiSttInput(params: { - env: NodeJS.ProcessEnv; - persisted: PersistedConfig; - providers: RequestedSpeechProviders; -}): Record { - const { env, persisted, providers } = params; +function resolveStt( + site: SttSite | undefined, + env: NodeJS.ProcessEnv, +): Partial | undefined { + const config = site?.provider === "openai" ? site : undefined; + const apiKey = firstString(config?.apiKey, env.OPENAI_STT_API_KEY, env.OPENAI_API_KEY); + if (!apiKey) return undefined; + const baseUrl = firstString(config?.baseUrl, env.OPENAI_STT_BASE_URL, env.OPENAI_BASE_URL); + const model = firstString(env.STT_MODEL, config?.model); + const confidenceThreshold = OptionalFiniteNumberSchema.parse( + firstString(env.STT_CONFIDENCE_THRESHOLD) ?? config?.confidenceThreshold, + ); return { - sttConfidenceThreshold: firstDefined([ - env.STT_CONFIDENCE_THRESHOLD, - persisted.features?.dictation?.stt?.confidenceThreshold, - ]), - sttModel: firstDefined([ - env.STT_MODEL, - pickIfOpenAi(providers.voiceStt, persisted.features?.voiceMode?.stt?.model), - pickIfOpenAi(providers.dictationStt, persisted.features?.dictation?.stt?.model), - ]), + apiKey, + ...(baseUrl ? { baseUrl } : {}), + ...(model ? { model } : {}), + ...(confidenceThreshold !== undefined ? { confidenceThreshold } : {}), }; } -function buildOpenAiTtsInput(params: { - env: NodeJS.ProcessEnv; - persisted: PersistedConfig; - providers: RequestedSpeechProviders; -}): Record { - const { env, persisted, providers } = params; +function resolveTts( + site: TtsSite | undefined, + env: NodeJS.ProcessEnv, +): Partial | undefined { + const config = site?.provider === "openai" ? site : undefined; + const apiKey = firstString(config?.apiKey, env.OPENAI_TTS_API_KEY, env.OPENAI_API_KEY); + if (!apiKey) return undefined; + const baseUrl = firstString(config?.baseUrl, env.OPENAI_TTS_BASE_URL, env.OPENAI_BASE_URL); return { - ttsVoice: firstDefined([ - env.TTS_VOICE, - pickIfOpenAi(providers.voiceTts, persisted.features?.voiceMode?.tts?.voice), - "alloy", - ]), - ttsModel: firstDefined([ - env.TTS_MODEL, - pickIfOpenAi(providers.voiceTts, persisted.features?.voiceMode?.tts?.model), - DEFAULT_OPENAI_TTS_MODEL, - ]), - }; -} - -function buildOpenAiResolutionInput(params: { - env: NodeJS.ProcessEnv; - persisted: PersistedConfig; - providers: RequestedSpeechProviders; -}): Record { - const { env } = params; - const openai = params.persisted.providers?.openai; - return { - sttApiKey: firstDefined([ - openai?.stt?.apiKey, - env.OPENAI_STT_API_KEY, - openai?.apiKey, - env.OPENAI_API_KEY, - ]), - sttBaseUrl: firstDefined([ - openai?.stt?.baseUrl, - env.OPENAI_STT_BASE_URL, - openai?.baseUrl, - env.OPENAI_BASE_URL, - ]), - ttsApiKey: firstDefined([ - openai?.tts?.apiKey, - env.OPENAI_TTS_API_KEY, - openai?.apiKey, - env.OPENAI_API_KEY, - ]), - ttsBaseUrl: firstDefined([ - openai?.tts?.baseUrl, - env.OPENAI_TTS_BASE_URL, - openai?.baseUrl, - env.OPENAI_BASE_URL, - ]), - ...buildOpenAiSttInput(params), - ...buildOpenAiTtsInput(params), + apiKey, + ...(baseUrl ? { baseUrl } : {}), + model: firstString(env.TTS_MODEL, config?.model) ?? DEFAULT_OPENAI_TTS_MODEL, + voice: firstString(env.TTS_VOICE, config?.voice) ?? "alloy", + responseFormat: "pcm", }; } @@ -155,46 +69,14 @@ export function resolveOpenAiSpeechConfig(params: { persisted: PersistedConfig; providers: RequestedSpeechProviders; }): OpenAiSpeechProviderConfig | undefined { - const input = buildOpenAiResolutionInput(params); - const keys = OpenAiEndpointKeysSchema.parse(input); - - if (!keys.sttApiKey && !keys.ttsApiKey) { - return undefined; - } - + const { env, persisted } = params; + const dictationStt = resolveStt(persisted.features?.dictation?.stt, env); + const stt = resolveStt(persisted.features?.voiceMode?.stt, env); + const tts = resolveTts(persisted.features?.voiceMode?.tts, env); + if (!dictationStt && !stt && !tts) return undefined; return { - ...(keys.sttApiKey ? { stt: buildSttConfig(keys.sttApiKey, keys.sttBaseUrl, input) } : {}), - ...(keys.ttsApiKey ? { tts: buildTtsConfig(keys.ttsApiKey, keys.ttsBaseUrl, input) } : {}), - }; -} - -function buildSttConfig( - apiKey: string, - baseUrl: string | undefined, - input: Record, -): OpenAiSpeechProviderConfig["stt"] { - const options = OpenAiSttOptionsSchema.parse(input); - return { - apiKey, - ...(baseUrl ? { baseUrl } : {}), - ...(options.sttConfidenceThreshold !== undefined - ? { confidenceThreshold: options.sttConfidenceThreshold } - : {}), - ...(options.sttModel ? { model: options.sttModel } : {}), - }; -} - -function buildTtsConfig( - apiKey: string, - baseUrl: string | undefined, - input: Record, -): OpenAiSpeechProviderConfig["tts"] { - const options = OpenAiTtsOptionsSchema.parse(input); - return { - apiKey, - ...(baseUrl ? { baseUrl } : {}), - voice: options.ttsVoice, - model: options.ttsModel, - responseFormat: "pcm", + ...(dictationStt ? { dictationStt } : {}), + ...(stt ? { stt } : {}), + ...(tts ? { tts } : {}), }; } diff --git a/packages/server/src/server/speech/providers/openai/runtime.test.ts b/packages/server/src/server/speech/providers/openai/runtime.test.ts index c947d86600..81b14ecaa0 100644 --- a/packages/server/src/server/speech/providers/openai/runtime.test.ts +++ b/packages/server/src/server/speech/providers/openai/runtime.test.ts @@ -1,6 +1,12 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; import pino from "pino"; -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test } from "vitest"; +import { mergeSpeechSettingsPatch } from "../../speech-settings.js"; +import { PersistedConfigSchema } from "../../../persisted-config.js"; +import type { SpeechToTextProvider } from "../../speech-provider.js"; +import { resolveOpenAiSpeechConfig } from "./config.js"; import { initializeOpenAiSpeechServices } from "./runtime.js"; import { OpenAISTT } from "./stt.js"; import { OpenAITTS } from "./tts.js"; @@ -16,6 +22,7 @@ describe("initializeOpenAiSpeechServices", () => { }, openaiConfig: { stt: { apiKey: "sk-test" }, + dictationStt: { apiKey: "sk-test" }, tts: { apiKey: "sk-test" }, }, existing: { @@ -32,3 +39,116 @@ describe("initializeOpenAiSpeechServices", () => { expect(services.ttsService).toBeInstanceOf(OpenAITTS); }); }); + +// An OpenAI-compatible transcription endpoint that records the model each request asks for. +async function startTranscriptionServer(): Promise<{ + server: Server; + baseUrl: string; + models: string[]; + keys: (string | undefined)[]; + paths: (string | undefined)[]; +}> { + const models: string[] = []; + const keys: (string | undefined)[] = []; + const paths: (string | undefined)[] = []; + const server = createServer((request, response) => { + let body = ""; + request.on("data", (chunk: Buffer) => { + body += chunk.toString("latin1"); + }); + request.on("end", () => { + const model = /name="model"\r\n\r\n([^\r]*)/.exec(body)?.[1]; + models.push(model ?? ""); + keys.push(request.headers.authorization); + paths.push(request.url); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ text: "hello" })); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { server, baseUrl: `http://127.0.0.1:${port}/v1`, models, keys, paths }; +} + +async function transcribeOnce(stt: SpeechToTextProvider | null): Promise { + if (!stt) throw new Error("STT service was not created"); + const session = stt.createSession({ logger: pino({ level: "silent" }), language: "en" }); + const transcript = new Promise((resolve, reject) => { + session.on("transcript", (event) => { + if (event.isFinal) resolve(); + }); + session.on("error", (error) => + reject(error instanceof Error ? error : new Error(String(error))), + ); + }); + await session.connect(); + session.appendPcm16(Buffer.alloc(3200)); + session.commit(); + await transcript; + session.close(); +} + +describe("OpenAI speech-to-text models", () => { + let server: Server | null = null; + + afterEach(async () => { + await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())); + server = null; + }); + + test("dictation and voice mode each transcribe with the model chosen for them", async () => { + const endpoint = await startTranscriptionServer(); + server = endpoint.server; + let persisted = PersistedConfigSchema.parse({ + features: { + dictation: { + stt: { + provider: "openai", + apiKey: "dictation-key", + baseUrl: `${endpoint.baseUrl}/dictation`, + model: "gpt-4o-mini-transcribe", + }, + }, + voiceMode: { + enabled: true, + stt: { + provider: "openai", + apiKey: "listening-key", + baseUrl: `${endpoint.baseUrl}/listening`, + model: "whisper-1", + }, + }, + }, + }); + persisted = mergeSpeechSettingsPatch(persisted, { + voiceMode: { stt: { provider: "openai", model: "whisper-1" } }, + }); + const providers = { + dictationStt: { provider: "openai", explicit: true }, + voiceTurnDetection: { provider: "local", explicit: false, enabled: false }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "local", explicit: false }, + } as const; + + const services = initializeOpenAiSpeechServices({ + providers, + openaiConfig: resolveOpenAiSpeechConfig({ env: {}, persisted, providers }), + existing: { + turnDetectionService: null, + sttService: null, + ttsService: null, + dictationSttService: null, + }, + logger: pino({ level: "silent" }), + }); + await transcribeOnce(services.dictationSttService); + await transcribeOnce(services.sttService); + + expect(endpoint.models).toEqual(["gpt-4o-mini-transcribe", "whisper-1"]); + expect(endpoint.keys).toEqual(["Bearer dictation-key", "Bearer listening-key"]); + expect(endpoint.paths).toEqual([ + "/v1/dictation/audio/transcriptions", + "/v1/listening/audio/transcriptions", + ]); + }); +}); diff --git a/packages/server/src/server/speech/providers/openai/runtime.ts b/packages/server/src/server/speech/providers/openai/runtime.ts index a1231ddbe6..ce11c8879c 100644 --- a/packages/server/src/server/speech/providers/openai/runtime.ts +++ b/packages/server/src/server/speech/providers/openai/runtime.ts @@ -3,7 +3,11 @@ import type { Logger } from "pino"; import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js"; import type { RequestedSpeechProviders } from "../../speech-types.js"; import type { TurnDetectionProvider } from "../../turn-detection-provider.js"; -import { DEFAULT_OPENAI_TTS_MODEL, type OpenAiSpeechProviderConfig } from "./config.js"; +import { + DEFAULT_OPENAI_TTS_MODEL, + type OpenAiSpeechProviderConfig, + type OpenAiSttFeature, +} from "./config.js"; import { OpenAISTT } from "./stt.js"; import { OpenAITTS } from "./tts.js"; @@ -33,7 +37,7 @@ function resolveOpenAiCredentials( return { openaiSttApiKey: sttApiKey, openaiTtsApiKey: openaiConfig?.tts?.apiKey, - openaiDictationApiKey: sttApiKey, + openaiDictationApiKey: openaiConfig?.dictationStt?.apiKey, }; } @@ -94,13 +98,16 @@ export function validateOpenAiCredentialRequirements(params: { } } -function createOpenAiStt( - apiKey: string, - openaiConfig: OpenAiSpeechProviderConfig | undefined, - logger: Logger, -): SpeechToTextProvider { - const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {}; - return new OpenAISTT({ apiKey, ...sttConfig }, logger); +function createOpenAiStt(params: { + apiKey: string; + openaiConfig: OpenAiSpeechProviderConfig | undefined; + feature: OpenAiSttFeature; + logger: Logger; +}): SpeechToTextProvider { + const site = + params.feature === "dictation" ? params.openaiConfig?.dictationStt : params.openaiConfig?.stt; + const { apiKey: _sttApiKey, ...sttConfig } = site ?? {}; + return new OpenAISTT({ apiKey: params.apiKey, ...sttConfig }, params.logger); } function createOpenAiTts( @@ -154,7 +161,12 @@ export function initializeOpenAiSpeechServices(params: { logger.info("OpenAI speech provider initialized"); if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) { - sttService = createOpenAiStt(openAiCredentials.openaiSttApiKey, openaiConfig, logger); + sttService = createOpenAiStt({ + apiKey: openAiCredentials.openaiSttApiKey, + openaiConfig, + feature: "voiceMode", + logger, + }); } if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) { @@ -162,11 +174,12 @@ export function initializeOpenAiSpeechServices(params: { } if (needsOpenAiDictation && openAiCredentials.openaiDictationApiKey) { - dictationSttService = createOpenAiStt( - openAiCredentials.openaiDictationApiKey, + dictationSttService = createOpenAiStt({ + apiKey: openAiCredentials.openaiDictationApiKey, openaiConfig, + feature: "dictation", logger, - ); + }); } } else if (needsAnyOpenAi) { // validateOpenAiCredentialRequirements already warned about missing credentials diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts index 0ee9df888a..c83d2c45fa 100644 --- a/packages/server/src/server/speech/speech-config-resolver.test.ts +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -6,7 +6,7 @@ import { PersistedConfigSchema } from "../persisted-config.js"; import { resolveSpeechConfig } from "./speech-config-resolver.js"; describe("resolveSpeechConfig", () => { - test("resolves local-first defaults without env overrides", () => { + test("resolves local-first defaults with voice mode off until turned on", () => { const paseoHome = "/tmp/paseo-home"; const persisted = PersistedConfigSchema.parse({}); const env = {} as NodeJS.ProcessEnv; @@ -26,17 +26,17 @@ describe("resolveSpeechConfig", () => { expect(result.speech.providers.voiceTurnDetection).toEqual({ provider: "local", explicit: false, - enabled: true, + enabled: false, }); expect(result.speech.providers.voiceStt).toEqual({ provider: "local", explicit: false, - enabled: true, + enabled: false, }); expect(result.speech.providers.voiceTts).toEqual({ provider: "local", explicit: false, - enabled: true, + enabled: false, }); expect(result.speech.local).toEqual({ modelsDir: path.join(paseoHome, "models", "local-speech"), @@ -61,13 +61,11 @@ describe("resolveSpeechConfig", () => { const persisted = PersistedConfigSchema.parse({ features: { voiceMode: { + enabled: true, turnDetection: { provider: "local" }, - stt: { provider: "openai", model: "gpt-4o-transcribe" }, + stt: { provider: "openai", apiKey: "persisted-key", model: "gpt-4o-transcribe" }, }, }, - providers: { - openai: { apiKey: "persisted-key" }, - }, }); const env = { PASEO_DICTATION_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8", @@ -130,40 +128,45 @@ describe("resolveSpeechConfig", () => { voice: "pt", }); expect(result.openai?.stt?.apiKey).toBe("persisted-key"); - expect(result.openai?.tts?.apiKey).toBe("persisted-key"); + expect(result.openai?.tts?.apiKey).toBe("env-key"); expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); }); - test("resolves STT language from env, settings, and voice-to-dictation fallback", () => { - const persisted = PersistedConfigSchema.parse({ - features: { - dictation: { - stt: { - language: "fr", + test.each(["local", "openai"])( + "resolves STT language from env, settings, and voice-to-dictation fallback (%s)", + (provider) => { + const persisted = PersistedConfigSchema.parse({ + features: { + dictation: { + stt: { + provider, + language: "fr", + }, + }, + voiceMode: { + stt: { + provider, + language: "de", + }, }, }, - voiceMode: { - stt: { - language: "de", - }, - }, - }, - }); + }); - const result = resolveSpeechConfig({ - paseoHome: "/tmp/paseo-home", - env: { - PASEO_DICTATION_LANGUAGE: "es", - PASEO_VOICE_LANGUAGE: " ", - } as NodeJS.ProcessEnv, - persisted, - }); + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env: { + PASEO_DICTATION_LANGUAGE: "es", + PASEO_VOICE_LANGUAGE: " ", + } as NodeJS.ProcessEnv, + persisted, + }); - expect(result.speech.sttLanguages).toEqual({ - dictation: "es", - voice: "es", - }); - }); + expect(result.speech.sttLanguages).toEqual({ + dictation: "es", + voice: "es", + }); + }, + ); test("respects disabled dictation and voice mode feature flags", () => { const persisted = PersistedConfigSchema.parse({ diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts index 95c78707a5..1257f46d4d 100644 --- a/packages/server/src/server/speech/speech-config-resolver.ts +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -43,8 +43,13 @@ const RequestedSpeechProvidersSchema = z.object({ voiceTts: OptionalSpeechProviderSchema.default("local"), }); -function resolveOptionalBooleanFlag(value: unknown): boolean { - return OptionalBooleanFlagSchema.parse(value) ?? true; +// Dictation is on unless turned off. Voice mode is opt-in: it downloads its own models and +// runs a hidden agent, so a host only pays for it once someone turns it on. +export const DICTATION_ENABLED_BY_DEFAULT = true; +export const VOICE_MODE_ENABLED_BY_DEFAULT = false; + +function resolveOptionalBooleanFlag(value: unknown, fallback: boolean): boolean { + return OptionalBooleanFlagSchema.parse(value) ?? fallback; } interface FeatureProviderInputs { @@ -70,6 +75,7 @@ function buildFeatureProviderInputs(params: { params.env.PASEO_VOICE_MODE_ENABLED, params.persisted.features?.voiceMode?.enabled, ]), + VOICE_MODE_ENABLED_BY_DEFAULT, ); return { dictationStt: { @@ -82,6 +88,7 @@ function buildFeatureProviderInputs(params: { params.env.PASEO_DICTATION_ENABLED, params.persisted.features?.dictation?.enabled, ]), + DICTATION_ENABLED_BY_DEFAULT, ), }, voiceTurnDetection: { @@ -119,7 +126,7 @@ function buildRequestedFeatureProvider( }; } -function resolveRequestedSpeechProviders(params: { +export function resolveRequestedSpeechProviders(params: { env: NodeJS.ProcessEnv; persisted: PersistedConfig; }): RequestedSpeechProviders { @@ -173,6 +180,7 @@ export function resolveSpeechConfig(params: { openai, speech: { providers, + localModelsDir: local.modelsDir, sttLanguages: local.sttLanguages, ...(local.local ? { local: local.local } : {}), }, diff --git a/packages/server/src/server/speech/speech-runtime.test.ts b/packages/server/src/server/speech/speech-runtime.test.ts index c9c21b3e89..cd3580e64a 100644 --- a/packages/server/src/server/speech/speech-runtime.test.ts +++ b/packages/server/src/server/speech/speech-runtime.test.ts @@ -2,13 +2,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import pino from "pino"; import type { PaseoSpeechConfig } from "../bootstrap.js"; +import type { LocalSpeechModelId } from "./providers/local/models.js"; import type { InitializedLocalSpeech } from "./providers/local/runtime.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js"; import type { TurnDetectionProvider } from "./turn-detection-provider.js"; -import { createSpeechService } from "./speech-runtime.js"; +import { createSpeechService, LocalSpeechModelInUseError } from "./speech-runtime.js"; -const { ensureLocalSpeechModelsMock, initializeLocalSpeechServicesMock } = vi.hoisted(() => ({ - ensureLocalSpeechModelsMock: vi.fn(async () => ({})), +const { ensureSherpaOnnxModelMock, initializeLocalSpeechServicesMock } = vi.hoisted(() => ({ + ensureSherpaOnnxModelMock: vi.fn(async (_options: { signal?: AbortSignal }) => ""), initializeLocalSpeechServicesMock: vi.fn<(args: unknown) => Promise>(), })); @@ -34,9 +35,10 @@ vi.mock("./providers/openai/runtime.js", () => ({ validateOpenAiCredentialRequirements: () => {}, })); -vi.mock("./providers/local/models.js", () => ({ - ensureLocalSpeechModels: ensureLocalSpeechModelsMock, - listMissingLocalSpeechModels: vi.fn(async ({ modelIds }: { modelIds: string[] }) => modelIds), +vi.mock("./providers/local/sherpa/model-downloader.js", () => ({ + ensureSherpaOnnxModel: ensureSherpaOnnxModelMock, + isSherpaOnnxModelInstalled: vi.fn(async () => false), + deleteSherpaOnnxModel: vi.fn(async () => {}), })); function createStubStt(id: string): SpeechToTextProvider { @@ -75,6 +77,7 @@ function rejectWhenAborted(signal?: AbortSignal): Promise { function createSpeechConfig(providers: PaseoSpeechConfig["providers"]): PaseoSpeechConfig { return { providers, + localModelsDir: "/tmp/missing-local-speech-models", sttLanguages: { dictation: "en", voice: "en", @@ -84,8 +87,8 @@ function createSpeechConfig(providers: PaseoSpeechConfig["providers"]): PaseoSpe describe("createSpeechService readiness", () => { beforeEach(() => { - ensureLocalSpeechModelsMock.mockReset(); - ensureLocalSpeechModelsMock.mockResolvedValue({}); + ensureSherpaOnnxModelMock.mockReset(); + ensureSherpaOnnxModelMock.mockResolvedValue(""); initializeLocalSpeechServicesMock.mockReset(); }); @@ -169,7 +172,7 @@ describe("createSpeechService readiness", () => { it("aborts and joins an in-flight model download when stopped", async () => { let downloadSignal: AbortSignal | undefined; - ensureLocalSpeechModelsMock.mockImplementation(({ signal }: { signal?: AbortSignal }) => { + ensureSherpaOnnxModelMock.mockImplementation(({ signal }: { signal?: AbortSignal }) => { downloadSignal = signal; return rejectWhenAborted(signal); }); @@ -201,10 +204,104 @@ describe("createSpeechService readiness", () => { }); runtime.start(); await runtime.ready; - expect(ensureLocalSpeechModelsMock).toHaveBeenCalledOnce(); + expect(ensureSherpaOnnxModelMock).toHaveBeenCalledOnce(); await runtime.stop(); expect(downloadSignal?.aborted).toBe(true); }); }); + +describe("createSpeechService settings changes", () => { + const dictationOnly = createSpeechConfig({ + dictationStt: { provider: "local", enabled: true, explicit: true }, + voiceTurnDetection: { provider: "local", enabled: false, explicit: true }, + voiceStt: { provider: "local", enabled: false, explicit: true }, + voiceTts: { provider: "local", enabled: false, explicit: true }, + }); + const nothingEnabled = createSpeechConfig({ + dictationStt: { provider: "local", enabled: false, explicit: true }, + voiceTurnDetection: { provider: "local", enabled: false, explicit: true }, + voiceStt: { provider: "local", enabled: false, explicit: true }, + voiceTts: { provider: "local", enabled: false, explicit: true }, + }); + + function localSpeech(input: { + dictationStt: SpeechToTextProvider | null; + defaultModelIds: LocalSpeechModelId[]; + }): InitializedLocalSpeech { + return { + turnDetectionService: null, + sttService: null, + ttsService: null, + dictationSttService: input.dictationStt, + localVoiceTtsProvider: null, + localModelConfig: { + modelsDir: "/tmp/missing-local-speech-models", + defaultModelIds: input.defaultModelIds, + }, + availability: { configured: true, modelsDir: "/tmp/missing-local-speech-models" }, + cleanup: () => {}, + }; + } + + beforeEach(() => { + ensureSherpaOnnxModelMock.mockReset(); + ensureSherpaOnnxModelMock.mockResolvedValue(""); + initializeLocalSpeechServicesMock.mockReset(); + }); + + it("applies new providers without a restart", async () => { + initializeLocalSpeechServicesMock.mockResolvedValue( + localSpeech({ dictationStt: createStubStt("dictation-local"), defaultModelIds: [] }), + ); + const runtime = createSpeechService({ + logger: pino({ level: "silent" }), + speechConfig: dictationOnly, + }); + runtime.start(); + await runtime.ready; + expect(runtime.getReadiness().dictation.available).toBe(true); + + const disabled = new Promise((resolve) => { + runtime.onReadinessChange((snapshot) => { + if (snapshot.dictation.reasonCode === "disabled") resolve(); + }); + }); + initializeLocalSpeechServicesMock.mockResolvedValue( + localSpeech({ dictationStt: null, defaultModelIds: [] }), + ); + runtime.reconfigure({ speechConfig: nothingEnabled }); + await disabled; + + expect(runtime.resolveDictationStt()).toBeNull(); + await runtime.stop(); + }); + + it("refuses to delete a model an enabled feature uses", async () => { + initializeLocalSpeechServicesMock.mockResolvedValue( + localSpeech({ + dictationStt: createStubStt("dictation-local"), + defaultModelIds: ["parakeet-tdt-0.6b-v2-int8"], + }), + ); + ensureSherpaOnnxModelMock.mockImplementation(({ signal }) => rejectWhenAborted(signal)); + const runtime = createSpeechService({ + logger: pino({ level: "silent" }), + speechConfig: dictationOnly, + }); + runtime.start(); + await runtime.ready; + + const models = await runtime.listLocalModels(); + expect(models.filter((model) => model.inUse).map((model) => model.spec.id)).toEqual([ + "parakeet-tdt-0.6b-v2-int8", + ]); + await expect(runtime.deleteLocalModel("parakeet-tdt-0.6b-v2-int8")).rejects.toBeInstanceOf( + LocalSpeechModelInUseError, + ); + await expect(runtime.deleteLocalModel("kokoro-int8-en-v0_19")).resolves.toBeUndefined(); + + await runtime.stop(); + }); +}); diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index 932f87c077..270b13b03c 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -2,7 +2,10 @@ import type { Logger } from "pino"; import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; import type { LocalSpeechModelId } from "./providers/local/config.js"; -import { ensureLocalSpeechModels, listMissingLocalSpeechModels } from "./providers/local/models.js"; +import { + LocalSpeechModelLibrary, + type LocalSpeechModelEntry, +} from "./providers/local/model-library.js"; import { initializeLocalSpeechServices } from "./providers/local/runtime.js"; import { getOpenAiSpeechAvailability, @@ -92,7 +95,7 @@ function buildRealtimeVoiceReadiness(params: { enabled: false, available: false, reasonCode: "disabled", - message: "Realtime voice is disabled in daemon config.", + message: "Voice mode is off. Turn it on in Settings > Voice.", retryable: false, missingModelIds: [], }; @@ -147,7 +150,7 @@ function buildDictationReadiness(params: { enabled: false, available: false, reasonCode: "disabled", - message: "Dictation is disabled in daemon config.", + message: "Dictation is off. Turn it on in Settings > Voice.", retryable: false, missingModelIds: [], }; @@ -292,6 +295,23 @@ function resolveEffectiveProviderIds(params: { }; } +export interface LocalSpeechModelStatus extends LocalSpeechModelEntry { + // Selected by an enabled feature, so the runtime keeps it installed. + inUse: boolean; +} + +export class LocalSpeechModelInUseError extends Error { + constructor(public readonly modelId: LocalSpeechModelId) { + super(`${modelId} is selected for dictation or voice mode. Choose another model first.`); + this.name = "LocalSpeechModelInUseError"; + } +} + +export interface SpeechServiceConfig { + openaiConfig?: PaseoOpenAIConfig; + speechConfig?: PaseoSpeechConfig; +} + export interface SpeechService { resolveStt: () => SpeechToTextProvider | null; resolveSttLanguage: () => string; @@ -301,37 +321,49 @@ export interface SpeechService { resolveDictationSttLanguage: () => string; getReadiness: () => SpeechReadinessSnapshot; onReadinessChange: (listener: (snapshot: SpeechReadinessSnapshot) => void) => () => void; + // Applies new provider/model settings without restarting the daemon. + reconfigure: (config: SpeechServiceConfig) => void; + listLocalModels: () => Promise; + downloadLocalModel: (modelId: LocalSpeechModelId) => void; + deleteLocalModel: (modelId: LocalSpeechModelId) => Promise; start: () => void; stop: () => Promise; ready: Promise; } -export function createSpeechService(params: { - logger: Logger; - openaiConfig?: PaseoOpenAIConfig; - speechConfig?: PaseoSpeechConfig; -}): SpeechService { +export function createSpeechService( + params: SpeechServiceConfig & { + logger: Logger; + }, +): SpeechService { const logger = params.logger.child({ module: "speech-runtime" }); - const speechConfig = params.speechConfig ?? null; - const openaiConfig = params.openaiConfig; - const providers = resolveRequestedSpeechProviders(speechConfig); - const requestedProviders = describeRequestedProviders(providers); + let speechConfig = params.speechConfig ?? null; + let openaiConfig = params.openaiConfig; + let providers = resolveRequestedSpeechProviders(speechConfig); + let requestedProviders = describeRequestedProviders(providers); + // The models directory is fixed for the daemon's lifetime; settings changes only pick + // which of its models are used. + const modelLibrary = speechConfig + ? new LocalSpeechModelLibrary(speechConfig.localModelsDir, logger) + : null; - validateOpenAiCredentialRequirements({ - providers, - openaiConfig, - logger, - }); - - logger.info( - { - requestedProviders, - availability: { - openai: getOpenAiSpeechAvailability(openaiConfig), + const logReconciliationStart = (): void => { + validateOpenAiCredentialRequirements({ + providers, + openaiConfig, + logger, + }); + logger.info( + { + requestedProviders, + availability: { + openai: getOpenAiSpeechAvailability(openaiConfig), + }, }, - }, - "Speech provider reconciliation started", - ); + "Speech provider reconciliation started", + ); + }; + logReconciliationStart(); let sttService: SpeechToTextProvider | null = null; let ttsService: TextToSpeechProvider | null = null; @@ -346,12 +378,11 @@ export function createSpeechService(params: { let missingLocalModelIds: LocalSpeechModelId[] = []; let backgroundDownloadInProgress = false; - let backgroundDownloadAbortController: AbortController | null = null; let backgroundDownloadPromise: Promise | null = null; let backgroundDownloadError: string | null = null; let stopped = false; let monitorTimeout: ReturnType | null = null; - let reconcileInFlight: Promise | null = null; + let reconcileQueue: Promise = Promise.resolve(); const readinessListeners = new Set<(snapshot: SpeechReadinessSnapshot) => void>(); let lastReadinessFingerprint: string | null = null; let lastPublishedReadinessSnapshot: SpeechReadinessSnapshot | null = null; @@ -445,12 +476,10 @@ export function createSpeechService(params: { }; const refreshMissingLocalModels = async (): Promise => { - missingLocalModelIds = localModelConfig - ? await listMissingLocalSpeechModels({ - modelsDir: localModelConfig.modelsDir, - modelIds: localModelConfig.defaultModelIds, - }) - : []; + missingLocalModelIds = + localModelConfig && modelLibrary + ? await modelLibrary.listMissing(localModelConfig.defaultModelIds) + : []; }; const reconcileServices = async (): Promise => { @@ -520,16 +549,12 @@ export function createSpeechService(params: { } }; + // Reconciles run one after another, and each starts after it was requested, so a + // reconfigure that lands mid-reconcile is applied instead of joining the stale run. const runReconcile = async (): Promise => { - if (reconcileInFlight) { - await reconcileInFlight; - publishReadinessIfChanged(); - return; - } - reconcileInFlight = reconcileServices().finally(() => { - reconcileInFlight = null; - }); - await reconcileInFlight; + const next = reconcileQueue.catch(() => undefined).then(reconcileServices); + reconcileQueue = next; + await next; publishReadinessIfChanged(); }; @@ -544,12 +569,12 @@ export function createSpeechService(params: { }; const startBackgroundDownload = (): void => { - if (stopped || backgroundDownloadInProgress) { + if (stopped || backgroundDownloadInProgress || !modelLibrary) { return; } - const modelsDir = localModelConfig?.modelsDir ?? null; + const modelsDir = modelLibrary.getModelsDir(); const modelIds = [...missingLocalModelIds]; - if (!modelsDir || modelIds.length === 0) { + if (modelIds.length === 0) { return; } @@ -565,16 +590,9 @@ export function createSpeechService(params: { "Starting background download for missing local speech models", ); - const abortController = new AbortController(); - backgroundDownloadAbortController = abortController; backgroundDownloadPromise = (async () => { try { - await ensureLocalSpeechModels({ - modelsDir, - modelIds, - logger, - signal: abortController.signal, - }); + await Promise.all(modelIds.map((modelId) => modelLibrary.download(modelId))); await runReconcile(); backgroundDownloadError = null; } catch (error) { @@ -588,7 +606,6 @@ export function createSpeechService(params: { "Background local speech model download failed", ); } finally { - backgroundDownloadAbortController = null; backgroundDownloadInProgress = false; await refreshMissingLocalModels().catch((error) => { logger.warn({ err: error }, "Failed to refresh local speech model status after download"); @@ -630,6 +647,62 @@ export function createSpeechService(params: { } }; + const reconcileAndFetchMissingModels = async (): Promise => { + await runReconcile(); + const snapshot = computeReadinessSnapshot(); + if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) { + if (missingLocalModelIds.length > 0) { + startBackgroundDownload(); + } + scheduleMonitor(); + } + }; + + const reconfigure = (next: SpeechServiceConfig): void => { + speechConfig = next.speechConfig ?? null; + openaiConfig = next.openaiConfig; + providers = resolveRequestedSpeechProviders(speechConfig); + requestedProviders = describeRequestedProviders(providers); + // A failure for the previous selection must not block downloading the new one. + backgroundDownloadError = null; + logReconciliationStart(); + if (!started || stopped) { + return; + } + void reconcileAndFetchMissingModels().catch((error) => { + logger.error({ err: error }, "Speech runtime failed to apply new configuration"); + }); + }; + + const requireModelLibrary = (): LocalSpeechModelLibrary => { + if (!modelLibrary) { + throw new Error("Local speech models are unavailable on this daemon"); + } + return modelLibrary; + }; + + const listLocalModels = async (): Promise => { + const inUse = new Set(localModelConfig?.defaultModelIds ?? []); + const entries = await requireModelLibrary().list(); + return entries.map(({ spec, state }) => ({ spec, state, inUse: inUse.has(spec.id) })); + }; + + const downloadLocalModel = (modelId: LocalSpeechModelId): void => { + void requireModelLibrary() + .download(modelId) + .then(() => runMonitorTick()) + .catch((error) => { + logger.warn({ err: error, modelId }, "Local speech model download failed"); + }); + }; + + const deleteLocalModel = async (modelId: LocalSpeechModelId): Promise => { + if (localModelConfig?.defaultModelIds.includes(modelId)) { + throw new LocalSpeechModelInUseError(modelId); + } + await requireModelLibrary().delete(modelId); + }; + const start = (): void => { if (started || stopped) { return; @@ -637,14 +710,7 @@ export function createSpeechService(params: { started = true; void (async () => { try { - await runReconcile(); - const snapshot = computeReadinessSnapshot(); - if (snapshot.voiceFeature.enabled && !snapshot.voiceFeature.available) { - if (missingLocalModelIds.length > 0) { - startBackgroundDownload(); - } - scheduleMonitor(); - } + await reconcileAndFetchMissingModels(); if (!readySettled) { readySettled = true; resolveReady(); @@ -666,7 +732,7 @@ export function createSpeechService(params: { monitorTimeout = null; } localCleanup(); - backgroundDownloadAbortController?.abort(); + await modelLibrary?.stop(); await backgroundDownloadPromise?.catch(() => undefined); backgroundDownloadPromise = null; }; @@ -680,6 +746,10 @@ export function createSpeechService(params: { resolveDictationSttLanguage: () => speechConfig?.sttLanguages?.dictation ?? "en", getReadiness: () => lastPublishedReadinessSnapshot ?? computeReadinessSnapshot(), onReadinessChange: subscribeSpeechReadiness, + reconfigure, + listLocalModels, + downloadLocalModel, + deleteLocalModel, start, stop, ready, diff --git a/packages/server/src/server/speech/speech-settings.ts b/packages/server/src/server/speech/speech-settings.ts new file mode 100644 index 0000000000..31995c46f6 --- /dev/null +++ b/packages/server/src/server/speech/speech-settings.ts @@ -0,0 +1,202 @@ +import type { SpeechSettings, SpeechSettingsPatch } from "@getpaseo/protocol/speech/rpc-schemas"; + +import { + PersistedConfigSchema, + SpeechSttConfigSchema, + SpeechTtsConfigSchema, + type PersistedConfig, +} from "../persisted-config.js"; +import { resolveRequestedSpeechProviders, resolveSpeechConfig } from "./speech-config-resolver.js"; +import type { SpeechService } from "./speech-runtime.js"; +import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/models.js"; + +interface SpeechRead { + features: SpeechSettings; + envOverrides: Record; +} + +// Owns the per-site wire boundary: credential redaction, env provenance and patch +// semantics. The store never assembles or merges provider objects itself. +export interface SpeechSettingsOwner { + read(persisted: PersistedConfig): SpeechRead; + apply(persisted: PersistedConfig): void; +} + +export function createSpeechSettingsOwner(params: { + paseoHome: string; + env: NodeJS.ProcessEnv; + speechService: Pick; +}): SpeechSettingsOwner { + const { paseoHome, env, speechService } = params; + return { + read: (persisted) => readSpeechSettings(persisted, env), + apply: (persisted) => { + const resolved = resolveSpeechConfig({ paseoHome, env, persisted }); + speechService.reconfigure({ openaiConfig: resolved.openai, speechConfig: resolved.speech }); + }, + }; +} + +export function readSpeechSettings(persisted: PersistedConfig, env: NodeJS.ProcessEnv): SpeechRead { + const dictation = persisted.features?.dictation; + const voiceMode = persisted.features?.voiceMode; + const providers = resolveRequestedSpeechProviders({ persisted, env }); + const envOverrides: Record = {}; + const recordOverride = (path: string, names: string[], fallback?: unknown) => { + if (fallback !== undefined) return; + const name = names.find((candidate) => env[candidate]?.trim()); + if (name) envOverrides[path] = name; + }; + recordOverride("features.dictation.enabled", ["PASEO_DICTATION_ENABLED"]); + recordOverride("features.voiceMode.enabled", ["PASEO_VOICE_MODE_ENABLED"]); + recordOverride("features.voiceMode.turnDetection.provider", [ + "PASEO_VOICE_TURN_DETECTION_PROVIDER", + ]); + + function readSite( + site: Record | undefined, + provider: string, + path: string, + kind: "stt" | "tts", + providerEnv: string, + localModelEnv: string, + ) { + recordOverride(`${path}.provider`, [providerEnv]); + const configured = site?.provider === provider ? site : undefined; + const { apiKey, ...fields } = configured ?? {}; + const openaiModelEnv = kind === "stt" ? "STT_MODEL" : "TTS_MODEL"; + recordOverride(`${path}.model`, [provider === "openai" ? openaiModelEnv : localModelEnv]); + if (kind === "stt") { + const languageVars = + path === "features.dictation.stt" + ? ["PASEO_DICTATION_LANGUAGE"] + : ["PASEO_VOICE_LANGUAGE", "PASEO_DICTATION_LANGUAGE"]; + recordOverride(`${path}.language`, languageVars); + } + if (provider === "openai") { + for (const [field, suffix] of [ + ["apiKey", "API_KEY"], + ["baseUrl", "BASE_URL"], + ] as const) { + recordOverride( + `${path}.${field}`, + [`OPENAI_${kind.toUpperCase()}_${suffix}`, `OPENAI_${suffix}`], + configured?.[field], + ); + } + if (kind === "stt") { + recordOverride(`${path}.confidenceThreshold`, ["STT_CONFIDENCE_THRESHOLD"]); + } else recordOverride(`${path}.voice`, ["TTS_VOICE"]); + } else if (kind === "tts") { + recordOverride(`${path}.speakerId`, ["PASEO_VOICE_LOCAL_TTS_SPEAKER_ID"]); + recordOverride(`${path}.speed`, ["PASEO_VOICE_LOCAL_TTS_SPEED"]); + } + return { + ...fields, + provider, + ...(provider === "openai" + ? { hasApiKey: typeof apiKey === "string" && apiKey.trim().length > 0 } + : {}), + }; + } + + return { + features: { + dictation: { + enabled: providers.dictationStt.enabled !== false, + stt: readSite( + dictation?.stt, + providers.dictationStt.provider, + "features.dictation.stt", + "stt", + "PASEO_DICTATION_STT_PROVIDER", + "PASEO_DICTATION_LOCAL_STT_MODEL", + ), + }, + voiceMode: { + enabled: providers.voiceStt.enabled !== false, + ...(voiceMode?.llm ? { llm: voiceMode.llm } : {}), + ...(voiceMode?.turnDetection + ? { + turnDetection: { + ...voiceMode.turnDetection, + provider: providers.voiceTurnDetection.provider, + }, + } + : {}), + stt: readSite( + voiceMode?.stt, + providers.voiceStt.provider, + "features.voiceMode.stt", + "stt", + "PASEO_VOICE_STT_PROVIDER", + "PASEO_VOICE_LOCAL_STT_MODEL", + ), + tts: readSite( + voiceMode?.tts, + providers.voiceTts.provider, + "features.voiceMode.tts", + "tts", + "PASEO_VOICE_TTS_PROVIDER", + "PASEO_VOICE_LOCAL_TTS_MODEL", + ), + }, + }, + envOverrides, + }; +} + +function mergeFields(current: Record | undefined, patch: Record) { + const next = { ...current }; + for (const [key, value] of Object.entries(patch)) { + if (value === undefined) continue; + if (value === null || (typeof value === "string" && !value.trim())) delete next[key]; + else next[key] = typeof value === "string" ? value.trim() : value; + } + return next; +} + +function mergeSite( + current: Record | undefined, + patch: Record, + kind: "stt" | "tts", +) { + const providerChanged = + patch.provider !== undefined && patch.provider !== (current?.provider ?? "local"); + const next = mergeFields(providerChanged ? undefined : current, patch); + next.provider ??= "local"; + const site = (kind === "stt" ? SpeechSttConfigSchema : SpeechTtsConfigSchema).parse(next); + if (site.provider === "local" && site.model !== undefined) { + const schema = kind === "stt" ? LocalSttModelIdSchema : LocalTtsModelIdSchema; + if (!schema.safeParse(site.model).success) { + throw new Error( + `Unknown local ${kind === "stt" ? "speech-to-text" : "voice"} model: ${site.model}`, + ); + } + } + return site; +} + +export function mergeSpeechSettingsPatch( + persisted: PersistedConfig, + patch: SpeechSettingsPatch, +): PersistedConfig { + const features = { ...persisted.features }; + if (patch.dictation) { + const { stt, ...rest } = patch.dictation; + const next = mergeFields(features.dictation, rest); + if (stt) next.stt = mergeSite(features.dictation?.stt, stt, "stt"); + features.dictation = PersistedConfigSchema.shape.features.unwrap().shape.dictation.parse(next); + } + if (patch.voiceMode) { + const { stt, tts, llm, turnDetection, ...rest } = patch.voiceMode; + const next = mergeFields(features.voiceMode, rest); + if (stt) next.stt = mergeSite(features.voiceMode?.stt, stt, "stt"); + if (tts) next.tts = mergeSite(features.voiceMode?.tts, tts, "tts"); + if (llm) next.llm = mergeFields(features.voiceMode?.llm, llm); + if (turnDetection) + next.turnDetection = mergeFields(features.voiceMode?.turnDetection, turnDetection); + features.voiceMode = PersistedConfigSchema.shape.features.unwrap().shape.voiceMode.parse(next); + } + return { ...persisted, features }; +} diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index 289d556bec..a1f8369b59 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -39,7 +39,8 @@ interface TestPaseoDaemonOptions { staticDir?: string; cleanup?: boolean; openai?: PaseoOpenAIConfig; - speech?: PaseoSpeechConfig; + // localModelsDir defaults to the local provider's modelsDir, then to a dir under paseoHome. + speech?: Omit & { localModelsDir?: string }; voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"]; voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; @@ -209,7 +210,15 @@ async function prepareTestDaemonConfig( webUi: options.webUi, trustedProxies: options.trustedProxies, openai: options.openai, - speech: options.speech, + speech: options.speech + ? { + ...options.speech, + localModelsDir: + options.speech.localModelsDir ?? + options.speech.local?.modelsDir ?? + path.join(paseoHome, "models", "local-speech"), + } + : undefined, voiceLlmProvider: options.voiceLlmProvider ?? null, voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false, voiceLlmModel: options.voiceLlmModel ?? null, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 69f116f25b..53172b2835 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -1620,6 +1620,7 @@ export class VoiceAssistantWebSocketServer { stt: () => this.speech?.resolveStt() ?? null, sttLanguage: this.speech?.resolveSttLanguage() ?? "en", tts: () => this.speech?.resolveTts() ?? null, + speechModels: this.speech, terminalManager: this.terminalManager, providerSnapshotManager: this.providerSnapshotManager, hubExecutionAgents: options.hubExecutionAgents, @@ -1991,6 +1992,8 @@ export class VoiceAssistantWebSocketServer { ...(this.advertiseDaemonStatusRpc ? { daemonStatusRpc: true } : {}), // COMPAT(daemonConfigReload): added in v0.4.0, remove gate after 2027-02-14. daemonConfigReload: true, + // COMPAT(speechSettings): added in v0.12.0, remove gate after 2027-04-18. + speechSettings: true, // COMPAT(relayConfig): added in v0.2.6, remove gate after 2027-01-31. ...(this.advertiseRelayConfig ? { relayConfig: true } : {}), // COMPAT(pushTokenRevocation): added in v0.3.2, remove gate after 2027-02-10. diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index a6d0ed4541..c1c32ed20a 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -257,48 +257,6 @@ "providers": { "type": "object", "properties": { - "openai": { - "type": "object", - "properties": { - "apiKey": { - "type": "string", - "minLength": 1 - }, - "baseUrl": { - "type": "string", - "minLength": 1 - }, - "stt": { - "type": "object", - "properties": { - "apiKey": { - "type": "string", - "minLength": 1 - }, - "baseUrl": { - "type": "string", - "minLength": 1 - } - }, - "additionalProperties": false - }, - "tts": { - "type": "object", - "properties": { - "apiKey": { - "type": "string", - "minLength": 1 - }, - "baseUrl": { - "type": "string", - "minLength": 1 - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - }, "local": { "type": "object", "properties": { @@ -639,24 +597,57 @@ "type": "boolean" }, "stt": { - "type": "object", - "properties": { - "provider": { - "type": "string" + "oneOf": [ + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "const": "local" + }, + "model": { + "type": "string", + "minLength": 1 + }, + "language": { + "type": "string", + "minLength": 1 + } + }, + "required": ["provider"], + "additionalProperties": false }, - "model": { - "type": "string", - "minLength": 1 - }, - "language": { - "type": "string", - "minLength": 1 - }, - "confidenceThreshold": { - "type": "number" + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "const": "openai" + }, + "apiKey": { + "type": "string", + "minLength": 1 + }, + "baseUrl": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "language": { + "type": "string", + "minLength": 1 + }, + "confidenceThreshold": { + "type": "number" + } + }, + "required": ["provider"], + "additionalProperties": false } - }, - "additionalProperties": false + ] } }, "additionalProperties": false @@ -681,21 +672,57 @@ "additionalProperties": false }, "stt": { - "type": "object", - "properties": { - "provider": { - "type": "string" + "oneOf": [ + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "const": "local" + }, + "model": { + "type": "string", + "minLength": 1 + }, + "language": { + "type": "string", + "minLength": 1 + } + }, + "required": ["provider"], + "additionalProperties": false }, - "model": { - "type": "string", - "minLength": 1 - }, - "language": { - "type": "string", - "minLength": 1 + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "const": "openai" + }, + "apiKey": { + "type": "string", + "minLength": 1 + }, + "baseUrl": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "language": { + "type": "string", + "minLength": 1 + }, + "confidenceThreshold": { + "type": "number" + } + }, + "required": ["provider"], + "additionalProperties": false } - }, - "additionalProperties": false + ] }, "turnDetection": { "type": "object", @@ -707,29 +734,58 @@ "additionalProperties": false }, "tts": { - "type": "object", - "properties": { - "provider": { - "type": "string" + "oneOf": [ + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "const": "local" + }, + "model": { + "type": "string", + "minLength": 1 + }, + "speakerId": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "speed": { + "type": "number" + } + }, + "required": ["provider"], + "additionalProperties": false }, - "model": { - "type": "string", - "minLength": 1 - }, - "voice": { - "type": "string", - "minLength": 1 - }, - "speakerId": { - "type": "integer", - "minimum": -9007199254740991, - "maximum": 9007199254740991 - }, - "speed": { - "type": "number" + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "const": "openai" + }, + "apiKey": { + "type": "string", + "minLength": 1 + }, + "baseUrl": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "voice": { + "type": "string", + "minLength": 1 + } + }, + "required": ["provider"], + "additionalProperties": false } - }, - "additionalProperties": false + ] } }, "additionalProperties": false diff --git a/public-docs/configuration.md b/public-docs/configuration.md index d9b3d59f8d..f89a12dcf7 100644 --- a/public-docs/configuration.md +++ b/public-docs/configuration.md @@ -116,7 +116,7 @@ Relative paths are resolved against `PASEO_HOME`. Existing worktrees remain wher ## Voice -Voice is configured through `features.dictation` and `features.voiceMode`, with provider credentials under `providers`. +Voice is configured through `features.dictation` and `features.voiceMode`. Provider credentials and options live inline at `features.dictation.stt`, `features.voiceMode.stt`, and `features.voiceMode.tts`. For voice philosophy, architecture, and complete local/OpenAI setup examples, see [Voice docs](/docs/voice). @@ -269,9 +269,9 @@ Set the persisted value in `config.json`: - `PASEO_LOG_FILE_ROTATE_SIZE`, override `log.file.rotate.maxSize` - `PASEO_LOG_FILE_ROTATE_COUNT`, override `log.file.rotate.maxFiles` - `PASEO_LOG`, `PASEO_LOG_FORMAT`, legacy log overrides (still supported) -- `OPENAI_API_KEY`, override OpenAI provider key -- `OPENAI_STT_API_KEY`, `OPENAI_STT_BASE_URL`, OpenAI speech-to-text endpoint (dictation + voice mode STT) -- `OPENAI_TTS_API_KEY`, `OPENAI_TTS_BASE_URL`, OpenAI text-to-speech endpoint (voice mode TTS) +- `OPENAI_API_KEY`, `OPENAI_BASE_URL`, fallback speech credentials and base URL when neither a site value nor an endpoint env value is set +- `OPENAI_STT_API_KEY`, `OPENAI_STT_BASE_URL`, speech-to-text key and base URL fallbacks when no value is saved at the dictation or voice-mode STT site +- `OPENAI_TTS_API_KEY`, `OPENAI_TTS_BASE_URL`, text-to-speech key and base URL fallbacks when no value is saved at the voice-mode TTS site - `PASEO_VOICE_LLM_PROVIDER`, override voice LLM provider (`claude`, `codex`, `opencode`) - `PASEO_DICTATION_STT_PROVIDER`, `PASEO_VOICE_STT_PROVIDER`, `PASEO_VOICE_TTS_PROVIDER`, override voice provider selection (`local` or `openai`) - `PASEO_LOCAL_MODELS_DIR`, control local model directory diff --git a/public-docs/voice.md b/public-docs/voice.md index 79022c15ad..c90e5196bc 100644 --- a/public-docs/voice.md +++ b/public-docs/voice.md @@ -27,27 +27,24 @@ This keeps credentials and execution in your environment and avoids introducing Local speech defaults to model IDs `parakeet-tdt-0.6b-v2-int8` (STT) and `kokoro-en-v0_19` (TTS, speaker 0 / voice 00). -Missing models are downloaded at daemon startup into `$PASEO_HOME/models/local-speech`. Downloads happen only for missing files. +Dictation is on by default. Voice mode is off until you turn it on in **Settings > Voice** or set `features.voiceMode.enabled: true`. + +Missing models for enabled features are downloaded at daemon startup into `$PASEO_HOME/models/local-speech`. Downloads happen only for missing files. **Settings > Voice** lists every local model with its size and languages, and lets you download or delete each one. ### Local STT models and language support -| Model ID | Languages | -| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `parakeet-tdt-0.6b-v2-int8` | English only (default). Includes punctuation and capitalization. | -| `parakeet-tdt-0.6b-v3-int8` | 25 European languages, auto-detected: Bulgarian, Croatian, Czech, Danish, Dutch, English, Estonian, Finnish, French, German, Greek, Hungarian, Italian, Latvian, Lithuanian, Maltese, Polish, Portuguese, Romanian, Russian, Slovak, Slovenian, Spanish, Swedish, Ukrainian. | - -**To use a non-English language, switch the local STT model to `parakeet-tdt-0.6b-v3-int8`.** v3 detects the spoken language automatically — there is no per-language setting for it. The `language` field below does **not** steer the local Parakeet model (v2 is English-only, v3 auto-detects); it only applies to the OpenAI STT provider. +**Settings > Voice** lists every local STT model with the languages it supports. The default, `parakeet-tdt-0.6b-v2-int8`, is English-only. For other languages, pick a multilingual model such as `parakeet-tdt-0.6b-v3-int8` (25 European languages) or `whisper-base` (99 languages). The `language` field is passed to STT sessions for both providers. Local recognizers currently detect the spoken language themselves; setting `language` does not expand a model’s supported languages. ```json { "version": 1, "features": { "dictation": { - "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v2-int8", "language": "en" } + "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v2-int8" } }, "voiceMode": { "llm": { "provider": "claude", "model": "haiku" }, - "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v2-int8", "language": "en" }, + "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v2-int8" }, "tts": { "provider": "local", "model": "kokoro-en-v0_19", "speakerId": 0 } } }, @@ -59,7 +56,7 @@ Missing models are downloaded at daemon startup into `$PASEO_HOME/models/local-s } ``` -For multilingual local dictation, set the model to v3 — it auto-detects the language, so no `language` field is needed: +For multilingual local dictation, set a multilingual model. No `language` field is needed: ```json { @@ -72,38 +69,53 @@ For multilingual local dictation, set the model to v3 — it auto-detects the la } ``` -The `language` field applies only to the OpenAI STT provider: set `features.dictation.stt.language` for dictation and `features.voiceMode.stt.language` for voice mode. If voice language is omitted, Paseo uses the dictation language before falling back to `en`. It has no effect on the local Parakeet models. +For either STT provider, set `features.dictation.stt.language` for dictation and `features.voiceMode.stt.language` for voice mode. If voice language is omitted, Paseo uses the dictation language before falling back to `en`. ## OpenAI Voice Option -You can switch dictation, voice STT, and voice TTS to OpenAI by setting provider fields to `openai` and providing OpenAI credentials. +Set `provider` to `openai` and enter credentials at each use site that needs them. +Dictation, listening and speaking each own their key, base URL and model: ```json { "version": 1, "features": { - "dictation": { "stt": { "provider": "openai" } }, + "dictation": { + "stt": { "provider": "openai", "apiKey": "sk-...", "model": "gpt-4o-mini-transcribe" } + }, "voiceMode": { - "stt": { "provider": "openai" }, - "tts": { "provider": "openai" } - } - }, - "providers": { - "openai": { + "enabled": true, "stt": { - "apiKey": "...", - "baseUrl": "https://api.openai.com/v1" + "provider": "openai", + "apiKey": "sk-...", + "baseUrl": "https://api.openai.com/v1", + "model": "gpt-4o-transcribe", + "language": "en" }, - "tts": { - "apiKey": "...", - "baseUrl": "https://api.openai.com/v1" - } + "tts": { "provider": "openai", "apiKey": "sk-...", "model": "tts-1", "voice": "alloy" } } } } ``` -`providers.openai.stt` covers dictation and voice mode speech-to-text, and `providers.openai.tts` covers voice mode text-to-speech. Because they resolve independently, you can point STT and TTS at different endpoints. Each falls back to `providers.openai.apiKey`/`baseUrl`, then `OPENAI_API_KEY`/`OPENAI_BASE_URL`, when unset. These settings configure only Paseo OpenAI speech traffic, without changing Codex or other OpenAI-backed tools. +| Provider | Speech-to-text fields | Text-to-speech fields | +| -------- | --------------------------------------------------------------- | ------------------------------------- | +| `local` | `model`, `language` | `model`, `speakerId`, `speed` | +| `openai` | `apiKey`, `baseUrl`, `model`, `language`, `confidenceThreshold` | `apiKey`, `baseUrl`, `model`, `voice` | + +Fields other than `provider` are optional. Omit `model` to use the provider's default. +OpenAI model and voice names accept custom IDs from compatible servers. Keys saved +on the host are write-only in the app. + +A saved `apiKey` or `baseUrl` wins over env settings. Without a saved value, STT +uses `OPENAI_STT_API_KEY` / `OPENAI_STT_BASE_URL`, TTS uses +`OPENAI_TTS_API_KEY` / `OPENAI_TTS_BASE_URL`, and both fall back to +`OPENAI_API_KEY` / `OPENAI_BASE_URL`. These settings affect Paseo speech traffic. + +Existing `providers.openai` settings load automatically into OpenAI use sites that +lack inline values. Endpoint settings take precedence over the old shared settings. +The next save writes the inline shape. A legacy key without an OpenAI site to +receive it is dropped with a warning; enter a key when you later select OpenAI. Paseo uses these paths under the configured OpenAI base URL: @@ -113,6 +125,14 @@ Paseo uses these paths under the configured OpenAI base URL: ## Environment Variables +Provider, enablement, model, language, voice and tuning env vars override config. +Keys and base URLs follow the fallback rule above. + +- `PASEO_DICTATION_ENABLED`, `PASEO_VOICE_MODE_ENABLED`, feature enablement +- `STT_MODEL`, model override for both OpenAI STT sites +- `STT_CONFIDENCE_THRESHOLD`, confidence threshold for both OpenAI STT sites +- `TTS_MODEL`, `TTS_VOICE`, OpenAI speaking model and voice + - `PASEO_VOICE_LLM_PROVIDER`, voice agent provider override - `PASEO_DICTATION_STT_PROVIDER`, `PASEO_VOICE_STT_PROVIDER`, `PASEO_VOICE_TTS_PROVIDER`, speech provider selection (`local` or `openai`) - `OPENAI_STT_API_KEY`, `OPENAI_STT_BASE_URL`, OpenAI speech-to-text endpoint (dictation + voice mode STT) @@ -120,8 +140,8 @@ Paseo uses these paths under the configured OpenAI base URL: - `PASEO_LOCAL_MODELS_DIR`, local model storage directory - `PASEO_DICTATION_LOCAL_STT_MODEL`, local dictation STT model ID - `PASEO_VOICE_LOCAL_STT_MODEL`, `PASEO_VOICE_LOCAL_TTS_MODEL`, local voice STT/TTS model IDs -- `PASEO_DICTATION_LANGUAGE`, dictation STT language (OpenAI STT only; ignored by local Parakeet) -- `PASEO_VOICE_LANGUAGE`, voice mode STT language; falls back to `PASEO_DICTATION_LANGUAGE` when unset (OpenAI STT only; ignored by local Parakeet) +- `PASEO_DICTATION_LANGUAGE`, dictation STT language +- `PASEO_VOICE_LANGUAGE`, voice mode STT language; falls back to `PASEO_DICTATION_LANGUAGE` when unset - `PASEO_VOICE_LOCAL_TTS_SPEAKER_ID`, `PASEO_VOICE_LOCAL_TTS_SPEED`, optional local voice TTS tuning ## Operational Notes From 033b2cdfc2b85e4e67a7f9d22b3af25200cf54a0 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 14:24:38 +0700 Subject: [PATCH 21/32] fix(app): keep command suggestions visible as the composer grows (#7910) Long command descriptions exhausted the available space above a tall draft, leaving no complete selectable row. Bound the preview and reserve list space. Let the popover own its composer anchor so layout changes refresh its position while suggestions remain open. --- docs/mobile-testing.md | 6 + .../autocomplete-placement.mjs | 9 +- .../app/e2e/mobile/composer-keyboard/ios.mjs | 161 +++++++++++++++--- .../components/ui/autocomplete-popover.tsx | 49 +++--- .../app/src/components/ui/autocomplete.tsx | 14 +- packages/app/src/composer/index.tsx | 36 ++-- 6 files changed, 207 insertions(+), 68 deletions(-) diff --git a/docs/mobile-testing.md b/docs/mobile-testing.md index 917ebf9b53..169803c0eb 100644 --- a/docs/mobile-testing.md +++ b/docs/mobile-testing.md @@ -94,6 +94,12 @@ and its Claude provider available. Set `PASEO_COMPOSER_KEYBOARD_SERVER_ID`, `_AGENT_ID` (an idle mock chat), and `_WORKSPACE_ID` (its workspace); `_DEVICE`, `_SESSION`, and `_DRAFT_ID` are optional. Artifacts go to `.dev/agent-device-artifacts/composer-keyboard-ios`. +With a Codex draft selected, set `_SCENARIO=growth` on a 402pt-wide iPhone to +check live wrapping, shrinking, the 12pt anchor gap, and command insertion. Set +`_SCENARIO=previews` on an SE-sized iPhone to check long and short descriptions +under an eight-line draft, selection with the keyboard shown and hidden, and +file mentions. These scenarios only need `_SERVER_ID` and `_WORKSPACE_ID`; +`_UDID` selects an exact simulator and `_ARTIFACTS_DIR` overrides the output path. The Android sidebar scroll-retention flow requires a connected host with enough workspaces to overflow the compact sidebar. Pass `TOP_WORKSPACE` and diff --git a/packages/app/e2e/mobile/composer-keyboard/autocomplete-placement.mjs b/packages/app/e2e/mobile/composer-keyboard/autocomplete-placement.mjs index 22c0b42e8f..a8450838eb 100644 --- a/packages/app/e2e/mobile/composer-keyboard/autocomplete-placement.mjs +++ b/packages/app/e2e/mobile/composer-keyboard/autocomplete-placement.mjs @@ -28,7 +28,10 @@ function contains(outer, inner) { * while suggestions are open, so its callers read the composer from `reference`: a * snapshot in the same keyboard state with an empty input. */ -export function resolveAutocompletePlacement(snapshot, { expected, keyboardTop, reference }) { +export function resolveAutocompletePlacement( + snapshot, + { expected, keyboardTop, reference, anchorGap }, +) { if (!snapshot.success || !snapshot.data?.nodes?.length) throw new Error("Empty device snapshot"); const nodes = snapshot.data.nodes; const popover = findById(nodes, "composer-autocomplete-popover").rect; @@ -40,6 +43,10 @@ export function resolveAutocompletePlacement(snapshot, { expected, keyboardTop, const keyboardY = keyboardTop ?? nodes.find((node) => node.type === "Keyboard" && node.rect)?.rect.y; const popoverBottom = bottomOf(popover); + const gap = composer.y - popoverBottom; + if (anchorGap !== undefined && Math.abs(gap - anchorGap) > 1) { + throw new Error(`Autocomplete lost its anchor gap: gap=${gap}, expected=${anchorGap}`); + } if (popoverBottom > composer.y + 1) { throw new Error( `Autocomplete overlaps the composer: popover bottom=${popoverBottom}, composer top=${composer.y}`, diff --git a/packages/app/e2e/mobile/composer-keyboard/ios.mjs b/packages/app/e2e/mobile/composer-keyboard/ios.mjs index e3fc4af072..513d68623e 100644 --- a/packages/app/e2e/mobile/composer-keyboard/ios.mjs +++ b/packages/app/e2e/mobile/composer-keyboard/ios.mjs @@ -14,7 +14,8 @@ const env = (name, fallback) => process.env[`PASEO_COMPOSER_KEYBOARD_${name}`] ? const serverId = env("SERVER_ID"); const agentId = env("AGENT_ID"); const workspaceId = env("WORKSPACE_ID"); -if (!serverId || !agentId || !workspaceId) { +const scenario = env("SCENARIO", "all"); +if (!serverId || !workspaceId || (scenario === "all" && !agentId)) { throw new Error( "Set PASEO_COMPOSER_KEYBOARD_SERVER_ID, _AGENT_ID (an idle chat) and _WORKSPACE_ID (its workspace)", ); @@ -22,7 +23,12 @@ if (!serverId || !agentId || !workspaceId) { const draftId = env("DRAFT_ID", "composer-keyboard"); const session = env("SESSION", "composer-keyboard-ios"); const device = env("DEVICE", "iPhone 16 Pro"); -const artifactsDir = path.join(repoRoot, ".dev/agent-device-artifacts/composer-keyboard-ios"); +const udid = env("UDID"); +const deviceArgs = udid ? ["--udid", udid] : ["--device", device]; +const artifactsDir = env( + "ARTIFACTS_DIR", + path.join(repoRoot, ".dev/agent-device-artifacts/composer-keyboard-ios"), +); const grownPrefix = "grown line one\ngrown line two\ngrown line three\n"; // Partial queries with known best matches. `inserted` is the composer text after // tapping the best match. @@ -35,7 +41,7 @@ const chatCommandCase = { text: "/mo", expected: "/mock", inserted: "/mock " }; fs.mkdirSync(artifactsDir, { recursive: true }); function ad(...args) { - return execFileSync("agent-device", [...args, "--session", session], { + return execFileSync("agent-device", [args[0], "--session", session, ...args.slice(1)], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, }); @@ -77,7 +83,7 @@ async function settledPlacement(name, keyboard) { } function assertPlacement(name, snap, expected) { - const placement = resolveAutocompletePlacement(snap, { expected }); + const placement = resolveAutocompletePlacement(snap, { expected, anchorGap: 12 }); process.stdout.write( `PASS ${name}: popover ${Math.round(placement.popover.y)}-${Math.round( placement.popover.y + placement.popover.height, @@ -123,21 +129,34 @@ async function clearInput(name) { )?.rect; if (!deleteKey) throw new Error("Missing keyboard delete key"); const length = composerText(keyboard).length; - ad( - "press", - String(Math.round(deleteKey.x + deleteKey.width / 2)), - String(Math.round(deleteKey.y + deleteKey.height / 2)), - "--count", - String(length + 2), - "--interval-ms", - "40", - ); + for (let remaining = length + 2; remaining > 0; remaining -= 100) { + ad( + "press", + String(Math.round(deleteKey.x + deleteKey.width / 2)), + String(Math.round(deleteKey.y + deleteKey.height / 2)), + "--count", + String(Math.min(remaining, 100)), + "--interval-ms", + "40", + ); + } } // The accessibility value drops a trailing space. const sameText = (actual, expected) => actual.trimEnd() === expected.trimEnd(); -async function runCase(host, kind, { text, expected, inserted }) { +async function selectRow(name, row, inserted) { + ad("press", String(row.x), String(row.y)); + const selected = await waitFor(`${name}-selected`, (snap) => + sameText(composerText(snap), inserted), + ); + if (!keyboardShown(selected)) throw new Error(`Selecting ${row.label} closed the keyboard`); + process.stdout.write( + `PASS ${name}: tapped ${row.label}, composer now ${JSON.stringify(inserted)}\n`, + ); +} + +async function runCase(host, kind, { text, expected, inserted, selectionKeyboard = "reopened" }) { const name = `${host}-${kind}`; await clearInput(name); await dismissKeyboard(snapshot(`${name}-empty`)); @@ -151,31 +170,32 @@ async function runCase(host, kind, { text, expected, inserted }) { await clearInput(`${name}-settled`); await waitFor(`${name}-settled-empty`, (snap) => composerText(snap) === ""); ad("type", text); - assertPlacement( + const opened = assertPlacement( `${name} opened after keyboard settled`, await settledPlacement(`${name}-open`, true), expected, ); + if (selectionKeyboard === "shown") { + await selectRow(name, opened, inserted); + return; + } await dismissKeyboard(snapshot(`${name}-before-close`)); - assertPlacement( + const closed = assertPlacement( `${name} keyboard closed`, await settledPlacement(`${name}-closed`, false), expected, ); + if (selectionKeyboard === "hidden") { + await selectRow(name, closed, inserted); + return; + } focusAtEnd(snapshot(`${name}-before-reopen`)); const reopened = assertPlacement( `${name} keyboard opened under open suggestions`, await settledPlacement(`${name}-reopened`, true), expected, ); - ad("press", String(reopened.x), String(reopened.y)); - const selected = await waitFor(`${name}-selected`, (snap) => - sameText(composerText(snap), inserted), - ); - if (!keyboardShown(selected)) throw new Error(`Selecting ${expected} closed the keyboard`); - process.stdout.write( - `PASS ${name}: tapped ${expected}, composer now ${JSON.stringify(inserted)}\n`, - ); + await selectRow(name, reopened, inserted); } // Drafts list commands for their selected provider, and a fresh install selects none. @@ -216,13 +236,104 @@ async function runHost(host) { await dismissKeyboard(snapshot(`${host}-done`)); } +// These cases use the selected draft provider's real skills. Run on a 402pt-wide +// iPhone for growth and a small phone (SE) for previews, with Codex selected. +async function runLiveGrowth() { + await clearInput("live-growth"); + ad("type", "Please do the following now /paseo"); + const before = await settledPlacement("live-growth-before", true); + assertPlacement("before growth", before, "/paseo"); + const inputHeight = (snap) => + snap.data.nodes.find((node) => node.type === "TextView").rect.height; + ad("type", "--", "-committee"); + const grown = await settledPlacement("live-growth-after", true); + if (inputHeight(grown) <= inputHeight(before)) + throw new Error("Draft did not wrap onto a new line"); + assertPlacement("live growth without reopening", grown, "/paseo-committee"); + const deleteKey = grown.data.nodes.find( + (node) => node.type === "Key" && node.label === "delete", + ).rect; + ad( + "press", + String(deleteKey.x + deleteKey.width / 2), + String(deleteKey.y + deleteKey.height / 2), + "--count", + "10", + "--interval-ms", + "40", + ); + const shrunk = await settledPlacement("live-growth-shrunk", true); + if (inputHeight(shrunk) !== inputHeight(before)) + throw new Error("Draft did not shrink back to one line"); + assertPlacement("live shrink without reopening", shrunk, "/paseo"); + ad("type", "--", "-committ"); + const partial = await settledPlacement("live-growth-partial", true); + if (!sameText(composerText(partial), "Please do the following now /paseo-committ")) { + throw new Error("Could not prepare a partial command for selection"); + } + const row = assertPlacement("partial command after growth", partial, "/paseo-committee"); + ad("press", String(row.x), String(row.y)); + await waitFor("live-growth-selected", (snap) => + sameText(composerText(snap), "Please do the following now /paseo-committee "), + ); + await clearInput("live-growth-done"); +} + +async function runBoundedPreview() { + const prefix = + "Please do the following / testing inline selection.\n" + + "Line two with details.\nLine three with details.\nLine four with details.\n" + + "Line five with details.\nLine six with details.\nLine seven with details.\nPlease use "; + await runCase("preview", "long", { + text: `${prefix}/openai`, + expected: "/openai-docs", + inserted: `${prefix}/openai-docs `, + selectionKeyboard: "shown", + }); + await runCase("preview", "long-hidden", { + text: `${prefix}/openai`, + expected: "/openai-docs", + inserted: `${prefix}/openai-docs `, + selectionKeyboard: "hidden", + }); + await runCase("preview", "longest", { + text: `${prefix}/paseo-plug`, + expected: "/paseo-plugin", + inserted: `${prefix}/paseo-plugin `, + selectionKeyboard: "shown", + }); + await runCase("preview", "short", { + text: `${prefix}/com`, + expected: "/commit", + inserted: `${prefix}/commit `, + selectionKeyboard: "shown", + }); + await runCase("preview", "files", fileCase); +} + +if (scenario !== "all") { + ad( + "open", + `paseo://h/${serverId}/workspace/${workspaceId}?open=draft%3A${draftId}`, + "--platform", + "ios", + ...deviceArgs, + ); + await waitFor("draft", (snap) => Boolean(nodeById(snap, "message-input-root")), 60_000); + if (scenario === "growth") await runLiveGrowth(); + else if (scenario === "previews") await runBoundedPreview(); + else throw new Error(`Unknown scenario: ${scenario}`); + process.stdout.write(`Composer ${scenario} regression passed. Artifacts: ${artifactsDir}\n`); + process.exit(0); +} + const hosts = [ ["chat", `paseo://h/${serverId}/agent/${agentId}`], ["workspace-draft", `paseo://h/${serverId}/workspace/${workspaceId}?open=draft%3A${draftId}`], ["new-workspace", `paseo://new?serverId=${serverId}`], ]; for (const [host, url] of hosts) { - ad("open", url, "--platform", "ios", "--device", device); + ad("open", url, "--platform", "ios", ...deviceArgs); await waitFor(host, (snap) => Boolean(nodeById(snap, "message-input-root")), 60_000); await runHost(host); } diff --git a/packages/app/src/components/ui/autocomplete-popover.tsx b/packages/app/src/components/ui/autocomplete-popover.tsx index fa5e724944..e74c068658 100644 --- a/packages/app/src/components/ui/autocomplete-popover.tsx +++ b/packages/app/src/components/ui/autocomplete-popover.tsx @@ -5,9 +5,9 @@ import { useRef, useState, type ReactElement, - type RefObject, + type ReactNode, } from "react"; -import { View, useWindowDimensions } from "react-native"; +import { View, useWindowDimensions, type StyleProp, type ViewStyle } from "react-native"; import { Portal } from "@gorhom/portal"; import Animated, { useAnimatedReaction, useAnimatedStyle } from "react-native-reanimated"; import { scheduleOnRN } from "react-native-worklets"; @@ -32,7 +32,8 @@ interface RelativeAnchorRect { interface AutocompletePopoverProps { visible: boolean; - anchorRef: RefObject; + children: ReactNode; + anchorStyle?: StyleProp; options: readonly AutocompleteOption[]; selectedIndex: number; onSelect: (option: AutocompleteOption) => void; @@ -44,7 +45,8 @@ interface AutocompletePopoverProps { export function AutocompletePopover({ visible, - anchorRef, + children, + anchorStyle, options, selectedIndex, onSelect, @@ -55,6 +57,7 @@ export function AutocompletePopover({ }: AutocompletePopoverProps): ReactElement | null { "use no memo"; // React Compiler memoizes effect captures by reading SharedValue.value during render. + const anchorRef = useRef(null); const [relativeAnchorRect, setRelativeAnchorRect] = useState(null); const windowDimensions = useWindowDimensions(); const safeAreaInsets = useSafeAreaInsets(); @@ -137,25 +140,29 @@ export function AutocompletePopover({ [anchorY, reservedLift, safeAreaInsets.top], ); - if (!visible || !relativeAnchorRect || !baseStyle) return null; - if (options.length > 0 && selectedIndex < 0) return null; + const showPopover = canMeasure && relativeAnchorRect && baseStyle; // The translate view is the popover's own frame, so it lifts only the popover. return ( - - - - - - - + + {children} + {showPopover ? ( + + + + + + + + ) : null} + ); } diff --git a/packages/app/src/components/ui/autocomplete.tsx b/packages/app/src/components/ui/autocomplete.tsx index 7b5b99b316..a9cb10f33e 100644 --- a/packages/app/src/components/ui/autocomplete.tsx +++ b/packages/app/src/components/ui/autocomplete.tsx @@ -247,14 +247,16 @@ export function Autocomplete({ {selectedOption?.kind === "command" && selectedOption.description ? ( - + {removeBoltGlyphs(selectedOption.label) ?? selectedOption.label} - + {removeBoltGlyphs(selectedOption.description)} {selectedOption.detail ? ( - {removeBoltGlyphs(selectedOption.detail)} + + {removeBoltGlyphs(selectedOption.detail)} + ) : null} ) : null} @@ -293,6 +295,9 @@ const styles = StyleSheet.create((theme: Theme) => ({ gap: theme.spacing[1], }, detailCard: { + flexShrink: 1, + minHeight: 0, + overflow: "hidden", backgroundColor: theme.colors.surface1, borderWidth: theme.borderWidth[1], borderColor: theme.colors.borderAccent, @@ -318,7 +323,8 @@ const styles = StyleSheet.create((theme: Theme) => ({ }, container: { flexShrink: 1, - minHeight: 0, + // Keep a complete selectable row (36), list padding (8), and borders (2). + minHeight: 46, backgroundColor: theme.colors.surface1, borderWidth: theme.borderWidth[1], borderColor: theme.colors.borderAccent, diff --git a/packages/app/src/composer/index.tsx b/packages/app/src/composer/index.tsx index ba85de2516..d000c269af 100644 --- a/packages/app/src/composer/index.tsx +++ b/packages/app/src/composer/index.tsx @@ -1003,7 +1003,8 @@ function ComposerAutocompleteBinding({ recall, configuration, inputRef, - anchorRef, + children, + anchorStyle, show, ref, }: { @@ -1015,7 +1016,8 @@ function ComposerAutocompleteBinding({ "userInput" | "cursorIndex" | "onAutocompleteApplied" >; inputRef: React.RefObject; - anchorRef: React.RefObject; + children: React.ReactNode; + anchorStyle: React.ComponentProps["style"]; show: boolean; ref: React.Ref; }) { @@ -1042,7 +1044,7 @@ function ComposerAutocompleteBinding({ return ( + > + {children} + ); } @@ -2379,7 +2383,6 @@ function ComposerContentImpl({ pluginClientSlashCommands, ], ); - const messageInputContainerRef = useRef(null); const isSubmitLoadingVisible = isProcessing || isSubmitLoading || isUploadingFile || pendingNativeImagePastes > 0; @@ -2438,17 +2441,16 @@ function ComposerContentImpl({ {isFullscreen ? null : queueList} {!isFullscreen && sendErrorNode} - - + {pluginAttachments.picker} - + From e958540b0a53604d63afcaebdf028f49f3575b2a Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 15:06:02 +0700 Subject: [PATCH 22/32] fix(app): remove SVG prop and nested button warnings on web (#7915) * fix(app): prevent web SVG prop and nested button warnings Consume Unistyles mapping props at the wrapper boundary and use web accessibility attributes for decorative rings. Register composer track rows as menu items so nested actions stay valid and keyboard selection remains available. * test(app): select archive actions by their menu role --- .../e2e/browser/context-window-meter.spec.ts | 9 + packages/app/e2e/support/helpers/subagents.ts | 6 +- .../src/components/context-window-meter.tsx | 5 +- .../composer/tracks-panel.browser.test.tsx | 22 +++ packages/app/src/composer/tracks.tsx | 8 +- .../git/pull-request-panel/checks-ring.tsx | 6 +- .../app/src/styles/unistyles-props.test.ts | 86 ++++++++++ patches/react-native-unistyles+3.2.4.patch | 157 ++++++++++++++++++ scripts/postinstall-patches.mjs | 3 +- 9 files changed, 293 insertions(+), 9 deletions(-) create mode 100644 packages/app/src/styles/unistyles-props.test.ts diff --git a/packages/app/e2e/browser/context-window-meter.spec.ts b/packages/app/e2e/browser/context-window-meter.spec.ts index 79a572a24b..be0835b116 100644 --- a/packages/app/e2e/browser/context-window-meter.spec.ts +++ b/packages/app/e2e/browser/context-window-meter.spec.ts @@ -42,6 +42,14 @@ const test = base.extend<{ agent: MockAgentSession }>({ }, }); +// The outer meter owns the accessible usage label; its SVG is decorative. +async function expectDecorativeContextRing(meter: Locator): Promise { + const ring = meter.locator("svg"); + await expect(ring).toHaveAttribute("aria-hidden", "true"); + await expect(ring).not.toHaveAttribute("accessibilityElementsHidden"); + await expect(ring).not.toHaveAttribute("importantForAccessibility"); +} + // Where the progress arc is painted, as its centroid relative to the ring's centre in pixels. // Reads the rendered pixels, so any rotation that does not reach the screen counts as none. async function progressArcCentroid(meter: Locator): Promise<{ x: number; y: number }> { @@ -100,6 +108,7 @@ test.describe("context window meter", () => { const meter = page.getByTestId("context-window-meter"); await expect(meter).toHaveAccessibleName(/25%/, { timeout: 30_000 }); + await expectDecorativeContextRing(meter); const centroid = await progressArcCentroid(meter); expect(centroid.x).toBeGreaterThan(1); expect(centroid.y).toBeLessThan(-1); diff --git a/packages/app/e2e/support/helpers/subagents.ts b/packages/app/e2e/support/helpers/subagents.ts index 3354bf3dd8..a6fc5bcf21 100644 --- a/packages/app/e2e/support/helpers/subagents.ts +++ b/packages/app/e2e/support/helpers/subagents.ts @@ -188,7 +188,7 @@ export async function expectManagedSubagentArchived( } export async function archiveFinishedSubagents(page: Page): Promise { - await page.getByRole("button", { name: "Archive finished subagents" }).click(); + await page.getByRole("menuitem", { name: "Archive finished subagents" }).click(); } export async function expectArchiveFinishedInProgress( @@ -196,7 +196,7 @@ export async function expectArchiveFinishedInProgress( completed: number, total: number, ): Promise { - await expect(page.getByRole("button", { name: "Archive finished subagents" })).toBeDisabled(); + await expect(page.getByRole("menuitem", { name: "Archive finished subagents" })).toBeDisabled(); await expect(page.getByText(`${completed}/${total}`, { exact: true })).toBeVisible(); } @@ -206,7 +206,7 @@ export async function expectArchiveFinishedRetry( total: number, ): Promise { await expect(page.getByText(`Retry (${failed}/${total})`, { exact: true })).toBeVisible(); - await expect(page.getByRole("button", { name: "Archive finished subagents" })).toBeEnabled(); + await expect(page.getByRole("menuitem", { name: "Archive finished subagents" })).toBeEnabled(); } export async function holdManagedSubagentArchiveRequest( diff --git a/packages/app/src/components/context-window-meter.tsx b/packages/app/src/components/context-window-meter.tsx index f9cfabcf8d..32c739c073 100644 --- a/packages/app/src/components/context-window-meter.tsx +++ b/packages/app/src/components/context-window-meter.tsx @@ -112,8 +112,9 @@ const ContextWindowRing = withUnistyles(function ContextWindowRing({ width={size} height={size} viewBox={`0 0 ${size} ${size}`} - accessibilityElementsHidden - importantForAccessibility="no-hide-descendants" + {...(isNative + ? { accessibilityElementsHidden: true, importantForAccessibility: "no-hide-descendants" } + : { "aria-hidden": true })} > { ); openPanel(); + expect(document.querySelector("button button")).toBeNull(); + expect(row("row")?.getAttribute("role")).toBe("menuitem"); click(row("row-archive") as Element); expect(onAction).toHaveBeenCalledTimes(1); expect(onPress).not.toHaveBeenCalled(); expect(isPanelOpen()).toBe(true); }); + + it.each(["Enter", " "])("selects a track row with %j", (key) => { + const select = vi.fn(); + mount( + + + Subagent one + + , + ); + openPanel(); + const target = row("row") as HTMLElement; + act(() => { + target.focus(); + target.dispatchEvent(new KeyboardEvent("keydown", { key, bubbles: true })); + target.dispatchEvent(new KeyboardEvent("keyup", { key, bubbles: true })); + }); + expect(select).toHaveBeenCalledTimes(1); + expect(isPanelOpen()).toBe(false); + }); }); describe("composer track pill status mark", () => { diff --git a/packages/app/src/composer/tracks.tsx b/packages/app/src/composer/tracks.tsx index 82bee6e3fa..4e06266058 100644 --- a/packages/app/src/composer/tracks.tsx +++ b/packages/app/src/composer/tracks.tsx @@ -233,6 +233,11 @@ export function ComposerTrackRow({ [closeOnSelect, onPress, selectItem], ); + const itemDataSet = useMemo( + () => ({ menuItem: "true", menuDisabled: disabled ? "true" : "false" }), + [disabled], + ); + const renderRow = useCallback( (active: boolean) => ( @@ -253,7 +258,8 @@ export function ComposerTrackRow({ return ( { + it("consumes uniProps while preserving mapped colors and explicit overrides", async () => { + const result = await renderWebProbe(` + import React from 'react'; + import { renderToStaticMarkup } from 'react-dom/server'; + import { StyleSheet, UnistylesRuntime, withUnistyles } from 'react-native-unistyles'; + import { Server } from 'lucide-react-native'; + StyleSheet.configure({ + themes: { light: { color: 'red' }, dark: { color: 'blue' } }, + settings: { initialTheme: 'light' }, + }); + const Icon = withUnistyles(Server); + const mapping = theme => ({ color: theme.color }); + const render = props => renderToStaticMarkup(React.createElement(Icon, { uniProps: mapping, ...props })); + const light = render(); + UnistylesRuntime.setTheme('dark'); + const dark = render(); + const overridden = render({ color: 'green' }); + const received = []; + const Probe = withUnistyles(props => { received.push(props); return null; }); + renderToStaticMarkup(React.createElement(Probe, { uniProps: mapping, id: 'probe' })); + console.log(JSON.stringify({ light, dark, overridden, keys: Object.keys(received[0]).sort() })); + `); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).not.toContain("React does not recognize"); + const rendered = JSON.parse(result.stdout); + expect(rendered.keys).toEqual(["color", "id", "ref"]); + expect(rendered.light).toContain('stroke="red"'); + expect(rendered.dark).toContain('stroke="blue"'); + expect(rendered.overridden).toContain('stroke="green"'); + }); +}); + +describe("decorative checks ring", () => { + it("hides its SVG from web accessibility without forwarding native props", async () => { + const result = await renderWebProbe(` + import React from 'react'; + import { renderToStaticMarkup } from 'react-dom/server'; + import { StyleSheet } from 'react-native-unistyles'; + import { ChecksRing } from './src/git/pull-request-panel/checks-ring'; + StyleSheet.configure({ + themes: { light: { colors: { surface3: 'gray', statusSuccess: 'green' } } }, + settings: { initialTheme: 'light' }, + }); + console.log(renderToStaticMarkup(React.createElement(ChecksRing, { + size: 20, summary: { total: 1, parts: [{ status: 'success', count: 1 }] }, + }))); + `); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('aria-hidden="true"'); + expect(result.stdout).not.toContain("importantForAccessibility"); + expect(result.stdout).not.toContain("accessibilityElementsHidden"); + expect(result.stdout).toContain('stroke="green"'); + expect(result.stderr).not.toContain("React does not recognize"); + }); +}); diff --git a/patches/react-native-unistyles+3.2.4.patch b/patches/react-native-unistyles+3.2.4.patch index 0195868340..7f40053f19 100644 --- a/patches/react-native-unistyles+3.2.4.patch +++ b/patches/react-native-unistyles+3.2.4.patch @@ -329,3 +329,160 @@ index 195954b..c4ac28c 100644 namespace "com.unistyles" defaultConfig { +diff --git a/node_modules/react-native-unistyles/src/core/withUnistyles/withUnistyles.tsx b/node_modules/react-native-unistyles/src/core/withUnistyles/withUnistyles.tsx +--- a/node_modules/react-native-unistyles/src/core/withUnistyles/withUnistyles.tsx ++++ b/node_modules/react-native-unistyles/src/core/withUnistyles/withUnistyles.tsx +@@ -28,13 +28,14 @@ + + return forwardRef, PropsWithUnistyles>((props, ref) => { + const narrowedProps = props as PropsWithUnistyles & UnistyleStyles ++ const { uniProps, ...componentProps } = narrowedProps + const styleClassNames = getClassName(narrowedProps.style, true) + const contentContainerStyleClassNames = getClassName(narrowedProps.contentContainerStyle) + const { proxifiedRuntime, proxifiedTheme } = useProxifiedUnistyles() + + const { key: mappingsKey, ...mappingsProps } = mappings ? mappings(proxifiedTheme, proxifiedRuntime) : {} +- const { key: uniPropsKey, ...unistyleProps } = narrowedProps.uniProps +- ? narrowedProps.uniProps(proxifiedTheme, proxifiedRuntime) ++ const { key: uniPropsKey, ...unistyleProps } = uniProps ++ ? uniProps(proxifiedTheme, proxifiedRuntime) + : {} + + const emptyStyles = narrowedProps.style +@@ -46,7 +47,7 @@ + : undefined + + const combinedProps = { +- ...deepMergeObjects(mappingsProps, unistyleProps, props), ++ ...deepMergeObjects(mappingsProps, unistyleProps, componentProps), + ...(narrowedProps.style + ? { + // Override default component styles with undefined values to reset them +diff --git a/node_modules/react-native-unistyles/src/core/withUnistyles/withUnistyles.native.tsx b/node_modules/react-native-unistyles/src/core/withUnistyles/withUnistyles.native.tsx +--- a/node_modules/react-native-unistyles/src/core/withUnistyles/withUnistyles.native.tsx ++++ b/node_modules/react-native-unistyles/src/core/withUnistyles/withUnistyles.native.tsx +@@ -63,6 +63,7 @@ + + return forwardRef, PropsWithUnistyles>((props, ref) => { + const narrowedProps = props as PropsWithUnistyles & UnistyleStyles ++ const { uniProps, ...componentProps } = narrowedProps + const NativeComponent = Component as ComponentType + + // @ts-ignore we don't know the type of the component +@@ -89,15 +90,15 @@ + }, [narrowedProps.style, narrowedProps.contentContainerStyle]) + + const { key: mappingsKey, ...mappingsProps } = mappings ? mappings(proxifiedTheme, proxifiedRuntime) : {} +- const { key: uniPropsKey, ...unistyleProps } = narrowedProps.uniProps +- ? narrowedProps.uniProps(proxifiedTheme, proxifiedRuntime) ++ const { key: uniPropsKey, ...unistyleProps } = uniProps ++ ? uniProps(proxifiedTheme, proxifiedRuntime) + : {} + + const styleSecrets = getSecrets(narrowedProps.style) + const contentContainerStyleSecrets = getSecrets(narrowedProps.contentContainerStyle) + + const finalProps = { +- ...deepMergeObjects(mappingsProps, unistyleProps, props), ++ ...deepMergeObjects(mappingsProps, unistyleProps, componentProps), + ...(narrowedProps.style + ? { + style: styleSecrets.styles, +diff --git a/node_modules/react-native-unistyles/lib/module/core/withUnistyles/withUnistyles.js b/node_modules/react-native-unistyles/lib/module/core/withUnistyles/withUnistyles.js +--- a/node_modules/react-native-unistyles/lib/module/core/withUnistyles/withUnistyles.js ++++ b/node_modules/react-native-unistyles/lib/module/core/withUnistyles/withUnistyles.js +@@ -13,6 +13,7 @@ + export const withUnistyles = (Component, mappings) => { + return /*#__PURE__*/forwardRef((props, ref) => { + const narrowedProps = props; ++ const { uniProps, ...componentProps } = narrowedProps; + const styleClassNames = getClassName(narrowedProps.style, true); + const contentContainerStyleClassNames = getClassName(narrowedProps.contentContainerStyle); + const { +@@ -26,10 +27,10 @@ + const { + key: uniPropsKey, + ...unistyleProps +- } = narrowedProps.uniProps ? narrowedProps.uniProps(proxifiedTheme, proxifiedRuntime) : {}; ++ } = uniProps ? uniProps(proxifiedTheme, proxifiedRuntime) : {}; + const emptyStyles = narrowedProps.style ? Object.fromEntries(Object.entries(Object.getOwnPropertyDescriptors(narrowedProps.style)).filter(([key]) => !key.startsWith('unistyles') && !key.startsWith('_')).map(([key]) => [key, undefined])) : undefined; + const combinedProps = { +- ...deepMergeObjects(mappingsProps, unistyleProps, props), ++ ...deepMergeObjects(mappingsProps, unistyleProps, componentProps), + ...(narrowedProps.style ? { + // Override default component styles with undefined values to reset them + style: emptyStyles +diff --git a/node_modules/react-native-unistyles/lib/module/core/withUnistyles/withUnistyles.native.js b/node_modules/react-native-unistyles/lib/module/core/withUnistyles/withUnistyles.native.js +--- a/node_modules/react-native-unistyles/lib/module/core/withUnistyles/withUnistyles.native.js ++++ b/node_modules/react-native-unistyles/lib/module/core/withUnistyles/withUnistyles.native.js +@@ -27,6 +27,7 @@ + }; + return /*#__PURE__*/forwardRef((props, ref) => { + const narrowedProps = props; ++ const { uniProps, ...componentProps } = narrowedProps; + const NativeComponent = Component; + + // @ts-ignore we don't know the type of the component +@@ -57,11 +58,11 @@ + const { + key: uniPropsKey, + ...unistyleProps +- } = narrowedProps.uniProps ? narrowedProps.uniProps(proxifiedTheme, proxifiedRuntime) : {}; ++ } = uniProps ? uniProps(proxifiedTheme, proxifiedRuntime) : {}; + const styleSecrets = getSecrets(narrowedProps.style); + const contentContainerStyleSecrets = getSecrets(narrowedProps.contentContainerStyle); + const finalProps = { +- ...deepMergeObjects(mappingsProps, unistyleProps, props), ++ ...deepMergeObjects(mappingsProps, unistyleProps, componentProps), + ...(narrowedProps.style ? { + style: styleSecrets.styles + } : {}), +diff --git a/node_modules/react-native-unistyles/lib/commonjs/core/withUnistyles/withUnistyles.js b/node_modules/react-native-unistyles/lib/commonjs/core/withUnistyles/withUnistyles.js +--- a/node_modules/react-native-unistyles/lib/commonjs/core/withUnistyles/withUnistyles.js ++++ b/node_modules/react-native-unistyles/lib/commonjs/core/withUnistyles/withUnistyles.js +@@ -18,6 +18,7 @@ + const withUnistyles = (Component, mappings) => { + return /*#__PURE__*/(0, _react.forwardRef)((props, ref) => { + const narrowedProps = props; ++ const { uniProps, ...componentProps } = narrowedProps; + const styleClassNames = (0, _getClassname.getClassName)(narrowedProps.style, true); + const contentContainerStyleClassNames = (0, _getClassname.getClassName)(narrowedProps.contentContainerStyle); + const { +@@ -31,10 +32,10 @@ + const { + key: uniPropsKey, + ...unistyleProps +- } = narrowedProps.uniProps ? narrowedProps.uniProps(proxifiedTheme, proxifiedRuntime) : {}; ++ } = uniProps ? uniProps(proxifiedTheme, proxifiedRuntime) : {}; + const emptyStyles = narrowedProps.style ? Object.fromEntries(Object.entries(Object.getOwnPropertyDescriptors(narrowedProps.style)).filter(([key]) => !key.startsWith('unistyles') && !key.startsWith('_')).map(([key]) => [key, undefined])) : undefined; + const combinedProps = { +- ...(0, _utils.deepMergeObjects)(mappingsProps, unistyleProps, props), ++ ...(0, _utils.deepMergeObjects)(mappingsProps, unistyleProps, componentProps), + ...(narrowedProps.style ? { + // Override default component styles with undefined values to reset them + style: emptyStyles +diff --git a/node_modules/react-native-unistyles/lib/commonjs/core/withUnistyles/withUnistyles.native.js b/node_modules/react-native-unistyles/lib/commonjs/core/withUnistyles/withUnistyles.native.js +--- a/node_modules/react-native-unistyles/lib/commonjs/core/withUnistyles/withUnistyles.native.js ++++ b/node_modules/react-native-unistyles/lib/commonjs/core/withUnistyles/withUnistyles.native.js +@@ -32,6 +32,7 @@ + }; + return /*#__PURE__*/(0, _react.forwardRef)((props, ref) => { + const narrowedProps = props; ++ const { uniProps, ...componentProps } = narrowedProps; + const NativeComponent = Component; + + // @ts-ignore we don't know the type of the component +@@ -62,11 +63,11 @@ + const { + key: uniPropsKey, + ...unistyleProps +- } = narrowedProps.uniProps ? narrowedProps.uniProps(proxifiedTheme, proxifiedRuntime) : {}; ++ } = uniProps ? uniProps(proxifiedTheme, proxifiedRuntime) : {}; + const styleSecrets = getSecrets(narrowedProps.style); + const contentContainerStyleSecrets = getSecrets(narrowedProps.contentContainerStyle); + const finalProps = { +- ...(0, _utils.deepMergeObjects)(mappingsProps, unistyleProps, props), ++ ...(0, _utils.deepMergeObjects)(mappingsProps, unistyleProps, componentProps), + ...(narrowedProps.style ? { + style: styleSecrets.styles + } : {}), diff --git a/scripts/postinstall-patches.mjs b/scripts/postinstall-patches.mjs index b009be9841..09ff2919b6 100644 --- a/scripts/postinstall-patches.mjs +++ b/scripts/postinstall-patches.mjs @@ -19,8 +19,9 @@ const patchedPackages = [ patchPrefix: "react-native-enriched-markdown+", cwd: "packages/app", }, - // Remove after react-native-unistyles ships + // Registry cleanup: remove after react-native-unistyles ships // https://github.com/jpudysz/react-native-unistyles/pull/1203. + // withUnistyles: also keep the uniProps filter until upstream consumes its control prop. { nodeModulesPath: "node_modules/react-native-unistyles", patchPrefix: "react-native-unistyles+", From e277656cbdda4f84320fd3cbf2f806e67d2bb742 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 15:46:04 +0700 Subject: [PATCH 23/32] fix(app): clean up native animation and layout lifecycles (#7914) Worklets 0.10.4 and Reanimated 4.5.5 while keeping the stale-view guard from #7907; backport the Enriched Markdown settled-row measurement fix (upstream #924); release each bottom sheet's Dimensions subscription on unmount. --- package-lock.json | 51 ++------ package.json | 4 +- packages/app/package.json | 12 +- .../index.test.tsx | 33 ++++++ patches/@gorhom+bottom-sheet+5.2.14.patch | 62 ++++++++++ ...own+1.2.0-nightly-20261014-766fc0e67.patch | 109 +++++++++++++++++- ...ch => react-native-reanimated+4.5.5.patch} | 0 scripts/postinstall-patches.mjs | 6 + 8 files changed, 226 insertions(+), 51 deletions(-) rename patches/{react-native-reanimated+4.5.1.patch => react-native-reanimated+4.5.5.patch} (100%) diff --git a/package-lock.json b/package-lock.json index 64d1415515..1f5cce1657 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30269,9 +30269,9 @@ } }, "node_modules/react-native-reanimated": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-4.5.1.tgz", - "integrity": "sha512-RnMvtDuR+68ig864gAvZCOdZehqhC5rFmMo0kn+ARfgVSTvFeF6IFLBVgMPUu0KwihaapEyW24WRi6nEyy1kSA==", + "version": "4.5.5", + "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-4.5.5.tgz", + "integrity": "sha512-xtJXZRZ1vkec1AIUVG02St2sbyZS5jc7227TlDu/HonqUYPeZAfiC4RI2jqL1dVMNBu1+7oBx/c9QkjF2kg6Lg==", "license": "MIT", "dependencies": { "react-native-is-edge-to-edge": "^1.3.1", @@ -30280,7 +30280,7 @@ "peerDependencies": { "react": "*", "react-native": "0.83 - 0.86", - "react-native-worklets": "0.10.x" + "react-native-worklets": "0.10.x - 0.11.x" } }, "node_modules/react-native-safe-area-context": { @@ -30406,9 +30406,9 @@ } }, "node_modules/react-native-worklets": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/react-native-worklets/-/react-native-worklets-0.10.1.tgz", - "integrity": "sha512-62mRM19bDpfpdI8HLkEErcdOsrAPDtE9lA/sw+5lLRpzBHNhxaoj9QyY2KjXqUmirelxkX4zuPGTC3VdA0feJA==", + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/react-native-worklets/-/react-native-worklets-0.10.4.tgz", + "integrity": "sha512-gbk0CxMrGywEp3SJAHO+2JVaCWtl4WyAYBIxLZy6Y113+mHHuq3FuT6iV+VcaBuJzuuN34mtdZ4j5JIlBDzmeA==", "license": "MIT", "dependencies": { "@babel/plugin-transform-arrow-functions": "^7.27.1", @@ -35780,14 +35780,14 @@ "react-native-gesture-handler": "~2.32.0", "react-native-keyboard-controller": "1.21.12", "react-native-nitro-modules": "0.35.5", - "react-native-reanimated": "4.5.1", + "react-native-reanimated": "4.5.5", "react-native-safe-area-context": "~5.7.0", "react-native-screens": "~4.26.0", "react-native-svg": "15.15.4", "react-native-unistyles": "^3.2.4", "react-native-web": "~0.21.0", "react-native-webview": "13.16.1", - "react-native-worklets": "0.10.1", + "react-native-worklets": "0.10.4", "tiny-invariant": "^1.3.3", "turndown": "^7.2.4", "turndown-plugin-gfm": "^1.0.2", @@ -41012,39 +41012,6 @@ } } }, - "packages/expo-two-way-audio/node_modules/expo-module-scripts/node_modules/react-native-worklets": { - "version": "0.13.0", - "resolved": "https://registry.npmjs.org/react-native-worklets/-/react-native-worklets-0.13.0.tgz", - "integrity": "sha512-1OrhAeAKaZF4XrBa4YbPQbKzNOpjh52HDlId8e6ete1OCHNdAlrQHzG4gznJ225EAj3dgQXHViJvK51M9B52jQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@babel/generator": "^7.27.1", - "@babel/plugin-transform-arrow-functions": "^7.27.1", - "@babel/plugin-transform-class-properties": "^7.28.6", - "@babel/plugin-transform-classes": "^7.28.6", - "@babel/plugin-transform-nullish-coalescing-operator": "^7.28.6", - "@babel/plugin-transform-optional-chaining": "^7.28.6", - "@babel/plugin-transform-shorthand-properties": "^7.27.1", - "@babel/plugin-transform-template-literals": "^7.27.1", - "@babel/plugin-transform-unicode-regex": "^7.27.1", - "@babel/preset-typescript": "^7.28.5", - "@babel/traverse": "^7.27.1", - "@babel/types": "^7.27.1", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "semver": "^7.7.4", - "whatwg-fetch": "^3.6.20" - }, - "peerDependencies": { - "@babel/core": "*", - "@react-native/metro-config": "*", - "react": "*", - "react-native": "0.86 - 0.88" - } - }, "packages/expo-two-way-audio/node_modules/expo-module-scripts/node_modules/react-native/node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", diff --git a/package.json b/package.json index 9c18bb0db8..904e7e6af8 100644 --- a/package.json +++ b/package.json @@ -154,7 +154,7 @@ "react-dom": "19.2.3", "react-native": "0.86.3", "react-native-gesture-handler": "~2.32.0", - "react-native-reanimated": "4.5.1", - "react-native-worklets": "0.10.1" + "react-native-reanimated": "4.5.5", + "react-native-worklets": "0.10.4" } } diff --git a/packages/app/package.json b/packages/app/package.json index d72685bbec..e2e39ec03d 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -126,14 +126,14 @@ "react-native-gesture-handler": "~2.32.0", "react-native-keyboard-controller": "1.21.12", "react-native-nitro-modules": "0.35.5", - "react-native-reanimated": "4.5.1", + "react-native-reanimated": "4.5.5", "react-native-safe-area-context": "~5.7.0", "react-native-screens": "~4.26.0", "react-native-svg": "15.15.4", "react-native-unistyles": "^3.2.4", "react-native-web": "~0.21.0", "react-native-webview": "13.16.1", - "react-native-worklets": "0.10.1", + "react-native-worklets": "0.10.4", "tiny-invariant": "^1.3.3", "turndown": "^7.2.4", "turndown-plugin-gfm": "^1.0.2", @@ -198,6 +198,14 @@ ], "enableVideo": false }, + "expo": { + "install": { + "exclude": [ + "react-native-reanimated", + "react-native-worklets" + ] + } + }, "reanimated": { "staticFeatureFlags": { "FORCE_REACT_RENDER_FOR_SETTLED_ANIMATIONS": false diff --git a/packages/app/src/components/ui/isolated-bottom-sheet-modal/index.test.tsx b/packages/app/src/components/ui/isolated-bottom-sheet-modal/index.test.tsx index cf820ac86c..6637c5e339 100644 --- a/packages/app/src/components/ui/isolated-bottom-sheet-modal/index.test.tsx +++ b/packages/app/src/components/ui/isolated-bottom-sheet-modal/index.test.tsx @@ -58,6 +58,7 @@ vi.mock("@gorhom/bottom-sheet", async () => { }); import { IsolatedBottomSheetModal, type ContextBridge } from "."; +import { useAnimatedLayout } from "@gorhom/bottom-sheet/src/hooks/useAnimatedLayout"; import { useIsInsideBottomSheet } from "@/components/ui/bottom-sheet-scope"; /** Renders whatever the portal is currently holding, from its own place in the tree. */ @@ -200,3 +201,35 @@ describe("IsolatedBottomSheetModal context bridging", () => { ).toEqual(["false", "true"]); }); }); + +describe("bottom sheet layout subscription", () => { + it("stops updating a dismissed sheet when the window resizes", () => { + const container = document.createElement("div"); + const root = createRoot(container); + let layout: ReturnType | undefined; + const viewport = document.documentElement; + const originalWidth = Object.getOwnPropertyDescriptor(viewport, "clientWidth"); + function resize(width: number) { + Object.defineProperty(viewport, "clientWidth", { configurable: true, value: width }); + act(() => window.dispatchEvent(new Event("resize"))); + } + function LayoutProbe() { + layout = useAnimatedLayout(undefined, 0, 0); + return null; + } + try { + act(() => root.render()); + if (!layout) throw new Error("Sheet layout did not mount"); + const mountedLayout = layout; + resize(640); + expect(mountedLayout.get().window.width).toBe(640); + act(() => root.render(null)); + resize(960); + expect(mountedLayout.get().window.width).toBe(640); + } finally { + act(() => root.unmount()); + if (originalWidth) Object.defineProperty(viewport, "clientWidth", originalWidth); + else Reflect.deleteProperty(viewport, "clientWidth"); + } + }); +}); diff --git a/patches/@gorhom+bottom-sheet+5.2.14.patch b/patches/@gorhom+bottom-sheet+5.2.14.patch index f84d46f8fd..e0af7b8dcb 100644 --- a/patches/@gorhom+bottom-sheet+5.2.14.patch +++ b/patches/@gorhom+bottom-sheet+5.2.14.patch @@ -13,6 +13,27 @@ index ee51e76..dc1bc17 100644 if (_providedOnChange) { _providedOnChange(_index, _position, _type); } +diff --git a/node_modules/@gorhom/bottom-sheet/lib/commonjs/hooks/useAnimatedLayout.js b/node_modules/@gorhom/bottom-sheet/lib/commonjs/hooks/useAnimatedLayout.js +index 973d05c..b484b5f 100644 +--- a/node_modules/@gorhom/bottom-sheet/lib/commonjs/hooks/useAnimatedLayout.js ++++ b/node_modules/@gorhom/bottom-sheet/lib/commonjs/hooks/useAnimatedLayout.js +@@ -86,7 +86,7 @@ function useAnimatedLayout(containerLayoutState, topInset, bottomInset, modal, s + }); + }, [state, verticalInset, modal]); + (0, _react.useEffect)(() => { +- _reactNative.Dimensions.addEventListener('change', ({ ++ const subscription = _reactNative.Dimensions.addEventListener('change', ({ + window + }) => { + state.modify(_state => { +@@ -96,6 +96,7 @@ function useAnimatedLayout(containerLayoutState, topInset, bottomInset, modal, s + return _state; + }); + }); ++ return () => subscription.remove(); + }, [state]); + //#endregion + diff --git a/node_modules/@gorhom/bottom-sheet/lib/commonjs/index.js b/node_modules/@gorhom/bottom-sheet/lib/commonjs/index.js index e44ad86..96555ef 100644 --- a/node_modules/@gorhom/bottom-sheet/lib/commonjs/index.js @@ -54,6 +75,27 @@ index 94fabdb..60ca66a 100644 if (_providedOnChange) { _providedOnChange(_index, _position, _type); } +diff --git a/node_modules/@gorhom/bottom-sheet/lib/module/hooks/useAnimatedLayout.js b/node_modules/@gorhom/bottom-sheet/lib/module/hooks/useAnimatedLayout.js +index b0eec93..67713a2 100644 +--- a/node_modules/@gorhom/bottom-sheet/lib/module/hooks/useAnimatedLayout.js ++++ b/node_modules/@gorhom/bottom-sheet/lib/module/hooks/useAnimatedLayout.js +@@ -82,7 +82,7 @@ export function useAnimatedLayout(containerLayoutState, topInset, bottomInset, m + }); + }, [state, verticalInset, modal]); + useEffect(() => { +- Dimensions.addEventListener('change', ({ ++ const subscription = Dimensions.addEventListener('change', ({ + window + }) => { + state.modify(_state => { +@@ -92,6 +92,7 @@ export function useAnimatedLayout(containerLayoutState, topInset, bottomInset, m + return _state; + }); + }); ++ return () => subscription.remove(); + }, [state]); + //#endregion + diff --git a/node_modules/@gorhom/bottom-sheet/lib/module/index.js b/node_modules/@gorhom/bottom-sheet/lib/module/index.js index 8259637..eb057fe 100644 --- a/node_modules/@gorhom/bottom-sheet/lib/module/index.js @@ -101,6 +143,26 @@ index 2954b3f..7260656 100644 if (_providedOnChange) { _providedOnChange(_index, _position, _type); +diff --git a/node_modules/@gorhom/bottom-sheet/src/hooks/useAnimatedLayout.ts b/node_modules/@gorhom/bottom-sheet/src/hooks/useAnimatedLayout.ts +index 68c674d..564833c 100644 +--- a/node_modules/@gorhom/bottom-sheet/src/hooks/useAnimatedLayout.ts ++++ b/node_modules/@gorhom/bottom-sheet/src/hooks/useAnimatedLayout.ts +@@ -106,13 +106,14 @@ export function useAnimatedLayout( + [state, verticalInset, modal] + ); + useEffect(() => { +- Dimensions.addEventListener('change', ({ window }) => { ++ const subscription = Dimensions.addEventListener('change', ({ window }) => { + state.modify(_state => { + 'worklet'; + _state.window = window; + return _state; + }); + }); ++ return () => subscription.remove(); + }, [state]); + //#endregion + diff --git a/node_modules/@gorhom/bottom-sheet/src/index.ts b/node_modules/@gorhom/bottom-sheet/src/index.ts index 7aaea57..6a5f5a4 100644 --- a/node_modules/@gorhom/bottom-sheet/src/index.ts diff --git a/patches/react-native-enriched-markdown+1.2.0-nightly-20261014-766fc0e67.patch b/patches/react-native-enriched-markdown+1.2.0-nightly-20261014-766fc0e67.patch index 737578cd69..7ec1ec33b5 100644 --- a/patches/react-native-enriched-markdown+1.2.0-nightly-20261014-766fc0e67.patch +++ b/patches/react-native-enriched-markdown+1.2.0-nightly-20261014-766fc0e67.patch @@ -39,9 +39,10 @@ index 0a1c6c2..5935112 100644 struct OnLinkLongPress { diff --git a/node_modules/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/EnrichedMarkdown.kt b/node_modules/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/EnrichedMarkdown.kt +index c12937c..8105833 100644 --- a/node_modules/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/EnrichedMarkdown.kt +++ b/node_modules/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/EnrichedMarkdown.kt -@@ -95,7 +95,7 @@ +@@ -95,7 +95,7 @@ class EnrichedMarkdown( private var accessibilityLabels = AccessibilityLabels() private var textBreakStrategy: String = BreakStrategyUtils.DEFAULT_STRATEGY @@ -50,7 +51,7 @@ diff --git a/node_modules/react-native-enriched-markdown/android/src/main/java/c private var onLinkLongPressCallback: ((String) -> Unit)? = null private var onTaskListItemPressCallback: ((Int, Boolean, String) -> Unit)? = null private var onLatexErrorCallback: LatexErrorReporter? = null -@@ -296,7 +296,7 @@ +@@ -296,7 +296,7 @@ class EnrichedMarkdown( } } @@ -59,7 +60,7 @@ diff --git a/node_modules/react-native-enriched-markdown/android/src/main/java/c onLinkPressCallback = callback } -@@ -405,7 +405,7 @@ +@@ -405,7 +405,7 @@ class EnrichedMarkdown( val codeLinkRegex = inlineCodeLinkRegex val renderId = ++currentRenderId @@ -68,7 +69,7 @@ diff --git a/node_modules/react-native-enriched-markdown/android/src/main/java/c try { val filtered = if (isStreaming) { -@@ -418,13 +418,13 @@ +@@ -418,13 +418,13 @@ class EnrichedMarkdown( if (renderableMarkdown.isEmpty()) { postToMain(renderId) { applyRenderedSegments(emptyList(), false) } @@ -84,7 +85,19 @@ diff --git a/node_modules/react-native-enriched-markdown/android/src/main/java/c } val segments = splitASTIntoSegments(ast) -@@ -646,8 +646,12 @@ +@@ -478,6 +478,11 @@ class EnrichedMarkdown( + if (forceHeight || topologyChanged || heightBefore != heightAfter) { + MeasurementStore.invalidate(id) + requestLayout() ++ stateWrapper?.let { ++ val state = Arguments.createMap() ++ state.putInt("forceHeightRecalculationCounter", ++forceHeightRecalculationCounter) ++ it.updateState(state) ++ } + } + } + } +@@ -646,8 +651,12 @@ class EnrichedMarkdown( renderId: Long, action: () -> Unit, ) { @@ -3671,3 +3684,89 @@ diff --git a/node_modules/react-native-enriched-markdown/android/src/main/java/c val preservedStyle = (paint.typeface?.style ?: 0) and (Typeface.BOLD or Typeface.ITALIC) paint.typeface = if (styleCache.codeFontFamily.isNotEmpty()) { +diff --git a/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownContainerShadowNode.cpp b/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownContainerShadowNode.cpp +index a430d1a..b30a017 100644 +--- a/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownContainerShadowNode.cpp ++++ b/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownContainerShadowNode.cpp +@@ -22,6 +22,14 @@ void MarkdownContainerShadowNode::dirtyLayoutIfNeeded() { + } + } + ++// Only new props change the rendered markdown. When a parent is cloned, React Native's adoptYogaChild ++// re-clones its children without new props, and dirtying those clones re-measures every settled message ++// whenever the list above grows. ++bool MarkdownContainerShadowNode::shouldNewRevisionDirtyMeasurement(const ShadowNode & /*sourceShadowNode*/, ++ const ShadowNodeFragment &fragment) const { ++ return fragment.props != nullptr; ++} ++ + Size MarkdownContainerShadowNode::measureContent(const LayoutContext &layoutContext, + const LayoutConstraints &layoutConstraints) const { + return measurementsManager_->measure(getSurfaceId(), getTag(), getConcreteProps(), layoutConstraints); +diff --git a/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownContainerShadowNode.h b/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownContainerShadowNode.h +index dea8bb2..6c4bd20 100644 +--- a/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownContainerShadowNode.h ++++ b/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownContainerShadowNode.h +@@ -19,6 +19,8 @@ public: + + MarkdownContainerShadowNode(ShadowNode const &sourceShadowNode, ShadowNodeFragment const &fragment) + : ConcreteViewShadowNode(sourceShadowNode, fragment) { ++ forceHeightRecalculationCounter_ = ++ static_cast(sourceShadowNode).forceHeightRecalculationCounter_; + dirtyLayoutIfNeeded(); + } + +@@ -35,6 +37,10 @@ public: + + Size measureContent(const LayoutContext &layoutContext, const LayoutConstraints &layoutConstraints) const override; + ++protected: ++ bool shouldNewRevisionDirtyMeasurement(const ShadowNode &sourceShadowNode, ++ const ShadowNodeFragment &fragment) const override; ++ + private: + int forceHeightRecalculationCounter_{0}; + std::shared_ptr measurementsManager_; +diff --git a/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownTextShadowNode.cpp b/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownTextShadowNode.cpp +index 3ca259b..ab51055 100644 +--- a/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownTextShadowNode.cpp ++++ b/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownTextShadowNode.cpp +@@ -22,6 +22,14 @@ void MarkdownTextShadowNode::dirtyLayoutIfNeeded() { + } + } + ++// Only new props change the rendered markdown. When a parent is cloned, React Native's adoptYogaChild ++// re-clones its children without new props, and dirtying those clones re-measures every settled message ++// whenever the list above grows. ++bool MarkdownTextShadowNode::shouldNewRevisionDirtyMeasurement(const ShadowNode & /*sourceShadowNode*/, ++ const ShadowNodeFragment &fragment) const { ++ return fragment.props != nullptr; ++} ++ + Size MarkdownTextShadowNode::measureContent(const LayoutContext &layoutContext, + const LayoutConstraints &layoutConstraints) const { + return measurementsManager_->measure(getSurfaceId(), getTag(), getConcreteProps(), layoutConstraints); +diff --git a/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownTextShadowNode.h b/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownTextShadowNode.h +index eaa3279..05a02eb 100644 +--- a/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownTextShadowNode.h ++++ b/node_modules/react-native-enriched-markdown/android/src/main/jni/react/renderer/components/EnrichedMarkdownTextSpec/MarkdownTextShadowNode.h +@@ -22,6 +22,8 @@ public: + + MarkdownTextShadowNode(ShadowNode const &sourceShadowNode, ShadowNodeFragment const &fragment) + : ConcreteViewShadowNode(sourceShadowNode, fragment) { ++ forceHeightRecalculationCounter_ = ++ static_cast(sourceShadowNode).forceHeightRecalculationCounter_; + dirtyLayoutIfNeeded(); + } + +@@ -39,6 +41,10 @@ public: + + Size measureContent(const LayoutContext &layoutContext, const LayoutConstraints &layoutConstraints) const override; + ++protected: ++ bool shouldNewRevisionDirtyMeasurement(const ShadowNode &sourceShadowNode, ++ const ShadowNodeFragment &fragment) const override; ++ + private: + int forceHeightRecalculationCounter_{0}; + std::shared_ptr measurementsManager_; diff --git a/patches/react-native-reanimated+4.5.1.patch b/patches/react-native-reanimated+4.5.5.patch similarity index 100% rename from patches/react-native-reanimated+4.5.1.patch rename to patches/react-native-reanimated+4.5.5.patch diff --git a/scripts/postinstall-patches.mjs b/scripts/postinstall-patches.mjs index 09ff2919b6..435f6331e0 100644 --- a/scripts/postinstall-patches.mjs +++ b/scripts/postinstall-patches.mjs @@ -14,6 +14,9 @@ const patchedPackages = [ nodeModulesPath: "node_modules/expo-modules-core", patchPrefix: "expo-modules-core+", }, + // Backport https://github.com/software-mansion/enriched-markdown/pull/924: preserve settled + // Android measurements across parent clones. Remove those hunks when upgrading past + // 1.2.0-nightly-20261019-926ef06f0; reconcile the other local patch hunks separately. { nodeModulesPath: "packages/app/node_modules/react-native-enriched-markdown", patchPrefix: "react-native-enriched-markdown+", @@ -46,6 +49,9 @@ const patchedPackages = [ // modal stack (see packages/app/src/components/ui/isolated-bottom-sheet-modal/host-scope.tsx), and // keeps a dismissing modal's status through its final onChange(-1), so a modal with // enableDismissOnClose=false that unmounts while presented still leaves the portal. + // Dimensions subscription cleanup: https://github.com/gorhom/react-native-bottom-sheet/issues/2734 + // and https://github.com/gorhom/react-native-bottom-sheet/pull/2751. Remove the useAnimatedLayout + // hunks once an installed release includes the cleanup; retain the other local hunks. { nodeModulesPath: "node_modules/@gorhom/bottom-sheet", patchPrefix: "@gorhom+bottom-sheet+", From 428ef707adc5c3c7e0396c19d21569b2e48ce5b6 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 15:46:10 +0700 Subject: [PATCH 24/32] fix(app): demand the host directory from the workspace route (#7918) A workspace restored at launch with no open chat had nothing demanding its host's directory on a phone, so the deck waited on hydration that never started. The workspace route now keeps its host's directory demanded while mounted, through the same hook the agent route uses. --- .../src/app/h/[serverId]/agent/[agentId].tsx | 7 +--- .../workspace/[workspaceId]/index.tsx | 5 +++ .../src/hooks/use-directory-demand.test.tsx | 40 +++++++++++++++++++ .../app/src/hooks/use-directory-demand.ts | 16 ++++++++ 4 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 packages/app/src/hooks/use-directory-demand.test.tsx create mode 100644 packages/app/src/hooks/use-directory-demand.ts diff --git a/packages/app/src/app/h/[serverId]/agent/[agentId].tsx b/packages/app/src/app/h/[serverId]/agent/[agentId].tsx index 008999a540..551e3b604d 100644 --- a/packages/app/src/app/h/[serverId]/agent/[agentId].tsx +++ b/packages/app/src/app/h/[serverId]/agent/[agentId].tsx @@ -5,6 +5,7 @@ import { useFetchQuery } from "@/data/query"; import { resolveAgentRoute, type AgentRouteLookup } from "@/navigation/agent-route-resolution"; import { AgentRouteResolutionView } from "@/navigation/agent-route-resolution-view"; import { useHasHydratedWorkspaces } from "@/stores/session-store-hooks"; +import { useDirectoryDemand } from "@/hooks/use-directory-demand"; import { useWorkspaceLayoutStoreHydrated } from "@/stores/workspace-layout-store"; import { useSessionStore } from "@/stores/session-store"; import { getHostRuntimeStore, useHostRuntimeSnapshot, useHosts } from "@/runtime/host-runtime"; @@ -87,11 +88,7 @@ function HostAgentReadyRouteContent() { ]); const hasHydratedWorkspaces = useHasHydratedWorkspaces(serverId); const hasHydratedLayout = useWorkspaceLayoutStoreHydrated(); - useEffect(() => { - if (!serverId) return; - // A compact app can open this route before its sidebar has demanded the directory. - return getHostRuntimeStore().acquireDirectoryDemand(serverId); - }, [serverId]); + useDirectoryDemand(serverId); const resolution = resolveAgentRoute({ hasHydratedWorkspaces, hasHydratedLayout, diff --git a/packages/app/src/app/h/[serverId]/workspace/[workspaceId]/index.tsx b/packages/app/src/app/h/[serverId]/workspace/[workspaceId]/index.tsx index 8f80f185c7..2ea41196aa 100644 --- a/packages/app/src/app/h/[serverId]/workspace/[workspaceId]/index.tsx +++ b/packages/app/src/app/h/[serverId]/workspace/[workspaceId]/index.tsx @@ -9,6 +9,7 @@ import { useActiveWorkspaceSelection, } from "@/stores/navigation-active-workspace-store"; import { useHasHydratedWorkspaces, useWorkspaceExists } from "@/stores/session-store-hooks"; +import { useDirectoryDemand } from "@/hooks/use-directory-demand"; import type { WorkspaceTabTarget } from "@/workspace-tabs/model"; import { WorkspaceScreen } from "@/screens/workspace/workspace-screen"; import { useWorkspaceLayoutStore } from "@/stores/workspace-layout-store"; @@ -110,6 +111,10 @@ function HostWorkspaceRouteContent() { ? (decodeWorkspaceIdFromPathSegment(workspaceValue) ?? "") : ""; const openValue = getParamValue(params.open); + // A restored workspace with no open chat has nothing else demanding its host's directory on a + // phone, where the sidebar only demands while its panel is active. Without demand the deck + // waits on workspace hydration that never starts. + useDirectoryDemand(serverId); useEffect(() => { if (!serverId || !workspaceId) { return; diff --git a/packages/app/src/hooks/use-directory-demand.test.tsx b/packages/app/src/hooks/use-directory-demand.test.tsx new file mode 100644 index 0000000000..7a03d83471 --- /dev/null +++ b/packages/app/src/hooks/use-directory-demand.test.tsx @@ -0,0 +1,40 @@ +/** + * @vitest-environment jsdom + */ +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { getHostRuntimeStore } from "@/runtime/host-runtime"; +import { useDirectoryDemand } from "./use-directory-demand"; + +function Probe({ serverId }: { serverId: string | null }) { + useDirectoryDemand(serverId); + return null; +} + +describe("useDirectoryDemand", () => { + let root: Root | null = null; + afterEach(() => { + act(() => root?.unmount()); + root = null; + vi.restoreAllMocks(); + }); + + it("demands the host directory while mounted and releases it on unmount", () => { + const release = vi.fn(); + const acquire = vi + .spyOn(getHostRuntimeStore(), "acquireDirectoryDemand") + .mockImplementation(() => release); + const container = document.createElement("div"); + root = createRoot(container); + + act(() => root?.render()); + expect(acquire).toHaveBeenCalledTimes(1); + expect(acquire).toHaveBeenCalledWith("srv_a"); + expect(release).not.toHaveBeenCalled(); + + act(() => root?.render()); + expect(release).toHaveBeenCalledTimes(1); + expect(acquire).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/app/src/hooks/use-directory-demand.ts b/packages/app/src/hooks/use-directory-demand.ts new file mode 100644 index 0000000000..f4d2953ab0 --- /dev/null +++ b/packages/app/src/hooks/use-directory-demand.ts @@ -0,0 +1,16 @@ +import { useEffect } from "react"; +import { getHostRuntimeStore } from "@/runtime/host-runtime"; + +/** + * Keeps a host's directory demanded while the caller is mounted. + * + * The sidebar only demands the directory while its panel is active, so a compact app that + * restores straight into a workspace or agent route has nothing else asking the host for + * workspaces and agents. Without demand the route waits on hydration that never starts. + */ +export function useDirectoryDemand(serverId: string | null | undefined): void { + useEffect(() => { + if (!serverId) return; + return getHostRuntimeStore().acquireDirectoryDemand(serverId); + }, [serverId]); +} From 902b0c7a1637cc7a6c2ee268dca8161965a35def Mon Sep 17 00:00:00 2001 From: "paseo-ai[bot]" <266920839+paseo-ai[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:55:29 +0000 Subject: [PATCH 25/32] fix: update lockfile signatures and Nix hash [skip ci] --- nix/npm-deps.hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/npm-deps.hash b/nix/npm-deps.hash index e99d4da9b6..05bf98631a 100644 --- a/nix/npm-deps.hash +++ b/nix/npm-deps.hash @@ -1 +1 @@ -sha256-SKL7TMIuuV3imTIaQc66UDf3n6V3FB0nGSBucJ6rlA0= +sha256-zXSLA7FJHhIW63kkY3HbWaTsCTtJ30l71O9/Gx5F7QM= From 287ca547494f89076acb5a282e7f337420407fd9 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 18:12:56 +0700 Subject: [PATCH 26/32] fix(app): enlarge sidebar kebab tap targets on compact layouts (#7928) On phones the project and workspace row kebabs were 24px and 18px boxes with a 14px glyph, too small to hit reliably. Compact layouts now use a 32px box and the density-sized glyph (18px); desktop sizes are unchanged. Negative margins keep the dots on the existing right rail and the rows at their 44px compact height. --- .../src/components/sidebar-workspace-list.tsx | 18 +++++++++------- .../sidebar/sidebar-workspace-menu.tsx | 21 ++++++++++++------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/app/src/components/sidebar-workspace-list.tsx b/packages/app/src/components/sidebar-workspace-list.tsx index 16294e0a03..333eb68221 100644 --- a/packages/app/src/components/sidebar-workspace-list.tsx +++ b/packages/app/src/components/sidebar-workspace-list.tsx @@ -30,7 +30,7 @@ import { type ActiveWorkspaceSelection, } from "@/stores/navigation-active-workspace-store"; import { StyleSheet, withUnistyles } from "react-native-unistyles"; -import { density, densityIconBox } from "@/styles/density"; +import { density, densityIconBox, densityIconProps } from "@/styles/density"; import type { Theme } from "@/styles/theme"; import type { SidebarSurfaceBackdrop } from "@/styles/surface-backdrop"; import { getSidebarRowBackdrop } from "@/components/sidebar/sidebar-row-backdrop"; @@ -463,12 +463,12 @@ const openInNewWindowLeadingIcon = ( ); +const kebabTriggerIconProps = densityIconProps("sm", foregroundMutedColorMapping); +const kebabTriggerIconHoveredProps = densityIconProps("sm", foregroundColorMapping); + function renderKebabTriggerIcon({ hovered }: { hovered?: boolean }) { return ( - + ); } @@ -2705,8 +2705,12 @@ const styles = StyleSheet.create((theme) => ({ marginRight: -6, }, projectKebabButton: { - width: 24, - height: 24, + // Larger on compact so the kebab is an easy tap target. The negative margins keep its + // dots on the same rail as the 24px desktop box and the row at its 44px height. + width: { xs: 32, md: 24 }, + height: { xs: 32, md: 24 }, + marginRight: { xs: -4, md: 0 }, + marginVertical: { xs: -4, md: 0 }, borderRadius: theme.borderRadius.md, alignItems: "center", justifyContent: "center", diff --git a/packages/app/src/components/sidebar/sidebar-workspace-menu.tsx b/packages/app/src/components/sidebar/sidebar-workspace-menu.tsx index e1543fac41..f74c43c980 100644 --- a/packages/app/src/components/sidebar/sidebar-workspace-menu.tsx +++ b/packages/app/src/components/sidebar/sidebar-workspace-menu.tsx @@ -17,6 +17,7 @@ import { isWeb } from "@/constants/platform"; import { getForgePresentation, normalizeForge } from "@/git/forge"; import type { SidebarWorkspaceEntry } from "@/hooks/use-sidebar-workspaces-list"; import { useAppSettings } from "@/hooks/use-settings"; +import { densityIconProps } from "@/styles/density"; import type { Theme } from "@/styles/theme"; import type { ShortcutKey } from "@/utils/format-shortcut"; import { @@ -70,13 +71,11 @@ const archiveLeadingIcon = ; const unpinLeadingIcon = ; +const triggerIconProps = densityIconProps("sm", foregroundMutedColorMapping); +const triggerIconHoveredProps = densityIconProps("sm", foregroundColorMapping); + function renderTriggerIcon({ hovered }: { hovered?: boolean }) { - return ( - - ); + return ; } export interface SidebarWorkspaceMenuProps { @@ -439,12 +438,18 @@ function triggerStyle({ hovered = false }: PressableStateCallbackType & { hovere const styles = StyleSheet.create((theme) => ({ trigger: { - padding: 2, + // Larger on compact so the kebab is an easy tap target. + width: { xs: 32, md: 18 }, + height: { xs: 32, md: 18 }, + alignItems: "center", + justifyContent: "center", borderRadius: 4, marginLeft: 2, + // Center the compact box on the 20px title line it is anchored to. + marginTop: { xs: -6, md: 0 }, // MoreVertical paints only around the center of its SVG. Keep the padded hit box, but // pull the painted dots through that unused view-box space onto the trailing-content rail. - marginRight: -7, + marginRight: { xs: -14, md: -7 }, }, triggerHovered: { backgroundColor: theme.colors.surface2, From 678fe308db3acfa3b58866cb7ae603a9a331a07b Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 18:35:29 +0700 Subject: [PATCH 27/32] chore(release): reconcile 0.13.2 metadata with main (#7931) * docs(release): add 0.13.2 native patch notes (cherry picked from commit 870c969fdb927f422b892841dd4b941ee7ce1aaa) * chore(release): cut 0.13.2 (cherry picked from commit d2941157a8f91512daf54fc688241d416e8f5ebd) * chore(release): refresh Nix dependency hash for 0.13.2 --- CHANGELOG.md | 9 +++ CHANGELOG.zh-CN.md | 9 +++ .../android/en-US/changelogs/130021.txt | 7 +++ .../android/en-US/changelogs/130022.txt | 7 +++ .../android/en-US/changelogs/130023.txt | 7 +++ .../android/en-US/changelogs/130024.txt | 7 +++ nix/npm-deps.hash | 2 +- package-lock.json | 56 +++++++++---------- package.json | 2 +- packages/app/package.json | 2 +- packages/cli/package.json | 8 +-- packages/client/package.json | 6 +- packages/desktop/package.json | 2 +- packages/expo-two-way-audio/package.json | 2 +- packages/highlight/package.json | 2 +- packages/plugin/package.json | 10 ++-- packages/protocol/package.json | 2 +- packages/relay/package.json | 2 +- packages/server/package.json | 12 ++-- packages/website/package.json | 2 +- plugins/package.json | 2 +- 21 files changed, 102 insertions(+), 56 deletions(-) create mode 100644 fastlane/metadata/android/en-US/changelogs/130021.txt create mode 100644 fastlane/metadata/android/en-US/changelogs/130022.txt create mode 100644 fastlane/metadata/android/en-US/changelogs/130023.txt create mode 100644 fastlane/metadata/android/en-US/changelogs/130024.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cab484e81..98d1c76900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.13.2 - 2026-10-26 + +### Fixed + +- Fixed repeated Android animation exceptions for views that have already unmounted ([#7907](https://github.com/getpaseo/paseo/pull/7907)) +- Fixed the Android animation run loop continuing while the app is in the background ([#7914](https://github.com/getpaseo/paseo/pull/7914)) +- Fixed finished Android chat messages being remeasured during streaming updates ([#7914](https://github.com/getpaseo/paseo/pull/7914)) +- Fixed bottom sheets retaining window-size listeners after closing ([#7914](https://github.com/getpaseo/paseo/pull/7914)) + ## 0.13.1 - 2026-10-26 ### Fixed diff --git a/CHANGELOG.zh-CN.md b/CHANGELOG.zh-CN.md index a46d3ea21a..421da176dd 100644 --- a/CHANGELOG.zh-CN.md +++ b/CHANGELOG.zh-CN.md @@ -1,5 +1,14 @@ # 更新日志 +## 0.13.2 - 2026-10-26 + +### 修复 + +- 修复 Android 对已卸载视图反复抛出动画异常的问题 ([#7907](https://github.com/getpaseo/paseo/pull/7907)) +- 修复 Android 应用进入后台后动画运行循环仍持续执行的问题 ([#7914](https://github.com/getpaseo/paseo/pull/7914)) +- 修复 Android 在流式更新时反复测量已完成聊天消息的问题 ([#7914](https://github.com/getpaseo/paseo/pull/7914)) +- 修复底部面板关闭后仍保留窗口尺寸监听器的问题 ([#7914](https://github.com/getpaseo/paseo/pull/7914)) + ## 0.13.1 - 2026-10-26 ### 修复 diff --git a/fastlane/metadata/android/en-US/changelogs/130021.txt b/fastlane/metadata/android/en-US/changelogs/130021.txt new file mode 100644 index 0000000000..e08d39f767 --- /dev/null +++ b/fastlane/metadata/android/en-US/changelogs/130021.txt @@ -0,0 +1,7 @@ +Fixed +- Fixed repeated Android animation exceptions for views that have already unmounted +- Fixed the Android animation run loop continuing while the app is in the background +- Fixed finished Android chat messages being remeasured during streaming updates +- Fixed bottom sheets retaining window-size listeners after closing + +Full notes: https://paseo.sh/changelog diff --git a/fastlane/metadata/android/en-US/changelogs/130022.txt b/fastlane/metadata/android/en-US/changelogs/130022.txt new file mode 100644 index 0000000000..e08d39f767 --- /dev/null +++ b/fastlane/metadata/android/en-US/changelogs/130022.txt @@ -0,0 +1,7 @@ +Fixed +- Fixed repeated Android animation exceptions for views that have already unmounted +- Fixed the Android animation run loop continuing while the app is in the background +- Fixed finished Android chat messages being remeasured during streaming updates +- Fixed bottom sheets retaining window-size listeners after closing + +Full notes: https://paseo.sh/changelog diff --git a/fastlane/metadata/android/en-US/changelogs/130023.txt b/fastlane/metadata/android/en-US/changelogs/130023.txt new file mode 100644 index 0000000000..e08d39f767 --- /dev/null +++ b/fastlane/metadata/android/en-US/changelogs/130023.txt @@ -0,0 +1,7 @@ +Fixed +- Fixed repeated Android animation exceptions for views that have already unmounted +- Fixed the Android animation run loop continuing while the app is in the background +- Fixed finished Android chat messages being remeasured during streaming updates +- Fixed bottom sheets retaining window-size listeners after closing + +Full notes: https://paseo.sh/changelog diff --git a/fastlane/metadata/android/en-US/changelogs/130024.txt b/fastlane/metadata/android/en-US/changelogs/130024.txt new file mode 100644 index 0000000000..e08d39f767 --- /dev/null +++ b/fastlane/metadata/android/en-US/changelogs/130024.txt @@ -0,0 +1,7 @@ +Fixed +- Fixed repeated Android animation exceptions for views that have already unmounted +- Fixed the Android animation run loop continuing while the app is in the background +- Fixed finished Android chat messages being remeasured during streaming updates +- Fixed bottom sheets retaining window-size listeners after closing + +Full notes: https://paseo.sh/changelog diff --git a/nix/npm-deps.hash b/nix/npm-deps.hash index 05bf98631a..3e19f893c2 100644 --- a/nix/npm-deps.hash +++ b/nix/npm-deps.hash @@ -1 +1 @@ -sha256-zXSLA7FJHhIW63kkY3HbWaTsCTtJ30l71O9/Gx5F7QM= +sha256-eNiKmiZ3XXZMRZRSoGxUbMn2RtY6AX079vUatBYM+qE= diff --git a/package-lock.json b/package-lock.json index 1f5cce1657..847bd24eb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "paseo", - "version": "0.13.1", + "version": "0.13.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "paseo", - "version": "0.13.1", + "version": "0.13.2", "hasInstallScript": true, "license": "Apache-2.0", "workspaces": [ @@ -35695,7 +35695,7 @@ }, "packages/app": { "name": "@getpaseo/app", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "@codemirror/commands": "6.10.4", "@codemirror/language": "6.12.4", @@ -37961,12 +37961,12 @@ }, "packages/cli": { "name": "@getpaseo/cli", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "@clack/prompts": "^1.0.0", - "@getpaseo/client": "0.13.1", - "@getpaseo/protocol": "0.13.1", - "@getpaseo/server": "0.13.1", + "@getpaseo/client": "0.13.2", + "@getpaseo/protocol": "0.13.2", + "@getpaseo/server": "0.13.2", "chalk": "^5.3.0", "commander": "^12.0.0", "mime-types": "^2.1.35", @@ -38213,10 +38213,10 @@ }, "packages/client": { "name": "@getpaseo/client", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { - "@getpaseo/protocol": "0.13.1", - "@getpaseo/relay": "0.13.1", + "@getpaseo/protocol": "0.13.2", + "@getpaseo/relay": "0.13.2", "ws": "^8.20.0", "zod": "^4.4.3" }, @@ -38228,7 +38228,7 @@ }, "packages/desktop": { "name": "@getpaseo/desktop", - "version": "0.13.1", + "version": "0.13.2", "license": "Apache-2.0", "dependencies": { "@getpaseo/cli": "*", @@ -38472,7 +38472,7 @@ }, "packages/expo-two-way-audio": { "name": "@getpaseo/expo-two-way-audio", - "version": "0.13.1", + "version": "0.13.2", "license": "MIT", "devDependencies": { "@types/jest": "^29.5.14", @@ -42091,7 +42091,7 @@ }, "packages/highlight": { "name": "@getpaseo/highlight", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "@codemirror/language": "6.12.4", "@codemirror/legacy-modes": "^6.5.3", @@ -42120,15 +42120,15 @@ }, "packages/plugin": { "name": "@getpaseo/plugin", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "@agentclientprotocol/sdk": "^1.4.0", "i18next": "^26.3.0", "use-sync-external-store": "^1.6.0" }, "devDependencies": { - "@getpaseo/client": "0.13.1", - "@getpaseo/protocol": "0.13.1", + "@getpaseo/client": "0.13.2", + "@getpaseo/protocol": "0.13.2", "@types/node": "^20.9.0", "@types/react": "~19.2.0", "react": "19.2.3", @@ -42138,8 +42138,8 @@ "zod": "^4.4.3" }, "peerDependencies": { - "@getpaseo/client": "0.13.1", - "@getpaseo/protocol": "0.13.1", + "@getpaseo/client": "0.13.2", + "@getpaseo/protocol": "0.13.2", "react": ">=19.1.0 <20", "react-native": ">=0.81.5", "zod": "^4.4.3" @@ -42161,7 +42161,7 @@ }, "packages/protocol": { "name": "@getpaseo/protocol", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "semver": "^7.8.5", "zod": "^4.4.3" @@ -42188,7 +42188,7 @@ }, "packages/relay": { "name": "@getpaseo/relay", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "@noble/ciphers": "^2.4.0", "@noble/curves": "^2.4.0", @@ -42409,16 +42409,16 @@ }, "packages/server": { "name": "@getpaseo/server", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "@agentclientprotocol/sdk": "^0.17.1", "@anthropic-ai/claude-agent-sdk": "0.3.246", "@anthropic-ai/sdk": "^0.104.2", - "@getpaseo/client": "0.13.1", - "@getpaseo/highlight": "0.13.1", - "@getpaseo/plugin": "0.13.1", - "@getpaseo/protocol": "0.13.1", - "@getpaseo/relay": "0.13.1", + "@getpaseo/client": "0.13.2", + "@getpaseo/highlight": "0.13.2", + "@getpaseo/plugin": "0.13.2", + "@getpaseo/protocol": "0.13.2", + "@getpaseo/relay": "0.13.2", "@isaacs/ttlcache": "^2.1.4", "@modelcontextprotocol/sdk": "1.29.0", "@opencode-ai/sdk": "1.14.46", @@ -43339,7 +43339,7 @@ }, "packages/website": { "name": "@getpaseo/website", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "@cloudflare/vite-plugin": "^1.29.1", "@cloudflare/workers-types": "^4.20260317.1", @@ -44949,7 +44949,7 @@ }, "plugins": { "name": "@getpaseo/builtin-plugins", - "version": "0.13.1", + "version": "0.13.2", "dependencies": { "@getpaseo/plugin": "*", "zod": "^4.4.3" diff --git a/package.json b/package.json index 904e7e6af8..beeff7c613 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "paseo", - "version": "0.13.1", + "version": "0.13.2", "private": true, "description": "Paseo: voice-controlled development environment for local AI coding agents", "keywords": [ diff --git a/packages/app/package.json b/packages/app/package.json index e2e39ec03d..3eddd9b53f 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/app", - "version": "0.13.1", + "version": "0.13.2", "private": true, "main": "index.ts", "scripts": { diff --git a/packages/cli/package.json b/packages/cli/package.json index f4eef852fa..be8188fff3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/cli", - "version": "0.13.1", + "version": "0.13.2", "description": "Paseo CLI - control your AI coding agents from the command line", "bin": { "paseo": "bin/paseo" @@ -28,9 +28,9 @@ }, "dependencies": { "@clack/prompts": "^1.0.0", - "@getpaseo/client": "0.13.1", - "@getpaseo/protocol": "0.13.1", - "@getpaseo/server": "0.13.1", + "@getpaseo/client": "0.13.2", + "@getpaseo/protocol": "0.13.2", + "@getpaseo/server": "0.13.2", "chalk": "^5.3.0", "commander": "^12.0.0", "mime-types": "^2.1.35", diff --git a/packages/client/package.json b/packages/client/package.json index 2a0927de17..df80ba532c 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/client", - "version": "0.13.1", + "version": "0.13.2", "description": "Paseo client SDK package", "files": [ "dist", @@ -47,8 +47,8 @@ "test": "vitest run" }, "dependencies": { - "@getpaseo/protocol": "0.13.1", - "@getpaseo/relay": "0.13.1", + "@getpaseo/protocol": "0.13.2", + "@getpaseo/relay": "0.13.2", "ws": "^8.20.0", "zod": "^4.4.3" }, diff --git a/packages/desktop/package.json b/packages/desktop/package.json index b4af89d742..4735fb5192 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/desktop", - "version": "0.13.1", + "version": "0.13.2", "private": true, "description": "Paseo desktop app (Electron wrapper)", "homepage": "https://paseo.sh", diff --git a/packages/expo-two-way-audio/package.json b/packages/expo-two-way-audio/package.json index c60fbe6eb3..1a38528c25 100644 --- a/packages/expo-two-way-audio/package.json +++ b/packages/expo-two-way-audio/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/expo-two-way-audio", - "version": "0.13.1", + "version": "0.13.2", "description": "Native module for two way audio streaming", "keywords": [ "ExpoTwoWayAudio", diff --git a/packages/highlight/package.json b/packages/highlight/package.json index 5d9aed78cf..d7708e5925 100644 --- a/packages/highlight/package.json +++ b/packages/highlight/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/highlight", - "version": "0.13.1", + "version": "0.13.2", "files": [ "dist", "!dist/**/*.map", diff --git a/packages/plugin/package.json b/packages/plugin/package.json index b40268e537..18421b41e1 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/plugin", - "version": "0.13.1", + "version": "0.13.2", "description": "Paseo plugin SDK package", "files": [ "dist", @@ -72,8 +72,8 @@ "use-sync-external-store": "^1.6.0" }, "devDependencies": { - "@getpaseo/client": "0.13.1", - "@getpaseo/protocol": "0.13.1", + "@getpaseo/client": "0.13.2", + "@getpaseo/protocol": "0.13.2", "@types/node": "^20.9.0", "@types/react": "~19.2.0", "react": "19.2.3", @@ -83,8 +83,8 @@ "zod": "^4.4.3" }, "peerDependencies": { - "@getpaseo/client": "0.13.1", - "@getpaseo/protocol": "0.13.1", + "@getpaseo/client": "0.13.2", + "@getpaseo/protocol": "0.13.2", "react": ">=19.1.0 <20", "react-native": ">=0.81.5", "zod": "^4.4.3" diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 99da1fc5bd..e463e681e7 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/protocol", - "version": "0.13.1", + "version": "0.13.2", "description": "Paseo shared protocol schemas and wire types", "files": [ "dist", diff --git a/packages/relay/package.json b/packages/relay/package.json index 82822177c1..b2e0c5187f 100644 --- a/packages/relay/package.json +++ b/packages/relay/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/relay", - "version": "0.13.1", + "version": "0.13.2", "description": "Paseo relay for bridging daemon and client connections", "files": [ "dist", diff --git a/packages/server/package.json b/packages/server/package.json index 5b8017659e..4aeb01dc3f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/server", - "version": "0.13.1", + "version": "0.13.2", "description": "Paseo backend server", "files": [ "dist/server", @@ -107,11 +107,11 @@ "@agentclientprotocol/sdk": "^0.17.1", "@anthropic-ai/claude-agent-sdk": "0.3.246", "@anthropic-ai/sdk": "^0.104.2", - "@getpaseo/client": "0.13.1", - "@getpaseo/highlight": "0.13.1", - "@getpaseo/plugin": "0.13.1", - "@getpaseo/protocol": "0.13.1", - "@getpaseo/relay": "0.13.1", + "@getpaseo/client": "0.13.2", + "@getpaseo/highlight": "0.13.2", + "@getpaseo/plugin": "0.13.2", + "@getpaseo/protocol": "0.13.2", + "@getpaseo/relay": "0.13.2", "@isaacs/ttlcache": "^2.1.4", "@modelcontextprotocol/sdk": "1.29.0", "@opencode-ai/sdk": "1.14.46", diff --git a/packages/website/package.json b/packages/website/package.json index 5574ad4a46..2166f1a22e 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/website", - "version": "0.13.1", + "version": "0.13.2", "private": true, "type": "module", "scripts": { diff --git a/plugins/package.json b/plugins/package.json index fd0a74d13b..30d9929eef 100644 --- a/plugins/package.json +++ b/plugins/package.json @@ -1,6 +1,6 @@ { "name": "@getpaseo/builtin-plugins", - "version": "0.13.1", + "version": "0.13.2", "private": true, "type": "module", "scripts": { From 42d74c254652c43c03cf9bcdf7c0bb253e4706c0 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 19:20:17 +0700 Subject: [PATCH 28/32] fix(ci): retry transient Apple staple connection failures (#7933) --- .github/workflows/desktop-release.yml | 28 +++++++ scripts/ci-workflow.test.mjs | 113 +++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index f6b07c8b37..d81c036e62 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -169,6 +169,34 @@ jobs: NODE_OPTIONS: --max-old-space-size=${{ matrix.node_heap_mb }} run: | set -euo pipefail + export PASEO_REAL_XCRUN="$(command -v xcrun)" + stapler_bin="$(mktemp -d)" + trap 'rm -rf "$stapler_bin"' EXIT + cat > "$stapler_bin/xcrun" <<'STAPLER' + #!/bin/bash + if [[ "${1:-}" != stapler || "${2:-}" != staple ]]; then + exec "$PASEO_REAL_XCRUN" "$@" + fi + output="$(mktemp)" + trap 'rm -f "$output"' EXIT + for attempt in 1 2 3 4; do + if "$PASEO_REAL_XCRUN" "$@" >"$output" 2>&1; then + cat "$output" + exit 0 + else + status=$? + fi + cat "$output" + if [[ "$status" != 68 || "$attempt" == 4 ]] || + ! grep -Fq 'NSURLErrorDomain Code=-1005' "$output"; then + exit "$status" + fi + echo "::warning::Apple staple connection lost; retrying the same signed app ($attempt/4)" >&2 + sleep 30 + done + STAPLER + chmod +x "$stapler_bin/xcrun" + export PATH="$stapler_bin:$PATH" build_args=(-- --publish never --mac --${{ matrix.electron_arch }}) build_args+=("-c.publish.releaseType=$RELEASE_TYPE") build_args+=("-c.publish.channel=$RELEASE_CHANNEL") diff --git a/scripts/ci-workflow.test.mjs b/scripts/ci-workflow.test.mjs index c16e772172..4842870cc3 100644 --- a/scripts/ci-workflow.test.mjs +++ b/scripts/ci-workflow.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; -import { readFileSync, readdirSync } from "node:fs"; +import { readFileSync, readdirSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { relative as relativePath } from "node:path"; import test from "node:test"; import { runInNewContext } from "node:vm"; @@ -530,3 +533,111 @@ test("Windows units and integration run independently with complete routed cover assert.equal(selected(integrationCondition, outputs, true), false); } }); + +test("macOS release retries only transient stapling failures without rebuilding the signed app", async (t) => { + const workflow = readFileSync(new URL(".github/workflows/desktop-release.yml", repoRoot), "utf8"); + const wrapper = workflow + .match(/<<'STAPLER'\n([\s\S]*?)\n STAPLER/)[1] + .replace(/^ /gm, ""); + const networkError = + 'Error Domain=NSURLErrorDomain Code=-1005 "The network connection was lost."'; + const cases = [ + { + name: "success", + outcomes: [0], + output: "stapled", + command: "stapler", + expected: 0, + attempts: 1, + }, + { + name: "transient then success", + outcomes: [68, 0], + output: networkError, + command: "stapler", + expected: 0, + attempts: 2, + }, + { + name: "exhausted network retries", + outcomes: [68, 68, 68, 68, 0], + output: networkError, + command: "stapler", + expected: 68, + attempts: 4, + }, + { + name: "non-network staple error", + outcomes: [68, 0], + output: "ticket not found", + command: "stapler", + expected: 68, + attempts: 1, + }, + { + name: "different exit code", + outcomes: [1, 0], + output: networkError, + command: "stapler", + expected: 1, + attempts: 1, + }, + { + name: "notary submission is not retried", + outcomes: [68, 0], + output: networkError, + command: "notarytool", + expected: 68, + attempts: 1, + }, + ]; + for (const scenario of cases) { + await t.test(scenario.name, () => { + const directory = mkdtempSync(join(tmpdir(), "stapler-retry-")); + try { + const statePath = join(directory, "state.json"); + const realXcrun = join(directory, "real-xcrun"); + const wrapperPath = join(directory, "xcrun"); + writeFileSync(wrapperPath, wrapper, { mode: 0o755 }); + writeFileSync(join(directory, "sleep"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync( + statePath, + JSON.stringify({ outcomes: scenario.outcomes, output: scenario.output, calls: [] }), + ); + writeFileSync( + realXcrun, + `#!/usr/bin/env node +const fs = require("node:fs"); +const state = JSON.parse(fs.readFileSync(process.env.STAPLER_STATE, "utf8")); +state.calls.push(process.argv.slice(2)); +const code = state.outcomes.shift(); +fs.writeFileSync(process.env.STAPLER_STATE, JSON.stringify(state)); +console.log(state.output); +process.exit(code); +`, + { mode: 0o755 }, + ); + const args = [scenario.command, "staple", "-v", "Paseo signed.app"]; + const result = spawnSync("bash", [wrapperPath, ...args], { + encoding: "utf8", + env: { + ...process.env, + PASEO_REAL_XCRUN: realXcrun, + STAPLER_STATE: statePath, + PATH: `${directory}:${process.env.PATH}`, + }, + timeout: 5000, + }); + assert.equal(result.status, scenario.expected, result.stdout + result.stderr); + const state = JSON.parse(readFileSync(statePath, "utf8")); + assert.deepEqual( + state.calls, + Array.from({ length: scenario.attempts }, () => args), + ); + assert.match(result.stdout, /stapled|network connection was lost|ticket not found/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + } +}); From e3bbed3c73fac7cbd2b5e340037ff6a9201206b8 Mon Sep 17 00:00:00 2001 From: Simon Gurcke Date: Tue, 27 Oct 2026 00:02:44 +1000 Subject: [PATCH 29/32] fix(pi): queue prompts until manual compaction finishes (#7848) * fix(pi): wait for manual compaction before sending prompts * test(pi): simplify compaction regressions and stabilize live smoke --- .../server/agent/providers/pi/agent.test.ts | 75 +++++++++++++++++++ .../src/server/agent/providers/pi/agent.ts | 14 +++- .../agent/providers/pi/test-utils/fake-pi.ts | 4 + .../src/server/daemon-e2e/pi.real.e2e.test.ts | 21 ++++-- 4 files changed, 106 insertions(+), 8 deletions(-) diff --git a/packages/server/src/server/agent/providers/pi/agent.test.ts b/packages/server/src/server/agent/providers/pi/agent.test.ts index 1bad16ba30..b9f31b61b2 100644 --- a/packages/server/src/server/agent/providers/pi/agent.test.ts +++ b/packages/server/src/server/agent/providers/pi/agent.test.ts @@ -3063,6 +3063,81 @@ describe("PiRpcAgentClient", () => { ]); }); + test.each([ + { outcome: "successful", error: null }, + { outcome: "failed", error: new Error("summarizer failed") }, + ])("waits for $outcome manual compaction before sending a new prompt", async ({ error }) => { + const { pi, session, events } = await createSession(); + onTestFinished(() => session.close()); + const fakeSession = pi.latestSession(); + fakeSession.compactError = error; + const compactionFinished = Promise.withResolvers(); + fakeSession.compactCompletion = compactionFinished.promise; + const handler = session.tryHandleOutOfBand("/compact"); + const compaction = handler!.run({ emit: () => {} }); + + const { turnId } = await session.startTurn("continue after compaction"); + await flushTurnScheduling(); + expect(fakeSession.compactRequests).toEqual([{}]); + expect(fakeSession.prompts).toEqual([]); + + compactionFinished.resolve(); + await compaction; + await flushTurnScheduling(); + expect(fakeSession.prompts).toEqual([{ message: "continue after compaction", imageCount: 0 }]); + + fakeSession.finishTurn(); + await events.nextTurnCompletion(); + expect(events.turnLifecycleEvents()).toContainEqual({ type: "turn_completed", turnId }); + expect(events.eventTypes()).not.toContain("turn_failed"); + }); + + test.each(["interrupt", "close"] as const)( + "does not send a prompt waiting on compaction after %s", + async (action) => { + const { pi, session } = await createSession(); + onTestFinished(() => session.close()); + const fakeSession = pi.latestSession(); + const compactionFinished = Promise.withResolvers(); + fakeSession.compactCompletion = compactionFinished.promise; + const handler = session.tryHandleOutOfBand("/compact"); + const compaction = handler!.run({ emit: () => {} }); + + await session.startTurn("do not send this prompt"); + await session[action](); + compactionFinished.resolve(); + await compaction; + await flushTurnScheduling(); + + expect(fakeSession.prompts).toEqual([]); + }, + ); + + test("does not send a prompt when compaction finishes during interruption", async () => { + const { pi, session, events } = await createSession(); + onTestFinished(() => session.close()); + const fakeSession = pi.latestSession(); + const compactionFinished = Promise.withResolvers(); + fakeSession.compactCompletion = compactionFinished.promise; + const clearQueueFinished = Promise.withResolvers(); + fakeSession.clearQueueCompletion = clearQueueFinished.promise; + const handler = session.tryHandleOutOfBand("/compact"); + const compaction = handler!.run({ emit: () => {} }); + const { turnId } = await session.startTurn("do not send this prompt"); + + const interruption = session.interrupt(); + compactionFinished.resolve(); + await compaction; + await flushTurnScheduling(); + expect(fakeSession.controlRequests).toEqual(["clear_queue"]); + expect(fakeSession.prompts).toEqual([]); + + clearQueueFinished.resolve(); + await interruption; + expect(events.turnLifecycleEvents()).toEqual([{ type: "turn_canceled", turnId }]); + expect(fakeSession.prompts).toEqual([]); + }); + test("closes Pi compact loading marker when RPC rejects after compaction starts", async () => { const { pi, session } = await createSession(); const fakeSession = pi.latestSession(); diff --git a/packages/server/src/server/agent/providers/pi/agent.ts b/packages/server/src/server/agent/providers/pi/agent.ts index cf193e213b..881f8cbb29 100644 --- a/packages/server/src/server/agent/providers/pi/agent.ts +++ b/packages/server/src/server/agent/providers/pi/agent.ts @@ -1189,6 +1189,7 @@ export class PiRpcAgentSession implements AgentSession { private readonly contextUserEntries: PiCapturedEntry[] = []; private readonly treeUserEntriesById = new Map(); private readonly pendingExtensionResults = new Map(); + private compactionPromise: Promise | null = null; private outOfBandCompactionEmit: ((event: AgentStreamEvent) => void) | null = null; private outOfBandCompactionStarted = false; private outOfBandCompactionCompleted = false; @@ -1296,6 +1297,15 @@ export class PiRpcAgentSession implements AgentSession { void (async () => { try { + if (this.compactionPromise) { + // Pi RPC rejects prompts until the manual compaction request settles. + await this.compactionPromise.catch(() => undefined); + const turnCanceled = + this.closed || this.activeTurnId !== turnId || this.interruptingTurn?.turnId === turnId; + if (turnCanceled) { + return; + } + } const ack = await this.runtimeSession.prompt(payload.text, payload.images); this.activePromptRequestId = ack.requestId ?? null; const correlatedResult = ack.requestId @@ -1804,7 +1814,8 @@ export class PiRpcAgentSession implements AgentSession { this.outOfBandCompactionStarted = false; this.outOfBandCompactionCompleted = false; try { - await this.runtimeSession.compact(customInstructions); + this.compactionPromise = this.runtimeSession.compact(customInstructions); + await this.compactionPromise; } catch (error) { const message = error instanceof Error ? error.message : String(error); if ( @@ -1830,6 +1841,7 @@ export class PiRpcAgentSession implements AgentSession { }, }); } finally { + this.compactionPromise = null; if (this.outOfBandCompactionEmit === emit && !this.outOfBandCompactionStarted) { this.outOfBandCompactionEmit = null; this.outOfBandCompactionStarted = false; diff --git a/packages/server/src/server/agent/providers/pi/test-utils/fake-pi.ts b/packages/server/src/server/agent/providers/pi/test-utils/fake-pi.ts index b8c7930bc4..983b86c19d 100644 --- a/packages/server/src/server/agent/providers/pi/test-utils/fake-pi.ts +++ b/packages/server/src/server/agent/providers/pi/test-utils/fake-pi.ts @@ -108,6 +108,7 @@ export class FakePiSession implements PiRuntimeSession { steerError: Error | null = null; readonly controlRequests: string[] = []; clearQueueError: Error | null = null; + clearQueueCompletion: Promise | null = null; readonly compactRequests: Array<{ customInstructions?: string }> = []; readonly setAutoCompactionRequests: boolean[] = []; readonly subagentSubscriptionRequests: FakePiSubagentSubscriptionLevel[] = []; @@ -140,6 +141,7 @@ export class FakePiSession implements PiRuntimeSession { subagentSubscriptionError: Error | null = null; setSessionNameError: Error | null = null; compactError: Error | null = null; + compactCompletion: Promise | null = null; emitCompactEnd = true; getStateError: Error | null = null; getSessionStatsError: Error | null = null; @@ -233,6 +235,7 @@ export class FakePiSession implements PiRuntimeSession { async clearQueue(): Promise { this.controlRequests.push("clear_queue"); + await this.clearQueueCompletion; if (this.clearQueueError) { throw this.clearQueueError; } @@ -241,6 +244,7 @@ export class FakePiSession implements PiRuntimeSession { async compact(customInstructions?: string): Promise { this.compactRequests.push(customInstructions === undefined ? {} : { customInstructions }); this.emit({ type: "compaction_start", reason: "manual" }); + await this.compactCompletion; if (this.emitCompactEnd) { this.emit({ type: "compaction_end", reason: "manual" }); } diff --git a/packages/server/src/server/daemon-e2e/pi.real.e2e.test.ts b/packages/server/src/server/daemon-e2e/pi.real.e2e.test.ts index 984aa2fdcc..d2d839122e 100644 --- a/packages/server/src/server/daemon-e2e/pi.real.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/pi.real.e2e.test.ts @@ -219,7 +219,7 @@ test( ); test( - "real Pi daemon executes manual compact out-of-band instead of prompt text", + "real Pi daemon accepts a prompt after manual compact", async () => { const cwd = tmpCwd("pi-manual-compact-"); @@ -234,20 +234,24 @@ test( cwd, title: "pi-manual-compact", provider: "pi", - model: PI_COMPACTION_TEST_MODEL, + model: PI_REAL_TEST_MODEL, }); await client.sendMessage(agent.id, "Reply exactly: compact-ready"); const finish = await client.waitForFinish(agent.id, PI_TEST_TIMEOUT_MS); - expect(finish.status).toBe("idle"); + expect(finish.status, JSON.stringify(await fetchCanonicalTimeline(client, agent.id))).toBe( + "idle", + ); await client.sendMessage(agent.id, "/compact summarize this e2e run"); - await waitForTimelineItem( client, agent.id, (item) => item.type === "compaction" && item.status === "completed", ); + await client.sendMessage(agent.id, "Reply exactly: after-compaction"); + const followUpFinish = await client.waitForFinish(agent.id, PI_TEST_TIMEOUT_MS); + expect(followUpFinish.status).toBe("idle"); const items = await fetchCanonicalTimeline(client, agent.id); expect(items).toEqual( @@ -263,9 +267,12 @@ test( }), ]), ); - expect( - items.some((item) => item.type === "user_message" && item.text.includes("/compact")), - ).toBe(false); + expect(items).toContainEqual( + expect.objectContaining({ + type: "assistant_message", + text: "after-compaction", + }), + ); expect( items .filter((item) => item.type === "assistant_message") From 28834a91aeb31c4058bfa92558f9b45b5e5121c0 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 23:13:25 +0700 Subject: [PATCH 30/32] fix(server): tolerate brief Windows atomic replace failures (#7955) A receipt could remain pending after provider delivery when Windows rejected the completed-state rename with EPERM. Retry transient replace errors within a one-second budget while retaining the original file and cleanup semantics. --- .../server/src/server/atomic-file.test.ts | 147 ++++++++++++++++++ packages/server/src/server/atomic-file.ts | 57 ++++++- 2 files changed, 201 insertions(+), 3 deletions(-) create mode 100644 packages/server/src/server/atomic-file.test.ts diff --git a/packages/server/src/server/atomic-file.test.ts b/packages/server/src/server/atomic-file.test.ts new file mode 100644 index 0000000000..a521a1cc3c --- /dev/null +++ b/packages/server/src/server/atomic-file.test.ts @@ -0,0 +1,147 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, expect, it } from "vitest"; +import { writeFileAtomic, writeJsonFileAtomic } from "./atomic-file.js"; + +let directory: string; +beforeEach(async () => { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "paseo-atomic-file-")); +}); +afterEach(async () => { + await fs.rm(directory, { recursive: true, force: true }); +}); + +it.each(["EPERM", "EACCES", "EBUSY"])( + "commits a completed receipt after transient Windows %s replace failures", + async (code) => { + const file = path.join(directory, "receipt.json"); + await writeJsonFileAtomic(file, { state: "pending" }); + let failures = 3; + let elapsed = 0; + await writeJsonFileAtomic( + file, + { state: "completed" }, + { + platform: "win32", + now: () => elapsed, + sleep: async (milliseconds) => { + elapsed += milliseconds; + }, + rename: async (source, destination) => { + if (failures-- > 0) { + expect(JSON.parse(await fs.readFile(destination, "utf8"))).toEqual({ + state: "pending", + }); + expect(JSON.parse(await fs.readFile(source, "utf8"))).toEqual({ state: "completed" }); + throw Object.assign(new Error("destination held open"), { code }); + } + await fs.rename(source, destination); + }, + }, + ); + expect(JSON.parse(await fs.readFile(file, "utf8"))).toEqual({ state: "completed" }); + expect(await fs.readdir(directory)).toEqual(["receipt.json"]); + expect(elapsed).toBeGreaterThan(0); + }, +); + +it.each(["EPERM", "EACCES", "EBUSY"])( + "preserves the previous file and removes the temp file when Windows %s persists", + async (code) => { + const file = path.join(directory, "receipt.json"); + await writeJsonFileAtomic(file, { state: "pending" }); + const error = Object.assign(new Error("destination held open"), { code }); + const waits: number[] = []; + let elapsed = 0; + const attempts: number[] = []; + await expect( + writeJsonFileAtomic( + file, + { state: "completed" }, + { + platform: "win32", + now: () => elapsed, + sleep: async (milliseconds) => { + waits.push(milliseconds); + elapsed += milliseconds; + }, + rename: async () => { + attempts.push(elapsed); + throw error; + }, + }, + ), + ).rejects.toBe(error); + expect(elapsed).toBe(1000); + expect(waits.slice(0, 5)).toEqual([10, 20, 40, 80, 100]); + expect(Math.max(...waits)).toBe(100); + expect(attempts.every((time) => time < 1000)).toBe(true); + expect(JSON.parse(await fs.readFile(file, "utf8"))).toEqual({ state: "pending" }); + expect(await fs.readdir(directory)).toEqual(["receipt.json"]); + }, +); + +it.each([ + ["linux", "EPERM"], + ["darwin", "EACCES"], + ["linux", "EBUSY"], + ["win32", "ENOENT"], + ["win32", "ENOSPC"], + ["win32", "EXDEV"], +] as const)("does not retry %s %s failures", async (platform, code) => { + const file = path.join(directory, "receipt.json"); + await writeJsonFileAtomic(file, { state: "pending" }); + const error = Object.assign(new Error("rename failed"), { code }); + let attempts = 0; + const waits: number[] = []; + await expect( + writeFileAtomic(file, "replacement", { + platform, + now: () => 0, + sleep: async (milliseconds) => { + waits.push(milliseconds); + }, + rename: async () => { + attempts++; + throw error; + }, + }), + ).rejects.toBe(error); + expect(attempts).toBe(1); + expect(waits).toEqual([]); + expect(JSON.parse(await fs.readFile(file, "utf8"))).toEqual({ state: "pending" }); + expect(await fs.readdir(directory)).toEqual(["receipt.json"]); +}); + +it("does not retry after a rename consumes the Windows retry budget", async () => { + const file = path.join(directory, "receipt.json"); + const error = Object.assign(new Error("slow rename failed"), { code: "EPERM" }); + let elapsed = 0; + const waits: number[] = []; + await expect( + writeFileAtomic(file, "replacement", { + platform: "win32", + now: () => elapsed, + sleep: async (milliseconds) => { + waits.push(milliseconds); + }, + rename: async () => { + elapsed += 1000; + throw error; + }, + }), + ).rejects.toBe(error); + expect(waits).toEqual([]); + expect(await fs.readdir(directory)).toEqual([]); +}); + +it("creates parent directories and replaces binary and JSON files through the default runtime", async () => { + const file = path.join(directory, "nested", "file"); + const bytes = Buffer.from([0, 255, 128, 1]); + await writeFileAtomic(file, bytes); + expect(await fs.readFile(file)).toEqual(bytes); + await writeJsonFileAtomic(file, { state: "completed" }); + expect(await fs.readFile(file, "utf8")).toBe('{\n "state": "completed"\n}'); + expect(await fs.readdir(path.dirname(file))).toEqual(["file"]); +}); diff --git a/packages/server/src/server/atomic-file.ts b/packages/server/src/server/atomic-file.ts index cedc8d51ea..9f9c9f9568 100644 --- a/packages/server/src/server/atomic-file.ts +++ b/packages/server/src/server/atomic-file.ts @@ -1,10 +1,26 @@ import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; + +interface AtomicFileRuntime { + platform: NodeJS.Platform; + rename: (source: string, destination: string) => Promise; + now: () => number; + sleep: (milliseconds: number) => Promise; +} + +const runtime: AtomicFileRuntime = { + platform: process.platform, + rename: fs.rename, + now: () => performance.now(), + sleep, +}; export async function writeFileAtomic( filePath: string, data: string | NodeJS.ArrayBufferView, + operations: AtomicFileRuntime = runtime, ): Promise { await fs.mkdir(path.dirname(filePath), { recursive: true }); const tempPath = path.join( @@ -13,13 +29,48 @@ export async function writeFileAtomic( ); try { await fs.writeFile(tempPath, data, "utf8"); - await fs.rename(tempPath, filePath); + await replaceFile(tempPath, filePath, operations); } catch (error) { await fs.rm(tempPath, { force: true }); throw error; } } -export async function writeJsonFileAtomic(filePath: string, value: unknown): Promise { - await writeFileAtomic(filePath, JSON.stringify(value, null, 2)); +async function replaceFile( + source: string, + destination: string, + operations: AtomicFileRuntime, +): Promise { + // Windows readers/scanners can briefly deny replacement. Yield with backoff, + // but keep interactive persistence failures bounded to a one-second retry budget. + const deadline = operations.now() + 1000; + let delay = 10; + for (;;) { + try { + await operations.rename(source, destination); + return; + } catch (error) { + if ( + operations.platform !== "win32" || + !(error instanceof Error) || + !("code" in error) || + !["EPERM", "EACCES", "EBUSY"].includes(String(error.code)) + ) { + throw error; + } + const remaining = deadline - operations.now(); + if (remaining <= 0) throw error; + await operations.sleep(Math.min(delay, remaining)); + if (operations.now() >= deadline) throw error; + delay = Math.min(delay * 2, 100); + } + } +} + +export async function writeJsonFileAtomic( + filePath: string, + value: unknown, + operations: AtomicFileRuntime = runtime, +): Promise { + await writeFileAtomic(filePath, JSON.stringify(value, null, 2), operations); } From 1f3266e2ba0292a63f19c8d37c978702b055b90d Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 26 Oct 2026 23:56:19 +0700 Subject: [PATCH 31/32] test(server): remove connected-host delivery timing assumptions (#7961) CLI startup is not bounded by the recipient quiet window. Exercise the three delivery sources in separate windows and retain exact timeline and durable receipt assertions. Await permission-finish cleanup after dispatch acknowledgment using the integration cleanup budget. --- .../server/connected-hosts/hosts.e2e.test.ts | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/server/src/server/connected-hosts/hosts.e2e.test.ts b/packages/server/src/server/connected-hosts/hosts.e2e.test.ts index d8d5533cf6..b6765051f6 100644 --- a/packages/server/src/server/connected-hosts/hosts.e2e.test.ts +++ b/packages/server/src/server/connected-hosts/hosts.e2e.test.ts @@ -1544,7 +1544,7 @@ test( ); test( - "remote agent reports, local CLI sends and automatic completion share the recipient batch", + "remote agent reports, local CLI sends and automatic completion reach the same recipient", { timeout: 30000 }, async () => { const parentPrompts: string[] = []; @@ -1633,7 +1633,10 @@ test( }); expect(report.isError).not.toBe(true); expect(report.structuredContent).toMatchObject({ deliveryStatus: "queued", serverId: aId }); - expect(parentPrompts).toEqual([]); + // A CLI process can take longer than the two-second quiet window to start. + // Exercise each delivery across windows; delivery.test.ts owns batching with a clock. + await expect.poll(() => parentPrompts, { timeout: 10000 }).toHaveLength(1); + expect(parentPrompts[0]).toContain("REMOTE_REPORT <ready>"); const cli = await promisify(execFile)( process.execPath, @@ -1656,7 +1659,8 @@ test( }, ); expect(JSON.parse(cli.stdout)).toMatchObject({ status: "queued" }); - expect(parentPrompts).toEqual([]); + await expect.poll(() => parentPrompts, { timeout: 10000 }).toHaveLength(2); + expect(parentPrompts[1]).toContain("LOCAL_CLI_REPORT"); const receiptStates = async () => { const directory = join(a.paseoHome, "agent-requests"); const receipts = await Promise.all( @@ -1671,13 +1675,10 @@ test( .map((receipt) => receipt.state) .sort(); }; - expect(await receiptStates()).toEqual(["pending", "pending"]); finishChild.resolve(); - await expect.poll(() => parentPrompts, { timeout: 10000 }).toHaveLength(1); - expect(parentPrompts[0]).toContain("REMOTE_REPORT <ready>"); - expect(parentPrompts[0]).toContain("LOCAL_CLI_REPORT"); - expect(parentPrompts[0]).toContain('event="finished"'); - expect(parentPrompts[0]).toContain(`source-agent-id="${child.id}"`); + await expect.poll(() => parentPrompts, { timeout: 10000 }).toHaveLength(3); + expect(parentPrompts[2]).toContain('event="finished"'); + expect(parentPrompts[2]).toContain(`source-agent-id="${child.id}"`); await expect .poll(receiptStates, { timeout: 10000, @@ -1870,6 +1871,7 @@ test( }); const a = await daemon({ mcpEnabled: true, + logger: completionLogger(), agentClients: createTestAgentClients({ onStartTurn: (prompt) => { if (String(prompt).includes('kind="agent-notification"')) @@ -1931,8 +1933,14 @@ test( await expect.poll(() => parentPrompts, { timeout: 10000 }).toHaveLength(2); expect(parentPrompts[1]).toContain('event="finished"'); await expect - .poll(async () => - JSON.parse(await readFile(join(a.paseoHome, "finish-subscriptions.json"), "utf8")), + // Provider start precedes dispatch acknowledgment and the durable removal commit. + .poll( + async () => + JSON.parse(await readFile(join(a.paseoHome, "finish-subscriptions.json"), "utf8")), + { + timeout: 10000, + message: "Parent accepted completion; waiting for durable intent removal", + }, ) .toEqual([]); }, From 4f9d9d6de2236486ecc2ad886a2c37f4b92271e0 Mon Sep 17 00:00:00 2001 From: "paseo-bot[bot]" <266920839+paseo-bot[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:04:42 +0000 Subject: [PATCH 32/32] Keep ACP terminal output from stalling daemon requests (#7953) * fix(acp): trim terminal output in linear time * test(acp): reproduce terminal output stalls with a larger workload The 128 KiB workload completes inside the regression deadline on faster machines even with quadratic trimming. One MiB exceeds that deadline on the old implementation while the bounded UTF-8 suffix implementation finishes promptly. Co-authored-by: liyang --------- Co-authored-by: liyang Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com> --- .../server/agent/providers/acp-agent.test.ts | 40 +++++++++++++++++++ .../src/server/agent/providers/acp-agent.ts | 12 ++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/packages/server/src/server/agent/providers/acp-agent.test.ts b/packages/server/src/server/agent/providers/acp-agent.test.ts index 3deaa60c11..e902493dcb 100644 --- a/packages/server/src/server/agent/providers/acp-agent.test.ts +++ b/packages/server/src/server/agent/providers/acp-agent.test.ts @@ -811,6 +811,46 @@ describe("ACPAgentSession terminal tools", () => { truncated: false, }); }); + + test("trims large terminal output without rescanning the retained suffix", async () => { + const session = createSession(); + const terminal = await session.createTerminal({ + sessionId: "session-1", + command: process.execPath, + args: ["-e", 'process.stdout.write("x".repeat(1024 * 1024))'], + cwd: process.cwd(), + outputByteLimit: 1024, + }); + await session.waitForTerminalExit({ sessionId: "session-1", terminalId: terminal.terminalId }); + + await expect( + session.terminalOutput({ sessionId: "session-1", terminalId: terminal.terminalId }), + ).resolves.toEqual({ + output: "x".repeat(1024), + truncated: true, + exitStatus: { exitCode: 0, signal: null }, + }); + }, 1000); + + test("trims terminal output at a UTF-8 character boundary", async () => { + const session = createSession(); + const terminal = await session.createTerminal({ + sessionId: "session-1", + command: process.execPath, + args: ["-e", 'process.stdout.write("😀ab")'], + cwd: process.cwd(), + outputByteLimit: 3, + }); + await session.waitForTerminalExit({ sessionId: "session-1", terminalId: terminal.terminalId }); + + await expect( + session.terminalOutput({ sessionId: "session-1", terminalId: terminal.terminalId }), + ).resolves.toEqual({ + output: "ab", + truncated: true, + exitStatus: { exitCode: 0, signal: null }, + }); + }); }); describe("mapACPUsage", () => { diff --git a/packages/server/src/server/agent/providers/acp-agent.ts b/packages/server/src/server/agent/providers/acp-agent.ts index ae765ecd2e..e26b6e4ade 100644 --- a/packages/server/src/server/agent/providers/acp-agent.ts +++ b/packages/server/src/server/agent/providers/acp-agent.ts @@ -4093,10 +4093,16 @@ function appendTerminalOutput(entry: TerminalEntry, chunk: string): void { if (!limit) { return; } - while (Buffer.byteLength(entry.output, "utf8") > limit && entry.output.length > 0) { - entry.output = entry.output.slice(1); - entry.truncated = true; + const output = Buffer.from(entry.output, "utf8"); + if (output.length <= limit) { + return; } + let start = output.length - limit; + while (start < output.length && (output[start] & 0xc0) === 0x80) { + start += 1; + } + entry.output = output.subarray(start).toString("utf8"); + entry.truncated = true; } function readRecord(value: unknown): Record | null {