From 83ed52669da0ea1354ac1bf7343ca15723a33368 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 28 Oct 2026 23:40:35 +0700 Subject: [PATCH] Resume interrupted agents after host restart (#8127) * feat(server): resume interrupted agents after host restart Continue saved running conversations after graceful restart or crash, with host opt-out and a configurable continuation prompt. Preserve explicit launch overrides when reopening the same provider session. Co-authored-by: coolwind <45729206+coolwindX@users.noreply.github.com> Co-authored-by: suntianc * fix(app): translate restart recovery host settings * test(app): synchronize sidebar drags and clean up added projects Wait for the drag sensor click blocker to expire before the next user action. Resolve UI project keys to daemon IDs for cleanup so empty projects do not contaminate later sidebar tests. * fix(app): tuck the resume prompt behind an edit sheet Match the System prompt row: an Edit button left of the restart recovery switch opens the prompt in a sheet, and is disabled while resume is off. * fix(server): resume interrupted agents without holding startup Startup now only chooses resume candidates before anything else reads or repairs persisted running state. Reopening and prompting run in the background after connections open, so a provider that is slow to reopen holds back only its own agent. Finish observation waits for a pending child's resume, so parents are not told the child was closed meanwhile. * style(server): format restart resume e2e test * test(server): wait for the fetch result before advancing the narrow refresh debounce The test released the held fetch, flushed six microtasks, then advanced the 1s refresh debounce. When the fetch result landed after that advance (slow Windows runners), the debounce started late and vi.waitFor's auto-advance reached it only at its real-time timeout. Wait on the service's in-flight fetch count instead, as the sibling test does. * test(server): retry stalled OpenCode fixture downloads within the test budget npm abandons a stalled registry download only after its 300s default fetch timeout, past the install's 180s budget. The killed install left the fixture directory locked on Windows, so cleanup failed with EBUSY and hid the timeout. Against a registry proxy that stalls the first tarball, a default install hung 225s; with --fetch-timeout it retried and finished in 16s. --------- Co-authored-by: suntianc --- docs/agent-lifecycle.md | 26 ++- .../app/e2e/browser/projects-settings.spec.ts | 17 +- .../e2e/browser/settings-host-page.spec.ts | 21 ++ .../app/e2e/browser/sidebar-reorder.spec.ts | 11 +- packages/app/e2e/support/helpers/settings.ts | 38 ++++ packages/app/src/i18n/resources/ar.ts | 8 + packages/app/src/i18n/resources/en.ts | 8 + packages/app/src/i18n/resources/es.ts | 8 + packages/app/src/i18n/resources/fr.ts | 8 + packages/app/src/i18n/resources/ja.ts | 8 + packages/app/src/i18n/resources/ko.ts | 8 + packages/app/src/i18n/resources/pt-BR.ts | 8 + packages/app/src/i18n/resources/ru.ts | 8 + packages/app/src/i18n/resources/zh-CN.ts | 8 + .../app/src/screens/settings/host-page.tsx | 2 + .../settings/restart-recovery-card.tsx | 199 ++++++++++++++++++ packages/protocol/src/messages.ts | 7 + .../server/src/server/agent/agent-loading.ts | 1 + .../src/server/agent/agent-manager.test.ts | 86 ++++++++ .../server/src/server/agent/agent-manager.ts | 16 +- .../src/server/agent/agent-projections.ts | 1 + .../server/src/server/agent/agent-storage.ts | 4 + .../internal/observation.ts | 4 + .../src/server/agent/mcp-server.test.ts | 1 + .../opencode-bridge.local.e2e.test.ts | 12 +- .../agent/restart-recovery/index.test.ts | 105 +++++++++ .../server/agent/restart-recovery/index.ts | 81 +++++++ .../server/src/server/bootstrap.smoke.test.ts | 4 +- packages/server/src/server/bootstrap.ts | 33 ++- packages/server/src/server/config.ts | 10 + .../src/server/daemon-config-store.test.ts | 9 + .../server/src/server/daemon-config-store.ts | 20 ++ .../daemon-restart-resume.e2e.test.ts | 147 ++++++++++++- .../server/src/server/persisted-config.ts | 2 + .../server/test-utils/fake-agent-client.ts | 2 + .../test-utils/finish-subscription-daemon.ts | 9 +- .../src/server/test-utils/paseo-daemon.ts | 4 + .../server/src/server/websocket-server.ts | 1 + .../src/server/workspace-git-service.test.ts | 6 +- .../public/schemas/paseo.config.v1.json | 6 + 40 files changed, 926 insertions(+), 31 deletions(-) create mode 100644 packages/app/src/screens/settings/restart-recovery-card.tsx create mode 100644 packages/server/src/server/agent/restart-recovery/index.test.ts create mode 100644 packages/server/src/server/agent/restart-recovery/index.ts diff --git a/docs/agent-lifecycle.md b/docs/agent-lifecycle.md index 96afdc54fa..2c2ce29403 100644 --- a/docs/agent-lifecycle.md +++ b/docs/agent-lifecycle.md @@ -12,6 +12,26 @@ initializing → idle → running → idle (or error → closed) Each live agent in `AgentManager` carries a `lastStatus` of `initializing`, `idle`, `running`, or `error`. `closed` is the persisted, resumable state for an agent record that has no live provider runtime. State transitions persist to disk and stream to subscribed clients via WebSocket. +## Daemon restart recovery + +Hosts resume interrupted agents by default after graceful shutdown or a crash. The daemon +reopens the saved provider conversation and settings, then sends `Resume working.`. Change +`agents.resumePrompt` or disable `agents.resumeOnStartup` in host config or Host settings → Agents. +Disabling recovery consumes pending intent; enabling it later does not revive old work. + +Only agents recorded as running qualify. Idle and initializing agents stay inactive. Archived +agents or workspaces, private helpers and Hub-owned executions retain their own lifecycle. +Explicit creation environment overrides are saved with the agent and reapplied when reopening. + +Startup chooses candidates before anything else reads or repairs persisted running state, then +opens connections and resumes every candidate in the background. A provider that is slow to reopen +holds back only its own agent; a failed reopen is logged. Graceful shutdown records intent before +closing runtimes. + +This is best-effort continuation. A crash between saving intent and dispatching a prompt can lose +or repeat a continuation. Provider resume owns conversation restoration; there is no recovery retry +loop or exactly-once turn guarantee. + ## Runtime residency An unarchived agent may be `closed` without being deleted or archived. Closing releases its provider @@ -139,8 +159,10 @@ caller, child, and host into one watch. Bootstrap restores these watches after repairing stale running records to `closed` and before opening ingress. Reading a closed child's result does not load its provider. A finish notification reports that -the child's run has settled; the parent decides whether its task is complete. Recovery never resumes -child work. Archived or deleted parents suppress delivery; parent-owned creation watches also stop +the child's run has settled; the parent decides whether its task is complete. A child chosen for restart +recovery is observed only after its resume was sent, so the parent never hears that it was closed +in between. Archived or deleted parents suppress delivery; +parent-owned creation watches also stop reporting after detachment. Intent remains stored until the parent's notification dispatch is acknowledged. Dispatch failure diff --git a/packages/app/e2e/browser/projects-settings.spec.ts b/packages/app/e2e/browser/projects-settings.spec.ts index 0612d5ce07..0732ec6732 100644 --- a/packages/app/e2e/browser/projects-settings.spec.ts +++ b/packages/app/e2e/browser/projects-settings.spec.ts @@ -204,19 +204,26 @@ test.describe("Projects settings", () => { }) => { const repo = await createTempGitRepo("projects-settings-empty-"); const client = await connectSeedClient(); - let projectId: string | null = null; + let projectViewKey: string | null = null; try { await gotoAppShell(page); - projectId = await addProjectFromSidebar(page, repo.path); - await openProjectSettingsFromSidebar(page, projectId); + projectViewKey = await addProjectFromSidebar(page, repo.path); + await openProjectSettingsFromSidebar(page, projectViewKey); await expectProjectSettingsFormVisible(page); await expect(page.getByTestId("project-settings-back-button")).not.toBeVisible(); } finally { - if (projectId) { - await client.removeProject(projectId).catch(() => undefined); + if (projectViewKey) { + const project = (await client.listProjects()).projects.find( + (entry) => entry.projectKey === projectViewKey, + ); + expect(project).toBeDefined(); + await client.removeProject(project!.projectId); + expect( + (await client.listProjects()).projects.map((entry) => entry.projectKey), + ).not.toContain(projectViewKey); } await client.close().catch(() => undefined); await repo.cleanup().catch(() => undefined); diff --git a/packages/app/e2e/browser/settings-host-page.spec.ts b/packages/app/e2e/browser/settings-host-page.spec.ts index 76e5b331e6..f9b358a307 100644 --- a/packages/app/e2e/browser/settings-host-page.spec.ts +++ b/packages/app/e2e/browser/settings-host-page.spec.ts @@ -30,6 +30,10 @@ import { addDirectHostFromSettings, expectHostOnlineWithoutError, expectIdleAgentRelease, + editRestartRecoveryPrompt, + expectRestartRecoveryEnabled, + expectRestartRecoveryPrompt, + setRestartRecoveryEnabled, setIdleAgentRelease, } from "../support/helpers/settings"; @@ -252,3 +256,20 @@ test.describe("Settings host page", () => { await expectHostActionCards(page, serverId); }); }); + +test("restart recovery host controls persist the opt-out and custom prompt", async ({ page }) => { + const serverId = getServerId(); + await gotoAppShell(page); + await openSettings(page); + await openSettingsHost(page, serverId); + await openHostSection(page, serverId, "agents"); + await expectRestartRecoveryEnabled(page, true); + await expectRestartRecoveryPrompt(page, "Resume working."); + await editRestartRecoveryPrompt(page, "Continue the interrupted work."); + await setRestartRecoveryEnabled(page, false); + await page.reload(); + await expectRestartRecoveryEnabled(page, false); + await setRestartRecoveryEnabled(page, true); + await expectRestartRecoveryPrompt(page, "Continue the interrupted work."); + await editRestartRecoveryPrompt(page, "Resume working."); +}); diff --git a/packages/app/e2e/browser/sidebar-reorder.spec.ts b/packages/app/e2e/browser/sidebar-reorder.spec.ts index 03ffe9f7db..56cdd3dc5a 100644 --- a/packages/app/e2e/browser/sidebar-reorder.spec.ts +++ b/packages/app/e2e/browser/sidebar-reorder.spec.ts @@ -53,11 +53,18 @@ async function quickDragFirstRowAfterSecond( await page.mouse.move(source.x, source.y + 7); await expect(trailingScrim).toHaveCount(0); await page.mouse.move(target.x, target.y, { steps: 4 }); - await page.mouse.up(); + await finishSidebarDrag(page); await expect.poll(() => rowTestIds(rows)).toEqual([before[1], before[0]]); } +async function finishSidebarDrag(page: Page): Promise { + await page.mouse.up(); + // dnd-kit's pointer sensor keeps its document click blocker for 50 ms after drop. + // Row order updates before that blocker is removed; wait before the next user action. + await page.waitForTimeout(60); +} + async function dragWithoutReordering(rows: Locator): Promise { const before = await rowTestIds(rows); const source = await visibleBoundingBox(rows.nth(0)); @@ -66,7 +73,7 @@ async function dragWithoutReordering(rows: Locator): Promise { await page.mouse.move(source.x + source.width / 2, source.y + source.height / 2); await page.mouse.down(); await page.mouse.move(target.x + target.width / 2, target.y + target.height / 2, { steps: 5 }); - await page.mouse.up(); + await finishSidebarDrag(page); await expect.poll(() => rowTestIds(rows)).toEqual(before); } diff --git a/packages/app/e2e/support/helpers/settings.ts b/packages/app/e2e/support/helpers/settings.ts index 07bd716105..a2befccfbf 100644 --- a/packages/app/e2e/support/helpers/settings.ts +++ b/packages/app/e2e/support/helpers/settings.ts @@ -560,3 +560,41 @@ export async function expectHostOnlineWithoutError(page: Page): Promise { await expect(page.getByTestId("host-page-identity")).toContainText("Online"); await expect(page.getByTestId("host-connection-error")).toHaveCount(0); } + +function restartRecoverySwitch(page: Page) { + return page.getByRole("switch", { name: "Resume agents after restart", exact: true }); +} + +function restartRecoveryEditButton(page: Page) { + return page.getByRole("button", { name: "Edit prompt", exact: true }); +} + +export async function setRestartRecoveryEnabled(page: Page, enabled: boolean) { + const toggle = restartRecoverySwitch(page); + await toggle.click(); + await expectRestartRecoveryEnabled(page, enabled); +} + +// The prompt is only editable while restart recovery is on. +export async function expectRestartRecoveryEnabled(page: Page, enabled: boolean) { + await expect(restartRecoverySwitch(page)).toBeChecked({ checked: enabled }); + await expect(restartRecoveryEditButton(page)).toBeEnabled({ enabled }); +} + +export async function editRestartRecoveryPrompt(page: Page, prompt: string) { + await restartRecoveryEditButton(page).click(); + const sheet = page.getByTestId("host-page-restart-recovery-sheet"); + await sheet.getByRole("textbox", { name: "Resume prompt", exact: true }).fill(prompt); + await sheet.getByRole("button", { name: "Save", exact: true }).click(); + await expect(sheet).toHaveCount(0); +} + +export async function expectRestartRecoveryPrompt(page: Page, prompt: string) { + await restartRecoveryEditButton(page).click(); + const sheet = page.getByTestId("host-page-restart-recovery-sheet"); + await expect(sheet.getByRole("textbox", { name: "Resume prompt", exact: true })).toHaveValue( + prompt, + ); + await sheet.getByRole("button", { name: "Cancel", exact: true }).click(); + await expect(sheet).toHaveCount(0); +} diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index 033ef38f2e..2f285ebcdb 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -2778,6 +2778,14 @@ export const ar: TranslationResources = { }, agents: { unavailable: "Connect to this host to manage agents", + resumeOnStartup: { + title: "استئناف الوكلاء بعد إعادة التشغيل", + info: "إعادة فتح الوكلاء الذين كانوا يعملون عند توقف المضيف وإرسال طلب الاستئناف. ينطبق ذلك بعد إعادة التشغيل العادية والتعطّل.", + hint: "متابعة العمل المنقطع تلقائيًا.", + prompt: "طلب الاستئناف", + edit: "تحرير", + editAccessibilityLabel: "تعديل الطلب", + }, idleRuntimeCleanup: { title: "تحرير الوكلاء الخاملين", hint: "تحرير الذاكرة بعد ساعة من الخمول.", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 2e0cd804ef..896e8ebac7 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -2870,6 +2870,14 @@ export const en = { }, agents: { unavailable: "Connect to this host to manage agents", + resumeOnStartup: { + title: "Resume agents after restart", + info: "Reopen agents that were working when the host stopped and send the resume prompt. Applies after graceful restarts and crashes.", + hint: "Continue interrupted work automatically.", + prompt: "Resume prompt", + edit: "Edit", + editAccessibilityLabel: "Edit prompt", + }, idleRuntimeCleanup: { title: "Release idle agents", hint: "Free up memory after an hour of inactivity.", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 655293e4f7..016eb99c6e 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -2800,6 +2800,14 @@ export const es: TranslationResources = { }, agents: { unavailable: "Connect to this host to manage agents", + resumeOnStartup: { + title: "Reanudar agentes tras reiniciar", + info: "Vuelve a abrir los agentes que estaban trabajando cuando se detuvo el host y envía el mensaje de reanudación. Se aplica tras reinicios normales y fallos.", + hint: "Continuar automáticamente el trabajo interrumpido.", + prompt: "Mensaje de reanudación", + edit: "Editar", + editAccessibilityLabel: "Editar mensaje", + }, idleRuntimeCleanup: { title: "Liberar agentes inactivos", hint: "Libera memoria tras una hora de inactividad.", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 927fa9299a..cbbf2d9ec8 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -2803,6 +2803,14 @@ export const fr: TranslationResources = { }, agents: { unavailable: "Connectez-vous à cet hôte pour gérer les agents", + resumeOnStartup: { + title: "Reprendre les agents après un redémarrage", + info: "Rouvre les agents qui travaillaient lorsque l’hôte s’est arrêté et envoie le message de reprise. S’applique après un redémarrage normal ou un plantage.", + hint: "Continuer automatiquement le travail interrompu.", + prompt: "Message de reprise", + edit: "Modifier", + editAccessibilityLabel: "Modifier le message", + }, idleRuntimeCleanup: { title: "Libérer les agents inactifs", hint: "Libère de la mémoire après une heure d’inactivité.", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index b9582cefc9..05a7aa457e 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -2767,6 +2767,14 @@ export const ja: TranslationResources = { }, agents: { unavailable: "エージェントを管理するにはこのホストに接続してください", + resumeOnStartup: { + title: "再起動後にエージェントを再開", + info: "ホストの停止時に作業中だったエージェントを開き直し、再開メッセージを送信します。通常の再起動とクラッシュの両方に適用されます。", + hint: "中断された作業を自動的に続行します。", + prompt: "再開メッセージ", + edit: "編集", + editAccessibilityLabel: "メッセージを編集", + }, idleRuntimeCleanup: { title: "アイドル状態のエージェントを解放", hint: "1 時間のアイドル状態の後にメモリを解放します。", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index 1f3ed0b3ef..d8a7491e92 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -2755,6 +2755,14 @@ export const ko: TranslationResources = { }, agents: { unavailable: "에이전트를 관리하려면 이 호스트에 연결하세요", + resumeOnStartup: { + title: "재시작 후 에이전트 재개", + info: "호스트가 중지될 때 작업 중이던 에이전트를 다시 열고 재개 메시지를 보냅니다. 정상 재시작과 비정상 종료 후 모두 적용됩니다.", + hint: "중단된 작업을 자동으로 계속합니다.", + prompt: "재개 메시지", + edit: "편집", + editAccessibilityLabel: "메시지 편집", + }, idleRuntimeCleanup: { title: "유휴 에이전트 해제", hint: "1시간 동안 활동이 없으면 메모리를 확보합니다.", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 139ac00a2b..b2abd39b25 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -2782,6 +2782,14 @@ export const ptBR: TranslationResources = { }, agents: { unavailable: "Conecte-se a este host para gerenciar agentes", + resumeOnStartup: { + title: "Retomar agentes após reiniciar", + info: "Reabre os agentes que estavam trabalhando quando o host parou e envia a mensagem de retomada. Aplica-se após reinicializações normais e falhas.", + hint: "Continuar automaticamente o trabalho interrompido.", + prompt: "Mensagem de retomada", + edit: "Editar", + editAccessibilityLabel: "Editar mensagem", + }, idleRuntimeCleanup: { title: "Liberar agentes ociosos", hint: "Libera memória após uma hora de inatividade.", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index b4d0eb817e..cdacf42173 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -2792,6 +2792,14 @@ export const ru: TranslationResources = { }, agents: { unavailable: "Подключитесь к этому хосту, чтобы управлять агентами", + resumeOnStartup: { + title: "Возобновлять агентов после перезапуска", + info: "Открывает агентов, которые работали при остановке хоста, и отправляет сообщение для продолжения. Применяется после обычного перезапуска и сбоя.", + hint: "Автоматически продолжать прерванную работу.", + prompt: "Сообщение для продолжения", + edit: "Редактировать", + editAccessibilityLabel: "Изменить сообщение", + }, idleRuntimeCleanup: { title: "Освобождать неактивных агентов", hint: "Освобождает память после часа бездействия.", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index a231456352..f8131c7dca 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -2713,6 +2713,14 @@ export const zhCN: TranslationResources = { }, agents: { unavailable: "连接到这个 Host 以管理 Agent", + resumeOnStartup: { + title: "重启后恢复智能体", + info: "重新打开主机停止时正在工作的智能体，并发送恢复提示。适用于正常重启和崩溃后的恢复。", + hint: "自动继续被中断的工作。", + prompt: "恢复提示", + edit: "编辑", + editAccessibilityLabel: "编辑提示", + }, idleRuntimeCleanup: { title: "释放空闲 Agent", hint: "空闲一小时后释放内存。", diff --git a/packages/app/src/screens/settings/host-page.tsx b/packages/app/src/screens/settings/host-page.tsx index cf26f68e6f..879ef2fe95 100644 --- a/packages/app/src/screens/settings/host-page.tsx +++ b/packages/app/src/screens/settings/host-page.tsx @@ -65,6 +65,7 @@ import { ICON_SIZE } from "@/styles/theme"; import type { Theme } from "@/styles/theme"; import { useProviderIcon } from "@/components/provider-icons"; import { BrowserToolsOptInCard } from "./browser-tools-card"; +import { RestartRecoveryCard } from "./restart-recovery-card"; import { IdleRuntimeCleanupCard } from "./idle-runtime-cleanup-card"; import { restartDaemonFromSettings, updateDaemonFromSettings } from "./daemon-lifecycle"; @@ -292,6 +293,7 @@ export function HostAgentsPage({ serverId }: { serverId: string }) { + ) : ( diff --git a/packages/app/src/screens/settings/restart-recovery-card.tsx b/packages/app/src/screens/settings/restart-recovery-card.tsx new file mode 100644 index 0000000000..9fa8a872b0 --- /dev/null +++ b/packages/app/src/screens/settings/restart-recovery-card.tsx @@ -0,0 +1,199 @@ +import { useCallback, useMemo, useState } from "react"; +import { Text, View } from "react-native"; +import { useMutation } from "@tanstack/react-query"; +import { useTranslation } from "react-i18next"; +import { StyleSheet } from "react-native-unistyles"; +import { AdaptiveModalSheet, type SheetHeader } from "@/components/adaptive-modal-sheet"; +import { SettingsTextAreaCard } from "@/components/settings-textarea"; +import { SettingsInfoTip } from "@/components/settings/headings/settings-info-tip"; +import { Button } from "@/components/ui/button"; +import { Switch } from "@/components/ui/switch"; +import { useDaemonConfig } from "@/hooks/use-daemon-config"; +import { useHostFeature } from "@/runtime/host-features"; +import { useHostRuntimeIsConnected } from "@/runtime/host-runtime"; +import { settingsStyles } from "@/styles/settings"; + +const DEFAULT_RESUME_PROMPT = "Resume working."; + +// COMPAT(restartRecovery): added in v0.14.0, remove gate after 2027-04-28. +export function RestartRecoveryCard({ serverId }: { serverId: string }) { + const { t } = useTranslation(); + const isConnected = useHostRuntimeIsConnected(serverId); + const isSupported = useHostFeature(serverId, "resumeOnStartupSetting"); + const { config, patchConfig } = useDaemonConfig(serverId); + const [isEditing, setIsEditing] = useState(false); + const mutation = useMutation({ + mutationFn: async (next: { resumeOnStartup?: boolean; resumePrompt?: string }) => { + const result = await patchConfig(next); + if (!result) { + throw new Error(t("workspace.terminal.hostDisconnected")); + } + return result; + }, + }); + + const handleValueChange = useCallback( + (next: boolean) => { + mutation.mutate({ resumeOnStartup: next }); + }, + [mutation], + ); + + const handleOpen = useCallback(() => { + mutation.reset(); + setIsEditing(true); + }, [mutation]); + + const handleClose = useCallback(() => { + if (mutation.isPending) return; + mutation.reset(); + setIsEditing(false); + }, [mutation]); + + const handleSave = useCallback( + (prompt: string) => { + mutation.mutate({ resumePrompt: prompt }, { onSuccess: () => setIsEditing(false) }); + }, + [mutation], + ); + + if (!isConnected || !isSupported || !config) return null; + + const isEnabled = config.resumeOnStartup !== false; + + return ( + <> + + + + + + {t("settings.host.agents.resumeOnStartup.title")} + + + + + {t("settings.host.agents.resumeOnStartup.hint")} + + {mutation.error && !isEditing ? ( + + {mutation.error.message} + + ) : null} + + + + + + + + + {isEditing ? ( + + ) : null} + + ); +} + +function ResumePromptSheet({ + initialPrompt, + isSaving, + error, + onSave, + onClose, +}: { + initialPrompt: string; + isSaving: boolean; + error: string | null; + onSave: (prompt: string) => void; + onClose: () => void; +}) { + const { t } = useTranslation(); + const [draft, setDraft] = useState(initialPrompt); + const header = useMemo( + () => ({ title: t("settings.host.agents.resumeOnStartup.prompt") }), + [t], + ); + const canSave = draft !== initialPrompt && draft.trim().length > 0 && !isSaving; + + const handleSave = useCallback(() => onSave(draft), [draft, onSave]); + + return ( + + + {error ? ( + + {error} + + ) : null} + + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + titleRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + trailing: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[3], + }, + sheetActions: { + flexDirection: "row", + justifyContent: "flex-end", + gap: theme.spacing[2], + }, +})); diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 26a90016a7..d20c91d773 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -237,6 +237,9 @@ export const MutableDaemonConfigSchema = z // COMPAT(idleRuntimeCleanupSetting): added in v0.13.0, keep optional while old daemons are supported. // Persisted host-wide value, applied live by the daemon. idleRuntimeCleanup: z.boolean().optional(), + // COMPAT(restartRecovery): added in v0.14.0, keep optional for older hosts. + resumeOnStartup: z.boolean().optional(), + resumePrompt: z.string().optional(), appendSystemPrompt: z.string().default(""), terminalProfiles: z.array(TerminalProfileSchema).optional(), agentProfiles: z.array(AgentProfileSchema).optional(), @@ -262,6 +265,8 @@ export const MutableDaemonConfigPatchSchema = z autoArchiveAfterMerge: z.boolean().optional(), enableTerminalAgentHooks: z.boolean().optional(), idleRuntimeCleanup: z.boolean().optional(), + resumeOnStartup: z.boolean().optional(), + resumePrompt: z.string().optional(), appendSystemPrompt: z.string().optional(), terminalProfiles: z.array(TerminalProfileSchema).optional(), agentProfiles: z.array(AgentProfileSchema).optional(), @@ -3731,6 +3736,8 @@ export const ServerInfoStatusPayloadSchema = z skillManagement: z.boolean().optional(), // COMPAT(idleRuntimeCleanupSetting): added in v0.13.0, remove gate after 2027-04-26. idleRuntimeCleanupSetting: z.boolean().optional(), + // COMPAT(restartRecovery): added in v0.14.0, remove gate after 2027-04-28. + resumeOnStartupSetting: z.boolean().optional(), // COMPAT(terminalRestoreModes): added in v0.1.81, remove gate after 2026-11-23. "terminal-restore-modes": z.boolean().optional(), // COMPAT(terminalInputModeReplay): added in v0.2.6, remove gate after 2027-02-02. diff --git a/packages/server/src/server/agent/agent-loading.ts b/packages/server/src/server/agent/agent-loading.ts index 761f1008bd..ba444a394d 100644 --- a/packages/server/src/server/agent/agent-loading.ts +++ b/packages/server/src/server/agent/agent-loading.ts @@ -135,6 +135,7 @@ export async function ensureAgentLoaded( labels: record.labels, workspaceId: record.workspaceId, owner: record.owner, + env: record.launchEnv, }); deps.logger.info({ agentId, provider: record.provider }, "Agent created from stored config"); } diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index eb1c05b50f..1d6f323deb 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -2929,6 +2929,92 @@ test("createAgent passes daemon launch env through the provider launch context", }); }); +test.each([ + { provider: "codex", hasHandle: true }, + { provider: "opencode", hasHandle: true }, + { provider: "codex", hasHandle: false }, + { provider: "opencode", hasHandle: false }, +])( + "$provider preserves explicit launch env across reloads and storage-backed resumes (handle: $hasHandle)", + async ({ provider, hasHandle }) => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-launch-env-")); + const storagePath = join(workdir, "agents"); + class LaunchEnvClient extends TestAgentClient { + lastLaunchContext: AgentLaunchContext | undefined; + + override async createSession( + config: AgentSessionConfig, + context?: AgentLaunchContext, + ): Promise { + this.lastLaunchContext = context; + const session = await super.createSession(config); + session.describePersistence = () => ({ provider, sessionId: session.id }); + return session; + } + + override async resumeSession( + handle: AgentPersistenceHandle, + config?: Partial, + context?: AgentLaunchContext, + ): Promise { + this.lastLaunchContext = context; + const session = await super.resumeSession(handle, config, context); + session.describePersistence = () => ({ provider, sessionId: handle.sessionId }); + return session; + } + } + + const client = new LaunchEnvClient(provider); + let storage = new AgentStorage(storagePath, logger); + let manager = new AgentManager({ clients: { [provider]: client }, registry: storage, logger }); + const env = { XDG_DATA_HOME: join(workdir, "agent-data"), CUSTOM_EMPTY: "" }; + const expectedEnv = { ...env }; + const agent = await manager.createAgent({ provider, cwd: workdir }, undefined, { + workspaceId: undefined, + env, + }); + onTestFinished(async () => { + await manager.closeAgent(agent.id); + await manager.flush(); + rmSync(workdir, { recursive: true, force: true }); + }); + env.XDG_DATA_HOME = "caller-mutated"; + + await manager.closeAgent(agent.id); + await manager.flush(); + + if (!hasHandle) { + const record = await storage.get(agent.id); + if (!record) throw new Error("Created agent was not persisted"); + await storage.upsert({ ...record, persistence: null }); + } + + // Fresh storage and manager instances must read the environment from disk. + for (let restart = 0; restart < 2; restart += 1) { + storage = new AgentStorage(storagePath, logger); + manager = new AgentManager({ clients: { [provider]: client }, registry: storage, logger }); + const resumed = await ensureAgentLoaded(agent.id, { + agentManager: manager, + agentStorage: storage, + logger, + }); + expect(client.lastLaunchContext).toEqual({ + agentId: agent.id, + env: { ...expectedEnv, PASEO_AGENT_ID: agent.id, PASEO_AGENT_CWD: workdir }, + }); + await manager.reloadAgentSession(agent.id); + expect(client.lastLaunchContext).toEqual({ + agentId: agent.id, + env: { ...expectedEnv, PASEO_AGENT_ID: agent.id, PASEO_AGENT_CWD: workdir }, + }); + expect(toAgentPayload(resumed)).not.toHaveProperty("launchEnv"); + expect(toAgentPayload(resumed).persistence?.metadata).not.toHaveProperty("launchEnv"); + await manager.closeAgent(agent.id); + await manager.flush(); + } + }, +); + test("createAgent passes persistSession to provider create options", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 966542ff62..46c822ea2d 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -421,6 +421,8 @@ interface ManagedAgentBase { owner?: AgentOwner; capabilities: AgentCapabilityFlags; config: AgentSessionConfig; + /** Explicit launch overrides, before per-opening hooks and daemon-owned variables. */ + launchEnv?: Record; runtimeInfo?: AgentRuntimeInfo; createdAt: Date; updatedAt: Date; @@ -1399,6 +1401,7 @@ export class AgentManager { initialTitle: options.initialTitle, workspaceId: options.workspaceId, owner: options.owner, + launchEnv: options.env ? { ...options.env } : undefined, historyPrimed: true, }); if (!agent.internal) { @@ -1487,10 +1490,11 @@ export class AgentManager { : resumeOptions; const purpose = currentResumeOptions?.purpose ?? "interactive"; + const launchEnv = record?.launchEnv; const { storedConfig, launchConfig, paseoToolPolicy } = await this.prepareSessionConfig( mergedConfig, resolvedAgentId, - { purpose }, + { purpose, env: launchEnv }, ); const client = this.requireClient(handle.provider); const available = await client.isAvailable(); @@ -1505,7 +1509,7 @@ export class AgentManager { client, storedConfig.cwd, paseoToolPolicy, - undefined, + launchEnv, { reason: "resume", purpose, @@ -1524,6 +1528,7 @@ export class AgentManager { await this.requireExternalMcpSupport(session, providerLaunchConfig, launchContext); return this.registerSession(session, storedConfig, resolvedAgentId, { ...options, + launchEnv, persistence: handle, restoring: true, }); @@ -1661,6 +1666,7 @@ export class AgentManager { const { storedConfig, launchConfig, paseoToolPolicy } = await this.prepareSessionConfig( refreshConfig, agentId, + { env: existing.launchEnv }, ); const hadPreviousPaseoToolPolicy = this.paseoToolPolicies.has(agentId); const previousPaseoToolPolicy = this.paseoToolPolicies.get(agentId); @@ -1669,7 +1675,7 @@ export class AgentManager { client, storedConfig.cwd, paseoToolPolicy, - undefined, + existing.launchEnv, { reason: "refresh", purpose: "interactive", @@ -1720,6 +1726,7 @@ export class AgentManager { labels: existing.labels, workspaceId: existing.workspaceId, owner: existing.owner, + launchEnv: existing.launchEnv, createdAt: existing.createdAt, updatedAt: existing.updatedAt, lastUserMessageAt: existing.lastUserMessageAt, @@ -3763,6 +3770,7 @@ export class AgentManager { publishWhenReady?: boolean; workspaceId?: string; owner?: AgentOwner; + launchEnv?: Record; }, ): Promise { let registered = false; @@ -3948,6 +3956,7 @@ export class AgentManager { persistence?: AgentPersistenceHandle; workspaceId?: string; owner?: AgentOwner; + launchEnv?: Record; } | undefined; }): ActiveManagedAgent { @@ -3958,6 +3967,7 @@ export class AgentManager { cwd: config.cwd, workspaceId: options?.workspaceId, owner: options?.owner, + launchEnv: options?.launchEnv, session, capabilities: session.capabilities, config, diff --git a/packages/server/src/server/agent/agent-projections.ts b/packages/server/src/server/agent/agent-projections.ts index 7d154e9c14..b388143d8e 100644 --- a/packages/server/src/server/agent/agent-projections.ts +++ b/packages/server/src/server/agent/agent-projections.ts @@ -95,6 +95,7 @@ export function toStoredAgentRecord( lastStatus: agent.lifecycle, lastModeId: agent.currentModeId ?? config?.modeId ?? null, config: config ?? null, + launchEnv: agent.launchEnv ? { ...agent.launchEnv } : undefined, runtimeInfo, features: normalizeFeatures(agent.features), persistence, diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index be94a31d49..a097bd7f82 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -54,8 +54,10 @@ const STORED_AGENT_SCHEMA = z.object({ title: z.string().nullable().optional(), labels: z.record(z.string(), z.string()).default({}), lastStatus: AgentStatusSchema.default("closed"), + resumeOnStartup: z.boolean().optional(), lastModeId: z.string().nullable().optional(), config: SERIALIZABLE_CONFIG_SCHEMA, + launchEnv: z.record(z.string(), z.string()).optional(), runtimeInfo: z .object({ provider: z.string(), @@ -259,6 +261,8 @@ export class AgentStorage { if (existing && existing.archivedAt !== undefined) { record.archivedAt = existing.archivedAt; } + record.resumeOnStartup = + existing?.resumeOnStartup && agent.lifecycle !== "idle" && agent.lifecycle !== "error"; return record; }); } diff --git a/packages/server/src/server/agent/finish-subscriptions/internal/observation.ts b/packages/server/src/server/agent/finish-subscriptions/internal/observation.ts index 887a4ba075..7abe483cf4 100644 --- a/packages/server/src/server/agent/finish-subscriptions/internal/observation.ts +++ b/packages/server/src/server/agent/finish-subscriptions/internal/observation.ts @@ -13,6 +13,8 @@ import type { FinishObservation } from "../index.js"; export function createFinishObservation(input: { agentManager: AgentManager; agentStorage: AgentStorage; + /** A child chosen for restart recovery reports its state only after its resume was sent. */ + waitForResume(agentId: string, signal: AbortSignal): Promise; messages(): AgentMessageDelivery; hosts(): ToolHosts; serverId: string; @@ -24,6 +26,7 @@ export function createFinishObservation(input: { hosts: input.hosts, local: { async wait(agentId, signal) { + await input.waitForResume(agentId, signal); await input.messages().waitForDispatch(agentId, signal); signal.throwIfAborted(); const live = agentManager.getAgent(agentId); @@ -52,6 +55,7 @@ export function createFinishObservation(input: { }; }, async observe(agentId, handler, signal, onError) { + await input.waitForResume(agentId, signal); const stored = await agentStorage.get(agentId); let updates = Promise.resolve(); const publish = (snapshot: AgentSnapshotPayload | null, event?: AgentStreamEvent) => { diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 3b72c8805a..86ee383f73 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -881,6 +881,7 @@ async function createAgentMcpServer( logger: options.logger, ...createFinishObservation({ ...options, + waitForResume: async () => {}, messages: () => messages, serverId: options.serverId ?? "self", hosts: () => { diff --git a/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts b/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts index aeb87388fd..749374b29f 100644 --- a/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts +++ b/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts @@ -461,9 +461,19 @@ test.each([ process.platform === "win32" ? `${windowsPackage}-${process.arch}@${cliPackage.slice(cliPackage.lastIndexOf("@") + 1)}` : cliPackage; + // npm abandons a stalled registry download only after its 5 minute default, past this + // test's budget, and the killed install keeps the fixture locked. Retry stalls sooner. await execCommand( "npm", - ["install", "--prefix", root, "--no-audit", "--no-fund", installPackage], + [ + "install", + "--prefix", + root, + "--no-audit", + "--no-fund", + "--fetch-timeout=30000", + installPackage, + ], { cwd: root, timeout: 180_000, diff --git a/packages/server/src/server/agent/restart-recovery/index.test.ts b/packages/server/src/server/agent/restart-recovery/index.test.ts new file mode 100644 index 0000000000..0b1658b47d --- /dev/null +++ b/packages/server/src/server/agent/restart-recovery/index.test.ts @@ -0,0 +1,105 @@ +import { expect, test } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AgentStorage, parseStoredAgentRecord } from "../agent-storage.js"; +import { createTestLogger } from "../../../test-utils/test-logger.js"; +import { createRestartRecovery } from "./index.js"; + +test("recovery saves intent before normalization, clears before dispatch, and resumes each agent independently", async () => { + const root = await mkdtemp(join(tmpdir(), "restart-recovery-")); + try { + const storage = new AgentStorage(root, createTestLogger()); + for (const [id, lastStatus] of [ + ["broken", "running"], + ["stuck", "running"], + ["working", "running"], + ["idle", "idle"], + ["initializing", "initializing"], + ] as const) { + await storage.upsert( + parseStoredAgentRecord({ + id, + lastStatus, + provider: "codex", + cwd: root, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }), + ); + } + const dispatched: string[] = []; + const recovery = createRestartRecovery({ + storage, + settings: () => ({ enabled: true, prompt: "Continue verbatim.\n" }), + eligible: async () => true, + dispatch: async (id, prompt) => { + expect((await storage.get(id))?.resumeOnStartup).toBe(false); + if (id === "broken") throw new Error("unavailable"); + if (id === "stuck") return new Promise(() => {}); + dispatched.push(id + prompt); + }, + report: () => {}, + }); + await recovery.prepareStartup(); + expect((await storage.get("working"))?.resumeOnStartup).toBe(true); + expect((await storage.get("working"))?.lastStatus).toBe("closed"); + recovery.resumePending(); + const running = new AbortController().signal; + await recovery.waitForResume("broken", running); + await recovery.waitForResume("working", running); + await recovery.waitForResume("idle", running); + expect(dispatched).toEqual(["workingContinue verbatim.\n"]); + const stopped = new AbortController(); + stopped.abort(); + await expect(recovery.waitForResume("stuck", stopped.signal)).rejects.toBeDefined(); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("opt-out consumes pending recovery and excludes archived, private, owned and archived-workspace agents", async () => { + const root = await mkdtemp(join(tmpdir(), "restart-recovery-")); + try { + const storage = new AgentStorage(root, createTestLogger()); + for (const extra of [ + { id: "running" }, + { id: "archived", archivedAt: "today" }, + { id: "private", internal: true }, + { id: "owned", owner: { kind: "daemon", daemonId: "hub", executionId: "run" } }, + { id: "workspace", workspaceId: "archived" }, + ]) { + await storage.upsert( + parseStoredAgentRecord({ + lastStatus: "running", + provider: "codex", + cwd: root, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + ...extra, + }), + ); + } + let enabled = true; + const dispatched: string[] = []; + const recovery = createRestartRecovery({ + storage, + settings: () => ({ enabled, prompt: "Resume working." }), + eligible: async (record) => record.workspaceId !== "archived", + dispatch: async (id) => { + dispatched.push(id); + }, + report: () => {}, + }); + await recovery.captureRunningAgents(); + expect((await storage.get("running"))?.resumeOnStartup).toBe(true); + enabled = false; + await recovery.prepareStartup(); + enabled = true; + recovery.resumePending(); + expect(dispatched).toEqual([]); + expect((await storage.list()).every((record) => !record.resumeOnStartup)).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/server/src/server/agent/restart-recovery/index.ts b/packages/server/src/server/agent/restart-recovery/index.ts new file mode 100644 index 0000000000..dc4e7d90c7 --- /dev/null +++ b/packages/server/src/server/agent/restart-recovery/index.ts @@ -0,0 +1,81 @@ +import type { AgentStorage, StoredAgentRecord } from "../agent-storage.js"; +interface Dependencies { + storage: AgentStorage; + settings(): { enabled: boolean; prompt: string }; + eligible(record: StoredAgentRecord): Promise; + dispatch(agentId: string, prompt: string): Promise; + report(agentId: string, error: unknown): void; +} +export function createRestartRecovery(deps: Dependencies) { + // Candidates are chosen at startup; each settles once its resume was sent or abandoned. + const resuming = new Map; resolve(): void }>(); + const eligible = async (record: StoredAgentRecord) => + !record.archivedAt && !record.internal && !record.owner && (await deps.eligible(record)); + async function resume(agentId: string) { + const record = await deps.storage.get(agentId); + if (!record?.resumeOnStartup) return; + await deps.storage.upsert({ ...record, resumeOnStartup: false }); + if (!deps.settings().enabled || !(await eligible(record))) return; + await deps.dispatch(agentId, deps.settings().prompt); + } + return { + /** Choose candidates before anything else reads or repairs persisted running status. */ + async prepareStartup() { + for (const record of await deps.storage.list()) { + const pending = + deps.settings().enabled && + (record.resumeOnStartup === true || record.lastStatus === "running") && + (await eligible(record)); + await deps.storage.upsert({ + ...record, + resumeOnStartup: pending, + lastStatus: + record.lastStatus === "running" || record.lastStatus === "initializing" + ? "closed" + : record.lastStatus, + }); + if (pending) { + let resolve!: () => void; + const promise = new Promise((settle) => { + resolve = settle; + }); + resuming.set(record.id, { promise, resolve }); + } + } + }, + async captureRunningAgents() { + await deps.storage.flush(); + for (const record of await deps.storage.list()) { + if (record.lastStatus === "running" && (await eligible(record))) + await deps.storage.upsert({ ...record, resumeOnStartup: deps.settings().enabled }); + } + }, + /** Resume every candidate in the background; a slow provider holds back only its agent. */ + resumePending() { + for (const [agentId, settled] of resuming) { + void resume(agentId) + .catch((error) => deps.report(agentId, error)) + .finally(() => { + resuming.delete(agentId); + settled.resolve(); + }); + } + }, + /** Wait until this agent's resume was sent or abandoned. Others return at once. */ + async waitForResume(agentId: string, signal: AbortSignal) { + const settled = resuming.get(agentId); + if (!settled) return; + let abort!: () => void; + const aborted = new Promise((_resolve, reject) => { + abort = () => reject(signal.reason); + if (signal.aborted) abort(); + else signal.addEventListener("abort", abort, { once: true }); + }); + try { + await Promise.race([settled.promise, aborted]); + } finally { + signal.removeEventListener("abort", abort); + } + }, + }; +} diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index f6304491ab..76fa8cc7d1 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -1069,7 +1069,7 @@ function probeWebSocketConnection( }); } -test("startup closes stale active records before serving status without resuming sessions", async () => { +test("disabled recovery closes stale active records without resuming sessions", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "paseo-stale-status-")); const logger = pino({ level: "silent" }); const storagePath = path.join(root, ".paseo", "agents"); @@ -1121,6 +1121,7 @@ test("startup closes stale active records before serving status without resuming let sessionsCreated = 0; const handle = await createTestPaseoDaemon({ paseoHomeRoot: root, + resumeOnStartup: false, agentClients: createTestAgentClients({ beforeCreateSession: async () => { sessionsCreated++; @@ -1131,6 +1132,7 @@ test("startup closes stale active records before serving status without resuming try { const expected = records.map((record) => ({ ...record, + resumeOnStartup: false, lastStatus: record.lastStatus === "running" || record.lastStatus === "initializing" ? "closed" diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 36ba9e2b06..8fe0f05b14 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -1,3 +1,5 @@ +import { createRestartRecovery } from "./agent/restart-recovery/index.js"; +import { sendPromptToAgent } from "./agent/agent-prompt.js"; import { createIdleRuntimeCleanup } from "./agent/idle-runtime-cleanup/index.js"; import { createFinishSubscriptions, @@ -464,6 +466,8 @@ export interface PaseoDaemonConfig { downloadTokenTtlMs?: number; agentProviderSettings?: AgentProviderRuntimeSettingsMap; idleRuntimeCleanup?: boolean; + resumeOnStartup?: boolean; + resumePrompt?: string; providerCatalogRefreshTimeoutMs?: number; metadataGeneration?: { providers?: Array<{ @@ -594,6 +598,8 @@ function createInitialMutableDaemonConfig(config: ResolvedDaemonSettings): Mutab autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false, enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false, idleRuntimeCleanup: config.idleRuntimeCleanup !== false, + resumeOnStartup: config.resumeOnStartup !== false, + resumePrompt: config.resumePrompt ?? "Resume working.", appendSystemPrompt: config.appendSystemPrompt ?? "", pluginsEnabled: config.pluginsEnabled ?? false, plugins: config.plugins ?? {}, @@ -1036,7 +1042,19 @@ export async function createPaseoDaemon( agentStorage, ); await agentStorage.initialize(); - await closeStaleAgentRecords(agentStorage); + const restartRecovery = createRestartRecovery({ + storage: agentStorage, + settings: () => ({ + enabled: daemonConfigStore.get().resumeOnStartup !== false, + prompt: daemonConfigStore.get().resumePrompt ?? "Resume working.", + }), + eligible: async (record) => + !record.workspaceId || !(await workspaceRegistry?.get(record.workspaceId))?.archivedAt, + dispatch: (agentId, prompt) => + sendPromptToAgent({ agentId, prompt, agentManager, agentStorage, logger, unarchive: false }), + report: (agentId, err) => logger.error({ agentId, err }, "Agent restart recovery failed"), + }); + await restartRecovery.prepareStartup(); logger.info({ elapsed: elapsed() }, "Agent storage initialized"); await bootstrapWorkspaceRegistries({ serverId, @@ -1470,6 +1488,7 @@ export async function createPaseoDaemon( ...createFinishObservation({ agentManager, agentStorage, + waitForResume: restartRecovery.waitForResume, serverId, logger, messages: () => { @@ -1869,6 +1888,7 @@ export async function createPaseoDaemon( pluginRuntime.bindPaseoSessionHost(wsServer); await pluginRuntime.start(); providerSnapshotManager.settlePluginProviders(); + restartRecovery.resumePending(); await finishSubscriptions.start(); wsServer.beginAcceptingConnections(); const relaySecurityLogger = logger.child({ module: "relay-security" }); @@ -1963,6 +1983,7 @@ export async function createPaseoDaemon( await finishSubscriptions.close(); await wsServer?.agentMessages.close(); agentManager.prepareForShutdown(); + await restartRecovery.captureRunningAgents(); await closeAllAgents(logger, agentManager); await withTimeout({ promise: pluginRuntime.drainEvents(), @@ -2029,16 +2050,6 @@ export async function createPaseoDaemon( }; } -async function closeStaleAgentRecords(agentStorage: AgentStorage): Promise { - // No provider runtimes are loaded at startup. Repair interrupted - // shutdowns before any status consumer sees the persisted active states. - for (const record of await agentStorage.list()) { - if (record.lastStatus === "running" || record.lastStatus === "initializing") { - await agentStorage.upsert({ ...record, lastStatus: "closed" }); - } - } -} - /** * Closing an agent asks its provider to close the session and waits for the * answer. A provider that never answers must not hold the daemon open, so a diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index add78aa7f5..3149fc6eb9 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -590,6 +590,8 @@ export type ResolvedDaemonSettings = Pick< | "relayRequireProtectedSecurity" | "appBaseUrl" | "idleRuntimeCleanup" + | "resumeOnStartup" + | "resumePrompt" | "providerCatalogRefreshTimeoutMs" | "metadataGeneration" | "providerOverrides" @@ -677,6 +679,7 @@ export function resolveReloadableConfig( relayEnabled: relay.enabled, relayRequireProtectedSecurity: relay.requireProtectedSecurity, idleRuntimeCleanup: persisted.agents?.idleRuntimeCleanup ?? true, + ...resolveRestartRecoveryConfig(persisted.agents), providerCatalogRefreshTimeoutMs: persisted.agents?.catalogRefreshTimeoutMs, metadataGeneration: persisted.agents?.metadataGeneration, providerOverrides: extractProviderOverrides( @@ -827,3 +830,10 @@ function resolveSpeechOverrideControlledPaths( if (env.PASEO_LOCAL_MODELS_DIR !== undefined) paths.push("providers.local.modelsDir"); return paths; } + +function resolveRestartRecoveryConfig(agents: PersistedConfig["agents"]) { + return { + resumeOnStartup: agents?.resumeOnStartup ?? true, + resumePrompt: agents?.resumePrompt ?? "Resume working.", + }; +} diff --git a/packages/server/src/server/daemon-config-store.test.ts b/packages/server/src/server/daemon-config-store.test.ts index e3b6f48895..89c3c7f977 100644 --- a/packages/server/src/server/daemon-config-store.test.ts +++ b/packages/server/src/server/daemon-config-store.test.ts @@ -940,6 +940,15 @@ describe("DaemonConfigStore", () => { await store.patch({ idleRuntimeCleanup: true }); expect(loadConfig(paseoHome, { env: {} }).idleRuntimeCleanup).toBe(true); + await store.patch({ resumeOnStartup: false, resumePrompt: "Keep going.\n" }); + expect(loadPersistedConfig(paseoHome).agents).toMatchObject({ + resumeOnStartup: false, + resumePrompt: "Keep going.\n", + }); + expect(loadConfig(paseoHome, { env: {} })).toMatchObject({ + resumeOnStartup: false, + resumePrompt: "Keep going.\n", + }); }); test("patch persists custom ACP provider overrides into config.json", async () => { diff --git a/packages/server/src/server/daemon-config-store.ts b/packages/server/src/server/daemon-config-store.ts index 57b9a6b916..1bd67822a3 100644 --- a/packages/server/src/server/daemon-config-store.ts +++ b/packages/server/src/server/daemon-config-store.ts @@ -29,6 +29,8 @@ interface SupportedMutableConfigPatch { autoArchiveAfterMerge?: boolean; enableTerminalAgentHooks?: boolean; idleRuntimeCleanup?: boolean; + resumeOnStartup?: boolean; + resumePrompt?: string; appendSystemPrompt?: string; terminalProfiles?: MutableDaemonConfig["terminalProfiles"]; agentProfiles?: MutableDaemonConfig["agentProfiles"]; @@ -193,6 +195,8 @@ const RELOADABLE_PATHS = [ "daemon.agentProfiles", "app.baseUrl", "agents.idleRuntimeCleanup", + "agents.resumeOnStartup", + "agents.resumePrompt", "agents.providers", "agents.catalogRefreshTimeoutMs", "agents.metadataGeneration", @@ -218,6 +222,8 @@ const PERSISTED_TO_MUTABLE_PATH = new Map([ ["daemon.agentProfiles", "agentProfiles"], ["app.baseUrl", "app.baseUrl"], ["agents.idleRuntimeCleanup", "idleRuntimeCleanup"], + ["agents.resumeOnStartup", "resumeOnStartup"], + ["agents.resumePrompt", "resumePrompt"], ["agents.providers", "providers"], ["agents.catalogRefreshTimeoutMs", "catalogRefreshTimeoutMs"], ["agents.metadataGeneration", "metadataGeneration"], @@ -292,6 +298,8 @@ function pickSupportedPatchFields(patch: MutableDaemonConfigPatch): SupportedMut ...(patch.enableTerminalAgentHooks !== undefined ? { enableTerminalAgentHooks: patch.enableTerminalAgentHooks } : {}), + ...(patch.resumeOnStartup !== undefined ? { resumeOnStartup: patch.resumeOnStartup } : {}), + ...(patch.resumePrompt !== undefined ? { resumePrompt: patch.resumePrompt } : {}), ...(patch.idleRuntimeCleanup !== undefined ? { idleRuntimeCleanup: patch.idleRuntimeCleanup } : {}), @@ -702,6 +710,7 @@ function mergeMutableAgentPatch( patch.providers === undefined && patch.metadataGeneration === undefined && patch.skills === undefined && + !hasRestartRecoveryPatch(patch) && patch.idleRuntimeCleanup === undefined && removeProviders.length === 0 ) { @@ -735,6 +744,7 @@ function mergeMutableAgentPatch( next["skills"] = { selection: patch.skills.selection }; } + Object.assign(next, restartRecoveryPatch(patch)); if (patch.idleRuntimeCleanup !== undefined) { next["idleRuntimeCleanup"] = patch.idleRuntimeCleanup; } @@ -771,3 +781,13 @@ function mergeMutableDaemonPatch( if (patch.agentProfiles !== undefined) next.agentProfiles = patch.agentProfiles; return Object.keys(next).length > 0 ? next : undefined; } + +function hasRestartRecoveryPatch(patch: SupportedMutableConfigPatch) { + return patch.resumeOnStartup !== undefined || patch.resumePrompt !== undefined; +} +function restartRecoveryPatch(patch: SupportedMutableConfigPatch) { + return { + ...(patch.resumeOnStartup !== undefined ? { resumeOnStartup: patch.resumeOnStartup } : {}), + ...(patch.resumePrompt !== undefined ? { resumePrompt: patch.resumePrompt } : {}), + }; +} diff --git a/packages/server/src/server/daemon-e2e/daemon-restart-resume.e2e.test.ts b/packages/server/src/server/daemon-e2e/daemon-restart-resume.e2e.test.ts index efc99c9b59..1658356eed 100644 --- a/packages/server/src/server/daemon-e2e/daemon-restart-resume.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/daemon-restart-resume.e2e.test.ts @@ -76,10 +76,14 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { DaemonClient } from "../test-utils/daemon-client.js"; -async function restartFixture(root: string) { +async function restartFixture( + root: string, + settings: { resumeOnStartup?: boolean; resumePrompt?: string } = {}, + fixture: { holdResume?: boolean } = {}, +) { const child = fork( fileURLToPath(new URL("../test-utils/finish-subscription-daemon.ts", import.meta.url)), - [root], + [root, JSON.stringify(settings), JSON.stringify(fixture)], { execArgv: ["--conditions=source", "--import", "tsx"], stdio: ["ignore", "pipe", "pipe", "ipc"], @@ -183,7 +187,25 @@ test.each([false, true])( await expect.poll(() => readPrompts(root), { timeout: 15000 }).toHaveLength(1); const prompts = await readPrompts(root); expect(prompts[0]).toContain(`Agent ${childId}`); - expect(prompts[0]).toContain("was closed"); + expect(prompts[0]).not.toContain("was closed"); + const recoveredPrompts = (await readFile(path.join(root, "all-prompts.jsonl"), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(recoveredPrompts.filter((entry) => entry.prompt === "Resume working.")).toHaveLength( + 1, + ); + const restored = recoveredPrompts.find((entry) => entry.prompt === "Resume working.").config; + const original = recoveredPrompts.find((entry) => + String(entry.prompt).includes("HOLD_CHILD"), + ).config; + expect(restored).toMatchObject({ + provider: original.provider, + model: original.model, + modeId: original.modeId, + cwd: original.cwd, + }); + await expect .poll( async () => @@ -196,7 +218,7 @@ test.each([false, true])( client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" }); await client.connect(); await client.fetchAgents({ subscribe: {} }); - expect((await client.fetchAgent(childId))?.agent.status).toBe("closed"); + expect((await client.fetchAgent(childId))?.agent.status).toBe("idle"); await client.close(); client = undefined; await stopFixture(second.child, false); @@ -220,3 +242,120 @@ test.each([false, true])( }, 60000, ); + +test.each([false, true])( + "host restart recovery settings (enabled=%s)", + async (enabled) => { + const root = await mkdtemp(path.join(tmpdir(), "resume-settings-process-")); + const processes: ChildProcess[] = []; + let client: DaemonClient | undefined; + const settings = { resumeOnStartup: enabled, resumePrompt: "Continue exactly.\n" }; + try { + const first = await restartFixture(root, settings); + processes.push(first.child); + client = new DaemonClient({ url: `ws://127.0.0.1:${first.port}/ws`, appVersion: "0.13.0" }); + await client.connect(); + const working = await client.createAgent({ + provider: "claude", + cwd: first.cwd, + modeId: "full-access", + }); + const idle = await client.createAgent({ + provider: "claude", + cwd: first.cwd, + modeId: "full-access", + }); + await client.sendMessage(working.id, "HOLD_CHILD"); + await expect + .poll( + async () => + ( + await new AgentStorage(path.join(first.paseoHome, "agents"), createTestLogger()).get( + working.id, + ) + )?.lastStatus, + ) + .toBe("running"); + const original = await new AgentStorage( + path.join(first.paseoHome, "agents"), + createTestLogger(), + ).get(working.id); + await client.close(); + client = undefined; + await stopFixture(first.child, false); + const second = await restartFixture(root, settings); + processes.push(second.child); + client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" }); + await client.connect(); + const storage = new AgentStorage(path.join(second.paseoHome, "agents"), createTestLogger()); + // Resume runs in the background; it is done once the saved intent is consumed. + await expect.poll(async () => (await storage.get(working.id))?.resumeOnStartup).toBe(false); + const resumePrompts = async () => + (await readFile(path.join(root, "all-prompts.jsonl"), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + .filter((entry) => entry.prompt === settings.resumePrompt); + await expect.poll(resumePrompts).toHaveLength(enabled ? 1 : 0); + expect((await client.fetchAgent(idle.id))?.agent.status).toBe("closed"); + const restored = await storage.get(working.id); + expect(restored?.persistence?.sessionId).toBe(original?.persistence?.sessionId); + } finally { + await client?.close(); + for (const child of processes.toReversed()) await stopFixture(child, true); + await rm(root, { recursive: true, force: true }); + } + }, + 60000, +); + +test("a provider that is slow to reopen does not hold daemon startup", async () => { + const root = await mkdtemp(path.join(tmpdir(), "resume-background-process-")); + const processes: ChildProcess[] = []; + let client: DaemonClient | undefined; + try { + const first = await restartFixture(root); + processes.push(first.child); + client = new DaemonClient({ url: `ws://127.0.0.1:${first.port}/ws`, appVersion: "0.13.0" }); + await client.connect(); + const working = await client.createAgent({ + provider: "claude", + cwd: first.cwd, + modeId: "full-access", + }); + await client.sendMessage(working.id, "HOLD_CHILD"); + await expect + .poll( + async () => + ( + await new AgentStorage(path.join(first.paseoHome, "agents"), createTestLogger()).get( + working.id, + ) + )?.lastStatus, + ) + .toBe("running"); + await client.close(); + client = undefined; + await stopFixture(first.child, false); + + const second = await restartFixture(root, {}, { holdResume: true }); + processes.push(second.child); + client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" }); + await client.connect(); + expect((await client.fetchAgent(working.id))?.agent.id).toBe(working.id); + second.child.send("release-resume"); + await expect + .poll(async () => + (await readFile(path.join(root, "all-prompts.jsonl"), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line).prompt) + .filter((prompt) => prompt === "Resume working."), + ) + .toHaveLength(1); + } finally { + await client?.close(); + for (const child of processes.toReversed()) await stopFixture(child, true); + await rm(root, { recursive: true, force: true }); + } +}, 30000); diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 51bfb3d3a4..f7b3982513 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -348,6 +348,8 @@ export const PersistedConfigSchema = z .object({ providers: z.preprocess(normalizeAgentProviders, ProviderOverridesSchema).optional(), idleRuntimeCleanup: z.boolean().optional(), + resumeOnStartup: z.boolean().optional(), + resumePrompt: z.string().optional(), catalogRefreshTimeoutMs: z.number().int().positive().max(2_147_483_647).optional(), metadataGeneration: AgentMetadataGenerationSchema.optional(), skills: z.object({ selection: AgentSkillSelectionSchema.optional() }).strict().optional(), diff --git a/packages/server/src/server/test-utils/fake-agent-client.ts b/packages/server/src/server/test-utils/fake-agent-client.ts index 42ac0c7c79..4304a767e1 100644 --- a/packages/server/src/server/test-utils/fake-agent-client.ts +++ b/packages/server/src/server/test-utils/fake-agent-client.ts @@ -67,6 +67,7 @@ interface FakeAgentSessionOptions { export interface TestAgentClientOptions { beforeCreateSession?: (config: AgentSessionConfig) => Promise; + beforeResumeSession?: (handle: AgentPersistenceHandle) => Promise; closeSession?: () => Promise; onStartTurn?: (prompt: AgentPromptInput, config: AgentSessionConfig) => void; beforeTurnComplete?: (prompt: AgentPromptInput) => Promise; @@ -1254,6 +1255,7 @@ class FakeAgentClient implements AgentClient { overrides?: Partial, _launchContext?: AgentLaunchContext, ): Promise { + await this.options.beforeResumeSession?.(handle); const cfg: AgentSessionConfig = { provider: this.provider, cwd: overrides?.cwd ?? process.cwd(), diff --git a/packages/server/src/server/test-utils/finish-subscription-daemon.ts b/packages/server/src/server/test-utils/finish-subscription-daemon.ts index d35bc55ec8..eafe11e74a 100644 --- a/packages/server/src/server/test-utils/finish-subscription-daemon.ts +++ b/packages/server/src/server/test-utils/finish-subscription-daemon.ts @@ -9,18 +9,24 @@ const root = process.argv[2]; if (!root || !process.send) throw new Error("Expected isolated test home and IPC parent"); await mkdir(join(root, "static"), { recursive: true }); const held = Promise.withResolvers(); +const fixture = JSON.parse(process.argv[4] ?? "{}") as { holdResume?: boolean }; +const resumeReleased = Promise.withResolvers(); +if (!fixture.holdResume) resumeReleased.resolve(); const daemon = await createTestPaseoDaemon({ paseoHomeRoot: root, + ...JSON.parse(process.argv[3] ?? "{}"), staticDir: join(root, "static"), logger: pino({ level: "warn" }, pino.destination(join(root, "fixture.log"))), cleanup: false, mcpEnabled: true, agentClients: createTestAgentClients({ - onStartTurn: (prompt) => { + onStartTurn: (prompt, config) => { + void appendFile(join(root, "all-prompts.jsonl"), `${JSON.stringify({ prompt, config })}\n`); if (String(prompt).includes('kind="agent-notification"')) { void appendFile(join(root, "parent-prompts.jsonl"), `${JSON.stringify(prompt)}\n`); } }, + beforeResumeSession: () => resumeReleased.promise, beforeTurnComplete: (prompt) => String(prompt).includes("HOLD_CHILD") ? held.promise : Promise.resolve(), }), @@ -28,6 +34,7 @@ const daemon = await createTestPaseoDaemon({ process.send({ port: daemon.port, paseoHome: daemon.paseoHome, cwd: daemon.staticDir }); process.on("message", (message) => { if (message === "finish") held.resolve(); + if (message === "release-resume") resumeReleased.resolve(); if (message === "stop") { void daemon.close().then( () => process.exit(0), diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index a1f8369b59..766fd49919 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -52,6 +52,8 @@ interface TestPaseoDaemonOptions { trustedProxies?: PaseoDaemonConfig["trustedProxies"]; agentProfiles?: AgentProfile[]; autoArchiveAfterMerge?: boolean; + resumeOnStartup?: boolean; + resumePrompt?: string; pluginsEnabled?: PaseoDaemonConfig["pluginsEnabled"]; builtinPlugins?: BuiltinPluginLoader; plugins?: PaseoDaemonConfig["plugins"]; @@ -226,6 +228,8 @@ async function prepareTestDaemonConfig( downloadTokenTtlMs: options.downloadTokenTtlMs, agentProfiles: options.agentProfiles, autoArchiveAfterMerge: options.autoArchiveAfterMerge, + resumeOnStartup: options.resumeOnStartup, + resumePrompt: options.resumePrompt, pluginsEnabled: options.pluginsEnabled, plugins: options.plugins, }; diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 6705c1cdd9..b7024a638b 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -2013,6 +2013,7 @@ export class VoiceAssistantWebSocketServer { skillManagement: true, // COMPAT(idleRuntimeCleanupSetting): added in v0.13.0, remove gate after 2027-04-26. idleRuntimeCleanupSetting: true, + resumeOnStartupSetting: true, // COMPAT(terminalRestoreModes): added in v0.1.81, remove gate after 2026-11-23. "terminal-restore-modes": true, // COMPAT(terminalInputModeReplay): added in v0.2.6, remove gate after 2027-02-02. diff --git a/packages/server/src/server/workspace-git-service.test.ts b/packages/server/src/server/workspace-git-service.test.ts index f3571f9932..4e6acff9cd 100644 --- a/packages/server/src/server/workspace-git-service.test.ts +++ b/packages/server/src/server/workspace-git-service.test.ts @@ -1171,10 +1171,14 @@ describe("WorkspaceGitServiceImpl", () => { const subscription = service.registerWorkspace({ cwd: REPO_CWD }, listener); await vi.waitFor(() => { expect(getCheckoutSnapshotFacts).toHaveBeenCalledTimes(1); + expect(service.getMetrics().fetchInFlightCount).toBe(1); }); releaseFetch.resolve(); - await flushPromises(); + // The refresh debounce starts when the fetch result is applied, not when it is released. + await vi.waitFor(() => { + expect(service.getMetrics().fetchInFlightCount).toBe(0); + }); await vi.advanceTimersByTimeAsync(1_000); await vi.waitFor(() => { expect(getCheckoutRefDerivedState).toHaveBeenCalledTimes(1); diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index c1c32ed20a..1b74066bae 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -513,6 +513,12 @@ "idleRuntimeCleanup": { "type": "boolean" }, + "resumeOnStartup": { + "type": "boolean" + }, + "resumePrompt": { + "type": "string" + }, "catalogRefreshTimeoutMs": { "type": "integer", "exclusiveMinimum": 0,