diff --git a/docs/agent-lifecycle.md b/docs/agent-lifecycle.md index 96afdc54fa..2c2ce29403 100644 --- a/docs/agent-lifecycle.md +++ b/docs/agent-lifecycle.md @@ -12,6 +12,26 @@ initializing → idle → running → idle (or error → closed) Each live agent in `AgentManager` carries a `lastStatus` of `initializing`, `idle`, `running`, or `error`. `closed` is the persisted, resumable state for an agent record that has no live provider runtime. State transitions persist to disk and stream to subscribed clients via WebSocket. +## Daemon restart recovery + +Hosts resume interrupted agents by default after graceful shutdown or a crash. The daemon +reopens the saved provider conversation and settings, then sends `Resume working.`. Change +`agents.resumePrompt` or disable `agents.resumeOnStartup` in host config or Host settings → Agents. +Disabling recovery consumes pending intent; enabling it later does not revive old work. + +Only agents recorded as running qualify. Idle and initializing agents stay inactive. Archived +agents or workspaces, private helpers and Hub-owned executions retain their own lifecycle. +Explicit creation environment overrides are saved with the agent and reapplied when reopening. + +Startup chooses candidates before anything else reads or repairs persisted running state, then +opens connections and resumes every candidate in the background. A provider that is slow to reopen +holds back only its own agent; a failed reopen is logged. Graceful shutdown records intent before +closing runtimes. + +This is best-effort continuation. A crash between saving intent and dispatching a prompt can lose +or repeat a continuation. Provider resume owns conversation restoration; there is no recovery retry +loop or exactly-once turn guarantee. + ## Runtime residency An unarchived agent may be `closed` without being deleted or archived. Closing releases its provider @@ -139,8 +159,10 @@ caller, child, and host into one watch. Bootstrap restores these watches after repairing stale running records to `closed` and before opening ingress. Reading a closed child's result does not load its provider. A finish notification reports that -the child's run has settled; the parent decides whether its task is complete. Recovery never resumes -child work. Archived or deleted parents suppress delivery; parent-owned creation watches also stop +the child's run has settled; the parent decides whether its task is complete. A child chosen for restart +recovery is observed only after its resume was sent, so the parent never hears that it was closed +in between. Archived or deleted parents suppress delivery; +parent-owned creation watches also stop reporting after detachment. Intent remains stored until the parent's notification dispatch is acknowledged. Dispatch failure diff --git a/packages/app/e2e/browser/projects-settings.spec.ts b/packages/app/e2e/browser/projects-settings.spec.ts index 0612d5ce07..0732ec6732 100644 --- a/packages/app/e2e/browser/projects-settings.spec.ts +++ b/packages/app/e2e/browser/projects-settings.spec.ts @@ -204,19 +204,26 @@ test.describe("Projects settings", () => { }) => { const repo = await createTempGitRepo("projects-settings-empty-"); const client = await connectSeedClient(); - let projectId: string | null = null; + let projectViewKey: string | null = null; try { await gotoAppShell(page); - projectId = await addProjectFromSidebar(page, repo.path); - await openProjectSettingsFromSidebar(page, projectId); + projectViewKey = await addProjectFromSidebar(page, repo.path); + await openProjectSettingsFromSidebar(page, projectViewKey); await expectProjectSettingsFormVisible(page); await expect(page.getByTestId("project-settings-back-button")).not.toBeVisible(); } finally { - if (projectId) { - await client.removeProject(projectId).catch(() => undefined); + if (projectViewKey) { + const project = (await client.listProjects()).projects.find( + (entry) => entry.projectKey === projectViewKey, + ); + expect(project).toBeDefined(); + await client.removeProject(project!.projectId); + expect( + (await client.listProjects()).projects.map((entry) => entry.projectKey), + ).not.toContain(projectViewKey); } await client.close().catch(() => undefined); await repo.cleanup().catch(() => undefined); diff --git a/packages/app/e2e/browser/settings-host-page.spec.ts b/packages/app/e2e/browser/settings-host-page.spec.ts index 76e5b331e6..f9b358a307 100644 --- a/packages/app/e2e/browser/settings-host-page.spec.ts +++ b/packages/app/e2e/browser/settings-host-page.spec.ts @@ -30,6 +30,10 @@ import { addDirectHostFromSettings, expectHostOnlineWithoutError, expectIdleAgentRelease, + editRestartRecoveryPrompt, + expectRestartRecoveryEnabled, + expectRestartRecoveryPrompt, + setRestartRecoveryEnabled, setIdleAgentRelease, } from "../support/helpers/settings"; @@ -252,3 +256,20 @@ test.describe("Settings host page", () => { await expectHostActionCards(page, serverId); }); }); + +test("restart recovery host controls persist the opt-out and custom prompt", async ({ page }) => { + const serverId = getServerId(); + await gotoAppShell(page); + await openSettings(page); + await openSettingsHost(page, serverId); + await openHostSection(page, serverId, "agents"); + await expectRestartRecoveryEnabled(page, true); + await expectRestartRecoveryPrompt(page, "Resume working."); + await editRestartRecoveryPrompt(page, "Continue the interrupted work."); + await setRestartRecoveryEnabled(page, false); + await page.reload(); + await expectRestartRecoveryEnabled(page, false); + await setRestartRecoveryEnabled(page, true); + await expectRestartRecoveryPrompt(page, "Continue the interrupted work."); + await editRestartRecoveryPrompt(page, "Resume working."); +}); diff --git a/packages/app/e2e/browser/sidebar-reorder.spec.ts b/packages/app/e2e/browser/sidebar-reorder.spec.ts index 03ffe9f7db..56cdd3dc5a 100644 --- a/packages/app/e2e/browser/sidebar-reorder.spec.ts +++ b/packages/app/e2e/browser/sidebar-reorder.spec.ts @@ -53,11 +53,18 @@ async function quickDragFirstRowAfterSecond( await page.mouse.move(source.x, source.y + 7); await expect(trailingScrim).toHaveCount(0); await page.mouse.move(target.x, target.y, { steps: 4 }); - await page.mouse.up(); + await finishSidebarDrag(page); await expect.poll(() => rowTestIds(rows)).toEqual([before[1], before[0]]); } +async function finishSidebarDrag(page: Page): Promise { + await page.mouse.up(); + // dnd-kit's pointer sensor keeps its document click blocker for 50 ms after drop. + // Row order updates before that blocker is removed; wait before the next user action. + await page.waitForTimeout(60); +} + async function dragWithoutReordering(rows: Locator): Promise { const before = await rowTestIds(rows); const source = await visibleBoundingBox(rows.nth(0)); @@ -66,7 +73,7 @@ async function dragWithoutReordering(rows: Locator): Promise { await page.mouse.move(source.x + source.width / 2, source.y + source.height / 2); await page.mouse.down(); await page.mouse.move(target.x + target.width / 2, target.y + target.height / 2, { steps: 5 }); - await page.mouse.up(); + await finishSidebarDrag(page); await expect.poll(() => rowTestIds(rows)).toEqual(before); } diff --git a/packages/app/e2e/support/helpers/settings.ts b/packages/app/e2e/support/helpers/settings.ts index 07bd716105..a2befccfbf 100644 --- a/packages/app/e2e/support/helpers/settings.ts +++ b/packages/app/e2e/support/helpers/settings.ts @@ -560,3 +560,41 @@ export async function expectHostOnlineWithoutError(page: Page): Promise { await expect(page.getByTestId("host-page-identity")).toContainText("Online"); await expect(page.getByTestId("host-connection-error")).toHaveCount(0); } + +function restartRecoverySwitch(page: Page) { + return page.getByRole("switch", { name: "Resume agents after restart", exact: true }); +} + +function restartRecoveryEditButton(page: Page) { + return page.getByRole("button", { name: "Edit prompt", exact: true }); +} + +export async function setRestartRecoveryEnabled(page: Page, enabled: boolean) { + const toggle = restartRecoverySwitch(page); + await toggle.click(); + await expectRestartRecoveryEnabled(page, enabled); +} + +// The prompt is only editable while restart recovery is on. +export async function expectRestartRecoveryEnabled(page: Page, enabled: boolean) { + await expect(restartRecoverySwitch(page)).toBeChecked({ checked: enabled }); + await expect(restartRecoveryEditButton(page)).toBeEnabled({ enabled }); +} + +export async function editRestartRecoveryPrompt(page: Page, prompt: string) { + await restartRecoveryEditButton(page).click(); + const sheet = page.getByTestId("host-page-restart-recovery-sheet"); + await sheet.getByRole("textbox", { name: "Resume prompt", exact: true }).fill(prompt); + await sheet.getByRole("button", { name: "Save", exact: true }).click(); + await expect(sheet).toHaveCount(0); +} + +export async function expectRestartRecoveryPrompt(page: Page, prompt: string) { + await restartRecoveryEditButton(page).click(); + const sheet = page.getByTestId("host-page-restart-recovery-sheet"); + await expect(sheet.getByRole("textbox", { name: "Resume prompt", exact: true })).toHaveValue( + prompt, + ); + await sheet.getByRole("button", { name: "Cancel", exact: true }).click(); + await expect(sheet).toHaveCount(0); +} diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index 033ef38f2e..2f285ebcdb 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -2778,6 +2778,14 @@ export const ar: TranslationResources = { }, agents: { unavailable: "Connect to this host to manage agents", + resumeOnStartup: { + title: "استئناف الوكلاء بعد إعادة التشغيل", + info: "إعادة فتح الوكلاء الذين كانوا يعملون عند توقف المضيف وإرسال طلب الاستئناف. ينطبق ذلك بعد إعادة التشغيل العادية والتعطّل.", + hint: "متابعة العمل المنقطع تلقائيًا.", + prompt: "طلب الاستئناف", + edit: "تحرير", + editAccessibilityLabel: "تعديل الطلب", + }, idleRuntimeCleanup: { title: "تحرير الوكلاء الخاملين", hint: "تحرير الذاكرة بعد ساعة من الخمول.", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 2e0cd804ef..896e8ebac7 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -2870,6 +2870,14 @@ export const en = { }, agents: { unavailable: "Connect to this host to manage agents", + resumeOnStartup: { + title: "Resume agents after restart", + info: "Reopen agents that were working when the host stopped and send the resume prompt. Applies after graceful restarts and crashes.", + hint: "Continue interrupted work automatically.", + prompt: "Resume prompt", + edit: "Edit", + editAccessibilityLabel: "Edit prompt", + }, idleRuntimeCleanup: { title: "Release idle agents", hint: "Free up memory after an hour of inactivity.", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 655293e4f7..016eb99c6e 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -2800,6 +2800,14 @@ export const es: TranslationResources = { }, agents: { unavailable: "Connect to this host to manage agents", + resumeOnStartup: { + title: "Reanudar agentes tras reiniciar", + info: "Vuelve a abrir los agentes que estaban trabajando cuando se detuvo el host y envía el mensaje de reanudación. Se aplica tras reinicios normales y fallos.", + hint: "Continuar automáticamente el trabajo interrumpido.", + prompt: "Mensaje de reanudación", + edit: "Editar", + editAccessibilityLabel: "Editar mensaje", + }, idleRuntimeCleanup: { title: "Liberar agentes inactivos", hint: "Libera memoria tras una hora de inactividad.", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 927fa9299a..cbbf2d9ec8 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -2803,6 +2803,14 @@ export const fr: TranslationResources = { }, agents: { unavailable: "Connectez-vous à cet hôte pour gérer les agents", + resumeOnStartup: { + title: "Reprendre les agents après un redémarrage", + info: "Rouvre les agents qui travaillaient lorsque l’hôte s’est arrêté et envoie le message de reprise. S’applique après un redémarrage normal ou un plantage.", + hint: "Continuer automatiquement le travail interrompu.", + prompt: "Message de reprise", + edit: "Modifier", + editAccessibilityLabel: "Modifier le message", + }, idleRuntimeCleanup: { title: "Libérer les agents inactifs", hint: "Libère de la mémoire après une heure d’inactivité.", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index b9582cefc9..05a7aa457e 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -2767,6 +2767,14 @@ export const ja: TranslationResources = { }, agents: { unavailable: "エージェントを管理するにはこのホストに接続してください", + resumeOnStartup: { + title: "再起動後にエージェントを再開", + info: "ホストの停止時に作業中だったエージェントを開き直し、再開メッセージを送信します。通常の再起動とクラッシュの両方に適用されます。", + hint: "中断された作業を自動的に続行します。", + prompt: "再開メッセージ", + edit: "編集", + editAccessibilityLabel: "メッセージを編集", + }, idleRuntimeCleanup: { title: "アイドル状態のエージェントを解放", hint: "1 時間のアイドル状態の後にメモリを解放します。", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index 1f3ed0b3ef..d8a7491e92 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -2755,6 +2755,14 @@ export const ko: TranslationResources = { }, agents: { unavailable: "에이전트를 관리하려면 이 호스트에 연결하세요", + resumeOnStartup: { + title: "재시작 후 에이전트 재개", + info: "호스트가 중지될 때 작업 중이던 에이전트를 다시 열고 재개 메시지를 보냅니다. 정상 재시작과 비정상 종료 후 모두 적용됩니다.", + hint: "중단된 작업을 자동으로 계속합니다.", + prompt: "재개 메시지", + edit: "편집", + editAccessibilityLabel: "메시지 편집", + }, idleRuntimeCleanup: { title: "유휴 에이전트 해제", hint: "1시간 동안 활동이 없으면 메모리를 확보합니다.", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 139ac00a2b..b2abd39b25 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -2782,6 +2782,14 @@ export const ptBR: TranslationResources = { }, agents: { unavailable: "Conecte-se a este host para gerenciar agentes", + resumeOnStartup: { + title: "Retomar agentes após reiniciar", + info: "Reabre os agentes que estavam trabalhando quando o host parou e envia a mensagem de retomada. Aplica-se após reinicializações normais e falhas.", + hint: "Continuar automaticamente o trabalho interrompido.", + prompt: "Mensagem de retomada", + edit: "Editar", + editAccessibilityLabel: "Editar mensagem", + }, idleRuntimeCleanup: { title: "Liberar agentes ociosos", hint: "Libera memória após uma hora de inatividade.", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index b4d0eb817e..cdacf42173 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -2792,6 +2792,14 @@ export const ru: TranslationResources = { }, agents: { unavailable: "Подключитесь к этому хосту, чтобы управлять агентами", + resumeOnStartup: { + title: "Возобновлять агентов после перезапуска", + info: "Открывает агентов, которые работали при остановке хоста, и отправляет сообщение для продолжения. Применяется после обычного перезапуска и сбоя.", + hint: "Автоматически продолжать прерванную работу.", + prompt: "Сообщение для продолжения", + edit: "Редактировать", + editAccessibilityLabel: "Изменить сообщение", + }, idleRuntimeCleanup: { title: "Освобождать неактивных агентов", hint: "Освобождает память после часа бездействия.", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index a231456352..f8131c7dca 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -2713,6 +2713,14 @@ export const zhCN: TranslationResources = { }, agents: { unavailable: "连接到这个 Host 以管理 Agent", + resumeOnStartup: { + title: "重启后恢复智能体", + info: "重新打开主机停止时正在工作的智能体,并发送恢复提示。适用于正常重启和崩溃后的恢复。", + hint: "自动继续被中断的工作。", + prompt: "恢复提示", + edit: "编辑", + editAccessibilityLabel: "编辑提示", + }, idleRuntimeCleanup: { title: "释放空闲 Agent", hint: "空闲一小时后释放内存。", diff --git a/packages/app/src/screens/settings/host-page.tsx b/packages/app/src/screens/settings/host-page.tsx index cf26f68e6f..879ef2fe95 100644 --- a/packages/app/src/screens/settings/host-page.tsx +++ b/packages/app/src/screens/settings/host-page.tsx @@ -65,6 +65,7 @@ import { ICON_SIZE } from "@/styles/theme"; import type { Theme } from "@/styles/theme"; import { useProviderIcon } from "@/components/provider-icons"; import { BrowserToolsOptInCard } from "./browser-tools-card"; +import { RestartRecoveryCard } from "./restart-recovery-card"; import { IdleRuntimeCleanupCard } from "./idle-runtime-cleanup-card"; import { restartDaemonFromSettings, updateDaemonFromSettings } from "./daemon-lifecycle"; @@ -292,6 +293,7 @@ export function HostAgentsPage({ serverId }: { serverId: string }) { + ) : ( diff --git a/packages/app/src/screens/settings/restart-recovery-card.tsx b/packages/app/src/screens/settings/restart-recovery-card.tsx new file mode 100644 index 0000000000..9fa8a872b0 --- /dev/null +++ b/packages/app/src/screens/settings/restart-recovery-card.tsx @@ -0,0 +1,199 @@ +import { useCallback, useMemo, useState } from "react"; +import { Text, View } from "react-native"; +import { useMutation } from "@tanstack/react-query"; +import { useTranslation } from "react-i18next"; +import { StyleSheet } from "react-native-unistyles"; +import { AdaptiveModalSheet, type SheetHeader } from "@/components/adaptive-modal-sheet"; +import { SettingsTextAreaCard } from "@/components/settings-textarea"; +import { SettingsInfoTip } from "@/components/settings/headings/settings-info-tip"; +import { Button } from "@/components/ui/button"; +import { Switch } from "@/components/ui/switch"; +import { useDaemonConfig } from "@/hooks/use-daemon-config"; +import { useHostFeature } from "@/runtime/host-features"; +import { useHostRuntimeIsConnected } from "@/runtime/host-runtime"; +import { settingsStyles } from "@/styles/settings"; + +const DEFAULT_RESUME_PROMPT = "Resume working."; + +// COMPAT(restartRecovery): added in v0.14.0, remove gate after 2027-04-28. +export function RestartRecoveryCard({ serverId }: { serverId: string }) { + const { t } = useTranslation(); + const isConnected = useHostRuntimeIsConnected(serverId); + const isSupported = useHostFeature(serverId, "resumeOnStartupSetting"); + const { config, patchConfig } = useDaemonConfig(serverId); + const [isEditing, setIsEditing] = useState(false); + const mutation = useMutation({ + mutationFn: async (next: { resumeOnStartup?: boolean; resumePrompt?: string }) => { + const result = await patchConfig(next); + if (!result) { + throw new Error(t("workspace.terminal.hostDisconnected")); + } + return result; + }, + }); + + const handleValueChange = useCallback( + (next: boolean) => { + mutation.mutate({ resumeOnStartup: next }); + }, + [mutation], + ); + + const handleOpen = useCallback(() => { + mutation.reset(); + setIsEditing(true); + }, [mutation]); + + const handleClose = useCallback(() => { + if (mutation.isPending) return; + mutation.reset(); + setIsEditing(false); + }, [mutation]); + + const handleSave = useCallback( + (prompt: string) => { + mutation.mutate({ resumePrompt: prompt }, { onSuccess: () => setIsEditing(false) }); + }, + [mutation], + ); + + if (!isConnected || !isSupported || !config) return null; + + const isEnabled = config.resumeOnStartup !== false; + + return ( + <> + + + + + + {t("settings.host.agents.resumeOnStartup.title")} + + + + + {t("settings.host.agents.resumeOnStartup.hint")} + + {mutation.error && !isEditing ? ( + + {mutation.error.message} + + ) : null} + + + + + + + + + {isEditing ? ( + + ) : null} + + ); +} + +function ResumePromptSheet({ + initialPrompt, + isSaving, + error, + onSave, + onClose, +}: { + initialPrompt: string; + isSaving: boolean; + error: string | null; + onSave: (prompt: string) => void; + onClose: () => void; +}) { + const { t } = useTranslation(); + const [draft, setDraft] = useState(initialPrompt); + const header = useMemo( + () => ({ title: t("settings.host.agents.resumeOnStartup.prompt") }), + [t], + ); + const canSave = draft !== initialPrompt && draft.trim().length > 0 && !isSaving; + + const handleSave = useCallback(() => onSave(draft), [draft, onSave]); + + return ( + + + {error ? ( + + {error} + + ) : null} + + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + titleRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + trailing: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[3], + }, + sheetActions: { + flexDirection: "row", + justifyContent: "flex-end", + gap: theme.spacing[2], + }, +})); diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 26a90016a7..d20c91d773 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -237,6 +237,9 @@ export const MutableDaemonConfigSchema = z // COMPAT(idleRuntimeCleanupSetting): added in v0.13.0, keep optional while old daemons are supported. // Persisted host-wide value, applied live by the daemon. idleRuntimeCleanup: z.boolean().optional(), + // COMPAT(restartRecovery): added in v0.14.0, keep optional for older hosts. + resumeOnStartup: z.boolean().optional(), + resumePrompt: z.string().optional(), appendSystemPrompt: z.string().default(""), terminalProfiles: z.array(TerminalProfileSchema).optional(), agentProfiles: z.array(AgentProfileSchema).optional(), @@ -262,6 +265,8 @@ export const MutableDaemonConfigPatchSchema = z autoArchiveAfterMerge: z.boolean().optional(), enableTerminalAgentHooks: z.boolean().optional(), idleRuntimeCleanup: z.boolean().optional(), + resumeOnStartup: z.boolean().optional(), + resumePrompt: z.string().optional(), appendSystemPrompt: z.string().optional(), terminalProfiles: z.array(TerminalProfileSchema).optional(), agentProfiles: z.array(AgentProfileSchema).optional(), @@ -3731,6 +3736,8 @@ export const ServerInfoStatusPayloadSchema = z skillManagement: z.boolean().optional(), // COMPAT(idleRuntimeCleanupSetting): added in v0.13.0, remove gate after 2027-04-26. idleRuntimeCleanupSetting: z.boolean().optional(), + // COMPAT(restartRecovery): added in v0.14.0, remove gate after 2027-04-28. + resumeOnStartupSetting: z.boolean().optional(), // COMPAT(terminalRestoreModes): added in v0.1.81, remove gate after 2026-11-23. "terminal-restore-modes": z.boolean().optional(), // COMPAT(terminalInputModeReplay): added in v0.2.6, remove gate after 2027-02-02. diff --git a/packages/server/src/server/agent/agent-loading.ts b/packages/server/src/server/agent/agent-loading.ts index 761f1008bd..ba444a394d 100644 --- a/packages/server/src/server/agent/agent-loading.ts +++ b/packages/server/src/server/agent/agent-loading.ts @@ -135,6 +135,7 @@ export async function ensureAgentLoaded( labels: record.labels, workspaceId: record.workspaceId, owner: record.owner, + env: record.launchEnv, }); deps.logger.info({ agentId, provider: record.provider }, "Agent created from stored config"); } diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index eb1c05b50f..1d6f323deb 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -2929,6 +2929,92 @@ test("createAgent passes daemon launch env through the provider launch context", }); }); +test.each([ + { provider: "codex", hasHandle: true }, + { provider: "opencode", hasHandle: true }, + { provider: "codex", hasHandle: false }, + { provider: "opencode", hasHandle: false }, +])( + "$provider preserves explicit launch env across reloads and storage-backed resumes (handle: $hasHandle)", + async ({ provider, hasHandle }) => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-launch-env-")); + const storagePath = join(workdir, "agents"); + class LaunchEnvClient extends TestAgentClient { + lastLaunchContext: AgentLaunchContext | undefined; + + override async createSession( + config: AgentSessionConfig, + context?: AgentLaunchContext, + ): Promise { + this.lastLaunchContext = context; + const session = await super.createSession(config); + session.describePersistence = () => ({ provider, sessionId: session.id }); + return session; + } + + override async resumeSession( + handle: AgentPersistenceHandle, + config?: Partial, + context?: AgentLaunchContext, + ): Promise { + this.lastLaunchContext = context; + const session = await super.resumeSession(handle, config, context); + session.describePersistence = () => ({ provider, sessionId: handle.sessionId }); + return session; + } + } + + const client = new LaunchEnvClient(provider); + let storage = new AgentStorage(storagePath, logger); + let manager = new AgentManager({ clients: { [provider]: client }, registry: storage, logger }); + const env = { XDG_DATA_HOME: join(workdir, "agent-data"), CUSTOM_EMPTY: "" }; + const expectedEnv = { ...env }; + const agent = await manager.createAgent({ provider, cwd: workdir }, undefined, { + workspaceId: undefined, + env, + }); + onTestFinished(async () => { + await manager.closeAgent(agent.id); + await manager.flush(); + rmSync(workdir, { recursive: true, force: true }); + }); + env.XDG_DATA_HOME = "caller-mutated"; + + await manager.closeAgent(agent.id); + await manager.flush(); + + if (!hasHandle) { + const record = await storage.get(agent.id); + if (!record) throw new Error("Created agent was not persisted"); + await storage.upsert({ ...record, persistence: null }); + } + + // Fresh storage and manager instances must read the environment from disk. + for (let restart = 0; restart < 2; restart += 1) { + storage = new AgentStorage(storagePath, logger); + manager = new AgentManager({ clients: { [provider]: client }, registry: storage, logger }); + const resumed = await ensureAgentLoaded(agent.id, { + agentManager: manager, + agentStorage: storage, + logger, + }); + expect(client.lastLaunchContext).toEqual({ + agentId: agent.id, + env: { ...expectedEnv, PASEO_AGENT_ID: agent.id, PASEO_AGENT_CWD: workdir }, + }); + await manager.reloadAgentSession(agent.id); + expect(client.lastLaunchContext).toEqual({ + agentId: agent.id, + env: { ...expectedEnv, PASEO_AGENT_ID: agent.id, PASEO_AGENT_CWD: workdir }, + }); + expect(toAgentPayload(resumed)).not.toHaveProperty("launchEnv"); + expect(toAgentPayload(resumed).persistence?.metadata).not.toHaveProperty("launchEnv"); + await manager.closeAgent(agent.id); + await manager.flush(); + } + }, +); + test("createAgent passes persistSession to provider create options", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 966542ff62..46c822ea2d 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -421,6 +421,8 @@ interface ManagedAgentBase { owner?: AgentOwner; capabilities: AgentCapabilityFlags; config: AgentSessionConfig; + /** Explicit launch overrides, before per-opening hooks and daemon-owned variables. */ + launchEnv?: Record; runtimeInfo?: AgentRuntimeInfo; createdAt: Date; updatedAt: Date; @@ -1399,6 +1401,7 @@ export class AgentManager { initialTitle: options.initialTitle, workspaceId: options.workspaceId, owner: options.owner, + launchEnv: options.env ? { ...options.env } : undefined, historyPrimed: true, }); if (!agent.internal) { @@ -1487,10 +1490,11 @@ export class AgentManager { : resumeOptions; const purpose = currentResumeOptions?.purpose ?? "interactive"; + const launchEnv = record?.launchEnv; const { storedConfig, launchConfig, paseoToolPolicy } = await this.prepareSessionConfig( mergedConfig, resolvedAgentId, - { purpose }, + { purpose, env: launchEnv }, ); const client = this.requireClient(handle.provider); const available = await client.isAvailable(); @@ -1505,7 +1509,7 @@ export class AgentManager { client, storedConfig.cwd, paseoToolPolicy, - undefined, + launchEnv, { reason: "resume", purpose, @@ -1524,6 +1528,7 @@ export class AgentManager { await this.requireExternalMcpSupport(session, providerLaunchConfig, launchContext); return this.registerSession(session, storedConfig, resolvedAgentId, { ...options, + launchEnv, persistence: handle, restoring: true, }); @@ -1661,6 +1666,7 @@ export class AgentManager { const { storedConfig, launchConfig, paseoToolPolicy } = await this.prepareSessionConfig( refreshConfig, agentId, + { env: existing.launchEnv }, ); const hadPreviousPaseoToolPolicy = this.paseoToolPolicies.has(agentId); const previousPaseoToolPolicy = this.paseoToolPolicies.get(agentId); @@ -1669,7 +1675,7 @@ export class AgentManager { client, storedConfig.cwd, paseoToolPolicy, - undefined, + existing.launchEnv, { reason: "refresh", purpose: "interactive", @@ -1720,6 +1726,7 @@ export class AgentManager { labels: existing.labels, workspaceId: existing.workspaceId, owner: existing.owner, + launchEnv: existing.launchEnv, createdAt: existing.createdAt, updatedAt: existing.updatedAt, lastUserMessageAt: existing.lastUserMessageAt, @@ -3763,6 +3770,7 @@ export class AgentManager { publishWhenReady?: boolean; workspaceId?: string; owner?: AgentOwner; + launchEnv?: Record; }, ): Promise { let registered = false; @@ -3948,6 +3956,7 @@ export class AgentManager { persistence?: AgentPersistenceHandle; workspaceId?: string; owner?: AgentOwner; + launchEnv?: Record; } | undefined; }): ActiveManagedAgent { @@ -3958,6 +3967,7 @@ export class AgentManager { cwd: config.cwd, workspaceId: options?.workspaceId, owner: options?.owner, + launchEnv: options?.launchEnv, session, capabilities: session.capabilities, config, diff --git a/packages/server/src/server/agent/agent-projections.ts b/packages/server/src/server/agent/agent-projections.ts index 7d154e9c14..b388143d8e 100644 --- a/packages/server/src/server/agent/agent-projections.ts +++ b/packages/server/src/server/agent/agent-projections.ts @@ -95,6 +95,7 @@ export function toStoredAgentRecord( lastStatus: agent.lifecycle, lastModeId: agent.currentModeId ?? config?.modeId ?? null, config: config ?? null, + launchEnv: agent.launchEnv ? { ...agent.launchEnv } : undefined, runtimeInfo, features: normalizeFeatures(agent.features), persistence, diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index be94a31d49..a097bd7f82 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -54,8 +54,10 @@ const STORED_AGENT_SCHEMA = z.object({ title: z.string().nullable().optional(), labels: z.record(z.string(), z.string()).default({}), lastStatus: AgentStatusSchema.default("closed"), + resumeOnStartup: z.boolean().optional(), lastModeId: z.string().nullable().optional(), config: SERIALIZABLE_CONFIG_SCHEMA, + launchEnv: z.record(z.string(), z.string()).optional(), runtimeInfo: z .object({ provider: z.string(), @@ -259,6 +261,8 @@ export class AgentStorage { if (existing && existing.archivedAt !== undefined) { record.archivedAt = existing.archivedAt; } + record.resumeOnStartup = + existing?.resumeOnStartup && agent.lifecycle !== "idle" && agent.lifecycle !== "error"; return record; }); } diff --git a/packages/server/src/server/agent/finish-subscriptions/internal/observation.ts b/packages/server/src/server/agent/finish-subscriptions/internal/observation.ts index 887a4ba075..7abe483cf4 100644 --- a/packages/server/src/server/agent/finish-subscriptions/internal/observation.ts +++ b/packages/server/src/server/agent/finish-subscriptions/internal/observation.ts @@ -13,6 +13,8 @@ import type { FinishObservation } from "../index.js"; export function createFinishObservation(input: { agentManager: AgentManager; agentStorage: AgentStorage; + /** A child chosen for restart recovery reports its state only after its resume was sent. */ + waitForResume(agentId: string, signal: AbortSignal): Promise; messages(): AgentMessageDelivery; hosts(): ToolHosts; serverId: string; @@ -24,6 +26,7 @@ export function createFinishObservation(input: { hosts: input.hosts, local: { async wait(agentId, signal) { + await input.waitForResume(agentId, signal); await input.messages().waitForDispatch(agentId, signal); signal.throwIfAborted(); const live = agentManager.getAgent(agentId); @@ -52,6 +55,7 @@ export function createFinishObservation(input: { }; }, async observe(agentId, handler, signal, onError) { + await input.waitForResume(agentId, signal); const stored = await agentStorage.get(agentId); let updates = Promise.resolve(); const publish = (snapshot: AgentSnapshotPayload | null, event?: AgentStreamEvent) => { diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 3b72c8805a..86ee383f73 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -881,6 +881,7 @@ async function createAgentMcpServer( logger: options.logger, ...createFinishObservation({ ...options, + waitForResume: async () => {}, messages: () => messages, serverId: options.serverId ?? "self", hosts: () => { diff --git a/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts b/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts index aeb87388fd..749374b29f 100644 --- a/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts +++ b/packages/server/src/server/agent/providers/opencode-bridge.local.e2e.test.ts @@ -461,9 +461,19 @@ test.each([ process.platform === "win32" ? `${windowsPackage}-${process.arch}@${cliPackage.slice(cliPackage.lastIndexOf("@") + 1)}` : cliPackage; + // npm abandons a stalled registry download only after its 5 minute default, past this + // test's budget, and the killed install keeps the fixture locked. Retry stalls sooner. await execCommand( "npm", - ["install", "--prefix", root, "--no-audit", "--no-fund", installPackage], + [ + "install", + "--prefix", + root, + "--no-audit", + "--no-fund", + "--fetch-timeout=30000", + installPackage, + ], { cwd: root, timeout: 180_000, diff --git a/packages/server/src/server/agent/restart-recovery/index.test.ts b/packages/server/src/server/agent/restart-recovery/index.test.ts new file mode 100644 index 0000000000..0b1658b47d --- /dev/null +++ b/packages/server/src/server/agent/restart-recovery/index.test.ts @@ -0,0 +1,105 @@ +import { expect, test } from "vitest"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AgentStorage, parseStoredAgentRecord } from "../agent-storage.js"; +import { createTestLogger } from "../../../test-utils/test-logger.js"; +import { createRestartRecovery } from "./index.js"; + +test("recovery saves intent before normalization, clears before dispatch, and resumes each agent independently", async () => { + const root = await mkdtemp(join(tmpdir(), "restart-recovery-")); + try { + const storage = new AgentStorage(root, createTestLogger()); + for (const [id, lastStatus] of [ + ["broken", "running"], + ["stuck", "running"], + ["working", "running"], + ["idle", "idle"], + ["initializing", "initializing"], + ] as const) { + await storage.upsert( + parseStoredAgentRecord({ + id, + lastStatus, + provider: "codex", + cwd: root, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }), + ); + } + const dispatched: string[] = []; + const recovery = createRestartRecovery({ + storage, + settings: () => ({ enabled: true, prompt: "Continue verbatim.\n" }), + eligible: async () => true, + dispatch: async (id, prompt) => { + expect((await storage.get(id))?.resumeOnStartup).toBe(false); + if (id === "broken") throw new Error("unavailable"); + if (id === "stuck") return new Promise(() => {}); + dispatched.push(id + prompt); + }, + report: () => {}, + }); + await recovery.prepareStartup(); + expect((await storage.get("working"))?.resumeOnStartup).toBe(true); + expect((await storage.get("working"))?.lastStatus).toBe("closed"); + recovery.resumePending(); + const running = new AbortController().signal; + await recovery.waitForResume("broken", running); + await recovery.waitForResume("working", running); + await recovery.waitForResume("idle", running); + expect(dispatched).toEqual(["workingContinue verbatim.\n"]); + const stopped = new AbortController(); + stopped.abort(); + await expect(recovery.waitForResume("stuck", stopped.signal)).rejects.toBeDefined(); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("opt-out consumes pending recovery and excludes archived, private, owned and archived-workspace agents", async () => { + const root = await mkdtemp(join(tmpdir(), "restart-recovery-")); + try { + const storage = new AgentStorage(root, createTestLogger()); + for (const extra of [ + { id: "running" }, + { id: "archived", archivedAt: "today" }, + { id: "private", internal: true }, + { id: "owned", owner: { kind: "daemon", daemonId: "hub", executionId: "run" } }, + { id: "workspace", workspaceId: "archived" }, + ]) { + await storage.upsert( + parseStoredAgentRecord({ + lastStatus: "running", + provider: "codex", + cwd: root, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + ...extra, + }), + ); + } + let enabled = true; + const dispatched: string[] = []; + const recovery = createRestartRecovery({ + storage, + settings: () => ({ enabled, prompt: "Resume working." }), + eligible: async (record) => record.workspaceId !== "archived", + dispatch: async (id) => { + dispatched.push(id); + }, + report: () => {}, + }); + await recovery.captureRunningAgents(); + expect((await storage.get("running"))?.resumeOnStartup).toBe(true); + enabled = false; + await recovery.prepareStartup(); + enabled = true; + recovery.resumePending(); + expect(dispatched).toEqual([]); + expect((await storage.list()).every((record) => !record.resumeOnStartup)).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/server/src/server/agent/restart-recovery/index.ts b/packages/server/src/server/agent/restart-recovery/index.ts new file mode 100644 index 0000000000..dc4e7d90c7 --- /dev/null +++ b/packages/server/src/server/agent/restart-recovery/index.ts @@ -0,0 +1,81 @@ +import type { AgentStorage, StoredAgentRecord } from "../agent-storage.js"; +interface Dependencies { + storage: AgentStorage; + settings(): { enabled: boolean; prompt: string }; + eligible(record: StoredAgentRecord): Promise; + dispatch(agentId: string, prompt: string): Promise; + report(agentId: string, error: unknown): void; +} +export function createRestartRecovery(deps: Dependencies) { + // Candidates are chosen at startup; each settles once its resume was sent or abandoned. + const resuming = new Map; resolve(): void }>(); + const eligible = async (record: StoredAgentRecord) => + !record.archivedAt && !record.internal && !record.owner && (await deps.eligible(record)); + async function resume(agentId: string) { + const record = await deps.storage.get(agentId); + if (!record?.resumeOnStartup) return; + await deps.storage.upsert({ ...record, resumeOnStartup: false }); + if (!deps.settings().enabled || !(await eligible(record))) return; + await deps.dispatch(agentId, deps.settings().prompt); + } + return { + /** Choose candidates before anything else reads or repairs persisted running status. */ + async prepareStartup() { + for (const record of await deps.storage.list()) { + const pending = + deps.settings().enabled && + (record.resumeOnStartup === true || record.lastStatus === "running") && + (await eligible(record)); + await deps.storage.upsert({ + ...record, + resumeOnStartup: pending, + lastStatus: + record.lastStatus === "running" || record.lastStatus === "initializing" + ? "closed" + : record.lastStatus, + }); + if (pending) { + let resolve!: () => void; + const promise = new Promise((settle) => { + resolve = settle; + }); + resuming.set(record.id, { promise, resolve }); + } + } + }, + async captureRunningAgents() { + await deps.storage.flush(); + for (const record of await deps.storage.list()) { + if (record.lastStatus === "running" && (await eligible(record))) + await deps.storage.upsert({ ...record, resumeOnStartup: deps.settings().enabled }); + } + }, + /** Resume every candidate in the background; a slow provider holds back only its agent. */ + resumePending() { + for (const [agentId, settled] of resuming) { + void resume(agentId) + .catch((error) => deps.report(agentId, error)) + .finally(() => { + resuming.delete(agentId); + settled.resolve(); + }); + } + }, + /** Wait until this agent's resume was sent or abandoned. Others return at once. */ + async waitForResume(agentId: string, signal: AbortSignal) { + const settled = resuming.get(agentId); + if (!settled) return; + let abort!: () => void; + const aborted = new Promise((_resolve, reject) => { + abort = () => reject(signal.reason); + if (signal.aborted) abort(); + else signal.addEventListener("abort", abort, { once: true }); + }); + try { + await Promise.race([settled.promise, aborted]); + } finally { + signal.removeEventListener("abort", abort); + } + }, + }; +} diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index f6304491ab..76fa8cc7d1 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -1069,7 +1069,7 @@ function probeWebSocketConnection( }); } -test("startup closes stale active records before serving status without resuming sessions", async () => { +test("disabled recovery closes stale active records without resuming sessions", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "paseo-stale-status-")); const logger = pino({ level: "silent" }); const storagePath = path.join(root, ".paseo", "agents"); @@ -1121,6 +1121,7 @@ test("startup closes stale active records before serving status without resuming let sessionsCreated = 0; const handle = await createTestPaseoDaemon({ paseoHomeRoot: root, + resumeOnStartup: false, agentClients: createTestAgentClients({ beforeCreateSession: async () => { sessionsCreated++; @@ -1131,6 +1132,7 @@ test("startup closes stale active records before serving status without resuming try { const expected = records.map((record) => ({ ...record, + resumeOnStartup: false, lastStatus: record.lastStatus === "running" || record.lastStatus === "initializing" ? "closed" diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 36ba9e2b06..8fe0f05b14 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -1,3 +1,5 @@ +import { createRestartRecovery } from "./agent/restart-recovery/index.js"; +import { sendPromptToAgent } from "./agent/agent-prompt.js"; import { createIdleRuntimeCleanup } from "./agent/idle-runtime-cleanup/index.js"; import { createFinishSubscriptions, @@ -464,6 +466,8 @@ export interface PaseoDaemonConfig { downloadTokenTtlMs?: number; agentProviderSettings?: AgentProviderRuntimeSettingsMap; idleRuntimeCleanup?: boolean; + resumeOnStartup?: boolean; + resumePrompt?: string; providerCatalogRefreshTimeoutMs?: number; metadataGeneration?: { providers?: Array<{ @@ -594,6 +598,8 @@ function createInitialMutableDaemonConfig(config: ResolvedDaemonSettings): Mutab autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false, enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false, idleRuntimeCleanup: config.idleRuntimeCleanup !== false, + resumeOnStartup: config.resumeOnStartup !== false, + resumePrompt: config.resumePrompt ?? "Resume working.", appendSystemPrompt: config.appendSystemPrompt ?? "", pluginsEnabled: config.pluginsEnabled ?? false, plugins: config.plugins ?? {}, @@ -1036,7 +1042,19 @@ export async function createPaseoDaemon( agentStorage, ); await agentStorage.initialize(); - await closeStaleAgentRecords(agentStorage); + const restartRecovery = createRestartRecovery({ + storage: agentStorage, + settings: () => ({ + enabled: daemonConfigStore.get().resumeOnStartup !== false, + prompt: daemonConfigStore.get().resumePrompt ?? "Resume working.", + }), + eligible: async (record) => + !record.workspaceId || !(await workspaceRegistry?.get(record.workspaceId))?.archivedAt, + dispatch: (agentId, prompt) => + sendPromptToAgent({ agentId, prompt, agentManager, agentStorage, logger, unarchive: false }), + report: (agentId, err) => logger.error({ agentId, err }, "Agent restart recovery failed"), + }); + await restartRecovery.prepareStartup(); logger.info({ elapsed: elapsed() }, "Agent storage initialized"); await bootstrapWorkspaceRegistries({ serverId, @@ -1470,6 +1488,7 @@ export async function createPaseoDaemon( ...createFinishObservation({ agentManager, agentStorage, + waitForResume: restartRecovery.waitForResume, serverId, logger, messages: () => { @@ -1869,6 +1888,7 @@ export async function createPaseoDaemon( pluginRuntime.bindPaseoSessionHost(wsServer); await pluginRuntime.start(); providerSnapshotManager.settlePluginProviders(); + restartRecovery.resumePending(); await finishSubscriptions.start(); wsServer.beginAcceptingConnections(); const relaySecurityLogger = logger.child({ module: "relay-security" }); @@ -1963,6 +1983,7 @@ export async function createPaseoDaemon( await finishSubscriptions.close(); await wsServer?.agentMessages.close(); agentManager.prepareForShutdown(); + await restartRecovery.captureRunningAgents(); await closeAllAgents(logger, agentManager); await withTimeout({ promise: pluginRuntime.drainEvents(), @@ -2029,16 +2050,6 @@ export async function createPaseoDaemon( }; } -async function closeStaleAgentRecords(agentStorage: AgentStorage): Promise { - // No provider runtimes are loaded at startup. Repair interrupted - // shutdowns before any status consumer sees the persisted active states. - for (const record of await agentStorage.list()) { - if (record.lastStatus === "running" || record.lastStatus === "initializing") { - await agentStorage.upsert({ ...record, lastStatus: "closed" }); - } - } -} - /** * Closing an agent asks its provider to close the session and waits for the * answer. A provider that never answers must not hold the daemon open, so a diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index add78aa7f5..3149fc6eb9 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -590,6 +590,8 @@ export type ResolvedDaemonSettings = Pick< | "relayRequireProtectedSecurity" | "appBaseUrl" | "idleRuntimeCleanup" + | "resumeOnStartup" + | "resumePrompt" | "providerCatalogRefreshTimeoutMs" | "metadataGeneration" | "providerOverrides" @@ -677,6 +679,7 @@ export function resolveReloadableConfig( relayEnabled: relay.enabled, relayRequireProtectedSecurity: relay.requireProtectedSecurity, idleRuntimeCleanup: persisted.agents?.idleRuntimeCleanup ?? true, + ...resolveRestartRecoveryConfig(persisted.agents), providerCatalogRefreshTimeoutMs: persisted.agents?.catalogRefreshTimeoutMs, metadataGeneration: persisted.agents?.metadataGeneration, providerOverrides: extractProviderOverrides( @@ -827,3 +830,10 @@ function resolveSpeechOverrideControlledPaths( if (env.PASEO_LOCAL_MODELS_DIR !== undefined) paths.push("providers.local.modelsDir"); return paths; } + +function resolveRestartRecoveryConfig(agents: PersistedConfig["agents"]) { + return { + resumeOnStartup: agents?.resumeOnStartup ?? true, + resumePrompt: agents?.resumePrompt ?? "Resume working.", + }; +} diff --git a/packages/server/src/server/daemon-config-store.test.ts b/packages/server/src/server/daemon-config-store.test.ts index e3b6f48895..89c3c7f977 100644 --- a/packages/server/src/server/daemon-config-store.test.ts +++ b/packages/server/src/server/daemon-config-store.test.ts @@ -940,6 +940,15 @@ describe("DaemonConfigStore", () => { await store.patch({ idleRuntimeCleanup: true }); expect(loadConfig(paseoHome, { env: {} }).idleRuntimeCleanup).toBe(true); + await store.patch({ resumeOnStartup: false, resumePrompt: "Keep going.\n" }); + expect(loadPersistedConfig(paseoHome).agents).toMatchObject({ + resumeOnStartup: false, + resumePrompt: "Keep going.\n", + }); + expect(loadConfig(paseoHome, { env: {} })).toMatchObject({ + resumeOnStartup: false, + resumePrompt: "Keep going.\n", + }); }); test("patch persists custom ACP provider overrides into config.json", async () => { diff --git a/packages/server/src/server/daemon-config-store.ts b/packages/server/src/server/daemon-config-store.ts index 57b9a6b916..1bd67822a3 100644 --- a/packages/server/src/server/daemon-config-store.ts +++ b/packages/server/src/server/daemon-config-store.ts @@ -29,6 +29,8 @@ interface SupportedMutableConfigPatch { autoArchiveAfterMerge?: boolean; enableTerminalAgentHooks?: boolean; idleRuntimeCleanup?: boolean; + resumeOnStartup?: boolean; + resumePrompt?: string; appendSystemPrompt?: string; terminalProfiles?: MutableDaemonConfig["terminalProfiles"]; agentProfiles?: MutableDaemonConfig["agentProfiles"]; @@ -193,6 +195,8 @@ const RELOADABLE_PATHS = [ "daemon.agentProfiles", "app.baseUrl", "agents.idleRuntimeCleanup", + "agents.resumeOnStartup", + "agents.resumePrompt", "agents.providers", "agents.catalogRefreshTimeoutMs", "agents.metadataGeneration", @@ -218,6 +222,8 @@ const PERSISTED_TO_MUTABLE_PATH = new Map([ ["daemon.agentProfiles", "agentProfiles"], ["app.baseUrl", "app.baseUrl"], ["agents.idleRuntimeCleanup", "idleRuntimeCleanup"], + ["agents.resumeOnStartup", "resumeOnStartup"], + ["agents.resumePrompt", "resumePrompt"], ["agents.providers", "providers"], ["agents.catalogRefreshTimeoutMs", "catalogRefreshTimeoutMs"], ["agents.metadataGeneration", "metadataGeneration"], @@ -292,6 +298,8 @@ function pickSupportedPatchFields(patch: MutableDaemonConfigPatch): SupportedMut ...(patch.enableTerminalAgentHooks !== undefined ? { enableTerminalAgentHooks: patch.enableTerminalAgentHooks } : {}), + ...(patch.resumeOnStartup !== undefined ? { resumeOnStartup: patch.resumeOnStartup } : {}), + ...(patch.resumePrompt !== undefined ? { resumePrompt: patch.resumePrompt } : {}), ...(patch.idleRuntimeCleanup !== undefined ? { idleRuntimeCleanup: patch.idleRuntimeCleanup } : {}), @@ -702,6 +710,7 @@ function mergeMutableAgentPatch( patch.providers === undefined && patch.metadataGeneration === undefined && patch.skills === undefined && + !hasRestartRecoveryPatch(patch) && patch.idleRuntimeCleanup === undefined && removeProviders.length === 0 ) { @@ -735,6 +744,7 @@ function mergeMutableAgentPatch( next["skills"] = { selection: patch.skills.selection }; } + Object.assign(next, restartRecoveryPatch(patch)); if (patch.idleRuntimeCleanup !== undefined) { next["idleRuntimeCleanup"] = patch.idleRuntimeCleanup; } @@ -771,3 +781,13 @@ function mergeMutableDaemonPatch( if (patch.agentProfiles !== undefined) next.agentProfiles = patch.agentProfiles; return Object.keys(next).length > 0 ? next : undefined; } + +function hasRestartRecoveryPatch(patch: SupportedMutableConfigPatch) { + return patch.resumeOnStartup !== undefined || patch.resumePrompt !== undefined; +} +function restartRecoveryPatch(patch: SupportedMutableConfigPatch) { + return { + ...(patch.resumeOnStartup !== undefined ? { resumeOnStartup: patch.resumeOnStartup } : {}), + ...(patch.resumePrompt !== undefined ? { resumePrompt: patch.resumePrompt } : {}), + }; +} diff --git a/packages/server/src/server/daemon-e2e/daemon-restart-resume.e2e.test.ts b/packages/server/src/server/daemon-e2e/daemon-restart-resume.e2e.test.ts index efc99c9b59..1658356eed 100644 --- a/packages/server/src/server/daemon-e2e/daemon-restart-resume.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/daemon-restart-resume.e2e.test.ts @@ -76,10 +76,14 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { DaemonClient } from "../test-utils/daemon-client.js"; -async function restartFixture(root: string) { +async function restartFixture( + root: string, + settings: { resumeOnStartup?: boolean; resumePrompt?: string } = {}, + fixture: { holdResume?: boolean } = {}, +) { const child = fork( fileURLToPath(new URL("../test-utils/finish-subscription-daemon.ts", import.meta.url)), - [root], + [root, JSON.stringify(settings), JSON.stringify(fixture)], { execArgv: ["--conditions=source", "--import", "tsx"], stdio: ["ignore", "pipe", "pipe", "ipc"], @@ -183,7 +187,25 @@ test.each([false, true])( await expect.poll(() => readPrompts(root), { timeout: 15000 }).toHaveLength(1); const prompts = await readPrompts(root); expect(prompts[0]).toContain(`Agent ${childId}`); - expect(prompts[0]).toContain("was closed"); + expect(prompts[0]).not.toContain("was closed"); + const recoveredPrompts = (await readFile(path.join(root, "all-prompts.jsonl"), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(recoveredPrompts.filter((entry) => entry.prompt === "Resume working.")).toHaveLength( + 1, + ); + const restored = recoveredPrompts.find((entry) => entry.prompt === "Resume working.").config; + const original = recoveredPrompts.find((entry) => + String(entry.prompt).includes("HOLD_CHILD"), + ).config; + expect(restored).toMatchObject({ + provider: original.provider, + model: original.model, + modeId: original.modeId, + cwd: original.cwd, + }); + await expect .poll( async () => @@ -196,7 +218,7 @@ test.each([false, true])( client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" }); await client.connect(); await client.fetchAgents({ subscribe: {} }); - expect((await client.fetchAgent(childId))?.agent.status).toBe("closed"); + expect((await client.fetchAgent(childId))?.agent.status).toBe("idle"); await client.close(); client = undefined; await stopFixture(second.child, false); @@ -220,3 +242,120 @@ test.each([false, true])( }, 60000, ); + +test.each([false, true])( + "host restart recovery settings (enabled=%s)", + async (enabled) => { + const root = await mkdtemp(path.join(tmpdir(), "resume-settings-process-")); + const processes: ChildProcess[] = []; + let client: DaemonClient | undefined; + const settings = { resumeOnStartup: enabled, resumePrompt: "Continue exactly.\n" }; + try { + const first = await restartFixture(root, settings); + processes.push(first.child); + client = new DaemonClient({ url: `ws://127.0.0.1:${first.port}/ws`, appVersion: "0.13.0" }); + await client.connect(); + const working = await client.createAgent({ + provider: "claude", + cwd: first.cwd, + modeId: "full-access", + }); + const idle = await client.createAgent({ + provider: "claude", + cwd: first.cwd, + modeId: "full-access", + }); + await client.sendMessage(working.id, "HOLD_CHILD"); + await expect + .poll( + async () => + ( + await new AgentStorage(path.join(first.paseoHome, "agents"), createTestLogger()).get( + working.id, + ) + )?.lastStatus, + ) + .toBe("running"); + const original = await new AgentStorage( + path.join(first.paseoHome, "agents"), + createTestLogger(), + ).get(working.id); + await client.close(); + client = undefined; + await stopFixture(first.child, false); + const second = await restartFixture(root, settings); + processes.push(second.child); + client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" }); + await client.connect(); + const storage = new AgentStorage(path.join(second.paseoHome, "agents"), createTestLogger()); + // Resume runs in the background; it is done once the saved intent is consumed. + await expect.poll(async () => (await storage.get(working.id))?.resumeOnStartup).toBe(false); + const resumePrompts = async () => + (await readFile(path.join(root, "all-prompts.jsonl"), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + .filter((entry) => entry.prompt === settings.resumePrompt); + await expect.poll(resumePrompts).toHaveLength(enabled ? 1 : 0); + expect((await client.fetchAgent(idle.id))?.agent.status).toBe("closed"); + const restored = await storage.get(working.id); + expect(restored?.persistence?.sessionId).toBe(original?.persistence?.sessionId); + } finally { + await client?.close(); + for (const child of processes.toReversed()) await stopFixture(child, true); + await rm(root, { recursive: true, force: true }); + } + }, + 60000, +); + +test("a provider that is slow to reopen does not hold daemon startup", async () => { + const root = await mkdtemp(path.join(tmpdir(), "resume-background-process-")); + const processes: ChildProcess[] = []; + let client: DaemonClient | undefined; + try { + const first = await restartFixture(root); + processes.push(first.child); + client = new DaemonClient({ url: `ws://127.0.0.1:${first.port}/ws`, appVersion: "0.13.0" }); + await client.connect(); + const working = await client.createAgent({ + provider: "claude", + cwd: first.cwd, + modeId: "full-access", + }); + await client.sendMessage(working.id, "HOLD_CHILD"); + await expect + .poll( + async () => + ( + await new AgentStorage(path.join(first.paseoHome, "agents"), createTestLogger()).get( + working.id, + ) + )?.lastStatus, + ) + .toBe("running"); + await client.close(); + client = undefined; + await stopFixture(first.child, false); + + const second = await restartFixture(root, {}, { holdResume: true }); + processes.push(second.child); + client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" }); + await client.connect(); + expect((await client.fetchAgent(working.id))?.agent.id).toBe(working.id); + second.child.send("release-resume"); + await expect + .poll(async () => + (await readFile(path.join(root, "all-prompts.jsonl"), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line).prompt) + .filter((prompt) => prompt === "Resume working."), + ) + .toHaveLength(1); + } finally { + await client?.close(); + for (const child of processes.toReversed()) await stopFixture(child, true); + await rm(root, { recursive: true, force: true }); + } +}, 30000); diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 51bfb3d3a4..f7b3982513 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -348,6 +348,8 @@ export const PersistedConfigSchema = z .object({ providers: z.preprocess(normalizeAgentProviders, ProviderOverridesSchema).optional(), idleRuntimeCleanup: z.boolean().optional(), + resumeOnStartup: z.boolean().optional(), + resumePrompt: z.string().optional(), catalogRefreshTimeoutMs: z.number().int().positive().max(2_147_483_647).optional(), metadataGeneration: AgentMetadataGenerationSchema.optional(), skills: z.object({ selection: AgentSkillSelectionSchema.optional() }).strict().optional(), diff --git a/packages/server/src/server/test-utils/fake-agent-client.ts b/packages/server/src/server/test-utils/fake-agent-client.ts index 42ac0c7c79..4304a767e1 100644 --- a/packages/server/src/server/test-utils/fake-agent-client.ts +++ b/packages/server/src/server/test-utils/fake-agent-client.ts @@ -67,6 +67,7 @@ interface FakeAgentSessionOptions { export interface TestAgentClientOptions { beforeCreateSession?: (config: AgentSessionConfig) => Promise; + beforeResumeSession?: (handle: AgentPersistenceHandle) => Promise; closeSession?: () => Promise; onStartTurn?: (prompt: AgentPromptInput, config: AgentSessionConfig) => void; beforeTurnComplete?: (prompt: AgentPromptInput) => Promise; @@ -1254,6 +1255,7 @@ class FakeAgentClient implements AgentClient { overrides?: Partial, _launchContext?: AgentLaunchContext, ): Promise { + await this.options.beforeResumeSession?.(handle); const cfg: AgentSessionConfig = { provider: this.provider, cwd: overrides?.cwd ?? process.cwd(), diff --git a/packages/server/src/server/test-utils/finish-subscription-daemon.ts b/packages/server/src/server/test-utils/finish-subscription-daemon.ts index d35bc55ec8..eafe11e74a 100644 --- a/packages/server/src/server/test-utils/finish-subscription-daemon.ts +++ b/packages/server/src/server/test-utils/finish-subscription-daemon.ts @@ -9,18 +9,24 @@ const root = process.argv[2]; if (!root || !process.send) throw new Error("Expected isolated test home and IPC parent"); await mkdir(join(root, "static"), { recursive: true }); const held = Promise.withResolvers(); +const fixture = JSON.parse(process.argv[4] ?? "{}") as { holdResume?: boolean }; +const resumeReleased = Promise.withResolvers(); +if (!fixture.holdResume) resumeReleased.resolve(); const daemon = await createTestPaseoDaemon({ paseoHomeRoot: root, + ...JSON.parse(process.argv[3] ?? "{}"), staticDir: join(root, "static"), logger: pino({ level: "warn" }, pino.destination(join(root, "fixture.log"))), cleanup: false, mcpEnabled: true, agentClients: createTestAgentClients({ - onStartTurn: (prompt) => { + onStartTurn: (prompt, config) => { + void appendFile(join(root, "all-prompts.jsonl"), `${JSON.stringify({ prompt, config })}\n`); if (String(prompt).includes('kind="agent-notification"')) { void appendFile(join(root, "parent-prompts.jsonl"), `${JSON.stringify(prompt)}\n`); } }, + beforeResumeSession: () => resumeReleased.promise, beforeTurnComplete: (prompt) => String(prompt).includes("HOLD_CHILD") ? held.promise : Promise.resolve(), }), @@ -28,6 +34,7 @@ const daemon = await createTestPaseoDaemon({ process.send({ port: daemon.port, paseoHome: daemon.paseoHome, cwd: daemon.staticDir }); process.on("message", (message) => { if (message === "finish") held.resolve(); + if (message === "release-resume") resumeReleased.resolve(); if (message === "stop") { void daemon.close().then( () => process.exit(0), diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index a1f8369b59..766fd49919 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -52,6 +52,8 @@ interface TestPaseoDaemonOptions { trustedProxies?: PaseoDaemonConfig["trustedProxies"]; agentProfiles?: AgentProfile[]; autoArchiveAfterMerge?: boolean; + resumeOnStartup?: boolean; + resumePrompt?: string; pluginsEnabled?: PaseoDaemonConfig["pluginsEnabled"]; builtinPlugins?: BuiltinPluginLoader; plugins?: PaseoDaemonConfig["plugins"]; @@ -226,6 +228,8 @@ async function prepareTestDaemonConfig( downloadTokenTtlMs: options.downloadTokenTtlMs, agentProfiles: options.agentProfiles, autoArchiveAfterMerge: options.autoArchiveAfterMerge, + resumeOnStartup: options.resumeOnStartup, + resumePrompt: options.resumePrompt, pluginsEnabled: options.pluginsEnabled, plugins: options.plugins, }; diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 6705c1cdd9..b7024a638b 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -2013,6 +2013,7 @@ export class VoiceAssistantWebSocketServer { skillManagement: true, // COMPAT(idleRuntimeCleanupSetting): added in v0.13.0, remove gate after 2027-04-26. idleRuntimeCleanupSetting: true, + resumeOnStartupSetting: true, // COMPAT(terminalRestoreModes): added in v0.1.81, remove gate after 2026-11-23. "terminal-restore-modes": true, // COMPAT(terminalInputModeReplay): added in v0.2.6, remove gate after 2027-02-02. diff --git a/packages/server/src/server/workspace-git-service.test.ts b/packages/server/src/server/workspace-git-service.test.ts index f3571f9932..4e6acff9cd 100644 --- a/packages/server/src/server/workspace-git-service.test.ts +++ b/packages/server/src/server/workspace-git-service.test.ts @@ -1171,10 +1171,14 @@ describe("WorkspaceGitServiceImpl", () => { const subscription = service.registerWorkspace({ cwd: REPO_CWD }, listener); await vi.waitFor(() => { expect(getCheckoutSnapshotFacts).toHaveBeenCalledTimes(1); + expect(service.getMetrics().fetchInFlightCount).toBe(1); }); releaseFetch.resolve(); - await flushPromises(); + // The refresh debounce starts when the fetch result is applied, not when it is released. + await vi.waitFor(() => { + expect(service.getMetrics().fetchInFlightCount).toBe(0); + }); await vi.advanceTimersByTimeAsync(1_000); await vi.waitFor(() => { expect(getCheckoutRefDerivedState).toHaveBeenCalledTimes(1); diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index c1c32ed20a..1b74066bae 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -513,6 +513,12 @@ "idleRuntimeCleanup": { "type": "boolean" }, + "resumeOnStartup": { + "type": "boolean" + }, + "resumePrompt": { + "type": "string" + }, "catalogRefreshTimeoutMs": { "type": "integer", "exclusiveMinimum": 0,