Resume interrupted agents after host restart (#8127)

* feat(server): resume interrupted agents after host restart

Continue saved running conversations after graceful restart or crash, with host opt-out and a configurable continuation prompt. Preserve explicit launch overrides when reopening the same provider session.

Co-authored-by: coolwind <45729206+coolwindX@users.noreply.github.com>

Co-authored-by: suntianc <suntc8985@gmail.com>

* fix(app): translate restart recovery host settings

* test(app): synchronize sidebar drags and clean up added projects

Wait for the drag sensor click blocker to expire before the next user action. Resolve UI project keys to daemon IDs for cleanup so empty projects do not contaminate later sidebar tests.

* fix(app): tuck the resume prompt behind an edit sheet

Match the System prompt row: an Edit button left of the restart recovery
switch opens the prompt in a sheet, and is disabled while resume is off.

* fix(server): resume interrupted agents without holding startup

Startup now only chooses resume candidates before anything else reads or
repairs persisted running state. Reopening and prompting run in the
background after connections open, so a provider that is slow to reopen
holds back only its own agent. Finish observation waits for a pending
child's resume, so parents are not told the child was closed meanwhile.

* style(server): format restart resume e2e test

* test(server): wait for the fetch result before advancing the narrow refresh debounce

The test released the held fetch, flushed six microtasks, then advanced
the 1s refresh debounce. When the fetch result landed after that advance
(slow Windows runners), the debounce started late and vi.waitFor's
auto-advance reached it only at its real-time timeout. Wait on the
service's in-flight fetch count instead, as the sibling test does.

* test(server): retry stalled OpenCode fixture downloads within the test budget

npm abandons a stalled registry download only after its 300s default
fetch timeout, past the install's 180s budget. The killed install left
the fixture directory locked on Windows, so cleanup failed with EBUSY and
hid the timeout. Against a registry proxy that stalls the first tarball,
a default install hung 225s; with --fetch-timeout it retried and finished
in 16s.

---------

Co-authored-by: suntianc <suntc8985@gmail.com>
This commit is contained in:
Mohamed Boudra
2026-10-28 23:40:35 +07:00
committed by GitHub
co-authored by suntianc
parent 5cfe8f1218
commit 83ed52669d
40 changed files with 926 additions and 31 deletions
@@ -135,6 +135,7 @@ export async function ensureAgentLoaded(
labels: record.labels,
workspaceId: record.workspaceId,
owner: record.owner,
env: record.launchEnv,
});
deps.logger.info({ agentId, provider: record.provider }, "Agent created from stored config");
}
@@ -2929,6 +2929,92 @@ test("createAgent passes daemon launch env through the provider launch context",
});
});
test.each([
{ provider: "codex", hasHandle: true },
{ provider: "opencode", hasHandle: true },
{ provider: "codex", hasHandle: false },
{ provider: "opencode", hasHandle: false },
])(
"$provider preserves explicit launch env across reloads and storage-backed resumes (handle: $hasHandle)",
async ({ provider, hasHandle }) => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-launch-env-"));
const storagePath = join(workdir, "agents");
class LaunchEnvClient extends TestAgentClient {
lastLaunchContext: AgentLaunchContext | undefined;
override async createSession(
config: AgentSessionConfig,
context?: AgentLaunchContext,
): Promise<AgentSession> {
this.lastLaunchContext = context;
const session = await super.createSession(config);
session.describePersistence = () => ({ provider, sessionId: session.id });
return session;
}
override async resumeSession(
handle: AgentPersistenceHandle,
config?: Partial<AgentSessionConfig>,
context?: AgentLaunchContext,
): Promise<AgentSession> {
this.lastLaunchContext = context;
const session = await super.resumeSession(handle, config, context);
session.describePersistence = () => ({ provider, sessionId: handle.sessionId });
return session;
}
}
const client = new LaunchEnvClient(provider);
let storage = new AgentStorage(storagePath, logger);
let manager = new AgentManager({ clients: { [provider]: client }, registry: storage, logger });
const env = { XDG_DATA_HOME: join(workdir, "agent-data"), CUSTOM_EMPTY: "" };
const expectedEnv = { ...env };
const agent = await manager.createAgent({ provider, cwd: workdir }, undefined, {
workspaceId: undefined,
env,
});
onTestFinished(async () => {
await manager.closeAgent(agent.id);
await manager.flush();
rmSync(workdir, { recursive: true, force: true });
});
env.XDG_DATA_HOME = "caller-mutated";
await manager.closeAgent(agent.id);
await manager.flush();
if (!hasHandle) {
const record = await storage.get(agent.id);
if (!record) throw new Error("Created agent was not persisted");
await storage.upsert({ ...record, persistence: null });
}
// Fresh storage and manager instances must read the environment from disk.
for (let restart = 0; restart < 2; restart += 1) {
storage = new AgentStorage(storagePath, logger);
manager = new AgentManager({ clients: { [provider]: client }, registry: storage, logger });
const resumed = await ensureAgentLoaded(agent.id, {
agentManager: manager,
agentStorage: storage,
logger,
});
expect(client.lastLaunchContext).toEqual({
agentId: agent.id,
env: { ...expectedEnv, PASEO_AGENT_ID: agent.id, PASEO_AGENT_CWD: workdir },
});
await manager.reloadAgentSession(agent.id);
expect(client.lastLaunchContext).toEqual({
agentId: agent.id,
env: { ...expectedEnv, PASEO_AGENT_ID: agent.id, PASEO_AGENT_CWD: workdir },
});
expect(toAgentPayload(resumed)).not.toHaveProperty("launchEnv");
expect(toAgentPayload(resumed).persistence?.metadata).not.toHaveProperty("launchEnv");
await manager.closeAgent(agent.id);
await manager.flush();
}
},
);
test("createAgent passes persistSession to provider create options", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
@@ -421,6 +421,8 @@ interface ManagedAgentBase {
owner?: AgentOwner;
capabilities: AgentCapabilityFlags;
config: AgentSessionConfig;
/** Explicit launch overrides, before per-opening hooks and daemon-owned variables. */
launchEnv?: Record<string, string>;
runtimeInfo?: AgentRuntimeInfo;
createdAt: Date;
updatedAt: Date;
@@ -1399,6 +1401,7 @@ export class AgentManager {
initialTitle: options.initialTitle,
workspaceId: options.workspaceId,
owner: options.owner,
launchEnv: options.env ? { ...options.env } : undefined,
historyPrimed: true,
});
if (!agent.internal) {
@@ -1487,10 +1490,11 @@ export class AgentManager {
: resumeOptions;
const purpose = currentResumeOptions?.purpose ?? "interactive";
const launchEnv = record?.launchEnv;
const { storedConfig, launchConfig, paseoToolPolicy } = await this.prepareSessionConfig(
mergedConfig,
resolvedAgentId,
{ purpose },
{ purpose, env: launchEnv },
);
const client = this.requireClient(handle.provider);
const available = await client.isAvailable();
@@ -1505,7 +1509,7 @@ export class AgentManager {
client,
storedConfig.cwd,
paseoToolPolicy,
undefined,
launchEnv,
{
reason: "resume",
purpose,
@@ -1524,6 +1528,7 @@ export class AgentManager {
await this.requireExternalMcpSupport(session, providerLaunchConfig, launchContext);
return this.registerSession(session, storedConfig, resolvedAgentId, {
...options,
launchEnv,
persistence: handle,
restoring: true,
});
@@ -1661,6 +1666,7 @@ export class AgentManager {
const { storedConfig, launchConfig, paseoToolPolicy } = await this.prepareSessionConfig(
refreshConfig,
agentId,
{ env: existing.launchEnv },
);
const hadPreviousPaseoToolPolicy = this.paseoToolPolicies.has(agentId);
const previousPaseoToolPolicy = this.paseoToolPolicies.get(agentId);
@@ -1669,7 +1675,7 @@ export class AgentManager {
client,
storedConfig.cwd,
paseoToolPolicy,
undefined,
existing.launchEnv,
{
reason: "refresh",
purpose: "interactive",
@@ -1720,6 +1726,7 @@ export class AgentManager {
labels: existing.labels,
workspaceId: existing.workspaceId,
owner: existing.owner,
launchEnv: existing.launchEnv,
createdAt: existing.createdAt,
updatedAt: existing.updatedAt,
lastUserMessageAt: existing.lastUserMessageAt,
@@ -3763,6 +3770,7 @@ export class AgentManager {
publishWhenReady?: boolean;
workspaceId?: string;
owner?: AgentOwner;
launchEnv?: Record<string, string>;
},
): Promise<ManagedAgent> {
let registered = false;
@@ -3948,6 +3956,7 @@ export class AgentManager {
persistence?: AgentPersistenceHandle;
workspaceId?: string;
owner?: AgentOwner;
launchEnv?: Record<string, string>;
}
| undefined;
}): ActiveManagedAgent {
@@ -3958,6 +3967,7 @@ export class AgentManager {
cwd: config.cwd,
workspaceId: options?.workspaceId,
owner: options?.owner,
launchEnv: options?.launchEnv,
session,
capabilities: session.capabilities,
config,
@@ -95,6 +95,7 @@ export function toStoredAgentRecord(
lastStatus: agent.lifecycle,
lastModeId: agent.currentModeId ?? config?.modeId ?? null,
config: config ?? null,
launchEnv: agent.launchEnv ? { ...agent.launchEnv } : undefined,
runtimeInfo,
features: normalizeFeatures(agent.features),
persistence,
@@ -54,8 +54,10 @@ const STORED_AGENT_SCHEMA = z.object({
title: z.string().nullable().optional(),
labels: z.record(z.string(), z.string()).default({}),
lastStatus: AgentStatusSchema.default("closed"),
resumeOnStartup: z.boolean().optional(),
lastModeId: z.string().nullable().optional(),
config: SERIALIZABLE_CONFIG_SCHEMA,
launchEnv: z.record(z.string(), z.string()).optional(),
runtimeInfo: z
.object({
provider: z.string(),
@@ -259,6 +261,8 @@ export class AgentStorage {
if (existing && existing.archivedAt !== undefined) {
record.archivedAt = existing.archivedAt;
}
record.resumeOnStartup =
existing?.resumeOnStartup && agent.lifecycle !== "idle" && agent.lifecycle !== "error";
return record;
});
}
@@ -13,6 +13,8 @@ import type { FinishObservation } from "../index.js";
export function createFinishObservation(input: {
agentManager: AgentManager;
agentStorage: AgentStorage;
/** A child chosen for restart recovery reports its state only after its resume was sent. */
waitForResume(agentId: string, signal: AbortSignal): Promise<void>;
messages(): AgentMessageDelivery;
hosts(): ToolHosts;
serverId: string;
@@ -24,6 +26,7 @@ export function createFinishObservation(input: {
hosts: input.hosts,
local: {
async wait(agentId, signal) {
await input.waitForResume(agentId, signal);
await input.messages().waitForDispatch(agentId, signal);
signal.throwIfAborted();
const live = agentManager.getAgent(agentId);
@@ -52,6 +55,7 @@ export function createFinishObservation(input: {
};
},
async observe(agentId, handler, signal, onError) {
await input.waitForResume(agentId, signal);
const stored = await agentStorage.get(agentId);
let updates = Promise.resolve();
const publish = (snapshot: AgentSnapshotPayload | null, event?: AgentStreamEvent) => {
@@ -881,6 +881,7 @@ async function createAgentMcpServer(
logger: options.logger,
...createFinishObservation({
...options,
waitForResume: async () => {},
messages: () => messages,
serverId: options.serverId ?? "self",
hosts: () => {
@@ -461,9 +461,19 @@ test.each([
process.platform === "win32"
? `${windowsPackage}-${process.arch}@${cliPackage.slice(cliPackage.lastIndexOf("@") + 1)}`
: cliPackage;
// npm abandons a stalled registry download only after its 5 minute default, past this
// test's budget, and the killed install keeps the fixture locked. Retry stalls sooner.
await execCommand(
"npm",
["install", "--prefix", root, "--no-audit", "--no-fund", installPackage],
[
"install",
"--prefix",
root,
"--no-audit",
"--no-fund",
"--fetch-timeout=30000",
installPackage,
],
{
cwd: root,
timeout: 180_000,
@@ -0,0 +1,105 @@
import { expect, test } from "vitest";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { AgentStorage, parseStoredAgentRecord } from "../agent-storage.js";
import { createTestLogger } from "../../../test-utils/test-logger.js";
import { createRestartRecovery } from "./index.js";
test("recovery saves intent before normalization, clears before dispatch, and resumes each agent independently", async () => {
const root = await mkdtemp(join(tmpdir(), "restart-recovery-"));
try {
const storage = new AgentStorage(root, createTestLogger());
for (const [id, lastStatus] of [
["broken", "running"],
["stuck", "running"],
["working", "running"],
["idle", "idle"],
["initializing", "initializing"],
] as const) {
await storage.upsert(
parseStoredAgentRecord({
id,
lastStatus,
provider: "codex",
cwd: root,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
}),
);
}
const dispatched: string[] = [];
const recovery = createRestartRecovery({
storage,
settings: () => ({ enabled: true, prompt: "Continue verbatim.\n" }),
eligible: async () => true,
dispatch: async (id, prompt) => {
expect((await storage.get(id))?.resumeOnStartup).toBe(false);
if (id === "broken") throw new Error("unavailable");
if (id === "stuck") return new Promise(() => {});
dispatched.push(id + prompt);
},
report: () => {},
});
await recovery.prepareStartup();
expect((await storage.get("working"))?.resumeOnStartup).toBe(true);
expect((await storage.get("working"))?.lastStatus).toBe("closed");
recovery.resumePending();
const running = new AbortController().signal;
await recovery.waitForResume("broken", running);
await recovery.waitForResume("working", running);
await recovery.waitForResume("idle", running);
expect(dispatched).toEqual(["workingContinue verbatim.\n"]);
const stopped = new AbortController();
stopped.abort();
await expect(recovery.waitForResume("stuck", stopped.signal)).rejects.toBeDefined();
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("opt-out consumes pending recovery and excludes archived, private, owned and archived-workspace agents", async () => {
const root = await mkdtemp(join(tmpdir(), "restart-recovery-"));
try {
const storage = new AgentStorage(root, createTestLogger());
for (const extra of [
{ id: "running" },
{ id: "archived", archivedAt: "today" },
{ id: "private", internal: true },
{ id: "owned", owner: { kind: "daemon", daemonId: "hub", executionId: "run" } },
{ id: "workspace", workspaceId: "archived" },
]) {
await storage.upsert(
parseStoredAgentRecord({
lastStatus: "running",
provider: "codex",
cwd: root,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
...extra,
}),
);
}
let enabled = true;
const dispatched: string[] = [];
const recovery = createRestartRecovery({
storage,
settings: () => ({ enabled, prompt: "Resume working." }),
eligible: async (record) => record.workspaceId !== "archived",
dispatch: async (id) => {
dispatched.push(id);
},
report: () => {},
});
await recovery.captureRunningAgents();
expect((await storage.get("running"))?.resumeOnStartup).toBe(true);
enabled = false;
await recovery.prepareStartup();
enabled = true;
recovery.resumePending();
expect(dispatched).toEqual([]);
expect((await storage.list()).every((record) => !record.resumeOnStartup)).toBe(true);
} finally {
await rm(root, { recursive: true, force: true });
}
});
@@ -0,0 +1,81 @@
import type { AgentStorage, StoredAgentRecord } from "../agent-storage.js";
interface Dependencies {
storage: AgentStorage;
settings(): { enabled: boolean; prompt: string };
eligible(record: StoredAgentRecord): Promise<boolean>;
dispatch(agentId: string, prompt: string): Promise<unknown>;
report(agentId: string, error: unknown): void;
}
export function createRestartRecovery(deps: Dependencies) {
// Candidates are chosen at startup; each settles once its resume was sent or abandoned.
const resuming = new Map<string, { promise: Promise<void>; resolve(): void }>();
const eligible = async (record: StoredAgentRecord) =>
!record.archivedAt && !record.internal && !record.owner && (await deps.eligible(record));
async function resume(agentId: string) {
const record = await deps.storage.get(agentId);
if (!record?.resumeOnStartup) return;
await deps.storage.upsert({ ...record, resumeOnStartup: false });
if (!deps.settings().enabled || !(await eligible(record))) return;
await deps.dispatch(agentId, deps.settings().prompt);
}
return {
/** Choose candidates before anything else reads or repairs persisted running status. */
async prepareStartup() {
for (const record of await deps.storage.list()) {
const pending =
deps.settings().enabled &&
(record.resumeOnStartup === true || record.lastStatus === "running") &&
(await eligible(record));
await deps.storage.upsert({
...record,
resumeOnStartup: pending,
lastStatus:
record.lastStatus === "running" || record.lastStatus === "initializing"
? "closed"
: record.lastStatus,
});
if (pending) {
let resolve!: () => void;
const promise = new Promise<void>((settle) => {
resolve = settle;
});
resuming.set(record.id, { promise, resolve });
}
}
},
async captureRunningAgents() {
await deps.storage.flush();
for (const record of await deps.storage.list()) {
if (record.lastStatus === "running" && (await eligible(record)))
await deps.storage.upsert({ ...record, resumeOnStartup: deps.settings().enabled });
}
},
/** Resume every candidate in the background; a slow provider holds back only its agent. */
resumePending() {
for (const [agentId, settled] of resuming) {
void resume(agentId)
.catch((error) => deps.report(agentId, error))
.finally(() => {
resuming.delete(agentId);
settled.resolve();
});
}
},
/** Wait until this agent's resume was sent or abandoned. Others return at once. */
async waitForResume(agentId: string, signal: AbortSignal) {
const settled = resuming.get(agentId);
if (!settled) return;
let abort!: () => void;
const aborted = new Promise<never>((_resolve, reject) => {
abort = () => reject(signal.reason);
if (signal.aborted) abort();
else signal.addEventListener("abort", abort, { once: true });
});
try {
await Promise.race([settled.promise, aborted]);
} finally {
signal.removeEventListener("abort", abort);
}
},
};
}
@@ -1069,7 +1069,7 @@ function probeWebSocketConnection(
});
}
test("startup closes stale active records before serving status without resuming sessions", async () => {
test("disabled recovery closes stale active records without resuming sessions", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "paseo-stale-status-"));
const logger = pino({ level: "silent" });
const storagePath = path.join(root, ".paseo", "agents");
@@ -1121,6 +1121,7 @@ test("startup closes stale active records before serving status without resuming
let sessionsCreated = 0;
const handle = await createTestPaseoDaemon({
paseoHomeRoot: root,
resumeOnStartup: false,
agentClients: createTestAgentClients({
beforeCreateSession: async () => {
sessionsCreated++;
@@ -1131,6 +1132,7 @@ test("startup closes stale active records before serving status without resuming
try {
const expected = records.map((record) => ({
...record,
resumeOnStartup: false,
lastStatus:
record.lastStatus === "running" || record.lastStatus === "initializing"
? "closed"
+22 -11
View File
@@ -1,3 +1,5 @@
import { createRestartRecovery } from "./agent/restart-recovery/index.js";
import { sendPromptToAgent } from "./agent/agent-prompt.js";
import { createIdleRuntimeCleanup } from "./agent/idle-runtime-cleanup/index.js";
import {
createFinishSubscriptions,
@@ -464,6 +466,8 @@ export interface PaseoDaemonConfig {
downloadTokenTtlMs?: number;
agentProviderSettings?: AgentProviderRuntimeSettingsMap;
idleRuntimeCleanup?: boolean;
resumeOnStartup?: boolean;
resumePrompt?: string;
providerCatalogRefreshTimeoutMs?: number;
metadataGeneration?: {
providers?: Array<{
@@ -594,6 +598,8 @@ function createInitialMutableDaemonConfig(config: ResolvedDaemonSettings): Mutab
autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false,
enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false,
idleRuntimeCleanup: config.idleRuntimeCleanup !== false,
resumeOnStartup: config.resumeOnStartup !== false,
resumePrompt: config.resumePrompt ?? "Resume working.",
appendSystemPrompt: config.appendSystemPrompt ?? "",
pluginsEnabled: config.pluginsEnabled ?? false,
plugins: config.plugins ?? {},
@@ -1036,7 +1042,19 @@ export async function createPaseoDaemon(
agentStorage,
);
await agentStorage.initialize();
await closeStaleAgentRecords(agentStorage);
const restartRecovery = createRestartRecovery({
storage: agentStorage,
settings: () => ({
enabled: daemonConfigStore.get().resumeOnStartup !== false,
prompt: daemonConfigStore.get().resumePrompt ?? "Resume working.",
}),
eligible: async (record) =>
!record.workspaceId || !(await workspaceRegistry?.get(record.workspaceId))?.archivedAt,
dispatch: (agentId, prompt) =>
sendPromptToAgent({ agentId, prompt, agentManager, agentStorage, logger, unarchive: false }),
report: (agentId, err) => logger.error({ agentId, err }, "Agent restart recovery failed"),
});
await restartRecovery.prepareStartup();
logger.info({ elapsed: elapsed() }, "Agent storage initialized");
await bootstrapWorkspaceRegistries({
serverId,
@@ -1470,6 +1488,7 @@ export async function createPaseoDaemon(
...createFinishObservation({
agentManager,
agentStorage,
waitForResume: restartRecovery.waitForResume,
serverId,
logger,
messages: () => {
@@ -1869,6 +1888,7 @@ export async function createPaseoDaemon(
pluginRuntime.bindPaseoSessionHost(wsServer);
await pluginRuntime.start();
providerSnapshotManager.settlePluginProviders();
restartRecovery.resumePending();
await finishSubscriptions.start();
wsServer.beginAcceptingConnections();
const relaySecurityLogger = logger.child({ module: "relay-security" });
@@ -1963,6 +1983,7 @@ export async function createPaseoDaemon(
await finishSubscriptions.close();
await wsServer?.agentMessages.close();
agentManager.prepareForShutdown();
await restartRecovery.captureRunningAgents();
await closeAllAgents(logger, agentManager);
await withTimeout({
promise: pluginRuntime.drainEvents(),
@@ -2029,16 +2050,6 @@ export async function createPaseoDaemon(
};
}
async function closeStaleAgentRecords(agentStorage: AgentStorage): Promise<void> {
// No provider runtimes are loaded at startup. Repair interrupted
// shutdowns before any status consumer sees the persisted active states.
for (const record of await agentStorage.list()) {
if (record.lastStatus === "running" || record.lastStatus === "initializing") {
await agentStorage.upsert({ ...record, lastStatus: "closed" });
}
}
}
/**
* Closing an agent asks its provider to close the session and waits for the
* answer. A provider that never answers must not hold the daemon open, so a
+10
View File
@@ -590,6 +590,8 @@ export type ResolvedDaemonSettings = Pick<
| "relayRequireProtectedSecurity"
| "appBaseUrl"
| "idleRuntimeCleanup"
| "resumeOnStartup"
| "resumePrompt"
| "providerCatalogRefreshTimeoutMs"
| "metadataGeneration"
| "providerOverrides"
@@ -677,6 +679,7 @@ export function resolveReloadableConfig(
relayEnabled: relay.enabled,
relayRequireProtectedSecurity: relay.requireProtectedSecurity,
idleRuntimeCleanup: persisted.agents?.idleRuntimeCleanup ?? true,
...resolveRestartRecoveryConfig(persisted.agents),
providerCatalogRefreshTimeoutMs: persisted.agents?.catalogRefreshTimeoutMs,
metadataGeneration: persisted.agents?.metadataGeneration,
providerOverrides: extractProviderOverrides(
@@ -827,3 +830,10 @@ function resolveSpeechOverrideControlledPaths(
if (env.PASEO_LOCAL_MODELS_DIR !== undefined) paths.push("providers.local.modelsDir");
return paths;
}
function resolveRestartRecoveryConfig(agents: PersistedConfig["agents"]) {
return {
resumeOnStartup: agents?.resumeOnStartup ?? true,
resumePrompt: agents?.resumePrompt ?? "Resume working.",
};
}
@@ -940,6 +940,15 @@ describe("DaemonConfigStore", () => {
await store.patch({ idleRuntimeCleanup: true });
expect(loadConfig(paseoHome, { env: {} }).idleRuntimeCleanup).toBe(true);
await store.patch({ resumeOnStartup: false, resumePrompt: "Keep going.\n" });
expect(loadPersistedConfig(paseoHome).agents).toMatchObject({
resumeOnStartup: false,
resumePrompt: "Keep going.\n",
});
expect(loadConfig(paseoHome, { env: {} })).toMatchObject({
resumeOnStartup: false,
resumePrompt: "Keep going.\n",
});
});
test("patch persists custom ACP provider overrides into config.json", async () => {
@@ -29,6 +29,8 @@ interface SupportedMutableConfigPatch {
autoArchiveAfterMerge?: boolean;
enableTerminalAgentHooks?: boolean;
idleRuntimeCleanup?: boolean;
resumeOnStartup?: boolean;
resumePrompt?: string;
appendSystemPrompt?: string;
terminalProfiles?: MutableDaemonConfig["terminalProfiles"];
agentProfiles?: MutableDaemonConfig["agentProfiles"];
@@ -193,6 +195,8 @@ const RELOADABLE_PATHS = [
"daemon.agentProfiles",
"app.baseUrl",
"agents.idleRuntimeCleanup",
"agents.resumeOnStartup",
"agents.resumePrompt",
"agents.providers",
"agents.catalogRefreshTimeoutMs",
"agents.metadataGeneration",
@@ -218,6 +222,8 @@ const PERSISTED_TO_MUTABLE_PATH = new Map<string, string>([
["daemon.agentProfiles", "agentProfiles"],
["app.baseUrl", "app.baseUrl"],
["agents.idleRuntimeCleanup", "idleRuntimeCleanup"],
["agents.resumeOnStartup", "resumeOnStartup"],
["agents.resumePrompt", "resumePrompt"],
["agents.providers", "providers"],
["agents.catalogRefreshTimeoutMs", "catalogRefreshTimeoutMs"],
["agents.metadataGeneration", "metadataGeneration"],
@@ -292,6 +298,8 @@ function pickSupportedPatchFields(patch: MutableDaemonConfigPatch): SupportedMut
...(patch.enableTerminalAgentHooks !== undefined
? { enableTerminalAgentHooks: patch.enableTerminalAgentHooks }
: {}),
...(patch.resumeOnStartup !== undefined ? { resumeOnStartup: patch.resumeOnStartup } : {}),
...(patch.resumePrompt !== undefined ? { resumePrompt: patch.resumePrompt } : {}),
...(patch.idleRuntimeCleanup !== undefined
? { idleRuntimeCleanup: patch.idleRuntimeCleanup }
: {}),
@@ -702,6 +710,7 @@ function mergeMutableAgentPatch(
patch.providers === undefined &&
patch.metadataGeneration === undefined &&
patch.skills === undefined &&
!hasRestartRecoveryPatch(patch) &&
patch.idleRuntimeCleanup === undefined &&
removeProviders.length === 0
) {
@@ -735,6 +744,7 @@ function mergeMutableAgentPatch(
next["skills"] = { selection: patch.skills.selection };
}
Object.assign(next, restartRecoveryPatch(patch));
if (patch.idleRuntimeCleanup !== undefined) {
next["idleRuntimeCleanup"] = patch.idleRuntimeCleanup;
}
@@ -771,3 +781,13 @@ function mergeMutableDaemonPatch(
if (patch.agentProfiles !== undefined) next.agentProfiles = patch.agentProfiles;
return Object.keys(next).length > 0 ? next : undefined;
}
function hasRestartRecoveryPatch(patch: SupportedMutableConfigPatch) {
return patch.resumeOnStartup !== undefined || patch.resumePrompt !== undefined;
}
function restartRecoveryPatch(patch: SupportedMutableConfigPatch) {
return {
...(patch.resumeOnStartup !== undefined ? { resumeOnStartup: patch.resumeOnStartup } : {}),
...(patch.resumePrompt !== undefined ? { resumePrompt: patch.resumePrompt } : {}),
};
}
@@ -76,10 +76,14 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
import { DaemonClient } from "../test-utils/daemon-client.js";
async function restartFixture(root: string) {
async function restartFixture(
root: string,
settings: { resumeOnStartup?: boolean; resumePrompt?: string } = {},
fixture: { holdResume?: boolean } = {},
) {
const child = fork(
fileURLToPath(new URL("../test-utils/finish-subscription-daemon.ts", import.meta.url)),
[root],
[root, JSON.stringify(settings), JSON.stringify(fixture)],
{
execArgv: ["--conditions=source", "--import", "tsx"],
stdio: ["ignore", "pipe", "pipe", "ipc"],
@@ -183,7 +187,25 @@ test.each([false, true])(
await expect.poll(() => readPrompts(root), { timeout: 15000 }).toHaveLength(1);
const prompts = await readPrompts(root);
expect(prompts[0]).toContain(`Agent ${childId}`);
expect(prompts[0]).toContain("was closed");
expect(prompts[0]).not.toContain("was closed");
const recoveredPrompts = (await readFile(path.join(root, "all-prompts.jsonl"), "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(recoveredPrompts.filter((entry) => entry.prompt === "Resume working.")).toHaveLength(
1,
);
const restored = recoveredPrompts.find((entry) => entry.prompt === "Resume working.").config;
const original = recoveredPrompts.find((entry) =>
String(entry.prompt).includes("HOLD_CHILD"),
).config;
expect(restored).toMatchObject({
provider: original.provider,
model: original.model,
modeId: original.modeId,
cwd: original.cwd,
});
await expect
.poll(
async () =>
@@ -196,7 +218,7 @@ test.each([false, true])(
client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" });
await client.connect();
await client.fetchAgents({ subscribe: {} });
expect((await client.fetchAgent(childId))?.agent.status).toBe("closed");
expect((await client.fetchAgent(childId))?.agent.status).toBe("idle");
await client.close();
client = undefined;
await stopFixture(second.child, false);
@@ -220,3 +242,120 @@ test.each([false, true])(
},
60000,
);
test.each([false, true])(
"host restart recovery settings (enabled=%s)",
async (enabled) => {
const root = await mkdtemp(path.join(tmpdir(), "resume-settings-process-"));
const processes: ChildProcess[] = [];
let client: DaemonClient | undefined;
const settings = { resumeOnStartup: enabled, resumePrompt: "Continue exactly.\n" };
try {
const first = await restartFixture(root, settings);
processes.push(first.child);
client = new DaemonClient({ url: `ws://127.0.0.1:${first.port}/ws`, appVersion: "0.13.0" });
await client.connect();
const working = await client.createAgent({
provider: "claude",
cwd: first.cwd,
modeId: "full-access",
});
const idle = await client.createAgent({
provider: "claude",
cwd: first.cwd,
modeId: "full-access",
});
await client.sendMessage(working.id, "HOLD_CHILD");
await expect
.poll(
async () =>
(
await new AgentStorage(path.join(first.paseoHome, "agents"), createTestLogger()).get(
working.id,
)
)?.lastStatus,
)
.toBe("running");
const original = await new AgentStorage(
path.join(first.paseoHome, "agents"),
createTestLogger(),
).get(working.id);
await client.close();
client = undefined;
await stopFixture(first.child, false);
const second = await restartFixture(root, settings);
processes.push(second.child);
client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" });
await client.connect();
const storage = new AgentStorage(path.join(second.paseoHome, "agents"), createTestLogger());
// Resume runs in the background; it is done once the saved intent is consumed.
await expect.poll(async () => (await storage.get(working.id))?.resumeOnStartup).toBe(false);
const resumePrompts = async () =>
(await readFile(path.join(root, "all-prompts.jsonl"), "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line))
.filter((entry) => entry.prompt === settings.resumePrompt);
await expect.poll(resumePrompts).toHaveLength(enabled ? 1 : 0);
expect((await client.fetchAgent(idle.id))?.agent.status).toBe("closed");
const restored = await storage.get(working.id);
expect(restored?.persistence?.sessionId).toBe(original?.persistence?.sessionId);
} finally {
await client?.close();
for (const child of processes.toReversed()) await stopFixture(child, true);
await rm(root, { recursive: true, force: true });
}
},
60000,
);
test("a provider that is slow to reopen does not hold daemon startup", async () => {
const root = await mkdtemp(path.join(tmpdir(), "resume-background-process-"));
const processes: ChildProcess[] = [];
let client: DaemonClient | undefined;
try {
const first = await restartFixture(root);
processes.push(first.child);
client = new DaemonClient({ url: `ws://127.0.0.1:${first.port}/ws`, appVersion: "0.13.0" });
await client.connect();
const working = await client.createAgent({
provider: "claude",
cwd: first.cwd,
modeId: "full-access",
});
await client.sendMessage(working.id, "HOLD_CHILD");
await expect
.poll(
async () =>
(
await new AgentStorage(path.join(first.paseoHome, "agents"), createTestLogger()).get(
working.id,
)
)?.lastStatus,
)
.toBe("running");
await client.close();
client = undefined;
await stopFixture(first.child, false);
const second = await restartFixture(root, {}, { holdResume: true });
processes.push(second.child);
client = new DaemonClient({ url: `ws://127.0.0.1:${second.port}/ws`, appVersion: "0.13.0" });
await client.connect();
expect((await client.fetchAgent(working.id))?.agent.id).toBe(working.id);
second.child.send("release-resume");
await expect
.poll(async () =>
(await readFile(path.join(root, "all-prompts.jsonl"), "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line).prompt)
.filter((prompt) => prompt === "Resume working."),
)
.toHaveLength(1);
} finally {
await client?.close();
for (const child of processes.toReversed()) await stopFixture(child, true);
await rm(root, { recursive: true, force: true });
}
}, 30000);
@@ -348,6 +348,8 @@ export const PersistedConfigSchema = z
.object({
providers: z.preprocess(normalizeAgentProviders, ProviderOverridesSchema).optional(),
idleRuntimeCleanup: z.boolean().optional(),
resumeOnStartup: z.boolean().optional(),
resumePrompt: z.string().optional(),
catalogRefreshTimeoutMs: z.number().int().positive().max(2_147_483_647).optional(),
metadataGeneration: AgentMetadataGenerationSchema.optional(),
skills: z.object({ selection: AgentSkillSelectionSchema.optional() }).strict().optional(),
@@ -67,6 +67,7 @@ interface FakeAgentSessionOptions {
export interface TestAgentClientOptions {
beforeCreateSession?: (config: AgentSessionConfig) => Promise<void>;
beforeResumeSession?: (handle: AgentPersistenceHandle) => Promise<void>;
closeSession?: () => Promise<void>;
onStartTurn?: (prompt: AgentPromptInput, config: AgentSessionConfig) => void;
beforeTurnComplete?: (prompt: AgentPromptInput) => Promise<void>;
@@ -1254,6 +1255,7 @@ class FakeAgentClient implements AgentClient {
overrides?: Partial<AgentSessionConfig>,
_launchContext?: AgentLaunchContext,
): Promise<AgentSession> {
await this.options.beforeResumeSession?.(handle);
const cfg: AgentSessionConfig = {
provider: this.provider,
cwd: overrides?.cwd ?? process.cwd(),
@@ -9,18 +9,24 @@ const root = process.argv[2];
if (!root || !process.send) throw new Error("Expected isolated test home and IPC parent");
await mkdir(join(root, "static"), { recursive: true });
const held = Promise.withResolvers<void>();
const fixture = JSON.parse(process.argv[4] ?? "{}") as { holdResume?: boolean };
const resumeReleased = Promise.withResolvers<void>();
if (!fixture.holdResume) resumeReleased.resolve();
const daemon = await createTestPaseoDaemon({
paseoHomeRoot: root,
...JSON.parse(process.argv[3] ?? "{}"),
staticDir: join(root, "static"),
logger: pino({ level: "warn" }, pino.destination(join(root, "fixture.log"))),
cleanup: false,
mcpEnabled: true,
agentClients: createTestAgentClients({
onStartTurn: (prompt) => {
onStartTurn: (prompt, config) => {
void appendFile(join(root, "all-prompts.jsonl"), `${JSON.stringify({ prompt, config })}\n`);
if (String(prompt).includes('kind="agent-notification"')) {
void appendFile(join(root, "parent-prompts.jsonl"), `${JSON.stringify(prompt)}\n`);
}
},
beforeResumeSession: () => resumeReleased.promise,
beforeTurnComplete: (prompt) =>
String(prompt).includes("HOLD_CHILD") ? held.promise : Promise.resolve(),
}),
@@ -28,6 +34,7 @@ const daemon = await createTestPaseoDaemon({
process.send({ port: daemon.port, paseoHome: daemon.paseoHome, cwd: daemon.staticDir });
process.on("message", (message) => {
if (message === "finish") held.resolve();
if (message === "release-resume") resumeReleased.resolve();
if (message === "stop") {
void daemon.close().then(
() => process.exit(0),
@@ -52,6 +52,8 @@ interface TestPaseoDaemonOptions {
trustedProxies?: PaseoDaemonConfig["trustedProxies"];
agentProfiles?: AgentProfile[];
autoArchiveAfterMerge?: boolean;
resumeOnStartup?: boolean;
resumePrompt?: string;
pluginsEnabled?: PaseoDaemonConfig["pluginsEnabled"];
builtinPlugins?: BuiltinPluginLoader;
plugins?: PaseoDaemonConfig["plugins"];
@@ -226,6 +228,8 @@ async function prepareTestDaemonConfig(
downloadTokenTtlMs: options.downloadTokenTtlMs,
agentProfiles: options.agentProfiles,
autoArchiveAfterMerge: options.autoArchiveAfterMerge,
resumeOnStartup: options.resumeOnStartup,
resumePrompt: options.resumePrompt,
pluginsEnabled: options.pluginsEnabled,
plugins: options.plugins,
};
@@ -2013,6 +2013,7 @@ export class VoiceAssistantWebSocketServer {
skillManagement: true,
// COMPAT(idleRuntimeCleanupSetting): added in v0.13.0, remove gate after 2027-04-26.
idleRuntimeCleanupSetting: true,
resumeOnStartupSetting: true,
// COMPAT(terminalRestoreModes): added in v0.1.81, remove gate after 2026-11-23.
"terminal-restore-modes": true,
// COMPAT(terminalInputModeReplay): added in v0.2.6, remove gate after 2027-02-02.
@@ -1171,10 +1171,14 @@ describe("WorkspaceGitServiceImpl", () => {
const subscription = service.registerWorkspace({ cwd: REPO_CWD }, listener);
await vi.waitFor(() => {
expect(getCheckoutSnapshotFacts).toHaveBeenCalledTimes(1);
expect(service.getMetrics().fetchInFlightCount).toBe(1);
});
releaseFetch.resolve();
await flushPromises();
// The refresh debounce starts when the fetch result is applied, not when it is released.
await vi.waitFor(() => {
expect(service.getMetrics().fetchInFlightCount).toBe(0);
});
await vi.advanceTimersByTimeAsync(1_000);
await vi.waitFor(() => {
expect(getCheckoutRefDerivedState).toHaveBeenCalledTimes(1);