mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-14 13:22:37 +00:00
Preserve daemon sessions when workers stall (#3263)
Stop treating heartbeat silence as proof that a live daemon worker should be destroyed. Crash recovery, explicit lifecycle requests, and supervisor-loss cleanup remain unchanged.
This commit is contained in:
@@ -10,15 +10,9 @@ import { resolveSupervisorLogFile } from "./supervisor-log-config.js";
|
||||
const repoRoot = path.resolve(fileURLToPath(new URL("../../..", import.meta.url)));
|
||||
const supervisorPath = fileURLToPath(new URL("./supervisor.ts", import.meta.url));
|
||||
|
||||
interface SupervisorSuspension {
|
||||
afterMs: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
async function runSupervisorFixture(options: {
|
||||
workerSource: string;
|
||||
restartOnCrash?: boolean;
|
||||
suspension?: SupervisorSuspension;
|
||||
timeoutMs?: number;
|
||||
}): Promise<{
|
||||
code: number | null;
|
||||
@@ -58,23 +52,10 @@ async function runSupervisorFixture(options: {
|
||||
const startedAt = Date.now();
|
||||
const child = spawn(process.execPath, ["--import", "tsx", runnerPath], {
|
||||
cwd: repoRoot,
|
||||
detached: options.suspension !== undefined,
|
||||
env: { ...process.env },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
if (options.suspension) {
|
||||
const supervisorPid = child.pid;
|
||||
if (supervisorPid === undefined) {
|
||||
throw new Error("Supervisor fixture did not start");
|
||||
}
|
||||
const { afterMs, durationMs } = options.suspension;
|
||||
setTimeout(() => {
|
||||
process.kill(-supervisorPid, "SIGSTOP");
|
||||
setTimeout(() => process.kill(-supervisorPid, "SIGCONT"), durationMs);
|
||||
}, afterMs);
|
||||
}
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.setEncoding("utf8");
|
||||
@@ -208,46 +189,21 @@ describe("supervisor durable logging", () => {
|
||||
expect(result.log).toContain('"workerPid":');
|
||||
});
|
||||
|
||||
test("keeps a worker alive while heartbeats continue", async () => {
|
||||
test("does not restart a worker based on heartbeat absence", async () => {
|
||||
const result = await runSupervisorFixture({
|
||||
timeoutMs: 10_000,
|
||||
workerSource: `
|
||||
setInterval(() => {
|
||||
process.send?.({ type: "paseo:worker-heartbeat" });
|
||||
}, 250);
|
||||
setTimeout(() => {
|
||||
process.send?.({ type: "paseo:shutdown", reason: "healthy_heartbeat_test_complete" });
|
||||
}, 6000);
|
||||
`,
|
||||
});
|
||||
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.signal).toBeNull();
|
||||
expect(result.log).toContain('"reason":"healthy_heartbeat_test_complete"');
|
||||
expect(result.log).not.toContain('"msg":"Worker heartbeat timed out; restarting worker"');
|
||||
}, 10_000);
|
||||
|
||||
test("tolerates a transient seven-second heartbeat pause", async () => {
|
||||
const result = await runSupervisorFixture({
|
||||
timeoutMs: 10_000,
|
||||
timeoutMs: 20_000,
|
||||
workerSource: `
|
||||
import { existsSync, writeFileSync } from "node:fs";
|
||||
|
||||
const marker = process.argv[1] + ".started";
|
||||
if (!existsSync(marker)) {
|
||||
writeFileSync(marker, "started");
|
||||
let heartbeatCount = 0;
|
||||
const heartbeat = setInterval(() => {
|
||||
process.send?.({ type: "paseo:worker-heartbeat" });
|
||||
heartbeatCount += 1;
|
||||
if (heartbeatCount === 3) clearInterval(heartbeat);
|
||||
}, 100);
|
||||
setTimeout(() => {
|
||||
process.send?.({ type: "paseo:shutdown", reason: "heartbeat_pause_tolerated" });
|
||||
}, 7_000);
|
||||
process.send?.({ type: "paseo:shutdown", reason: "silent_worker_test_complete" });
|
||||
}, 16_000);
|
||||
setInterval(() => {}, 1_000);
|
||||
} else {
|
||||
process.send?.({ type: "paseo:shutdown", reason: "unexpected_heartbeat_restart" });
|
||||
process.send?.({ type: "paseo:shutdown", reason: "unexpected_silent_worker_restart" });
|
||||
setInterval(() => {}, 1_000);
|
||||
}
|
||||
`,
|
||||
@@ -255,44 +211,10 @@ describe("supervisor durable logging", () => {
|
||||
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.signal).toBeNull();
|
||||
expect(result.log).toContain('"reason":"heartbeat_pause_tolerated"');
|
||||
expect(result.log).not.toContain('"reason":"unexpected_heartbeat_restart"');
|
||||
expect(result.log).toContain('"reason":"silent_worker_test_complete"');
|
||||
expect(result.log).not.toContain('"reason":"unexpected_silent_worker_restart"');
|
||||
expect(result.log).not.toContain('"msg":"Worker heartbeat timed out; restarting worker"');
|
||||
}, 10_000);
|
||||
|
||||
test.skipIf(isPlatform("win32"))(
|
||||
"keeps a healthy worker alive after the host resumes from sleep",
|
||||
async () => {
|
||||
const result = await runSupervisorFixture({
|
||||
suspension: { afterMs: 1_000, durationMs: 16_000 },
|
||||
timeoutMs: 25_000,
|
||||
workerSource: `
|
||||
import { existsSync, writeFileSync } from "node:fs";
|
||||
|
||||
const marker = process.argv[1] + ".started";
|
||||
if (!existsSync(marker)) {
|
||||
writeFileSync(marker, "started");
|
||||
setInterval(() => {
|
||||
process.send?.({ type: "paseo:worker-heartbeat" });
|
||||
}, 250);
|
||||
setTimeout(() => {
|
||||
process.send?.({ type: "paseo:shutdown", reason: "sleep_resume_test_complete" });
|
||||
}, 17_000);
|
||||
} else {
|
||||
process.send?.({ type: "paseo:shutdown", reason: "unexpected_sleep_restart" });
|
||||
}
|
||||
setInterval(() => {}, 1_000);
|
||||
`,
|
||||
});
|
||||
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.signal).toBeNull();
|
||||
expect(result.log).toContain('"reason":"sleep_resume_test_complete"');
|
||||
expect(result.log).not.toContain('"reason":"unexpected_sleep_restart"');
|
||||
expect(result.log).not.toContain('"msg":"Worker heartbeat timed out; restarting worker"');
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
}, 25_000);
|
||||
|
||||
test.skipIf(isPlatform("win32"))(
|
||||
"forces shutdown when a worker ignores SIGTERM",
|
||||
@@ -315,42 +237,6 @@ describe("supervisor durable logging", () => {
|
||||
20_000,
|
||||
);
|
||||
|
||||
// POSIX-only: the watchdog uses SIGKILL after its graceful shutdown window.
|
||||
test.skipIf(isPlatform("win32"))(
|
||||
"restarts a worker that stops heartbeating",
|
||||
async () => {
|
||||
const result = await runSupervisorFixture({
|
||||
timeoutMs: 35_000,
|
||||
workerSource: `
|
||||
import { existsSync, writeFileSync } from "node:fs";
|
||||
|
||||
const marker = process.argv[1] + ".started";
|
||||
if (!existsSync(marker)) {
|
||||
writeFileSync(marker, "started");
|
||||
let heartbeatCount = 0;
|
||||
const heartbeat = setInterval(() => {
|
||||
process.send?.({ type: "paseo:worker-heartbeat" });
|
||||
heartbeatCount += 1;
|
||||
if (heartbeatCount === 3) clearInterval(heartbeat);
|
||||
}, 100);
|
||||
process.on("SIGTERM", () => {});
|
||||
setInterval(() => {}, 1000);
|
||||
} else {
|
||||
process.send?.({ type: "paseo:shutdown", reason: "watchdog_test_complete" });
|
||||
setInterval(() => {}, 1000);
|
||||
}
|
||||
`,
|
||||
});
|
||||
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.signal).toBeNull();
|
||||
expect(result.log).toContain('"msg":"Worker heartbeat timed out; restarting worker"');
|
||||
expect(result.log).toContain('"msg":"Worker did not exit after SIGTERM; forcing SIGKILL"');
|
||||
expect(result.log).toContain('"signal":"SIGKILL"');
|
||||
},
|
||||
40_000,
|
||||
);
|
||||
|
||||
test.skipIf(isPlatform("win32"))(
|
||||
"restarts after worker exit while a descendant retains the worker stdio",
|
||||
async () => {
|
||||
|
||||
@@ -5,7 +5,6 @@ import { createStream as createRotatingFileStream } from "rotating-file-stream";
|
||||
import { signalProcessTree } from "../src/utils/tree-kill.js";
|
||||
|
||||
const WORKER_HEARTBEAT_INTERVAL_MS = 1_000;
|
||||
const WORKER_HEARTBEAT_MISSED_CHECK_LIMIT = 15;
|
||||
const WORKER_TERMINATION_GRACE_MS = 10_000;
|
||||
|
||||
interface SupervisorLogFileOptions {
|
||||
@@ -34,10 +33,6 @@ interface SupervisorHeartbeatMessage {
|
||||
type: "paseo:supervisor-heartbeat";
|
||||
}
|
||||
|
||||
interface WorkerHeartbeatMessage {
|
||||
type: "paseo:worker-heartbeat";
|
||||
}
|
||||
|
||||
interface SupervisorOptions {
|
||||
name: string;
|
||||
startupMessage: string;
|
||||
@@ -93,15 +88,6 @@ function parseLifecycleMessage(msg: unknown): WorkerLifecycleMessage | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function isWorkerHeartbeatMessage(msg: unknown): msg is WorkerHeartbeatMessage {
|
||||
return (
|
||||
typeof msg === "object" &&
|
||||
msg !== null &&
|
||||
"type" in msg &&
|
||||
(msg as { type?: unknown }).type === "paseo:worker-heartbeat"
|
||||
);
|
||||
}
|
||||
|
||||
function toRotatingFileStreamSize(size: string): string {
|
||||
const trimmed = size.trim();
|
||||
const match = trimmed.match(/^(\d+)\s*([bBkKmMgG])?$/);
|
||||
@@ -250,7 +236,6 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
}
|
||||
|
||||
const currentChild = child;
|
||||
let missedWorkerHeartbeatChecks = 0;
|
||||
const heartbeat = setInterval(() => {
|
||||
const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" };
|
||||
if (currentChild.connected) {
|
||||
@@ -267,23 +252,6 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
}, WORKER_HEARTBEAT_INTERVAL_MS);
|
||||
heartbeat.unref();
|
||||
|
||||
const workerWatchdog = setInterval(() => {
|
||||
if (child !== currentChild || restarting || shuttingDown) {
|
||||
return;
|
||||
}
|
||||
missedWorkerHeartbeatChecks += 1;
|
||||
if (missedWorkerHeartbeatChecks < WORKER_HEARTBEAT_MISSED_CHECK_LIMIT) {
|
||||
return;
|
||||
}
|
||||
writeLifecycleLog("Worker heartbeat timed out; restarting worker", {
|
||||
missedHeartbeatChecks: missedWorkerHeartbeatChecks,
|
||||
supervisorPid: process.pid,
|
||||
workerPid: currentChild.pid ?? null,
|
||||
});
|
||||
requestRestart("worker_heartbeat_timeout");
|
||||
}, WORKER_HEARTBEAT_INTERVAL_MS);
|
||||
workerWatchdog.unref();
|
||||
|
||||
child.on("disconnect", () => {
|
||||
writeLifecycleLog("Worker IPC channel disconnected");
|
||||
});
|
||||
@@ -299,10 +267,6 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
});
|
||||
|
||||
child.on("message", (msg: unknown) => {
|
||||
if (isWorkerHeartbeatMessage(msg)) {
|
||||
missedWorkerHeartbeatChecks = 0;
|
||||
return;
|
||||
}
|
||||
const lifecycleMessage = parseLifecycleMessage(msg);
|
||||
if (!lifecycleMessage) {
|
||||
return;
|
||||
@@ -333,7 +297,6 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
|
||||
child.on("exit", (code, signal) => {
|
||||
clearInterval(heartbeat);
|
||||
clearInterval(workerWatchdog);
|
||||
clearForceKillTimer();
|
||||
const exitDescriptor = describeExit(code, signal);
|
||||
writeLifecycleLog("Worker exited", { code, signal, exit: exitDescriptor });
|
||||
|
||||
@@ -27,10 +27,6 @@ interface SupervisorHeartbeatMessage {
|
||||
type: "paseo:supervisor-heartbeat";
|
||||
}
|
||||
|
||||
interface WorkerHeartbeatMessage {
|
||||
type: "paseo:worker-heartbeat";
|
||||
}
|
||||
|
||||
interface BootstrapResult {
|
||||
paseoHome: string;
|
||||
logger: ReturnType<typeof createRootLogger>;
|
||||
@@ -232,13 +228,6 @@ async function main() {
|
||||
const supervisorPid = process.ppid;
|
||||
let lastSupervisorHeartbeatAt = Date.now();
|
||||
let supervisorExitRequested = false;
|
||||
const sendWorkerHeartbeat = () => {
|
||||
try {
|
||||
process.send?.({ type: "paseo:worker-heartbeat" } satisfies WorkerHeartbeatMessage);
|
||||
} catch {
|
||||
// The disconnect handler below owns supervisor-loss shutdown.
|
||||
}
|
||||
};
|
||||
const exitAfterSupervisorLoss = (reason: string) => {
|
||||
if (supervisorExitRequested) {
|
||||
return;
|
||||
@@ -272,10 +261,6 @@ async function main() {
|
||||
});
|
||||
process.on("disconnect", () => exitAfterSupervisorLoss("ipc_disconnect_event"));
|
||||
|
||||
sendWorkerHeartbeat();
|
||||
const workerHeartbeat = setInterval(sendWorkerHeartbeat, 1_000);
|
||||
workerHeartbeat.unref();
|
||||
|
||||
const timer = setInterval(() => {
|
||||
const ipcConnected = typeof process.connected === "boolean" ? process.connected : true;
|
||||
const heartbeatExpired = Date.now() - lastSupervisorHeartbeatAt > 3500;
|
||||
|
||||
@@ -6,9 +6,7 @@ user-invocable: true
|
||||
|
||||
# Committee Skill
|
||||
|
||||
Two agents from contrasting profiles, fresh context, planning a solution in parallel. They stay alive for review after implementation.
|
||||
|
||||
The purpose is to step back, not double down. The committee may propose a completely different approach.
|
||||
Two agents from contrasting profiles, fresh context, planning a solution in parallel.
|
||||
|
||||
**User's additional context:** $ARGUMENTS
|
||||
|
||||
@@ -35,49 +33,14 @@ If the user names profiles, use those. If fewer than two suitable profiles are c
|
||||
This is analysis only. Do NOT edit, create, or delete any files. Do NOT write code.
|
||||
```
|
||||
|
||||
- **Trust the wait.** Do not poll, send hurry-ups, or interrupt. GPT-5.4 can reason 15–30 minutes; Opus does extended thinking. Long waits mean it found something worth thinking about.
|
||||
- **You are the middleman.** Drive plan → implement → review without yielding to the user, except for divergences that need their call.
|
||||
- **Trust the finish notification.** Do not poll, send hurry-ups, or interrupt. Models can reason for 15–30 minutes. You can go idle and Paseo will notify you.
|
||||
|
||||
## Phase 1: Plan
|
||||
## Workflow
|
||||
|
||||
Write a problem-level prompt:
|
||||
1. Write a problem-level prompt
|
||||
2. Create both agents in parallel via Paseo with `[Committee] <task>` titles and the same prompt
|
||||
3. Wait for both responses
|
||||
4. Resolve disagreements by passing their arguments between each other
|
||||
5. Keep going until they converge into a response
|
||||
|
||||
- High-level goal and acceptance criteria
|
||||
- Constraints
|
||||
- Symptoms (if a bug)
|
||||
- What you tried and why it failed
|
||||
- Explicit: "do root cause analysis"
|
||||
- Explicit: "state assumptions, ask why three levels deep, check whether you're patching a symptom or removing the problem"
|
||||
|
||||
Create both agents in parallel via Paseo with `[Committee] <task>` titles and the same prompt. Wait for both — not just whichever finishes first.
|
||||
|
||||
Read both responses. Challenge them — do not accept at face value:
|
||||
|
||||
- "Why does <underlying thing> happen? Symptom or cause?"
|
||||
- Verify any assumption the plan makes about the code.
|
||||
- "What did you consider and reject?"
|
||||
|
||||
Send follow-ups until the plan addresses root cause.
|
||||
|
||||
Synthesize:
|
||||
|
||||
- Convergence → unified plan.
|
||||
- Significant divergence → involve the user.
|
||||
|
||||
Confirm the merged plan with both members. Multi-turn until consensus.
|
||||
|
||||
## Phase 2: Implement
|
||||
|
||||
Default: implement yourself. If the user said **"delegate"**, launch one impl agent and pass the merged plan.
|
||||
|
||||
The committee stays clean — not involved in implementation.
|
||||
|
||||
## Phase 3: Review
|
||||
|
||||
Send the diff to the committee:
|
||||
|
||||
> Implementation is done. Review changes against the plan. Flag drift or missing pieces. <no-edits suffix>
|
||||
|
||||
Apply feedback yourself, or send to the impl agent. Repeat 2 → 3 until consensus.
|
||||
|
||||
After ~10 iterations without convergence, start a fresh committee with the full history of what was tried — the current committee's context may have drifted too far.
|
||||
Share the consensus with the user. Summarize where the agents diverged and how they resolved it.
|
||||
|
||||
@@ -62,6 +62,4 @@ Prepare the handoff in a dedicated workspace:
|
||||
|
||||
Do not encode independence as a create mode and do not invoke CLI or wire-level detach operations. Detach is a user gesture in the subagents track.
|
||||
|
||||
Leave `notifyOnFinish` omitted unless the user explicitly wants no callback.
|
||||
|
||||
Don't wait by default — the user decides whether to follow along or move on. Tell them the agent ID and how to follow along (the paseo skill explains).
|
||||
Do not wait or poll for the agent to finish.
|
||||
|
||||
@@ -3,7 +3,7 @@ name: paseo
|
||||
description: Paseo reference for managing workspaces, workspace scripts, agents, schedules, and heartbeats.
|
||||
---
|
||||
|
||||
Paseo is a daemon that supervises AI coding agents on your machine. Control it through tools or a CLI.
|
||||
Paseo is a remote daemon that manages coding agents, terminals. Control it through MCP tools or the CLI.
|
||||
|
||||
## Workspaces
|
||||
|
||||
@@ -13,7 +13,7 @@ Paseo is a daemon that supervises AI coding agents on your machine. Control it t
|
||||
|
||||
**`archive_workspace`** — `{ workspaceId }`. Archives the workspace, its agents, and its terminals. Local directories remain; Paseo removes an owned worktree only after its final active workspace reference is archived.
|
||||
|
||||
Worktree creation and reference accounting are implementation details of `isolation: "worktree"`.
|
||||
**`rename_workspace`** — `{ workspaceId, name }`. Rename workspace.
|
||||
|
||||
## Workspace scripts
|
||||
|
||||
|
||||
Reference in New Issue
Block a user