mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-06 08:24:46 +00:00
fix(daemon): keep lifecycle operations on the selected instance
Restart reconstructed launch inputs and address fallback could redirect a home-selected shutdown to another daemon. Let supervisors publish bound endpoints, retain their worker restart contract, and share captured local lifecycle ownership between CLI and Desktop.
This commit is contained in:
@@ -18,13 +18,11 @@ process.title = "Paseo Supervisor";
|
||||
|
||||
interface DaemonRunnerConfig {
|
||||
devMode: boolean;
|
||||
reclaimStalePidLock: boolean;
|
||||
workerArgs: string[];
|
||||
}
|
||||
|
||||
function parseConfig(argv: string[]): DaemonRunnerConfig {
|
||||
let devMode = false;
|
||||
let reclaimStalePidLock = false;
|
||||
const workerArgs: string[] = [];
|
||||
|
||||
for (const arg of argv) {
|
||||
@@ -33,13 +31,14 @@ function parseConfig(argv: string[]): DaemonRunnerConfig {
|
||||
continue;
|
||||
}
|
||||
if (arg === "--reclaim-stale-pid-lock") {
|
||||
reclaimStalePidLock = true;
|
||||
continue;
|
||||
throw new Error(
|
||||
"--reclaim-stale-pid-lock was removed: stop the existing supervisor before starting another.",
|
||||
);
|
||||
}
|
||||
workerArgs.push(arg);
|
||||
}
|
||||
|
||||
return { devMode, reclaimStalePidLock, workerArgs };
|
||||
return { devMode, workerArgs };
|
||||
}
|
||||
|
||||
function resolveWorkerEntry(): string {
|
||||
@@ -116,7 +115,6 @@ async function main(): Promise<void> {
|
||||
try {
|
||||
await acquirePidLock(paseoHome, null, {
|
||||
ownerPid: process.pid,
|
||||
reclaimStaleDesktopLock: config.reclaimStalePidLock,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof PidLockError) {
|
||||
@@ -177,6 +175,7 @@ async function main(): Promise<void> {
|
||||
onWorkerReady: async ({ listen }) => {
|
||||
await updatePidLock(paseoHome, { listen }, { ownerPid: process.pid });
|
||||
},
|
||||
onWorkerExit: () => updatePidLock(paseoHome, { listen: null }, { ownerPid: process.pid }),
|
||||
onSupervisorExit: releaseLock,
|
||||
});
|
||||
requestSupervisorShutdown = supervisor.requestShutdown;
|
||||
|
||||
@@ -36,7 +36,10 @@ async function runSupervisorFixture(options: {
|
||||
const workerPath = path.join(tempDir, "worker.mjs");
|
||||
const runnerPath = path.join(tempDir, "runner.mjs");
|
||||
|
||||
await writeFile(workerPath, options.workerSource);
|
||||
await writeFile(
|
||||
workerPath,
|
||||
`process.send?.({ type: "paseo:ready", listen: "fixture" });\n${options.workerSource}`,
|
||||
);
|
||||
await writeFile(
|
||||
runnerPath,
|
||||
`
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { expect, test } from "vitest";
|
||||
|
||||
async function runWorker(worker: string, publicationDelay = 0) {
|
||||
const home = await mkdtemp(path.join(tmpdir(), "paseo-readiness-"));
|
||||
const workerPath = path.join(home, "worker.mjs");
|
||||
const runnerPath = path.join(home, "runner.mjs");
|
||||
const eventsPath = path.join(home, "events.jsonl");
|
||||
const supervisor = new URL("./supervisor.ts", import.meta.url).href;
|
||||
await writeFile(workerPath, worker);
|
||||
await writeFile(
|
||||
runnerPath,
|
||||
`
|
||||
import { appendFile } from "node:fs/promises";
|
||||
import { runSupervisor } from ${JSON.stringify(supervisor)};
|
||||
const record = (event) => appendFile(${JSON.stringify(eventsPath)}, JSON.stringify(event) + "\\n");
|
||||
runSupervisor({
|
||||
name: "ReadinessTest", startupMessage: "starting", restartOnCrash: true,
|
||||
resolveWorkerEntry: () => ${JSON.stringify(workerPath)}, workerArgs: [], workerExecArgv: [],
|
||||
onWorkerReady: async ({ listen }) => {
|
||||
await new Promise(resolve => setTimeout(resolve, ${publicationDelay}));
|
||||
await record(listen);
|
||||
},
|
||||
onWorkerExit: () => record(null),
|
||||
});
|
||||
`,
|
||||
);
|
||||
const env = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([key]) => !key.startsWith("PASEO_")),
|
||||
);
|
||||
const child = spawn(process.execPath, ["--import", "tsx", runnerPath], {
|
||||
cwd: fileURLToPath(new URL("../../../", import.meta.url)),
|
||||
env: { ...env, HOME: home, PASEO_HOME: home },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let output = "";
|
||||
child.stdout.on("data", (chunk) => {
|
||||
output += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
output += chunk;
|
||||
});
|
||||
try {
|
||||
const code = await new Promise<number | null>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
reject(new Error(output));
|
||||
}, 5_000);
|
||||
child.once("error", reject);
|
||||
child.once("close", (exitCode) => {
|
||||
clearTimeout(timer);
|
||||
resolve(exitCode);
|
||||
});
|
||||
});
|
||||
const events = await readFile(eventsPath, "utf8").catch(() => "");
|
||||
return {
|
||||
code,
|
||||
output,
|
||||
events: events
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line)),
|
||||
};
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("a worker that exits before ready fails the supervisor without respawning", async () => {
|
||||
const result = await runWorker(`
|
||||
import { existsSync, writeFileSync } from "node:fs";
|
||||
const marker = process.argv[1] + ".started";
|
||||
if (existsSync(marker)) process.exit(0);
|
||||
writeFileSync(marker, "started");
|
||||
process.exit(1);
|
||||
`);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.output).not.toContain("Restarting worker");
|
||||
});
|
||||
|
||||
test("a replacement worker must independently reach ready", async () => {
|
||||
const result = await runWorker(`
|
||||
import { existsSync, writeFileSync } from "node:fs";
|
||||
const marker = process.argv[1] + ".started";
|
||||
if (existsSync(marker)) process.exit(0);
|
||||
writeFileSync(marker, "started");
|
||||
process.on("message", message => {
|
||||
if (message.type === "paseo:graceful-shutdown") process.exit(0);
|
||||
});
|
||||
process.send({ type: "paseo:ready", listen: "test-endpoint" });
|
||||
process.send({ type: "paseo:restart" });
|
||||
`);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.events, result.output).toEqual(["test-endpoint", null, null]);
|
||||
});
|
||||
|
||||
test("exit clears publication after an in-flight ready write", async () => {
|
||||
const result = await runWorker(
|
||||
`
|
||||
process.send({ type: "paseo:ready", listen: "test-endpoint" }, () => process.exit(0));
|
||||
`,
|
||||
100,
|
||||
);
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.events.at(-1)).toBeNull();
|
||||
});
|
||||
|
||||
test("requested shutdown before first readiness exits successfully", async () => {
|
||||
const result = await runWorker(`
|
||||
process.on("message", message => {
|
||||
if (message.type === "paseo:graceful-shutdown") process.exit(0);
|
||||
});
|
||||
process.send({ type: "paseo:shutdown", reason: "cancelled_start" });
|
||||
`);
|
||||
expect(result.code).toBe(0);
|
||||
expect(result.events).toEqual([null]);
|
||||
});
|
||||
@@ -51,6 +51,7 @@ interface SupervisorOptions {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
} | null;
|
||||
onWorkerReady?: (message: { listen: string }) => Promise<void> | void;
|
||||
onWorkerExit?: () => Promise<void> | void;
|
||||
restartOnCrash?: boolean;
|
||||
onSupervisorExit?: () => Promise<void> | void;
|
||||
logFile?: SupervisorLogFileOptions;
|
||||
@@ -129,6 +130,8 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
let restarting = false;
|
||||
let shuttingDown = false;
|
||||
let exiting = false;
|
||||
let lifecycleFailed = false;
|
||||
let publication = Promise.resolve();
|
||||
let forceKillTimer: NodeJS.Timeout | null = null;
|
||||
const logStream = createSupervisorLogStream(options.logFile);
|
||||
|
||||
@@ -244,6 +247,8 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
}
|
||||
|
||||
const currentChild = child;
|
||||
let reachedReady = false;
|
||||
// Serialize endpoint writes with exit/clear before allowing another worker to spawn.
|
||||
const heartbeat = setInterval(() => {
|
||||
const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" };
|
||||
if (currentChild.connected) {
|
||||
@@ -276,18 +281,21 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
|
||||
child.on("message", (msg: unknown) => {
|
||||
const lifecycleMessage = parseLifecycleMessage(msg);
|
||||
if (!lifecycleMessage) {
|
||||
if (!lifecycleMessage || child !== currentChild) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (lifecycleMessage.type === "paseo:ready") {
|
||||
reachedReady = true;
|
||||
writeLifecycleLog("Worker ready", { listen: lifecycleMessage.listen });
|
||||
Promise.resolve(options.onWorkerReady?.({ listen: lifecycleMessage.listen })).catch(
|
||||
(error) => {
|
||||
publication = publication
|
||||
.then(() => options.onWorkerReady?.({ listen: lifecycleMessage.listen }))
|
||||
.catch((error) => {
|
||||
lifecycleFailed = true;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
log(`Worker ready callback failed: ${message}`);
|
||||
},
|
||||
);
|
||||
requestShutdown("endpoint_publication_failed");
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -303,35 +311,57 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
requestRestart(reason);
|
||||
});
|
||||
|
||||
child.on("error", (error) => {
|
||||
lifecycleFailed = true;
|
||||
log(`Worker spawn failed: ${error.message}`);
|
||||
exitSupervisor(1);
|
||||
});
|
||||
|
||||
child.on("exit", (code, signal) => {
|
||||
clearInterval(heartbeat);
|
||||
clearForceKillTimer();
|
||||
const exitDescriptor = describeExit(code, signal);
|
||||
writeLifecycleLog("Worker exited", { code, signal, exit: exitDescriptor });
|
||||
child = null;
|
||||
publication = publication
|
||||
.then(() => options.onWorkerExit?.())
|
||||
.catch((error) => {
|
||||
lifecycleFailed = true;
|
||||
log(`Worker exit callback failed: ${String(error)}`);
|
||||
});
|
||||
void publication.then(() => {
|
||||
const exitDescriptor = describeExit(code, signal);
|
||||
writeLifecycleLog("Worker exited", { code, signal, exit: exitDescriptor });
|
||||
|
||||
if (shuttingDown) {
|
||||
log(`Worker exited (${exitDescriptor}). Supervisor shutting down.`);
|
||||
exitSupervisor(0);
|
||||
if (lifecycleFailed || (!reachedReady && !shuttingDown)) {
|
||||
log(`Worker exited before successful readiness (${exitDescriptor}). Supervisor exiting.`);
|
||||
exitSupervisor(1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (shuttingDown) {
|
||||
log(`Worker exited (${exitDescriptor}). Supervisor shutting down.`);
|
||||
exitSupervisor(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const crashed =
|
||||
restartOnCrash &&
|
||||
((code !== 0 && code !== null) || (signal !== null && signal !== "SIGTERM"));
|
||||
|
||||
if (restarting || crashed) {
|
||||
restarting = false;
|
||||
log(
|
||||
crashed
|
||||
? `Worker crashed (${exitDescriptor}). Restarting worker...`
|
||||
: `Worker exited (${exitDescriptor}). Restarting worker...`,
|
||||
);
|
||||
spawnWorker();
|
||||
return;
|
||||
}
|
||||
|
||||
log(`Worker exited (${exitDescriptor}). Supervisor exiting.`);
|
||||
exitSupervisor(typeof code === "number" ? code : 1);
|
||||
return;
|
||||
}
|
||||
|
||||
const crashed =
|
||||
restartOnCrash &&
|
||||
((code !== 0 && code !== null) || (signal !== null && signal !== "SIGTERM"));
|
||||
|
||||
if (restarting || crashed) {
|
||||
restarting = false;
|
||||
log(
|
||||
crashed
|
||||
? `Worker crashed (${exitDescriptor}). Restarting worker...`
|
||||
: `Worker exited (${exitDescriptor}). Restarting worker...`,
|
||||
);
|
||||
spawnWorker();
|
||||
return;
|
||||
}
|
||||
|
||||
log(`Worker exited (${exitDescriptor}). Supervisor exiting.`);
|
||||
exitSupervisor(typeof code === "number" ? code : 1);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
@@ -389,7 +419,10 @@ export function runSupervisor(options: SupervisorOptions): SupervisorController
|
||||
writeLifecycleLog("Supervisor shutdown requested", { reason });
|
||||
log(`${reason}. Stopping worker...`);
|
||||
if (!child) {
|
||||
exitSupervisor(0);
|
||||
void publication.then(() => {
|
||||
exitSupervisor(lifecycleFailed ? 1 : 0);
|
||||
return;
|
||||
});
|
||||
return;
|
||||
}
|
||||
requestWorkerShutdown(reason);
|
||||
|
||||
Reference in New Issue
Block a user