From b0954d1616129a4d113eb75b4e177001507b40f2 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 8 Jun 2026 23:02:36 +0700 Subject: [PATCH] Log daemon supervisor exit reasons --- packages/server/scripts/supervisor.ts | 14 ++++- packages/server/src/server/daemon-worker.ts | 62 +++++++++++++++++---- 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/packages/server/scripts/supervisor.ts b/packages/server/scripts/supervisor.ts index c9c0158b3..1f8a78656 100644 --- a/packages/server/scripts/supervisor.ts +++ b/packages/server/scripts/supervisor.ts @@ -191,11 +191,23 @@ export function runSupervisor(options: SupervisorOptions): void { const heartbeat = setInterval(() => { const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" }; if (currentChild.connected) { - currentChild.send?.(message, () => undefined); + currentChild.send?.(message, (error) => { + if (error) { + writeLifecycleLog("Worker heartbeat IPC send failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + }); + } else { + writeLifecycleLog("Worker heartbeat skipped because IPC channel is disconnected"); } }, 1000); heartbeat.unref(); + child.on("disconnect", () => { + writeLifecycleLog("Worker IPC channel disconnected"); + }); + child.stdout?.on("data", (chunk: Buffer) => { process.stdout.write(chunk); writeDurableChunk(chunk); diff --git a/packages/server/src/server/daemon-worker.ts b/packages/server/src/server/daemon-worker.ts index ff094da23..c9cb89354 100644 --- a/packages/server/src/server/daemon-worker.ts +++ b/packages/server/src/server/daemon-worker.ts @@ -1,3 +1,5 @@ +import { appendFileSync, mkdirSync } from "node:fs"; +import path from "node:path"; import { createPaseoDaemon } from "./bootstrap.js"; import { loadConfig } from "./config.js"; import { resolvePaseoHome } from "./paseo-home.js"; @@ -41,6 +43,31 @@ function isPidAlive(pid: number): boolean { } } +function writeWorkerLifecycleLog( + paseoHome: string, + message: string, + fields: Record = {}, +): void { + try { + const logPath = path.join(paseoHome, "daemon.log"); + mkdirSync(path.dirname(logPath), { recursive: true }); + appendFileSync( + logPath, + `${JSON.stringify({ + level: "warn", + time: new Date().toISOString(), + pid: process.pid, + name: "DaemonWorker", + msg: message, + ...fields, + })}\n`, + "utf8", + ); + } catch { + // Exit-reason logging must never prevent the worker from exiting. + } +} + function bootstrapFromEnvironment(): BootstrapResult { try { const paseoHome = resolvePaseoHome(); @@ -70,7 +97,7 @@ function applyCliFlagOverrides(config: ReturnType): void { } async function main() { - const { logger, config } = bootstrapFromEnvironment(); + const { paseoHome, logger, config } = bootstrapFromEnvironment(); let daemon: Awaited> | null = null; let shutdownPromise: Promise | null = null; let exitHookInstalled = false; @@ -174,12 +201,20 @@ async function main() { const supervisorPid = process.ppid; let lastSupervisorHeartbeatAt = Date.now(); let supervisorExitRequested = false; - const exitAfterSupervisorLoss = () => { + const exitAfterSupervisorLoss = (reason: string) => { if (supervisorExitRequested) { return; } supervisorExitRequested = true; + writeWorkerLifecycleLog(paseoHome, "Supervisor liveness lost; worker exiting", { + reason, + supervisorPid, + currentParentPid: process.ppid, + ipcConnected: typeof process.connected === "boolean" ? process.connected : null, + heartbeatAgeMs: Date.now() - lastSupervisorHeartbeatAt, + }); + // The supervisor owns the worker's stdout/stderr pipes. Once it is gone, // logging during graceful shutdown can block on the broken pipe and leave // the daemon orphaned, so supervisor loss is a hard process boundary. @@ -196,19 +231,26 @@ async function main() { lastSupervisorHeartbeatAt = Date.now(); } }); - process.on("disconnect", exitAfterSupervisorLoss); + process.on("disconnect", () => exitAfterSupervisorLoss("ipc_disconnect_event")); const timer = setInterval(() => { const ipcConnected = typeof process.connected === "boolean" ? process.connected : true; const heartbeatExpired = Date.now() - lastSupervisorHeartbeatAt > 3500; const supervisorChanged = process.ppid !== supervisorPid; - if ( - ipcConnected === false || - supervisorChanged || - !isPidAlive(supervisorPid) || - heartbeatExpired - ) { - exitAfterSupervisorLoss(); + if (ipcConnected === false) { + exitAfterSupervisorLoss("ipc_disconnected"); + return; + } + if (supervisorChanged) { + exitAfterSupervisorLoss("supervisor_parent_pid_changed"); + return; + } + if (!isPidAlive(supervisorPid)) { + exitAfterSupervisorLoss("supervisor_pid_dead"); + return; + } + if (heartbeatExpired) { + exitAfterSupervisorLoss("supervisor_heartbeat_expired"); } }, 1000); timer.unref();