Make daemon shutdowns easier to diagnose (#1790)

* fix(daemon): log stop reasons and client identity

Record websocket client identity, process memory/uptime, and shutdown reasons across CLI, desktop, supervisor, and worker paths so daemon drops can be traced from the triggering client to worker termination.

* fix(daemon): keep shutdown diagnostics in sync

Test drift: supervisor and relay tests still asserted the old log text and metadata shape after shutdown diagnostics started logging structured reasons and relay connection ids. Update those assertions to the new diagnostic contract.

Also centralize client lifecycle reason normalization and derive desktop daemon stop reasons from one tuple so future changes cannot silently drift.
This commit is contained in:
Mohamed Boudra
2026-06-29 17:31:11 +02:00
committed by GitHub
parent b613bea9f6
commit e63a971968
27 changed files with 567 additions and 86 deletions

View File

@@ -59,6 +59,8 @@ export interface StopLocalDaemonResult {
home: string;
pid: number | null;
forced: boolean;
usedLifecycleRpc: boolean;
reason: "not_running" | "lifecycle_shutdown_rpc" | "owner_pid_signal" | "owner_pid_sigkill";
message: string;
}
@@ -230,6 +232,15 @@ function resolveStopMessage(
return fallbackMessage ?? "Daemon stopped via owner PID signal";
}
function resolveStopReason(
forced: boolean,
lifecycleRequested: boolean,
): StopLocalDaemonResult["reason"] {
if (forced) return "owner_pid_sigkill";
if (lifecycleRequested) return "lifecycle_shutdown_rpc";
return "owner_pid_signal";
}
function readPidFile(pidPath: string): LocalDaemonPidInfo | null {
try {
const parsed = JSON.parse(readFileSync(pidPath, "utf-8")) as Record<string, unknown>;
@@ -421,6 +432,8 @@ function createNotRunningStopResult(
home: state.home,
pid,
forced: false,
usedLifecycleRpc: false,
reason: "not_running",
message,
};
}
@@ -741,6 +754,8 @@ export async function stopLocalDaemon(
home: state.home,
pid,
forced,
usedLifecycleRpc: lifecycleRequested,
reason: resolveStopReason(forced, lifecycleRequested),
message: resolveStopMessage(forced, lifecycleRequested, fallbackMessage),
};
}

View File

@@ -16,6 +16,8 @@ interface StopResult {
home: string;
pid: string;
forced: boolean;
usedLifecycleRpc: boolean;
reason: "not_running" | "lifecycle_shutdown_rpc" | "owner_pid_signal" | "owner_pid_sigkill";
message: string;
}
@@ -75,6 +77,8 @@ export async function runStopCommand(
home: result.home,
pid: result.pid === null ? "-" : String(result.pid),
forced: result.forced,
usedLifecycleRpc: result.usedLifecycleRpc,
reason: result.reason,
message: result.message,
},
schema: stopResultSchema,

View File

@@ -84,6 +84,11 @@ async function readDaemonStatus(paseoHome: string): Promise<DaemonStatus> {
}
}
async function readCapturedSupervisorLogs(paseoHome: string, recentLogs: string): Promise<string> {
const durableLogs = await readFile(join(paseoHome, "daemon.log"), "utf8").catch(() => "");
return `${recentLogs}\n${durableLogs}`;
}
async function waitFor(
check: () => Promise<boolean> | boolean,
timeoutMs: number,
@@ -206,13 +211,20 @@ try {
"stopped",
"daemon should remain stopped after stop command",
);
const capturedSupervisorLogs = await readCapturedSupervisorLogs(paseoHome, recentSupervisorLogs);
assert(
recentSupervisorLogs.includes("Shutdown requested by worker. Stopping worker..."),
`stop should request lifecycle shutdown from daemon worker, logs:\n${recentSupervisorLogs}`,
capturedSupervisorLogs.includes('"msg":"Worker requested shutdown"') &&
capturedSupervisorLogs.includes('"reason":"client_shutdown_rpc"'),
`stop should log lifecycle shutdown reason from daemon worker, logs:\n${capturedSupervisorLogs}`,
);
assert(
!recentSupervisorLogs.includes("cli_shutdown"),
`supervisor logs should not route shutdown by reason string:\n${recentSupervisorLogs}`,
capturedSupervisorLogs.includes('"msg":"Supervisor sending signal to worker"') &&
capturedSupervisorLogs.includes('"signal":"SIGTERM"'),
`stop should log supervisor signal dispatch, logs:\n${capturedSupervisorLogs}`,
);
assert(
!capturedSupervisorLogs.includes("cli_shutdown"),
`supervisor logs should not route shutdown by reason string:\n${capturedSupervisorLogs}`,
);
console.log("✓ stop leaves supervised daemon stopped (no respawn)\n");
} finally {

View File

@@ -229,8 +229,14 @@ try {
);
const capturedSupervisorLogs = await readCapturedSupervisorLogs(paseoHome, recentSupervisorLogs);
assert(
capturedSupervisorLogs.includes("Restart requested by worker. Stopping worker for restart..."),
`restart should route through supervisor restart intent, logs:\n${capturedSupervisorLogs}`,
capturedSupervisorLogs.includes('"msg":"Worker requested restart"') &&
capturedSupervisorLogs.includes('"reason":"settings_update"'),
`restart should log lifecycle restart reason from daemon worker, logs:\n${capturedSupervisorLogs}`,
);
assert(
capturedSupervisorLogs.includes('"msg":"Supervisor sending signal to worker"') &&
capturedSupervisorLogs.includes('"signal":"SIGTERM"'),
`restart should log supervisor signal dispatch, logs:\n${capturedSupervisorLogs}`,
);
console.log("✓ app-style restart keeps daemon healthy and restarts worker\n");
} finally {

View File

@@ -144,17 +144,23 @@ try {
const parsed = JSON.parse(stopResult.stdout) as {
action?: unknown;
forced?: unknown;
usedLifecycleRpc?: unknown;
reason?: unknown;
message?: unknown;
};
assert.deepStrictEqual(
{
action: parsed.action,
forced: parsed.forced,
usedLifecycleRpc: parsed.usedLifecycleRpc,
reason: parsed.reason,
message: parsed.message,
},
{
action: "stopped",
forced: true,
usedLifecycleRpc: false,
reason: "owner_pid_sigkill",
message: "Daemon owner process was force-stopped",
},
`stop should report forced tree cleanup: ${stopResult.stdout}`,