From 893e3376b0c3ef56878a89879123b6848a1e91bd Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sun, 7 Jun 2026 23:18:48 +0700 Subject: [PATCH] Recover stale supervised daemons Teach daemon stop to use lifecycle shutdown when the API is reachable even if the owner pid is stale, then wait for the API to disappear and clean the stale pidfile. Launch desktop-managed daemons detached from desktop stdio, preserve stale reachable daemon ownership for version checks, and allow desktop stop/restart to use the CLI recovery path. Add supervisor heartbeats so supervised workers shut down when their supervisor disappears instead of surviving as orphaned reachable daemons. --- .../cli/src/commands/daemon/local-daemon.ts | 185 ++++++++++++---- .../34-daemon-stop-stale-reachable.test.ts | 207 ++++++++++++++++++ ...aemon-worker-supervisor-disconnect.test.ts | 192 ++++++++++++++++ .../desktop/src/daemon/daemon-manager.test.ts | 166 ++++++++++++-- packages/desktop/src/daemon/daemon-manager.ts | 64 +----- packages/server/scripts/supervisor.ts | 14 ++ packages/server/src/server/daemon-worker.ts | 56 +++++ 7 files changed, 769 insertions(+), 115 deletions(-) create mode 100644 packages/cli/tests/34-daemon-stop-stale-reachable.test.ts create mode 100644 packages/cli/tests/35-daemon-worker-supervisor-disconnect.test.ts diff --git a/packages/cli/src/commands/daemon/local-daemon.ts b/packages/cli/src/commands/daemon/local-daemon.ts index f153df13f..3968d3eed 100644 --- a/packages/cli/src/commands/daemon/local-daemon.ts +++ b/packages/cli/src/commands/daemon/local-daemon.ts @@ -1,5 +1,5 @@ import { spawnSync, type ChildProcess } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, unlinkSync } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { loadConfig, resolvePaseoHome, spawnProcess } from "@getpaseo/server"; @@ -351,6 +351,123 @@ async function waitForPidExit(pid: number, timeoutMs: number): Promise return poll(); } +async function waitForDaemonUnreachable( + state: LocalDaemonState, + timeoutMs: number, +): Promise { + const host = resolveTcpHostFromListen(state.listen); + if (!host) { + return true; + } + + const reachableHost = host; + const deadline = Date.now() + timeoutMs; + async function poll(): Promise { + const client = await tryConnectToDaemon({ host: reachableHost, timeout: 500 }); + if (!client) { + return true; + } + await client.close().catch(() => undefined); + if (Date.now() >= deadline) { + const finalClient = await tryConnectToDaemon({ + host: reachableHost, + timeout: PID_POLL_INTERVAL_MS, + }); + if (!finalClient) { + return true; + } + await finalClient.close().catch(() => undefined); + return false; + } + await sleep(PID_POLL_INTERVAL_MS); + return poll(); + } + + return poll(); +} + +function removeStalePidFile(state: LocalDaemonState): void { + if (!state.stalePidFile) { + return; + } + + try { + unlinkSync(state.pidPath); + } catch { + // Best-effort cleanup only. The successful lifecycle stop is authoritative. + } +} + +function createNotRunningStopResult( + state: LocalDaemonState, + pid: number | null, + message: string, +): StopLocalDaemonResult { + return { + action: "not_running", + home: state.home, + pid, + forced: false, + message, + }; +} + +function createStopTimeoutError( + state: LocalDaemonState, + pid: number | null, + timeoutMs: number, +): Error { + if (!state.running) { + const host = resolveTcpHostFromListen(state.listen); + return new Error( + `Timed out waiting for daemon${host ? ` at ${host}` : ""} to stop after ${Math.ceil( + timeoutMs / 1000, + )}s`, + ); + } + return new Error( + `Timed out waiting for daemon PID ${pid} to stop after ${Math.ceil(timeoutMs / 1000)}s`, + ); +} + +async function signalDaemonOwnerForStop( + state: LocalDaemonState, + pid: number | null, +): Promise { + if (pid === null) { + return createNotRunningStopResult(state, null, "Daemon is not running"); + } + + const signaled = await signalProcessTreeOrOwnerSafely(pid, "SIGTERM"); + if (signaled) { + return null; + } + + return createNotRunningStopResult(state, pid, "Daemon process was already stopped"); +} + +async function waitForStopAfterRequest(args: { + state: LocalDaemonState; + pid: number | null; + timeoutMs: number; + killTimeoutMs: number; + force?: boolean; +}): Promise<{ stopped: boolean; forced: boolean }> { + const { state, pid, timeoutMs, killTimeoutMs, force } = args; + let stopped = + state.running && pid !== null + ? await waitForPidExit(pid, timeoutMs) + : await waitForDaemonUnreachable(state, timeoutMs); + + if (!stopped && force && state.running && pid !== null) { + await signalProcessTreeOrOwnerSafely(pid, "SIGKILL"); + stopped = await waitForPidExit(pid, killTimeoutMs); + return { stopped, forced: true }; + } + + return { stopped, forced: false }; +} + type LifecycleShutdownAttempt = { requested: true } | { requested: false; reason: string }; function getErrorMessage(error: unknown): string { @@ -567,49 +684,41 @@ export async function stopLocalDaemon( const killTimeoutMs = options.killTimeoutMs ?? DEFAULT_KILL_TIMEOUT_MS; const state = resolveLocalDaemonState({ home: options.home }); - if (!state.pidInfo || !state.running) { - const staleSuffix = - state.stalePidFile && state.pidInfo ? ` (stale PID file for ${state.pidInfo.pid})` : ""; - return { - action: "not_running", - home: state.home, - pid: state.pidInfo?.pid ?? null, - forced: false, - message: `Daemon is not running${staleSuffix}`, - }; - } - - const pid = state.pidInfo.pid; const shutdownAttempt = await requestLifecycleShutdown(state, timeoutMs); const lifecycleRequested = shutdownAttempt.requested; - const fallbackMessage = shutdownAttempt.requested ? null : shutdownAttempt.reason; - let forced = false; - if (!lifecycleRequested) { - const signaled = await signalProcessTreeOrOwnerSafely(pid, "SIGTERM"); - if (!signaled) { - return { - action: "not_running", - home: state.home, - pid, - forced: false, - message: "Daemon process was already stopped", - }; - } - } - let stopped = await waitForPidExit(pid, timeoutMs); - if (!stopped && options.force) { - forced = true; - await signalProcessTreeOrOwnerSafely(pid, "SIGKILL"); - stopped = await waitForPidExit(pid, killTimeoutMs); - } - - if (!stopped) { - throw new Error( - `Timed out waiting for daemon PID ${pid} to stop after ${Math.ceil(timeoutMs / 1000)}s`, + if (!state.pidInfo || (!state.running && !lifecycleRequested)) { + const staleSuffix = + state.stalePidFile && state.pidInfo ? ` (stale PID file for ${state.pidInfo.pid})` : ""; + return createNotRunningStopResult( + state, + state.pidInfo?.pid ?? null, + `Daemon is not running${staleSuffix}`, ); } + const pid = state.pidInfo?.pid ?? null; + const fallbackMessage = shutdownAttempt.requested ? null : shutdownAttempt.reason; + if (!lifecycleRequested) { + const notRunningResult = await signalDaemonOwnerForStop(state, pid); + if (notRunningResult) return notRunningResult; + } + + const { stopped, forced } = await waitForStopAfterRequest({ + state, + pid, + timeoutMs, + killTimeoutMs, + force: options.force, + }); + if (!stopped) { + throw createStopTimeoutError(state, pid, timeoutMs); + } + + if (lifecycleRequested) { + removeStalePidFile(state); + } + return { action: "stopped", home: state.home, diff --git a/packages/cli/tests/34-daemon-stop-stale-reachable.test.ts b/packages/cli/tests/34-daemon-stop-stale-reachable.test.ts new file mode 100644 index 000000000..91d789d1d --- /dev/null +++ b/packages/cli/tests/34-daemon-stop-stale-reachable.test.ts @@ -0,0 +1,207 @@ +#!/usr/bin/env npx tsx + +/** + * Regression: `paseo daemon stop` must stop a reachable daemon even when the + * local pid file points at a dead supervisor owner. + */ + +import assert from "node:assert"; +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { $ } from "zx"; +import { getAvailablePort } from "./helpers/network.ts"; + +$.verbose = false; + +const pollIntervalMs = 100; +const testEnv = { + PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD: process.env.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD ?? "0", + PASEO_DICTATION_ENABLED: process.env.PASEO_DICTATION_ENABLED ?? "0", + PASEO_VOICE_MODE_ENABLED: process.env.PASEO_VOICE_MODE_ENABLED ?? "0", +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isProcessRunning(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) { + return false; + } + + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitFor( + check: () => Promise | boolean, + timeoutMs: number, + message: string, +): Promise { + const deadline = Date.now() + timeoutMs; + + async function poll(): Promise { + if (await check()) return; + if (Date.now() >= deadline) throw new Error(message); + await sleep(pollIntervalMs); + return poll(); + } + + return poll(); +} + +interface DaemonStatus { + localDaemon: string | null; + connectedDaemon: string | null; + pid: number | null; +} + +async function readDaemonStatus(paseoHome: string): Promise { + const result = + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon status --home ${paseoHome} --json`.nothrow(); + if (result.exitCode !== 0) { + return { localDaemon: null, connectedDaemon: null, pid: null }; + } + + try { + const parsed = JSON.parse(result.stdout) as { + localDaemon?: unknown; + connectedDaemon?: unknown; + pid?: unknown; + }; + return { + localDaemon: typeof parsed.localDaemon === "string" ? parsed.localDaemon : null, + connectedDaemon: typeof parsed.connectedDaemon === "string" ? parsed.connectedDaemon : null, + pid: + typeof parsed.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0 + ? parsed.pid + : null, + }; + } catch { + return { localDaemon: null, connectedDaemon: null, pid: null }; + } +} + +function findUnusedPid(): number { + for (let pid = 999_999; pid > 900_000; pid--) { + if (!isProcessRunning(pid)) { + return pid; + } + } + throw new Error("Unable to find unused pid for stale pid fixture"); +} + +console.log("=== Daemon Stop (stale pid, reachable worker regression) ===\n"); + +const port = await getAvailablePort(); +const paseoHome = await mkdtemp(join(tmpdir(), "paseo-stop-stale-reachable-")); +const cliRoot = join(import.meta.dirname, ".."); +const host = `127.0.0.1:${port}`; +const pidPath = join(paseoHome, "paseo.pid"); +const stalePid = findUnusedPid(); + +let workerProcess: ChildProcess | null = null; + +try { + console.log("Test 1: start daemon worker with stale supervisor pid file"); + + await writeFile( + pidPath, + `${JSON.stringify( + { + pid: stalePid, + startedAt: new Date().toISOString(), + hostname: "stale-supervisor-fixture.local", + uid: typeof process.getuid === "function" ? process.getuid() : undefined, + listen: host, + }, + null, + 2, + )}\n`, + ); + + workerProcess = spawn( + process.execPath, + ["--import", "tsx", "../server/src/server/daemon-worker.ts"], + { + cwd: cliRoot, + env: { + ...process.env, + ...testEnv, + PASEO_HOME: paseoHome, + PASEO_LISTEN: host, + PASEO_RELAY_ENABLED: "false", + CI: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + await waitFor( + async () => { + const status = await readDaemonStatus(paseoHome); + return status.localDaemon === "stale_pid" && status.connectedDaemon === "reachable"; + }, + 120000, + "daemon did not enter stale_pid + reachable state in time", + ); + + const statusBeforeStop = await readDaemonStatus(paseoHome); + assert.strictEqual(statusBeforeStop.pid, stalePid, "status should report the stale owner pid"); + assert(workerProcess.pid && isProcessRunning(workerProcess.pid), "worker should be running"); + console.log(`✓ fixture has stale pid ${stalePid} and live worker ${workerProcess.pid}\n`); + + console.log( + "Test 2: `paseo daemon stop` should stop reachable worker instead of saying not_running", + ); + const stopResult = + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon stop --home ${paseoHome} --json`.nothrow(); + assert.strictEqual(stopResult.exitCode, 0, `stop should succeed: ${stopResult.stderr}`); + const stopJson = JSON.parse(stopResult.stdout) as { + action?: unknown; + pid?: unknown; + message?: unknown; + }; + assert.strictEqual(stopJson.action, "stopped", "stop should report stopped action"); + assert.strictEqual( + stopJson.pid, + String(stalePid), + "stop should report the stale pid it recovered from", + ); + assert.strictEqual( + stopJson.message, + "Daemon stopped gracefully", + "stop should route through lifecycle shutdown", + ); + + await waitFor( + () => !isProcessRunning(workerProcess?.pid ?? -1), + 15000, + "worker remained running after stop", + ); + assert.strictEqual(existsSync(pidPath), false, "stale pid file should be removed after stop"); + console.log("✓ stop recovered stale supervisor pid state\n"); +} finally { + if (workerProcess?.pid && isProcessRunning(workerProcess.pid)) { + workerProcess.kill("SIGTERM"); + await waitFor( + () => !isProcessRunning(workerProcess!.pid ?? -1), + 5000, + "worker cleanup timed out", + ).catch(() => { + workerProcess?.kill("SIGKILL"); + }); + } + + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon stop --home ${paseoHome} --force`.nothrow(); + await rm(paseoHome, { recursive: true, force: true }); +} + +console.log("=== Stale reachable stop regression test passed ==="); diff --git a/packages/cli/tests/35-daemon-worker-supervisor-disconnect.test.ts b/packages/cli/tests/35-daemon-worker-supervisor-disconnect.test.ts new file mode 100644 index 000000000..fea7133ed --- /dev/null +++ b/packages/cli/tests/35-daemon-worker-supervisor-disconnect.test.ts @@ -0,0 +1,192 @@ +#!/usr/bin/env npx tsx + +/** + * Regression: a supervised daemon worker must exit when its supervisor IPC + * channel closes, instead of becoming an orphaned daemon. + */ + +import assert from "node:assert"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { $ } from "zx"; +import { getAvailablePort } from "./helpers/network.ts"; + +$.verbose = false; + +const pollIntervalMs = 100; +const testEnv = { + PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD: process.env.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD ?? "0", + PASEO_DICTATION_ENABLED: process.env.PASEO_DICTATION_ENABLED ?? "0", + PASEO_VOICE_MODE_ENABLED: process.env.PASEO_VOICE_MODE_ENABLED ?? "0", +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isProcessRunning(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) { + return false; + } + + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function readWorkerPid(supervisorPid: number): number | null { + if (!Number.isInteger(supervisorPid) || supervisorPid <= 0) { + return null; + } + + const result = spawnSync("ps", ["ax", "-o", "pid=,ppid="], { encoding: "utf8" }); + if (result.status !== 0 || result.error) { + return null; + } + + for (const line of result.stdout.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + const [pidToken, ppidToken] = trimmed.split(/\s+/); + const pid = Number.parseInt(pidToken ?? "", 10); + const ppid = Number.parseInt(ppidToken ?? "", 10); + if (ppid === supervisorPid && pid > 0) { + return pid; + } + } + + return null; +} + +async function waitFor( + check: () => Promise | boolean, + timeoutMs: number, + message: string, +): Promise { + const deadline = Date.now() + timeoutMs; + + async function poll(): Promise { + if (await check()) return; + if (Date.now() >= deadline) throw new Error(message); + await sleep(pollIntervalMs); + return poll(); + } + + return poll(); +} + +interface DaemonStatus { + localDaemon: string | null; + pid: number | null; +} + +async function readDaemonStatus(paseoHome: string): Promise { + const result = + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon status --home ${paseoHome} --json`.nothrow(); + if (result.exitCode !== 0) { + return { localDaemon: null, pid: null }; + } + + try { + const parsed = JSON.parse(result.stdout) as { localDaemon?: unknown; pid?: unknown }; + return { + localDaemon: typeof parsed.localDaemon === "string" ? parsed.localDaemon : null, + pid: + typeof parsed.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0 + ? parsed.pid + : null, + }; + } catch { + return { localDaemon: null, pid: null }; + } +} + +console.log("=== Daemon Worker Supervisor Disconnect Regression ===\n"); + +const port = await getAvailablePort(); +const paseoHome = await mkdtemp(join(tmpdir(), "paseo-worker-supervisor-disconnect-")); +const cliRoot = join(import.meta.dirname, ".."); + +let supervisorProcess: ChildProcess | null = null; +let recentSupervisorLogs = ""; + +try { + console.log("Test 1: start supervised daemon with isolated PASEO_HOME"); + + supervisorProcess = spawn( + process.execPath, + ["--import", "tsx", "../server/scripts/supervisor-entrypoint.ts", "--dev"], + { + cwd: cliRoot, + env: { + ...process.env, + ...testEnv, + PASEO_HOME: paseoHome, + PASEO_LISTEN: `127.0.0.1:${port}`, + PASEO_RELAY_ENABLED: "false", + CI: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + supervisorProcess.stdout?.on("data", (chunk) => { + recentSupervisorLogs = (recentSupervisorLogs + chunk.toString()).slice(-8000); + }); + supervisorProcess.stderr?.on("data", (chunk) => { + recentSupervisorLogs = (recentSupervisorLogs + chunk.toString()).slice(-8000); + }); + + await waitFor( + async () => { + const status = await readDaemonStatus(paseoHome); + return ( + status.localDaemon === "running" && status.pid !== null && isProcessRunning(status.pid) + ); + }, + 120000, + "daemon did not become running in time", + ); + + const statusBeforeKill = await readDaemonStatus(paseoHome); + const supervisorPid = statusBeforeKill.pid; + assert(supervisorPid !== null, "supervisor pid should exist once daemon starts"); + const workerPid = readWorkerPid(supervisorPid); + assert(workerPid !== null, "supervisor should have a worker process"); + assert(isProcessRunning(workerPid), "worker process should be running"); + console.log(`✓ daemon running with supervisor ${supervisorPid} and worker ${workerPid}\n`); + + console.log("Test 2: killing supervisor should make worker exit via IPC disconnect"); + supervisorProcess.kill("SIGKILL"); + await waitFor( + () => !isProcessRunning(supervisorPid), + 15000, + "supervisor remained running after SIGKILL", + ); + await waitFor( + () => !isProcessRunning(workerPid), + 15000, + "worker remained running after supervisor IPC disconnect", + ); + console.log("✓ worker exited after supervisor disconnect\n"); +} finally { + if (supervisorProcess?.pid && isProcessRunning(supervisorProcess.pid)) { + supervisorProcess.kill("SIGKILL"); + } + + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon stop --home ${paseoHome} --force`.nothrow(); + await rm(paseoHome, { recursive: true, force: true }); +} + +if (recentSupervisorLogs.trim().length === 0) { + console.log("(no supervisor logs captured)"); +} + +console.log("=== Worker supervisor disconnect regression test passed ==="); diff --git a/packages/desktop/src/daemon/daemon-manager.test.ts b/packages/desktop/src/daemon/daemon-manager.test.ts index e2393b6c0..d7a6e78d6 100644 --- a/packages/desktop/src/daemon/daemon-manager.test.ts +++ b/packages/desktop/src/daemon/daemon-manager.test.ts @@ -1,10 +1,12 @@ import { EventEmitter } from "node:events"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_DESKTOP_SETTINGS } from "../settings/desktop-settings"; import { createDaemonCommandHandlers } from "./daemon-manager"; const mocks = vi.hoisted(() => ({ + paseoHome: "/tmp/paseo-desktop-daemon-manager-test-home", settings: { releaseChannel: "stable", daemon: { @@ -21,7 +23,7 @@ vi.mock("electron", () => ({ app: { getPath: vi.fn(() => "/tmp/paseo-user-data"), getVersion: vi.fn(() => "1.2.3"), - isPackaged: false, + isPackaged: true, }, ipcMain: { handle: vi.fn() }, powerMonitor: { getSystemIdleTime: vi.fn(() => 0) }, @@ -32,7 +34,7 @@ vi.mock("electron-log/main", () => ({ })); vi.mock("@getpaseo/server", () => ({ - resolvePaseoHome: vi.fn(() => "/tmp/paseo-home"), + resolvePaseoHome: vi.fn(() => mocks.paseoHome), spawnProcess: mocks.spawnProcess, })); @@ -72,8 +74,6 @@ function desktopSettingsWithManagement(enabled: boolean) { } type MockChildProcess = EventEmitter & { - stdout: EventEmitter; - stderr: EventEmitter; pid: number; spawnfile: string; spawnargs: string[]; @@ -82,8 +82,6 @@ type MockChildProcess = EventEmitter & { function createMockChildProcess(): MockChildProcess { const child = new EventEmitter() as MockChildProcess; - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); child.pid = 1234; child.spawnfile = "node"; child.spawnargs = ["node", "daemon.js"]; @@ -91,10 +89,8 @@ function createMockChildProcess(): MockChildProcess { return child; } -function scheduleFailedStartupOutput(child: MockChildProcess): void { +function scheduleFailedStartup(child: MockChildProcess): void { setImmediate(() => { - child.stdout.emit("data", Buffer.from(`${"x".repeat(80_000)}stdout-tail`)); - child.stderr.emit("data", Buffer.from(`${"y".repeat(80_000)}stderr-tail`)); child.emit("exit", 1, null); }); } @@ -105,6 +101,11 @@ describe("daemon-manager commands", () => { mocks.runExternalCliJsonCommand.mockReset(); mocks.runExternalCliTextCommand.mockReset(); mocks.spawnProcess.mockReset(); + rmSync(mocks.paseoHome, { recursive: true, force: true }); + }); + + afterEach(() => { + rmSync(mocks.paseoHome, { recursive: true, force: true }); }); it("refuses start and restart while built-in daemon management is disabled", async () => { @@ -136,7 +137,7 @@ describe("daemon-manager commands", () => { listen: null, hostname: null, pid: null, - home: "/tmp/paseo-home", + home: mocks.paseoHome, version: null, desktopManaged: false, error: null, @@ -167,7 +168,7 @@ describe("daemon-manager commands", () => { listen: null, hostname: null, pid: null, - home: "/tmp/paseo-home", + home: mocks.paseoHome, version: null, desktopManaged: false, error: null, @@ -195,7 +196,50 @@ describe("daemon-manager commands", () => { ]); }); - it("uses a reachable daemon when the PID file is stale", async () => { + it("routes stale reachable desktop daemon stops through external CLI daemon stop", async () => { + mocks.runExternalCliJsonCommand + .mockResolvedValueOnce({ + localDaemon: "stale_pid", + connectedDaemon: "reachable", + serverId: "server-1", + pid: 7675, + listen: "127.0.0.1:6767", + daemonVersion: "1.2.2", + desktopManaged: true, + }) + .mockResolvedValueOnce({ action: "stopped" }) + .mockResolvedValueOnce({ + localDaemon: "stopped", + connectedDaemon: "unreachable", + serverId: "", + }); + const handlers = createDaemonCommandHandlers(); + + await expect(handlers.stop_desktop_daemon()).resolves.toEqual({ + serverId: "", + status: "stopped", + listen: null, + hostname: null, + pid: null, + home: mocks.paseoHome, + version: null, + desktopManaged: false, + error: null, + }); + + expect(mocks.runExternalCliJsonCommand).toHaveBeenNthCalledWith(2, [ + "daemon", + "stop", + "--json", + "--timeout", + "5", + "--force", + "--kill-timeout", + "5", + ]); + }); + + it("uses a stale reachable desktop daemon when the version matches", async () => { mocks.runExternalCliJsonCommand.mockResolvedValue({ localDaemon: "stale_pid", connectedDaemon: "reachable", @@ -203,7 +247,7 @@ describe("daemon-manager commands", () => { pid: 7675, listen: "127.0.0.1:6767", hostname: "dev-host", - daemonVersion: "1.2.2", + daemonVersion: "1.2.3", desktopManaged: true, }); const handlers = createDaemonCommandHandlers(); @@ -214,16 +258,86 @@ describe("daemon-manager commands", () => { listen: "127.0.0.1:6767", hostname: "dev-host", pid: null, - home: "/tmp/paseo-home", - version: "1.2.2", - desktopManaged: false, + home: mocks.paseoHome, + version: "1.2.3", + desktopManaged: true, error: null, }); expect(mocks.spawnProcess).not.toHaveBeenCalled(); }); - it("bounds captured daemon startup output", async () => { + it("restarts a stale reachable desktop daemon when the version differs", async () => { + mocks.runExternalCliJsonCommand + .mockResolvedValueOnce({ + localDaemon: "stale_pid", + connectedDaemon: "reachable", + serverId: "server-1", + pid: 7675, + listen: "127.0.0.1:6767", + hostname: "dev-host", + daemonVersion: "1.2.2", + desktopManaged: true, + }) + .mockResolvedValueOnce({ + localDaemon: "stale_pid", + connectedDaemon: "reachable", + serverId: "server-1", + pid: 7675, + listen: "127.0.0.1:6767", + daemonVersion: "1.2.2", + desktopManaged: true, + }) + .mockResolvedValueOnce({ action: "stopped" }) + .mockResolvedValueOnce({ + localDaemon: "stopped", + connectedDaemon: "unreachable", + serverId: "", + }) + .mockResolvedValueOnce({ + localDaemon: "running", + connectedDaemon: "reachable", + serverId: "server-2", + pid: 8888, + listen: "127.0.0.1:6767", + hostname: "dev-host", + daemonVersion: "1.2.3", + desktopManaged: true, + }); + mocks.spawnProcess.mockReturnValue(createMockChildProcess()); + const handlers = createDaemonCommandHandlers(); + + await expect(handlers.start_desktop_daemon()).resolves.toEqual({ + serverId: "server-2", + status: "running", + listen: "127.0.0.1:6767", + hostname: "dev-host", + pid: 8888, + home: mocks.paseoHome, + version: "1.2.3", + desktopManaged: true, + error: null, + }); + + expect(mocks.runExternalCliJsonCommand).toHaveBeenNthCalledWith(3, [ + "daemon", + "stop", + "--json", + "--timeout", + "5", + "--force", + "--kill-timeout", + "5", + ]); + expect(mocks.spawnProcess).toHaveBeenCalled(); + }); + + it("starts the managed daemon detached from desktop stdio and reports daemon log failures", async () => { + mkdirSync(mocks.paseoHome, { recursive: true }); + writeFileSync( + `${mocks.paseoHome}/daemon.log`, + ["old log line", "recent daemon failure"].join("\n"), + ); mocks.runExternalCliJsonCommand.mockResolvedValue({ localDaemon: "stopped", connectedDaemon: "unreachable", @@ -231,7 +345,7 @@ describe("daemon-manager commands", () => { }); mocks.spawnProcess.mockImplementation(() => { const child = createMockChildProcess(); - scheduleFailedStartupOutput(child); + scheduleFailedStartup(child); return child; }); const handlers = createDaemonCommandHandlers(); @@ -246,9 +360,15 @@ describe("daemon-manager commands", () => { expect(thrown).toBeInstanceOf(Error); const message = thrown?.message ?? ""; expect(message).toContain("Daemon failed to start: exit code 1"); - expect(message).toContain("output truncated to the last 65536 chars"); - expect(message).toContain("stdout-tail"); - expect(message).toContain("stderr-tail"); - expect(message.length).toBeLessThan(150_000); + expect(message).toContain(`Recent logs (${mocks.paseoHome}/daemon.log):`); + expect(message).toContain("recent daemon failure"); + expect(mocks.spawnProcess).toHaveBeenCalledWith( + "node", + [], + expect.objectContaining({ + detached: true, + stdio: ["ignore", "ignore", "ignore"], + }), + ); }); }); diff --git a/packages/desktop/src/daemon/daemon-manager.ts b/packages/desktop/src/daemon/daemon-manager.ts index e24940583..3e21f5dfe 100644 --- a/packages/desktop/src/daemon/daemon-manager.ts +++ b/packages/desktop/src/daemon/daemon-manager.ts @@ -43,7 +43,6 @@ const DAEMON_LOG_FILENAME = "daemon.log"; const STARTUP_POLL_INTERVAL_MS = 200; const STARTUP_POLL_MAX_ATTEMPTS = 150; const DETACHED_STARTUP_GRACE_MS = 1200; -const STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS = 64 * 1024; type DesktopDaemonState = "starting" | "running" | "stopped" | "errored"; @@ -70,11 +69,6 @@ interface DesktopPairingOffer { qr: string | null; } -interface StartupOutputCapture { - text: string; - truncated: boolean; -} - function parseReleaseChannel( args: Record | undefined, ): AppReleaseChannel | undefined { @@ -151,30 +145,6 @@ function tailFile(filePath: string, lines = 50): string { } } -function createStartupOutputCapture(): StartupOutputCapture { - return { text: "", truncated: false }; -} - -function appendStartupOutput(capture: StartupOutputCapture, chunk: Buffer): StartupOutputCapture { - const nextText = capture.text + chunk.toString(); - if (nextText.length <= STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS) { - return { text: nextText, truncated: capture.truncated }; - } - - return { - text: nextText.slice(-STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS), - truncated: true, - }; -} - -function formatStartupOutput(capture: StartupOutputCapture): string { - if (!capture.truncated) { - return capture.text; - } - - return `[output truncated to the last ${STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS} chars]\n${capture.text}`; -} - function logDesktopDaemonLifecycle(message: string, details?: Record): void { log.info("[desktop daemon]", message, { pid: process.pid, @@ -231,6 +201,7 @@ export async function resolveDesktopDaemonStatus(): Promise typeof payload.connectedDaemon === "string" ? payload.connectedDaemon : "not_probed"; const hasRunningLocalProcess = localDaemon === "running"; const hasLocalProcess = hasRunningLocalProcess || localDaemon === "unresponsive"; + const desktopManaged = payload.desktopManaged === true; const apiReachable = connectedDaemon === "reachable"; let status: DesktopDaemonState = "stopped"; if (apiReachable || hasRunningLocalProcess) { @@ -248,7 +219,7 @@ export async function resolveDesktopDaemonStatus(): Promise pid: hasLocalProcess && typeof payload.pid === "number" ? payload.pid : null, home, version: typeof payload.daemonVersion === "string" ? payload.daemonVersion : null, - desktopManaged: hasRunningLocalProcess && payload.desktopManaged === true, + desktopManaged, error: null, }; } catch (error) { @@ -287,19 +258,15 @@ function assertBuiltInDaemonManagementEnabled(settings: DesktopSettings): void { } } -function buildStartupFailureError( - result: { code: number | null; signal: string | null; error?: Error }, - stdout: StartupOutputCapture, - stderr: StartupOutputCapture, -): Error { +function buildStartupFailureError(result: { + code: number | null; + signal: string | null; + error?: Error; +}): Error { const reason = result.error ? result.error.message : `exit code ${result.code ?? "unknown"}${result.signal ? ` (${result.signal})` : ""}`; const parts = [`Daemon failed to start: ${reason}`]; - const formattedStderr = formatStartupOutput(stderr).trim(); - const formattedStdout = formatStartupOutput(stdout).trim(); - if (formattedStderr) parts.push(`stderr:\n${formattedStderr}`); - if (formattedStdout) parts.push(`stdout:\n${formattedStdout}`); const logs = tailFile(logFilePath(), 15); if (logs) parts.push(`Recent logs (${logFilePath()}):\n${logs}`); return new Error(parts.join("\n\n")); @@ -377,16 +344,7 @@ async function startDaemon(): Promise { envMode: "internal", env: invocation.env, envOverlay: { PASEO_DESKTOP_MANAGED: "1" }, - stdio: ["ignore", "pipe", "pipe"], - }); - - let stdout = createStartupOutputCapture(); - let stderr = createStartupOutputCapture(); - child.stdout!.on("data", (data: Buffer) => { - stdout = appendStartupOutput(stdout, data); - }); - child.stderr!.on("data", (data: Buffer) => { - stderr = appendStartupOutput(stderr, data); + stdio: ["ignore", "ignore", "ignore"], }); logDesktopDaemonLifecycle("detached spawn returned", { @@ -424,8 +382,6 @@ async function startDaemon(): Promise { logDesktopDaemonLifecycle("detached startup grace period completed", { childPid: child.pid ?? null, exitedEarly: result.exitedEarly, - stdout: formatStartupOutput(stdout).slice(0, 2000), - stderr: formatStartupOutput(stderr).slice(0, 2000), ...(result.exitedEarly ? { exitCode: result.code, @@ -436,7 +392,7 @@ async function startDaemon(): Promise { }); if (result.exitedEarly) { - throw buildStartupFailureError(result, stdout, stderr); + throw buildStartupFailureError(result); } return pollForRunningDaemon(); @@ -444,7 +400,7 @@ async function startDaemon(): Promise { export async function stopDesktopDaemon(): Promise { const status = await resolveDesktopDaemonStatus(); - if (status.status !== "running" || !status.pid) return status; + if (status.status !== "running") return status; await stopDesktopDaemonViaCli(); return await resolveDesktopDaemonStatus(); diff --git a/packages/server/scripts/supervisor.ts b/packages/server/scripts/supervisor.ts index a50643e5a..c9c0158b3 100644 --- a/packages/server/scripts/supervisor.ts +++ b/packages/server/scripts/supervisor.ts @@ -24,6 +24,10 @@ type WorkerLifecycleMessage = reason?: string; }; +interface SupervisorHeartbeatMessage { + type: "paseo:supervisor-heartbeat"; +} + interface SupervisorOptions { name: string; startupMessage: string; @@ -183,6 +187,15 @@ export function runSupervisor(options: SupervisorOptions): void { }); } + const currentChild = child; + const heartbeat = setInterval(() => { + const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" }; + if (currentChild.connected) { + currentChild.send?.(message, () => undefined); + } + }, 1000); + heartbeat.unref(); + child.stdout?.on("data", (chunk: Buffer) => { process.stdout.write(chunk); writeDurableChunk(chunk); @@ -224,6 +237,7 @@ export function runSupervisor(options: SupervisorOptions): void { }); child.on("close", (code, signal) => { + clearInterval(heartbeat); const exitDescriptor = describeExit(code, signal); writeLifecycleLog("Worker exited", { code, signal, exit: exitDescriptor }); diff --git a/packages/server/src/server/daemon-worker.ts b/packages/server/src/server/daemon-worker.ts index 2a74cd46e..34c082552 100644 --- a/packages/server/src/server/daemon-worker.ts +++ b/packages/server/src/server/daemon-worker.ts @@ -19,12 +19,28 @@ type SupervisorLifecycleMessage = reason?: string; }; +interface SupervisorHeartbeatMessage { + type: "paseo:supervisor-heartbeat"; +} + interface BootstrapResult { paseoHome: string; logger: ReturnType; config: ReturnType; } +function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + if (typeof err === "object" && err !== null && "code" in err && err.code === "EPERM") { + return true; + } + return false; + } +} + function bootstrapFromEnvironment(): BootstrapResult { try { const paseoHome = resolvePaseoHome(); @@ -150,6 +166,46 @@ async function main() { beginShutdown("restart lifecycle intent", { successExitCode: 0 }); }; + const installSupervisorLivenessGuard = () => { + if (typeof process.send !== "function") { + return; + } + + const supervisorPid = process.ppid; + let lastSupervisorHeartbeatAt = Date.now(); + let supervisorShutdownRequested = false; + const requestSupervisorShutdown = (reason: string) => { + if (supervisorShutdownRequested) { + return; + } + supervisorShutdownRequested = true; + beginShutdown(reason); + }; + + process.on("message", (message: unknown) => { + if ( + typeof message === "object" && + message !== null && + "type" in message && + (message as SupervisorHeartbeatMessage).type === "paseo:supervisor-heartbeat" + ) { + lastSupervisorHeartbeatAt = Date.now(); + } + }); + process.on("disconnect", () => requestSupervisorShutdown("supervisor disconnect")); + + const timer = setInterval(() => { + const ipcConnected = typeof process.connected === "boolean" ? process.connected : true; + const heartbeatExpired = Date.now() - lastSupervisorHeartbeatAt > 3500; + if (ipcConnected === false || !isPidAlive(supervisorPid) || heartbeatExpired) { + requestSupervisorShutdown("supervisor disconnect"); + } + }, 1000); + timer.unref(); + }; + + installSupervisorLivenessGuard(); + try { daemon = await createPaseoDaemon( {