diff --git a/packages/cli/src/commands/daemon/local-daemon.ts b/packages/cli/src/commands/daemon/local-daemon.ts index f153df13f..3968d3eed 100644 --- a/packages/cli/src/commands/daemon/local-daemon.ts +++ b/packages/cli/src/commands/daemon/local-daemon.ts @@ -1,5 +1,5 @@ import { spawnSync, type ChildProcess } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readFileSync, unlinkSync } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { loadConfig, resolvePaseoHome, spawnProcess } from "@getpaseo/server"; @@ -351,6 +351,123 @@ async function waitForPidExit(pid: number, timeoutMs: number): Promise return poll(); } +async function waitForDaemonUnreachable( + state: LocalDaemonState, + timeoutMs: number, +): Promise { + const host = resolveTcpHostFromListen(state.listen); + if (!host) { + return true; + } + + const reachableHost = host; + const deadline = Date.now() + timeoutMs; + async function poll(): Promise { + const client = await tryConnectToDaemon({ host: reachableHost, timeout: 500 }); + if (!client) { + return true; + } + await client.close().catch(() => undefined); + if (Date.now() >= deadline) { + const finalClient = await tryConnectToDaemon({ + host: reachableHost, + timeout: PID_POLL_INTERVAL_MS, + }); + if (!finalClient) { + return true; + } + await finalClient.close().catch(() => undefined); + return false; + } + await sleep(PID_POLL_INTERVAL_MS); + return poll(); + } + + return poll(); +} + +function removeStalePidFile(state: LocalDaemonState): void { + if (!state.stalePidFile) { + return; + } + + try { + unlinkSync(state.pidPath); + } catch { + // Best-effort cleanup only. The successful lifecycle stop is authoritative. + } +} + +function createNotRunningStopResult( + state: LocalDaemonState, + pid: number | null, + message: string, +): StopLocalDaemonResult { + return { + action: "not_running", + home: state.home, + pid, + forced: false, + message, + }; +} + +function createStopTimeoutError( + state: LocalDaemonState, + pid: number | null, + timeoutMs: number, +): Error { + if (!state.running) { + const host = resolveTcpHostFromListen(state.listen); + return new Error( + `Timed out waiting for daemon${host ? ` at ${host}` : ""} to stop after ${Math.ceil( + timeoutMs / 1000, + )}s`, + ); + } + return new Error( + `Timed out waiting for daemon PID ${pid} to stop after ${Math.ceil(timeoutMs / 1000)}s`, + ); +} + +async function signalDaemonOwnerForStop( + state: LocalDaemonState, + pid: number | null, +): Promise { + if (pid === null) { + return createNotRunningStopResult(state, null, "Daemon is not running"); + } + + const signaled = await signalProcessTreeOrOwnerSafely(pid, "SIGTERM"); + if (signaled) { + return null; + } + + return createNotRunningStopResult(state, pid, "Daemon process was already stopped"); +} + +async function waitForStopAfterRequest(args: { + state: LocalDaemonState; + pid: number | null; + timeoutMs: number; + killTimeoutMs: number; + force?: boolean; +}): Promise<{ stopped: boolean; forced: boolean }> { + const { state, pid, timeoutMs, killTimeoutMs, force } = args; + let stopped = + state.running && pid !== null + ? await waitForPidExit(pid, timeoutMs) + : await waitForDaemonUnreachable(state, timeoutMs); + + if (!stopped && force && state.running && pid !== null) { + await signalProcessTreeOrOwnerSafely(pid, "SIGKILL"); + stopped = await waitForPidExit(pid, killTimeoutMs); + return { stopped, forced: true }; + } + + return { stopped, forced: false }; +} + type LifecycleShutdownAttempt = { requested: true } | { requested: false; reason: string }; function getErrorMessage(error: unknown): string { @@ -567,49 +684,41 @@ export async function stopLocalDaemon( const killTimeoutMs = options.killTimeoutMs ?? DEFAULT_KILL_TIMEOUT_MS; const state = resolveLocalDaemonState({ home: options.home }); - if (!state.pidInfo || !state.running) { - const staleSuffix = - state.stalePidFile && state.pidInfo ? ` (stale PID file for ${state.pidInfo.pid})` : ""; - return { - action: "not_running", - home: state.home, - pid: state.pidInfo?.pid ?? null, - forced: false, - message: `Daemon is not running${staleSuffix}`, - }; - } - - const pid = state.pidInfo.pid; const shutdownAttempt = await requestLifecycleShutdown(state, timeoutMs); const lifecycleRequested = shutdownAttempt.requested; - const fallbackMessage = shutdownAttempt.requested ? null : shutdownAttempt.reason; - let forced = false; - if (!lifecycleRequested) { - const signaled = await signalProcessTreeOrOwnerSafely(pid, "SIGTERM"); - if (!signaled) { - return { - action: "not_running", - home: state.home, - pid, - forced: false, - message: "Daemon process was already stopped", - }; - } - } - let stopped = await waitForPidExit(pid, timeoutMs); - if (!stopped && options.force) { - forced = true; - await signalProcessTreeOrOwnerSafely(pid, "SIGKILL"); - stopped = await waitForPidExit(pid, killTimeoutMs); - } - - if (!stopped) { - throw new Error( - `Timed out waiting for daemon PID ${pid} to stop after ${Math.ceil(timeoutMs / 1000)}s`, + if (!state.pidInfo || (!state.running && !lifecycleRequested)) { + const staleSuffix = + state.stalePidFile && state.pidInfo ? ` (stale PID file for ${state.pidInfo.pid})` : ""; + return createNotRunningStopResult( + state, + state.pidInfo?.pid ?? null, + `Daemon is not running${staleSuffix}`, ); } + const pid = state.pidInfo?.pid ?? null; + const fallbackMessage = shutdownAttempt.requested ? null : shutdownAttempt.reason; + if (!lifecycleRequested) { + const notRunningResult = await signalDaemonOwnerForStop(state, pid); + if (notRunningResult) return notRunningResult; + } + + const { stopped, forced } = await waitForStopAfterRequest({ + state, + pid, + timeoutMs, + killTimeoutMs, + force: options.force, + }); + if (!stopped) { + throw createStopTimeoutError(state, pid, timeoutMs); + } + + if (lifecycleRequested) { + removeStalePidFile(state); + } + return { action: "stopped", home: state.home, diff --git a/packages/cli/tests/34-daemon-stop-stale-reachable.test.ts b/packages/cli/tests/34-daemon-stop-stale-reachable.test.ts new file mode 100644 index 000000000..91d789d1d --- /dev/null +++ b/packages/cli/tests/34-daemon-stop-stale-reachable.test.ts @@ -0,0 +1,207 @@ +#!/usr/bin/env npx tsx + +/** + * Regression: `paseo daemon stop` must stop a reachable daemon even when the + * local pid file points at a dead supervisor owner. + */ + +import assert from "node:assert"; +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { $ } from "zx"; +import { getAvailablePort } from "./helpers/network.ts"; + +$.verbose = false; + +const pollIntervalMs = 100; +const testEnv = { + PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD: process.env.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD ?? "0", + PASEO_DICTATION_ENABLED: process.env.PASEO_DICTATION_ENABLED ?? "0", + PASEO_VOICE_MODE_ENABLED: process.env.PASEO_VOICE_MODE_ENABLED ?? "0", +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isProcessRunning(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) { + return false; + } + + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitFor( + check: () => Promise | boolean, + timeoutMs: number, + message: string, +): Promise { + const deadline = Date.now() + timeoutMs; + + async function poll(): Promise { + if (await check()) return; + if (Date.now() >= deadline) throw new Error(message); + await sleep(pollIntervalMs); + return poll(); + } + + return poll(); +} + +interface DaemonStatus { + localDaemon: string | null; + connectedDaemon: string | null; + pid: number | null; +} + +async function readDaemonStatus(paseoHome: string): Promise { + const result = + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon status --home ${paseoHome} --json`.nothrow(); + if (result.exitCode !== 0) { + return { localDaemon: null, connectedDaemon: null, pid: null }; + } + + try { + const parsed = JSON.parse(result.stdout) as { + localDaemon?: unknown; + connectedDaemon?: unknown; + pid?: unknown; + }; + return { + localDaemon: typeof parsed.localDaemon === "string" ? parsed.localDaemon : null, + connectedDaemon: typeof parsed.connectedDaemon === "string" ? parsed.connectedDaemon : null, + pid: + typeof parsed.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0 + ? parsed.pid + : null, + }; + } catch { + return { localDaemon: null, connectedDaemon: null, pid: null }; + } +} + +function findUnusedPid(): number { + for (let pid = 999_999; pid > 900_000; pid--) { + if (!isProcessRunning(pid)) { + return pid; + } + } + throw new Error("Unable to find unused pid for stale pid fixture"); +} + +console.log("=== Daemon Stop (stale pid, reachable worker regression) ===\n"); + +const port = await getAvailablePort(); +const paseoHome = await mkdtemp(join(tmpdir(), "paseo-stop-stale-reachable-")); +const cliRoot = join(import.meta.dirname, ".."); +const host = `127.0.0.1:${port}`; +const pidPath = join(paseoHome, "paseo.pid"); +const stalePid = findUnusedPid(); + +let workerProcess: ChildProcess | null = null; + +try { + console.log("Test 1: start daemon worker with stale supervisor pid file"); + + await writeFile( + pidPath, + `${JSON.stringify( + { + pid: stalePid, + startedAt: new Date().toISOString(), + hostname: "stale-supervisor-fixture.local", + uid: typeof process.getuid === "function" ? process.getuid() : undefined, + listen: host, + }, + null, + 2, + )}\n`, + ); + + workerProcess = spawn( + process.execPath, + ["--import", "tsx", "../server/src/server/daemon-worker.ts"], + { + cwd: cliRoot, + env: { + ...process.env, + ...testEnv, + PASEO_HOME: paseoHome, + PASEO_LISTEN: host, + PASEO_RELAY_ENABLED: "false", + CI: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + await waitFor( + async () => { + const status = await readDaemonStatus(paseoHome); + return status.localDaemon === "stale_pid" && status.connectedDaemon === "reachable"; + }, + 120000, + "daemon did not enter stale_pid + reachable state in time", + ); + + const statusBeforeStop = await readDaemonStatus(paseoHome); + assert.strictEqual(statusBeforeStop.pid, stalePid, "status should report the stale owner pid"); + assert(workerProcess.pid && isProcessRunning(workerProcess.pid), "worker should be running"); + console.log(`✓ fixture has stale pid ${stalePid} and live worker ${workerProcess.pid}\n`); + + console.log( + "Test 2: `paseo daemon stop` should stop reachable worker instead of saying not_running", + ); + const stopResult = + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon stop --home ${paseoHome} --json`.nothrow(); + assert.strictEqual(stopResult.exitCode, 0, `stop should succeed: ${stopResult.stderr}`); + const stopJson = JSON.parse(stopResult.stdout) as { + action?: unknown; + pid?: unknown; + message?: unknown; + }; + assert.strictEqual(stopJson.action, "stopped", "stop should report stopped action"); + assert.strictEqual( + stopJson.pid, + String(stalePid), + "stop should report the stale pid it recovered from", + ); + assert.strictEqual( + stopJson.message, + "Daemon stopped gracefully", + "stop should route through lifecycle shutdown", + ); + + await waitFor( + () => !isProcessRunning(workerProcess?.pid ?? -1), + 15000, + "worker remained running after stop", + ); + assert.strictEqual(existsSync(pidPath), false, "stale pid file should be removed after stop"); + console.log("✓ stop recovered stale supervisor pid state\n"); +} finally { + if (workerProcess?.pid && isProcessRunning(workerProcess.pid)) { + workerProcess.kill("SIGTERM"); + await waitFor( + () => !isProcessRunning(workerProcess!.pid ?? -1), + 5000, + "worker cleanup timed out", + ).catch(() => { + workerProcess?.kill("SIGKILL"); + }); + } + + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon stop --home ${paseoHome} --force`.nothrow(); + await rm(paseoHome, { recursive: true, force: true }); +} + +console.log("=== Stale reachable stop regression test passed ==="); diff --git a/packages/cli/tests/35-daemon-worker-supervisor-disconnect.test.ts b/packages/cli/tests/35-daemon-worker-supervisor-disconnect.test.ts new file mode 100644 index 000000000..fea7133ed --- /dev/null +++ b/packages/cli/tests/35-daemon-worker-supervisor-disconnect.test.ts @@ -0,0 +1,192 @@ +#!/usr/bin/env npx tsx + +/** + * Regression: a supervised daemon worker must exit when its supervisor IPC + * channel closes, instead of becoming an orphaned daemon. + */ + +import assert from "node:assert"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { $ } from "zx"; +import { getAvailablePort } from "./helpers/network.ts"; + +$.verbose = false; + +const pollIntervalMs = 100; +const testEnv = { + PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD: process.env.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD ?? "0", + PASEO_DICTATION_ENABLED: process.env.PASEO_DICTATION_ENABLED ?? "0", + PASEO_VOICE_MODE_ENABLED: process.env.PASEO_VOICE_MODE_ENABLED ?? "0", +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isProcessRunning(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) { + return false; + } + + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function readWorkerPid(supervisorPid: number): number | null { + if (!Number.isInteger(supervisorPid) || supervisorPid <= 0) { + return null; + } + + const result = spawnSync("ps", ["ax", "-o", "pid=,ppid="], { encoding: "utf8" }); + if (result.status !== 0 || result.error) { + return null; + } + + for (const line of result.stdout.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + const [pidToken, ppidToken] = trimmed.split(/\s+/); + const pid = Number.parseInt(pidToken ?? "", 10); + const ppid = Number.parseInt(ppidToken ?? "", 10); + if (ppid === supervisorPid && pid > 0) { + return pid; + } + } + + return null; +} + +async function waitFor( + check: () => Promise | boolean, + timeoutMs: number, + message: string, +): Promise { + const deadline = Date.now() + timeoutMs; + + async function poll(): Promise { + if (await check()) return; + if (Date.now() >= deadline) throw new Error(message); + await sleep(pollIntervalMs); + return poll(); + } + + return poll(); +} + +interface DaemonStatus { + localDaemon: string | null; + pid: number | null; +} + +async function readDaemonStatus(paseoHome: string): Promise { + const result = + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon status --home ${paseoHome} --json`.nothrow(); + if (result.exitCode !== 0) { + return { localDaemon: null, pid: null }; + } + + try { + const parsed = JSON.parse(result.stdout) as { localDaemon?: unknown; pid?: unknown }; + return { + localDaemon: typeof parsed.localDaemon === "string" ? parsed.localDaemon : null, + pid: + typeof parsed.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0 + ? parsed.pid + : null, + }; + } catch { + return { localDaemon: null, pid: null }; + } +} + +console.log("=== Daemon Worker Supervisor Disconnect Regression ===\n"); + +const port = await getAvailablePort(); +const paseoHome = await mkdtemp(join(tmpdir(), "paseo-worker-supervisor-disconnect-")); +const cliRoot = join(import.meta.dirname, ".."); + +let supervisorProcess: ChildProcess | null = null; +let recentSupervisorLogs = ""; + +try { + console.log("Test 1: start supervised daemon with isolated PASEO_HOME"); + + supervisorProcess = spawn( + process.execPath, + ["--import", "tsx", "../server/scripts/supervisor-entrypoint.ts", "--dev"], + { + cwd: cliRoot, + env: { + ...process.env, + ...testEnv, + PASEO_HOME: paseoHome, + PASEO_LISTEN: `127.0.0.1:${port}`, + PASEO_RELAY_ENABLED: "false", + CI: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + + supervisorProcess.stdout?.on("data", (chunk) => { + recentSupervisorLogs = (recentSupervisorLogs + chunk.toString()).slice(-8000); + }); + supervisorProcess.stderr?.on("data", (chunk) => { + recentSupervisorLogs = (recentSupervisorLogs + chunk.toString()).slice(-8000); + }); + + await waitFor( + async () => { + const status = await readDaemonStatus(paseoHome); + return ( + status.localDaemon === "running" && status.pid !== null && isProcessRunning(status.pid) + ); + }, + 120000, + "daemon did not become running in time", + ); + + const statusBeforeKill = await readDaemonStatus(paseoHome); + const supervisorPid = statusBeforeKill.pid; + assert(supervisorPid !== null, "supervisor pid should exist once daemon starts"); + const workerPid = readWorkerPid(supervisorPid); + assert(workerPid !== null, "supervisor should have a worker process"); + assert(isProcessRunning(workerPid), "worker process should be running"); + console.log(`✓ daemon running with supervisor ${supervisorPid} and worker ${workerPid}\n`); + + console.log("Test 2: killing supervisor should make worker exit via IPC disconnect"); + supervisorProcess.kill("SIGKILL"); + await waitFor( + () => !isProcessRunning(supervisorPid), + 15000, + "supervisor remained running after SIGKILL", + ); + await waitFor( + () => !isProcessRunning(workerPid), + 15000, + "worker remained running after supervisor IPC disconnect", + ); + console.log("✓ worker exited after supervisor disconnect\n"); +} finally { + if (supervisorProcess?.pid && isProcessRunning(supervisorProcess.pid)) { + supervisorProcess.kill("SIGKILL"); + } + + await $`PASEO_HOME=${paseoHome} PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD=${testEnv.PASEO_LOCAL_SPEECH_AUTO_DOWNLOAD} PASEO_DICTATION_ENABLED=${testEnv.PASEO_DICTATION_ENABLED} PASEO_VOICE_MODE_ENABLED=${testEnv.PASEO_VOICE_MODE_ENABLED} npx paseo daemon stop --home ${paseoHome} --force`.nothrow(); + await rm(paseoHome, { recursive: true, force: true }); +} + +if (recentSupervisorLogs.trim().length === 0) { + console.log("(no supervisor logs captured)"); +} + +console.log("=== Worker supervisor disconnect regression test passed ==="); diff --git a/packages/desktop/src/daemon/daemon-manager.test.ts b/packages/desktop/src/daemon/daemon-manager.test.ts index e2393b6c0..d7a6e78d6 100644 --- a/packages/desktop/src/daemon/daemon-manager.test.ts +++ b/packages/desktop/src/daemon/daemon-manager.test.ts @@ -1,10 +1,12 @@ import { EventEmitter } from "node:events"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_DESKTOP_SETTINGS } from "../settings/desktop-settings"; import { createDaemonCommandHandlers } from "./daemon-manager"; const mocks = vi.hoisted(() => ({ + paseoHome: "/tmp/paseo-desktop-daemon-manager-test-home", settings: { releaseChannel: "stable", daemon: { @@ -21,7 +23,7 @@ vi.mock("electron", () => ({ app: { getPath: vi.fn(() => "/tmp/paseo-user-data"), getVersion: vi.fn(() => "1.2.3"), - isPackaged: false, + isPackaged: true, }, ipcMain: { handle: vi.fn() }, powerMonitor: { getSystemIdleTime: vi.fn(() => 0) }, @@ -32,7 +34,7 @@ vi.mock("electron-log/main", () => ({ })); vi.mock("@getpaseo/server", () => ({ - resolvePaseoHome: vi.fn(() => "/tmp/paseo-home"), + resolvePaseoHome: vi.fn(() => mocks.paseoHome), spawnProcess: mocks.spawnProcess, })); @@ -72,8 +74,6 @@ function desktopSettingsWithManagement(enabled: boolean) { } type MockChildProcess = EventEmitter & { - stdout: EventEmitter; - stderr: EventEmitter; pid: number; spawnfile: string; spawnargs: string[]; @@ -82,8 +82,6 @@ type MockChildProcess = EventEmitter & { function createMockChildProcess(): MockChildProcess { const child = new EventEmitter() as MockChildProcess; - child.stdout = new EventEmitter(); - child.stderr = new EventEmitter(); child.pid = 1234; child.spawnfile = "node"; child.spawnargs = ["node", "daemon.js"]; @@ -91,10 +89,8 @@ function createMockChildProcess(): MockChildProcess { return child; } -function scheduleFailedStartupOutput(child: MockChildProcess): void { +function scheduleFailedStartup(child: MockChildProcess): void { setImmediate(() => { - child.stdout.emit("data", Buffer.from(`${"x".repeat(80_000)}stdout-tail`)); - child.stderr.emit("data", Buffer.from(`${"y".repeat(80_000)}stderr-tail`)); child.emit("exit", 1, null); }); } @@ -105,6 +101,11 @@ describe("daemon-manager commands", () => { mocks.runExternalCliJsonCommand.mockReset(); mocks.runExternalCliTextCommand.mockReset(); mocks.spawnProcess.mockReset(); + rmSync(mocks.paseoHome, { recursive: true, force: true }); + }); + + afterEach(() => { + rmSync(mocks.paseoHome, { recursive: true, force: true }); }); it("refuses start and restart while built-in daemon management is disabled", async () => { @@ -136,7 +137,7 @@ describe("daemon-manager commands", () => { listen: null, hostname: null, pid: null, - home: "/tmp/paseo-home", + home: mocks.paseoHome, version: null, desktopManaged: false, error: null, @@ -167,7 +168,7 @@ describe("daemon-manager commands", () => { listen: null, hostname: null, pid: null, - home: "/tmp/paseo-home", + home: mocks.paseoHome, version: null, desktopManaged: false, error: null, @@ -195,7 +196,50 @@ describe("daemon-manager commands", () => { ]); }); - it("uses a reachable daemon when the PID file is stale", async () => { + it("routes stale reachable desktop daemon stops through external CLI daemon stop", async () => { + mocks.runExternalCliJsonCommand + .mockResolvedValueOnce({ + localDaemon: "stale_pid", + connectedDaemon: "reachable", + serverId: "server-1", + pid: 7675, + listen: "127.0.0.1:6767", + daemonVersion: "1.2.2", + desktopManaged: true, + }) + .mockResolvedValueOnce({ action: "stopped" }) + .mockResolvedValueOnce({ + localDaemon: "stopped", + connectedDaemon: "unreachable", + serverId: "", + }); + const handlers = createDaemonCommandHandlers(); + + await expect(handlers.stop_desktop_daemon()).resolves.toEqual({ + serverId: "", + status: "stopped", + listen: null, + hostname: null, + pid: null, + home: mocks.paseoHome, + version: null, + desktopManaged: false, + error: null, + }); + + expect(mocks.runExternalCliJsonCommand).toHaveBeenNthCalledWith(2, [ + "daemon", + "stop", + "--json", + "--timeout", + "5", + "--force", + "--kill-timeout", + "5", + ]); + }); + + it("uses a stale reachable desktop daemon when the version matches", async () => { mocks.runExternalCliJsonCommand.mockResolvedValue({ localDaemon: "stale_pid", connectedDaemon: "reachable", @@ -203,7 +247,7 @@ describe("daemon-manager commands", () => { pid: 7675, listen: "127.0.0.1:6767", hostname: "dev-host", - daemonVersion: "1.2.2", + daemonVersion: "1.2.3", desktopManaged: true, }); const handlers = createDaemonCommandHandlers(); @@ -214,16 +258,86 @@ describe("daemon-manager commands", () => { listen: "127.0.0.1:6767", hostname: "dev-host", pid: null, - home: "/tmp/paseo-home", - version: "1.2.2", - desktopManaged: false, + home: mocks.paseoHome, + version: "1.2.3", + desktopManaged: true, error: null, }); expect(mocks.spawnProcess).not.toHaveBeenCalled(); }); - it("bounds captured daemon startup output", async () => { + it("restarts a stale reachable desktop daemon when the version differs", async () => { + mocks.runExternalCliJsonCommand + .mockResolvedValueOnce({ + localDaemon: "stale_pid", + connectedDaemon: "reachable", + serverId: "server-1", + pid: 7675, + listen: "127.0.0.1:6767", + hostname: "dev-host", + daemonVersion: "1.2.2", + desktopManaged: true, + }) + .mockResolvedValueOnce({ + localDaemon: "stale_pid", + connectedDaemon: "reachable", + serverId: "server-1", + pid: 7675, + listen: "127.0.0.1:6767", + daemonVersion: "1.2.2", + desktopManaged: true, + }) + .mockResolvedValueOnce({ action: "stopped" }) + .mockResolvedValueOnce({ + localDaemon: "stopped", + connectedDaemon: "unreachable", + serverId: "", + }) + .mockResolvedValueOnce({ + localDaemon: "running", + connectedDaemon: "reachable", + serverId: "server-2", + pid: 8888, + listen: "127.0.0.1:6767", + hostname: "dev-host", + daemonVersion: "1.2.3", + desktopManaged: true, + }); + mocks.spawnProcess.mockReturnValue(createMockChildProcess()); + const handlers = createDaemonCommandHandlers(); + + await expect(handlers.start_desktop_daemon()).resolves.toEqual({ + serverId: "server-2", + status: "running", + listen: "127.0.0.1:6767", + hostname: "dev-host", + pid: 8888, + home: mocks.paseoHome, + version: "1.2.3", + desktopManaged: true, + error: null, + }); + + expect(mocks.runExternalCliJsonCommand).toHaveBeenNthCalledWith(3, [ + "daemon", + "stop", + "--json", + "--timeout", + "5", + "--force", + "--kill-timeout", + "5", + ]); + expect(mocks.spawnProcess).toHaveBeenCalled(); + }); + + it("starts the managed daemon detached from desktop stdio and reports daemon log failures", async () => { + mkdirSync(mocks.paseoHome, { recursive: true }); + writeFileSync( + `${mocks.paseoHome}/daemon.log`, + ["old log line", "recent daemon failure"].join("\n"), + ); mocks.runExternalCliJsonCommand.mockResolvedValue({ localDaemon: "stopped", connectedDaemon: "unreachable", @@ -231,7 +345,7 @@ describe("daemon-manager commands", () => { }); mocks.spawnProcess.mockImplementation(() => { const child = createMockChildProcess(); - scheduleFailedStartupOutput(child); + scheduleFailedStartup(child); return child; }); const handlers = createDaemonCommandHandlers(); @@ -246,9 +360,15 @@ describe("daemon-manager commands", () => { expect(thrown).toBeInstanceOf(Error); const message = thrown?.message ?? ""; expect(message).toContain("Daemon failed to start: exit code 1"); - expect(message).toContain("output truncated to the last 65536 chars"); - expect(message).toContain("stdout-tail"); - expect(message).toContain("stderr-tail"); - expect(message.length).toBeLessThan(150_000); + expect(message).toContain(`Recent logs (${mocks.paseoHome}/daemon.log):`); + expect(message).toContain("recent daemon failure"); + expect(mocks.spawnProcess).toHaveBeenCalledWith( + "node", + [], + expect.objectContaining({ + detached: true, + stdio: ["ignore", "ignore", "ignore"], + }), + ); }); }); diff --git a/packages/desktop/src/daemon/daemon-manager.ts b/packages/desktop/src/daemon/daemon-manager.ts index e24940583..3e21f5dfe 100644 --- a/packages/desktop/src/daemon/daemon-manager.ts +++ b/packages/desktop/src/daemon/daemon-manager.ts @@ -43,7 +43,6 @@ const DAEMON_LOG_FILENAME = "daemon.log"; const STARTUP_POLL_INTERVAL_MS = 200; const STARTUP_POLL_MAX_ATTEMPTS = 150; const DETACHED_STARTUP_GRACE_MS = 1200; -const STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS = 64 * 1024; type DesktopDaemonState = "starting" | "running" | "stopped" | "errored"; @@ -70,11 +69,6 @@ interface DesktopPairingOffer { qr: string | null; } -interface StartupOutputCapture { - text: string; - truncated: boolean; -} - function parseReleaseChannel( args: Record | undefined, ): AppReleaseChannel | undefined { @@ -151,30 +145,6 @@ function tailFile(filePath: string, lines = 50): string { } } -function createStartupOutputCapture(): StartupOutputCapture { - return { text: "", truncated: false }; -} - -function appendStartupOutput(capture: StartupOutputCapture, chunk: Buffer): StartupOutputCapture { - const nextText = capture.text + chunk.toString(); - if (nextText.length <= STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS) { - return { text: nextText, truncated: capture.truncated }; - } - - return { - text: nextText.slice(-STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS), - truncated: true, - }; -} - -function formatStartupOutput(capture: StartupOutputCapture): string { - if (!capture.truncated) { - return capture.text; - } - - return `[output truncated to the last ${STARTUP_OUTPUT_CAPTURE_LIMIT_CHARS} chars]\n${capture.text}`; -} - function logDesktopDaemonLifecycle(message: string, details?: Record): void { log.info("[desktop daemon]", message, { pid: process.pid, @@ -231,6 +201,7 @@ export async function resolveDesktopDaemonStatus(): Promise typeof payload.connectedDaemon === "string" ? payload.connectedDaemon : "not_probed"; const hasRunningLocalProcess = localDaemon === "running"; const hasLocalProcess = hasRunningLocalProcess || localDaemon === "unresponsive"; + const desktopManaged = payload.desktopManaged === true; const apiReachable = connectedDaemon === "reachable"; let status: DesktopDaemonState = "stopped"; if (apiReachable || hasRunningLocalProcess) { @@ -248,7 +219,7 @@ export async function resolveDesktopDaemonStatus(): Promise pid: hasLocalProcess && typeof payload.pid === "number" ? payload.pid : null, home, version: typeof payload.daemonVersion === "string" ? payload.daemonVersion : null, - desktopManaged: hasRunningLocalProcess && payload.desktopManaged === true, + desktopManaged, error: null, }; } catch (error) { @@ -287,19 +258,15 @@ function assertBuiltInDaemonManagementEnabled(settings: DesktopSettings): void { } } -function buildStartupFailureError( - result: { code: number | null; signal: string | null; error?: Error }, - stdout: StartupOutputCapture, - stderr: StartupOutputCapture, -): Error { +function buildStartupFailureError(result: { + code: number | null; + signal: string | null; + error?: Error; +}): Error { const reason = result.error ? result.error.message : `exit code ${result.code ?? "unknown"}${result.signal ? ` (${result.signal})` : ""}`; const parts = [`Daemon failed to start: ${reason}`]; - const formattedStderr = formatStartupOutput(stderr).trim(); - const formattedStdout = formatStartupOutput(stdout).trim(); - if (formattedStderr) parts.push(`stderr:\n${formattedStderr}`); - if (formattedStdout) parts.push(`stdout:\n${formattedStdout}`); const logs = tailFile(logFilePath(), 15); if (logs) parts.push(`Recent logs (${logFilePath()}):\n${logs}`); return new Error(parts.join("\n\n")); @@ -377,16 +344,7 @@ async function startDaemon(): Promise { envMode: "internal", env: invocation.env, envOverlay: { PASEO_DESKTOP_MANAGED: "1" }, - stdio: ["ignore", "pipe", "pipe"], - }); - - let stdout = createStartupOutputCapture(); - let stderr = createStartupOutputCapture(); - child.stdout!.on("data", (data: Buffer) => { - stdout = appendStartupOutput(stdout, data); - }); - child.stderr!.on("data", (data: Buffer) => { - stderr = appendStartupOutput(stderr, data); + stdio: ["ignore", "ignore", "ignore"], }); logDesktopDaemonLifecycle("detached spawn returned", { @@ -424,8 +382,6 @@ async function startDaemon(): Promise { logDesktopDaemonLifecycle("detached startup grace period completed", { childPid: child.pid ?? null, exitedEarly: result.exitedEarly, - stdout: formatStartupOutput(stdout).slice(0, 2000), - stderr: formatStartupOutput(stderr).slice(0, 2000), ...(result.exitedEarly ? { exitCode: result.code, @@ -436,7 +392,7 @@ async function startDaemon(): Promise { }); if (result.exitedEarly) { - throw buildStartupFailureError(result, stdout, stderr); + throw buildStartupFailureError(result); } return pollForRunningDaemon(); @@ -444,7 +400,7 @@ async function startDaemon(): Promise { export async function stopDesktopDaemon(): Promise { const status = await resolveDesktopDaemonStatus(); - if (status.status !== "running" || !status.pid) return status; + if (status.status !== "running") return status; await stopDesktopDaemonViaCli(); return await resolveDesktopDaemonStatus(); diff --git a/packages/server/scripts/supervisor.ts b/packages/server/scripts/supervisor.ts index a50643e5a..c9c0158b3 100644 --- a/packages/server/scripts/supervisor.ts +++ b/packages/server/scripts/supervisor.ts @@ -24,6 +24,10 @@ type WorkerLifecycleMessage = reason?: string; }; +interface SupervisorHeartbeatMessage { + type: "paseo:supervisor-heartbeat"; +} + interface SupervisorOptions { name: string; startupMessage: string; @@ -183,6 +187,15 @@ export function runSupervisor(options: SupervisorOptions): void { }); } + const currentChild = child; + const heartbeat = setInterval(() => { + const message: SupervisorHeartbeatMessage = { type: "paseo:supervisor-heartbeat" }; + if (currentChild.connected) { + currentChild.send?.(message, () => undefined); + } + }, 1000); + heartbeat.unref(); + child.stdout?.on("data", (chunk: Buffer) => { process.stdout.write(chunk); writeDurableChunk(chunk); @@ -224,6 +237,7 @@ export function runSupervisor(options: SupervisorOptions): void { }); child.on("close", (code, signal) => { + clearInterval(heartbeat); const exitDescriptor = describeExit(code, signal); writeLifecycleLog("Worker exited", { code, signal, exit: exitDescriptor }); diff --git a/packages/server/src/server/daemon-worker.ts b/packages/server/src/server/daemon-worker.ts index 2a74cd46e..34c082552 100644 --- a/packages/server/src/server/daemon-worker.ts +++ b/packages/server/src/server/daemon-worker.ts @@ -19,12 +19,28 @@ type SupervisorLifecycleMessage = reason?: string; }; +interface SupervisorHeartbeatMessage { + type: "paseo:supervisor-heartbeat"; +} + interface BootstrapResult { paseoHome: string; logger: ReturnType; config: ReturnType; } +function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + if (typeof err === "object" && err !== null && "code" in err && err.code === "EPERM") { + return true; + } + return false; + } +} + function bootstrapFromEnvironment(): BootstrapResult { try { const paseoHome = resolvePaseoHome(); @@ -150,6 +166,46 @@ async function main() { beginShutdown("restart lifecycle intent", { successExitCode: 0 }); }; + const installSupervisorLivenessGuard = () => { + if (typeof process.send !== "function") { + return; + } + + const supervisorPid = process.ppid; + let lastSupervisorHeartbeatAt = Date.now(); + let supervisorShutdownRequested = false; + const requestSupervisorShutdown = (reason: string) => { + if (supervisorShutdownRequested) { + return; + } + supervisorShutdownRequested = true; + beginShutdown(reason); + }; + + process.on("message", (message: unknown) => { + if ( + typeof message === "object" && + message !== null && + "type" in message && + (message as SupervisorHeartbeatMessage).type === "paseo:supervisor-heartbeat" + ) { + lastSupervisorHeartbeatAt = Date.now(); + } + }); + process.on("disconnect", () => requestSupervisorShutdown("supervisor disconnect")); + + const timer = setInterval(() => { + const ipcConnected = typeof process.connected === "boolean" ? process.connected : true; + const heartbeatExpired = Date.now() - lastSupervisorHeartbeatAt > 3500; + if (ipcConnected === false || !isPidAlive(supervisorPid) || heartbeatExpired) { + requestSupervisorShutdown("supervisor disconnect"); + } + }, 1000); + timer.unref(); + }; + + installSupervisorLivenessGuard(); + try { daemon = await createPaseoDaemon( {