diff --git a/packages/desktop/src/daemon/daemon-manager.test.ts b/packages/desktop/src/daemon/daemon-manager.test.ts index c5bc67926b..29bf02d7ad 100644 --- a/packages/desktop/src/daemon/daemon-manager.test.ts +++ b/packages/desktop/src/daemon/daemon-manager.test.ts @@ -16,6 +16,11 @@ const mocks = vi.hoisted(() => ({ }, runExternalCliJsonCommand: vi.fn(), runExternalCliTextCommand: vi.fn(), + createNodeEntrypointInvocation: vi.fn(() => ({ + command: "node", + args: [], + env: {}, + })), spawnProcess: vi.fn(), logInfo: vi.fn(), logError: vi.fn(), @@ -59,11 +64,7 @@ vi.mock("../settings/desktop-settings-electron.js", () => ({ })); vi.mock("./runtime-paths.js", () => ({ - createNodeEntrypointInvocation: vi.fn(() => ({ - command: "node", - args: [], - env: {}, - })), + createNodeEntrypointInvocation: mocks.createNodeEntrypointInvocation, resolveDaemonRunnerEntrypoint: vi.fn(() => ({ entryPath: "/tmp/daemon.js", execArgv: [], @@ -112,6 +113,8 @@ describe("daemon-manager commands", () => { mocks.settings = DEFAULT_DESKTOP_SETTINGS; mocks.runExternalCliJsonCommand.mockReset(); mocks.runExternalCliTextCommand.mockReset(); + mocks.createNodeEntrypointInvocation.mockReset(); + mocks.createNodeEntrypointInvocation.mockReturnValue({ command: "node", args: [], env: {} }); mocks.spawnProcess.mockReset(); mocks.logInfo.mockReset(); mocks.logError.mockReset(); @@ -439,6 +442,9 @@ describe("daemon-manager commands", () => { expect(message).toContain("Daemon failed to start: exit code 1"); expect(recentLogsLabel?.split(/[\\/]/).at(-1)).toBe("daemon.log"); expect(message).toContain("recent daemon failure"); + expect(mocks.createNodeEntrypointInvocation).toHaveBeenCalledWith( + expect.objectContaining({ args: ["--reclaim-stale-pid-lock"] }), + ); expect(mocks.spawnProcess).toHaveBeenCalledWith( "node", [], diff --git a/packages/desktop/src/daemon/daemon-manager.ts b/packages/desktop/src/daemon/daemon-manager.ts index 331454e457..717c34ef9c 100644 --- a/packages/desktop/src/daemon/daemon-manager.ts +++ b/packages/desktop/src/daemon/daemon-manager.ts @@ -388,7 +388,8 @@ async function startDaemon(): Promise { const invocation = createNodeEntrypointInvocation({ entrypoint: daemonRunner, argvMode: "node-script", - args: [], + // The status probe above confirmed that the recorded daemon is unreachable. + args: ["--reclaim-stale-pid-lock"], baseEnv: process.env, }); diff --git a/packages/server/scripts/supervisor-entrypoint.ts b/packages/server/scripts/supervisor-entrypoint.ts index c6c1491227..b379e6179f 100644 --- a/packages/server/scripts/supervisor-entrypoint.ts +++ b/packages/server/scripts/supervisor-entrypoint.ts @@ -18,11 +18,13 @@ process.title = "Paseo Supervisor"; interface DaemonRunnerConfig { devMode: boolean; + reclaimStalePidLock: boolean; workerArgs: string[]; } function parseConfig(argv: string[]): DaemonRunnerConfig { let devMode = false; + let reclaimStalePidLock = false; const workerArgs: string[] = []; for (const arg of argv) { @@ -30,10 +32,14 @@ function parseConfig(argv: string[]): DaemonRunnerConfig { devMode = true; continue; } + if (arg === "--reclaim-stale-pid-lock") { + reclaimStalePidLock = true; + continue; + } workerArgs.push(arg); } - return { devMode, workerArgs }; + return { devMode, reclaimStalePidLock, workerArgs }; } function resolveWorkerEntry(): string { @@ -110,7 +116,7 @@ async function main(): Promise { try { await acquirePidLock(paseoHome, null, { ownerPid: process.pid, - reclaimLegacyDesktopLock: process.env.PASEO_DESKTOP_MANAGED === "1", + reclaimStaleDesktopLock: config.reclaimStalePidLock, }); } catch (error) { if (error instanceof PidLockError) { diff --git a/packages/server/src/server/pid-lock.test.ts b/packages/server/src/server/pid-lock.test.ts index f37b375b44..1dc6c260bb 100644 --- a/packages/server/src/server/pid-lock.test.ts +++ b/packages/server/src/server/pid-lock.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm, utimes, writeFile } from "node:fs/promises"; +import { mkdtemp, open, rm, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "vitest"; @@ -6,6 +6,7 @@ import { describe, expect, test } from "vitest"; import { acquirePidLock, getPidLockInfo, + isLocked, PidLockError, refreshPidLock, releasePidLock, @@ -58,7 +59,7 @@ describe("pid-lock ownership", () => { } }); - test("reclaims a stale heartbeat lock when the recorded pid is alive", async () => { + test("keeps a stale heartbeat lock when the recorded pid is alive without a reachability check", async () => { const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-stale-heartbeat-")); const replacementOwnerPid = process.pid + 10_000; @@ -79,7 +80,43 @@ describe("pid-lock ownership", () => { const staleTime = new Date(Date.now() - 10 * 60_000); await utimes(pidPath, staleTime, staleTime); - await acquirePidLock(paseoHome, null, { ownerPid: replacementOwnerPid }); + await expect(isLocked(paseoHome)).resolves.toMatchObject({ locked: true }); + await expect( + acquirePidLock(paseoHome, null, { ownerPid: replacementOwnerPid }), + ).rejects.toThrow("Another Paseo daemon is already running"); + + const lock = await getPidLockInfo(paseoHome); + expect(lock?.pid).toBe(process.pid); + } finally { + await rm(paseoHome, { recursive: true, force: true }); + } + }); + + test("reclaims a stale desktop heartbeat lock after desktop confirms the daemon is unreachable", async () => { + const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-stale-desktop-heartbeat-")); + const replacementOwnerPid = process.pid + 10_000; + + try { + const pidPath = join(paseoHome, "paseo.pid"); + await writeFile( + pidPath, + JSON.stringify({ + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + hostname: "old-host", + uid: process.getuid?.() ?? 0, + listen: "127.0.0.1:6767", + desktopManaged: true, + heartbeat: true, + }), + ); + const staleTime = new Date(Date.now() - 10 * 60_000); + await utimes(pidPath, staleTime, staleTime); + + await acquirePidLock(paseoHome, null, { + ownerPid: replacementOwnerPid, + reclaimStaleDesktopLock: true, + }); const lock = await getPidLockInfo(paseoHome); expect(lock?.pid).toBe(replacementOwnerPid); @@ -141,7 +178,7 @@ describe("pid-lock ownership", () => { await acquirePidLock(paseoHome, null, { ownerPid: replacementOwnerPid, - reclaimLegacyDesktopLock: true, + reclaimStaleDesktopLock: true, }); const lock = await getPidLockInfo(paseoHome); @@ -166,6 +203,29 @@ describe("pid-lock ownership", () => { } }); + test("retries a heartbeat refresh while its owner is rewriting the lock", async () => { + const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-refresh-rewrite-")); + const pidPath = join(paseoHome, "paseo.pid"); + + try { + await acquirePidLock(paseoHome, null, { ownerPid: process.pid }); + const lock = await getPidLockInfo(paseoHome); + expect(lock).not.toBeNull(); + + const rewriteHandle = await open(pidPath, "r+"); + await rewriteHandle.truncate(0); + + const refresh = refreshPidLock(paseoHome, { ownerPid: process.pid }); + await new Promise((resolve) => setTimeout(resolve, 250)); + await rewriteHandle.writeFile(JSON.stringify(lock)); + await rewriteHandle.close(); + + await expect(refresh).resolves.toBeUndefined(); + } finally { + await rm(paseoHome, { recursive: true, force: true }); + } + }); + test("keeps a fresh lock when the recorded pid is alive", async () => { const paseoHome = await mkdtemp(join(tmpdir(), "paseo-pid-lock-fresh-heartbeat-")); diff --git a/packages/server/src/server/pid-lock.ts b/packages/server/src/server/pid-lock.ts index 7a3dd9aee9..9202f49dfd 100644 --- a/packages/server/src/server/pid-lock.ts +++ b/packages/server/src/server/pid-lock.ts @@ -1,4 +1,5 @@ import { open, readFile, stat, unlink, mkdir, utimes } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { hostname } from "node:os"; @@ -38,6 +39,8 @@ export class PidLockError extends Error { // Stale recovery is for abandoned locks, so keep this well above ordinary event-loop stalls. const PID_LOCK_STALE_MS = 5 * 60_000; const PID_LOCK_HEARTBEAT_INTERVAL_MS = 30_000; +const PID_LOCK_READ_RETRY_ATTEMPTS = 10; +const PID_LOCK_READ_RETRY_DELAY_MS = 50; function isPidRunning(pid: number): boolean { try { @@ -84,20 +87,16 @@ function resolveOwnerPid(ownerPid?: number): number { interface AcquirePidLockOptions { ownerPid?: number; - reclaimLegacyDesktopLock?: boolean; + reclaimStaleDesktopLock?: boolean; } function canReclaimLiveLock( lock: PidLockInfo, options: AcquirePidLockOptions | undefined, ): boolean { - if (lock.heartbeat === true) { - return true; - } - // COMPAT(pidLockHeartbeat): v0.1.108 desktop startup has already confirmed the old daemon is // unreachable before it launches the supervisor. Remove after 2027-01-15. - return options?.reclaimLegacyDesktopLock === true && lock.desktopManaged === true; + return options?.reclaimStaleDesktopLock === true && lock.desktopManaged === true; } function isSamePidLock(left: PidLockInfo, right: PidLockInfo): boolean { @@ -222,12 +221,7 @@ export async function refreshPidLock( } try { - let lock: PidLockInfo | null = null; - try { - lock = parsePidLockInfo(JSON.parse(await fd.readFile("utf-8"))); - } catch { - // Report malformed contents as an ownership failure below. - } + const lock = await readPidLockFromHandleWithRetry(fd); if (!lock) { throw new PidLockError("Cannot refresh PID lock: invalid lock file"); } @@ -241,6 +235,33 @@ export async function refreshPidLock( } } +async function readPidLockFromHandle(fd: FileHandle): Promise { + try { + const { size } = await fd.stat(); + if (size === 0) { + return null; + } + const content = Buffer.alloc(size); + const { bytesRead } = await fd.read(content, 0, size, 0); + return parsePidLockInfo(JSON.parse(content.subarray(0, bytesRead).toString("utf-8"))); + } catch { + return null; + } +} + +async function readPidLockFromHandleWithRetry(fd: FileHandle): Promise { + for (let attempt = 0; attempt < PID_LOCK_READ_RETRY_ATTEMPTS; attempt += 1) { + const lock = await readPidLockFromHandle(fd); + if (lock) { + return lock; + } + if (attempt < PID_LOCK_READ_RETRY_ATTEMPTS - 1) { + await new Promise((resolve) => setTimeout(resolve, PID_LOCK_READ_RETRY_DELAY_MS)); + } + } + return null; +} + export function startPidLockHeartbeat( paseoHome: string, options?: { @@ -282,23 +303,23 @@ export async function updatePidLock( ): Promise { const pidPath = getPidFilePath(paseoHome); const lockOwnerPid = resolveOwnerPid(options?.ownerPid); - const content = await readFile(pidPath, "utf-8"); - const existingLock = parsePidLockInfo(JSON.parse(content)); - if (!existingLock) { - throw new PidLockError("Cannot update PID lock: invalid lock file"); - } - - if (existingLock.pid !== lockOwnerPid) { - throw new PidLockError(`Cannot update PID lock owned by PID ${existingLock.pid}`, existingLock); - } - - const updatedLock: PidLockInfo = { - ...existingLock, - ...patch, - }; - const fd = await open(pidPath, "r+"); try { + const existingLock = await readPidLockFromHandleWithRetry(fd); + if (!existingLock) { + throw new PidLockError("Cannot update PID lock: invalid lock file"); + } + if (existingLock.pid !== lockOwnerPid) { + throw new PidLockError( + `Cannot update PID lock owned by PID ${existingLock.pid}`, + existingLock, + ); + } + + const updatedLock: PidLockInfo = { + ...existingLock, + ...patch, + }; await fd.truncate(0); await fd.writeFile(JSON.stringify(updatedLock)); } finally { @@ -336,12 +357,8 @@ export async function isLocked( if (!info) { return { locked: false }; } - const pidPath = getPidFilePath(paseoHome); if (!isPidRunning(info.pid)) { return { locked: false, info }; } - if (info.heartbeat === true && !(await isPidLockFresh(pidPath))) { - return { locked: false, info }; - } return { locked: true, info }; }