diff --git a/packages/server/src/server/agent/providers/omp/agent.test.ts b/packages/server/src/server/agent/providers/omp/agent.test.ts index 537c94f2a6..c0e6193bb5 100644 --- a/packages/server/src/server/agent/providers/omp/agent.test.ts +++ b/packages/server/src/server/agent/providers/omp/agent.test.ts @@ -459,7 +459,9 @@ describe("OMP agent client and session", () => { test("fails a turn when the provider idle gate passes its deadline", async () => { const scheduler = new ManualIdleScheduler(); - const omp = new OmpHarness({ providerIdleScheduler: scheduler, providerIdleDeadlineMs: 1 }); + // Long enough that the gate's first check, made right after the turn ends, cannot already + // be past it on a slow runner. + const omp = new OmpHarness({ providerIdleScheduler: scheduler, providerIdleDeadlineMs: 50 }); await omp.start(); const { completion } = await omp.startPromptUntilProviderIdle("first", "first done", { isStreaming: true, @@ -473,7 +475,7 @@ describe("OMP agent client and session", () => { args: { command: "sleep 30" }, }); expect(omp.runningToolCallIds()).toEqual(["tool-at-deadline"]); - await new Promise((resolve) => setTimeout(resolve, 2)); + await new Promise((resolve) => setTimeout(resolve, 60)); scheduler.retry(); await expect(completion).rejects.toThrow(/provider idle/i); expect(omp.runningToolCallIds()).toEqual([]); diff --git a/packages/server/src/server/daemon-instance.test.ts b/packages/server/src/server/daemon-instance.test.ts index 9d7ea66c5a..ee428b2e58 100644 --- a/packages/server/src/server/daemon-instance.test.ts +++ b/packages/server/src/server/daemon-instance.test.ts @@ -3,7 +3,7 @@ import { existsSync } from "node:fs"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir, uptime } from "node:os"; import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { readDaemonInstance, stopDaemonInstance } from "./daemon-instance.js"; import { acquirePidLock, getPidLockInfo, isLocked, type PidLockInfo } from "./pid-lock.js"; @@ -94,3 +94,42 @@ describe("daemon instance identity across a reboot", () => { await expect(readFile(join(paseoHome, "paseo.pid"), "utf-8")).rejects.toThrow(/ENOENT/); }); }); + +// Linux keeps a boot's id while a VM is paused for a host sleep, though the VM's wall clock +// runs ahead of its uptime once it resumes. A sleep longer than the time between boot and +// the supervisor's start puts the wall-clock boot instant after the lock's startedAt. +describe.runIf(process.platform === "linux")("daemon instance identity on Linux", () => { + let paseoHome: string; + + beforeEach(async () => { + paseoHome = await mkdtemp(join(tmpdir(), "paseo-daemon-instance-linux-")); + }); + + afterEach(async () => { + vi.useRealTimers(); + await rm(paseoHome, { recursive: true, force: true }); + }); + + test("a supervisor still holds the lock after its paused VM resumes", async () => { + await acquirePidLock(paseoHome, null, { ownerPid: process.pid }); + + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + uptime() * 1000 + 12 * 60 * 60_000); + + expect(await readDaemonInstance(paseoHome)).toMatchObject({ pid: process.pid }); + expect(await isLocked(paseoHome)).toMatchObject({ locked: true }); + await expect( + acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 }), + ).rejects.toThrow("Another Paseo daemon is already running"); + }); + + test("a lock written during another boot has no running owner", async () => { + await writeLock(paseoHome, { + ...lockFor(process.pid, new Date()), + bootId: "00000000-0000-0000-0000-000000000000", + }); + + expect(await readDaemonInstance(paseoHome)).toBeNull(); + expect(await isLocked(paseoHome)).toMatchObject({ locked: false }); + }); +}); diff --git a/packages/server/src/server/pid-lock.ts b/packages/server/src/server/pid-lock.ts index db95d4786d..314b44ea47 100644 --- a/packages/server/src/server/pid-lock.ts +++ b/packages/server/src/server/pid-lock.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { open, readFile, stat, unlink, utimes } from "node:fs/promises"; import type { FileHandle } from "node:fs/promises"; import { ensurePrivateDirectory } from "./private-files.js"; @@ -15,6 +16,7 @@ export const pidLockInfoSchema = z.object({ serverId: z.string().nullable().optional(), desktopManaged: z.boolean().optional(), heartbeat: z.literal(true).optional(), + bootId: z.string().optional(), }); export interface PidLockInfo extends z.infer {} @@ -62,16 +64,38 @@ function precedesThisBoot(startedAt: string): boolean { return stamped < Date.now() - uptime() * 1000 - BOOT_INSTANT_TOLERANCE_MS; } +let cachedBootId: string | null | undefined; + +// Linux names each boot. The wall-clock boot instant is not enough there: a VM paused while +// its host sleeps keeps its uptime, so after it resumes the instant lands after locks its +// running supervisor wrote. +function currentBootId(): string | null { + if (cachedBootId === undefined) { + try { + cachedBootId = readFileSync("/proc/sys/kernel/random/boot_id", "utf-8").trim() || null; + } catch { + cachedBootId = null; + } + } + return cachedBootId; +} + +function writtenDuringThisBoot(lock: PidLockInfo): boolean { + const thisBoot = currentBootId(); + if (lock.bootId && thisBoot) return lock.bootId === thisBoot; + return !precedesThisBoot(lock.startedAt); +} + /** * Whether the process that wrote this lock is still running. * * A PID alone does not identify the supervisor: the operating system hands the number to * something else once the supervisor is gone, and a reboot reassigns it freely. A process - * cannot predate the boot it runs under, so a lock stamped before this boot is abandoned + * cannot predate the boot it runs under, so a lock written during another boot is abandoned * however alive its PID looks. */ export function isPidLockOwnerRunning(lock: PidLockInfo): boolean { - if (precedesThisBoot(lock.startedAt)) return false; + if (!writtenDuringThisBoot(lock)) return false; return isPidRunning(lock.pid); } @@ -214,6 +238,7 @@ export async function acquirePidLock( } // Create new lock with exclusive flag + const bootId = currentBootId(); const lockInfo: PidLockInfo = { pid: lockOwnerPid, startedAt: new Date().toISOString(), @@ -221,6 +246,7 @@ export async function acquirePidLock( uid: process.getuid?.() ?? 0, listen, heartbeat: true, + ...(bootId ? { bootId } : {}), ...(process.env.PASEO_DESKTOP_MANAGED === "1" ? { desktopManaged: true } : {}), };