From 2323ee68445c00afc737e021c130bcf0c0da461d Mon Sep 17 00:00:00 2001 From: "paseo-bot[bot]" <266920839+paseo-bot[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:51:12 +0200 Subject: [PATCH] Keep reporting a Linux daemon as running after its VM resumes from a host sleep (#6279) * Keep reporting a Linux daemon as running after its VM resumes from a host sleep A VM paused while its host sleeps keeps its uptime, but its wall clock catches up on resume. The boot instant derived from the two then lands after the running supervisor's lock, so daemon status reported it as stopped and daemon stop released its lock without stopping it. The lock now records Linux's boot id, and a lock that carries one is judged by it. Locks without one keep the wall-clock check. Closes #6019 * Give the OMP idle-deadline test a deadline a slow runner cannot pass before the first wait With a 1 ms deadline, the gate's first check, made synchronously when the turn ends, could already be past it on a loaded Windows runner. The turn then failed before the test's first scheduler wait, and the test timed out. --------- Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com> --- .../server/agent/providers/omp/agent.test.ts | 6 ++- .../server/src/server/daemon-instance.test.ts | 41 ++++++++++++++++++- packages/server/src/server/pid-lock.ts | 30 +++++++++++++- 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/packages/server/src/server/agent/providers/omp/agent.test.ts b/packages/server/src/server/agent/providers/omp/agent.test.ts index 537c94f2a6..c0e6193bb5 100644 --- a/packages/server/src/server/agent/providers/omp/agent.test.ts +++ b/packages/server/src/server/agent/providers/omp/agent.test.ts @@ -459,7 +459,9 @@ describe("OMP agent client and session", () => { test("fails a turn when the provider idle gate passes its deadline", async () => { const scheduler = new ManualIdleScheduler(); - const omp = new OmpHarness({ providerIdleScheduler: scheduler, providerIdleDeadlineMs: 1 }); + // Long enough that the gate's first check, made right after the turn ends, cannot already + // be past it on a slow runner. + const omp = new OmpHarness({ providerIdleScheduler: scheduler, providerIdleDeadlineMs: 50 }); await omp.start(); const { completion } = await omp.startPromptUntilProviderIdle("first", "first done", { isStreaming: true, @@ -473,7 +475,7 @@ describe("OMP agent client and session", () => { args: { command: "sleep 30" }, }); expect(omp.runningToolCallIds()).toEqual(["tool-at-deadline"]); - await new Promise((resolve) => setTimeout(resolve, 2)); + await new Promise((resolve) => setTimeout(resolve, 60)); scheduler.retry(); await expect(completion).rejects.toThrow(/provider idle/i); expect(omp.runningToolCallIds()).toEqual([]); diff --git a/packages/server/src/server/daemon-instance.test.ts b/packages/server/src/server/daemon-instance.test.ts index 9d7ea66c5a..ee428b2e58 100644 --- a/packages/server/src/server/daemon-instance.test.ts +++ b/packages/server/src/server/daemon-instance.test.ts @@ -3,7 +3,7 @@ import { existsSync } from "node:fs"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir, uptime } from "node:os"; import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { readDaemonInstance, stopDaemonInstance } from "./daemon-instance.js"; import { acquirePidLock, getPidLockInfo, isLocked, type PidLockInfo } from "./pid-lock.js"; @@ -94,3 +94,42 @@ describe("daemon instance identity across a reboot", () => { await expect(readFile(join(paseoHome, "paseo.pid"), "utf-8")).rejects.toThrow(/ENOENT/); }); }); + +// Linux keeps a boot's id while a VM is paused for a host sleep, though the VM's wall clock +// runs ahead of its uptime once it resumes. A sleep longer than the time between boot and +// the supervisor's start puts the wall-clock boot instant after the lock's startedAt. +describe.runIf(process.platform === "linux")("daemon instance identity on Linux", () => { + let paseoHome: string; + + beforeEach(async () => { + paseoHome = await mkdtemp(join(tmpdir(), "paseo-daemon-instance-linux-")); + }); + + afterEach(async () => { + vi.useRealTimers(); + await rm(paseoHome, { recursive: true, force: true }); + }); + + test("a supervisor still holds the lock after its paused VM resumes", async () => { + await acquirePidLock(paseoHome, null, { ownerPid: process.pid }); + + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + uptime() * 1000 + 12 * 60 * 60_000); + + expect(await readDaemonInstance(paseoHome)).toMatchObject({ pid: process.pid }); + expect(await isLocked(paseoHome)).toMatchObject({ locked: true }); + await expect( + acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 }), + ).rejects.toThrow("Another Paseo daemon is already running"); + }); + + test("a lock written during another boot has no running owner", async () => { + await writeLock(paseoHome, { + ...lockFor(process.pid, new Date()), + bootId: "00000000-0000-0000-0000-000000000000", + }); + + expect(await readDaemonInstance(paseoHome)).toBeNull(); + expect(await isLocked(paseoHome)).toMatchObject({ locked: false }); + }); +}); diff --git a/packages/server/src/server/pid-lock.ts b/packages/server/src/server/pid-lock.ts index db95d4786d..314b44ea47 100644 --- a/packages/server/src/server/pid-lock.ts +++ b/packages/server/src/server/pid-lock.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { open, readFile, stat, unlink, utimes } from "node:fs/promises"; import type { FileHandle } from "node:fs/promises"; import { ensurePrivateDirectory } from "./private-files.js"; @@ -15,6 +16,7 @@ export const pidLockInfoSchema = z.object({ serverId: z.string().nullable().optional(), desktopManaged: z.boolean().optional(), heartbeat: z.literal(true).optional(), + bootId: z.string().optional(), }); export interface PidLockInfo extends z.infer {} @@ -62,16 +64,38 @@ function precedesThisBoot(startedAt: string): boolean { return stamped < Date.now() - uptime() * 1000 - BOOT_INSTANT_TOLERANCE_MS; } +let cachedBootId: string | null | undefined; + +// Linux names each boot. The wall-clock boot instant is not enough there: a VM paused while +// its host sleeps keeps its uptime, so after it resumes the instant lands after locks its +// running supervisor wrote. +function currentBootId(): string | null { + if (cachedBootId === undefined) { + try { + cachedBootId = readFileSync("/proc/sys/kernel/random/boot_id", "utf-8").trim() || null; + } catch { + cachedBootId = null; + } + } + return cachedBootId; +} + +function writtenDuringThisBoot(lock: PidLockInfo): boolean { + const thisBoot = currentBootId(); + if (lock.bootId && thisBoot) return lock.bootId === thisBoot; + return !precedesThisBoot(lock.startedAt); +} + /** * Whether the process that wrote this lock is still running. * * A PID alone does not identify the supervisor: the operating system hands the number to * something else once the supervisor is gone, and a reboot reassigns it freely. A process - * cannot predate the boot it runs under, so a lock stamped before this boot is abandoned + * cannot predate the boot it runs under, so a lock written during another boot is abandoned * however alive its PID looks. */ export function isPidLockOwnerRunning(lock: PidLockInfo): boolean { - if (precedesThisBoot(lock.startedAt)) return false; + if (!writtenDuringThisBoot(lock)) return false; return isPidRunning(lock.pid); } @@ -214,6 +238,7 @@ export async function acquirePidLock( } // Create new lock with exclusive flag + const bootId = currentBootId(); const lockInfo: PidLockInfo = { pid: lockOwnerPid, startedAt: new Date().toISOString(), @@ -221,6 +246,7 @@ export async function acquirePidLock( uid: process.getuid?.() ?? 0, listen, heartbeat: true, + ...(bootId ? { bootId } : {}), ...(process.env.PASEO_DESKTOP_MANAGED === "1" ? { desktopManaged: true } : {}), };