Keep reporting a Linux daemon as running after its VM resumes from a host sleep (#6279)

* Keep reporting a Linux daemon as running after its VM resumes from a host sleep

A VM paused while its host sleeps keeps its uptime, but its wall clock
catches up on resume. The boot instant derived from the two then lands
after the running supervisor's lock, so daemon status reported it as
stopped and daemon stop released its lock without stopping it.

The lock now records Linux's boot id, and a lock that carries one is
judged by it. Locks without one keep the wall-clock check.

Closes #6019

* Give the OMP idle-deadline test a deadline a slow runner cannot pass before the first wait

With a 1 ms deadline, the gate's first check, made synchronously when the
turn ends, could already be past it on a loaded Windows runner. The turn
then failed before the test's first scheduler wait, and the test timed out.

---------

Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com>

packages/server/src/server/agent/providers/omp/agent.test.ts

@@ -459,7 +459,9 @@ describe("OMP agent client and session", () => {
 
   test("fails a turn when the provider idle gate passes its deadline", async () => {
     const scheduler = new ManualIdleScheduler();
-    const omp = new OmpHarness({ providerIdleScheduler: scheduler, providerIdleDeadlineMs: 1 });
+    // Long enough that the gate's first check, made right after the turn ends, cannot already
+    // be past it on a slow runner.
+    const omp = new OmpHarness({ providerIdleScheduler: scheduler, providerIdleDeadlineMs: 50 });
     await omp.start();
     const { completion } = await omp.startPromptUntilProviderIdle("first", "first done", {
       isStreaming: true,
@@ -473,7 +475,7 @@ describe("OMP agent client and session", () => {
       args: { command: "sleep 30" },
     });
     expect(omp.runningToolCallIds()).toEqual(["tool-at-deadline"]);
-    await new Promise((resolve) => setTimeout(resolve, 2));
+    await new Promise((resolve) => setTimeout(resolve, 60));
     scheduler.retry();
     await expect(completion).rejects.toThrow(/provider idle/i);
     expect(omp.runningToolCallIds()).toEqual([]);

packages/server/src/server/daemon-instance.test.ts

@@ -3,7 +3,7 @@ import { existsSync } from "node:fs";
 import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir, uptime } from "node:os";
 import { join } from "node:path";
-import { afterEach, beforeEach, describe, expect, test } from "vitest";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
 
 import { readDaemonInstance, stopDaemonInstance } from "./daemon-instance.js";
 import { acquirePidLock, getPidLockInfo, isLocked, type PidLockInfo } from "./pid-lock.js";
@@ -94,3 +94,42 @@ describe("daemon instance identity across a reboot", () => {
     await expect(readFile(join(paseoHome, "paseo.pid"), "utf-8")).rejects.toThrow(/ENOENT/);
   });
 });
+
+// Linux keeps a boot's id while a VM is paused for a host sleep, though the VM's wall clock
+// runs ahead of its uptime once it resumes. A sleep longer than the time between boot and
+// the supervisor's start puts the wall-clock boot instant after the lock's startedAt.
+describe.runIf(process.platform === "linux")("daemon instance identity on Linux", () => {
+  let paseoHome: string;
+
+  beforeEach(async () => {
+    paseoHome = await mkdtemp(join(tmpdir(), "paseo-daemon-instance-linux-"));
+  });
+
+  afterEach(async () => {
+    vi.useRealTimers();
+    await rm(paseoHome, { recursive: true, force: true });
+  });
+
+  test("a supervisor still holds the lock after its paused VM resumes", async () => {
+    await acquirePidLock(paseoHome, null, { ownerPid: process.pid });
+
+    vi.useFakeTimers({ toFake: ["Date"] });
+    vi.setSystemTime(Date.now() + uptime() * 1000 + 12 * 60 * 60_000);
+
+    expect(await readDaemonInstance(paseoHome)).toMatchObject({ pid: process.pid });
+    expect(await isLocked(paseoHome)).toMatchObject({ locked: true });
+    await expect(
+      acquirePidLock(paseoHome, null, { ownerPid: process.pid + 10_000 }),
+    ).rejects.toThrow("Another Paseo daemon is already running");
+  });
+
+  test("a lock written during another boot has no running owner", async () => {
+    await writeLock(paseoHome, {
+      ...lockFor(process.pid, new Date()),
+      bootId: "00000000-0000-0000-0000-000000000000",
+    });
+
+    expect(await readDaemonInstance(paseoHome)).toBeNull();
+    expect(await isLocked(paseoHome)).toMatchObject({ locked: false });
+  });
+});

packages/server/src/server/pid-lock.ts

@@ -1,3 +1,4 @@
+import { readFileSync } from "node:fs";
 import { open, readFile, stat, unlink, utimes } from "node:fs/promises";
 import type { FileHandle } from "node:fs/promises";
 import { ensurePrivateDirectory } from "./private-files.js";
@@ -15,6 +16,7 @@ export const pidLockInfoSchema = z.object({
   serverId: z.string().nullable().optional(),
   desktopManaged: z.boolean().optional(),
   heartbeat: z.literal(true).optional(),
+  bootId: z.string().optional(),
 });
 
 export interface PidLockInfo extends z.infer<typeof pidLockInfoSchema> {}
@@ -62,16 +64,38 @@ function precedesThisBoot(startedAt: string): boolean {
   return stamped < Date.now() - uptime() * 1000 - BOOT_INSTANT_TOLERANCE_MS;
 }
 
+let cachedBootId: string | null | undefined;
+
+// Linux names each boot. The wall-clock boot instant is not enough there: a VM paused while
+// its host sleeps keeps its uptime, so after it resumes the instant lands after locks its
+// running supervisor wrote.
+function currentBootId(): string | null {
+  if (cachedBootId === undefined) {
+    try {
+      cachedBootId = readFileSync("/proc/sys/kernel/random/boot_id", "utf-8").trim() || null;
+    } catch {
+      cachedBootId = null;
+    }
+  }
+  return cachedBootId;
+}
+
+function writtenDuringThisBoot(lock: PidLockInfo): boolean {
+  const thisBoot = currentBootId();
+  if (lock.bootId && thisBoot) return lock.bootId === thisBoot;
+  return !precedesThisBoot(lock.startedAt);
+}
+
 /**
  * Whether the process that wrote this lock is still running.
  *
  * A PID alone does not identify the supervisor: the operating system hands the number to
  * something else once the supervisor is gone, and a reboot reassigns it freely. A process
- * cannot predate the boot it runs under, so a lock stamped before this boot is abandoned
+ * cannot predate the boot it runs under, so a lock written during another boot is abandoned
  * however alive its PID looks.
  */
 export function isPidLockOwnerRunning(lock: PidLockInfo): boolean {
-  if (precedesThisBoot(lock.startedAt)) return false;
+  if (!writtenDuringThisBoot(lock)) return false;
   return isPidRunning(lock.pid);
 }
 
@@ -214,6 +238,7 @@ export async function acquirePidLock(
   }
 
   // Create new lock with exclusive flag
+  const bootId = currentBootId();
   const lockInfo: PidLockInfo = {
     pid: lockOwnerPid,
     startedAt: new Date().toISOString(),
@@ -221,6 +246,7 @@ export async function acquirePidLock(
     uid: process.getuid?.() ?? 0,
     listen,
     heartbeat: true,
+    ...(bootId ? { bootId } : {}),
     ...(process.env.PASEO_DESKTOP_MANAGED === "1" ? { desktopManaged: true } : {}),
   };