From cbb8295cebafaa1448ffdab2d2966c21389fe149 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Thu, 13 Aug 2026 00:46:16 +0200 Subject: [PATCH] fix(workspaces): unblock desktop runtime lifecycle Electron-hosted Local helpers bypassed the sanitized self-Node launch policy, so they lacked ELECTRON_RUN_AS_NODE and provider probes hung. Command-runtime adapters lacked exact detached POSIX process-group ownership, so forced lifecycle cleanup could leave adapter descendants alive. --- packages/desktop/e2e/browser-tabs.e2e.mjs | 93 ++++++++++++++++--- .../fixture-workspace-runtime/src/index.mjs | 14 ++- .../command/internal/command-runtime.ts | 36 +++++-- .../server/workspace-runtime/drivers/index.ts | 7 +- .../internal/host-helper.test.ts | 12 +++ .../workspace-runtime/internal/host-helper.ts | 13 ++- .../internal/local-runtime.ts | 4 +- .../workspace-runtime/internal/service.ts | 6 +- .../internal/worktree-runtime.ts | 4 +- .../workspace-runtime.command.test.ts | 65 +++++++++++++ .../workspace-runtime.docker.e2e.test.ts | 76 +++++++++++++++ 11 files changed, 295 insertions(+), 35 deletions(-) create mode 100644 packages/server/src/server/workspace-runtime/internal/host-helper.test.ts diff --git a/packages/desktop/e2e/browser-tabs.e2e.mjs b/packages/desktop/e2e/browser-tabs.e2e.mjs index 6c8ec44d7e..cde56586d4 100644 --- a/packages/desktop/e2e/browser-tabs.e2e.mjs +++ b/packages/desktop/e2e/browser-tabs.e2e.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import { createServer } from "node:http"; import net from "node:net"; @@ -10,6 +10,7 @@ import process from "node:process"; import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { experimental_createMCPClient } from "ai"; +import { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { chromium } from "playwright"; import { runAppearanceFontSizeRegression } from "./appearance-font-size.electron.mjs"; @@ -180,7 +181,13 @@ async function waitForDesktopStatus(page) { if (typeof window.paseoDesktop?.invoke !== "function") return null; return await window.paseoDesktop.invoke("desktop_daemon_status"); }); - if (typeof status?.serverId === "string") return status; + if ( + status?.status === "running" && + status.desktopManaged === true && + typeof status.serverId === "string" + ) { + return status; + } } catch (error) { // Metro may replace the renderer execution context during its initial load. lastError = error; @@ -192,6 +199,68 @@ async function waitForDesktopStatus(page) { ); } +function processDescendants(parentPid) { + const rows = spawnSync("ps", ["-axo", "pid=,ppid=,command="], { encoding: "utf8" }) + .stdout.trim() + .split("\n") + .map((line) => { + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/); + return match + ? { pid: Number(match[1]), parentPid: Number(match[2]), command: match[3] } + : null; + }) + .filter(Boolean); + const descendants = []; + const pending = [parentPid]; + while (pending.length > 0) { + const current = pending.pop(); + for (const row of rows) { + if (row.parentPid !== current) continue; + descendants.push(row); + pending.push(row.pid); + } + } + return descendants; +} + +async function proveManagedLocalProbe(status, daemonPort) { + assert(status.desktopManaged === true, "Electron did not start its managed daemon"); + assert(Number.isInteger(status.pid), "Desktop daemon status did not expose its PID"); + const daemonClient = new DaemonClient({ + url: `ws://127.0.0.1:${daemonPort}/ws`, + clientId: "desktop-managed-local-probe-e2e", + appVersion: "0.3.1", + reconnect: { enabled: false }, + }); + try { + await daemonClient.connect(); + const ensured = await Promise.race([ + daemonClient.ensureWorkspaceRuntimeProbe({ + projectId: `project-${workspaceIds[0]}`, + runtimeId: "local", + }), + delay(10_000).then(() => { + throw new Error("Electron-managed explicit Local provider probe exceeded 10 seconds"); + }), + ]); + assert( + ensured.status === "ready" && typeof ensured.workspaceId === "string", + `Explicit Local provider probe failed: ${ensured.error ?? ensured.status}`, + ); + await delay(250); + const helpers = processDescendants(status.pid).filter((processInfo) => + processInfo.command.includes("workspace-helper/executable.mjs"), + ); + assert( + helpers.length === 0, + `Explicit Local provider probe left Electron helper children: ${helpers.map(({ pid }) => pid).join(", ")}`, + ); + return ensured.workspaceId; + } finally { + await daemonClient.close().catch(() => undefined); + } +} + async function startTargetPage() { const server = createServer((_request, response) => { response.writeHead(200, { "content-type": "text/html; charset=utf-8" }); @@ -723,6 +792,7 @@ async function main() { const children = []; let browser = null; let client = null; + let page = null; try { const commonEnv = { @@ -737,16 +807,6 @@ async function main() { FORCE_COLOR: "0", NO_COLOR: "1", }; - const daemon = spawnLogged( - "daemon", - process.execPath, - ["--import", "tsx", path.join(rootDir, "packages/server/scripts/dev-runner.ts")], - { cwd: rootDir, env: { ...commonEnv, PASEO_NODE_ENV: "development" } }, - artifactDir, - ); - children.push(daemon.child); - await waitForPort(daemonPort, "daemon", daemon); - const desktopArgs = [ process.execPath, devRunner, @@ -778,8 +838,10 @@ async function main() { await waitForPort(cdpPort, "Electron CDP", desktop); browser = await chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`); - const page = await waitForAppPage(browser, expoPort); + page = await waitForAppPage(browser, expoPort); const status = await waitForDesktopStatus(page); + await waitForPort(daemonPort, "Desktop-managed daemon"); + const localProbeWorkspaceId = await proveManagedLocalProbe(status, daemonPort); await runAppearanceFontSizeRegression(page); @@ -797,7 +859,7 @@ async function main() { targetUrl: target.url, callerAgentId, }); - writeJson(path.join(artifactDir, "result.json"), report); + writeJson(path.join(artifactDir, "result.json"), { ...report, localProbeWorkspaceId }); console.log( `Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click passed.`, ); @@ -807,6 +869,9 @@ async function main() { throw error; } finally { await client?.close().catch(() => undefined); + await page + ?.evaluate(() => window.paseoDesktop?.invoke("stop_desktop_daemon", { reason: "quit" })) + .catch(() => undefined); await browser?.close().catch(() => undefined); for (const child of children.toReversed()) stopProcess(child); await closeServer(target.server); diff --git a/packages/fixture-workspace-runtime/src/index.mjs b/packages/fixture-workspace-runtime/src/index.mjs index 5142ebc253..8c7be218d9 100755 --- a/packages/fixture-workspace-runtime/src/index.mjs +++ b/packages/fixture-workspace-runtime/src/index.mjs @@ -448,7 +448,19 @@ async function signal(id, execId, signalName) { await waitForProcessExit(pid); await rm(execFile(id, execId, options), { force: true }); if (options.signalHelperFailure === "error") throw new Error("fixture signal helper failed"); - if (options.signalHelperFailure === "hang") await new Promise(() => {}); + if (options.signalHelperFailure === "hang") { + if (options.signalHelperDescendantPidFileName) { + const state = await readState(id, options); + const descendant = spawn(process.execPath, ["-e", "setInterval(()=>{},1000)"], { + stdio: "ignore", + }); + await writeFile( + path.join(state.root, options.signalHelperDescendantPidFileName), + String(descendant.pid), + ); + } + await new Promise(() => setInterval(() => {}, 1_000)); + } } async function waitForProcessExit(pid) { diff --git a/packages/server/src/server/workspace-runtime/command/internal/command-runtime.ts b/packages/server/src/server/workspace-runtime/command/internal/command-runtime.ts index d8de96eeda..10a02e72c6 100644 --- a/packages/server/src/server/workspace-runtime/command/internal/command-runtime.ts +++ b/packages/server/src/server/workspace-runtime/command/internal/command-runtime.ts @@ -24,7 +24,7 @@ import type { WorkspacePtyProcess, WorkspaceRuntimeDriver, } from "../../drivers/index.js"; -const COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS = 1_000; +const COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS = 750; export interface CommandRuntimeConfig { command: readonly [string, ...string[]]; @@ -76,7 +76,10 @@ export function createCommandRuntime( id: runtimeId, requiresGitProject: true, reconciliationDomainId: JSON.stringify({ command: config.command, options: config.options }), - workspaceHelperCommand: config.helperCommand ?? ["paseo-workspace-helper"], + workspaceHelper: { + command: config.helperCommand ?? ["paseo-workspace-helper"], + env: {}, + }, scriptTerminal: { kind: "direct-command", command: "/bin/sh", argsPrefix: ["-lc"] }, async create(input) { const response = await lifecycle("create", input.workspaceId, input); @@ -204,6 +207,7 @@ function spawnCommandPty( [...command.slice(1), "exec", "--workspace-id", input.workspaceId], { env: commandEnvironment(), + detached: process.platform !== "win32", shell: false, stdio: ["pipe", "pipe", "pipe", "pipe", "pipe"], }, @@ -421,7 +425,7 @@ async function forceKillCommandRuntime( workspaceId: string, execId: string, options: Readonly>, - wrapper: { kill(signal?: NodeJS.Signals): boolean }, + wrapper: { pid?: number; kill(signal?: NodeJS.Signals): boolean }, wrapperClosed: Promise, ): Promise { const signalCommand = spawn( @@ -436,7 +440,12 @@ async function forceKillCommandRuntime( "--signal", "SIGKILL", ], - { env: commandEnvironment(), shell: false, stdio: ["pipe", "ignore", "pipe"] }, + { + env: commandEnvironment(), + detached: process.platform !== "win32", + shell: false, + stdio: ["pipe", "ignore", "pipe"], + }, ); let signalStderr = ""; signalCommand.stderr.on("data", (chunk: Buffer | string) => { @@ -471,12 +480,12 @@ async function forceKillCommandRuntime( let cleanupError: Error | null = null; if (helper.timedOut) { cleanupError = new Error("signal helper timed out"); - signalCommand.kill("SIGKILL"); + killOwnedProcessGroup(signalCommand, "SIGKILL"); } else { cleanupError = helper.value; - if (cleanupError) signalCommand.kill("SIGKILL"); + if (cleanupError) killOwnedProcessGroup(signalCommand, "SIGKILL"); } - wrapper.kill("SIGKILL"); + killOwnedProcessGroup(wrapper, "SIGKILL"); const wrapperResult = await waitBounded(wrapperClosed, COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS); if (wrapperResult.timedOut) { const wrapperError = new Error("wrapper remained alive after SIGKILL"); @@ -538,6 +547,7 @@ function spawnCommandProcess( [...command.slice(1), "exec", "--workspace-id", input.workspaceId], { env: commandEnvironment(), + detached: process.platform !== "win32", shell: false, stdio: ["pipe", "pipe", "pipe", "pipe", "pipe"], }, @@ -702,6 +712,18 @@ function spawnCommandProcess( } } +function killOwnedProcessGroup( + child: { pid?: number; kill(signal?: NodeJS.Signals): boolean }, + signal: NodeJS.Signals, +): void { + try { + if (process.platform === "win32" || !child.pid) child.kill(signal); + else process.kill(-child.pid, signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } +} + async function collect(stream: Readable): Promise { const chunks: Buffer[] = []; for await (const chunk of stream) diff --git a/packages/server/src/server/workspace-runtime/drivers/index.ts b/packages/server/src/server/workspace-runtime/drivers/index.ts index f7aa7c8d18..41c3d1bac0 100644 --- a/packages/server/src/server/workspace-runtime/drivers/index.ts +++ b/packages/server/src/server/workspace-runtime/drivers/index.ts @@ -99,8 +99,11 @@ export interface WorkspaceRuntimeDriver { readonly id: WorkspaceRuntimeId; readonly requiresGitProject: boolean; readonly reconciliationDomainId?: string; - /** Runtime-local command for Paseo's compatible workspace helper. */ - readonly workspaceHelperCommand: readonly [string, ...string[]]; + /** Runtime-local launch specification for Paseo's compatible workspace helper. */ + readonly workspaceHelper: { + command: readonly [string, ...string[]]; + env: Readonly>; + }; readonly scriptTerminal: import("../index.js").WorkspaceScriptTerminal; create(input: WorkspaceDriverCreateInput): Promise; inspect(workspaceId: WorkspaceId): Promise; diff --git a/packages/server/src/server/workspace-runtime/internal/host-helper.test.ts b/packages/server/src/server/workspace-runtime/internal/host-helper.test.ts new file mode 100644 index 0000000000..40c1fc4f1c --- /dev/null +++ b/packages/server/src/server/workspace-runtime/internal/host-helper.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, test } from "vitest"; + +import { hostWorkspaceHelper } from "./host-helper.js"; + +describe("hostWorkspaceHelper", () => { + test("uses the external host environment plus Electron Node mode", () => { + expect(hostWorkspaceHelper.env.PATH).toBe(process.env.PATH); + expect(hostWorkspaceHelper.env.ELECTRON_RUN_AS_NODE).toBe("1"); + expect(hostWorkspaceHelper.env.PASEO_DESKTOP_MANAGED).toBeUndefined(); + expect(hostWorkspaceHelper.env.PASEO_SUPERVISED).toBeUndefined(); + }); +}); diff --git a/packages/server/src/server/workspace-runtime/internal/host-helper.ts b/packages/server/src/server/workspace-runtime/internal/host-helper.ts index e8d8bb9568..ec299de3cb 100644 --- a/packages/server/src/server/workspace-runtime/internal/host-helper.ts +++ b/packages/server/src/server/workspace-runtime/internal/host-helper.ts @@ -1,6 +1,11 @@ import { fileURLToPath } from "node:url"; -export const hostWorkspaceHelperCommand = [ - process.execPath, - fileURLToPath(new URL("../../workspace-helper/executable.mjs", import.meta.url)), -] as const; +import { buildSelfNodeCommand } from "../../paseo-env.js"; + +const executable = fileURLToPath(new URL("../../workspace-helper/executable.mjs", import.meta.url)); +const launch = buildSelfNodeCommand([executable]); + +export const hostWorkspaceHelper = { + command: [launch.command, ...launch.args] as readonly [string, ...string[]], + env: launch.env, +}; diff --git a/packages/server/src/server/workspace-runtime/internal/local-runtime.ts b/packages/server/src/server/workspace-runtime/internal/local-runtime.ts index 9754523d60..037f102f7c 100644 --- a/packages/server/src/server/workspace-runtime/internal/local-runtime.ts +++ b/packages/server/src/server/workspace-runtime/internal/local-runtime.ts @@ -9,7 +9,7 @@ import type { WorkspaceRuntimeDriver, } from "../drivers/index.js"; import { resolveRuntimeCwd, spawnHostProcess, spawnHostPty } from "./host-process.js"; -import { hostWorkspaceHelperCommand } from "./host-helper.js"; +import { hostWorkspaceHelper } from "./host-helper.js"; import type { HostGitObservationOwner } from "./host-git-observation.js"; import { createRuntimeStateStore } from "./runtime-state.js"; @@ -55,7 +55,7 @@ export function createLocalRuntime( return { id: "local", requiresGitProject: false, - workspaceHelperCommand: hostWorkspaceHelperCommand, + workspaceHelper: hostWorkspaceHelper, scriptTerminal: { kind: "persistent-shell" }, async create(input: WorkspaceDriverCreateInput) { const existing = await inspect(input.workspaceId); diff --git a/packages/server/src/server/workspace-runtime/internal/service.ts b/packages/server/src/server/workspace-runtime/internal/service.ts index c485dc5e17..f83179b670 100644 --- a/packages/server/src/server/workspace-runtime/internal/service.ts +++ b/packages/server/src/server/workspace-runtime/internal/service.ts @@ -183,7 +183,7 @@ export function createService( ownsNewResource = before.status === "missing"; if (ready.state.lifecycle === "ready") { const helper = bindWorkspaceHelper({ - command: driver.workspaceHelperCommand, + command: driver.workspaceHelper.command, launch: (argv) => launchHelper(driver, input.workspaceId, argv), }); try { @@ -544,7 +544,7 @@ export function createService( } if (cached) await cached.client.close(); const client = bindWorkspaceHelper({ - command: driver.workspaceHelperCommand, + command: driver.workspaceHelper.command, launch: async (argv) => { return launchHelper(driver, workspaceId, argv); }, @@ -626,7 +626,7 @@ export function createService( const runtimeProcess = await driver.spawn({ workspaceId, argv, - env: {}, + env: driver.workspaceHelper.env, purpose: { kind: "workspace-helper" }, stdio: { kind: "pipes" }, }); diff --git a/packages/server/src/server/workspace-runtime/internal/worktree-runtime.ts b/packages/server/src/server/workspace-runtime/internal/worktree-runtime.ts index 6510a3585a..bc3658ad62 100644 --- a/packages/server/src/server/workspace-runtime/internal/worktree-runtime.ts +++ b/packages/server/src/server/workspace-runtime/internal/worktree-runtime.ts @@ -15,7 +15,7 @@ import type { WorkspaceRuntimeDriver, } from "../drivers/index.js"; import { resolveRuntimeCwd, spawnHostProcess, spawnHostPty } from "./host-process.js"; -import { hostWorkspaceHelperCommand } from "./host-helper.js"; +import { hostWorkspaceHelper } from "./host-helper.js"; import type { HostGitObservationOwner } from "./host-git-observation.js"; import { createRuntimeStateStore } from "./runtime-state.js"; import { writePaseoWorktreeFirstAgentBranchAutoNameMetadata } from "../../../utils/worktree-metadata.js"; @@ -66,7 +66,7 @@ export function createWorktreeRuntime(options: { return { id: "worktree", requiresGitProject: true, - workspaceHelperCommand: hostWorkspaceHelperCommand, + workspaceHelper: hostWorkspaceHelper, scriptTerminal: { kind: "persistent-shell" }, async create(input: WorkspaceDriverCreateInput) { const existing = await inspect(input.workspaceId); diff --git a/packages/server/src/server/workspace-runtime/workspace-runtime.command.test.ts b/packages/server/src/server/workspace-runtime/workspace-runtime.command.test.ts index 634eb84de3..1abfd9df58 100644 --- a/packages/server/src/server/workspace-runtime/workspace-runtime.command.test.ts +++ b/packages/server/src/server/workspace-runtime/workspace-runtime.command.test.ts @@ -655,6 +655,60 @@ test.each(["error", "hang"] as const)( 5_000, ); +test("archive and permanent deletion reap runtime-owned processes when forced cleanup hangs", async () => { + const descendantPidFileName = "signal-helper-descendant.pid"; + const fixture = await createFixture("lifecycle-hanging-cleanup", false, "pty", { + signalHelperFailure: "hang", + signalHelperDescendantPidFileName: descendantPidFileName, + }); + const pidFile = path.join(fixture.source, "lifecycle-workload.pid"); + const descendantPidFile = path.join(fixture.source, descendantPidFileName); + await fixture.service.create(fixture.createInput); + const workload = await fixture.service.run({ + workspaceId: fixture.workspaceId, + argv: [ + processExecPath(), + "-e", + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000)`, + ], + env: {}, + purpose: { kind: "provider-probe", provider: "codex" }, + }); + workload.stdin.end(); + const workloadPid = await vi.waitFor(async () => { + const pid = Number(await readFile(pidFile, "utf8")); + expect(processExists(pid)).toBe(true); + return pid; + }); + let descendantPid: number | undefined; + + try { + await fixture.service.archive(fixture.workspaceId); + descendantPid = Number(await readFile(descendantPidFile, "utf8")); + expect(fixture.archivedWorkspaceIds).toContain(fixture.workspaceId); + await expect(workload.exited).rejects.toThrow("forced process cleanup failed"); + expect(processExists(workloadPid)).toBe(false); + expect(processExists(descendantPid)).toBe(false); + expect(runtimeAdapterProcesses(fixture.workspaceId)).toEqual([]); + + await fixture.service.destroy(fixture.workspaceId); + await fixture.service.destroy(fixture.workspaceId); + expect(fixture.archivedWorkspaceIds).not.toContain(fixture.workspaceId); + expect(await readdir(fixture.stateDirectory)).toEqual([]); + expect(runtimeAdapterProcesses(fixture.workspaceId)).toEqual([]); + } finally { + if (processExists(workloadPid)) { + try { + process.kill(-workloadPid, "SIGKILL"); + } catch { + // Failure-only containment for the process ownership assertion above. + } + } + if (descendantPid && processExists(descendantPid)) process.kill(descendantPid, "SIGKILL"); + await fixture.service.destroy(fixture.workspaceId).catch(() => undefined); + } +}, 10_000); + test("run admission racing pause cannot leave an unregistered workload running", async () => { const fixture = await createFixture("race", true); await fixture.service.create(fixture.createInput); @@ -714,6 +768,7 @@ async function createFixture( const barrierDirectory = path.join(root, "barrier"); await Promise.all([mkdir(source), mkdir(stateDirectory), mkdir(barrierDirectory)]); const runtimeIds = new Map(); + const archivedWorkspaceIds = new Set(); const workspaceId = `${name}-workspace`; const service = createWorkspaceRuntimeService({ paseoHome: path.join(root, "paseo-home"), @@ -722,8 +777,11 @@ async function createFixture( runtimeIds.set(id, runtimeId); }, beginWorkspaceDeletion: async () => {}, + archiveWorkspaceRecord: async (id) => void archivedWorkspaceIds.add(id), + restoreWorkspaceRecord: async (id) => void archivedWorkspaceIds.delete(id), removeWorkspaceRecord: async (id) => { runtimeIds.delete(id); + archivedWorkspaceIds.delete(id); }, externalRuntimes: { fixture: { @@ -751,6 +809,7 @@ async function createFixture( stateDirectory, barrierDirectory, workspaceId, + archivedWorkspaceIds, service, createInput: { workspaceId, @@ -780,6 +839,12 @@ function processExists(pid: number): boolean { } } +function runtimeAdapterProcesses(workspaceId: string): string[] { + return execFileSync("ps", ["-axo", "command="], { encoding: "utf8" }) + .split("\n") + .filter((command) => command.includes(fixtureExecutable) && command.includes(workspaceId)); +} + function nextFile(directory: string, expectedName: string): Promise { return new Promise((resolve, reject) => { const watcher = watch(directory, (_event, filename) => { diff --git a/packages/server/src/server/workspace-runtime/workspace-runtime.docker.e2e.test.ts b/packages/server/src/server/workspace-runtime/workspace-runtime.docker.e2e.test.ts index 2d78121f6b..4b703ae31f 100644 --- a/packages/server/src/server/workspace-runtime/workspace-runtime.docker.e2e.test.ts +++ b/packages/server/src/server/workspace-runtime/workspace-runtime.docker.e2e.test.ts @@ -367,6 +367,82 @@ test("the command runtime owns Docker workspace materialization, pipes, and life } }, 180_000); +test("production archive and deletion reap Docker runtime processes and owned resources", async () => { + const root = await mkdtemp(path.join(tmpdir(), "paseo-runtime-docker-lifecycle-cleanup-")); + const repo = await createRepository(root); + const paseoHome = path.join(root, "paseo-home"); + const workspaceId = `docker-lifecycle-cleanup-${Date.now()}`; + const runtimeIds = new Map(); + const archivedWorkspaceIds = new Set(); + const service = createWorkspaceRuntimeService({ + paseoHome, + resolveRuntimeId: async (id) => runtimeIds.get(id) ?? null, + persistRuntimeId: async (id, runtimeId) => void runtimeIds.set(id, runtimeId), + beginWorkspaceDeletion: async () => {}, + archiveWorkspaceRecord: async (id) => void archivedWorkspaceIds.add(id), + restoreWorkspaceRecord: async (id) => void archivedWorkspaceIds.delete(id), + removeWorkspaceRecord: async (id) => { + runtimeIds.delete(id); + archivedWorkspaceIds.delete(id); + }, + externalRuntimes: { docker: { type: "docker", image } }, + }); + + onTestFinished(async () => { + await service.destroy(workspaceId).catch(() => undefined); + const leakedContainers = dockerResourceCount("ps", workspaceId); + const leakedVolumes = dockerResourceCount("volume", workspaceId); + const leakedProcesses = dockerTestProcessPids(new Set([workspaceId])); + await rm(root, { recursive: true, force: true }); + if (leakedContainers || leakedVolumes || leakedProcesses.length) { + throw new Error( + `Production lifecycle leaked containers=${leakedContainers} volumes=${leakedVolumes} processes=${leakedProcesses.join(",")}`, + ); + } + }, 30_000); + + await service.create({ + workspaceId, + runtimeId: "docker", + project: { + id: "docker-lifecycle-cleanup-project", + source: { kind: "host-directory", path: repo }, + }, + placement: { kind: "existing" }, + purpose: "provider-probe", + }); + const runtimeProcess = await service.run({ + workspaceId, + argv: [ + "/usr/local/bin/node", + "-e", + "process.on('SIGTERM',()=>{});console.log('owned-ready');setInterval(()=>{},1000)", + ], + env: {}, + purpose: { kind: "provider-probe", provider: "codex" }, + }); + runtimeProcess.stdin.end(); + await expect(waitForOutput(runtimeProcess.stdout, "owned-ready")).resolves.toContain( + "owned-ready", + ); + + await service.archive(workspaceId); + expect(archivedWorkspaceIds).toContain(workspaceId); + await expect(runtimeProcess.exited).resolves.toEqual({ code: null, signal: "SIGKILL" }); + expect(dockerRunningContainerCount(workspaceId)).toBe(0); + expect(dockerResourceCount("ps", workspaceId)).toBe(1); + expect(dockerResourceCount("volume", workspaceId)).toBe(1); + expect(dockerTestProcessPids(new Set([workspaceId]))).toEqual([]); + + await service.destroy(workspaceId); + await service.destroy(workspaceId); + expect(runtimeIds.has(workspaceId)).toBe(false); + expect(archivedWorkspaceIds.has(workspaceId)).toBe(false); + expect(dockerResourceCount("ps", workspaceId)).toBe(0); + expect(dockerResourceCount("volume", workspaceId)).toBe(0); + expect(dockerTestProcessPids(new Set([workspaceId]))).toEqual([]); +}, 60_000); + test("Docker buffers immediate pipe signals until the workload identity is authoritative", async () => { const fixture = await dockerTestFixture("paseo-runtime-docker-started-"); const root = fixture.root;