mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-24 18:22:38 +00:00
fix(workspaces): unblock desktop runtime lifecycle
Electron-hosted Local helpers bypassed the sanitized self-Node launch policy, so they lacked ELECTRON_RUN_AS_NODE and provider probes hung. Command-runtime adapters lacked exact detached POSIX process-group ownership, so forced lifecycle cleanup could leave adapter descendants alive.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import net from "node:net";
|
||||
@@ -10,6 +10,7 @@ import process from "node:process";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { experimental_createMCPClient } from "ai";
|
||||
import { DaemonClient } from "@getpaseo/client/internal/daemon-client";
|
||||
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
||||
import { chromium } from "playwright";
|
||||
import { runAppearanceFontSizeRegression } from "./appearance-font-size.electron.mjs";
|
||||
@@ -180,7 +181,13 @@ async function waitForDesktopStatus(page) {
|
||||
if (typeof window.paseoDesktop?.invoke !== "function") return null;
|
||||
return await window.paseoDesktop.invoke("desktop_daemon_status");
|
||||
});
|
||||
if (typeof status?.serverId === "string") return status;
|
||||
if (
|
||||
status?.status === "running" &&
|
||||
status.desktopManaged === true &&
|
||||
typeof status.serverId === "string"
|
||||
) {
|
||||
return status;
|
||||
}
|
||||
} catch (error) {
|
||||
// Metro may replace the renderer execution context during its initial load.
|
||||
lastError = error;
|
||||
@@ -192,6 +199,68 @@ async function waitForDesktopStatus(page) {
|
||||
);
|
||||
}
|
||||
|
||||
function processDescendants(parentPid) {
|
||||
const rows = spawnSync("ps", ["-axo", "pid=,ppid=,command="], { encoding: "utf8" })
|
||||
.stdout.trim()
|
||||
.split("\n")
|
||||
.map((line) => {
|
||||
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/);
|
||||
return match
|
||||
? { pid: Number(match[1]), parentPid: Number(match[2]), command: match[3] }
|
||||
: null;
|
||||
})
|
||||
.filter(Boolean);
|
||||
const descendants = [];
|
||||
const pending = [parentPid];
|
||||
while (pending.length > 0) {
|
||||
const current = pending.pop();
|
||||
for (const row of rows) {
|
||||
if (row.parentPid !== current) continue;
|
||||
descendants.push(row);
|
||||
pending.push(row.pid);
|
||||
}
|
||||
}
|
||||
return descendants;
|
||||
}
|
||||
|
||||
async function proveManagedLocalProbe(status, daemonPort) {
|
||||
assert(status.desktopManaged === true, "Electron did not start its managed daemon");
|
||||
assert(Number.isInteger(status.pid), "Desktop daemon status did not expose its PID");
|
||||
const daemonClient = new DaemonClient({
|
||||
url: `ws://127.0.0.1:${daemonPort}/ws`,
|
||||
clientId: "desktop-managed-local-probe-e2e",
|
||||
appVersion: "0.3.1",
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
try {
|
||||
await daemonClient.connect();
|
||||
const ensured = await Promise.race([
|
||||
daemonClient.ensureWorkspaceRuntimeProbe({
|
||||
projectId: `project-${workspaceIds[0]}`,
|
||||
runtimeId: "local",
|
||||
}),
|
||||
delay(10_000).then(() => {
|
||||
throw new Error("Electron-managed explicit Local provider probe exceeded 10 seconds");
|
||||
}),
|
||||
]);
|
||||
assert(
|
||||
ensured.status === "ready" && typeof ensured.workspaceId === "string",
|
||||
`Explicit Local provider probe failed: ${ensured.error ?? ensured.status}`,
|
||||
);
|
||||
await delay(250);
|
||||
const helpers = processDescendants(status.pid).filter((processInfo) =>
|
||||
processInfo.command.includes("workspace-helper/executable.mjs"),
|
||||
);
|
||||
assert(
|
||||
helpers.length === 0,
|
||||
`Explicit Local provider probe left Electron helper children: ${helpers.map(({ pid }) => pid).join(", ")}`,
|
||||
);
|
||||
return ensured.workspaceId;
|
||||
} finally {
|
||||
await daemonClient.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async function startTargetPage() {
|
||||
const server = createServer((_request, response) => {
|
||||
response.writeHead(200, { "content-type": "text/html; charset=utf-8" });
|
||||
@@ -723,6 +792,7 @@ async function main() {
|
||||
const children = [];
|
||||
let browser = null;
|
||||
let client = null;
|
||||
let page = null;
|
||||
|
||||
try {
|
||||
const commonEnv = {
|
||||
@@ -737,16 +807,6 @@ async function main() {
|
||||
FORCE_COLOR: "0",
|
||||
NO_COLOR: "1",
|
||||
};
|
||||
const daemon = spawnLogged(
|
||||
"daemon",
|
||||
process.execPath,
|
||||
["--import", "tsx", path.join(rootDir, "packages/server/scripts/dev-runner.ts")],
|
||||
{ cwd: rootDir, env: { ...commonEnv, PASEO_NODE_ENV: "development" } },
|
||||
artifactDir,
|
||||
);
|
||||
children.push(daemon.child);
|
||||
await waitForPort(daemonPort, "daemon", daemon);
|
||||
|
||||
const desktopArgs = [
|
||||
process.execPath,
|
||||
devRunner,
|
||||
@@ -778,8 +838,10 @@ async function main() {
|
||||
await waitForPort(cdpPort, "Electron CDP", desktop);
|
||||
|
||||
browser = await chromium.connectOverCDP(`http://127.0.0.1:${cdpPort}`);
|
||||
const page = await waitForAppPage(browser, expoPort);
|
||||
page = await waitForAppPage(browser, expoPort);
|
||||
const status = await waitForDesktopStatus(page);
|
||||
await waitForPort(daemonPort, "Desktop-managed daemon");
|
||||
const localProbeWorkspaceId = await proveManagedLocalProbe(status, daemonPort);
|
||||
|
||||
await runAppearanceFontSizeRegression(page);
|
||||
|
||||
@@ -797,7 +859,7 @@ async function main() {
|
||||
targetUrl: target.url,
|
||||
callerAgentId,
|
||||
});
|
||||
writeJson(path.join(artifactDir, "result.json"), report);
|
||||
writeJson(path.join(artifactDir, "result.json"), { ...report, localProbeWorkspaceId });
|
||||
console.log(
|
||||
`Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click passed.`,
|
||||
);
|
||||
@@ -807,6 +869,9 @@ async function main() {
|
||||
throw error;
|
||||
} finally {
|
||||
await client?.close().catch(() => undefined);
|
||||
await page
|
||||
?.evaluate(() => window.paseoDesktop?.invoke("stop_desktop_daemon", { reason: "quit" }))
|
||||
.catch(() => undefined);
|
||||
await browser?.close().catch(() => undefined);
|
||||
for (const child of children.toReversed()) stopProcess(child);
|
||||
await closeServer(target.server);
|
||||
|
||||
@@ -448,7 +448,19 @@ async function signal(id, execId, signalName) {
|
||||
await waitForProcessExit(pid);
|
||||
await rm(execFile(id, execId, options), { force: true });
|
||||
if (options.signalHelperFailure === "error") throw new Error("fixture signal helper failed");
|
||||
if (options.signalHelperFailure === "hang") await new Promise(() => {});
|
||||
if (options.signalHelperFailure === "hang") {
|
||||
if (options.signalHelperDescendantPidFileName) {
|
||||
const state = await readState(id, options);
|
||||
const descendant = spawn(process.execPath, ["-e", "setInterval(()=>{},1000)"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
await writeFile(
|
||||
path.join(state.root, options.signalHelperDescendantPidFileName),
|
||||
String(descendant.pid),
|
||||
);
|
||||
}
|
||||
await new Promise(() => setInterval(() => {}, 1_000));
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForProcessExit(pid) {
|
||||
|
||||
@@ -24,7 +24,7 @@ import type {
|
||||
WorkspacePtyProcess,
|
||||
WorkspaceRuntimeDriver,
|
||||
} from "../../drivers/index.js";
|
||||
const COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS = 1_000;
|
||||
const COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS = 750;
|
||||
|
||||
export interface CommandRuntimeConfig {
|
||||
command: readonly [string, ...string[]];
|
||||
@@ -76,7 +76,10 @@ export function createCommandRuntime(
|
||||
id: runtimeId,
|
||||
requiresGitProject: true,
|
||||
reconciliationDomainId: JSON.stringify({ command: config.command, options: config.options }),
|
||||
workspaceHelperCommand: config.helperCommand ?? ["paseo-workspace-helper"],
|
||||
workspaceHelper: {
|
||||
command: config.helperCommand ?? ["paseo-workspace-helper"],
|
||||
env: {},
|
||||
},
|
||||
scriptTerminal: { kind: "direct-command", command: "/bin/sh", argsPrefix: ["-lc"] },
|
||||
async create(input) {
|
||||
const response = await lifecycle("create", input.workspaceId, input);
|
||||
@@ -204,6 +207,7 @@ function spawnCommandPty(
|
||||
[...command.slice(1), "exec", "--workspace-id", input.workspaceId],
|
||||
{
|
||||
env: commandEnvironment(),
|
||||
detached: process.platform !== "win32",
|
||||
shell: false,
|
||||
stdio: ["pipe", "pipe", "pipe", "pipe", "pipe"],
|
||||
},
|
||||
@@ -421,7 +425,7 @@ async function forceKillCommandRuntime(
|
||||
workspaceId: string,
|
||||
execId: string,
|
||||
options: Readonly<Record<string, unknown>>,
|
||||
wrapper: { kill(signal?: NodeJS.Signals): boolean },
|
||||
wrapper: { pid?: number; kill(signal?: NodeJS.Signals): boolean },
|
||||
wrapperClosed: Promise<void>,
|
||||
): Promise<void> {
|
||||
const signalCommand = spawn(
|
||||
@@ -436,7 +440,12 @@ async function forceKillCommandRuntime(
|
||||
"--signal",
|
||||
"SIGKILL",
|
||||
],
|
||||
{ env: commandEnvironment(), shell: false, stdio: ["pipe", "ignore", "pipe"] },
|
||||
{
|
||||
env: commandEnvironment(),
|
||||
detached: process.platform !== "win32",
|
||||
shell: false,
|
||||
stdio: ["pipe", "ignore", "pipe"],
|
||||
},
|
||||
);
|
||||
let signalStderr = "";
|
||||
signalCommand.stderr.on("data", (chunk: Buffer | string) => {
|
||||
@@ -471,12 +480,12 @@ async function forceKillCommandRuntime(
|
||||
let cleanupError: Error | null = null;
|
||||
if (helper.timedOut) {
|
||||
cleanupError = new Error("signal helper timed out");
|
||||
signalCommand.kill("SIGKILL");
|
||||
killOwnedProcessGroup(signalCommand, "SIGKILL");
|
||||
} else {
|
||||
cleanupError = helper.value;
|
||||
if (cleanupError) signalCommand.kill("SIGKILL");
|
||||
if (cleanupError) killOwnedProcessGroup(signalCommand, "SIGKILL");
|
||||
}
|
||||
wrapper.kill("SIGKILL");
|
||||
killOwnedProcessGroup(wrapper, "SIGKILL");
|
||||
const wrapperResult = await waitBounded(wrapperClosed, COMMAND_RUNTIME_CLEANUP_TIMEOUT_MS);
|
||||
if (wrapperResult.timedOut) {
|
||||
const wrapperError = new Error("wrapper remained alive after SIGKILL");
|
||||
@@ -538,6 +547,7 @@ function spawnCommandProcess(
|
||||
[...command.slice(1), "exec", "--workspace-id", input.workspaceId],
|
||||
{
|
||||
env: commandEnvironment(),
|
||||
detached: process.platform !== "win32",
|
||||
shell: false,
|
||||
stdio: ["pipe", "pipe", "pipe", "pipe", "pipe"],
|
||||
},
|
||||
@@ -702,6 +712,18 @@ function spawnCommandProcess(
|
||||
}
|
||||
}
|
||||
|
||||
function killOwnedProcessGroup(
|
||||
child: { pid?: number; kill(signal?: NodeJS.Signals): boolean },
|
||||
signal: NodeJS.Signals,
|
||||
): void {
|
||||
try {
|
||||
if (process.platform === "win32" || !child.pid) child.kill(signal);
|
||||
else process.kill(-child.pid, signal);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function collect(stream: Readable): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream)
|
||||
|
||||
@@ -99,8 +99,11 @@ export interface WorkspaceRuntimeDriver {
|
||||
readonly id: WorkspaceRuntimeId;
|
||||
readonly requiresGitProject: boolean;
|
||||
readonly reconciliationDomainId?: string;
|
||||
/** Runtime-local command for Paseo's compatible workspace helper. */
|
||||
readonly workspaceHelperCommand: readonly [string, ...string[]];
|
||||
/** Runtime-local launch specification for Paseo's compatible workspace helper. */
|
||||
readonly workspaceHelper: {
|
||||
command: readonly [string, ...string[]];
|
||||
env: Readonly<Record<string, string>>;
|
||||
};
|
||||
readonly scriptTerminal: import("../index.js").WorkspaceScriptTerminal;
|
||||
create(input: WorkspaceDriverCreateInput): Promise<WorkspaceDriverCreation>;
|
||||
inspect(workspaceId: WorkspaceId): Promise<WorkspaceDriverInspection>;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { hostWorkspaceHelper } from "./host-helper.js";
|
||||
|
||||
describe("hostWorkspaceHelper", () => {
|
||||
test("uses the external host environment plus Electron Node mode", () => {
|
||||
expect(hostWorkspaceHelper.env.PATH).toBe(process.env.PATH);
|
||||
expect(hostWorkspaceHelper.env.ELECTRON_RUN_AS_NODE).toBe("1");
|
||||
expect(hostWorkspaceHelper.env.PASEO_DESKTOP_MANAGED).toBeUndefined();
|
||||
expect(hostWorkspaceHelper.env.PASEO_SUPERVISED).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,11 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const hostWorkspaceHelperCommand = [
|
||||
process.execPath,
|
||||
fileURLToPath(new URL("../../workspace-helper/executable.mjs", import.meta.url)),
|
||||
] as const;
|
||||
import { buildSelfNodeCommand } from "../../paseo-env.js";
|
||||
|
||||
const executable = fileURLToPath(new URL("../../workspace-helper/executable.mjs", import.meta.url));
|
||||
const launch = buildSelfNodeCommand([executable]);
|
||||
|
||||
export const hostWorkspaceHelper = {
|
||||
command: [launch.command, ...launch.args] as readonly [string, ...string[]],
|
||||
env: launch.env,
|
||||
};
|
||||
|
||||
@@ -9,7 +9,7 @@ import type {
|
||||
WorkspaceRuntimeDriver,
|
||||
} from "../drivers/index.js";
|
||||
import { resolveRuntimeCwd, spawnHostProcess, spawnHostPty } from "./host-process.js";
|
||||
import { hostWorkspaceHelperCommand } from "./host-helper.js";
|
||||
import { hostWorkspaceHelper } from "./host-helper.js";
|
||||
import type { HostGitObservationOwner } from "./host-git-observation.js";
|
||||
import { createRuntimeStateStore } from "./runtime-state.js";
|
||||
|
||||
@@ -55,7 +55,7 @@ export function createLocalRuntime(
|
||||
return {
|
||||
id: "local",
|
||||
requiresGitProject: false,
|
||||
workspaceHelperCommand: hostWorkspaceHelperCommand,
|
||||
workspaceHelper: hostWorkspaceHelper,
|
||||
scriptTerminal: { kind: "persistent-shell" },
|
||||
async create(input: WorkspaceDriverCreateInput) {
|
||||
const existing = await inspect(input.workspaceId);
|
||||
|
||||
@@ -183,7 +183,7 @@ export function createService(
|
||||
ownsNewResource = before.status === "missing";
|
||||
if (ready.state.lifecycle === "ready") {
|
||||
const helper = bindWorkspaceHelper({
|
||||
command: driver.workspaceHelperCommand,
|
||||
command: driver.workspaceHelper.command,
|
||||
launch: (argv) => launchHelper(driver, input.workspaceId, argv),
|
||||
});
|
||||
try {
|
||||
@@ -544,7 +544,7 @@ export function createService(
|
||||
}
|
||||
if (cached) await cached.client.close();
|
||||
const client = bindWorkspaceHelper({
|
||||
command: driver.workspaceHelperCommand,
|
||||
command: driver.workspaceHelper.command,
|
||||
launch: async (argv) => {
|
||||
return launchHelper(driver, workspaceId, argv);
|
||||
},
|
||||
@@ -626,7 +626,7 @@ export function createService(
|
||||
const runtimeProcess = await driver.spawn({
|
||||
workspaceId,
|
||||
argv,
|
||||
env: {},
|
||||
env: driver.workspaceHelper.env,
|
||||
purpose: { kind: "workspace-helper" },
|
||||
stdio: { kind: "pipes" },
|
||||
});
|
||||
|
||||
@@ -15,7 +15,7 @@ import type {
|
||||
WorkspaceRuntimeDriver,
|
||||
} from "../drivers/index.js";
|
||||
import { resolveRuntimeCwd, spawnHostProcess, spawnHostPty } from "./host-process.js";
|
||||
import { hostWorkspaceHelperCommand } from "./host-helper.js";
|
||||
import { hostWorkspaceHelper } from "./host-helper.js";
|
||||
import type { HostGitObservationOwner } from "./host-git-observation.js";
|
||||
import { createRuntimeStateStore } from "./runtime-state.js";
|
||||
import { writePaseoWorktreeFirstAgentBranchAutoNameMetadata } from "../../../utils/worktree-metadata.js";
|
||||
@@ -66,7 +66,7 @@ export function createWorktreeRuntime(options: {
|
||||
return {
|
||||
id: "worktree",
|
||||
requiresGitProject: true,
|
||||
workspaceHelperCommand: hostWorkspaceHelperCommand,
|
||||
workspaceHelper: hostWorkspaceHelper,
|
||||
scriptTerminal: { kind: "persistent-shell" },
|
||||
async create(input: WorkspaceDriverCreateInput) {
|
||||
const existing = await inspect(input.workspaceId);
|
||||
|
||||
@@ -655,6 +655,60 @@ test.each(["error", "hang"] as const)(
|
||||
5_000,
|
||||
);
|
||||
|
||||
test("archive and permanent deletion reap runtime-owned processes when forced cleanup hangs", async () => {
|
||||
const descendantPidFileName = "signal-helper-descendant.pid";
|
||||
const fixture = await createFixture("lifecycle-hanging-cleanup", false, "pty", {
|
||||
signalHelperFailure: "hang",
|
||||
signalHelperDescendantPidFileName: descendantPidFileName,
|
||||
});
|
||||
const pidFile = path.join(fixture.source, "lifecycle-workload.pid");
|
||||
const descendantPidFile = path.join(fixture.source, descendantPidFileName);
|
||||
await fixture.service.create(fixture.createInput);
|
||||
const workload = await fixture.service.run({
|
||||
workspaceId: fixture.workspaceId,
|
||||
argv: [
|
||||
processExecPath(),
|
||||
"-e",
|
||||
`require('node:fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000)`,
|
||||
],
|
||||
env: {},
|
||||
purpose: { kind: "provider-probe", provider: "codex" },
|
||||
});
|
||||
workload.stdin.end();
|
||||
const workloadPid = await vi.waitFor(async () => {
|
||||
const pid = Number(await readFile(pidFile, "utf8"));
|
||||
expect(processExists(pid)).toBe(true);
|
||||
return pid;
|
||||
});
|
||||
let descendantPid: number | undefined;
|
||||
|
||||
try {
|
||||
await fixture.service.archive(fixture.workspaceId);
|
||||
descendantPid = Number(await readFile(descendantPidFile, "utf8"));
|
||||
expect(fixture.archivedWorkspaceIds).toContain(fixture.workspaceId);
|
||||
await expect(workload.exited).rejects.toThrow("forced process cleanup failed");
|
||||
expect(processExists(workloadPid)).toBe(false);
|
||||
expect(processExists(descendantPid)).toBe(false);
|
||||
expect(runtimeAdapterProcesses(fixture.workspaceId)).toEqual([]);
|
||||
|
||||
await fixture.service.destroy(fixture.workspaceId);
|
||||
await fixture.service.destroy(fixture.workspaceId);
|
||||
expect(fixture.archivedWorkspaceIds).not.toContain(fixture.workspaceId);
|
||||
expect(await readdir(fixture.stateDirectory)).toEqual([]);
|
||||
expect(runtimeAdapterProcesses(fixture.workspaceId)).toEqual([]);
|
||||
} finally {
|
||||
if (processExists(workloadPid)) {
|
||||
try {
|
||||
process.kill(-workloadPid, "SIGKILL");
|
||||
} catch {
|
||||
// Failure-only containment for the process ownership assertion above.
|
||||
}
|
||||
}
|
||||
if (descendantPid && processExists(descendantPid)) process.kill(descendantPid, "SIGKILL");
|
||||
await fixture.service.destroy(fixture.workspaceId).catch(() => undefined);
|
||||
}
|
||||
}, 10_000);
|
||||
|
||||
test("run admission racing pause cannot leave an unregistered workload running", async () => {
|
||||
const fixture = await createFixture("race", true);
|
||||
await fixture.service.create(fixture.createInput);
|
||||
@@ -714,6 +768,7 @@ async function createFixture(
|
||||
const barrierDirectory = path.join(root, "barrier");
|
||||
await Promise.all([mkdir(source), mkdir(stateDirectory), mkdir(barrierDirectory)]);
|
||||
const runtimeIds = new Map<string, string>();
|
||||
const archivedWorkspaceIds = new Set<string>();
|
||||
const workspaceId = `${name}-workspace`;
|
||||
const service = createWorkspaceRuntimeService({
|
||||
paseoHome: path.join(root, "paseo-home"),
|
||||
@@ -722,8 +777,11 @@ async function createFixture(
|
||||
runtimeIds.set(id, runtimeId);
|
||||
},
|
||||
beginWorkspaceDeletion: async () => {},
|
||||
archiveWorkspaceRecord: async (id) => void archivedWorkspaceIds.add(id),
|
||||
restoreWorkspaceRecord: async (id) => void archivedWorkspaceIds.delete(id),
|
||||
removeWorkspaceRecord: async (id) => {
|
||||
runtimeIds.delete(id);
|
||||
archivedWorkspaceIds.delete(id);
|
||||
},
|
||||
externalRuntimes: {
|
||||
fixture: {
|
||||
@@ -751,6 +809,7 @@ async function createFixture(
|
||||
stateDirectory,
|
||||
barrierDirectory,
|
||||
workspaceId,
|
||||
archivedWorkspaceIds,
|
||||
service,
|
||||
createInput: {
|
||||
workspaceId,
|
||||
@@ -780,6 +839,12 @@ function processExists(pid: number): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeAdapterProcesses(workspaceId: string): string[] {
|
||||
return execFileSync("ps", ["-axo", "command="], { encoding: "utf8" })
|
||||
.split("\n")
|
||||
.filter((command) => command.includes(fixtureExecutable) && command.includes(workspaceId));
|
||||
}
|
||||
|
||||
function nextFile(directory: string, expectedName: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const watcher = watch(directory, (_event, filename) => {
|
||||
|
||||
@@ -367,6 +367,82 @@ test("the command runtime owns Docker workspace materialization, pipes, and life
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
test("production archive and deletion reap Docker runtime processes and owned resources", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "paseo-runtime-docker-lifecycle-cleanup-"));
|
||||
const repo = await createRepository(root);
|
||||
const paseoHome = path.join(root, "paseo-home");
|
||||
const workspaceId = `docker-lifecycle-cleanup-${Date.now()}`;
|
||||
const runtimeIds = new Map<string, string>();
|
||||
const archivedWorkspaceIds = new Set<string>();
|
||||
const service = createWorkspaceRuntimeService({
|
||||
paseoHome,
|
||||
resolveRuntimeId: async (id) => runtimeIds.get(id) ?? null,
|
||||
persistRuntimeId: async (id, runtimeId) => void runtimeIds.set(id, runtimeId),
|
||||
beginWorkspaceDeletion: async () => {},
|
||||
archiveWorkspaceRecord: async (id) => void archivedWorkspaceIds.add(id),
|
||||
restoreWorkspaceRecord: async (id) => void archivedWorkspaceIds.delete(id),
|
||||
removeWorkspaceRecord: async (id) => {
|
||||
runtimeIds.delete(id);
|
||||
archivedWorkspaceIds.delete(id);
|
||||
},
|
||||
externalRuntimes: { docker: { type: "docker", image } },
|
||||
});
|
||||
|
||||
onTestFinished(async () => {
|
||||
await service.destroy(workspaceId).catch(() => undefined);
|
||||
const leakedContainers = dockerResourceCount("ps", workspaceId);
|
||||
const leakedVolumes = dockerResourceCount("volume", workspaceId);
|
||||
const leakedProcesses = dockerTestProcessPids(new Set([workspaceId]));
|
||||
await rm(root, { recursive: true, force: true });
|
||||
if (leakedContainers || leakedVolumes || leakedProcesses.length) {
|
||||
throw new Error(
|
||||
`Production lifecycle leaked containers=${leakedContainers} volumes=${leakedVolumes} processes=${leakedProcesses.join(",")}`,
|
||||
);
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
await service.create({
|
||||
workspaceId,
|
||||
runtimeId: "docker",
|
||||
project: {
|
||||
id: "docker-lifecycle-cleanup-project",
|
||||
source: { kind: "host-directory", path: repo },
|
||||
},
|
||||
placement: { kind: "existing" },
|
||||
purpose: "provider-probe",
|
||||
});
|
||||
const runtimeProcess = await service.run({
|
||||
workspaceId,
|
||||
argv: [
|
||||
"/usr/local/bin/node",
|
||||
"-e",
|
||||
"process.on('SIGTERM',()=>{});console.log('owned-ready');setInterval(()=>{},1000)",
|
||||
],
|
||||
env: {},
|
||||
purpose: { kind: "provider-probe", provider: "codex" },
|
||||
});
|
||||
runtimeProcess.stdin.end();
|
||||
await expect(waitForOutput(runtimeProcess.stdout, "owned-ready")).resolves.toContain(
|
||||
"owned-ready",
|
||||
);
|
||||
|
||||
await service.archive(workspaceId);
|
||||
expect(archivedWorkspaceIds).toContain(workspaceId);
|
||||
await expect(runtimeProcess.exited).resolves.toEqual({ code: null, signal: "SIGKILL" });
|
||||
expect(dockerRunningContainerCount(workspaceId)).toBe(0);
|
||||
expect(dockerResourceCount("ps", workspaceId)).toBe(1);
|
||||
expect(dockerResourceCount("volume", workspaceId)).toBe(1);
|
||||
expect(dockerTestProcessPids(new Set([workspaceId]))).toEqual([]);
|
||||
|
||||
await service.destroy(workspaceId);
|
||||
await service.destroy(workspaceId);
|
||||
expect(runtimeIds.has(workspaceId)).toBe(false);
|
||||
expect(archivedWorkspaceIds.has(workspaceId)).toBe(false);
|
||||
expect(dockerResourceCount("ps", workspaceId)).toBe(0);
|
||||
expect(dockerResourceCount("volume", workspaceId)).toBe(0);
|
||||
expect(dockerTestProcessPids(new Set([workspaceId]))).toEqual([]);
|
||||
}, 60_000);
|
||||
|
||||
test("Docker buffers immediate pipe signals until the workload identity is authoritative", async () => {
|
||||
const fixture = await dockerTestFixture("paseo-runtime-docker-started-");
|
||||
const root = fixture.root;
|
||||
|
||||
Reference in New Issue
Block a user