feat(agents/orb): harden Docker sandbox lifecycle, proof staging, and tests
Fix detached process startup in docker-sandbox.ts: capture the real process group PID via setsid, redirect stdout/stderr to durable log files, refresh status and exit code after natural completion, make getProcessLogs return real logs, and implement bounded TERM/KILL cleanup with no leftover containers. Switch the adapter to node:child_process (supported by Bun) so tests execute under vitest, add -i to docker exec so staged files reach cat, and assert the launcher is nonempty before launch. Ensure Orb creation disposes the VM before its sandbox on partial failure (AgentOS creation or OpenCode linking), and dispose the VM before its sandbox in OrbHandle.dispose. Fix the Effect state callbacks (tapError/sync/asVoid) and prepareRepository branch/base shell quoting. Split normalizeSessionEvent to stay under the complexity limit without suppressing the rule. Raw outbound prompts stay unchanged; only persisted and logged event copies are redacted. Rewrite scripts/orb-proof.ts to emit explicit stable stage markers for Docker, AgentOS/OpenCode, and the model turn; ORB_GATEWAY_* fall back safely to AGENT_MODEL_* without printing secrets; an unreachable gateway is BLOCKED not passed; container removal is verified after disposal. Strengthen tests: drop the any-failure createOrb test and add focused Docker lifecycle coverage (PID, logs, status, exit code, stop/kill, removal) guarded only when Docker is unavailable.
This commit is contained in:
-Puter
405
scripts/orb-proof.ts
Normal file
405
scripts/orb-proof.ts
Normal file
@@ -0,0 +1,405 @@
|
||||
/**
|
||||
* Orb Runtime Proof Fixture
|
||||
*
|
||||
* Opt-in integration proof that exercises the Orb lifecycle in distinct,
|
||||
* independently reported stages, each with a stable marker:
|
||||
*
|
||||
* STAGE 1 Docker sandbox lifecycle ORB_STAGE_DOCKER_OK / _BLOCKED
|
||||
* STAGE 2 AgentOS VM + OpenCode session ORB_STAGE_AGENTOS_OK / _BLOCKED
|
||||
* STAGE 3 Model turn via the gateway ORB_STAGE_MODEL_TURN_OK / _BLOCKED
|
||||
*
|
||||
* A missing or unreachable model gateway is BLOCKED, never passed. The proof
|
||||
* only prints ORB_PROOF_PASSED when the model turn completes. It never prints
|
||||
* gateway secrets; only provider/model identifiers and reachability labels.
|
||||
*
|
||||
* Gateway credentials resolve from ORB_GATEWAY_* with safe fallbacks to the
|
||||
* shared AGENT_MODEL_* variables. Container removal is always verified after
|
||||
* disposal.
|
||||
*
|
||||
* Usage:
|
||||
* ORB_PROOF=1 \
|
||||
* ORB_GATEWAY_API_KEY=... \
|
||||
* ORB_GATEWAY_BASE_URL=https://ai.example.com/v1 \
|
||||
* ORB_GATEWAY_MODEL=glm-5.2 \
|
||||
* ORB_GATEWAY_PROVIDER=cheaptricks \
|
||||
* bun run scripts/orb-proof.ts
|
||||
*/
|
||||
/* eslint-disable no-console -- proof fixture is a CLI tool */
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { Effect } from "effect";
|
||||
|
||||
import { DockerSandboxProvider } from "../packages/agents/src/orb/docker-sandbox";
|
||||
import type { DockerSandboxOptions } from "../packages/agents/src/orb/docker-sandbox";
|
||||
import type { OrbEvent } from "../packages/agents/src/orb/events";
|
||||
import { OrbRuntime } from "../packages/agents/src/orb/runtime";
|
||||
import type { OrbHandle } from "../packages/agents/src/orb/runtime";
|
||||
|
||||
const MARKER = {
|
||||
blocked: "ORB_PROOF_BLOCKED",
|
||||
failed: "ORB_PROOF_FAILED",
|
||||
passed: "ORB_PROOF_PASSED",
|
||||
} as const;
|
||||
|
||||
const STAGE = {
|
||||
agentosBlocked: "ORB_STAGE_AGENTOS_BLOCKED",
|
||||
agentosOk: "ORB_STAGE_AGENTOS_OK",
|
||||
dockerBlocked: "ORB_STAGE_DOCKER_BLOCKED",
|
||||
dockerOk: "ORB_STAGE_DOCKER_OK",
|
||||
modelBlocked: "ORB_STAGE_MODEL_TURN_BLOCKED",
|
||||
modelOk: "ORB_STAGE_MODEL_TURN_OK",
|
||||
} as const;
|
||||
|
||||
interface Gateway {
|
||||
apiKey: string;
|
||||
baseUrl: string;
|
||||
model: string;
|
||||
provider: string;
|
||||
}
|
||||
|
||||
const envValue = (key: string): string | undefined => process.env[key];
|
||||
|
||||
const resolveGateway = (): Gateway | null => {
|
||||
const apiKey =
|
||||
envValue("ORB_GATEWAY_API_KEY") ?? envValue("AGENT_MODEL_API_KEY");
|
||||
const baseUrl =
|
||||
envValue("ORB_GATEWAY_BASE_URL") ?? envValue("AGENT_MODEL_BASE_URL");
|
||||
const model = envValue("ORB_GATEWAY_MODEL") ?? envValue("AGENT_MODEL_NAME");
|
||||
const provider =
|
||||
envValue("ORB_GATEWAY_PROVIDER") ?? envValue("AGENT_MODEL_PROVIDER");
|
||||
if (
|
||||
!apiKey?.trim() ||
|
||||
!baseUrl?.trim() ||
|
||||
!model?.trim() ||
|
||||
!provider?.trim()
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return { apiKey, baseUrl, model, provider };
|
||||
};
|
||||
|
||||
const runCli = async (args: readonly string[]): Promise<string> => {
|
||||
try {
|
||||
const proc = Bun.spawn([...args], { stderr: "pipe", stdout: "pipe" });
|
||||
const stdout = await new Response(proc.stdout).text();
|
||||
await proc.exited;
|
||||
return stdout;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
};
|
||||
|
||||
const dockerVersion = async (): Promise<string | null> => {
|
||||
const out = await runCli([
|
||||
"docker",
|
||||
"version",
|
||||
"--format",
|
||||
"{{.Server.Version}}",
|
||||
]);
|
||||
const trimmed = out.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
};
|
||||
|
||||
const containerExists = async (name: string): Promise<boolean> => {
|
||||
const listing = await runCli([
|
||||
"docker",
|
||||
"ps",
|
||||
"-a",
|
||||
"--filter",
|
||||
`name=^${name}$`,
|
||||
"--format",
|
||||
"{{.ID}}",
|
||||
]);
|
||||
return listing.trim().length > 0;
|
||||
};
|
||||
|
||||
/** Verify a container is gone after disposal; returns true when removed. */
|
||||
const verifyRemoved = async (name: string): Promise<boolean> => {
|
||||
const present = await containerExists(name);
|
||||
console.log(
|
||||
` [verify] container ${name} ${present ? "STILL PRESENT" : "removed"}`
|
||||
);
|
||||
return !present;
|
||||
};
|
||||
|
||||
const TINY_PROJECT: Record<string, string> = {
|
||||
"index.test.ts": `import { add } from "./index";\nimport { expect, test } from "bun:test";\n\ntest("add", () => {\n expect(add(1, 2)).toBe(3);\n});\n`,
|
||||
"index.ts": `export function add(a: number, b: number): number {\n return a + b;\n}\n`,
|
||||
"package.json": `{"name":"tiny","scripts":{"test":"bun test"}}\n`,
|
||||
};
|
||||
|
||||
const prepareProject = async (hostWorkspace: string): Promise<void> => {
|
||||
await mkdir(path.join(hostWorkspace, "repository"), { recursive: true });
|
||||
for (const [file, content] of Object.entries(TINY_PROJECT)) {
|
||||
// eslint-disable-next-line no-await-in-loop -- sequential file writes are intentional
|
||||
await writeFile(path.join(hostWorkspace, "repository", file), content);
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// STAGE 1 — Docker sandbox lifecycle (standalone, no sidecar required)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const stageDocker = async (image: string | undefined): Promise<boolean> => {
|
||||
const version = await dockerVersion();
|
||||
if (!version) {
|
||||
console.log("[docker] Docker daemon not available");
|
||||
console.log(STAGE.dockerBlocked);
|
||||
return false;
|
||||
}
|
||||
console.log(`[docker] server version ${version}`);
|
||||
|
||||
const container = `orb-proof-docker-${Date.now()}`;
|
||||
const hostWorkspace = `/tmp/orb-proof-docker-${Date.now()}`;
|
||||
const options: DockerSandboxOptions =
|
||||
image === undefined
|
||||
? { containerName: container, hostWorkspacePath: hostWorkspace }
|
||||
: { containerName: container, hostWorkspacePath: hostWorkspace, image };
|
||||
|
||||
try {
|
||||
await prepareProject(hostWorkspace);
|
||||
const provider = await Effect.runPromise(
|
||||
DockerSandboxProvider.create(options)
|
||||
);
|
||||
const client = await provider.start();
|
||||
|
||||
console.log("[docker] running bun install in sandbox...");
|
||||
const install = await client.runProcess({
|
||||
command: "bun install",
|
||||
cwd: "/workspace/repository",
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
console.log(` bun install exit: ${install.exitCode}`);
|
||||
|
||||
console.log("[docker] running bun test in sandbox...");
|
||||
const test = await client.runProcess({
|
||||
command: "bun test",
|
||||
cwd: "/workspace/repository",
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
console.log(` bun test exit: ${test.exitCode}`);
|
||||
|
||||
await provider.dispose();
|
||||
const removed = await verifyRemoved(container);
|
||||
if (install.exitCode !== 0 || test.exitCode !== 0 || !removed) {
|
||||
console.log("[docker] sandbox lifecycle did not complete cleanly");
|
||||
console.log(STAGE.dockerBlocked);
|
||||
return false;
|
||||
}
|
||||
console.log(STAGE.dockerOk);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(
|
||||
`[docker] stage failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
await runCli(["docker", "rm", "-f", container]);
|
||||
await verifyRemoved(container);
|
||||
console.log(STAGE.dockerBlocked);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// STAGE 2 — AgentOS VM + OpenCode session
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const stageAgentOs = async (
|
||||
gateway: Gateway,
|
||||
image: string | undefined,
|
||||
events: OrbEvent[]
|
||||
): Promise<{ container: string; handle: OrbHandle } | null> => {
|
||||
const container = `orb-proof-${Date.now()}`;
|
||||
const hostWorkspace = `/tmp/orb-proof-${Date.now()}`;
|
||||
const runtime = new OrbRuntime(
|
||||
image === undefined ? {} : { dockerImage: image }
|
||||
);
|
||||
|
||||
const createResult = await Effect.runPromise(
|
||||
Effect.match(
|
||||
runtime.createOrb({
|
||||
context: {
|
||||
artifacts: [],
|
||||
contextFiles: [],
|
||||
issueBody: "Run bun test and report the result.",
|
||||
issueTitle: "Proof: run tests",
|
||||
},
|
||||
docker: { containerName: container, hostWorkspacePath: hostWorkspace },
|
||||
gateway,
|
||||
identity: {
|
||||
projectId: "proof",
|
||||
runId: `proof-${Date.now()}`,
|
||||
workUnitId: "proof-wu",
|
||||
},
|
||||
}),
|
||||
{
|
||||
onFailure: (error) => ({ error, ok: false as const }),
|
||||
onSuccess: (handle) => ({ handle, ok: true as const }),
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
if (!createResult.ok) {
|
||||
console.log(
|
||||
`[agentos] creation failed (${createResult.error.reason}): ${createResult.error.message}`
|
||||
);
|
||||
await verifyRemoved(container);
|
||||
console.log(STAGE.agentosBlocked);
|
||||
return null;
|
||||
}
|
||||
|
||||
const { handle } = createResult;
|
||||
handle.onEvent((event) => {
|
||||
events.push(event);
|
||||
const summary = (event.text ?? event.command ?? event.type).slice(0, 160);
|
||||
console.log(` [event] ${event.type}: ${summary}`);
|
||||
});
|
||||
|
||||
try {
|
||||
const vm = handle.vmInstance;
|
||||
if (!vm) {
|
||||
throw new Error("AgentOS VM is not attached");
|
||||
}
|
||||
const agents = await vm.listAgents();
|
||||
const opencode = agents.find((a) => a.id === "opencode" && a.installed);
|
||||
if (!opencode) {
|
||||
throw new Error("OpenCode agent is not installed in the VM");
|
||||
}
|
||||
console.log("[agentos] OpenCode agent available");
|
||||
|
||||
await Effect.runPromise(handle.openSession());
|
||||
console.log(`[agentos] session opened: ${handle.currentSessionId ?? "?"}`);
|
||||
console.log(STAGE.agentosOk);
|
||||
return { container, handle };
|
||||
} catch (error) {
|
||||
console.log(
|
||||
`[agentos] stage failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
await Effect.runPromise(handle.dispose().pipe(Effect.ignore)).catch(() => {
|
||||
// best-effort cleanup
|
||||
});
|
||||
await verifyRemoved(container);
|
||||
console.log(STAGE.agentosBlocked);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// STAGE 3 — Model turn through the gateway
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const stageModelTurn = async (handle: OrbHandle): Promise<boolean> => {
|
||||
const vm = handle.vmInstance;
|
||||
const sessionId = handle.currentSessionId;
|
||||
if (!vm || !sessionId) {
|
||||
console.log("[model] no active session to drive a model turn");
|
||||
console.log(STAGE.modelBlocked);
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
console.log("[model] sending bounded turn to gateway...");
|
||||
const result = await vm.prompt({
|
||||
content: [
|
||||
{ text: "Reply with exactly: ORB_PROOF_MODEL_ECHO", type: "text" },
|
||||
],
|
||||
sessionId,
|
||||
});
|
||||
console.log(
|
||||
`[model] turn completed (stopReason=${result.stopReason ?? "?"})`
|
||||
);
|
||||
console.log(STAGE.modelOk);
|
||||
return true;
|
||||
} catch (error) {
|
||||
// An unreachable/missing gateway surfaces here as a prompt failure.
|
||||
console.log(
|
||||
`[model] gateway turn failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
console.log(STAGE.modelBlocked);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Orchestration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const runProof = async (): Promise<number> => {
|
||||
const image = envValue("ORB_DOCKER_IMAGE");
|
||||
const events: OrbEvent[] = [];
|
||||
|
||||
// STAGE 1 — Docker sandbox.
|
||||
console.log("[stage 1] Docker sandbox lifecycle...");
|
||||
const dockerOk = await stageDocker(image);
|
||||
if (!dockerOk) {
|
||||
console.log(`\n${MARKER.blocked}`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
// STAGE 2 + 3 — AgentOS/OpenCode + model turn.
|
||||
const gateway = resolveGateway();
|
||||
if (!gateway) {
|
||||
console.error(
|
||||
"[gateway] missing config: set ORB_GATEWAY_* (or AGENT_MODEL_* fallbacks)."
|
||||
);
|
||||
console.log(STAGE.agentosBlocked);
|
||||
console.log(`\n${MARKER.blocked}`);
|
||||
return 2;
|
||||
}
|
||||
// Identifiers only — never the key or base URL.
|
||||
console.log(`[gateway] provider=${gateway.provider} model=${gateway.model}`);
|
||||
|
||||
console.log("[stage 2] AgentOS VM + OpenCode session...");
|
||||
const orb = await stageAgentOs(gateway, image, events);
|
||||
if (!orb) {
|
||||
console.log(`\n${MARKER.blocked}`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
console.log("[stage 3] model turn...");
|
||||
let modelOk = false;
|
||||
try {
|
||||
modelOk = await stageModelTurn(orb.handle);
|
||||
} finally {
|
||||
console.log("[dispose] disposing Orb...");
|
||||
await Effect.runPromise(orb.handle.dispose().pipe(Effect.ignore)).catch(
|
||||
() => {
|
||||
// best-effort cleanup
|
||||
}
|
||||
);
|
||||
const removed = await verifyRemoved(orb.container);
|
||||
if (!removed) {
|
||||
console.log("[dispose] container removal could not be verified");
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\nStreamed ${events.length} normalized events (${[...new Set(events.map((e) => e.type))].join(", ")}).`
|
||||
);
|
||||
|
||||
if (modelOk) {
|
||||
console.log(`\n${MARKER.passed}`);
|
||||
return 0;
|
||||
}
|
||||
console.log(`\n${MARKER.blocked}`);
|
||||
return 2;
|
||||
};
|
||||
|
||||
// Entry point
|
||||
if (envValue("ORB_PROOF") !== "1") {
|
||||
console.error(
|
||||
"Set ORB_PROOF=1 to run the proof fixture (requires Docker + model gateway)."
|
||||
);
|
||||
console.error(MARKER.blocked);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const code = await runProof();
|
||||
process.exit(code);
|
||||
} catch (error) {
|
||||
console.error("[proof] unexpected failure:", error);
|
||||
console.error(MARKER.failed);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
Reference in New Issue
Block a user