test: add Zopu integration smoke harness
This commit is contained in:
-Puter
829
scripts/zopu-smoke.ts
Normal file
829
scripts/zopu-smoke.ts
Normal file
@@ -0,0 +1,829 @@
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
import { api } from "@code/backend/convex/_generated/api";
|
||||
import { parseSmokeEnv } from "@code/env/smoke";
|
||||
import type { SmokeEnv } from "@code/env/smoke";
|
||||
import {
|
||||
advanceSmokeLoop,
|
||||
decodeSmokeChatText,
|
||||
decodeSmokeModelCatalog,
|
||||
decodeSmokePlan,
|
||||
decodeSmokeReview,
|
||||
decodeSmokeWorkerReport,
|
||||
initialSmokeLoopState,
|
||||
redactSmokeText,
|
||||
selectSmokeModelRoles,
|
||||
validateSmokeCapabilities,
|
||||
} from "@code/primitives/smoke";
|
||||
import type {
|
||||
SmokeCapabilityCheck,
|
||||
SmokeLoopState,
|
||||
SmokeModelRoles,
|
||||
} from "@code/primitives/smoke";
|
||||
import { createFlueClient } from "@flue/sdk";
|
||||
import { ConvexHttpClient } from "convex/browser";
|
||||
import { Effect } from "effect";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const REQUIRED_ARTIFACTS = [
|
||||
"agent.md",
|
||||
"work.md",
|
||||
"steps.md",
|
||||
"artifacts.md",
|
||||
"signals.md",
|
||||
"agent-manager.md",
|
||||
"context.md",
|
||||
"card.md",
|
||||
] as const;
|
||||
|
||||
type SmokeStatus =
|
||||
| "preflight-passed"
|
||||
| "completed"
|
||||
| "contract-blocked"
|
||||
| "failed";
|
||||
|
||||
interface SmokeReport {
|
||||
readonly checks: readonly SmokeCapabilityCheck[];
|
||||
readonly featureRequest: string;
|
||||
readonly issueId?: string;
|
||||
readonly loop: SmokeLoopState;
|
||||
readonly mode: "preflight" | "run";
|
||||
readonly roles?: Pick<SmokeModelRoles, "plannerReviewer" | "worker">;
|
||||
readonly review?: {
|
||||
readonly approved: boolean;
|
||||
readonly findingsCount: number;
|
||||
};
|
||||
readonly status: SmokeStatus;
|
||||
readonly worker?: {
|
||||
readonly status: "completed" | "blocked" | "failed";
|
||||
};
|
||||
}
|
||||
|
||||
interface ChatMessage {
|
||||
readonly content: string;
|
||||
readonly role: "system" | "user";
|
||||
}
|
||||
|
||||
const { values, positionals } = parseArgs({
|
||||
allowPositionals: true,
|
||||
args: process.argv.slice(2),
|
||||
options: {
|
||||
help: { default: false, short: "h", type: "boolean" },
|
||||
report: { type: "string" },
|
||||
run: { default: false, type: "boolean" },
|
||||
"timeout-ms": { default: String(DEFAULT_TIMEOUT_MS), type: "string" },
|
||||
},
|
||||
strict: true,
|
||||
});
|
||||
|
||||
const usage = `Usage: bun run smoke:zopu [--run] [--report <path>] [--timeout-ms <ms>]
|
||||
|
||||
Without --run, the command only probes the local web/Convex/Flue/AgentOS and CPA contracts.
|
||||
With --run, it creates one issue in the explicit smoke project, drives the worker, and sends the result to the GLM planner/reviewer.
|
||||
|
||||
Required environment:
|
||||
ZOPU_SMOKE_ACCESS_TOKEN Better Auth/Convex bearer token
|
||||
ZOPU_SMOKE_PROJECT_ID disposable/test project id
|
||||
ZOPU_SMOKE_CPA_API_KEY CPA key (or AGENT_MODEL_API_KEY)
|
||||
ZOPU_SMOKE_CPA_BASE_URL CPA OpenAI-compatible /v1 URL (or AGENT_MODEL_BASE_URL)
|
||||
ZOPU_SMOKE_CONVEX_URL Convex URL (or CONVEX_URL)
|
||||
ZOPU_SMOKE_DAEMON_ID online local daemon id (or DAEMON_ID)
|
||||
|
||||
Optional environment:
|
||||
ZOPU_SMOKE_FEATURE_REQUEST override the tiny feature request
|
||||
ZOPU_SMOKE_WEB_URL default: SITE_URL or http://localhost:5173
|
||||
ZOPU_SMOKE_FLUE_URL default: VITE_FLUE_URL or http://localhost:3583
|
||||
`;
|
||||
|
||||
if (values.help || positionals.length > 0) {
|
||||
console.log(usage);
|
||||
process.exit(positionals.length > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
const runEffect = <A, E>(effect: Effect.Effect<A, E>): A =>
|
||||
Effect.runSync(effect);
|
||||
|
||||
const errorMessage = (error: unknown, secrets: readonly string[]): string => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return redactSmokeText(message, secrets).replaceAll(/\s+/gu, " ").trim();
|
||||
};
|
||||
|
||||
const withTimeout = (timeoutMs: number): AbortSignal =>
|
||||
AbortSignal.timeout(timeoutMs);
|
||||
|
||||
const normalizeBaseUrl = (value: string): URL => {
|
||||
const url = new URL(value);
|
||||
if (!url.pathname.endsWith("/")) {
|
||||
url.pathname = `${url.pathname}/`;
|
||||
}
|
||||
return url;
|
||||
};
|
||||
|
||||
const endpoint = (baseUrl: URL, path: string): string =>
|
||||
new URL(path, baseUrl).toString();
|
||||
|
||||
const addCheck = (
|
||||
checks: SmokeCapabilityCheck[],
|
||||
id: string,
|
||||
passed: boolean,
|
||||
message: string
|
||||
) => {
|
||||
checks.push({ id, message, passed });
|
||||
};
|
||||
|
||||
const probeHttp = async (
|
||||
checks: SmokeCapabilityCheck[],
|
||||
id: string,
|
||||
label: string,
|
||||
url: string,
|
||||
timeoutMs: number
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
redirect: "manual",
|
||||
signal: withTimeout(timeoutMs),
|
||||
});
|
||||
const passed = response.status >= 200 && response.status < 400;
|
||||
addCheck(
|
||||
checks,
|
||||
id,
|
||||
passed,
|
||||
passed
|
||||
? `${label} responded with HTTP ${response.status}`
|
||||
: `${label} returned HTTP ${response.status}`
|
||||
);
|
||||
} catch (error) {
|
||||
addCheck(
|
||||
checks,
|
||||
id,
|
||||
false,
|
||||
`${label} is unreachable: ${errorMessage(error, [])}`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const callOpenAiChat = async (input: {
|
||||
readonly apiKey: string;
|
||||
readonly baseUrl: URL;
|
||||
readonly messages: readonly ChatMessage[];
|
||||
readonly model: string;
|
||||
readonly timeoutMs: number;
|
||||
}): Promise<string> => {
|
||||
const response = await fetch(endpoint(input.baseUrl, "chat/completions"), {
|
||||
body: JSON.stringify({
|
||||
max_tokens: 512,
|
||||
messages: input.messages,
|
||||
model: input.model,
|
||||
temperature: 0,
|
||||
}),
|
||||
headers: {
|
||||
Authorization: `Bearer ${input.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method: "POST",
|
||||
signal: withTimeout(input.timeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`CPA ${input.model} completion returned HTTP ${response.status}`
|
||||
);
|
||||
}
|
||||
const body = (await response.json()) as unknown;
|
||||
return await Effect.runPromise(decodeSmokeChatText(body));
|
||||
};
|
||||
|
||||
const parseJsonObject = (text: string): unknown => {
|
||||
const withoutFence = text
|
||||
.trim()
|
||||
.replace(/^```(?:json)?\s*/iu, "")
|
||||
.replace(/\s*```$/u, "");
|
||||
const start = withoutFence.indexOf("{");
|
||||
const end = withoutFence.lastIndexOf("}");
|
||||
if (start === -1 || end <= start) {
|
||||
throw new Error("model response did not contain a JSON object");
|
||||
}
|
||||
return JSON.parse(withoutFence.slice(start, end + 1)) as unknown;
|
||||
};
|
||||
|
||||
const buildPlanPrompt = (featureRequest: string): readonly ChatMessage[] => [
|
||||
{
|
||||
content:
|
||||
"You are the planner/reviewer for a disposable Zopu web-project smoke. Return JSON only with summary, steps, acceptanceCriteria, and nonGoals. Keep the plan tiny and testable.",
|
||||
role: "system",
|
||||
},
|
||||
{ content: featureRequest, role: "user" },
|
||||
];
|
||||
|
||||
const buildWorkerPrompt = (input: {
|
||||
readonly featureRequest: string;
|
||||
readonly plan: {
|
||||
readonly acceptanceCriteria: readonly string[];
|
||||
readonly nonGoals: readonly string[];
|
||||
readonly steps: readonly string[];
|
||||
readonly summary: string;
|
||||
};
|
||||
}): string => `Execute this one disposable web-project smoke issue.
|
||||
|
||||
Feature request: ${input.featureRequest}
|
||||
|
||||
Plan:
|
||||
${JSON.stringify(input.plan)}
|
||||
|
||||
Rules:
|
||||
- Work only in the bound project workspace.
|
||||
- Use one tool call at a time; never issue parallel tool calls because this worker is minimax-m3.
|
||||
- Inspect the bound repository before editing.
|
||||
- Run the narrowest relevant test or check.
|
||||
- Do not claim completion without observed changed files and command evidence.
|
||||
- Return JSON only: {"status":"completed"|"blocked"|"failed","summary":"..."}`;
|
||||
|
||||
const buildReviewPrompt = (input: {
|
||||
readonly featureRequest: string;
|
||||
readonly plan: unknown;
|
||||
readonly workerReport: unknown;
|
||||
}): readonly ChatMessage[] => [
|
||||
{
|
||||
content:
|
||||
"You are the independent GLM-5.2 planner/reviewer. Review the worker report against the feature request and plan. Return JSON only with approved, summary, and findings (an array of concise strings). Reject missing evidence or a non-completed worker.",
|
||||
role: "system",
|
||||
},
|
||||
{
|
||||
content: JSON.stringify({
|
||||
featureRequest: input.featureRequest,
|
||||
plan: input.plan,
|
||||
workerReport: input.workerReport,
|
||||
}),
|
||||
role: "user",
|
||||
},
|
||||
];
|
||||
|
||||
const printReport = (report: SmokeReport): void => {
|
||||
console.log(`ZOPU_SMOKE_${report.status.toUpperCase().replaceAll("-", "_")}`);
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
};
|
||||
|
||||
const writeReport = async (
|
||||
path: string | undefined,
|
||||
report: SmokeReport
|
||||
): Promise<void> => {
|
||||
if (path !== undefined) {
|
||||
await Bun.write(path, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
};
|
||||
|
||||
const waitForIssueStatus = async (
|
||||
convex: ConvexHttpClient,
|
||||
projectId: string,
|
||||
issueId: string,
|
||||
timeoutMs: number
|
||||
): Promise<string | null> => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
// oxlint-disable-next-line no-await-in-loop -- polling intentionally waits between durable reads.
|
||||
const issues = await convex.query(api.projectIssues.list, {
|
||||
projectId: projectId as never,
|
||||
});
|
||||
const issue = issues.find((candidate) => candidate._id === issueId);
|
||||
if (issue?.status === "completed" || issue?.status === "failed") {
|
||||
return issue.status;
|
||||
}
|
||||
// oxlint-disable-next-line no-await-in-loop -- polling intentionally waits between durable reads.
|
||||
await Bun.sleep(500);
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const probeModelRoles = (
|
||||
env: SmokeEnv,
|
||||
cpaBaseUrl: URL,
|
||||
checks: SmokeCapabilityCheck[],
|
||||
secrets: readonly string[]
|
||||
): SmokeModelRoles | undefined => {
|
||||
try {
|
||||
const roles = runEffect(
|
||||
selectSmokeModelRoles({
|
||||
plannerReviewerModel: "glm-5.2",
|
||||
plannerReviewerProvider: "cheaptricks",
|
||||
workerModel: env.agentModelName ?? "",
|
||||
workerProvider: env.agentModelProvider ?? "cheaptricks",
|
||||
workerToolCallMode: "serial",
|
||||
})
|
||||
);
|
||||
addCheck(
|
||||
checks,
|
||||
"model-roles",
|
||||
true,
|
||||
"worker=minimax-m3; planner-reviewer=glm-5.2"
|
||||
);
|
||||
const workerGatewayMatches =
|
||||
env.agentModelBaseUrl !== undefined &&
|
||||
normalizeBaseUrl(env.agentModelBaseUrl).toString() ===
|
||||
cpaBaseUrl.toString();
|
||||
addCheck(
|
||||
checks,
|
||||
"worker-gateway-config",
|
||||
workerGatewayMatches,
|
||||
workerGatewayMatches
|
||||
? "the local worker points at the configured CPA base URL"
|
||||
: "AGENT_MODEL_BASE_URL must match the CPA base URL; the harness will not rewrite it"
|
||||
);
|
||||
return roles;
|
||||
} catch (error) {
|
||||
addCheck(checks, "model-roles", false, errorMessage(error, secrets));
|
||||
addCheck(
|
||||
checks,
|
||||
"worker-gateway-config",
|
||||
false,
|
||||
"worker gateway configuration could not be checked because the model role contract failed"
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const probeControlPlane = async (input: {
|
||||
readonly checks: SmokeCapabilityCheck[];
|
||||
readonly convex: ConvexHttpClient;
|
||||
readonly daemonId: string;
|
||||
readonly projectId: string | undefined;
|
||||
readonly secrets: readonly string[];
|
||||
}): Promise<void> => {
|
||||
const { checks, convex, daemonId, projectId, secrets } = input;
|
||||
try {
|
||||
const health = await convex.query(api.healthCheck.get, {});
|
||||
addCheck(
|
||||
checks,
|
||||
"convex",
|
||||
health === "OK",
|
||||
health === "OK"
|
||||
? "Convex health query returned OK"
|
||||
: "Convex health query returned an unexpected value"
|
||||
);
|
||||
} catch (error) {
|
||||
addCheck(
|
||||
checks,
|
||||
"convex",
|
||||
false,
|
||||
`Convex health query failed: ${errorMessage(error, secrets)}`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const organization = await convex.query(api.organizations.getCurrent, {});
|
||||
addCheck(
|
||||
checks,
|
||||
"organization",
|
||||
organization !== null,
|
||||
organization
|
||||
? "authenticated personal organization is available"
|
||||
: "no personal organization; sign in through the web app before running the smoke"
|
||||
);
|
||||
} catch (error) {
|
||||
addCheck(
|
||||
checks,
|
||||
"organization",
|
||||
false,
|
||||
`organization query failed: ${errorMessage(error, secrets)}`
|
||||
);
|
||||
}
|
||||
|
||||
if (projectId === undefined) {
|
||||
addCheck(
|
||||
checks,
|
||||
"project-contract",
|
||||
false,
|
||||
"ZOPU_SMOKE_PROJECT_ID is required and must point at a disposable/test project"
|
||||
);
|
||||
} else {
|
||||
// eslint-disable-next-line no-use-before-define -- the project probe is a focused helper below the control-plane flow.
|
||||
await probeProjectContract({ checks, convex, projectId, secrets });
|
||||
}
|
||||
|
||||
try {
|
||||
const daemon = await convex.query(api.daemons.get, { daemonId });
|
||||
const online = daemon.presence?.status === "online";
|
||||
addCheck(
|
||||
checks,
|
||||
"agentos-daemon",
|
||||
online,
|
||||
online
|
||||
? `daemon ${daemonId} is online`
|
||||
: `daemon ${daemonId} is not online; start the local AgentOS daemon without changing CPA`
|
||||
);
|
||||
} catch (error) {
|
||||
addCheck(
|
||||
checks,
|
||||
"agentos-daemon",
|
||||
false,
|
||||
`daemon probe failed: ${errorMessage(error, secrets)}`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const probeProjectContract = async (input: {
|
||||
readonly checks: SmokeCapabilityCheck[];
|
||||
readonly convex: ConvexHttpClient;
|
||||
readonly projectId: string;
|
||||
readonly secrets: readonly string[];
|
||||
}): Promise<void> => {
|
||||
const { checks, convex, projectId, secrets } = input;
|
||||
try {
|
||||
const project = await convex.query(api.projects.get, {
|
||||
projectId: projectId as never,
|
||||
});
|
||||
const hasSource = (project?.sources.length ?? 0) > 0;
|
||||
let projectMessage =
|
||||
"project has no connected repository source; land the project backend lane first";
|
||||
if (project === null) {
|
||||
projectMessage =
|
||||
"project was not found or is not accessible to the supplied token";
|
||||
} else if (hasSource) {
|
||||
projectMessage = "project source is available";
|
||||
}
|
||||
addCheck(
|
||||
checks,
|
||||
"project-contract",
|
||||
project !== null && hasSource,
|
||||
projectMessage
|
||||
);
|
||||
|
||||
const artifacts = await convex.query(api.projectArtifacts.list, {
|
||||
projectId: projectId as never,
|
||||
});
|
||||
const paths = new Set(artifacts.map((artifact) => artifact.path));
|
||||
const missing = REQUIRED_ARTIFACTS.filter((path) => !paths.has(path));
|
||||
addCheck(
|
||||
checks,
|
||||
"agentos-workspace-contract",
|
||||
missing.length === 0,
|
||||
missing.length === 0
|
||||
? "issue-scoped AgentOS artifact contract is present"
|
||||
: `missing AgentOS artifacts: ${missing.join(", ")}; land the project loop lane first`
|
||||
);
|
||||
} catch (error) {
|
||||
addCheck(
|
||||
checks,
|
||||
"project-contract",
|
||||
false,
|
||||
`project query failed: ${errorMessage(error, secrets)}`
|
||||
);
|
||||
addCheck(
|
||||
checks,
|
||||
"agentos-workspace-contract",
|
||||
false,
|
||||
"project artifacts could not be inspected"
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const probeFlue = async (input: {
|
||||
readonly accessToken: string;
|
||||
readonly checks: SmokeCapabilityCheck[];
|
||||
readonly flueUrl: string;
|
||||
readonly secrets: readonly string[];
|
||||
}): Promise<ReturnType<typeof createFlueClient> | undefined> => {
|
||||
const { accessToken, checks, flueUrl, secrets } = input;
|
||||
const flueClient = createFlueClient({
|
||||
baseUrl: flueUrl,
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
});
|
||||
try {
|
||||
await flueClient.agents.history(
|
||||
"project-manager",
|
||||
"zopu-smoke-capability-probe"
|
||||
);
|
||||
addCheck(
|
||||
checks,
|
||||
"flue",
|
||||
true,
|
||||
"project-manager history endpoint is reachable"
|
||||
);
|
||||
return flueClient;
|
||||
} catch (error) {
|
||||
const message = errorMessage(error, secrets);
|
||||
const streamMissing = message.includes("[stream_not_found]");
|
||||
addCheck(
|
||||
checks,
|
||||
"flue",
|
||||
streamMissing,
|
||||
streamMissing
|
||||
? "project-manager history route is reachable; the probe stream is absent as expected"
|
||||
: `Flue project-manager endpoint failed: ${message}`
|
||||
);
|
||||
return streamMissing ? flueClient : undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const probeGateway = async (input: {
|
||||
readonly apiKey: string;
|
||||
readonly baseUrl: URL;
|
||||
readonly checks: SmokeCapabilityCheck[];
|
||||
readonly secrets: readonly string[];
|
||||
readonly timeoutMs: number;
|
||||
}): Promise<void> => {
|
||||
const { apiKey, baseUrl, checks, secrets, timeoutMs } = input;
|
||||
try {
|
||||
const response = await fetch(endpoint(baseUrl, "models"), {
|
||||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
signal: withTimeout(timeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`CPA models returned HTTP ${response.status}`);
|
||||
}
|
||||
const catalog = await Effect.runPromise(
|
||||
decodeSmokeModelCatalog(await response.json())
|
||||
);
|
||||
const modelIds = new Set(catalog.data.map((entry) => entry.id));
|
||||
const missing = ["minimax-m3", "glm-5.2"].filter(
|
||||
(model) => !modelIds.has(model)
|
||||
);
|
||||
addCheck(
|
||||
checks,
|
||||
"cpa-model-catalog",
|
||||
missing.length === 0,
|
||||
missing.length === 0
|
||||
? "CPA exposes minimax-m3 and glm-5.2"
|
||||
: `CPA is missing canonical models: ${missing.join(", ")}`
|
||||
);
|
||||
} catch (error) {
|
||||
addCheck(
|
||||
checks,
|
||||
"cpa-model-catalog",
|
||||
false,
|
||||
`CPA model probe failed: ${errorMessage(error, secrets)}`
|
||||
);
|
||||
}
|
||||
|
||||
for (const model of ["minimax-m3", "glm-5.2"] as const) {
|
||||
try {
|
||||
// oxlint-disable-next-line no-await-in-loop -- M3 must never be probed in parallel with another worker call.
|
||||
const text = await callOpenAiChat({
|
||||
apiKey,
|
||||
baseUrl,
|
||||
messages: [{ content: "Reply with exactly OK.", role: "user" }],
|
||||
model,
|
||||
timeoutMs,
|
||||
});
|
||||
addCheck(
|
||||
checks,
|
||||
`cpa-completion-${model}`,
|
||||
text.trim().length > 0,
|
||||
`${model} accepted a no-tools OpenAI-compatible completion`
|
||||
);
|
||||
} catch (error) {
|
||||
addCheck(
|
||||
checks,
|
||||
`cpa-completion-${model}`,
|
||||
false,
|
||||
errorMessage(error, secrets)
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const runSmoke = async (input: {
|
||||
readonly baseReport: Omit<SmokeReport, "status">;
|
||||
readonly convex: ConvexHttpClient;
|
||||
readonly cpaBaseUrl: URL;
|
||||
readonly env: SmokeEnv;
|
||||
readonly flueClient: ReturnType<typeof createFlueClient>;
|
||||
readonly loop: SmokeLoopState;
|
||||
readonly roles: SmokeModelRoles;
|
||||
readonly secrets: readonly string[];
|
||||
readonly timeoutMs: number;
|
||||
}): Promise<number> => {
|
||||
const {
|
||||
baseReport,
|
||||
convex,
|
||||
cpaBaseUrl,
|
||||
env,
|
||||
flueClient,
|
||||
roles,
|
||||
secrets,
|
||||
timeoutMs,
|
||||
} = input;
|
||||
const { projectId } = env;
|
||||
if (projectId === undefined) {
|
||||
throw new Error("run requested without ZOPU_SMOKE_PROJECT_ID");
|
||||
}
|
||||
let { loop } = input;
|
||||
let issueId: string | undefined;
|
||||
let worker: SmokeReport["worker"];
|
||||
let review: SmokeReport["review"];
|
||||
|
||||
try {
|
||||
const planText = await callOpenAiChat({
|
||||
apiKey: env.cpaApiKey,
|
||||
baseUrl: cpaBaseUrl,
|
||||
messages: buildPlanPrompt(env.featureRequest),
|
||||
model: roles.plannerReviewer.model,
|
||||
timeoutMs,
|
||||
});
|
||||
const plan = await Effect.runPromise(
|
||||
decodeSmokePlan(parseJsonObject(planText))
|
||||
);
|
||||
loop = runEffect(advanceSmokeLoop(loop, "plan-validated"));
|
||||
|
||||
issueId = await convex.mutation(api.projectIssues.create, {
|
||||
body: `${env.featureRequest}\n\nSmoke plan:\n${JSON.stringify(plan)}`,
|
||||
projectId: projectId as never,
|
||||
title: "Zopu smoke: tiny web feature",
|
||||
});
|
||||
await convex.mutation(api.projectIssues.begin, {
|
||||
issueId: issueId as never,
|
||||
});
|
||||
|
||||
const workerResult = await flueClient.agents.prompt(
|
||||
"project-manager",
|
||||
issueId,
|
||||
{
|
||||
message: buildWorkerPrompt({
|
||||
featureRequest: env.featureRequest,
|
||||
plan,
|
||||
}),
|
||||
signal: withTimeout(timeoutMs),
|
||||
}
|
||||
);
|
||||
const workerReport = await Effect.runPromise(
|
||||
decodeSmokeWorkerReport(parseJsonObject(workerResult.result.text))
|
||||
);
|
||||
worker = { status: workerReport.status };
|
||||
if (workerReport.status !== "completed") {
|
||||
throw new Error(
|
||||
`worker returned ${workerReport.status}: ${workerReport.summary}`
|
||||
);
|
||||
}
|
||||
loop = runEffect(advanceSmokeLoop(loop, "worker-succeeded"));
|
||||
|
||||
const issueStatus = await waitForIssueStatus(
|
||||
convex,
|
||||
projectId,
|
||||
issueId,
|
||||
timeoutMs
|
||||
);
|
||||
if (issueStatus !== "completed") {
|
||||
throw new Error(
|
||||
`worker reported completed but durable issue status is ${issueStatus ?? "missing"}`
|
||||
);
|
||||
}
|
||||
|
||||
const reviewText = await callOpenAiChat({
|
||||
apiKey: env.cpaApiKey,
|
||||
baseUrl: cpaBaseUrl,
|
||||
messages: buildReviewPrompt({
|
||||
featureRequest: env.featureRequest,
|
||||
plan,
|
||||
workerReport,
|
||||
}),
|
||||
model: roles.plannerReviewer.model,
|
||||
timeoutMs,
|
||||
});
|
||||
const reviewReport = await Effect.runPromise(
|
||||
decodeSmokeReview(parseJsonObject(reviewText))
|
||||
);
|
||||
review = {
|
||||
approved: reviewReport.approved,
|
||||
findingsCount: reviewReport.findings.length,
|
||||
};
|
||||
if (!reviewReport.approved) {
|
||||
throw new Error(
|
||||
`GLM review rejected the worker result: ${reviewReport.summary}`
|
||||
);
|
||||
}
|
||||
loop = runEffect(advanceSmokeLoop(loop, "review-approved"));
|
||||
} catch (error) {
|
||||
const message = errorMessage(error, secrets);
|
||||
if (issueId !== undefined) {
|
||||
await convex.mutation(api.projectIssues.markDispatchFailed, {
|
||||
error: message,
|
||||
issueId: issueId as never,
|
||||
});
|
||||
}
|
||||
loop = runEffect(advanceSmokeLoop(loop, "runtime-failed", message));
|
||||
const report: SmokeReport = {
|
||||
...baseReport,
|
||||
issueId,
|
||||
loop,
|
||||
status: "failed",
|
||||
...(review === undefined ? {} : { review }),
|
||||
...(worker === undefined ? {} : { worker }),
|
||||
};
|
||||
await writeReport(values.report, report);
|
||||
printReport(report);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const report: SmokeReport = {
|
||||
...baseReport,
|
||||
issueId,
|
||||
loop,
|
||||
review,
|
||||
status: "completed",
|
||||
worker,
|
||||
};
|
||||
await writeReport(values.report, report);
|
||||
printReport(report);
|
||||
return 0;
|
||||
};
|
||||
|
||||
const main = async (): Promise<number> => {
|
||||
const timeoutMs = Number(values["timeout-ms"] ?? DEFAULT_TIMEOUT_MS);
|
||||
if (!Number.isInteger(timeoutMs) || timeoutMs <= 0) {
|
||||
console.error(
|
||||
"ZOPU_SMOKE_CONTRACT_BLOCKED: --timeout-ms must be a positive integer"
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
|
||||
let env: SmokeEnv;
|
||||
try {
|
||||
env = parseSmokeEnv(process.env);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`ZOPU_SMOKE_CONTRACT_BLOCKED: environment contract failed: ${errorMessage(error, [])}`
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
|
||||
const secrets = [env.accessToken, env.cpaApiKey];
|
||||
const checks: SmokeCapabilityCheck[] = [];
|
||||
const convex = new ConvexHttpClient(env.convexUrl);
|
||||
convex.setAuth(env.accessToken);
|
||||
const cpaBaseUrl = normalizeBaseUrl(env.cpaBaseUrl);
|
||||
const roles = probeModelRoles(env, cpaBaseUrl, checks, secrets);
|
||||
await probeHttp(checks, "web", "web app", env.webUrl, timeoutMs);
|
||||
await probeControlPlane({
|
||||
checks,
|
||||
convex,
|
||||
daemonId: env.daemonId,
|
||||
projectId: env.projectId,
|
||||
secrets,
|
||||
});
|
||||
const flueClient = await probeFlue({
|
||||
accessToken: env.accessToken,
|
||||
checks,
|
||||
flueUrl: env.flueUrl,
|
||||
secrets,
|
||||
});
|
||||
await probeGateway({
|
||||
apiKey: env.cpaApiKey,
|
||||
baseUrl: cpaBaseUrl,
|
||||
checks,
|
||||
secrets,
|
||||
timeoutMs,
|
||||
});
|
||||
|
||||
const initialLoop = initialSmokeLoopState();
|
||||
let loop: SmokeLoopState;
|
||||
try {
|
||||
runEffect(validateSmokeCapabilities(checks));
|
||||
loop = runEffect(advanceSmokeLoop(initialLoop, "preflight-passed"));
|
||||
} catch (error) {
|
||||
loop = runEffect(
|
||||
advanceSmokeLoop(
|
||||
initialLoop,
|
||||
"contract-failed",
|
||||
errorMessage(error, secrets)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const baseReport = {
|
||||
checks,
|
||||
featureRequest: redactSmokeText(env.featureRequest, secrets),
|
||||
loop,
|
||||
mode: values.run ? ("run" as const) : ("preflight" as const),
|
||||
...(roles === undefined ? {} : { roles }),
|
||||
};
|
||||
if (loop.phase === "failed") {
|
||||
const report = { ...baseReport, status: "contract-blocked" as const };
|
||||
await writeReport(values.report, report);
|
||||
printReport(report);
|
||||
return 2;
|
||||
}
|
||||
if (!values.run) {
|
||||
const report = { ...baseReport, status: "preflight-passed" as const };
|
||||
await writeReport(values.report, report);
|
||||
printReport(report);
|
||||
return 0;
|
||||
}
|
||||
if (roles === undefined || flueClient === undefined) {
|
||||
throw new Error("run prerequisites were not retained after preflight");
|
||||
}
|
||||
return await runSmoke({
|
||||
baseReport,
|
||||
convex,
|
||||
cpaBaseUrl,
|
||||
env,
|
||||
flueClient,
|
||||
loop,
|
||||
roles,
|
||||
secrets,
|
||||
timeoutMs,
|
||||
});
|
||||
};
|
||||
|
||||
const exitCode = await main().catch((error: unknown) => {
|
||||
console.error(`ZOPU_SMOKE_FAILED: ${errorMessage(error, [])}`);
|
||||
return 1;
|
||||
});
|
||||
process.exitCode = exitCode;
|
||||
Reference in New Issue
Block a user