382 lines
13 KiB
TypeScript
382 lines
13 KiB
TypeScript
import { convexTest } from "convex-test";
|
|
import { anyApi } from "convex/server";
|
|
import { describe, expect, test } from "vitest";
|
|
|
|
import schema from "./schema";
|
|
|
|
const modules = import.meta.glob("./**/*.ts");
|
|
const api = anyApi;
|
|
|
|
type TestContext = ReturnType<typeof convexTest>;
|
|
|
|
interface Seeded {
|
|
attemptId: string;
|
|
runId: string;
|
|
t: TestContext;
|
|
workId: string;
|
|
}
|
|
|
|
// Seed an in-flight real attempt so each mutation under test starts from a
|
|
// realistic running state: Work "executing", Run "running", Attempt "running".
|
|
const seedRunningAttempt = async (
|
|
overrides: {
|
|
attemptStatus?: "queued" | "claimed" | "running" | "terminal";
|
|
runStatus?: "ready" | "running" | "terminal" | "cancelled";
|
|
} = {}
|
|
): Promise<Seeded> => {
|
|
const t = convexTest({ modules, schema });
|
|
const seeded = await t.run(async (ctx) => {
|
|
const organizationId = await ctx.db.insert("organizations", {
|
|
createdAt: 1,
|
|
createdBy: "user",
|
|
kind: "personal",
|
|
name: "Personal",
|
|
});
|
|
const projectId = await ctx.db.insert("projects", {
|
|
createdAt: 1,
|
|
name: "Zopu",
|
|
normalizedSourceUrl: "https://github.com/puter/zopu",
|
|
organizationId,
|
|
repositoryPath: "puter/zopu",
|
|
sourceHost: "github.com",
|
|
sourceUrl: "https://github.com/puter/zopu",
|
|
updatedAt: 1,
|
|
});
|
|
const workId = await ctx.db.insert("works", {
|
|
createdAt: 1,
|
|
objective: "Implement a real slice",
|
|
organizationId,
|
|
projectId,
|
|
status: "executing",
|
|
title: "Real execution",
|
|
updatedAt: 1,
|
|
});
|
|
const sliceRowId = await ctx.db.insert("workSlices", {
|
|
designVersion: 1,
|
|
objective: "Change the repo",
|
|
observableBehavior: "A diff exists",
|
|
ordinal: 0,
|
|
payloadJson: "{}",
|
|
sliceId: "slice-1",
|
|
status: "running",
|
|
title: "Implementation",
|
|
workId,
|
|
});
|
|
const runId = await ctx.db.insert("workRuns", {
|
|
createdAt: 1,
|
|
designVersion: 1,
|
|
executionKind: "real",
|
|
kitId: "coding-v0",
|
|
kitVersion: "1",
|
|
scenario: "success",
|
|
sliceId: "slice-1",
|
|
sliceRowId,
|
|
status: overrides.runStatus ?? "running",
|
|
workId,
|
|
});
|
|
const attemptId = await ctx.db.insert("workAttempts", {
|
|
number: 1,
|
|
runId,
|
|
status: overrides.attemptStatus ?? "running",
|
|
workId,
|
|
workspaceKey: "workspace-1",
|
|
});
|
|
return { attemptId, runId, workId };
|
|
});
|
|
return { ...seeded, t };
|
|
};
|
|
|
|
const successResult = {
|
|
baseRevision: "base123",
|
|
candidateRevision: "candidate456",
|
|
changedFiles: ["src/index.ts"],
|
|
diff: "+export const ready = true;",
|
|
environmentId: "workspace-1",
|
|
events: [
|
|
{
|
|
kind: "runtime.completed",
|
|
message: "completed",
|
|
metadata: {},
|
|
occurredAt: 2,
|
|
sequence: 0,
|
|
},
|
|
],
|
|
summary: "Changed one file",
|
|
};
|
|
|
|
describe("real work execution persistence", () => {
|
|
test("records revisions, activity, diff, and terminal state atomically", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
await t.mutation(api.workExecutionWorkflow.completeAttempt, {
|
|
attemptId: seeded.attemptId,
|
|
result: successResult,
|
|
});
|
|
|
|
const state = await t.run(async (ctx) => ({
|
|
artifacts: await ctx.db.query("workArtifacts").collect(),
|
|
attempt: await ctx.db.get(seeded.attemptId as any),
|
|
run: await ctx.db.get(seeded.runId as any),
|
|
work: await ctx.db.get(seeded.workId as any),
|
|
}));
|
|
expect(state.attempt?.classification).toBe("Succeeded");
|
|
expect(state.run).toMatchObject({
|
|
baseRevision: "base123",
|
|
candidateRevision: "candidate456",
|
|
terminalClassification: "Succeeded",
|
|
});
|
|
expect(state.work?.status).toBe("completed");
|
|
expect(state.artifacts[0]).toMatchObject({
|
|
kind: "diff",
|
|
sourceRevision: "candidate456",
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("classified failure mapping", () => {
|
|
test("maps a transient runtime reason to a retryable classification", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
await t.mutation(api.workExecutionWorkflow.failAttempt, {
|
|
attemptId: seeded.attemptId,
|
|
reason: "ProviderUnavailable",
|
|
retryable: true,
|
|
summary: "model provider timed out",
|
|
});
|
|
const attempt = await t.run((ctx) => ctx.db.get(seeded.attemptId as any));
|
|
expect(attempt?.classification).toBe("RetryableFailure");
|
|
expect(attempt?.failureReason).toBe("ProviderUnavailable");
|
|
});
|
|
|
|
test("queues a new attempt without terminalizing a retryable run", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
await t.mutation(api.workExecutionWorkflow.failAttempt, {
|
|
attemptId: seeded.attemptId,
|
|
reason: "ProviderUnavailable",
|
|
retryable: true,
|
|
summary: "provider unavailable",
|
|
});
|
|
const state = await t.run(async (ctx) => ({
|
|
attempts: await ctx.db.query("workAttempts").collect(),
|
|
run: await ctx.db.get(seeded.runId as any),
|
|
work: await ctx.db.get(seeded.workId as any),
|
|
}));
|
|
expect(state.attempts).toHaveLength(2);
|
|
expect(state.attempts[1]).toMatchObject({
|
|
number: 2,
|
|
status: "queued",
|
|
workspaceKey: "workspace-1",
|
|
});
|
|
expect(state.run?.status).toBe("running");
|
|
expect(state.work?.status).toBe("executing");
|
|
});
|
|
test("maps an authentication reason to a permanent failure", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
await t.mutation(api.workExecutionWorkflow.failAttempt, {
|
|
attemptId: seeded.attemptId,
|
|
reason: "Authentication",
|
|
retryable: false,
|
|
summary: "token rejected",
|
|
});
|
|
const state = await t.run(async (ctx) => ({
|
|
attempt: await ctx.db.get(seeded.attemptId as any),
|
|
run: await ctx.db.get(seeded.runId as any),
|
|
work: await ctx.db.get(seeded.workId as any),
|
|
}));
|
|
expect(state.attempt?.classification).toBe("PermanentFailure");
|
|
expect(state.attempt?.failureReason).toBe("Authentication");
|
|
expect(state.run?.terminalClassification).toBe("PermanentFailure");
|
|
expect(state.work?.status).toBe("failed");
|
|
});
|
|
});
|
|
|
|
describe("cancellation fencing", () => {
|
|
test("a cancelled attempt cannot later settle success", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
// Simulate cancellation marking the attempt/run terminal as Cancelled.
|
|
await t.run(async (ctx) => {
|
|
await ctx.db.patch(seeded.attemptId as any, {
|
|
classification: "Cancelled",
|
|
endedAt: 5,
|
|
status: "terminal",
|
|
summary: "Execution cancelled",
|
|
});
|
|
await ctx.db.patch(seeded.runId as any, {
|
|
endedAt: 5,
|
|
status: "cancelled",
|
|
terminalClassification: "Cancelled",
|
|
terminalSummary: "Execution cancelled",
|
|
});
|
|
});
|
|
// A late-arriving completion must be ignored.
|
|
await t.mutation(api.workExecutionWorkflow.completeAttempt, {
|
|
attemptId: seeded.attemptId,
|
|
result: successResult,
|
|
});
|
|
const attempt = await t.run((ctx) => ctx.db.get(seeded.attemptId as any));
|
|
expect(attempt?.classification).toBe("Cancelled");
|
|
expect(attempt?.status).toBe("terminal");
|
|
});
|
|
|
|
test("a late failure does not overwrite a cancelled run", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
await t.run(async (ctx) => {
|
|
await ctx.db.patch(seeded.attemptId as any, {
|
|
classification: "Cancelled",
|
|
endedAt: 5,
|
|
status: "terminal",
|
|
});
|
|
await ctx.db.patch(seeded.runId as any, { status: "cancelled" });
|
|
});
|
|
await t.mutation(api.workExecutionWorkflow.failAttempt, {
|
|
attemptId: seeded.attemptId,
|
|
reason: "HarnessFailed",
|
|
retryable: true,
|
|
summary: "late failure",
|
|
});
|
|
const attempt = await t.run((ctx) => ctx.db.get(seeded.attemptId as any));
|
|
expect(attempt?.classification).toBe("Cancelled");
|
|
});
|
|
});
|
|
|
|
describe("empty-change rejection", () => {
|
|
test("a no-op result with no changed files fails as permanent", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
await t.mutation(api.workExecutionWorkflow.completeAttempt, {
|
|
attemptId: seeded.attemptId,
|
|
result: {
|
|
...successResult,
|
|
changedFiles: [],
|
|
candidateRevision: "base123",
|
|
},
|
|
});
|
|
const state = await t.run(async (ctx) => ({
|
|
attempt: await ctx.db.get(seeded.attemptId as any),
|
|
work: await ctx.db.get(seeded.workId as any),
|
|
}));
|
|
expect(state.attempt?.classification).toBe("PermanentFailure");
|
|
expect(state.attempt?.failureReason).toBe("InvalidInput");
|
|
expect(state.work?.status).toBe("failed");
|
|
});
|
|
|
|
test("identical base and candidate revisions are rejected", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
await t.mutation(api.workExecutionWorkflow.completeAttempt, {
|
|
attemptId: seeded.attemptId,
|
|
result: {
|
|
...successResult,
|
|
baseRevision: "same",
|
|
candidateRevision: "same",
|
|
},
|
|
});
|
|
const attempt = await t.run((ctx) => ctx.db.get(seeded.attemptId as any));
|
|
expect(attempt?.classification).toBe("PermanentFailure");
|
|
});
|
|
});
|
|
|
|
describe("attempt re-entry", () => {
|
|
test("markAttemptRunning re-claims an already-running attempt", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt();
|
|
// First claim (already "running" from seed) must still succeed so a
|
|
// workflow replay can re-enter the same live attempt.
|
|
const first = await t.mutation(
|
|
api.workExecutionWorkflow.markAttemptRunning,
|
|
{ attemptId: seeded.attemptId }
|
|
);
|
|
expect(first).toBe(true);
|
|
const attempt = await t.run((ctx) => ctx.db.get(seeded.attemptId as any));
|
|
expect(attempt?.status).toBe("running");
|
|
expect(attempt?.leaseExpiresAt).toBeGreaterThan(0);
|
|
});
|
|
|
|
test("a terminal attempt is not re-runnable", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt({
|
|
attemptStatus: "terminal",
|
|
});
|
|
const result = await t.mutation(
|
|
api.workExecutionWorkflow.markAttemptRunning,
|
|
{ attemptId: seeded.attemptId }
|
|
);
|
|
expect(result).toBe(false);
|
|
});
|
|
test("a cancelled run cannot re-enter a queued attempt", async () => {
|
|
const { t, ...seeded } = await seedRunningAttempt({
|
|
attemptStatus: "queued",
|
|
runStatus: "cancelled",
|
|
});
|
|
const result = await t.mutation(
|
|
api.workExecutionWorkflow.markAttemptRunning,
|
|
{ attemptId: seeded.attemptId }
|
|
);
|
|
expect(result).toBe(false);
|
|
const attempt = await t.run((ctx) => ctx.db.get(seeded.attemptId as any));
|
|
expect(attempt?.status).toBe("queued");
|
|
});
|
|
});
|
|
|
|
describe("forge credential validation", () => {
|
|
const identity = { tokenIdentifier: "https://convex.test|forge-user" };
|
|
|
|
const seedForgableProject = async (
|
|
provider: "github" | "gitea",
|
|
sourceHost: string
|
|
) => {
|
|
const t = convexTest({ modules, schema }).withIdentity(identity);
|
|
const ids = await t.run(async (ctx) => {
|
|
const organizationId = await ctx.db.insert("organizations", {
|
|
createdAt: 1,
|
|
createdBy: identity.tokenIdentifier,
|
|
kind: "personal",
|
|
name: "Personal",
|
|
});
|
|
await ctx.db.insert("organizationMembers", {
|
|
createdAt: 1,
|
|
organizationId,
|
|
role: "owner",
|
|
userId: identity.tokenIdentifier,
|
|
});
|
|
const projectId = await ctx.db.insert("projects", {
|
|
createdAt: 1,
|
|
name: "Zopu",
|
|
normalizedSourceUrl: `https://${sourceHost}/puter/zopu`,
|
|
organizationId,
|
|
repositoryPath: "puter/zopu",
|
|
sourceHost,
|
|
sourceUrl: `https://${sourceHost}/puter/zopu`,
|
|
updatedAt: 1,
|
|
});
|
|
const connectionId = await ctx.db.insert("gitConnections", {
|
|
connectedAt: 1,
|
|
credentialCiphertext: "x",
|
|
credentialIv: "y",
|
|
credentialKind: provider === "github" ? "oauth" : "token",
|
|
organizationId,
|
|
provider,
|
|
serverUrl: `https://${sourceHost}`,
|
|
updatedAt: 1,
|
|
username: "zopu",
|
|
});
|
|
return { connectionId, projectId };
|
|
});
|
|
return { t, ids };
|
|
};
|
|
|
|
test("rejects a git connection whose provider mismatches the project forge", async () => {
|
|
// Gitea token attached to a GitHub-hosted project.
|
|
const { t, ids } = await seedForgableProject("gitea", "github.com");
|
|
await expect(
|
|
t.mutation(api.gitConnectionData.attachToProject, {
|
|
connectionId: ids.connectionId,
|
|
projectId: ids.projectId,
|
|
})
|
|
).rejects.toThrow(/does not match this project's forge/u);
|
|
});
|
|
|
|
test("accepts a matching provider for the project forge", async () => {
|
|
const { t, ids } = await seedForgableProject("github", "github.com");
|
|
const result = await t.mutation(api.gitConnectionData.attachToProject, {
|
|
connectionId: ids.connectionId,
|
|
projectId: ids.projectId,
|
|
});
|
|
expect(result).toMatchObject({ attached: true });
|
|
});
|
|
});
|