import { WorkAttemptExecutionError } from "@code/primitives/execution-runtime"; import type { WorkAttemptExecutionErrorReason } from "@code/primitives/execution-runtime"; import { defaultCodingKitV0, resolveOutcome } from "@code/primitives/resolver"; import type { AttemptClassification } from "@code/primitives/resolver"; import { WorkflowManager } from "@convex-dev/workflow"; import type { WorkflowId } from "@convex-dev/workflow"; import { ConvexError, v } from "convex/values"; import { components, internal } from "./_generated/api"; import type { Doc, Id } from "./_generated/dataModel"; import { internalMutation, internalQuery, mutation } from "./_generated/server"; import type { MutationCtx } from "./_generated/server"; import { requireProjectMember } from "./authz"; import { forgeForHost } from "./gitConnectionData"; export const workflow = new WorkflowManager(components.workflow); // Lease window for a running real attempt. Long enough to outlast a single // agent turn; the workflow's own retry/lease reconciliation covers crashes. const LEASE_MS = 5 * 60_000; // Runtime failure reason -> durable attempt classification. The Convex // workflow stores classifications, not provider-specific reasons, so the // resolver and Work lifecycle stay forge-agnostic. `retryable` from the // runtime is carried alongside and consulted by the kit retry policy. const FAILURE_REASON_CLASSIFICATION = { Authentication: "PermanentFailure", Cancelled: "Cancelled", HarnessFailed: "RetryableFailure", InvalidInput: "PermanentFailure", ProviderUnavailable: "RetryableFailure", RepositoryFailed: "PermanentFailure", Timeout: "RetryableFailure", } as const satisfies Record< WorkAttemptExecutionErrorReason, AttemptClassification >; export const classifyFailure = ( reason: WorkAttemptExecutionErrorReason ): AttemptClassification => FAILURE_REASON_CLASSIFICATION[reason]; // Normalize an error thrown from the agent action into the durable failure // triple (reason/retryable/summary). WorkAttemptExecutionError is the // classified runtime failure; anything else is a Convex/infrastructure error // treated as a transient HarnessFailed so the workflow retry policy decides. const toExecutionFailure = ( error: unknown ): { message: string; reason: WorkAttemptExecutionErrorReason; retryable: boolean; } => error instanceof WorkAttemptExecutionError ? { message: error.message, reason: error.reason, retryable: error.retryable, } : { message: error instanceof Error ? error.message : "Execution failed", reason: "HarnessFailed", retryable: true, }; const failureReasonValues = v.union( v.literal("Authentication"), v.literal("Cancelled"), v.literal("HarnessFailed"), v.literal("InvalidInput"), v.literal("ProviderUnavailable"), v.literal("RepositoryFailed"), v.literal("Timeout") ); const resolveReadySlice = async ( ctx: MutationCtx, work: Doc<"works">, sliceId?: string ) => { if (work.designVersion === undefined) { throw new ConvexError("Work has no approved Design to execute"); } const slices = await ctx.db .query("workSlices") .withIndex("by_workId_and_designVersion", (q) => q.eq("workId", work._id).eq("designVersion", work.designVersion!) ) .collect(); const slice = sliceId ? slices.find((candidate) => candidate.sliceId === sliceId) : slices .sort((left, right) => left.ordinal - right.ordinal) .find((candidate) => candidate.status === "ready"); if (!slice || slice.status !== "ready") { throw new ConvexError("Only the next ready slice can be executed"); } return slice; }; // Deployment prerequisite gate for real execution. Rejects before any // workflow/sandbox starts when the project lacks an attached, forge-matched // Git connection with a cloneable source URL and default branch. Returns the // validated connection so the caller can pass it to the agent unchanged. const validateProjectDeployment = async ( ctx: MutationCtx, work: Doc<"works"> ): Promise> => { const project = await ctx.db.get(work.projectId); if (!project) { throw new ConvexError("Project not found"); } if (!project.gitConnectionId) { throw new ConvexError("Connect Git credentials to this project first"); } const connection = await ctx.db.get(project.gitConnectionId); if (!connection) { throw new ConvexError("Git connection not found"); } const expected = forgeForHost(project.sourceHost); if (expected && connection.provider !== expected) { throw new ConvexError( `Git credential provider (${connection.provider}) does not match this project's forge (${expected})` ); } return connection; }; export const execute = workflow .define({ args: { attemptId: v.id("workAttempts") } }) .handler(async (step, args): Promise => { try { const result = await step.runAction( internal.workExecutionAgent.executeAttempt, args, { retry: true } ); await step.runMutation(internal.workExecutionWorkflow.completeAttempt, { attemptId: args.attemptId, result: { ...result, changedFiles: [...result.changedFiles], events: result.events.map((item) => ({ ...item, metadata: { ...item.metadata }, })), }, }); } catch (error) { const failure = toExecutionFailure(error); await step.runMutation(internal.workExecutionWorkflow.failAttempt, { attemptId: args.attemptId, reason: failure.reason, retryable: failure.retryable, summary: failure.message, }); } }); export const startExecution = mutation({ args: { sliceId: v.optional(v.string()), workId: v.id("works"), }, handler: async ( ctx, args ): Promise<{ attemptId: Id<"workAttempts">; runId: Id<"workRuns">; workflowId: WorkflowId; }> => { const work = await ctx.db.get(args.workId); if (!work) { throw new ConvexError("Work not found"); } await requireProjectMember(ctx, work.projectId); if (work.status !== "ready") { throw new ConvexError("Work must be Ready before execution"); } if ( work.definitionApprovalVersion !== work.definitionVersion || work.designApprovalVersion !== work.designVersion ) { throw new ConvexError("Execution requires exact approved versions"); } await validateProjectDeployment(ctx, work); const slice = await resolveReadySlice(ctx, work, args.sliceId); const createdAt = Date.now(); const runId = await ctx.db.insert("workRuns", { createdAt, designVersion: slice.designVersion, executionKind: "real", kitId: defaultCodingKitV0.id, kitVersion: defaultCodingKitV0.version, scenario: "success", sliceId: slice.sliceId, sliceRowId: slice._id, status: "running", workId: work._id, }); const workspaceKey = `work-${work._id}-run-${runId}`; const attemptId = await ctx.db.insert("workAttempts", { number: 1, runId, status: "queued", workId: work._id, workspaceKey, }); const workflowId: WorkflowId = await workflow.start( ctx, internal.workExecutionWorkflow.execute, { attemptId } ); await ctx.db.patch(runId, { startedAt: createdAt, workflowId }); await ctx.db.patch(slice._id, { status: "running" }); await ctx.db.patch(work._id, { status: "executing", updatedAt: createdAt }); await ctx.db.insert("workEvents", { createdAt, idempotencyKey: `real-run-started:${runId}`, kind: "run.started", referenceId: String(runId), workId: work._id, }); return { attemptId, runId, workflowId }; }, }); export const executionContext = internalQuery({ args: { attemptId: v.id("workAttempts") }, handler: async (ctx, args) => { const attempt = await ctx.db.get(args.attemptId); if (!attempt) { throw new ConvexError("Attempt not found"); } const run = await ctx.db.get(attempt.runId); const work = await ctx.db.get(attempt.workId); if (!run || !work || !run.sliceRowId) { throw new ConvexError("Execution records are incomplete"); } const [slice, project] = await Promise.all([ ctx.db.get(run.sliceRowId), ctx.db.get(work.projectId), ]); if (!slice || !project?.gitConnectionId) { throw new ConvexError("Project execution configuration is incomplete"); } const connection = await ctx.db.get(project.gitConnectionId); if (!connection) { throw new ConvexError("Git connection not found"); } const expectedForge = forgeForHost(project.sourceHost); if (expectedForge && connection.provider !== expectedForge) { throw new ConvexError( `Git credential provider (${connection.provider}) does not match this project's forge (${expectedForge})` ); } return { attempt, connection, project, prompt: [ `Implement this approved Zopu slice: ${slice.title}`, `Objective: ${slice.objective}`, `Observable behavior: ${slice.observableBehavior}`, "Inspect the repository instructions first. Make focused changes and run relevant checks.", ].join("\n\n"), run, work, }; }, }); export const markAttemptRunning = internalMutation({ args: { attemptId: v.id("workAttempts") }, handler: async (ctx, args) => { const attempt = await ctx.db.get(args.attemptId); if (!attempt || attempt.status === "terminal") { return false; } const run = await ctx.db.get(attempt.runId); if (!run || run.status !== "running") { return false; } await ctx.db.patch(attempt._id, { leaseExpiresAt: Date.now() + LEASE_MS, startedAt: attempt.startedAt ?? Date.now(), status: "running", }); return true; }, }); const settleSliceAndWork = async ( ctx: MutationCtx, run: Doc<"workRuns">, succeeded: boolean, workStatus: Doc<"works">["status"] ) => { if (!run.sliceRowId) { return; } const slice = await ctx.db.get(run.sliceRowId); if (!slice) { return; } let sliceStatus: "blocked" | "completed" | "ready" = "ready"; if (succeeded) { sliceStatus = "completed"; } else if (workStatus === "blocked") { sliceStatus = "blocked"; } await ctx.db.patch(slice._id, { status: sliceStatus }); const work = await ctx.db.get(run.workId); if (!work) { return; } let status = workStatus; if (succeeded) { const slices = await ctx.db .query("workSlices") .withIndex("by_workId_and_designVersion", (q) => q.eq("workId", work._id).eq("designVersion", slice.designVersion) ) .collect(); const next = slices .sort((left, right) => left.ordinal - right.ordinal) .find((candidate) => candidate.status === "planned"); if (next) { await ctx.db.patch(next._id, { status: "ready" }); status = "ready"; } } await ctx.db.patch(work._id, { status, updatedAt: Date.now() }); }; export const completeAttempt = internalMutation({ args: { attemptId: v.id("workAttempts"), result: v.object({ baseRevision: v.string(), candidateRevision: v.string(), changedFiles: v.array(v.string()), diff: v.string(), environmentId: v.string(), events: v.array( v.object({ kind: v.string(), message: v.string(), metadata: v.record(v.string(), v.string()), occurredAt: v.number(), sequence: v.number(), }) ), summary: v.string(), }), }, handler: async (ctx, args) => { const attempt = await ctx.db.get(args.attemptId); // Cancellation fencing: a terminal or cancelled attempt/run must never // later settle success, even if the agent action raced to completion. if (!attempt || attempt.status === "terminal") { return; } const run = await ctx.db.get(attempt.runId); const work = await ctx.db.get(attempt.workId); if (!run || !work) { throw new ConvexError("Execution records not found"); } if (run.status === "terminal" || run.status === "cancelled") { return; } // Empty-change rejection: a no-op result (no changed files, or identical // base/candidate revisions) is not a successful implementation. Fail it // as a permanent InvalidInput so the resolver does not mark Work done. const noOp = args.result.changedFiles.length === 0 || args.result.baseRevision === args.result.candidateRevision; if (noOp) { await ctx.db.patch(attempt._id, { classification: "PermanentFailure", endedAt: Date.now(), failureReason: "InvalidInput", leaseExpiresAt: undefined, status: "terminal", summary: "Execution produced no repository changes", }); await ctx.db.patch(run._id, { endedAt: Date.now(), status: "terminal", terminalClassification: "PermanentFailure", terminalSummary: "Execution produced no repository changes", }); await settleSliceAndWork(ctx, run, false, "failed"); return; } for (const item of args.result.events) { await ctx.db.insert("workAttemptEvents", { attemptId: attempt._id, kind: item.kind, message: item.message, metadataJson: JSON.stringify(item.metadata), occurredAt: item.occurredAt, sequence: item.sequence, }); } const endedAt = Date.now(); await ctx.db.patch(attempt._id, { classification: "Succeeded", endedAt, leaseExpiresAt: undefined, status: "terminal", summary: args.result.summary, }); await ctx.db.patch(run._id, { baseRevision: args.result.baseRevision, candidateRevision: args.result.candidateRevision, endedAt, environmentId: args.result.environmentId, status: "terminal", terminalClassification: "Succeeded", terminalSummary: args.result.summary, }); await ctx.db.insert("workArtifacts", { attemptId: attempt._id, createdAt: endedAt, designVersion: run.designVersion, environmentId: args.result.environmentId, idempotencyKey: `real-diff:${attempt._id}`, kind: "diff", metadataJson: JSON.stringify({ changedFiles: args.result.changedFiles }), organizationId: work.organizationId, producer: "agentos-pi", projectId: work.projectId, provenanceJson: JSON.stringify({ baseRevision: args.result.baseRevision, }), runId: run._id, sliceId: run.sliceId, sourceRevision: args.result.candidateRevision, title: "Implementation diff", verificationStatus: "unverified", workId: work._id, ...(args.result.diff.length > 0 ? { uri: `data:text/plain;charset=utf-8,${encodeURIComponent(args.result.diff.slice(0, 50_000))}`, } : {}), }); await settleSliceAndWork(ctx, run, true, "completed"); await ctx.db.insert("workEvents", { createdAt: endedAt, idempotencyKey: `real-run-completed:${run._id}`, kind: "run.completed", payloadJson: JSON.stringify({ classification: "Succeeded" }), referenceId: String(run._id), workId: work._id, }); }, }); export const launchAttemptWorkflow = internalMutation({ args: { attemptId: v.id("workAttempts") }, handler: async (ctx, args) => { const attempt = await ctx.db.get(args.attemptId); if (!attempt || attempt.status === "terminal") { return; } const run = await ctx.db.get(attempt.runId); if (!run || run.status !== "running") { return; } const workflowId: WorkflowId = await workflow.start( ctx, internal.workExecutionWorkflow.execute, args ); await ctx.db.patch(run._id, { workflowId }); }, }); export const failAttempt = internalMutation({ args: { attemptId: v.id("workAttempts"), reason: failureReasonValues, retryable: v.boolean(), summary: v.string(), }, handler: async (ctx, args) => { const attempt = await ctx.db.get(args.attemptId); // Cancellation fencing: once an attempt or run is terminal/cancelled, a // late failure from the workflow catch block must not overwrite the // settled state. This also prevents a cancelled attempt from settling // as a different classification after cancelExecution ran. if (!attempt || attempt.status === "terminal") { return; } const run = await ctx.db.get(attempt.runId); if (!run || run.status === "terminal" || run.status === "cancelled") { return; } const classification = classifyFailure(args.reason); const endedAt = Date.now(); await ctx.db.patch(attempt._id, { classification, endedAt, failureReason: args.reason, leaseExpiresAt: undefined, status: "terminal", summary: args.summary, }); const outcome = resolveOutcome( { classification, retryable: args.retryable, summary: args.summary }, attempt.number, defaultCodingKitV0.retryPolicy ); await ctx.db.insert("resolverDecisions", { attemptId: attempt._id, attemptNumber: attempt.number, classification, createdAt: endedAt, decision: outcome.kind, ...(outcome.kind === "terminal" ? { resultingWorkStatus: outcome.workStatus } : {}), runId: run._id, summary: args.summary, workId: attempt.workId, }); if (outcome.kind === "retry") { const nextAttemptId = await ctx.db.insert("workAttempts", { number: attempt.number + 1, runId: run._id, status: "queued", workId: attempt.workId, workspaceKey: attempt.workspaceKey, }); await ctx.scheduler.runAfter( 0, internal.workExecutionWorkflow.launchAttemptWorkflow, { attemptId: nextAttemptId } ); return; } await ctx.db.patch(run._id, { endedAt, status: "terminal", terminalClassification: classification, terminalSummary: args.summary, }); await settleSliceAndWork(ctx, run, false, outcome.workStatus); const work = await ctx.db.get(run.workId); if (work) { await ctx.db.insert("workEvents", { createdAt: endedAt, idempotencyKey: `real-run-failed:${run._id}`, kind: "run.completed", payloadJson: JSON.stringify({ classification }), referenceId: String(run._id), workId: work._id, }); } }, }); export const cancelExecution = mutation({ args: { runId: v.id("workRuns") }, handler: async (ctx, args) => { const run = await ctx.db.get(args.runId); if (!run) { throw new ConvexError("Run not found"); } await requireProjectMember(ctx, (await ctx.db.get(run.workId))!.projectId); if (run.status !== "running") { return { cancelled: false }; } if (run.workflowId) { await workflow.cancel(ctx, run.workflowId as WorkflowId); } const attempts = await ctx.db .query("workAttempts") .withIndex("by_runId_and_number", (q) => q.eq("runId", run._id)) .collect(); const attempt = attempts.find( (candidate) => candidate.status !== "terminal" ); if (attempt?.workspaceKey) { await ctx.scheduler.runAfter( 0, internal.workExecutionAgent.cancelAttempt, { attemptId: String(attempt._id), workspaceKey: attempt.workspaceKey, } ); const cancelledAt = Date.now(); await ctx.db.patch(attempt._id, { classification: "Cancelled", endedAt: cancelledAt, failureReason: "Cancelled", leaseExpiresAt: undefined, status: "terminal", summary: "Execution cancelled", }); await ctx.db.insert("resolverDecisions", { attemptId: attempt._id, attemptNumber: attempt.number, classification: "Cancelled", createdAt: cancelledAt, decision: "terminal", resultingWorkStatus: "ready", runId: run._id, summary: "Execution cancelled", workId: run.workId, }); } await ctx.db.patch(run._id, { endedAt: Date.now(), status: "cancelled", terminalClassification: "Cancelled", terminalSummary: "Execution cancelled", }); if (run.sliceRowId) { await ctx.db.patch(run.sliceRowId, { status: "ready" }); } const work = await ctx.db.get(run.workId); if (work) { await ctx.db.patch(work._id, { status: "ready", updatedAt: Date.now() }); } return { cancelled: true }; }, });