puter/zopu-code
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Slice 4 hardening: bounded resolver, terminal states, reconciliation

Browse Source 
Addresses review findings before Slice 5 sandbox work.

Resolver primitives:
- resolveOutcome maps an attempt outcome to retry-or-terminal; removed the
  no-op isTerminalClassification.
- WORK_STATUS_FOR_OUTCOME preserves real terminal Work status instead of
  collapsing everything back to ready.
- Lifecycle: completed is terminal, cancelled reopens only to ready, and the
  invalid awaiting-design-approval -> awaiting-definition-approval reopening
  is gone.

Execution (workExecution.ts):
- finishAttempt replaces completeAttempt: auto-retries within the kit budget
  and settles Work at the real classification (Succeeded->completed,
  PermanentFailure/BudgetExhausted->failed, NeedsInput->needs-input,
  Blocked->blocked). No Work stays silently runnable after a failure.
- startSimulatedExecution validates sliceId against the current approved
  Design (foreign ids rejected; defaults to the first slice) and advances
  slice status running -> completed/ready.
- retrySimulatedExecution is now an explicit user restart with a retryable-
  state guard, not the automatic resolver path.
- reconcileExpiredAttempts is bounded: a dead lease terminates as Blocked,
  resumes within budget or settles the Run/Work so nothing runs forever.
- crons.ts runs the reconciler every 30s, string-referenced so it does not
  depend on the stale codegen.

Planning (workPlanning.ts):
- saveDefinition/saveDesign reject revision while a Run is executing.
- submitQuestion validates and persists a durable workQuestions row instead
  of discarding planner questions.
- Design revision invalidates active design approvals.

Verification: primitives 69 tests (+4), backend 22 tests (+10) covering
retry/cancel/failure/restart, slice validation, reconciliation, the
revision guard, and question persistence. Root typecheck, web+agents build,
and the configured Ultracite gate (53 files) all pass.

package.json: the recurring check gate now covers workExecution, workPlanning,
crons, resolver, and work-lifecycle.
This commit is contained in:
-Puter
2026-07-28 00:07:35 +05:30
parent 005b26fa32
commit 85633a5124
9 changed files with 717 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
packages/backend/convex/workExecution.ts
View File

@@ -2,12 +2,16 @@ import {
FakeHarnessLive,
type FakeScenario,
} from "@code/primitives/harness-runtime";
import { defaultCodingKitV0 } from "@code/primitives/resolver";
import {
type AttemptClassification,
defaultCodingKitV0,
resolveOutcome,
} from "@code/primitives/resolver";
import { makeFunctionReference } from "convex/server";
import { ConvexError, v } from "convex/values";
import { Effect } from "effect";
import type { Id } from "./_generated/dataModel";
import type { Doc, Id } from "./_generated/dataModel";
import {
internalAction,
internalMutation,
@@ -37,16 +41,17 @@ const checkpointRef = makeFunctionReference<
},
unknown
>("workExecution:checkpointAttempt");
const completeRef = makeFunctionReference<
const finishRef = makeFunctionReference<
"mutation",
{
attemptId: Id<"workAttempts">;
classification: string;
owner: string;
retryable: boolean;
summary: string;
},
unknown
>("workExecution:completeAttempt");
>("workExecution:finishAttempt");
const now = () => Date.now();
const requireWorkForMember = async (ctx: any, workId: Id<"works">) => {
@@ -81,6 +86,54 @@ const appendWorkEvent = async (
});
};
// Slice the Run targets for the currently approved Design. `sliceId` is
// optional only to let the resolver default to the first ready slice.
const resolveRunSlice = async (
ctx: any,
work: Doc<"works">,
sliceId?: string
) => {
const currentDesignVersion = work.designVersion;
if (currentDesignVersion === undefined)
throw new ConvexError("Work has no approved Design to execute");
const slices = await ctx.db
.query("workSlices")
.withIndex("by_work_and_designVersion", (q: any) =>
q.eq("workId", work._id).eq("designVersion", currentDesignVersion)
)
.collect();
if (slices.length === 0)
throw new ConvexError("No slices found for the current approved Design");
const slice = sliceId
? slices.find((row: any) => row.sliceId === sliceId)
: slices.sort((a: any, b: any) => a.ordinal - b.ordinal)[0];
if (!slice)
throw new ConvexError(
"Requested slice does not belong to the current approved Design"
);
return slice as Doc<"workSlices">;
};
const setSliceStatus = async (
ctx: any,
workId: Id<"works">,
sliceId: string | undefined,
status: any
) => {
if (!sliceId) return;
const slice = (
await ctx.db
.query("workSlices")
.withIndex("by_work_and_designVersion", (q: any) =>
q.eq("workId", workId)
)
.collect()
).find((row: any) => row.sliceId === sliceId);
if (slice) await ctx.db.patch(slice._id, { status });
};
const RETRYABLE_WORK_STATUSES = ["failed", "needs-input", "blocked", "ready"];
export const startSimulatedExecution = mutation({
args: {
workId: v.id("works"),
@@ -104,10 +157,11 @@ export const startSimulatedExecution = mutation({
throw new ConvexError(
"Execution requires exact approved Definition and Design versions"
);
const slice = await resolveRunSlice(ctx, work, args.sliceId);
const createdAt = now();
const runId = await ctx.db.insert("workRuns", {
workId: work._id,
sliceId: args.sliceId,
sliceId: slice.sliceId,
status: "ready",
scenario: args.scenario,
kitId: defaultCodingKitV0.id,
@@ -120,7 +174,11 @@ export const startSimulatedExecution = mutation({
number: 1,
status: "queued",
});
await ctx.db.patch(work._id, { status: "executing", updatedAt: createdAt });
await ctx.db.patch(work._id, {
status: "executing",
updatedAt: createdAt,
});
await ctx.db.patch(slice._id, { status: "running" });
await ctx.db.patch(runId, { status: "running", startedAt: createdAt });
await appendWorkEvent(
ctx,
@@ -163,6 +221,7 @@ export const cancelSimulatedExecution = mutation({
classification: "Cancelled",
summary: "Simulation cancelled",
});
await setSliceStatus(ctx, work._id, run.sliceId, "ready");
await ctx.db.patch(work._id, { status: "ready", updatedAt: now() });
await appendWorkEvent(
ctx,
@@ -175,6 +234,8 @@ export const cancelSimulatedExecution = mutation({
},
});
// User-initiated restart of a terminal Run. Unlike the automatic resolver
// retry, this is an explicit decision and may exceed the kit budget.
export const retrySimulatedExecution = mutation({
args: { runId: v.id("workRuns") },
handler: async (ctx, args) => {
@@ -183,6 +244,8 @@ export const retrySimulatedExecution = mutation({
const work = await requireWorkForMember(ctx, run.workId);
if (run.status !== "terminal")
throw new ConvexError("Only terminal Runs can be retried");
if (!RETRYABLE_WORK_STATUSES.includes(work.status))
throw new ConvexError("Work is not in a retryable state");
const attempts = await ctx.db
.query("workAttempts")
.withIndex("by_run_and_number", (q: any) => q.eq("runId", run._id))
@@ -202,6 +265,7 @@ export const retrySimulatedExecution = mutation({
terminalSummary: undefined,
});
await ctx.db.patch(work._id, { status: "executing", updatedAt: now() });
await setSliceStatus(ctx, work._id, run.sliceId, "running");
await ctx.scheduler.runAfter(0, executeRef, {
attemptId,
scenario: run.scenario,
@@ -274,11 +338,16 @@ export const checkpointAttempt = internalMutation({
},
});
export const completeAttempt = internalMutation({
// Single resolver mutation: record the attempt outcome, then either schedule
// the next attempt within the kit retry policy or settle the Run/Work at the
// terminal classification. Replaces the old completeAttempt that always reset
// Work to "ready" and ignored the retry policy.
export const finishAttempt = internalMutation({
args: {
attemptId: v.id("workAttempts"),
owner: v.string(),
classification: v.string(),
retryable: v.boolean(),
summary: v.string(),
},
handler: async (ctx, args) => {
@@ -288,7 +357,7 @@ export const completeAttempt = internalMutation({
attempt.leaseOwner !== args.owner ||
attempt.status === "terminal"
)
return false;
return null;
const endedAt = now();
await ctx.db.patch(attempt._id, {
status: "terminal",
@@ -297,8 +366,44 @@ export const completeAttempt = internalMutation({
summary: args.summary,
leaseExpiresAt: undefined,
});
const outcome = {
classification: args.classification as AttemptClassification,
retryable: args.retryable,
summary: args.summary,
};
const resolution = resolveOutcome(
outcome,
attempt.number,
defaultCodingKitV0.retryPolicy
);
const run = await ctx.db.get(attempt.runId);
if (!run) return false;
if (!run) return null;
await appendWorkEvent(
ctx,
attempt.workId,
"attempt.completed",
`attempt-completed:${attempt._id}`,
String(attempt._id),
JSON.stringify({
classification: args.classification,
retried: resolution.kind === "retry",
summary: args.summary,
})
);
if (resolution.kind === "retry") {
// Keep Run running and Work executing; spin the next attempt.
const nextAttemptId = await ctx.db.insert("workAttempts", {
runId: run._id,
workId: attempt.workId,
number: attempt.number + 1,
status: "queued",
});
await ctx.scheduler.runAfter(0, executeRef, {
attemptId: nextAttemptId,
scenario: run.scenario,
});
return { nextAttemptId, retried: true };
}
await ctx.db.patch(run._id, {
status: "terminal",
endedAt,
@@ -308,22 +413,16 @@ export const completeAttempt = internalMutation({
const work = await ctx.db.get(run.workId);
if (work)
await ctx.db.patch(work._id, {
status: args.classification === "NeedsInput" ? "needs-input" : "ready",
status: resolution.workStatus,
updatedAt: endedAt,
});
if (work)
await ctx.db.insert("workEvents", {
workId: work._id,
kind: "attempt.completed",
idempotencyKey: `attempt-completed:${attempt._id}`,
referenceId: String(attempt._id),
payloadJson: JSON.stringify({
classification: args.classification,
summary: args.summary,
}),
createdAt: endedAt,
});
return true;
await setSliceStatus(
ctx,
attempt.workId,
run.sliceId,
args.classification === "Succeeded" ? "completed" : "ready"
);
return { retried: false };
},
});
@@ -355,22 +454,26 @@ export const executeFakeAttempt = internalAction({
for (const item of events)
await ctx.runMutation(checkpointRef, {
attemptId: args.attemptId,
owner,
sequence: item.sequence,
kind: item.kind,
message: item.message,
metadataJson: JSON.stringify(item.metadata),
owner,
sequence: item.sequence,
});
await ctx.runMutation(completeRef, {
await ctx.runMutation(finishRef, {
attemptId: args.attemptId,
owner,
classification: outcome.classification,
owner,
retryable: outcome.retryable,
summary: outcome.summary,
});
return null;
},
});
// Bounded recovery: a claimed/running attempt whose lease expired means the
// worker died mid-flight. Terminate that attempt, then either resume within
// the kit budget or settle the Run/Work so nothing stays "running" forever.
export const reconcileExpiredAttempts = internalMutation({
args: {},
handler: async (ctx) => {
@@ -383,17 +486,56 @@ export const reconcileExpiredAttempts = internalMutation({
attempt.leaseExpiresAt < now()
) {
await ctx.db.patch(attempt._id, {
status: "queued",
status: "terminal",
endedAt: now(),
classification: "Blocked",
summary: "Attempt lease expired and was reconciled",
leaseOwner: undefined,
leaseExpiresAt: undefined,
});
const run = await ctx.db.get(attempt.runId);
if (run)
await appendWorkEvent(
ctx,
attempt.workId,
"attempt.reconciled",
`attempt-reconciled:${attempt._id}`,
String(attempt._id),
JSON.stringify({ attemptNumber: attempt.number })
);
reconciled += 1;
if (!run) continue;
const attempts = await ctx.db
.query("workAttempts")
.withIndex("by_run_and_number", (q: any) => q.eq("runId", run._id))
.collect();
const withinBudget =
attempts.length < defaultCodingKitV0.retryPolicy.maxAttempts;
if (run.status === "running" && withinBudget) {
const nextAttemptId = await ctx.db.insert("workAttempts", {
runId: run._id,
workId: attempt.workId,
number: attempts.length + 1,
status: "queued",
});
await ctx.scheduler.runAfter(0, executeRef, {
attemptId: attempt._id,
attemptId: nextAttemptId,
scenario: run.scenario,
});
reconciled += 1;
} else {
await ctx.db.patch(run._id, {
status: "terminal",
endedAt: now(),
terminalClassification: "Blocked",
terminalSummary: "Run reconciled after expired lease",
});
const work = await ctx.db.get(run.workId);
if (work)
await ctx.db.patch(work._id, {
status: "blocked",
updatedAt: now(),
});
await setSliceStatus(ctx, attempt.workId, run.sliceId, "ready");
}
}
}
return { reconciled };