mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-04 23:44:41 +00:00
fix(codex): fail fully rejected automation runs
A successful native turn can contain only a blocked prompt. Fail that attempt with its rejection reason while retaining successful outcomes once the native turn accepts input or starts work.
This commit is contained in:
@@ -82,7 +82,93 @@ describe("Codex prompt hook rejection", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("keeps the rejection visible in the manager timeline and permits a corrected submission", async () => {
|
||||
test.each(["before", "after"])(
|
||||
"keeps an accepted input successful when rejected input arrives %s it, even without a reply",
|
||||
async (order) => {
|
||||
const appServer = createFakeCodexAppServer();
|
||||
const { session } = await startPublicSteeringSession(appServer);
|
||||
try {
|
||||
function rejectInput(): void {
|
||||
appServer.completesHook({
|
||||
threadId: "thread-1",
|
||||
turnId: "native-A",
|
||||
eventName: "userPromptSubmit",
|
||||
status: "blocked",
|
||||
entries: [{ kind: "feedback", text: "Rejected follow-up" }],
|
||||
});
|
||||
}
|
||||
if (order === "before") rejectInput();
|
||||
appServer.acceptsUserMessage({
|
||||
threadId: "thread-1",
|
||||
turnId: "native-A",
|
||||
itemId: "accepted-input",
|
||||
text: "Accepted prompt",
|
||||
});
|
||||
if (order === "after") rejectInput();
|
||||
const completion = waitForNextTurnCompletion(session);
|
||||
appServer.completeTurn();
|
||||
await expect(completion).resolves.toMatchObject({ type: "turn_completed" });
|
||||
appServer.assertNoErrors();
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test.each([
|
||||
{ status: "interrupted", expected: { type: "turn_canceled", reason: "interrupted" } },
|
||||
{ status: "failed", expected: { type: "turn_failed", error: "Native failure" } },
|
||||
] as const)(
|
||||
"preserves native $status outcomes after a rejected prompt",
|
||||
async ({ status, expected }) => {
|
||||
const appServer = createFakeCodexAppServer();
|
||||
const { session } = await startPublicSteeringSession(appServer);
|
||||
try {
|
||||
appServer.completesHook({
|
||||
threadId: "thread-1",
|
||||
turnId: "native-A",
|
||||
eventName: "userPromptSubmit",
|
||||
status: "blocked",
|
||||
entries: [{ kind: "feedback", text: "Rejected prompt" }],
|
||||
});
|
||||
const completion = waitForNextTurnCompletion(session);
|
||||
appServer.completeTurn({ status, error: { message: "Native failure" } });
|
||||
await expect(completion).resolves.toMatchObject(expected);
|
||||
appServer.assertNoErrors();
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test.each([
|
||||
{ threadId: "thread-1", turnId: "older-turn" },
|
||||
{ threadId: "child-thread", turnId: "native-A" },
|
||||
])("does not count foreign input as accepted work: $threadId $turnId", async (input) => {
|
||||
const appServer = createFakeCodexAppServer();
|
||||
const { session } = await startPublicSteeringSession(appServer);
|
||||
try {
|
||||
appServer.acceptsUserMessage({ ...input, itemId: "foreign-input", text: "Unrelated input" });
|
||||
appServer.completesHook({
|
||||
threadId: "thread-1",
|
||||
turnId: "native-A",
|
||||
eventName: "userPromptSubmit",
|
||||
status: "blocked",
|
||||
entries: [{ kind: "feedback", text: "Rejected prompt" }],
|
||||
});
|
||||
const completion = waitForNextTurnCompletion(session);
|
||||
appServer.completeTurn();
|
||||
await expect(completion).resolves.toMatchObject({
|
||||
type: "turn_failed",
|
||||
error: "Rejected prompt",
|
||||
});
|
||||
appServer.assertNoErrors();
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("fails a fully rejected manager run and permits a corrected submission", async () => {
|
||||
const appServer = createFakeCodexAppServer();
|
||||
const provider = createProviderWithFakeAppServer(appServer);
|
||||
const client: AgentClient = {
|
||||
@@ -102,6 +188,7 @@ describe("Codex prompt hook rejection", () => {
|
||||
const run = manager.runAgent(agent.id, "Disallowed prompt", {
|
||||
clientMessageId: "blocked-prompt",
|
||||
});
|
||||
const failure = expect(run).rejects.toThrow("Remove the disallowed text and retry.");
|
||||
await appServer.waitForTurnStart();
|
||||
appServer.startsTurn({ threadId: "thread-1", turnId: "native-A" });
|
||||
appServer.completesHook({
|
||||
@@ -112,7 +199,7 @@ describe("Codex prompt hook rejection", () => {
|
||||
entries: [{ kind: "feedback", text: "Remove the disallowed text and retry." }],
|
||||
});
|
||||
appServer.completeTurn();
|
||||
await expect(run).resolves.toMatchObject({ finalText: "", canceled: false });
|
||||
await failure;
|
||||
|
||||
expect(manager.getTimeline(agent.id)).toEqual([
|
||||
{
|
||||
@@ -122,9 +209,10 @@ describe("Codex prompt hook rejection", () => {
|
||||
clientMessageId: "blocked-prompt",
|
||||
},
|
||||
{ type: "notification", level: "error", message: "Remove the disallowed text and retry." },
|
||||
{ type: "assistant_message", text: "[System Error] Remove the disallowed text and retry." },
|
||||
]);
|
||||
expect(manager.hasInFlightRun(agent.id)).toBe(false);
|
||||
expect(manager.getAgent(agent.id)?.lastError).toBeUndefined();
|
||||
expect(manager.getAgent(agent.id)?.lastError).toBe("Remove the disallowed text and retry.");
|
||||
|
||||
const retry = manager.runAgent(agent.id, "Corrected prompt", {
|
||||
clientMessageId: "corrected-prompt",
|
||||
@@ -171,9 +259,9 @@ describe("Codex prompt hook rejection", () => {
|
||||
turnId: paseoTurnId,
|
||||
},
|
||||
{
|
||||
type: "turn_completed",
|
||||
type: "turn_failed",
|
||||
provider: "codex",
|
||||
usage: undefined,
|
||||
error: "Remove the disallowed text and retry.",
|
||||
turnId: paseoTurnId,
|
||||
},
|
||||
]);
|
||||
@@ -274,9 +362,9 @@ describe("Codex prompt hook rejection", () => {
|
||||
turnId: paseoTurnId,
|
||||
},
|
||||
{
|
||||
type: "turn_completed",
|
||||
type: "turn_failed",
|
||||
provider: "codex",
|
||||
usage: undefined,
|
||||
error: "Prompt rejected by a UserPromptSubmit hook.",
|
||||
turnId: paseoTurnId,
|
||||
},
|
||||
]);
|
||||
|
||||
@@ -3521,6 +3521,7 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
// in this app-server, so no other app-server can resume it and it is not reported.
|
||||
private savedThreadId: string | null = null;
|
||||
private currentTurnId: string | null = null;
|
||||
private turnPromptOutcome: "unobserved" | "accepted" | { rejection: string } = "unobserved";
|
||||
private pendingForegroundTurnIdentification: {
|
||||
foregroundTurnId: string;
|
||||
promise: Promise<string | null>;
|
||||
@@ -5846,9 +5847,11 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
this.handlePatchApplyCompletedNotification(parsed);
|
||||
return;
|
||||
case "item_completed":
|
||||
this.recordAcceptedTurnWork(parsed);
|
||||
await this.handleItemCompletedNotification(parsed);
|
||||
return;
|
||||
case "item_started":
|
||||
this.recordAcceptedTurnWork(parsed);
|
||||
await this.handleItemStartedNotification(parsed);
|
||||
return;
|
||||
case "invalid_payload":
|
||||
@@ -6451,14 +6454,14 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
.map((entry) => entry.text.trim())
|
||||
.filter((text) => text.length > 0)
|
||||
.join("\n\n");
|
||||
const message = feedback || "Prompt rejected by a UserPromptSubmit hook.";
|
||||
if (this.turnPromptOutcome !== "accepted") {
|
||||
this.turnPromptOutcome = { rejection: message };
|
||||
}
|
||||
this.emitEvent({
|
||||
type: "timeline",
|
||||
provider: CODEX_PROVIDER,
|
||||
item: {
|
||||
type: "notification",
|
||||
level: "error",
|
||||
message: feedback || "Prompt rejected by a UserPromptSubmit hook.",
|
||||
},
|
||||
item: { type: "notification", level: "error", message },
|
||||
});
|
||||
}
|
||||
|
||||
@@ -6492,6 +6495,12 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
} else if (parsed.status === "interrupted") {
|
||||
this.dismissInterruptedAsyncQuestions();
|
||||
this.emitEvent({ type: "turn_canceled", provider: CODEX_PROVIDER, reason: "interrupted" });
|
||||
} else if (typeof this.turnPromptOutcome === "object") {
|
||||
this.emitEvent({
|
||||
type: "turn_failed",
|
||||
provider: CODEX_PROVIDER,
|
||||
error: this.turnPromptOutcome.rejection,
|
||||
});
|
||||
} else {
|
||||
if (this.planModeEnabled && this.latestPlanResult?.text) {
|
||||
this.emitSyntheticPlanApprovalRequest(this.latestPlanResult.text);
|
||||
@@ -6511,7 +6520,22 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
this.resetTurnTrackingState();
|
||||
}
|
||||
|
||||
private recordAcceptedTurnWork(
|
||||
parsed: Extract<ParsedCodexNotification, { kind: "item_started" | "item_completed" }>,
|
||||
): void {
|
||||
// Rejected prompts produce no user-message or work items. Any accepted input or
|
||||
// autonomous work in this native turn keeps a rejected follow-up from failing it.
|
||||
if (
|
||||
this.currentTurnId !== null &&
|
||||
parsed.threadId === this.currentThreadId &&
|
||||
(parsed.turnId === null || parsed.turnId === this.currentTurnId)
|
||||
) {
|
||||
this.turnPromptOutcome = "accepted";
|
||||
}
|
||||
}
|
||||
|
||||
private resetTurnTrackingState(): void {
|
||||
this.turnPromptOutcome = "unobserved";
|
||||
this.latestPlanResult = null;
|
||||
this.emittedItemStartedIds.clear();
|
||||
this.emittedItemCompletedIds.clear();
|
||||
|
||||
@@ -81,6 +81,12 @@ export interface FakeCodexAppServer {
|
||||
startsTurn(params: { threadId: string; turnId?: string }): void;
|
||||
startsCompaction(params: { threadId: string; itemId: string }): void;
|
||||
updatesPlan(params: { threadId: string; steps: string[] }): void;
|
||||
acceptsUserMessage(params: {
|
||||
threadId: string;
|
||||
turnId: string;
|
||||
itemId: string;
|
||||
text: string;
|
||||
}): void;
|
||||
completesHook(params: FakeHookCompletion): void;
|
||||
completeTurn(params?: {
|
||||
threadId?: string;
|
||||
@@ -428,6 +434,15 @@ export function createFakeCodexAppServer(
|
||||
})}\n`,
|
||||
);
|
||||
},
|
||||
acceptsUserMessage({ threadId, turnId, itemId, text }) {
|
||||
const params = {
|
||||
threadId,
|
||||
turnId,
|
||||
item: { type: "userMessage", id: itemId, content: [{ type: "text", text }] },
|
||||
};
|
||||
writeNotification("item/started", params);
|
||||
writeNotification("item/completed", params);
|
||||
},
|
||||
completesHook({ threadId, turnId, ...run }) {
|
||||
writeNotification("hook/completed", { threadId, turnId, run });
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user