From 3beb7707d613782789bea6e0de2fc1f991b01f10 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 4 Nov 2026 09:22:09 +0100 Subject: [PATCH] test(server): the test providers list two models with thinking options, and the live switches run The fake providers listed one model and no thinking options, so once the daemon validated selections against the catalog the live model and thinking switches had nothing valid to switch to and skipped. Every fake provider now lists two models with thinking options, and the switches run for all three providers without depending on installed binaries. --- .../daemon-e2e/live-preferences.e2e.test.ts | 248 ++++++++---------- .../server/test-utils/fake-agent-client.ts | 55 ++-- 2 files changed, 137 insertions(+), 166 deletions(-) diff --git a/packages/server/src/server/daemon-e2e/live-preferences.e2e.test.ts b/packages/server/src/server/daemon-e2e/live-preferences.e2e.test.ts index 8d80b07448..7ab7abeb83 100644 --- a/packages/server/src/server/daemon-e2e/live-preferences.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/live-preferences.e2e.test.ts @@ -2,7 +2,6 @@ import { describe, test, expect, beforeEach, afterEach } from "vitest"; import { mkdtempSync, rmSync } from "fs"; import { tmpdir } from "os"; import path from "path"; -import { execFileSync } from "node:child_process"; import { createDaemonTestContext, type DaemonTestContext } from "../test-utils/index.js"; import type { AgentSnapshotPayload, SessionOutboundMessage } from "../messages.js"; @@ -39,18 +38,13 @@ function waitForAgentUpdate( }); } -// The daemon only switches to a model the provider lists, so a provider that lists one model has -// nothing to switch to. -function pickModelSwitchPair( - provider: string, - models: Array<{ id: string }>, -): [string, string] | null { - const ids = Array.from(new Set(models.map((m) => m.id))).filter(Boolean); - const [first, second] = ids; - if (!first) { - throw new Error(`No models returned for provider ${provider}`); +// The daemon only switches to a model the provider lists. +function pickModelSwitchPair(provider: string, models: Array<{ id: string }>): [string, string] { + const [first, second] = Array.from(new Set(models.map((m) => m.id))).filter(Boolean); + if (!first || !second) { + throw new Error(`Provider ${provider} lists fewer than two models`); } - return second ? [first, second] : null; + return [first, second]; } function pickThinkingSwitchOption( @@ -59,13 +53,12 @@ function pickThinkingSwitchOption( thinkingOptions?: Array<{ id: string }>; defaultThinkingOptionId?: string; }>, -): { model: (typeof models)[number]; thinkingOptionId: string } | null { - if (models.length === 0) { - throw new Error(`No ${provider} models returned`); - } +): { model: (typeof models)[number]; thinkingOptionId: string } { // The daemon only switches to a thinking option the model lists. const modelWithOptions = models.find((m) => (m.thinkingOptions?.length ?? 0) > 0); - if (!modelWithOptions) return null; + if (!modelWithOptions) { + throw new Error(`No ${provider} model lists thinking options`); + } const defaultThinkingId = modelWithOptions.defaultThinkingOptionId ?? "default"; const thinkingOptionId = @@ -77,18 +70,6 @@ function pickThinkingSwitchOption( return { model: modelWithOptions, thinkingOptionId }; } -function isBinaryInstalled(binary: string): boolean { - try { - const out = execFileSync("which", [binary], { encoding: "utf8" }).trim(); - return out.length > 0; - } catch { - return false; - } -} - -const hasCodex = isBinaryInstalled("codex"); -const hasOpenCode = isBinaryInstalled("opencode"); - let ctx: DaemonTestContext; let messages: SessionOutboundMessage[] = []; let unsubscribe: (() => void) | null = null; @@ -108,60 +89,47 @@ afterEach(async () => { }, 60000); describe.each(["claude", "codex", "opencode"] as const)("live model switching (%s)", (provider) => { - const shouldRun = - provider === "claude" || - (provider === "codex" && hasCodex) || - (provider === "opencode" && hasOpenCode); - - test.runIf(shouldRun)( - "updates agent model without restarting", - async ({ skip }) => { - const cwd = tmpCwd(); - try { - const modelList = await ctx.client.listProviderModels(provider); - if (!modelList.models || modelList.models.length === 0) { - throw new Error(`No models returned for provider ${provider}`); - } - const pair = pickModelSwitchPair(provider, modelList.models); - if (!pair) skip(`${provider} lists one model, so there is nothing to switch to`); - const [modelA, modelB] = pair; - - const agent = await ctx.client.createAgent({ - provider, - cwd, - title: `Model Switch (${provider})`, - model: modelA, - }); - - const startIndex = messages.length; - await ctx.client.setAgentModel(agent.id, modelB); - - const updated = await waitForAgentUpdate( - messages, - startIndex, - (a) => a.id === agent.id && a.model === modelB, - { timeoutMs: 20000 }, - ); - - expect(updated.model).toBe(modelB); - } finally { - rmSync(cwd, { recursive: true, force: true }); + test("updates agent model without restarting", async () => { + const cwd = tmpCwd(); + try { + const modelList = await ctx.client.listProviderModels(provider); + if (!modelList.models || modelList.models.length === 0) { + throw new Error(`No models returned for provider ${provider}`); } - }, - 180000, - ); + const [modelA, modelB] = pickModelSwitchPair(provider, modelList.models); + + const agent = await ctx.client.createAgent({ + provider, + cwd, + title: `Model Switch (${provider})`, + model: modelA, + }); + + const startIndex = messages.length; + await ctx.client.setAgentModel(agent.id, modelB); + + const updated = await waitForAgentUpdate( + messages, + startIndex, + (a) => a.id === agent.id && a.model === modelB, + { timeoutMs: 20000 }, + ); + + expect(updated.model).toBe(modelB); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } + }, 180000); }); -test("live thinking switching works for Claude (default -> non-default)", async ({ skip }) => { +test("live thinking switching works for Claude (default -> non-default)", async () => { const cwd = tmpCwd(); try { const modelList = await ctx.client.listProviderModels("claude"); if (!modelList.models || modelList.models.length === 0) { throw new Error("No Claude models returned"); } - const pick = pickThinkingSwitchOption("Claude", modelList.models); - if (!pick) skip("No Claude model lists thinking options"); - const { model, thinkingOptionId } = pick; + const { model, thinkingOptionId } = pickThinkingSwitchOption("Claude", modelList.models); const agent = await ctx.client.createAgent({ provider: "claude", @@ -186,78 +154,72 @@ test("live thinking switching works for Claude (default -> non-default)", async } }, 120000); -test.runIf(hasCodex)( - "live thinking switching works for Codex (default -> non-default)", - async ({ skip }) => { - const cwd = tmpCwd(); - try { - const modelList = await ctx.client.listProviderModels("codex"); - if (!modelList.models || modelList.models.length === 0) { - throw new Error("No Codex models returned"); - } - - const pick = pickThinkingSwitchOption("Codex", modelList.models); - if (!pick) skip("No Codex model lists thinking options"); - const { model: modelWithOptions, thinkingOptionId } = pick; - - const agent = await ctx.client.createAgent({ - provider: "codex", - cwd, - title: "Codex Thinking Switch", - model: modelWithOptions.id, - }); - - const startIndex = messages.length; - await ctx.client.setAgentThinkingOption(agent.id, thinkingOptionId); - - const updated = await waitForAgentUpdate( - messages, - startIndex, - (a) => a.id === agent.id && a.thinkingOptionId === thinkingOptionId, - { timeoutMs: 20000 }, - ); - - expect(updated.thinkingOptionId).toBe(thinkingOptionId); - } finally { - rmSync(cwd, { recursive: true, force: true }); +test("live thinking switching works for Codex (default -> non-default)", async () => { + const cwd = tmpCwd(); + try { + const modelList = await ctx.client.listProviderModels("codex"); + if (!modelList.models || modelList.models.length === 0) { + throw new Error("No Codex models returned"); } - }, - 120000, -); -test.runIf(hasOpenCode)( - "live thinking switching works for OpenCode", - async ({ skip }) => { - const cwd = tmpCwd(); - try { - const modelList = await ctx.client.listProviderModels("opencode"); - if (!modelList.models || modelList.models.length === 0) { - throw new Error("No OpenCode models returned"); - } + const { model: modelWithOptions, thinkingOptionId } = pickThinkingSwitchOption( + "Codex", + modelList.models, + ); - const pick = pickThinkingSwitchOption("OpenCode", modelList.models); - if (!pick) skip("No OpenCode model lists thinking options"); - const { model: modelWithThinkingOptions, thinkingOptionId } = pick; + const agent = await ctx.client.createAgent({ + provider: "codex", + cwd, + title: "Codex Thinking Switch", + model: modelWithOptions.id, + }); - const agent = await ctx.client.createAgent({ - provider: "opencode", - cwd, - title: "OpenCode Preferences Switch", - model: modelWithThinkingOptions.id, - }); + const startIndex = messages.length; + await ctx.client.setAgentThinkingOption(agent.id, thinkingOptionId); - const startIndex = messages.length; - await ctx.client.setAgentThinkingOption(agent.id, thinkingOptionId); - const updatedThinking = await waitForAgentUpdate( - messages, - startIndex, - (a) => a.id === agent.id && a.thinkingOptionId === thinkingOptionId, - { timeoutMs: 20000 }, - ); - expect(updatedThinking.thinkingOptionId).toBe(thinkingOptionId); - } finally { - rmSync(cwd, { recursive: true, force: true }); + const updated = await waitForAgentUpdate( + messages, + startIndex, + (a) => a.id === agent.id && a.thinkingOptionId === thinkingOptionId, + { timeoutMs: 20000 }, + ); + + expect(updated.thinkingOptionId).toBe(thinkingOptionId); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } +}, 120000); + +test("live thinking switching works for OpenCode", async () => { + const cwd = tmpCwd(); + try { + const modelList = await ctx.client.listProviderModels("opencode"); + if (!modelList.models || modelList.models.length === 0) { + throw new Error("No OpenCode models returned"); } - }, - 180000, -); + + const { model: modelWithThinkingOptions, thinkingOptionId } = pickThinkingSwitchOption( + "OpenCode", + modelList.models, + ); + + const agent = await ctx.client.createAgent({ + provider: "opencode", + cwd, + title: "OpenCode Preferences Switch", + model: modelWithThinkingOptions.id, + }); + + const startIndex = messages.length; + await ctx.client.setAgentThinkingOption(agent.id, thinkingOptionId); + const updatedThinking = await waitForAgentUpdate( + messages, + startIndex, + (a) => a.id === agent.id && a.thinkingOptionId === thinkingOptionId, + { timeoutMs: 20000 }, + ); + expect(updatedThinking.thinkingOptionId).toBe(thinkingOptionId); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } +}, 180000); diff --git a/packages/server/src/server/test-utils/fake-agent-client.ts b/packages/server/src/server/test-utils/fake-agent-client.ts index 01e17836e4..a7a237b9de 100644 --- a/packages/server/src/server/test-utils/fake-agent-client.ts +++ b/packages/server/src/server/test-utils/fake-agent-client.ts @@ -16,6 +16,7 @@ import type { AgentRunOptions, AgentRunResult, AgentSession, + AgentSelectOption, AgentSessionConfig, AgentStreamEvent, AgentSlashCommand, @@ -38,6 +39,26 @@ const TEST_CAPABILITIES: AgentCapabilityFlags = { }; const TEST_FEATURE_ID = "test_feature"; +const TEST_THINKING_OPTIONS: AgentSelectOption[] = [ + { id: "low", label: "Low", isDefault: true }, + { id: "high", label: "High" }, +]; + +const TEST_MODELS: Record> = { + claude: [ + { id: "haiku", label: "Haiku" }, + { id: "sonnet", label: "Sonnet" }, + ], + codex: [ + { id: "gpt-5.4-mini", label: "gpt-5.4-mini" }, + { id: "gpt-5.4", label: "gpt-5.4" }, + ], + opencode: [ + { id: "test-model", label: "Test Model" }, + { id: "test-model-alt", label: "Test Model Alt" }, + ], +}; + const TEST_MODES: AgentMode[] = [ { id: "bypassPermissions", label: "Bypass", description: "No permissions" }, { id: "default", label: "Default", description: "Ask for permissions" }, @@ -1280,30 +1301,18 @@ class FakeAgentClient implements AgentClient { async fetchCatalog( _options: FetchCatalogOptions, ): Promise<{ models: AgentModelDefinition[]; modes: AgentMode[] }> { - if (this.provider === "claude") { - return { - models: [ - { provider: this.provider, id: "haiku", label: "Haiku", isDefault: true }, - { provider: this.provider, id: "sonnet", label: "Sonnet", isDefault: false }, - ], - modes: TEST_MODES, - }; - } - if (this.provider === "codex") { - return { - models: [ - { - provider: this.provider, - id: "gpt-5.4-mini", - label: "gpt-5.4-mini", - isDefault: true, - }, - ], - modes: TEST_MODES, - }; - } + // Every provider lists two models that each list thinking options, so the live model and + // thinking switches have listed values to switch between. + const listed = TEST_MODELS[this.provider] ?? TEST_MODELS.opencode; return { - models: [{ provider: this.provider, id: "test-model", label: "Test Model", isDefault: true }], + models: listed.map((model, index) => ({ + provider: this.provider, + id: model.id, + label: model.label, + isDefault: index === 0, + thinkingOptions: TEST_THINKING_OPTIONS, + defaultThinkingOptionId: "low", + })), modes: TEST_MODES, }; }