From 0f3f6ce92e8989ea8a1491fb6014a678577b97c7 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Thu, 8 Oct 2026 20:27:02 +0200 Subject: [PATCH] feat(claude): gate Mythos on advertised model identity --- docs/providers.md | 2 + packages/cli/tests/15-provider.test.ts | 5 - .../agent/providers/claude/agent.test.ts | 24 +++ .../server/agent/providers/claude/agent.ts | 33 ++- .../agent/providers/claude/model-discovery.ts | 89 ++++++++ .../agent/providers/claude/model-manifest.ts | 13 +- .../agent/providers/claude/models.test.ts | 195 ++++++++++++++++++ .../server/agent/providers/claude/models.ts | 3 +- 8 files changed, 355 insertions(+), 9 deletions(-) create mode 100644 packages/server/src/server/agent/providers/claude/model-discovery.ts diff --git a/docs/providers.md b/docs/providers.md index 4609246922..298bee651b 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -61,6 +61,8 @@ Core direct providers: `claude` (in `providers/claude/agent.ts`), `codex` (`code Claude first-party model metadata lives in `packages/server/src/server/agent/providers/claude/model-manifest.ts`. When adding or updating a Claude model, update that manifest only; the model picker thinking options and Claude-specific feature gates are derived from the manifest. Do not add model-specific Claude capability lists in feature code. +Keep the curated Claude catalog: `supportedModels()` has historically omitted older versions and carries less metadata. Discovery is only an opt-in visibility gate, not a replacement catalog or an entitlement guarantee. Claude Code v2.1.197 added `resolvedModel` to disambiguate alias rows; a bare family alias does not establish access to a particular version. Explicit user-configured models must remain selectable when discovery fails or omits them. + Paseo tools are not implemented as MCP tools internally. They live in a shared tool catalog under `packages/server/src/server/agent/tools/`; MCP is only the fallback adapter. The daemon resolves `agents.providers..paseoTools` by the exact provider ID. The catalog policy belongs to the caller: it filters the tools exposed to the current agent. When that agent calls `create_agent`, the child receives the policy for the child provider ID; the caller's policy is not inherited. A provider that can register runtime tools directly should set `supportsNativePaseoTools: true` and consume the already-filtered `launchContext.paseoTools` in `createSession`/`resumeSession`. When native tools are present, `AgentManager` strips the internal Paseo MCP server from the provider launch config so the provider does not receive the same tools twice. Providers that only know MCP should keep `supportsMcpServers: true` and let the daemon inject `/mcp/agents`; the MCP server builds the same policy-filtered catalog for that caller. Filtering is enforced at catalog registration in both paths. Browser tools remain subject to the daemon browser-tools setting and browser-host availability. diff --git a/packages/cli/tests/15-provider.test.ts b/packages/cli/tests/15-provider.test.ts index c4731e1de6..3081ee3584 100644 --- a/packages/cli/tests/15-provider.test.ts +++ b/packages/cli/tests/15-provider.test.ts @@ -62,11 +62,6 @@ const EXPECTED_CLAUDE_MODELS = [ model: "Opus 5", descriptionFragment: "Previous release", }, - { - id: "claude-mythos-5-1", - model: "Mythos 5.1", - descriptionFragment: "Requires verified access", - }, { id: "claude-fable-5-1", model: "Fable 5.1", diff --git a/packages/server/src/server/agent/providers/claude/agent.test.ts b/packages/server/src/server/agent/providers/claude/agent.test.ts index dc90b742d3..5a5e1132a1 100644 --- a/packages/server/src/server/agent/providers/claude/agent.test.ts +++ b/packages/server/src/server/agent/providers/claude/agent.test.ts @@ -426,6 +426,14 @@ describe("ClaudeAgentClient.fetchCatalog", () => { try { const client = new ClaudeAgentClient({ logger, + discoverModels: async () => [ + { + value: "mythos", + resolvedModel: "claude-mythos-5-1", + displayName: "Mythos", + description: "", + }, + ], resolveBinary: async () => "/test/claude/bin", resolveVersion: async () => "2.1.219", runtimeSettings: { env: { CLAUDE_CONFIG_DIR: emptyConfigDir } }, @@ -473,6 +481,14 @@ describe("ClaudeAgentClient.fetchCatalog", () => { try { const client = new ClaudeAgentClient({ logger, + discoverModels: async () => [ + { + value: "mythos", + resolvedModel: "claude-mythos-5-1", + displayName: "Mythos", + description: "", + }, + ], resolveVersion: async () => { throw new Error("unrecognized version output"); }, @@ -496,6 +512,14 @@ describe("ClaudeAgentClient.fetchCatalog", () => { try { const client = new ClaudeAgentClient({ logger, + discoverModels: async () => [ + { + value: "mythos", + resolvedModel: "claude-mythos-5-1", + displayName: "Mythos", + description: "", + }, + ], resolveBinary: async () => "/test/claude/bin", resolveVersion: async () => "2.1.293", runtimeSettings: { env: { CLAUDE_CONFIG_DIR: emptyConfigDir } }, diff --git a/packages/server/src/server/agent/providers/claude/agent.ts b/packages/server/src/server/agent/providers/claude/agent.ts index b797ce2c94..6c73f1f643 100644 --- a/packages/server/src/server/agent/providers/claude/agent.ts +++ b/packages/server/src/server/agent/providers/claude/agent.ts @@ -76,6 +76,7 @@ import { type ClaudeProviderOptions, } from "./options.js"; import { renderPromptAttachmentAsText } from "../../prompt-attachments.js"; +import { discoverClaudeModels } from "./model-discovery.js"; import { claudeQuery, type ClaudeOptions, type ClaudeQueryFactory } from "./query.js"; import { realClaudeRewindSdk, @@ -412,6 +413,7 @@ export interface ClaudeContentChunk { } interface ClaudeAgentClientOptions { + discoverModels?: typeof discoverClaudeModels; defaults?: { agents?: Record }; logger: Logger; runtimeSettings?: ProviderRuntimeSettings; @@ -1514,6 +1516,7 @@ export function readEventIdentifiers(message: SDKMessage): EventIdentifiers { } export class ClaudeAgentClient implements AgentClient { + private readonly discoverModels: typeof discoverClaudeModels; readonly provider = "claude" as const; readonly capabilities = CLAUDE_CAPABILITIES; @@ -1526,6 +1529,7 @@ export class ClaudeAgentClient implements AgentClient { private readonly rewindSdk: ClaudeRewindSdk; constructor(options: ClaudeAgentClientOptions) { + this.discoverModels = options.discoverModels ?? discoverClaudeModels; this.defaults = options.defaults; this.logger = options.logger.child({ module: "agent", provider: "claude" }); this.runtimeSettings = options.runtimeSettings; @@ -1608,8 +1612,35 @@ export class ClaudeAgentClient implements AgentClient { this.logger.warn({ err: error }, "Failed to resolve Claude Code version for model catalog"); } const env = this.buildProviderEnv(); + let discoveredModelIds = new Set(); + try { + const advertisedModels = await runProviderRefreshActivity(context, "models", () => + this.discoverModels({ + resolveBinary: this.resolveBinary, + runtimeSettings: this.runtimeSettings, + env, + signal: context?.signal, + }), + ); + discoveredModelIds = new Set( + advertisedModels + .map((model) => normalizeClaudeRuntimeModelId(model.resolvedModel ?? model.value)) + .filter((id): id is string => id !== null), + ); + } catch (error) { + context?.signal.throwIfAborted(); + this.logger.warn( + { err: error }, + "Claude model discovery failed; omitting discovery-required models", + ); + } const models = await runProviderRefreshActivity(context, "settings", () => - getClaudeModelsWithSettings(this.logger, claudeConfigDir(env), claudeCodeVersion), + getClaudeModelsWithSettings( + this.logger, + claudeConfigDir(env), + claudeCodeVersion, + discoveredModelIds, + ), ); const modeCatalog = claudeModeCatalog(env); return { diff --git a/packages/server/src/server/agent/providers/claude/model-discovery.ts b/packages/server/src/server/agent/providers/claude/model-discovery.ts new file mode 100644 index 0000000000..d74898472b --- /dev/null +++ b/packages/server/src/server/agent/providers/claude/model-discovery.ts @@ -0,0 +1,89 @@ +import type { ChildProcess } from "node:child_process"; +import os from "node:os"; +import type { ModelInfo, Query, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; +import { raceProviderRefreshAbort } from "../../provider-refresh-deadline.js"; +import type { ProviderRuntimeSettings } from "../../provider-launch-config.js"; +import { terminateWithTreeKill } from "../../../../utils/tree-kill.js"; +import { claudeQuery, type ClaudeQueryInput, type ClaudeQueryContext } from "./query.js"; + +interface ClaudeModelDiscoveryOptions { + resolveBinary: () => Promise; + runtimeSettings?: ProviderRuntimeSettings; + env: NodeJS.ProcessEnv; + signal?: AbortSignal; +} + +type OpenDiscoveryQuery = ( + input: ClaudeQueryInput, + context: ClaudeQueryContext, +) => Pick; + +// The curated catalog survives a failed or slow probe. This process never receives +// a prompt and must not persist a session or launch user MCP servers/hooks. +export async function discoverClaudeModels( + options: ClaudeModelDiscoveryOptions, + openQuery: OpenDiscoveryQuery = claudeQuery, +): Promise { + const controller = new AbortController(); + const signal = options.signal + ? AbortSignal.any([options.signal, controller.signal]) + : controller.signal; + const timer = setTimeout( + () => controller.abort(new Error("Claude model discovery timed out")), + 5_000, + ); + let endInput!: () => void; + const ended = new Promise((resolve) => { + endInput = resolve; + }); + const input: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next: async () => { + await ended; + return { done: true, value: undefined }; + }, + }; + }, + }; + let query: ReturnType | undefined; + let child: ChildProcess | undefined; + try { + signal.throwIfAborted(); + const binary = await raceProviderRefreshAbort(signal, options.resolveBinary()); + query = openQuery( + { + prompt: input, + options: { + pathToClaudeCodeExecutable: binary, + cwd: os.homedir(), + env: options.env, + persistSession: false, + tools: [], + mcpServers: {}, + strictMcpConfig: true, + settingSources: ["user"], + settings: { disableAllHooks: true }, + abortController: controller, + }, + }, + { + runtimeSettings: options.runtimeSettings, + onChildProcess: (process) => { + child = process; + }, + }, + ); + return await raceProviderRefreshAbort(signal, query.supportedModels()); + } finally { + clearTimeout(timer); + endInput(); + try { + query?.close(); + } finally { + if (child) { + await terminateWithTreeKill(child, { gracefulTimeoutMs: 200, forceTimeoutMs: 200 }); + } + } + } +} diff --git a/packages/server/src/server/agent/providers/claude/model-manifest.ts b/packages/server/src/server/agent/providers/claude/model-manifest.ts index f77c7de2fb..45a5441109 100644 --- a/packages/server/src/server/agent/providers/claude/model-manifest.ts +++ b/packages/server/src/server/agent/providers/claude/model-manifest.ts @@ -9,6 +9,7 @@ interface ClaudeModelManifestEntry { description: string; defaultPriority?: number; minimumClaudeCodeVersion?: string; + requiresDiscovery?: boolean; contextWindowMaxTokens?: number; effortLevels?: readonly ClaudeEffortLevel[]; defaultThinkingOptionId?: ClaudeEffortLevel; @@ -60,6 +61,7 @@ export const CLAUDE_MODEL_MANIFEST = [ { id: "claude-mythos-5-1", label: "Mythos 5.1", + requiresDiscovery: true, description: "Mythos 5.1 ยท Requires verified access", contextWindowMaxTokens: 1_000_000, effortLevels: CLAUDE_EFFORT_LEVELS.xhigh, @@ -224,9 +226,16 @@ function buildThinkingOptions( return options; } -export function getClaudeManifestModels(claudeCodeVersion?: string): AgentModelDefinition[] { +export function getClaudeManifestModels( + claudeCodeVersion?: string, + discoveredModelIds?: ReadonlySet, +): AgentModelDefinition[] { const availableModels: readonly ClaudeModelManifestEntry[] = CLAUDE_MODEL_MANIFEST.filter( - (model) => isModelAvailableInClaudeCode(model, claudeCodeVersion), + (model) => + isModelAvailableInClaudeCode(model, claudeCodeVersion) && + (discoveredModelIds === undefined || + !("requiresDiscovery" in model && model.requiresDiscovery) || + discoveredModelIds.has(model.id)), ); const defaultModel = availableModels.reduce( (selected, candidate) => diff --git a/packages/server/src/server/agent/providers/claude/models.test.ts b/packages/server/src/server/agent/providers/claude/models.test.ts index ce8c583917..988c3a6469 100644 --- a/packages/server/src/server/agent/providers/claude/models.test.ts +++ b/packages/server/src/server/agent/providers/claude/models.test.ts @@ -13,6 +13,8 @@ import { parseClaudeCodeVersion, resolveClaudeDisabledThinkingForModel, } from "./model-manifest.js"; +import { discoverClaudeModels } from "./model-discovery.js"; +import type { ModelInfo } from "@anthropic-ai/claude-agent-sdk"; import { findClaudeModel, getClaudeModels, normalizeClaudeRuntimeModelId } from "./models.js"; const createdClaudeConfigDirs: string[] = []; @@ -43,6 +45,14 @@ function createCatalogClient(claudeCodeVersion = "2.1.293"): ClaudeAgentClient { return new ClaudeAgentClient({ logger: createTestLogger(), resolveVersion: async () => claudeCodeVersion, + discoverModels: async () => [ + { + value: "mythos", + resolvedModel: "claude-mythos-5-1", + displayName: "Mythos", + description: "", + }, + ], }); } @@ -223,6 +233,116 @@ describe("getClaudeModels", () => { }); describe("ClaudeAgentClient.fetchCatalog", () => { + it.each([ + { value: "mythos", resolvedModel: "claude-mythos-5-1", expected: true }, + { value: "claude-mythos-5-1", expected: true }, + { value: "claude-mythos-5-1[1m]", expected: true }, + { value: "global.anthropic.claude-mythos-5-1", expected: true }, + { value: "mythos", expected: false }, + { value: "mythos", resolvedModel: "claude-mythos-5-2", expected: false }, + { value: "claude-mythos-5-1", resolvedModel: "claude-fable-5-1", expected: false }, + ])( + "uses advertised identity $value / $resolvedModel to gate Mythos", + async ({ value, resolvedModel, expected }) => { + const configDir = await createClaudeConfigDir({}); + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + resolveVersion: async () => "2.1.293", + runtimeSettings: { env: { CLAUDE_CONFIG_DIR: configDir } }, + discoverModels: async () => [ + { value, resolvedModel, displayName: "Untrusted label", description: "" }, + ], + }); + const { models } = await client.fetchCatalog({ + scope: "workspace", + cwd: configDir, + force: false, + }); + expect(models.find((model) => model.id === "claude-mythos-5-1")).toEqual( + expected ? findClaudeModel("claude-mythos-5-1") : undefined, + ); + }, + ); + + it("preserves the curated catalog and explicitly configured models after discovery failure", async () => { + const configDir = await createClaudeConfigDir({ model: "claude-mythos-5-1" }); + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + resolveVersion: async () => "2.1.293", + runtimeSettings: { env: { CLAUDE_CONFIG_DIR: configDir } }, + discoverModels: async () => { + throw new Error("discovery unavailable"); + }, + }); + const { models } = await client.fetchCatalog({ + scope: "workspace", + cwd: configDir, + force: false, + }); + expect(models.filter((model) => model.id !== "claude-mythos-5-1")).toEqual( + getClaudeModels("2.1.293").filter((model) => model.id !== "claude-mythos-5-1"), + ); + expect(models.find((model) => model.id === "claude-mythos-5-1")?.description).toBe( + "From Claude settings.json model", + ); + }); + + it("omits unconfirmed Mythos after a failed probe and retries on refresh", async () => { + const configDir = await createClaudeConfigDir({}); + let fail = true; + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + resolveVersion: async () => "2.1.293", + runtimeSettings: { env: { CLAUDE_CONFIG_DIR: configDir } }, + discoverModels: async () => { + if (fail) throw new Error("discovery unavailable"); + return [ + { + value: "mythos", + resolvedModel: "claude-mythos-5-1", + displayName: "Mythos", + description: "", + }, + ]; + }, + }); + const options = { scope: "workspace" as const, cwd: configDir, force: true }; + expect( + (await client.fetchCatalog(options)).models.some((model) => model.id === "claude-mythos-5-1"), + ).toBe(false); + fail = false; + expect( + (await client.fetchCatalog(options)).models.find((model) => model.id === "claude-mythos-5-1"), + ).toEqual(findClaudeModel("claude-mythos-5-1")); + }); + + it("keeps the curated catalog when discovery does not advertise Mythos", async () => { + const configDir = await createClaudeConfigDir({}); + const client = new ClaudeAgentClient({ + logger: createTestLogger(), + resolveVersion: async () => "2.1.293", + runtimeSettings: { env: { CLAUDE_CONFIG_DIR: configDir } }, + discoverModels: async () => [ + { + value: "opus", + resolvedModel: "claude-opus-5-5", + displayName: "Opus", + description: "Latest", + }, + ], + }); + const { models } = await client.fetchCatalog({ + scope: "workspace", + cwd: configDir, + force: false, + }); + expect(models.map((model) => model.id)).toEqual( + getClaudeModels("2.1.293") + .filter((model) => model.id !== "claude-mythos-5-1") + .map((model) => model.id), + ); + }); + it("appends concrete models from Claude settings.json", async () => { const configDir = await createClaudeConfigDir({ model: "us.anthropic.claude-opus-4-7[1m]", @@ -300,6 +420,7 @@ describe("ClaudeAgentClient.fetchCatalog", () => { const client = new ClaudeAgentClient({ logger: createTestLogger(), resolveVersion: async () => "2.1.280", + discoverModels: async () => [], runtimeSettings: { env: { CLAUDE_CONFIG_DIR: providerConfigDir } }, }); @@ -773,3 +894,77 @@ describe("Claude Mythos 5.1 catalog", () => { }, ); }); + +describe("Claude model discovery process", () => { + it("does not send a prompt or persist a session and closes after discovery", async () => { + let closed = false; + let promptEnded: Promise> | undefined; + const advertised: ModelInfo[] = [ + { + value: "mythos", + resolvedModel: "claude-mythos-5-1", + displayName: "Mythos", + description: "", + }, + ]; + const models = await discoverClaudeModels( + { resolveBinary: async () => "/claude", env: { CLAUDE_CONFIG_DIR: "/provider-config" } }, + (input) => { + expect(input.options).toMatchObject({ + pathToClaudeCodeExecutable: "/claude", + env: { CLAUDE_CONFIG_DIR: "/provider-config" }, + persistSession: false, + tools: [], + mcpServers: {}, + strictMcpConfig: true, + settingSources: ["user"], + settings: { disableAllHooks: true }, + }); + if (typeof input.prompt === "string") throw new Error("unexpected inference prompt"); + promptEnded = input.prompt[Symbol.asyncIterator]().next(); + return { + supportedModels: async () => advertised, + close: () => { + closed = true; + }, + }; + }, + ); + expect(models).toEqual(advertised); + expect(closed).toBe(true); + await expect(promptEnded).resolves.toEqual({ done: true, value: undefined }); + }); + + it("times out and closes an unresponsive discovery process", async () => { + let closed = false; + await expect( + discoverClaudeModels({ resolveBinary: async () => "/claude", env: {} }, () => ({ + supportedModels: () => new Promise(() => {}), + close: () => { + closed = true; + }, + })), + ).rejects.toThrow("Claude model discovery timed out"); + expect(closed).toBe(true); + }, 10_000); + + it("closes a stalled probe when its refresh is canceled", async () => { + const controller = new AbortController(); + const failure = new Error("refresh canceled"); + let closed = false; + const result = discoverClaudeModels( + { resolveBinary: async () => "/claude", env: {}, signal: controller.signal }, + () => ({ + supportedModels: () => { + controller.abort(failure); + return new Promise(() => {}); + }, + close: () => { + closed = true; + }, + }), + ); + await expect(result).rejects.toBe(failure); + expect(closed).toBe(true); + }); +}); diff --git a/packages/server/src/server/agent/providers/claude/models.ts b/packages/server/src/server/agent/providers/claude/models.ts index 6b341fa9ea..342c3cab37 100644 --- a/packages/server/src/server/agent/providers/claude/models.ts +++ b/packages/server/src/server/agent/providers/claude/models.ts @@ -52,8 +52,9 @@ export async function getClaudeModelsWithSettings( logger: Logger, configDir: string, claudeCodeVersion?: string, + discoveredModelIds: ReadonlySet = new Set(), ): Promise { - const hardcodedModels = getClaudeModels(claudeCodeVersion); + const hardcodedModels = getClaudeManifestModels(claudeCodeVersion, discoveredModelIds); const settingsModels = await readClaudeSettingsModels(logger, configDir); if (settingsModels.length === 0) { return hardcodedModels;