common/hermes
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(aux): remove hardcoded Codex fallback model, drop Codex from auto chain (#17765)

Browse Source 
The _CODEX_AUX_MODEL constant had already rotated twice in 6 weeks
(gpt-5.3-codex -> gpt-5.2-codex -> now broken again at gpt-5.2-codex)
because ChatGPT-account Codex gates which models it accepts via an
undocumented, shifting allow-list that OpenAI publishes no changelog
for.  Any pinned default will keep going stale.  Issue #17533 reports
the current breakage: every ChatGPT-account auxiliary fallback fails
with HTTP 400 "model is not supported" and the 60s pause loop degrades
long sessions.

Rather than reset the clock with another stale pin (PR #17544 proposes
gpt-5.2-codex -> gpt-5.4), remove the hardcoded second-order Codex
fallback entirely:

- Delete `_CODEX_AUX_MODEL`.
- Drop `_try_codex` from `_get_provider_chain()` (the auto chain now
  ends at api-key providers; 4 rungs instead of 5).
- Rename `_try_codex() -> _build_codex_client(model)` and require an
  explicit model from the caller.  No more guessing.
- `resolve_provider_client("openai-codex", model=None)` now warns and
  returns (None, None) instead of silently guessing a stale model ID.
- Remove `_try_codex` from the `provider="custom"` fallback ladder
  (same stale-constant trap).
- `_resolve_strict_vision_backend("openai-codex")` routes through
  `resolve_provider_client` so the caller's explicit model is honored.

Codex-main users are unaffected: Step 1 of `_resolve_auto` already
uses `main_provider` + `main_model` directly and passes the user's
configured Codex model through `resolve_provider_client`, which never
touched `_CODEX_AUX_MODEL`.  Per-task overrides (`auxiliary.<task>.provider/model`)
continue to work and are the supported way to route specific aux tasks
through Codex.

Users whose main provider fails with a payment/connection error and
who have ONLY ChatGPT-account Codex auth will now see the 60s pause
without a stale-model-rejection noise line in between -- same outcome,
cleaner failure.

Closes #17533.  Supersedes #17544 (which resets the clock on the
same stale-constant problem).
This commit is contained in:
Teknium
2026-04-29 23:23:50 -07:00
committed by GitHub
parent f73364b1c4
commit ce0c3ae493
4 changed files with 123 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
tests/agent/test_auxiliary_client.py
View File

@@ -259,7 +259,7 @@ class TestAnthropicOAuthFlag:
assert mock_build.call_args.args[0] == "sk-ant-oat01-pooled"
class TestTryCodex:
class TestBuildCodexClient:
def test_pool_without_selected_entry_falls_back_to_auth_store(self):
with (
patch("agent.auxiliary_client._select_pool_entry", return_value=(True, None)),
@@ -267,15 +267,23 @@ class TestTryCodex:
patch("agent.auxiliary_client.OpenAI") as mock_openai,
):
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import _try_codex
from agent.auxiliary_client import _build_codex_client
client, model = _try_codex()
client, model = _build_codex_client("gpt-5.4")
assert client is not None
assert model == "gpt-5.2-codex"
assert model == "gpt-5.4"
assert mock_openai.call_args.kwargs["api_key"] == "codex-auth-token"
assert mock_openai.call_args.kwargs["base_url"] == "https://chatgpt.com/backend-api/codex"
def test_rejects_missing_model(self):
"""Callers must pass an explicit model; no hardcoded default."""
from agent.auxiliary_client import _build_codex_client
client, model = _build_codex_client("")
assert client is None
assert model is None
class TestExpiredCodexFallback:
"""Test that expired Codex tokens don't block the auto chain."""
@@ -507,14 +515,14 @@ class TestGetTextAuxiliaryClient:
patch("agent.auxiliary_client.OpenAI"),
patch("hermes_cli.auth._read_codex_tokens", side_effect=AssertionError("legacy codex store should not run")),
):
from agent.auxiliary_client import _try_codex
from agent.auxiliary_client import _build_codex_client
client, model = _try_codex()
client, model = _build_codex_client("gpt-5.4")
from agent.auxiliary_client import CodexAuxiliaryClient
assert isinstance(client, CodexAuxiliaryClient)
assert model == "gpt-5.2-codex"
assert model == "gpt-5.4"
def test_returns_none_when_nothing_available(self, monkeypatch):
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
@@ -783,11 +791,15 @@ class TestIsPaymentError:
class TestGetProviderChain:
"""_get_provider_chain() resolves functions at call time (testable)."""
def test_returns_five_entries(self):
def test_returns_four_entries(self):
chain = _get_provider_chain()
assert len(chain) == 5
assert len(chain) == 4
labels = [label for label, _ in chain]
assert labels == ["openrouter", "nous", "local/custom", "openai-codex", "api-key"]
assert labels == ["openrouter", "nous", "local/custom", "api-key"]
# Codex is deliberately NOT in this chain — see _get_provider_chain
# docstring. ChatGPT-account Codex has a shifting model allow-list;
# guessing a model to fall back on breaks more often than it helps.
assert "openai-codex" not in labels
def test_picks_up_patched_functions(self):
"""Patches on _try_* functions must be visible in the chain."""
@@ -814,7 +826,6 @@ class TestTryPaymentFallback:
with patch("agent.auxiliary_client._try_openrouter", return_value=(None, None)), \
patch("agent.auxiliary_client._try_nous", return_value=(None, None)), \
patch("agent.auxiliary_client._try_custom_endpoint", return_value=(None, None)), \
patch("agent.auxiliary_client._try_codex", return_value=(None, None)), \
patch("agent.auxiliary_client._resolve_api_key_provider", return_value=(None, None)), \
patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"):
client, model, label = _try_payment_fallback("openrouter")
@@ -825,23 +836,26 @@ class TestTryPaymentFallback:
"""'codex' should map to 'openai-codex' in the skip set."""
mock_client = MagicMock()
with patch("agent.auxiliary_client._try_openrouter", return_value=(mock_client, "or-model")), \
patch("agent.auxiliary_client._try_codex", return_value=(None, None)), \
patch("agent.auxiliary_client._read_main_provider", return_value="openai-codex"):
client, model, label = _try_payment_fallback("openai-codex", task="vision")
assert client is mock_client
assert label == "openrouter"
def test_skips_to_codex_when_or_and_nous_fail(self):
mock_codex = MagicMock()
def test_codex_not_in_fallback_chain(self):
"""Codex is deliberately NOT a fallback rung (shifting model allow-list).
When OR/Nous/custom/api-key all fail, payment-fallback returns None —
Codex is never tried with a guessed model.
"""
with patch("agent.auxiliary_client._try_openrouter", return_value=(None, None)), \
patch("agent.auxiliary_client._try_nous", return_value=(None, None)), \
patch("agent.auxiliary_client._try_custom_endpoint", return_value=(None, None)), \
patch("agent.auxiliary_client._try_codex", return_value=(mock_codex, "gpt-5.2-codex")), \
patch("agent.auxiliary_client._resolve_api_key_provider", return_value=(None, None)), \
patch("agent.auxiliary_client._read_main_provider", return_value="openrouter"):
client, model, label = _try_payment_fallback("openrouter")
assert client is mock_codex
assert model == "gpt-5.2-codex"
assert label == "openai-codex"
assert client is None
assert model is None
assert label == ""
class TestCallLlmPaymentFallback:
@@ -1360,14 +1374,14 @@ class TestAuxiliaryAuthRefreshRetry:
with (
patch(
"agent.auxiliary_client.resolve_vision_provider_client",
side_effect=[("openai-codex", failing_client, "gpt-5.2-codex"), ("openai-codex", fresh_client, "gpt-5.2-codex")],
side_effect=[("openai-codex", failing_client, "gpt-5.4"), ("openai-codex", fresh_client, "gpt-5.4")],
),
patch("agent.auxiliary_client._refresh_provider_credentials", return_value=True) as mock_refresh,
):
resp = call_llm(
task="vision",
provider="openai-codex",
model="gpt-5.2-codex",
model="gpt-5.4",
messages=[{"role": "user", "content": "hi"}],
)
@@ -1384,14 +1398,14 @@ class TestAuxiliaryAuthRefreshRetry:
fresh_client.chat.completions.create.return_value = _DummyResponse("fresh-non-vision")
with (
patch("agent.auxiliary_client._resolve_task_provider_model", return_value=("openai-codex", "gpt-5.2-codex", None, None, None)),
patch("agent.auxiliary_client._get_cached_client", side_effect=[(stale_client, "gpt-5.2-codex"), (fresh_client, "gpt-5.2-codex")]),
patch("agent.auxiliary_client._resolve_task_provider_model", return_value=("openai-codex", "gpt-5.4", None, None, None)),
patch("agent.auxiliary_client._get_cached_client", side_effect=[(stale_client, "gpt-5.4"), (fresh_client, "gpt-5.4")]),
patch("agent.auxiliary_client._refresh_provider_credentials", return_value=True) as mock_refresh,
):
resp = call_llm(
task="compression",
provider="openai-codex",
model="gpt-5.2-codex",
model="gpt-5.4",
messages=[{"role": "user", "content": "hi"}],
)
@@ -1439,14 +1453,14 @@ class TestAuxiliaryAuthRefreshRetry:
with (
patch(
"agent.auxiliary_client.resolve_vision_provider_client",
side_effect=[("openai-codex", failing_client, "gpt-5.2-codex"), ("openai-codex", fresh_client, "gpt-5.2-codex")],
side_effect=[("openai-codex", failing_client, "gpt-5.4"), ("openai-codex", fresh_client, "gpt-5.4")],
),
patch("agent.auxiliary_client._refresh_provider_credentials", return_value=True) as mock_refresh,
):
resp = await async_call_llm(
task="vision",
provider="openai-codex",
model="gpt-5.2-codex",
model="gpt-5.4",
messages=[{"role": "user", "content": "hi"}],
)

13
tests/agent/test_codex_cloudflare_headers.py
View File

@@ -10,7 +10,7 @@ of auth correctness.
``_codex_cloudflare_headers`` in ``agent.auxiliary_client`` centralizes the
header set so the primary chat client (``run_agent.AIAgent.__init__`` +
``_apply_client_headers_for_base_url``) and the auxiliary client paths
(``_try_codex`` and the ``raw_codex`` branch of ``resolve_provider_client``)
(``_build_codex_client`` and the ``raw_codex`` branch of ``resolve_provider_client``)
all emit the same headers.
These tests pin:
@@ -207,9 +207,10 @@ class TestPrimaryClientWiring:
# ---------------------------------------------------------------------------
class TestAuxiliaryClientWiring:
def test_try_codex_passes_codex_headers(self, monkeypatch):
"""_try_codex builds the OpenAI client used for compression / vision /
title generation when routed through Codex. Must emit codex headers."""
def test_build_codex_client_passes_codex_headers(self, monkeypatch):
"""_build_codex_client builds the OpenAI client used for compression /
vision / title generation when routed through Codex. Must emit codex
headers."""
from agent import auxiliary_client
token = _make_codex_jwt("acct-aux-try-codex")
@@ -225,7 +226,7 @@ class TestAuxiliaryClientWiring:
)
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
client, model = auxiliary_client._try_codex()
client, model = auxiliary_client._build_codex_client("gpt-5.4")
assert client is not None
headers = mock_openai.call_args.kwargs.get("default_headers") or {}
assert headers.get("originator") == "codex_cli_rs"
@@ -244,7 +245,7 @@ class TestAuxiliaryClientWiring:
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
client, model = auxiliary_client.resolve_provider_client(
"openai-codex", raw_codex=True,
"openai-codex", model="gpt-5.4", raw_codex=True,
)
assert client is not None
headers = mock_openai.call_args.kwargs.get("default_headers") or {}

16
tests/run_agent/test_provider_parity.py
View File

@@ -966,17 +966,25 @@ class TestAuxiliaryClientProviderPriority:
client, model = get_text_auxiliary_client()
assert mock.call_args.kwargs["base_url"] == "http://localhost:1234/v1"
def test_codex_fallback_last_resort(self, monkeypatch):
def test_codex_not_in_auto_fallback(self, monkeypatch):
"""Codex is deliberately NOT part of the auto fallback chain.
ChatGPT-account Codex gates which models it accepts via an
undocumented, shifting allow-list, so falling through to Codex with
a hardcoded default model breaks silently whenever OpenAI rotates
the list. When nothing else is available, ``get_text_auxiliary_client``
now returns (None, None) rather than guessing a Codex model.
"""
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
from agent.auxiliary_client import get_text_auxiliary_client, CodexAuxiliaryClient
from agent.auxiliary_client import get_text_auxiliary_client
with patch("agent.auxiliary_client._read_nous_auth", return_value=None), \
patch("agent.auxiliary_client._read_codex_access_token", return_value="codex-tok"), \
patch("agent.auxiliary_client.OpenAI"):
client, model = get_text_auxiliary_client()
assert model == "gpt-5.2-codex"
assert isinstance(client, CodexAuxiliaryClient)
assert client is None
assert model is None
# ── Provider routing tests ───────────────────────────────────────────────────