Merge origin/main into pr-27248 (resolving run_agent.py = ours)
run_agent.py taken from HEAD (the extracted forwarder structure). The 25 run_agent.py fixes that landed on main during the PR's life need to be ported into the agent/* extracted modules in follow-up commits.
This commit is contained in:
@@ -59,7 +59,7 @@ class TestTruncatedAnthropicResponseNormalization:
|
||||
nr = get_transport("anthropic_messages").normalize_response(response)
|
||||
|
||||
# The continuation block checks these two attributes:
|
||||
# assistant_message.content → appended to truncated_response_prefix
|
||||
# assistant_message.content → appended to truncated_response_parts
|
||||
# assistant_message.tool_calls → guards the text-retry branch
|
||||
assert nr.content is not None
|
||||
assert "partial response" in nr.content
|
||||
|
||||
@@ -193,3 +193,51 @@ def test_background_review_summary_is_attributed_to_self_improvement_loop(monkey
|
||||
assert captured_bg_callback[0].startswith("💾 Self-improvement review:"), (
|
||||
captured_bg_callback[0]
|
||||
)
|
||||
|
||||
|
||||
def test_background_review_fork_skips_external_memory_plugins(monkeypatch):
|
||||
"""The background review fork must NOT touch external memory plugins.
|
||||
|
||||
Without skip_memory=True on the fork constructor, AIAgent.__init__
|
||||
rebuilds its own _memory_manager from config, scoped to the parent's
|
||||
session_id. The review fork's run_conversation() then leaks the
|
||||
harness prompt into the user's real memory namespace via three
|
||||
ingestion sites: on_turn_start (cadence + turn message),
|
||||
prefetch_all (recall query), and sync_all (harness prompt + review
|
||||
output recorded as a (user, assistant) turn pair). The fix is a
|
||||
single kwarg on the fork constructor — this test guards it.
|
||||
"""
|
||||
captured_kwargs: dict = {}
|
||||
|
||||
class FakeReviewAgent:
|
||||
def __init__(self, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
self._session_messages = []
|
||||
|
||||
def run_conversation(self, **kwargs):
|
||||
pass
|
||||
|
||||
def shutdown_memory_provider(self):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(run_agent_module, "AIAgent", FakeReviewAgent)
|
||||
monkeypatch.setattr(run_agent_module.threading, "Thread", ImmediateThread)
|
||||
|
||||
agent = _bare_agent()
|
||||
|
||||
AIAgent._spawn_background_review(
|
||||
agent,
|
||||
messages_snapshot=[{"role": "user", "content": "hello"}],
|
||||
review_memory=True,
|
||||
)
|
||||
|
||||
assert captured_kwargs.get("skip_memory") is True, (
|
||||
"Background review fork must be constructed with skip_memory=True "
|
||||
"so AIAgent.__init__ does not rebuild a _memory_manager wired to "
|
||||
"external plugins (honcho, mem0, supermemory, ...). Without this "
|
||||
"the fork leaks harness prompts into the user's real memory "
|
||||
"namespace via on_turn_start / prefetch_all / sync_all."
|
||||
)
|
||||
|
||||
544
tests/run_agent/test_codex_xai_oauth_recovery.py
Normal file
544
tests/run_agent/test_codex_xai_oauth_recovery.py
Normal file
@@ -0,0 +1,544 @@
|
||||
"""Regression tests for the May 2026 xAI OAuth (SuperGrok / X Premium) bugs.
|
||||
|
||||
Three distinct failure modes the user community hit during rollout:
|
||||
|
||||
1. ``RuntimeError("Expected to have received `response.created` before
|
||||
`error`")`` on multi-turn xAI OAuth conversations. The OpenAI SDK's
|
||||
Responses streaming state machine collapses an upstream ``error`` SSE
|
||||
frame into a generic stream-ordering error. ``_run_codex_stream``
|
||||
now treats this the same way it already treats the missing
|
||||
``response.completed`` postlude — fall back to a non-stream
|
||||
``responses.create(stream=True)`` which surfaces the real provider
|
||||
error. Also closes #8133 (``response.in_progress`` prelude on custom
|
||||
relays) and #14634 (``codex.rate_limits`` prelude on codex-lb).
|
||||
|
||||
2. The HTTP 403 entitlement error xAI returns when an OAuth token lacks
|
||||
SuperGrok / X Premium ("You have either run out of available
|
||||
resources or do not have an active Grok subscription") used to read
|
||||
as a confusing wall of JSON. ``_summarize_api_error`` now appends a
|
||||
one-line hint pointing the user at https://grok.com and ``/model``.
|
||||
|
||||
3. Multi-turn replay of ``codex_reasoning_items`` (with
|
||||
``encrypted_content``) is now suppressed for ``is_xai_responses=True``
|
||||
in ``_chat_messages_to_responses_input``. xAI's OAuth/SuperGrok
|
||||
surface rejects replayed encrypted reasoning items; Grok still
|
||||
reasons natively each turn, so coherence rides on visible message
|
||||
text.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix A: prelude error fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_codex_agent():
|
||||
"""Build a minimal AIAgent wired for codex_responses streaming tests."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://api.x.ai/v1",
|
||||
model="grok-4.3",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
agent.api_mode = "codex_responses"
|
||||
agent.provider = "xai-oauth"
|
||||
agent._interrupt_requested = False
|
||||
return agent
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prelude_event_type",
|
||||
[
|
||||
"error", # xAI OAuth multi-turn
|
||||
"codex.rate_limits", # codex-lb relays (#14634)
|
||||
"response.in_progress", # custom Responses relays (#8133)
|
||||
],
|
||||
)
|
||||
def test_codex_stream_prelude_error_falls_back_to_create_stream(prelude_event_type):
|
||||
"""The SDK's prelude RuntimeError must trigger the non-stream fallback.
|
||||
|
||||
When the first SSE event isn't ``response.created``, openai-python
|
||||
raises RuntimeError before our event loop sees anything. We must
|
||||
detect that, retry once, then fall back to ``create(stream=True)``
|
||||
which surfaces the real provider error or a real response.
|
||||
"""
|
||||
agent = _make_codex_agent()
|
||||
|
||||
prelude_error = RuntimeError(
|
||||
f"Expected to have received `response.created` before `{prelude_event_type}`"
|
||||
)
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.responses.stream.side_effect = prelude_error
|
||||
|
||||
fallback_response = SimpleNamespace(
|
||||
output=[SimpleNamespace(
|
||||
type="message",
|
||||
content=[SimpleNamespace(type="output_text", text="fallback ok")],
|
||||
)],
|
||||
status="completed",
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
agent, "_run_codex_create_stream_fallback", return_value=fallback_response
|
||||
) as mock_fallback:
|
||||
result = agent._run_codex_stream({}, client=mock_client)
|
||||
|
||||
assert result is fallback_response
|
||||
mock_fallback.assert_called_once_with({}, client=mock_client)
|
||||
|
||||
|
||||
def test_codex_stream_prelude_error_retries_once_before_fallback():
|
||||
"""The retry path must fire one extra stream attempt before falling back."""
|
||||
agent = _make_codex_agent()
|
||||
|
||||
call_count = {"n": 0}
|
||||
|
||||
def stream_side_effect(**kwargs):
|
||||
call_count["n"] += 1
|
||||
raise RuntimeError(
|
||||
"Expected to have received `response.created` before `error`"
|
||||
)
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.responses.stream.side_effect = stream_side_effect
|
||||
|
||||
fallback_response = SimpleNamespace(output=[], status="completed")
|
||||
with patch.object(
|
||||
agent, "_run_codex_create_stream_fallback", return_value=fallback_response
|
||||
) as mock_fallback:
|
||||
agent._run_codex_stream({}, client=mock_client)
|
||||
|
||||
# max_stream_retries=1 → one retry + final attempt → 2 stream calls,
|
||||
# THEN the fallback path runs.
|
||||
assert call_count["n"] == 2
|
||||
mock_fallback.assert_called_once()
|
||||
|
||||
|
||||
def test_codex_stream_unrelated_runtimeerror_still_raises():
|
||||
"""RuntimeErrors that aren't prelude/postlude shape must propagate."""
|
||||
agent = _make_codex_agent()
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.responses.stream.side_effect = RuntimeError("something else broke")
|
||||
|
||||
with patch.object(agent, "_run_codex_create_stream_fallback") as mock_fallback:
|
||||
with pytest.raises(RuntimeError, match="something else broke"):
|
||||
agent._run_codex_stream({}, client=mock_client)
|
||||
|
||||
mock_fallback.assert_not_called()
|
||||
|
||||
|
||||
def test_codex_stream_postlude_error_still_falls_back():
|
||||
"""Existing ``response.completed`` fallback must not regress."""
|
||||
agent = _make_codex_agent()
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.responses.stream.side_effect = RuntimeError(
|
||||
"Didn't receive a `response.completed` event."
|
||||
)
|
||||
|
||||
fallback_response = SimpleNamespace(output=[], status="completed")
|
||||
with patch.object(
|
||||
agent, "_run_codex_create_stream_fallback", return_value=fallback_response
|
||||
) as mock_fallback:
|
||||
result = agent._run_codex_stream({}, client=mock_client)
|
||||
|
||||
assert result is fallback_response
|
||||
mock_fallback.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix B: surface xAI's entitlement body verbatim (no editorializing)
|
||||
#
|
||||
# The original PR #26644 appended a hint that led with "X Premium+ does NOT
|
||||
# include xAI API access — only standalone SuperGrok subscribers can use this
|
||||
# provider." xAI announced on 2026-05-16 that X Premium subs now work in
|
||||
# Hermes (https://x.ai/news/grok-hermes), making that hint actively wrong:
|
||||
# a Premium+ user hitting a real entitlement issue (no Grok sub, wrong tier,
|
||||
# exhausted quota) would be misdirected to switch subscriptions when their
|
||||
# Premium sub is in fact valid. We now surface xAI's own body text verbatim
|
||||
# (which already says "Manage subscriptions at https://grok.com/?_s=usage")
|
||||
# and leave the diagnosis to xAI's wording.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_summarize_api_error_surfaces_xai_entitlement_body_verbatim():
|
||||
"""xAI's OAuth 403 body must surface as-is, with no Hermes-side hint."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
error = RuntimeError(
|
||||
"HTTP 403: Error code: 403 - {'code': 'The caller does not have permission "
|
||||
"to execute the specified operation', 'error': 'You have either run out of "
|
||||
"available resources or do not have an active Grok subscription. Manage "
|
||||
"subscriptions at https://grok.com'}"
|
||||
)
|
||||
summary = AIAgent._summarize_api_error(error)
|
||||
# xAI's own body text must reach the user — they need it to diagnose.
|
||||
assert "do not have an active Grok subscription" in summary
|
||||
# No stale claim that X Premium is incompatible with Hermes.
|
||||
assert "X Premium+ does NOT include" not in summary
|
||||
assert "standalone SuperGrok subscribers" not in summary
|
||||
|
||||
|
||||
def test_summarize_api_error_xai_body_message_unwrapped():
|
||||
"""SDK-style error with structured body surfaces the message cleanly."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
class _XaiErr(Exception):
|
||||
status_code = 403
|
||||
body = {
|
||||
"error": {
|
||||
"message": (
|
||||
"You have either run out of available resources or do "
|
||||
"not have an active Grok subscription. Manage at "
|
||||
"https://grok.com"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
summary = AIAgent._summarize_api_error(_XaiErr("403"))
|
||||
assert "HTTP 403" in summary
|
||||
assert "do not have an active Grok subscription" in summary
|
||||
# No editorializing on top of xAI's own wording.
|
||||
assert "X Premium+ does NOT include" not in summary
|
||||
|
||||
|
||||
def test_summarize_api_error_passes_through_unrelated_errors():
|
||||
"""Non-xAI / non-entitlement errors must not be touched."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
error = RuntimeError("HTTP 500: upstream is sad")
|
||||
summary = AIAgent._summarize_api_error(error)
|
||||
assert "SuperGrok" not in summary
|
||||
assert "grok.com" not in summary
|
||||
assert "upstream is sad" in summary
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix C: reasoning replay gating for xai-oauth
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _assistant_msg_with_encrypted_reasoning(text="hi from grok", encrypted="enc_blob"):
|
||||
return {
|
||||
"role": "assistant",
|
||||
"content": text,
|
||||
"codex_reasoning_items": [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_xai_001",
|
||||
"encrypted_content": encrypted,
|
||||
"summary": [],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_codex_reasoning_replay_default_includes_encrypted_content():
|
||||
"""Native Codex backend (default) must still replay encrypted reasoning."""
|
||||
from agent.codex_responses_adapter import _chat_messages_to_responses_input
|
||||
|
||||
msgs = [
|
||||
{"role": "user", "content": "hi"},
|
||||
_assistant_msg_with_encrypted_reasoning(),
|
||||
{"role": "user", "content": "what's your name?"},
|
||||
]
|
||||
|
||||
items = _chat_messages_to_responses_input(msgs)
|
||||
reasoning = [it for it in items if it.get("type") == "reasoning"]
|
||||
assert len(reasoning) == 1
|
||||
assert reasoning[0]["encrypted_content"] == "enc_blob"
|
||||
|
||||
|
||||
def test_codex_reasoning_replay_stripped_for_xai_oauth():
|
||||
"""xAI OAuth surface must NOT receive replayed encrypted reasoning."""
|
||||
from agent.codex_responses_adapter import _chat_messages_to_responses_input
|
||||
|
||||
msgs = [
|
||||
{"role": "user", "content": "hi"},
|
||||
_assistant_msg_with_encrypted_reasoning(),
|
||||
{"role": "user", "content": "what's your name?"},
|
||||
]
|
||||
|
||||
items = _chat_messages_to_responses_input(msgs, is_xai_responses=True)
|
||||
reasoning = [it for it in items if it.get("type") == "reasoning"]
|
||||
assert reasoning == []
|
||||
|
||||
# The assistant's visible text must still survive — coherence across
|
||||
# turns rides on the message text alone.
|
||||
assistant_items = [
|
||||
it for it in items
|
||||
if it.get("role") == "assistant" or it.get("type") == "message"
|
||||
]
|
||||
assert assistant_items, "assistant message must still be present"
|
||||
|
||||
|
||||
def test_codex_transport_xai_request_omits_encrypted_content_include():
|
||||
"""Verify the xAI ``include`` array no longer requests encrypted reasoning."""
|
||||
from agent.transports.codex import ResponsesApiTransport
|
||||
|
||||
transport = ResponsesApiTransport()
|
||||
kwargs = transport.build_kwargs(
|
||||
model="grok-4.3",
|
||||
messages=[
|
||||
{"role": "system", "content": "you are a helpful assistant"},
|
||||
{"role": "user", "content": "hi"},
|
||||
],
|
||||
tools=None,
|
||||
instructions="you are a helpful assistant",
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
is_xai_responses=True,
|
||||
)
|
||||
# Without this gate, xAI would echo back encrypted_content blobs we'd
|
||||
# then store in codex_reasoning_items and replay next turn — which is
|
||||
# exactly the multi-turn failure mode we're closing.
|
||||
assert kwargs["include"] == []
|
||||
|
||||
|
||||
def test_codex_transport_xai_strips_replayed_reasoning_in_input():
|
||||
"""End-to-end: build_kwargs on xai-oauth must strip prior reasoning."""
|
||||
from agent.transports.codex import ResponsesApiTransport
|
||||
|
||||
transport = ResponsesApiTransport()
|
||||
kwargs = transport.build_kwargs(
|
||||
model="grok-4.3",
|
||||
messages=[
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "hi"},
|
||||
_assistant_msg_with_encrypted_reasoning(text="hi from grok"),
|
||||
{"role": "user", "content": "what's your name?"},
|
||||
],
|
||||
tools=None,
|
||||
instructions="sys",
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
is_xai_responses=True,
|
||||
)
|
||||
input_items = kwargs["input"]
|
||||
reasoning_items = [it for it in input_items if it.get("type") == "reasoning"]
|
||||
assert reasoning_items == []
|
||||
|
||||
|
||||
def test_codex_transport_native_codex_still_replays_reasoning_in_input():
|
||||
"""Regression guard: openai-codex must keep the existing replay path."""
|
||||
from agent.transports.codex import ResponsesApiTransport
|
||||
|
||||
transport = ResponsesApiTransport()
|
||||
kwargs = transport.build_kwargs(
|
||||
model="gpt-5-codex",
|
||||
messages=[
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "hi"},
|
||||
_assistant_msg_with_encrypted_reasoning(text="hi from codex"),
|
||||
{"role": "user", "content": "next"},
|
||||
],
|
||||
tools=None,
|
||||
instructions="sys",
|
||||
reasoning_config={"enabled": True, "effort": "medium"},
|
||||
is_xai_responses=False,
|
||||
)
|
||||
input_items = kwargs["input"]
|
||||
reasoning_items = [it for it in input_items if it.get("type") == "reasoning"]
|
||||
assert len(reasoning_items) == 1
|
||||
assert reasoning_items[0]["encrypted_content"] == "enc_blob"
|
||||
# Native Codex still asks for encrypted_content back.
|
||||
assert "reasoning.encrypted_content" in kwargs.get("include", [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix D: entitlement 403 must NOT trigger credential-pool refresh loop
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
# The exact wire text RaidenTyler and Don Piedro captured.
|
||||
"You have either run out of available resources or do not have an "
|
||||
"active Grok subscription. Manage at https://grok.com",
|
||||
# Permission-style variant from the same 403 body.
|
||||
"The caller does not have permission to execute the specified "
|
||||
"operation for grok-4.3",
|
||||
],
|
||||
)
|
||||
def test_is_entitlement_failure_matches_real_xai_bodies(message):
|
||||
from run_agent import AIAgent
|
||||
|
||||
assert AIAgent._is_entitlement_failure(
|
||||
{"message": message, "reason": "permission_denied"},
|
||||
403,
|
||||
)
|
||||
|
||||
|
||||
def test_is_entitlement_failure_false_for_status_other_than_401_403():
|
||||
"""200/429/500 must never be classified as entitlement, even if body matches."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
body = {
|
||||
"message": "do not have an active Grok subscription",
|
||||
}
|
||||
assert not AIAgent._is_entitlement_failure(body, 500)
|
||||
assert not AIAgent._is_entitlement_failure(body, 429)
|
||||
assert not AIAgent._is_entitlement_failure(body, 200)
|
||||
|
||||
|
||||
def test_is_entitlement_failure_false_for_unrelated_auth_errors():
|
||||
"""A real auth failure (expired token, wrong key) must keep refreshing."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
# Generic Anthropic-style auth failure
|
||||
assert not AIAgent._is_entitlement_failure(
|
||||
{"message": "Invalid API key", "reason": "authentication_error"},
|
||||
401,
|
||||
)
|
||||
# OAuth token expired
|
||||
assert not AIAgent._is_entitlement_failure(
|
||||
{"message": "Token has expired", "reason": "unauthorized"},
|
||||
401,
|
||||
)
|
||||
# Empty context
|
||||
assert not AIAgent._is_entitlement_failure({}, 401)
|
||||
assert not AIAgent._is_entitlement_failure(None, 401)
|
||||
|
||||
|
||||
def test_recover_with_credential_pool_skips_refresh_on_entitlement_403():
|
||||
"""The recovery path must NOT call pool.try_refresh_current() on entitlement 403.
|
||||
|
||||
Before the fix, an unsubscribed xAI OAuth account would burn the agent
|
||||
loop indefinitely: refresh → 403 → refresh → 403, infinitely. With
|
||||
the entitlement guard, recovery returns False so the error surfaces
|
||||
normally with the friendly hint from _summarize_api_error.
|
||||
"""
|
||||
from run_agent import AIAgent
|
||||
from agent.error_classifier import FailoverReason
|
||||
|
||||
agent = _make_codex_agent()
|
||||
|
||||
# Wire a fake credential pool that records refresh attempts.
|
||||
refresh_calls = {"n": 0}
|
||||
|
||||
class _FakePool:
|
||||
def try_refresh_current(self):
|
||||
refresh_calls["n"] += 1
|
||||
return MagicMock(id="should_not_be_called")
|
||||
|
||||
def mark_exhausted_and_rotate(self, **_kwargs):
|
||||
return None
|
||||
|
||||
def has_available(self):
|
||||
return False
|
||||
|
||||
agent._credential_pool = _FakePool()
|
||||
|
||||
error_context = {
|
||||
"reason": "The caller does not have permission to execute the specified operation",
|
||||
"message": "You have either run out of available resources or do not have an "
|
||||
"active Grok subscription. Manage at https://grok.com",
|
||||
}
|
||||
|
||||
recovered, _retried_429 = agent._recover_with_credential_pool(
|
||||
status_code=403,
|
||||
has_retried_429=False,
|
||||
classified_reason=FailoverReason.auth,
|
||||
error_context=error_context,
|
||||
)
|
||||
|
||||
assert recovered is False, "Entitlement 403 must surface, not silently recover"
|
||||
assert refresh_calls["n"] == 0, "try_refresh_current must NOT be called on entitlement 403"
|
||||
|
||||
|
||||
def test_recover_with_credential_pool_still_refreshes_genuine_auth_failure():
|
||||
"""Regression guard: legitimate auth errors must still trigger refresh."""
|
||||
from run_agent import AIAgent
|
||||
from agent.error_classifier import FailoverReason
|
||||
|
||||
agent = _make_codex_agent()
|
||||
|
||||
refresh_calls = {"n": 0}
|
||||
|
||||
class _FakePool:
|
||||
def try_refresh_current(self):
|
||||
refresh_calls["n"] += 1
|
||||
# Return a fake refreshed entry — semantically "refresh worked"
|
||||
entry = MagicMock()
|
||||
entry.id = "entry_refreshed"
|
||||
return entry
|
||||
|
||||
def mark_exhausted_and_rotate(self, **_kwargs):
|
||||
return None
|
||||
|
||||
def has_available(self):
|
||||
return False
|
||||
|
||||
agent._credential_pool = _FakePool()
|
||||
# _swap_credential is called by the recovery path — stub it out
|
||||
agent._swap_credential = MagicMock()
|
||||
|
||||
error_context = {
|
||||
"reason": "authentication_error",
|
||||
"message": "Invalid API key",
|
||||
}
|
||||
|
||||
recovered, _retried_429 = agent._recover_with_credential_pool(
|
||||
status_code=401,
|
||||
has_retried_429=False,
|
||||
classified_reason=FailoverReason.auth,
|
||||
error_context=error_context,
|
||||
)
|
||||
|
||||
assert recovered is True, "Genuine auth failure must still recover via refresh"
|
||||
assert refresh_calls["n"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix E: grok-4.3 context length must be 1M, not 256K
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_grok_4_3_context_length_is_1m():
|
||||
"""grok-4.3 ships with 1M context per docs.x.ai/developers/models/grok-4.3.
|
||||
|
||||
Hermes' substring-match fallback used to return 256k (from the
|
||||
"grok-4" catch-all) which under-reported the model's real capacity.
|
||||
"""
|
||||
from agent.model_metadata import DEFAULT_CONTEXT_LENGTHS
|
||||
|
||||
# The entry exists with the expected value.
|
||||
assert DEFAULT_CONTEXT_LENGTHS["grok-4.3"] == 1_000_000
|
||||
|
||||
# And longest-first substring matching resolves grok-4.3 and
|
||||
# grok-4.3-latest to the new value, NOT the grok-4 catch-all.
|
||||
for slug in ("grok-4.3", "grok-4.3-latest"):
|
||||
matched_key = max(
|
||||
(k for k in DEFAULT_CONTEXT_LENGTHS if k in slug.lower()),
|
||||
key=len,
|
||||
)
|
||||
assert matched_key == "grok-4.3", (
|
||||
f"Expected longest-first match to land on grok-4.3 for {slug}, "
|
||||
f"got {matched_key}"
|
||||
)
|
||||
assert DEFAULT_CONTEXT_LENGTHS[matched_key] == 1_000_000
|
||||
|
||||
|
||||
def test_grok_4_still_resolves_to_256k():
|
||||
"""Regression guard: grok-4 (non-.3) must still resolve to 256k."""
|
||||
from agent.model_metadata import DEFAULT_CONTEXT_LENGTHS
|
||||
|
||||
for slug in ("grok-4", "grok-4-0709"):
|
||||
matched_key = max(
|
||||
(k for k in DEFAULT_CONTEXT_LENGTHS if k in slug.lower()),
|
||||
key=len,
|
||||
)
|
||||
# grok-4-0709 contains "grok-4" but not "grok-4.3"; matched key
|
||||
# must be "grok-4" (or a more specific variant family if one is
|
||||
# ever added). The 256k contract must hold.
|
||||
assert DEFAULT_CONTEXT_LENGTHS[matched_key] == 256_000
|
||||
@@ -123,6 +123,26 @@ class TestRestorePrimaryRuntime:
|
||||
assert agent._fallback_activated is False
|
||||
assert agent._restore_primary_runtime() is False
|
||||
|
||||
def test_resets_index_when_fallback_not_activated(self):
|
||||
"""Regression for #20465: failed activation leaves _fallback_index advanced
|
||||
with _fallback_activated=False; the next turn's restore must reset the index."""
|
||||
fbs = [{"provider": "custom", "model": "gpt-oss:20b",
|
||||
"base_url": "http://host.docker.internal:11434/v1", "api_key": "ollama"}]
|
||||
agent = _make_agent(fallback_model=fbs)
|
||||
|
||||
# resolve_provider_client returns None → _try_activate_fallback returns False
|
||||
# but _fallback_index has already been incremented to 1
|
||||
with patch("agent.auxiliary_client.resolve_provider_client", return_value=(None, None)):
|
||||
assert agent._try_activate_fallback() is False
|
||||
|
||||
assert agent._fallback_activated is False
|
||||
assert agent._fallback_index == 1 # advanced past the only entry
|
||||
|
||||
# _restore_primary_runtime must reset the index so the next turn can retry
|
||||
result = agent._restore_primary_runtime()
|
||||
assert result is False # still no-op (primary was never left)
|
||||
assert agent._fallback_index == 0 # chain available again
|
||||
|
||||
def test_restores_model_and_provider(self):
|
||||
agent = _make_agent(
|
||||
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
Mirrors the OpenRouter pattern for the Vercel AI Gateway so that
|
||||
referrerUrl / appName / User-Agent flow into gateway analytics.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from run_agent import AIAgent
|
||||
@@ -65,6 +66,73 @@ def test_routermint_base_url_applies_user_agent_header(mock_openai):
|
||||
assert headers["User-Agent"].startswith("HermesAgent/")
|
||||
|
||||
|
||||
@patch("run_agent.OpenAI")
|
||||
def test_nvidia_cloud_base_url_applies_billing_origin_header(mock_openai):
|
||||
mock_openai.return_value = MagicMock()
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://integrate.api.nvidia.com/v1",
|
||||
model="nvidia/test-model",
|
||||
provider="nvidia",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
|
||||
assert agent._client_kwargs["default_headers"]["X-BILLING-INVOKE-ORIGIN"] == "HermesAgent"
|
||||
|
||||
agent._apply_client_headers_for_base_url("https://integrate.api.nvidia.com/v1")
|
||||
|
||||
headers = agent._client_kwargs["default_headers"]
|
||||
assert headers["X-BILLING-INVOKE-ORIGIN"] == "HermesAgent"
|
||||
|
||||
|
||||
@patch("run_agent.OpenAI")
|
||||
def test_nvidia_local_base_url_does_not_apply_billing_origin_header(mock_openai):
|
||||
mock_openai.return_value = MagicMock()
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://integrate.api.nvidia.com/v1",
|
||||
model="nvidia/test-model",
|
||||
provider="nvidia",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
agent._client_kwargs["default_headers"] = {
|
||||
"X-BILLING-INVOKE-ORIGIN": "HermesAgent",
|
||||
}
|
||||
|
||||
agent._apply_client_headers_for_base_url("http://localhost:8000/v1")
|
||||
|
||||
assert "default_headers" not in agent._client_kwargs
|
||||
|
||||
|
||||
@patch("run_agent.OpenAI")
|
||||
def test_routed_client_preserves_openai_sdk_custom_headers(mock_openai):
|
||||
mock_openai.return_value = MagicMock()
|
||||
routed_client = SimpleNamespace(
|
||||
api_key="test-key",
|
||||
base_url="https://integrate.api.nvidia.com/v1",
|
||||
_custom_headers={"X-BILLING-INVOKE-ORIGIN": "HermesAgent"},
|
||||
)
|
||||
|
||||
with patch("agent.auxiliary_client.resolve_provider_client", return_value=(
|
||||
routed_client,
|
||||
"nvidia/test-model",
|
||||
)):
|
||||
agent = AIAgent(
|
||||
provider="nvidia",
|
||||
model="nvidia/test-model",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
|
||||
headers = agent._client_kwargs["default_headers"]
|
||||
assert headers["X-BILLING-INVOKE-ORIGIN"] == "HermesAgent"
|
||||
|
||||
|
||||
@patch("run_agent.OpenAI")
|
||||
def test_gmi_base_url_picks_up_profile_user_agent(mock_openai):
|
||||
"""GMI declares User-Agent on its ProviderProfile.default_headers.
|
||||
|
||||
@@ -61,6 +61,8 @@ def _make_agent(monkeypatch, provider, api_mode="chat_completions", base_url="ht
|
||||
)
|
||||
if model:
|
||||
kwargs["model"] = model
|
||||
elif provider == "nous":
|
||||
kwargs["model"] = "gpt-5"
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
api_key="test-key",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
|
||||
@@ -2269,6 +2269,60 @@ class TestParallelScopePathNormalization:
|
||||
assert not _should_parallelize_tool_batch([tc1, tc2])
|
||||
|
||||
|
||||
class TestMcpParallelToolBatch:
|
||||
"""Integration test: _should_parallelize_tool_batch respects MCP parallel flag."""
|
||||
|
||||
def test_mcp_tools_default_sequential(self):
|
||||
"""MCP tools without supports_parallel_tool_calls are sequential."""
|
||||
from run_agent import _should_parallelize_tool_batch
|
||||
tc1 = _mock_tool_call(name="mcp_github_list_repos", arguments='{"org":"openai"}', call_id="c1")
|
||||
tc2 = _mock_tool_call(name="mcp_github_search_code", arguments='{"q":"test"}', call_id="c2")
|
||||
assert not _should_parallelize_tool_batch([tc1, tc2])
|
||||
|
||||
def test_mcp_tools_parallel_when_server_opted_in(self):
|
||||
"""MCP tools from a parallel-safe server can run concurrently."""
|
||||
from run_agent import _should_parallelize_tool_batch
|
||||
from tools.mcp_tool import _parallel_safe_servers, _lock
|
||||
with _lock:
|
||||
_parallel_safe_servers.add("github")
|
||||
try:
|
||||
tc1 = _mock_tool_call(name="mcp_github_list_repos", arguments='{"org":"openai"}', call_id="c1")
|
||||
tc2 = _mock_tool_call(name="mcp_github_search_code", arguments='{"q":"test"}', call_id="c2")
|
||||
assert _should_parallelize_tool_batch([tc1, tc2])
|
||||
finally:
|
||||
with _lock:
|
||||
_parallel_safe_servers.discard("github")
|
||||
|
||||
def test_mixed_mcp_and_builtin_parallel(self):
|
||||
"""MCP parallel tools mixed with built-in parallel-safe tools."""
|
||||
from run_agent import _should_parallelize_tool_batch
|
||||
from tools.mcp_tool import _parallel_safe_servers, _lock
|
||||
with _lock:
|
||||
_parallel_safe_servers.add("docs")
|
||||
try:
|
||||
tc1 = _mock_tool_call(name="mcp_docs_search", arguments='{"query":"api"}', call_id="c1")
|
||||
tc2 = _mock_tool_call(name="web_search", arguments='{"query":"test"}', call_id="c2")
|
||||
assert _should_parallelize_tool_batch([tc1, tc2])
|
||||
finally:
|
||||
with _lock:
|
||||
_parallel_safe_servers.discard("docs")
|
||||
|
||||
def test_mixed_parallel_and_serial_mcp_servers(self):
|
||||
"""One parallel MCP server + one non-parallel MCP server = sequential."""
|
||||
from run_agent import _should_parallelize_tool_batch
|
||||
from tools.mcp_tool import _parallel_safe_servers, _lock
|
||||
with _lock:
|
||||
_parallel_safe_servers.add("docs")
|
||||
# "github" is NOT in _parallel_safe_servers
|
||||
try:
|
||||
tc1 = _mock_tool_call(name="mcp_docs_search", arguments='{"query":"api"}', call_id="c1")
|
||||
tc2 = _mock_tool_call(name="mcp_github_list_repos", arguments='{"org":"openai"}', call_id="c2")
|
||||
assert not _should_parallelize_tool_batch([tc1, tc2])
|
||||
finally:
|
||||
with _lock:
|
||||
_parallel_safe_servers.discard("docs")
|
||||
|
||||
|
||||
class TestHandleMaxIterations:
|
||||
def test_returns_summary(self, agent):
|
||||
resp = _mock_response(content="Here is a summary of what I did.")
|
||||
@@ -2524,8 +2578,9 @@ class TestRunConversation:
|
||||
assert [call["api_call_count"] for call in pre_request_calls] == [1, 2]
|
||||
assert [call["api_call_count"] for call in post_request_calls] == [1, 2]
|
||||
assert all(call["session_id"] == agent.session_id for call in pre_request_calls)
|
||||
assert all("message_count" in c and "messages" not in c for c in pre_request_calls)
|
||||
assert all("usage" in c and "response" not in c for c in post_request_calls)
|
||||
assert all("message_count" in c and isinstance(c.get("request_messages"), list) for c in pre_request_calls)
|
||||
assert any(msg.get("role") == "user" and msg.get("content") == "search something" for msg in pre_request_calls[0]["request_messages"])
|
||||
assert all("usage" in c and "response" in c and "assistant_message" in c for c in post_request_calls)
|
||||
|
||||
def test_content_with_tool_calls_stays_silent_for_non_cli_quiet_mode(self, agent):
|
||||
self._setup_agent(agent)
|
||||
@@ -3691,6 +3746,37 @@ class TestCredentialPoolRecovery:
|
||||
assert retry_same is False
|
||||
agent._swap_credential.assert_called_once_with(next_entry)
|
||||
|
||||
def test_recover_with_pool_rotates_usage_limit_429_immediately(self, agent):
|
||||
next_entry = SimpleNamespace(label="secondary")
|
||||
captured = {}
|
||||
|
||||
class _Pool:
|
||||
def current(self):
|
||||
return SimpleNamespace(label="primary")
|
||||
|
||||
def mark_exhausted_and_rotate(self, *, status_code, error_context=None):
|
||||
captured["status_code"] = status_code
|
||||
captured["error_context"] = error_context
|
||||
return next_entry
|
||||
|
||||
agent._credential_pool = _Pool()
|
||||
agent._swap_credential = MagicMock()
|
||||
|
||||
recovered, retry_same = agent._recover_with_credential_pool(
|
||||
status_code=429,
|
||||
has_retried_429=False,
|
||||
error_context={
|
||||
"reason": "usage_limit_reached",
|
||||
"message": "The usage limit has been reached",
|
||||
},
|
||||
)
|
||||
|
||||
assert recovered is True
|
||||
assert retry_same is False
|
||||
assert captured["status_code"] == 429
|
||||
assert captured["error_context"]["reason"] == "usage_limit_reached"
|
||||
agent._swap_credential.assert_called_once_with(next_entry)
|
||||
|
||||
|
||||
def test_recover_with_pool_refreshes_on_401(self, agent):
|
||||
"""401 with successful refresh should swap to refreshed credential."""
|
||||
@@ -3777,6 +3863,22 @@ class TestCredentialPoolRecovery:
|
||||
assert context["message"] == "Weekly credits exhausted."
|
||||
assert context["reset_at"] == "2026-04-12T10:30:00Z"
|
||||
|
||||
def test_extract_api_error_context_uses_type_as_reason(self, agent):
|
||||
error = SimpleNamespace(
|
||||
body={
|
||||
"error": {
|
||||
"type": "usage_limit_reached",
|
||||
"message": "The usage limit has been reached",
|
||||
}
|
||||
},
|
||||
response=SimpleNamespace(headers={}),
|
||||
)
|
||||
|
||||
context = agent._extract_api_error_context(error)
|
||||
|
||||
assert context["reason"] == "usage_limit_reached"
|
||||
assert context["message"] == "The usage limit has been reached"
|
||||
|
||||
def test_recover_with_pool_passes_error_context_on_rotated_429(self, agent):
|
||||
next_entry = SimpleNamespace(label="secondary")
|
||||
captured = {}
|
||||
|
||||
@@ -578,6 +578,197 @@ def test_run_conversation_codex_refreshes_after_401_and_retries(monkeypatch):
|
||||
assert result["final_response"] == "Recovered after refresh"
|
||||
|
||||
|
||||
def _build_xai_oauth_agent(monkeypatch):
|
||||
_patch_agent_bootstrap(monkeypatch)
|
||||
agent = run_agent.AIAgent(
|
||||
model="grok-4.3",
|
||||
provider="xai-oauth",
|
||||
api_mode="codex_responses",
|
||||
base_url="https://api.x.ai/v1",
|
||||
api_key="xai-oauth-token",
|
||||
quiet_mode=True,
|
||||
max_iterations=4,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
agent._cleanup_task_resources = lambda task_id: None
|
||||
agent._persist_session = lambda messages, history=None: None
|
||||
agent._save_trajectory = lambda messages, user_message, completed: None
|
||||
agent._save_session_log = lambda messages: None
|
||||
return agent
|
||||
|
||||
|
||||
def test_build_api_kwargs_xai_oauth_sends_cache_key_via_extra_body(monkeypatch):
|
||||
"""xai-oauth + codex_responses must route prompt caching via the
|
||||
``prompt_cache_key`` body field on /v1/responses (xAI's documented
|
||||
Responses-API cache key — see docs.x.ai prompt-caching/maximizing-
|
||||
cache-hits).
|
||||
|
||||
We pass it through ``extra_body`` rather than as a top-level kwarg so
|
||||
the body field is serialized into JSON regardless of whether the
|
||||
installed openai SDK build still accepts ``prompt_cache_key`` on
|
||||
``Responses.stream()``. Older or trimmed SDK builds drop it from the
|
||||
signature and would otherwise raise ``TypeError`` before the request
|
||||
reaches api.x.ai. The ``x-grok-conv-id`` header is retained as a
|
||||
belt-and-braces fallback for clients/proxies that route on headers."""
|
||||
agent = _build_xai_oauth_agent(monkeypatch)
|
||||
kwargs = agent._build_api_kwargs(
|
||||
[
|
||||
{"role": "system", "content": "You are Hermes."},
|
||||
{"role": "user", "content": "Ping"},
|
||||
]
|
||||
)
|
||||
|
||||
assert kwargs.get("model") == "grok-4.3"
|
||||
# Top-level kwarg must NOT be set — that's the openai SDK
|
||||
# incompatibility this whole indirection exists to dodge.
|
||||
assert "prompt_cache_key" not in kwargs
|
||||
extra_body = kwargs.get("extra_body") or {}
|
||||
assert extra_body.get("prompt_cache_key"), (
|
||||
"xAI prompt-cache routing must travel via extra_body.prompt_cache_key "
|
||||
"for /v1/responses — body field is the documented surface."
|
||||
)
|
||||
headers = kwargs.get("extra_headers") or {}
|
||||
assert "x-grok-conv-id" in headers, (
|
||||
"x-grok-conv-id header kept as belt-and-braces fallback for clients "
|
||||
"that route on headers."
|
||||
)
|
||||
|
||||
|
||||
def test_run_conversation_xai_oauth_refreshes_after_401_and_retries(monkeypatch):
|
||||
"""xai-oauth speaks the Responses API just like codex. When the access
|
||||
token is rejected mid-call (401), the same proactive refresh-and-retry
|
||||
handler that fires for openai-codex must also fire for xai-oauth — the
|
||||
bug it caught: the gating condition checked only ``provider == "openai-codex"``,
|
||||
so xai-oauth 401s leaked straight to non-retryable abort path with no
|
||||
chance to swap in a freshly refreshed access token."""
|
||||
agent = _build_xai_oauth_agent(monkeypatch)
|
||||
calls = {"api": 0, "refresh": 0}
|
||||
|
||||
class _UnauthorizedError(RuntimeError):
|
||||
def __init__(self):
|
||||
super().__init__("Error code: 401 - unauthorized")
|
||||
self.status_code = 401
|
||||
|
||||
def _fake_api_call(api_kwargs):
|
||||
calls["api"] += 1
|
||||
if calls["api"] == 1:
|
||||
raise _UnauthorizedError()
|
||||
return _codex_message_response("Recovered after xAI refresh")
|
||||
|
||||
def _fake_refresh(*, force=True):
|
||||
calls["refresh"] += 1
|
||||
assert force is True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(agent, "_interruptible_api_call", _fake_api_call)
|
||||
monkeypatch.setattr(agent, "_try_refresh_codex_client_credentials", _fake_refresh)
|
||||
|
||||
result = agent.run_conversation("Say OK")
|
||||
|
||||
assert calls["api"] == 2
|
||||
assert calls["refresh"] == 1
|
||||
assert result["completed"] is True
|
||||
assert result["final_response"] == "Recovered after xAI refresh"
|
||||
|
||||
|
||||
def test_try_refresh_codex_client_credentials_handles_xai_oauth(monkeypatch):
|
||||
"""``_try_refresh_codex_client_credentials`` must rebuild the OpenAI
|
||||
client with freshly resolved xAI OAuth credentials when the active
|
||||
provider is xai-oauth. The function name is shared between codex and
|
||||
xai-oauth (both speak codex_responses) — covering both cases prevents
|
||||
silent regressions where the function gets gated to a single provider."""
|
||||
agent = _build_xai_oauth_agent(monkeypatch)
|
||||
closed = {"value": False}
|
||||
rebuilt = {"kwargs": None}
|
||||
|
||||
class _ExistingClient:
|
||||
def close(self):
|
||||
closed["value"] = True
|
||||
|
||||
class _RebuiltClient:
|
||||
pass
|
||||
|
||||
def _fake_openai(**kwargs):
|
||||
rebuilt["kwargs"] = kwargs
|
||||
return _RebuiltClient()
|
||||
|
||||
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
|
||||
# The pre-refresh guard reads the singleton with refresh_if_expiring=False
|
||||
# to verify that the agent's active key still matches; the actual
|
||||
# refresh later passes force_refresh=True. Both calls must succeed.
|
||||
return {
|
||||
"api_key": "fresh-xai-token" if force_refresh else agent.api_key,
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth.resolve_xai_oauth_runtime_credentials",
|
||||
_fake_resolve,
|
||||
)
|
||||
monkeypatch.setattr(run_agent, "OpenAI", _fake_openai)
|
||||
|
||||
agent.client = _ExistingClient()
|
||||
ok = agent._try_refresh_codex_client_credentials(force=True)
|
||||
|
||||
assert ok is True
|
||||
assert closed["value"] is True
|
||||
assert rebuilt["kwargs"]["api_key"] == "fresh-xai-token"
|
||||
assert rebuilt["kwargs"]["base_url"] == "https://api.x.ai/v1"
|
||||
assert isinstance(agent.client, _RebuiltClient)
|
||||
assert agent.api_key == "fresh-xai-token"
|
||||
|
||||
|
||||
def test_try_refresh_codex_client_credentials_skips_xai_oauth_when_singleton_differs(monkeypatch):
|
||||
"""An xai-oauth agent constructed with a non-singleton credential
|
||||
(e.g. a manual pool entry whose tokens belong to a different account
|
||||
than the loopback_pkce singleton, or an explicit ``api_key=`` arg)
|
||||
MUST NOT silently adopt the singleton's tokens on a 401 reactive
|
||||
refresh. Otherwise a 401 mid-conversation would re-route the rest
|
||||
of the conversation onto a different account, with no user feedback.
|
||||
|
||||
The credential pool's reactive recovery is the right channel for
|
||||
pool-managed credentials; this fallback path is for the singleton-
|
||||
only case and must short-circuit when the active key differs."""
|
||||
agent = _build_xai_oauth_agent(monkeypatch)
|
||||
# Agent is using "xai-oauth-token" (per the builder); singleton holds
|
||||
# a *different* account's token. No force_refresh should fire.
|
||||
refresh_calls = {"count": 0}
|
||||
|
||||
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
|
||||
if force_refresh:
|
||||
refresh_calls["count"] += 1
|
||||
return {
|
||||
"api_key": "singleton-account-token",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
}
|
||||
# The pre-refresh guard read — return the singleton's view of the
|
||||
# singleton's token, which is NOT what the agent is currently using.
|
||||
return {
|
||||
"api_key": "singleton-account-token",
|
||||
"base_url": "https://api.x.ai/v1",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth.resolve_xai_oauth_runtime_credentials",
|
||||
_fake_resolve,
|
||||
)
|
||||
|
||||
pre_refresh_key = agent.api_key
|
||||
ok = agent._try_refresh_codex_client_credentials(force=True)
|
||||
|
||||
assert ok is False, (
|
||||
"must not refresh when the active credential isn't the singleton; "
|
||||
"otherwise the conversation silently swaps accounts mid-flight."
|
||||
)
|
||||
assert refresh_calls["count"] == 0, (
|
||||
"force_refresh must not run — that would mutate the singleton's "
|
||||
"tokens on disk and consume its single-use refresh_token for an "
|
||||
"agent that wasn't even using the singleton."
|
||||
)
|
||||
assert agent.api_key == pre_refresh_key
|
||||
|
||||
|
||||
def test_run_conversation_copilot_refreshes_after_401_and_retries(monkeypatch):
|
||||
agent = _build_copilot_agent(monkeypatch)
|
||||
calls = {"api": 0, "refresh": 0}
|
||||
@@ -624,12 +815,18 @@ def test_try_refresh_codex_client_credentials_rebuilds_client(monkeypatch):
|
||||
rebuilt["kwargs"] = kwargs
|
||||
return _RebuiltClient()
|
||||
|
||||
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
|
||||
# Pre-refresh guard reads the singleton (refresh_if_expiring=False).
|
||||
# It must report the agent's current api_key so the equality check
|
||||
# passes; only then does the actual force_refresh run.
|
||||
return {
|
||||
"api_key": "new-codex-token" if force_refresh else agent.api_key,
|
||||
"base_url": "https://chatgpt.com/backend-api/codex",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.auth.resolve_codex_runtime_credentials",
|
||||
lambda force_refresh=True: {
|
||||
"api_key": "new-codex-token",
|
||||
"base_url": "https://chatgpt.com/backend-api/codex",
|
||||
},
|
||||
_fake_resolve,
|
||||
)
|
||||
monkeypatch.setattr(run_agent, "OpenAI", _fake_openai)
|
||||
|
||||
|
||||
@@ -999,6 +999,88 @@ class TestAnthropicStreamCallbacks:
|
||||
|
||||
assert touch_calls.count("receiving stream response") == len(events)
|
||||
|
||||
@patch("run_agent.AIAgent._replace_primary_openai_client")
|
||||
def test_anthropic_stream_parser_valueerror_retries_before_delivery(
|
||||
self, mock_replace, monkeypatch,
|
||||
):
|
||||
"""Malformed Anthropic event-stream frames retry instead of surfacing HTTP None."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://api.minimax.io/anthropic",
|
||||
provider="minimax",
|
||||
model="MiniMax-M2.7",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
agent.api_mode = "anthropic_messages"
|
||||
agent._interrupt_requested = False
|
||||
monkeypatch.setenv("HERMES_STREAM_RETRIES", "1")
|
||||
|
||||
class _BadStream:
|
||||
response = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def __iter__(self):
|
||||
raise ValueError("expected ident at line 1 column 149")
|
||||
|
||||
final_message = SimpleNamespace(content=[], stop_reason="end_turn")
|
||||
good_stream = MagicMock()
|
||||
good_stream.__enter__ = MagicMock(return_value=good_stream)
|
||||
good_stream.__exit__ = MagicMock(return_value=False)
|
||||
good_stream.__iter__ = MagicMock(return_value=iter([]))
|
||||
good_stream.get_final_message.return_value = final_message
|
||||
|
||||
agent._anthropic_client = MagicMock()
|
||||
agent._anthropic_client.messages.stream.side_effect = [
|
||||
_BadStream(),
|
||||
good_stream,
|
||||
]
|
||||
|
||||
response = agent._interruptible_streaming_api_call({})
|
||||
|
||||
assert response is final_message
|
||||
assert agent._anthropic_client.messages.stream.call_count == 2
|
||||
assert mock_replace.call_count == 1
|
||||
|
||||
@patch("run_agent.AIAgent._replace_primary_openai_client")
|
||||
def test_generic_anthropic_valueerror_still_propagates_without_stream_retry(
|
||||
self, mock_replace, monkeypatch,
|
||||
):
|
||||
"""Only known provider stream parser ValueErrors are treated as transient."""
|
||||
from run_agent import AIAgent
|
||||
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://api.minimax.io/anthropic",
|
||||
provider="minimax",
|
||||
model="MiniMax-M2.7",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
agent.api_mode = "anthropic_messages"
|
||||
agent._interrupt_requested = False
|
||||
monkeypatch.setenv("HERMES_STREAM_RETRIES", "1")
|
||||
|
||||
agent._anthropic_client = MagicMock()
|
||||
agent._anthropic_client.messages.stream.side_effect = ValueError(
|
||||
"invalid local request shape"
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="invalid local request shape"):
|
||||
agent._interruptible_streaming_api_call({})
|
||||
|
||||
assert agent._anthropic_client.messages.stream.call_count == 1
|
||||
assert mock_replace.call_count == 0
|
||||
|
||||
|
||||
class TestPartialToolCallWarning:
|
||||
"""Regression: when a stream dies mid tool-call argument generation after
|
||||
@@ -1505,3 +1587,144 @@ class TestCopilotACPStreamingDecision:
|
||||
|
||||
assert _use_streaming is True
|
||||
|
||||
|
||||
class TestCodexFallbackErrorEvent:
|
||||
"""Provider ``error`` SSE frames must surface the real message,
|
||||
not the generic "did not emit a terminal response" RuntimeError.
|
||||
|
||||
xAI emits ``type=error`` as the FIRST frame on the Responses stream
|
||||
when an OAuth account is unsubscribed/exhausted (May 2026
|
||||
SuperGrok rollout). The SDK helper raises
|
||||
``RuntimeError("Expected to have received response.created before
|
||||
error")`` which the caller catches and routes to
|
||||
``_run_codex_create_stream_fallback``. The fallback then opens a
|
||||
NEW stream that emits the same ``type=error`` frame; before this
|
||||
fix it ignored the event entirely and raised a useless RuntimeError.
|
||||
"""
|
||||
|
||||
def _make_agent(self):
|
||||
from run_agent import AIAgent
|
||||
agent = AIAgent(
|
||||
api_key="test-key",
|
||||
base_url="https://api.x.ai/v1",
|
||||
provider="xai-oauth",
|
||||
model="grok-4.3",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
agent.api_mode = "codex_responses"
|
||||
agent._touch_activity = lambda desc: None
|
||||
return agent
|
||||
|
||||
def test_fallback_raises_synthesized_error_with_xai_subscription_message(self):
|
||||
from run_agent import _StreamErrorEvent
|
||||
|
||||
agent = self._make_agent()
|
||||
|
||||
error_event = SimpleNamespace(
|
||||
type="error",
|
||||
message=(
|
||||
"Forbidden: The caller does not have permission to execute the specified operation. "
|
||||
"'You have either run out of available resources or do not have an active Grok subscription.'"
|
||||
),
|
||||
code="permission_denied",
|
||||
param=None,
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
class _FakeStream:
|
||||
def __iter__(self_inner):
|
||||
return iter([error_event])
|
||||
def close(self_inner):
|
||||
return None
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.responses.create.return_value = _FakeStream()
|
||||
|
||||
with pytest.raises(_StreamErrorEvent) as excinfo:
|
||||
agent._run_codex_create_stream_fallback(
|
||||
{"model": "grok-4.3", "instructions": "hi", "input": []},
|
||||
client=mock_client,
|
||||
)
|
||||
|
||||
exc = excinfo.value
|
||||
assert "active Grok subscription" in str(exc)
|
||||
assert exc.code == "permission_denied"
|
||||
assert isinstance(exc.body, dict)
|
||||
assert exc.body["error"]["message"] == error_event.message
|
||||
# _extract_api_error_context reads .body["error"]["message"] — make sure
|
||||
# the entitlement detector will find the subscription phrase there.
|
||||
assert "active Grok subscription" in exc.body["error"]["message"]
|
||||
|
||||
def test_fallback_dict_event_payload_is_also_handled(self):
|
||||
"""Some relays deliver events as plain dicts instead of model
|
||||
objects; the dict branch in the loop must surface them too."""
|
||||
from run_agent import _StreamErrorEvent
|
||||
|
||||
agent = self._make_agent()
|
||||
|
||||
error_event = {
|
||||
"type": "error",
|
||||
"message": "rate_limited",
|
||||
"code": "rate_limit_exceeded",
|
||||
}
|
||||
|
||||
class _FakeStream:
|
||||
def __iter__(self_inner):
|
||||
return iter([error_event])
|
||||
def close(self_inner):
|
||||
return None
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.responses.create.return_value = _FakeStream()
|
||||
|
||||
with pytest.raises(_StreamErrorEvent) as excinfo:
|
||||
agent._run_codex_create_stream_fallback(
|
||||
{"model": "grok-4.3", "instructions": "hi", "input": []},
|
||||
client=mock_client,
|
||||
)
|
||||
|
||||
assert "rate_limited" in str(excinfo.value)
|
||||
assert excinfo.value.code == "rate_limit_exceeded"
|
||||
|
||||
def test_fallback_surfaces_message_useful_to_summarizer(self):
|
||||
"""The synthesized exception must be readable by
|
||||
``_summarize_api_error`` so the user-facing log line shows the
|
||||
real provider message instead of a generic class name."""
|
||||
from run_agent import AIAgent, _StreamErrorEvent
|
||||
|
||||
agent = self._make_agent()
|
||||
exc = _StreamErrorEvent(
|
||||
"You have either run out of available resources or do not have an active Grok subscription.",
|
||||
code="permission_denied",
|
||||
)
|
||||
|
||||
summary = AIAgent._summarize_api_error(exc)
|
||||
assert "active Grok subscription" in summary
|
||||
|
||||
def test_fallback_still_raises_terminal_error_when_no_error_event(self):
|
||||
"""Streams that simply end without any terminal event (and no
|
||||
``error`` frame) must continue to raise the original
|
||||
``"did not emit a terminal response"`` RuntimeError so callers
|
||||
can distinguish "stream truncated mid-flight" from "provider
|
||||
rejected the call"."""
|
||||
agent = self._make_agent()
|
||||
|
||||
# Empty stream — no events at all
|
||||
class _FakeStream:
|
||||
def __iter__(self_inner):
|
||||
return iter([])
|
||||
def close(self_inner):
|
||||
return None
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.responses.create.return_value = _FakeStream()
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
agent._run_codex_create_stream_fallback(
|
||||
{"model": "grok-4.3", "instructions": "hi", "input": []},
|
||||
client=mock_client,
|
||||
)
|
||||
|
||||
assert "did not emit a terminal response" in str(excinfo.value)
|
||||
|
||||
Reference in New Issue
Block a user