Merge origin/main into pr-27248 (resolving run_agent.py = ours)

run_agent.py taken from HEAD (the extracted forwarder structure). The 25
run_agent.py fixes that landed on main during the PR's life need to be
ported into the agent/* extracted modules in follow-up commits.
This commit is contained in:
teknium1
2026-05-16 23:16:52 -07:00
355 changed files with 32716 additions and 4195 deletions

View File

@@ -59,7 +59,7 @@ class TestTruncatedAnthropicResponseNormalization:
nr = get_transport("anthropic_messages").normalize_response(response)
# The continuation block checks these two attributes:
# assistant_message.content → appended to truncated_response_prefix
# assistant_message.content → appended to truncated_response_parts
# assistant_message.tool_calls → guards the text-retry branch
assert nr.content is not None
assert "partial response" in nr.content

View File

@@ -193,3 +193,51 @@ def test_background_review_summary_is_attributed_to_self_improvement_loop(monkey
assert captured_bg_callback[0].startswith("💾 Self-improvement review:"), (
captured_bg_callback[0]
)
def test_background_review_fork_skips_external_memory_plugins(monkeypatch):
"""The background review fork must NOT touch external memory plugins.
Without skip_memory=True on the fork constructor, AIAgent.__init__
rebuilds its own _memory_manager from config, scoped to the parent's
session_id. The review fork's run_conversation() then leaks the
harness prompt into the user's real memory namespace via three
ingestion sites: on_turn_start (cadence + turn message),
prefetch_all (recall query), and sync_all (harness prompt + review
output recorded as a (user, assistant) turn pair). The fix is a
single kwarg on the fork constructor — this test guards it.
"""
captured_kwargs: dict = {}
class FakeReviewAgent:
def __init__(self, **kwargs):
captured_kwargs.update(kwargs)
self._session_messages = []
def run_conversation(self, **kwargs):
pass
def shutdown_memory_provider(self):
pass
def close(self):
pass
monkeypatch.setattr(run_agent_module, "AIAgent", FakeReviewAgent)
monkeypatch.setattr(run_agent_module.threading, "Thread", ImmediateThread)
agent = _bare_agent()
AIAgent._spawn_background_review(
agent,
messages_snapshot=[{"role": "user", "content": "hello"}],
review_memory=True,
)
assert captured_kwargs.get("skip_memory") is True, (
"Background review fork must be constructed with skip_memory=True "
"so AIAgent.__init__ does not rebuild a _memory_manager wired to "
"external plugins (honcho, mem0, supermemory, ...). Without this "
"the fork leaks harness prompts into the user's real memory "
"namespace via on_turn_start / prefetch_all / sync_all."
)

View File

@@ -0,0 +1,544 @@
"""Regression tests for the May 2026 xAI OAuth (SuperGrok / X Premium) bugs.
Three distinct failure modes the user community hit during rollout:
1. ``RuntimeError("Expected to have received `response.created` before
`error`")`` on multi-turn xAI OAuth conversations. The OpenAI SDK's
Responses streaming state machine collapses an upstream ``error`` SSE
frame into a generic stream-ordering error. ``_run_codex_stream``
now treats this the same way it already treats the missing
``response.completed`` postlude — fall back to a non-stream
``responses.create(stream=True)`` which surfaces the real provider
error. Also closes #8133 (``response.in_progress`` prelude on custom
relays) and #14634 (``codex.rate_limits`` prelude on codex-lb).
2. The HTTP 403 entitlement error xAI returns when an OAuth token lacks
SuperGrok / X Premium ("You have either run out of available
resources or do not have an active Grok subscription") used to read
as a confusing wall of JSON. ``_summarize_api_error`` now appends a
one-line hint pointing the user at https://grok.com and ``/model``.
3. Multi-turn replay of ``codex_reasoning_items`` (with
``encrypted_content``) is now suppressed for ``is_xai_responses=True``
in ``_chat_messages_to_responses_input``. xAI's OAuth/SuperGrok
surface rejects replayed encrypted reasoning items; Grok still
reasons natively each turn, so coherence rides on visible message
text.
"""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Fix A: prelude error fallback
# ---------------------------------------------------------------------------
def _make_codex_agent():
"""Build a minimal AIAgent wired for codex_responses streaming tests."""
from run_agent import AIAgent
agent = AIAgent(
api_key="test-key",
base_url="https://api.x.ai/v1",
model="grok-4.3",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
agent.api_mode = "codex_responses"
agent.provider = "xai-oauth"
agent._interrupt_requested = False
return agent
@pytest.mark.parametrize(
"prelude_event_type",
[
"error", # xAI OAuth multi-turn
"codex.rate_limits", # codex-lb relays (#14634)
"response.in_progress", # custom Responses relays (#8133)
],
)
def test_codex_stream_prelude_error_falls_back_to_create_stream(prelude_event_type):
"""The SDK's prelude RuntimeError must trigger the non-stream fallback.
When the first SSE event isn't ``response.created``, openai-python
raises RuntimeError before our event loop sees anything. We must
detect that, retry once, then fall back to ``create(stream=True)``
which surfaces the real provider error or a real response.
"""
agent = _make_codex_agent()
prelude_error = RuntimeError(
f"Expected to have received `response.created` before `{prelude_event_type}`"
)
mock_client = MagicMock()
mock_client.responses.stream.side_effect = prelude_error
fallback_response = SimpleNamespace(
output=[SimpleNamespace(
type="message",
content=[SimpleNamespace(type="output_text", text="fallback ok")],
)],
status="completed",
)
with patch.object(
agent, "_run_codex_create_stream_fallback", return_value=fallback_response
) as mock_fallback:
result = agent._run_codex_stream({}, client=mock_client)
assert result is fallback_response
mock_fallback.assert_called_once_with({}, client=mock_client)
def test_codex_stream_prelude_error_retries_once_before_fallback():
"""The retry path must fire one extra stream attempt before falling back."""
agent = _make_codex_agent()
call_count = {"n": 0}
def stream_side_effect(**kwargs):
call_count["n"] += 1
raise RuntimeError(
"Expected to have received `response.created` before `error`"
)
mock_client = MagicMock()
mock_client.responses.stream.side_effect = stream_side_effect
fallback_response = SimpleNamespace(output=[], status="completed")
with patch.object(
agent, "_run_codex_create_stream_fallback", return_value=fallback_response
) as mock_fallback:
agent._run_codex_stream({}, client=mock_client)
# max_stream_retries=1 → one retry + final attempt → 2 stream calls,
# THEN the fallback path runs.
assert call_count["n"] == 2
mock_fallback.assert_called_once()
def test_codex_stream_unrelated_runtimeerror_still_raises():
"""RuntimeErrors that aren't prelude/postlude shape must propagate."""
agent = _make_codex_agent()
mock_client = MagicMock()
mock_client.responses.stream.side_effect = RuntimeError("something else broke")
with patch.object(agent, "_run_codex_create_stream_fallback") as mock_fallback:
with pytest.raises(RuntimeError, match="something else broke"):
agent._run_codex_stream({}, client=mock_client)
mock_fallback.assert_not_called()
def test_codex_stream_postlude_error_still_falls_back():
"""Existing ``response.completed`` fallback must not regress."""
agent = _make_codex_agent()
mock_client = MagicMock()
mock_client.responses.stream.side_effect = RuntimeError(
"Didn't receive a `response.completed` event."
)
fallback_response = SimpleNamespace(output=[], status="completed")
with patch.object(
agent, "_run_codex_create_stream_fallback", return_value=fallback_response
) as mock_fallback:
result = agent._run_codex_stream({}, client=mock_client)
assert result is fallback_response
mock_fallback.assert_called_once()
# ---------------------------------------------------------------------------
# Fix B: surface xAI's entitlement body verbatim (no editorializing)
#
# The original PR #26644 appended a hint that led with "X Premium+ does NOT
# include xAI API access — only standalone SuperGrok subscribers can use this
# provider." xAI announced on 2026-05-16 that X Premium subs now work in
# Hermes (https://x.ai/news/grok-hermes), making that hint actively wrong:
# a Premium+ user hitting a real entitlement issue (no Grok sub, wrong tier,
# exhausted quota) would be misdirected to switch subscriptions when their
# Premium sub is in fact valid. We now surface xAI's own body text verbatim
# (which already says "Manage subscriptions at https://grok.com/?_s=usage")
# and leave the diagnosis to xAI's wording.
# ---------------------------------------------------------------------------
def test_summarize_api_error_surfaces_xai_entitlement_body_verbatim():
"""xAI's OAuth 403 body must surface as-is, with no Hermes-side hint."""
from run_agent import AIAgent
error = RuntimeError(
"HTTP 403: Error code: 403 - {'code': 'The caller does not have permission "
"to execute the specified operation', 'error': 'You have either run out of "
"available resources or do not have an active Grok subscription. Manage "
"subscriptions at https://grok.com'}"
)
summary = AIAgent._summarize_api_error(error)
# xAI's own body text must reach the user — they need it to diagnose.
assert "do not have an active Grok subscription" in summary
# No stale claim that X Premium is incompatible with Hermes.
assert "X Premium+ does NOT include" not in summary
assert "standalone SuperGrok subscribers" not in summary
def test_summarize_api_error_xai_body_message_unwrapped():
"""SDK-style error with structured body surfaces the message cleanly."""
from run_agent import AIAgent
class _XaiErr(Exception):
status_code = 403
body = {
"error": {
"message": (
"You have either run out of available resources or do "
"not have an active Grok subscription. Manage at "
"https://grok.com"
)
}
}
summary = AIAgent._summarize_api_error(_XaiErr("403"))
assert "HTTP 403" in summary
assert "do not have an active Grok subscription" in summary
# No editorializing on top of xAI's own wording.
assert "X Premium+ does NOT include" not in summary
def test_summarize_api_error_passes_through_unrelated_errors():
"""Non-xAI / non-entitlement errors must not be touched."""
from run_agent import AIAgent
error = RuntimeError("HTTP 500: upstream is sad")
summary = AIAgent._summarize_api_error(error)
assert "SuperGrok" not in summary
assert "grok.com" not in summary
assert "upstream is sad" in summary
# ---------------------------------------------------------------------------
# Fix C: reasoning replay gating for xai-oauth
# ---------------------------------------------------------------------------
def _assistant_msg_with_encrypted_reasoning(text="hi from grok", encrypted="enc_blob"):
return {
"role": "assistant",
"content": text,
"codex_reasoning_items": [
{
"type": "reasoning",
"id": "rs_xai_001",
"encrypted_content": encrypted,
"summary": [],
}
],
}
def test_codex_reasoning_replay_default_includes_encrypted_content():
"""Native Codex backend (default) must still replay encrypted reasoning."""
from agent.codex_responses_adapter import _chat_messages_to_responses_input
msgs = [
{"role": "user", "content": "hi"},
_assistant_msg_with_encrypted_reasoning(),
{"role": "user", "content": "what's your name?"},
]
items = _chat_messages_to_responses_input(msgs)
reasoning = [it for it in items if it.get("type") == "reasoning"]
assert len(reasoning) == 1
assert reasoning[0]["encrypted_content"] == "enc_blob"
def test_codex_reasoning_replay_stripped_for_xai_oauth():
"""xAI OAuth surface must NOT receive replayed encrypted reasoning."""
from agent.codex_responses_adapter import _chat_messages_to_responses_input
msgs = [
{"role": "user", "content": "hi"},
_assistant_msg_with_encrypted_reasoning(),
{"role": "user", "content": "what's your name?"},
]
items = _chat_messages_to_responses_input(msgs, is_xai_responses=True)
reasoning = [it for it in items if it.get("type") == "reasoning"]
assert reasoning == []
# The assistant's visible text must still survive — coherence across
# turns rides on the message text alone.
assistant_items = [
it for it in items
if it.get("role") == "assistant" or it.get("type") == "message"
]
assert assistant_items, "assistant message must still be present"
def test_codex_transport_xai_request_omits_encrypted_content_include():
"""Verify the xAI ``include`` array no longer requests encrypted reasoning."""
from agent.transports.codex import ResponsesApiTransport
transport = ResponsesApiTransport()
kwargs = transport.build_kwargs(
model="grok-4.3",
messages=[
{"role": "system", "content": "you are a helpful assistant"},
{"role": "user", "content": "hi"},
],
tools=None,
instructions="you are a helpful assistant",
reasoning_config={"enabled": True, "effort": "medium"},
is_xai_responses=True,
)
# Without this gate, xAI would echo back encrypted_content blobs we'd
# then store in codex_reasoning_items and replay next turn — which is
# exactly the multi-turn failure mode we're closing.
assert kwargs["include"] == []
def test_codex_transport_xai_strips_replayed_reasoning_in_input():
"""End-to-end: build_kwargs on xai-oauth must strip prior reasoning."""
from agent.transports.codex import ResponsesApiTransport
transport = ResponsesApiTransport()
kwargs = transport.build_kwargs(
model="grok-4.3",
messages=[
{"role": "system", "content": "sys"},
{"role": "user", "content": "hi"},
_assistant_msg_with_encrypted_reasoning(text="hi from grok"),
{"role": "user", "content": "what's your name?"},
],
tools=None,
instructions="sys",
reasoning_config={"enabled": True, "effort": "medium"},
is_xai_responses=True,
)
input_items = kwargs["input"]
reasoning_items = [it for it in input_items if it.get("type") == "reasoning"]
assert reasoning_items == []
def test_codex_transport_native_codex_still_replays_reasoning_in_input():
"""Regression guard: openai-codex must keep the existing replay path."""
from agent.transports.codex import ResponsesApiTransport
transport = ResponsesApiTransport()
kwargs = transport.build_kwargs(
model="gpt-5-codex",
messages=[
{"role": "system", "content": "sys"},
{"role": "user", "content": "hi"},
_assistant_msg_with_encrypted_reasoning(text="hi from codex"),
{"role": "user", "content": "next"},
],
tools=None,
instructions="sys",
reasoning_config={"enabled": True, "effort": "medium"},
is_xai_responses=False,
)
input_items = kwargs["input"]
reasoning_items = [it for it in input_items if it.get("type") == "reasoning"]
assert len(reasoning_items) == 1
assert reasoning_items[0]["encrypted_content"] == "enc_blob"
# Native Codex still asks for encrypted_content back.
assert "reasoning.encrypted_content" in kwargs.get("include", [])
# ---------------------------------------------------------------------------
# Fix D: entitlement 403 must NOT trigger credential-pool refresh loop
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"message",
[
# The exact wire text RaidenTyler and Don Piedro captured.
"You have either run out of available resources or do not have an "
"active Grok subscription. Manage at https://grok.com",
# Permission-style variant from the same 403 body.
"The caller does not have permission to execute the specified "
"operation for grok-4.3",
],
)
def test_is_entitlement_failure_matches_real_xai_bodies(message):
from run_agent import AIAgent
assert AIAgent._is_entitlement_failure(
{"message": message, "reason": "permission_denied"},
403,
)
def test_is_entitlement_failure_false_for_status_other_than_401_403():
"""200/429/500 must never be classified as entitlement, even if body matches."""
from run_agent import AIAgent
body = {
"message": "do not have an active Grok subscription",
}
assert not AIAgent._is_entitlement_failure(body, 500)
assert not AIAgent._is_entitlement_failure(body, 429)
assert not AIAgent._is_entitlement_failure(body, 200)
def test_is_entitlement_failure_false_for_unrelated_auth_errors():
"""A real auth failure (expired token, wrong key) must keep refreshing."""
from run_agent import AIAgent
# Generic Anthropic-style auth failure
assert not AIAgent._is_entitlement_failure(
{"message": "Invalid API key", "reason": "authentication_error"},
401,
)
# OAuth token expired
assert not AIAgent._is_entitlement_failure(
{"message": "Token has expired", "reason": "unauthorized"},
401,
)
# Empty context
assert not AIAgent._is_entitlement_failure({}, 401)
assert not AIAgent._is_entitlement_failure(None, 401)
def test_recover_with_credential_pool_skips_refresh_on_entitlement_403():
"""The recovery path must NOT call pool.try_refresh_current() on entitlement 403.
Before the fix, an unsubscribed xAI OAuth account would burn the agent
loop indefinitely: refresh → 403 → refresh → 403, infinitely. With
the entitlement guard, recovery returns False so the error surfaces
normally with the friendly hint from _summarize_api_error.
"""
from run_agent import AIAgent
from agent.error_classifier import FailoverReason
agent = _make_codex_agent()
# Wire a fake credential pool that records refresh attempts.
refresh_calls = {"n": 0}
class _FakePool:
def try_refresh_current(self):
refresh_calls["n"] += 1
return MagicMock(id="should_not_be_called")
def mark_exhausted_and_rotate(self, **_kwargs):
return None
def has_available(self):
return False
agent._credential_pool = _FakePool()
error_context = {
"reason": "The caller does not have permission to execute the specified operation",
"message": "You have either run out of available resources or do not have an "
"active Grok subscription. Manage at https://grok.com",
}
recovered, _retried_429 = agent._recover_with_credential_pool(
status_code=403,
has_retried_429=False,
classified_reason=FailoverReason.auth,
error_context=error_context,
)
assert recovered is False, "Entitlement 403 must surface, not silently recover"
assert refresh_calls["n"] == 0, "try_refresh_current must NOT be called on entitlement 403"
def test_recover_with_credential_pool_still_refreshes_genuine_auth_failure():
"""Regression guard: legitimate auth errors must still trigger refresh."""
from run_agent import AIAgent
from agent.error_classifier import FailoverReason
agent = _make_codex_agent()
refresh_calls = {"n": 0}
class _FakePool:
def try_refresh_current(self):
refresh_calls["n"] += 1
# Return a fake refreshed entry — semantically "refresh worked"
entry = MagicMock()
entry.id = "entry_refreshed"
return entry
def mark_exhausted_and_rotate(self, **_kwargs):
return None
def has_available(self):
return False
agent._credential_pool = _FakePool()
# _swap_credential is called by the recovery path — stub it out
agent._swap_credential = MagicMock()
error_context = {
"reason": "authentication_error",
"message": "Invalid API key",
}
recovered, _retried_429 = agent._recover_with_credential_pool(
status_code=401,
has_retried_429=False,
classified_reason=FailoverReason.auth,
error_context=error_context,
)
assert recovered is True, "Genuine auth failure must still recover via refresh"
assert refresh_calls["n"] == 1
# ---------------------------------------------------------------------------
# Fix E: grok-4.3 context length must be 1M, not 256K
# ---------------------------------------------------------------------------
def test_grok_4_3_context_length_is_1m():
"""grok-4.3 ships with 1M context per docs.x.ai/developers/models/grok-4.3.
Hermes' substring-match fallback used to return 256k (from the
"grok-4" catch-all) which under-reported the model's real capacity.
"""
from agent.model_metadata import DEFAULT_CONTEXT_LENGTHS
# The entry exists with the expected value.
assert DEFAULT_CONTEXT_LENGTHS["grok-4.3"] == 1_000_000
# And longest-first substring matching resolves grok-4.3 and
# grok-4.3-latest to the new value, NOT the grok-4 catch-all.
for slug in ("grok-4.3", "grok-4.3-latest"):
matched_key = max(
(k for k in DEFAULT_CONTEXT_LENGTHS if k in slug.lower()),
key=len,
)
assert matched_key == "grok-4.3", (
f"Expected longest-first match to land on grok-4.3 for {slug}, "
f"got {matched_key}"
)
assert DEFAULT_CONTEXT_LENGTHS[matched_key] == 1_000_000
def test_grok_4_still_resolves_to_256k():
"""Regression guard: grok-4 (non-.3) must still resolve to 256k."""
from agent.model_metadata import DEFAULT_CONTEXT_LENGTHS
for slug in ("grok-4", "grok-4-0709"):
matched_key = max(
(k for k in DEFAULT_CONTEXT_LENGTHS if k in slug.lower()),
key=len,
)
# grok-4-0709 contains "grok-4" but not "grok-4.3"; matched key
# must be "grok-4" (or a more specific variant family if one is
# ever added). The 256k contract must hold.
assert DEFAULT_CONTEXT_LENGTHS[matched_key] == 256_000

View File

@@ -123,6 +123,26 @@ class TestRestorePrimaryRuntime:
assert agent._fallback_activated is False
assert agent._restore_primary_runtime() is False
def test_resets_index_when_fallback_not_activated(self):
"""Regression for #20465: failed activation leaves _fallback_index advanced
with _fallback_activated=False; the next turn's restore must reset the index."""
fbs = [{"provider": "custom", "model": "gpt-oss:20b",
"base_url": "http://host.docker.internal:11434/v1", "api_key": "ollama"}]
agent = _make_agent(fallback_model=fbs)
# resolve_provider_client returns None → _try_activate_fallback returns False
# but _fallback_index has already been incremented to 1
with patch("agent.auxiliary_client.resolve_provider_client", return_value=(None, None)):
assert agent._try_activate_fallback() is False
assert agent._fallback_activated is False
assert agent._fallback_index == 1 # advanced past the only entry
# _restore_primary_runtime must reset the index so the next turn can retry
result = agent._restore_primary_runtime()
assert result is False # still no-op (primary was never left)
assert agent._fallback_index == 0 # chain available again
def test_restores_model_and_provider(self):
agent = _make_agent(
fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},

View File

@@ -3,6 +3,7 @@
Mirrors the OpenRouter pattern for the Vercel AI Gateway so that
referrerUrl / appName / User-Agent flow into gateway analytics.
"""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from run_agent import AIAgent
@@ -65,6 +66,73 @@ def test_routermint_base_url_applies_user_agent_header(mock_openai):
assert headers["User-Agent"].startswith("HermesAgent/")
@patch("run_agent.OpenAI")
def test_nvidia_cloud_base_url_applies_billing_origin_header(mock_openai):
mock_openai.return_value = MagicMock()
agent = AIAgent(
api_key="test-key",
base_url="https://integrate.api.nvidia.com/v1",
model="nvidia/test-model",
provider="nvidia",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
assert agent._client_kwargs["default_headers"]["X-BILLING-INVOKE-ORIGIN"] == "HermesAgent"
agent._apply_client_headers_for_base_url("https://integrate.api.nvidia.com/v1")
headers = agent._client_kwargs["default_headers"]
assert headers["X-BILLING-INVOKE-ORIGIN"] == "HermesAgent"
@patch("run_agent.OpenAI")
def test_nvidia_local_base_url_does_not_apply_billing_origin_header(mock_openai):
mock_openai.return_value = MagicMock()
agent = AIAgent(
api_key="test-key",
base_url="https://integrate.api.nvidia.com/v1",
model="nvidia/test-model",
provider="nvidia",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
agent._client_kwargs["default_headers"] = {
"X-BILLING-INVOKE-ORIGIN": "HermesAgent",
}
agent._apply_client_headers_for_base_url("http://localhost:8000/v1")
assert "default_headers" not in agent._client_kwargs
@patch("run_agent.OpenAI")
def test_routed_client_preserves_openai_sdk_custom_headers(mock_openai):
mock_openai.return_value = MagicMock()
routed_client = SimpleNamespace(
api_key="test-key",
base_url="https://integrate.api.nvidia.com/v1",
_custom_headers={"X-BILLING-INVOKE-ORIGIN": "HermesAgent"},
)
with patch("agent.auxiliary_client.resolve_provider_client", return_value=(
routed_client,
"nvidia/test-model",
)):
agent = AIAgent(
provider="nvidia",
model="nvidia/test-model",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
headers = agent._client_kwargs["default_headers"]
assert headers["X-BILLING-INVOKE-ORIGIN"] == "HermesAgent"
@patch("run_agent.OpenAI")
def test_gmi_base_url_picks_up_profile_user_agent(mock_openai):
"""GMI declares User-Agent on its ProviderProfile.default_headers.

View File

@@ -61,6 +61,8 @@ def _make_agent(monkeypatch, provider, api_mode="chat_completions", base_url="ht
)
if model:
kwargs["model"] = model
elif provider == "nous":
kwargs["model"] = "gpt-5"
base_url="https://openrouter.ai/api/v1",
api_key="test-key",
base_url="https://openrouter.ai/api/v1",

View File

@@ -2269,6 +2269,60 @@ class TestParallelScopePathNormalization:
assert not _should_parallelize_tool_batch([tc1, tc2])
class TestMcpParallelToolBatch:
"""Integration test: _should_parallelize_tool_batch respects MCP parallel flag."""
def test_mcp_tools_default_sequential(self):
"""MCP tools without supports_parallel_tool_calls are sequential."""
from run_agent import _should_parallelize_tool_batch
tc1 = _mock_tool_call(name="mcp_github_list_repos", arguments='{"org":"openai"}', call_id="c1")
tc2 = _mock_tool_call(name="mcp_github_search_code", arguments='{"q":"test"}', call_id="c2")
assert not _should_parallelize_tool_batch([tc1, tc2])
def test_mcp_tools_parallel_when_server_opted_in(self):
"""MCP tools from a parallel-safe server can run concurrently."""
from run_agent import _should_parallelize_tool_batch
from tools.mcp_tool import _parallel_safe_servers, _lock
with _lock:
_parallel_safe_servers.add("github")
try:
tc1 = _mock_tool_call(name="mcp_github_list_repos", arguments='{"org":"openai"}', call_id="c1")
tc2 = _mock_tool_call(name="mcp_github_search_code", arguments='{"q":"test"}', call_id="c2")
assert _should_parallelize_tool_batch([tc1, tc2])
finally:
with _lock:
_parallel_safe_servers.discard("github")
def test_mixed_mcp_and_builtin_parallel(self):
"""MCP parallel tools mixed with built-in parallel-safe tools."""
from run_agent import _should_parallelize_tool_batch
from tools.mcp_tool import _parallel_safe_servers, _lock
with _lock:
_parallel_safe_servers.add("docs")
try:
tc1 = _mock_tool_call(name="mcp_docs_search", arguments='{"query":"api"}', call_id="c1")
tc2 = _mock_tool_call(name="web_search", arguments='{"query":"test"}', call_id="c2")
assert _should_parallelize_tool_batch([tc1, tc2])
finally:
with _lock:
_parallel_safe_servers.discard("docs")
def test_mixed_parallel_and_serial_mcp_servers(self):
"""One parallel MCP server + one non-parallel MCP server = sequential."""
from run_agent import _should_parallelize_tool_batch
from tools.mcp_tool import _parallel_safe_servers, _lock
with _lock:
_parallel_safe_servers.add("docs")
# "github" is NOT in _parallel_safe_servers
try:
tc1 = _mock_tool_call(name="mcp_docs_search", arguments='{"query":"api"}', call_id="c1")
tc2 = _mock_tool_call(name="mcp_github_list_repos", arguments='{"org":"openai"}', call_id="c2")
assert not _should_parallelize_tool_batch([tc1, tc2])
finally:
with _lock:
_parallel_safe_servers.discard("docs")
class TestHandleMaxIterations:
def test_returns_summary(self, agent):
resp = _mock_response(content="Here is a summary of what I did.")
@@ -2524,8 +2578,9 @@ class TestRunConversation:
assert [call["api_call_count"] for call in pre_request_calls] == [1, 2]
assert [call["api_call_count"] for call in post_request_calls] == [1, 2]
assert all(call["session_id"] == agent.session_id for call in pre_request_calls)
assert all("message_count" in c and "messages" not in c for c in pre_request_calls)
assert all("usage" in c and "response" not in c for c in post_request_calls)
assert all("message_count" in c and isinstance(c.get("request_messages"), list) for c in pre_request_calls)
assert any(msg.get("role") == "user" and msg.get("content") == "search something" for msg in pre_request_calls[0]["request_messages"])
assert all("usage" in c and "response" in c and "assistant_message" in c for c in post_request_calls)
def test_content_with_tool_calls_stays_silent_for_non_cli_quiet_mode(self, agent):
self._setup_agent(agent)
@@ -3691,6 +3746,37 @@ class TestCredentialPoolRecovery:
assert retry_same is False
agent._swap_credential.assert_called_once_with(next_entry)
def test_recover_with_pool_rotates_usage_limit_429_immediately(self, agent):
next_entry = SimpleNamespace(label="secondary")
captured = {}
class _Pool:
def current(self):
return SimpleNamespace(label="primary")
def mark_exhausted_and_rotate(self, *, status_code, error_context=None):
captured["status_code"] = status_code
captured["error_context"] = error_context
return next_entry
agent._credential_pool = _Pool()
agent._swap_credential = MagicMock()
recovered, retry_same = agent._recover_with_credential_pool(
status_code=429,
has_retried_429=False,
error_context={
"reason": "usage_limit_reached",
"message": "The usage limit has been reached",
},
)
assert recovered is True
assert retry_same is False
assert captured["status_code"] == 429
assert captured["error_context"]["reason"] == "usage_limit_reached"
agent._swap_credential.assert_called_once_with(next_entry)
def test_recover_with_pool_refreshes_on_401(self, agent):
"""401 with successful refresh should swap to refreshed credential."""
@@ -3777,6 +3863,22 @@ class TestCredentialPoolRecovery:
assert context["message"] == "Weekly credits exhausted."
assert context["reset_at"] == "2026-04-12T10:30:00Z"
def test_extract_api_error_context_uses_type_as_reason(self, agent):
error = SimpleNamespace(
body={
"error": {
"type": "usage_limit_reached",
"message": "The usage limit has been reached",
}
},
response=SimpleNamespace(headers={}),
)
context = agent._extract_api_error_context(error)
assert context["reason"] == "usage_limit_reached"
assert context["message"] == "The usage limit has been reached"
def test_recover_with_pool_passes_error_context_on_rotated_429(self, agent):
next_entry = SimpleNamespace(label="secondary")
captured = {}

View File

@@ -578,6 +578,197 @@ def test_run_conversation_codex_refreshes_after_401_and_retries(monkeypatch):
assert result["final_response"] == "Recovered after refresh"
def _build_xai_oauth_agent(monkeypatch):
_patch_agent_bootstrap(monkeypatch)
agent = run_agent.AIAgent(
model="grok-4.3",
provider="xai-oauth",
api_mode="codex_responses",
base_url="https://api.x.ai/v1",
api_key="xai-oauth-token",
quiet_mode=True,
max_iterations=4,
skip_context_files=True,
skip_memory=True,
)
agent._cleanup_task_resources = lambda task_id: None
agent._persist_session = lambda messages, history=None: None
agent._save_trajectory = lambda messages, user_message, completed: None
agent._save_session_log = lambda messages: None
return agent
def test_build_api_kwargs_xai_oauth_sends_cache_key_via_extra_body(monkeypatch):
"""xai-oauth + codex_responses must route prompt caching via the
``prompt_cache_key`` body field on /v1/responses (xAI's documented
Responses-API cache key — see docs.x.ai prompt-caching/maximizing-
cache-hits).
We pass it through ``extra_body`` rather than as a top-level kwarg so
the body field is serialized into JSON regardless of whether the
installed openai SDK build still accepts ``prompt_cache_key`` on
``Responses.stream()``. Older or trimmed SDK builds drop it from the
signature and would otherwise raise ``TypeError`` before the request
reaches api.x.ai. The ``x-grok-conv-id`` header is retained as a
belt-and-braces fallback for clients/proxies that route on headers."""
agent = _build_xai_oauth_agent(monkeypatch)
kwargs = agent._build_api_kwargs(
[
{"role": "system", "content": "You are Hermes."},
{"role": "user", "content": "Ping"},
]
)
assert kwargs.get("model") == "grok-4.3"
# Top-level kwarg must NOT be set — that's the openai SDK
# incompatibility this whole indirection exists to dodge.
assert "prompt_cache_key" not in kwargs
extra_body = kwargs.get("extra_body") or {}
assert extra_body.get("prompt_cache_key"), (
"xAI prompt-cache routing must travel via extra_body.prompt_cache_key "
"for /v1/responses — body field is the documented surface."
)
headers = kwargs.get("extra_headers") or {}
assert "x-grok-conv-id" in headers, (
"x-grok-conv-id header kept as belt-and-braces fallback for clients "
"that route on headers."
)
def test_run_conversation_xai_oauth_refreshes_after_401_and_retries(monkeypatch):
"""xai-oauth speaks the Responses API just like codex. When the access
token is rejected mid-call (401), the same proactive refresh-and-retry
handler that fires for openai-codex must also fire for xai-oauth — the
bug it caught: the gating condition checked only ``provider == "openai-codex"``,
so xai-oauth 401s leaked straight to non-retryable abort path with no
chance to swap in a freshly refreshed access token."""
agent = _build_xai_oauth_agent(monkeypatch)
calls = {"api": 0, "refresh": 0}
class _UnauthorizedError(RuntimeError):
def __init__(self):
super().__init__("Error code: 401 - unauthorized")
self.status_code = 401
def _fake_api_call(api_kwargs):
calls["api"] += 1
if calls["api"] == 1:
raise _UnauthorizedError()
return _codex_message_response("Recovered after xAI refresh")
def _fake_refresh(*, force=True):
calls["refresh"] += 1
assert force is True
return True
monkeypatch.setattr(agent, "_interruptible_api_call", _fake_api_call)
monkeypatch.setattr(agent, "_try_refresh_codex_client_credentials", _fake_refresh)
result = agent.run_conversation("Say OK")
assert calls["api"] == 2
assert calls["refresh"] == 1
assert result["completed"] is True
assert result["final_response"] == "Recovered after xAI refresh"
def test_try_refresh_codex_client_credentials_handles_xai_oauth(monkeypatch):
"""``_try_refresh_codex_client_credentials`` must rebuild the OpenAI
client with freshly resolved xAI OAuth credentials when the active
provider is xai-oauth. The function name is shared between codex and
xai-oauth (both speak codex_responses) — covering both cases prevents
silent regressions where the function gets gated to a single provider."""
agent = _build_xai_oauth_agent(monkeypatch)
closed = {"value": False}
rebuilt = {"kwargs": None}
class _ExistingClient:
def close(self):
closed["value"] = True
class _RebuiltClient:
pass
def _fake_openai(**kwargs):
rebuilt["kwargs"] = kwargs
return _RebuiltClient()
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
# The pre-refresh guard reads the singleton with refresh_if_expiring=False
# to verify that the agent's active key still matches; the actual
# refresh later passes force_refresh=True. Both calls must succeed.
return {
"api_key": "fresh-xai-token" if force_refresh else agent.api_key,
"base_url": "https://api.x.ai/v1",
}
monkeypatch.setattr(
"hermes_cli.auth.resolve_xai_oauth_runtime_credentials",
_fake_resolve,
)
monkeypatch.setattr(run_agent, "OpenAI", _fake_openai)
agent.client = _ExistingClient()
ok = agent._try_refresh_codex_client_credentials(force=True)
assert ok is True
assert closed["value"] is True
assert rebuilt["kwargs"]["api_key"] == "fresh-xai-token"
assert rebuilt["kwargs"]["base_url"] == "https://api.x.ai/v1"
assert isinstance(agent.client, _RebuiltClient)
assert agent.api_key == "fresh-xai-token"
def test_try_refresh_codex_client_credentials_skips_xai_oauth_when_singleton_differs(monkeypatch):
"""An xai-oauth agent constructed with a non-singleton credential
(e.g. a manual pool entry whose tokens belong to a different account
than the loopback_pkce singleton, or an explicit ``api_key=`` arg)
MUST NOT silently adopt the singleton's tokens on a 401 reactive
refresh. Otherwise a 401 mid-conversation would re-route the rest
of the conversation onto a different account, with no user feedback.
The credential pool's reactive recovery is the right channel for
pool-managed credentials; this fallback path is for the singleton-
only case and must short-circuit when the active key differs."""
agent = _build_xai_oauth_agent(monkeypatch)
# Agent is using "xai-oauth-token" (per the builder); singleton holds
# a *different* account's token. No force_refresh should fire.
refresh_calls = {"count": 0}
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
if force_refresh:
refresh_calls["count"] += 1
return {
"api_key": "singleton-account-token",
"base_url": "https://api.x.ai/v1",
}
# The pre-refresh guard read — return the singleton's view of the
# singleton's token, which is NOT what the agent is currently using.
return {
"api_key": "singleton-account-token",
"base_url": "https://api.x.ai/v1",
}
monkeypatch.setattr(
"hermes_cli.auth.resolve_xai_oauth_runtime_credentials",
_fake_resolve,
)
pre_refresh_key = agent.api_key
ok = agent._try_refresh_codex_client_credentials(force=True)
assert ok is False, (
"must not refresh when the active credential isn't the singleton; "
"otherwise the conversation silently swaps accounts mid-flight."
)
assert refresh_calls["count"] == 0, (
"force_refresh must not run — that would mutate the singleton's "
"tokens on disk and consume its single-use refresh_token for an "
"agent that wasn't even using the singleton."
)
assert agent.api_key == pre_refresh_key
def test_run_conversation_copilot_refreshes_after_401_and_retries(monkeypatch):
agent = _build_copilot_agent(monkeypatch)
calls = {"api": 0, "refresh": 0}
@@ -624,12 +815,18 @@ def test_try_refresh_codex_client_credentials_rebuilds_client(monkeypatch):
rebuilt["kwargs"] = kwargs
return _RebuiltClient()
def _fake_resolve(force_refresh=False, refresh_if_expiring=True, **_):
# Pre-refresh guard reads the singleton (refresh_if_expiring=False).
# It must report the agent's current api_key so the equality check
# passes; only then does the actual force_refresh run.
return {
"api_key": "new-codex-token" if force_refresh else agent.api_key,
"base_url": "https://chatgpt.com/backend-api/codex",
}
monkeypatch.setattr(
"hermes_cli.auth.resolve_codex_runtime_credentials",
lambda force_refresh=True: {
"api_key": "new-codex-token",
"base_url": "https://chatgpt.com/backend-api/codex",
},
_fake_resolve,
)
monkeypatch.setattr(run_agent, "OpenAI", _fake_openai)

View File

@@ -999,6 +999,88 @@ class TestAnthropicStreamCallbacks:
assert touch_calls.count("receiving stream response") == len(events)
@patch("run_agent.AIAgent._replace_primary_openai_client")
def test_anthropic_stream_parser_valueerror_retries_before_delivery(
self, mock_replace, monkeypatch,
):
"""Malformed Anthropic event-stream frames retry instead of surfacing HTTP None."""
from run_agent import AIAgent
agent = AIAgent(
api_key="test-key",
base_url="https://api.minimax.io/anthropic",
provider="minimax",
model="MiniMax-M2.7",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
agent.api_mode = "anthropic_messages"
agent._interrupt_requested = False
monkeypatch.setenv("HERMES_STREAM_RETRIES", "1")
class _BadStream:
response = None
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def __iter__(self):
raise ValueError("expected ident at line 1 column 149")
final_message = SimpleNamespace(content=[], stop_reason="end_turn")
good_stream = MagicMock()
good_stream.__enter__ = MagicMock(return_value=good_stream)
good_stream.__exit__ = MagicMock(return_value=False)
good_stream.__iter__ = MagicMock(return_value=iter([]))
good_stream.get_final_message.return_value = final_message
agent._anthropic_client = MagicMock()
agent._anthropic_client.messages.stream.side_effect = [
_BadStream(),
good_stream,
]
response = agent._interruptible_streaming_api_call({})
assert response is final_message
assert agent._anthropic_client.messages.stream.call_count == 2
assert mock_replace.call_count == 1
@patch("run_agent.AIAgent._replace_primary_openai_client")
def test_generic_anthropic_valueerror_still_propagates_without_stream_retry(
self, mock_replace, monkeypatch,
):
"""Only known provider stream parser ValueErrors are treated as transient."""
from run_agent import AIAgent
agent = AIAgent(
api_key="test-key",
base_url="https://api.minimax.io/anthropic",
provider="minimax",
model="MiniMax-M2.7",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
agent.api_mode = "anthropic_messages"
agent._interrupt_requested = False
monkeypatch.setenv("HERMES_STREAM_RETRIES", "1")
agent._anthropic_client = MagicMock()
agent._anthropic_client.messages.stream.side_effect = ValueError(
"invalid local request shape"
)
with pytest.raises(ValueError, match="invalid local request shape"):
agent._interruptible_streaming_api_call({})
assert agent._anthropic_client.messages.stream.call_count == 1
assert mock_replace.call_count == 0
class TestPartialToolCallWarning:
"""Regression: when a stream dies mid tool-call argument generation after
@@ -1505,3 +1587,144 @@ class TestCopilotACPStreamingDecision:
assert _use_streaming is True
class TestCodexFallbackErrorEvent:
"""Provider ``error`` SSE frames must surface the real message,
not the generic "did not emit a terminal response" RuntimeError.
xAI emits ``type=error`` as the FIRST frame on the Responses stream
when an OAuth account is unsubscribed/exhausted (May 2026
SuperGrok rollout). The SDK helper raises
``RuntimeError("Expected to have received response.created before
error")`` which the caller catches and routes to
``_run_codex_create_stream_fallback``. The fallback then opens a
NEW stream that emits the same ``type=error`` frame; before this
fix it ignored the event entirely and raised a useless RuntimeError.
"""
def _make_agent(self):
from run_agent import AIAgent
agent = AIAgent(
api_key="test-key",
base_url="https://api.x.ai/v1",
provider="xai-oauth",
model="grok-4.3",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
agent.api_mode = "codex_responses"
agent._touch_activity = lambda desc: None
return agent
def test_fallback_raises_synthesized_error_with_xai_subscription_message(self):
from run_agent import _StreamErrorEvent
agent = self._make_agent()
error_event = SimpleNamespace(
type="error",
message=(
"Forbidden: The caller does not have permission to execute the specified operation. "
"'You have either run out of available resources or do not have an active Grok subscription.'"
),
code="permission_denied",
param=None,
sequence_number=1,
)
class _FakeStream:
def __iter__(self_inner):
return iter([error_event])
def close(self_inner):
return None
mock_client = MagicMock()
mock_client.responses.create.return_value = _FakeStream()
with pytest.raises(_StreamErrorEvent) as excinfo:
agent._run_codex_create_stream_fallback(
{"model": "grok-4.3", "instructions": "hi", "input": []},
client=mock_client,
)
exc = excinfo.value
assert "active Grok subscription" in str(exc)
assert exc.code == "permission_denied"
assert isinstance(exc.body, dict)
assert exc.body["error"]["message"] == error_event.message
# _extract_api_error_context reads .body["error"]["message"] — make sure
# the entitlement detector will find the subscription phrase there.
assert "active Grok subscription" in exc.body["error"]["message"]
def test_fallback_dict_event_payload_is_also_handled(self):
"""Some relays deliver events as plain dicts instead of model
objects; the dict branch in the loop must surface them too."""
from run_agent import _StreamErrorEvent
agent = self._make_agent()
error_event = {
"type": "error",
"message": "rate_limited",
"code": "rate_limit_exceeded",
}
class _FakeStream:
def __iter__(self_inner):
return iter([error_event])
def close(self_inner):
return None
mock_client = MagicMock()
mock_client.responses.create.return_value = _FakeStream()
with pytest.raises(_StreamErrorEvent) as excinfo:
agent._run_codex_create_stream_fallback(
{"model": "grok-4.3", "instructions": "hi", "input": []},
client=mock_client,
)
assert "rate_limited" in str(excinfo.value)
assert excinfo.value.code == "rate_limit_exceeded"
def test_fallback_surfaces_message_useful_to_summarizer(self):
"""The synthesized exception must be readable by
``_summarize_api_error`` so the user-facing log line shows the
real provider message instead of a generic class name."""
from run_agent import AIAgent, _StreamErrorEvent
agent = self._make_agent()
exc = _StreamErrorEvent(
"You have either run out of available resources or do not have an active Grok subscription.",
code="permission_denied",
)
summary = AIAgent._summarize_api_error(exc)
assert "active Grok subscription" in summary
def test_fallback_still_raises_terminal_error_when_no_error_event(self):
"""Streams that simply end without any terminal event (and no
``error`` frame) must continue to raise the original
``"did not emit a terminal response"`` RuntimeError so callers
can distinguish "stream truncated mid-flight" from "provider
rejected the call"."""
agent = self._make_agent()
# Empty stream — no events at all
class _FakeStream:
def __iter__(self_inner):
return iter([])
def close(self_inner):
return None
mock_client = MagicMock()
mock_client.responses.create.return_value = _FakeStream()
with pytest.raises(RuntimeError) as excinfo:
agent._run_codex_create_stream_fallback(
{"model": "grok-4.3", "instructions": "hi", "input": []},
client=mock_client,
)
assert "did not emit a terminal response" in str(excinfo.value)