test(tui): fix stale mocks + xdist flakes in TUI test suite
All 61 TUI-related tests green across 3 consecutive xdist runs.
tests/tui_gateway/test_protocol.py:
- rename `get_messages` → `get_messages_as_conversation` on mock DB (method
was renamed in the real backend, test was still stubbing the old name)
- update tool-message shape expectation: `{role, name, context}` matches
current `_history_to_messages` output, not the legacy `{role, text}`
tests/hermes_cli/test_tui_resume_flow.py:
- `cmd_chat` grew a first-run provider-gate that bailed to "Run: hermes
setup" before `_launch_tui` was ever reached; 3 tests stubbed
`_resolve_last_session` + `_launch_tui` but not the gate
- factored a `main_mod` fixture that stubs `_has_any_provider_configured`,
reused by all three tests
tests/test_tui_gateway_server.py:
- `test_config_set_personality_resets_history_and_returns_info` was flaky
under xdist because the real `_write_config_key` touches
`~/.hermes/config.yaml`, racing with any other worker that writes
config. Stub it in the test.
This commit is contained in:
1032
tests/agent/test_gemini_cloudcode.py
Normal file
1032
tests/agent/test_gemini_cloudcode.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -141,3 +141,116 @@ class TestCliApprovalUi:
|
||||
assert "archive-" in rendered
|
||||
assert "keyring.gpg" in rendered
|
||||
assert "status=progress" in rendered
|
||||
|
||||
def test_approval_display_preserves_command_and_choices_with_long_description(self):
|
||||
"""Regression: long tirith descriptions used to push approve/deny off-screen.
|
||||
|
||||
The panel must always render the command and every choice, even when
|
||||
the description would otherwise wrap into 10+ lines. The description
|
||||
gets truncated with a marker instead.
|
||||
"""
|
||||
cli = _make_cli_stub()
|
||||
long_desc = (
|
||||
"Security scan — [CRITICAL] Destructive shell command with wildcard expansion: "
|
||||
"The command performs a recursive deletion of log files which may contain "
|
||||
"audit information relevant to active incident investigations, running services "
|
||||
"that rely on log files for state, rotated archives, and other system artifacts. "
|
||||
"Review whether this is intended before approving. Consider whether a targeted "
|
||||
"deletion with more specific filters would better match the intent."
|
||||
)
|
||||
cli._approval_state = {
|
||||
"command": "rm -rf /var/log/apache2/*.log",
|
||||
"description": long_desc,
|
||||
"choices": ["once", "session", "always", "deny"],
|
||||
"selected": 0,
|
||||
"response_queue": queue.Queue(),
|
||||
}
|
||||
|
||||
# Simulate a compact terminal where the old unbounded panel would overflow.
|
||||
import shutil as _shutil
|
||||
|
||||
with patch("cli.shutil.get_terminal_size",
|
||||
return_value=_shutil.os.terminal_size((100, 20))):
|
||||
fragments = cli._get_approval_display_fragments()
|
||||
|
||||
rendered = "".join(text for _style, text in fragments)
|
||||
|
||||
# Command must be fully visible (rm -rf /var/log/apache2/*.log is short).
|
||||
assert "rm -rf /var/log/apache2/*.log" in rendered
|
||||
|
||||
# Every choice must render — this is the core bug: approve/deny were
|
||||
# getting clipped off the bottom of the panel.
|
||||
assert "Allow once" in rendered
|
||||
assert "Allow for this session" in rendered
|
||||
assert "Add to permanent allowlist" in rendered
|
||||
assert "Deny" in rendered
|
||||
|
||||
# The bottom border must render (i.e. the panel is self-contained).
|
||||
assert rendered.rstrip().endswith("╯")
|
||||
|
||||
# The description gets truncated — marker should appear.
|
||||
assert "(description truncated)" in rendered
|
||||
|
||||
def test_approval_display_skips_description_on_very_short_terminal(self):
|
||||
"""On a 12-row terminal, only the command and choices have room.
|
||||
|
||||
The description is dropped entirely rather than partially shown, so the
|
||||
choices never get clipped.
|
||||
"""
|
||||
cli = _make_cli_stub()
|
||||
cli._approval_state = {
|
||||
"command": "rm -rf /var/log/apache2/*.log",
|
||||
"description": "recursive delete",
|
||||
"choices": ["once", "session", "always", "deny"],
|
||||
"selected": 0,
|
||||
"response_queue": queue.Queue(),
|
||||
}
|
||||
|
||||
import shutil as _shutil
|
||||
|
||||
with patch("cli.shutil.get_terminal_size",
|
||||
return_value=_shutil.os.terminal_size((100, 12))):
|
||||
fragments = cli._get_approval_display_fragments()
|
||||
|
||||
rendered = "".join(text for _style, text in fragments)
|
||||
|
||||
# Command visible.
|
||||
assert "rm -rf /var/log/apache2/*.log" in rendered
|
||||
# All four choices visible.
|
||||
for label in ("Allow once", "Allow for this session",
|
||||
"Add to permanent allowlist", "Deny"):
|
||||
assert label in rendered, f"choice {label!r} missing"
|
||||
|
||||
def test_approval_display_truncates_giant_command_in_view_mode(self):
|
||||
"""If the user hits /view on a massive command, choices still render.
|
||||
|
||||
The command gets truncated with a marker; the description gets dropped
|
||||
if there's no remaining row budget.
|
||||
"""
|
||||
cli = _make_cli_stub()
|
||||
# 50 lines of command when wrapped at ~64 chars.
|
||||
giant_cmd = "bash -c 'echo " + ("x" * 3000) + "'"
|
||||
cli._approval_state = {
|
||||
"command": giant_cmd,
|
||||
"description": "shell command via -c/-lc flag",
|
||||
"choices": ["once", "session", "always", "deny"],
|
||||
"selected": 0,
|
||||
"show_full": True,
|
||||
"response_queue": queue.Queue(),
|
||||
}
|
||||
|
||||
import shutil as _shutil
|
||||
|
||||
with patch("cli.shutil.get_terminal_size",
|
||||
return_value=_shutil.os.terminal_size((100, 24))):
|
||||
fragments = cli._get_approval_display_fragments()
|
||||
|
||||
rendered = "".join(text for _style, text in fragments)
|
||||
|
||||
# All four choices visible even with a huge command.
|
||||
for label in ("Allow once", "Allow for this session",
|
||||
"Add to permanent allowlist", "Deny"):
|
||||
assert label in rendered, f"choice {label!r} missing"
|
||||
|
||||
# Command got truncated with a marker.
|
||||
assert "(command truncated" in rendered
|
||||
|
||||
@@ -200,6 +200,22 @@ class TestCommandBypassActiveSession:
|
||||
"/background response was not sent back to the user"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_bypasses_guard(self):
|
||||
"""/queue must bypass so it can queue without interrupting."""
|
||||
adapter = _make_adapter()
|
||||
sk = _session_key()
|
||||
adapter._active_sessions[sk] = asyncio.Event()
|
||||
|
||||
await adapter.handle_message(_make_event("/queue follow up"))
|
||||
|
||||
assert sk not in adapter._pending_messages, (
|
||||
"/queue was queued as a pending message instead of being dispatched"
|
||||
)
|
||||
assert any("handled:queue" in r for r in adapter.sent_responses), (
|
||||
"/queue response was not sent back to the user"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: non-bypass messages still get queued
|
||||
|
||||
@@ -370,6 +370,8 @@ class TestCopilotNormalization:
|
||||
assert opencode_model_api_mode("opencode-zen", "minimax-m2.5") == "chat_completions"
|
||||
|
||||
def test_opencode_go_api_modes_match_docs(self):
|
||||
assert opencode_model_api_mode("opencode-go", "glm-5.1") == "chat_completions"
|
||||
assert opencode_model_api_mode("opencode-go", "opencode-go/glm-5.1") == "chat_completions"
|
||||
assert opencode_model_api_mode("opencode-go", "glm-5") == "chat_completions"
|
||||
assert opencode_model_api_mode("opencode-go", "opencode-go/glm-5") == "chat_completions"
|
||||
assert opencode_model_api_mode("opencode-go", "kimi-k2.5") == "chat_completions"
|
||||
|
||||
@@ -15,7 +15,7 @@ def test_opencode_go_appears_when_api_key_set():
|
||||
opencode_go = next((p for p in providers if p["slug"] == "opencode-go"), None)
|
||||
|
||||
assert opencode_go is not None, "opencode-go should appear when OPENCODE_GO_API_KEY is set"
|
||||
assert opencode_go["models"] == ["glm-5", "kimi-k2.5", "mimo-v2-pro", "mimo-v2-omni", "minimax-m2.7", "minimax-m2.5"]
|
||||
assert opencode_go["models"] == ["glm-5.1", "glm-5", "kimi-k2.5", "mimo-v2-pro", "mimo-v2-omni", "minimax-m2.7", "minimax-m2.5"]
|
||||
# opencode-go can appear as "built-in" (from PROVIDER_TO_MODELS_DEV when
|
||||
# models.dev is reachable) or "hermes" (from HERMES_OVERLAYS fallback when
|
||||
# the API is unavailable, e.g. in CI).
|
||||
|
||||
@@ -15,9 +15,20 @@ def _args(**overrides):
|
||||
return Namespace(**base)
|
||||
|
||||
|
||||
def test_cmd_chat_tui_continue_uses_latest_tui_session(monkeypatch):
|
||||
import hermes_cli.main as main_mod
|
||||
@pytest.fixture
|
||||
def main_mod(monkeypatch):
|
||||
"""cmd_chat entry with the first-run provider-gate stubbed past.
|
||||
|
||||
`cmd_chat` now early-exits when no API key is configured (post-merge);
|
||||
these tests exercise the post-config routing so we fake the check out.
|
||||
"""
|
||||
import hermes_cli.main as mod
|
||||
|
||||
monkeypatch.setattr(mod, "_has_any_provider_configured", lambda: True)
|
||||
return mod
|
||||
|
||||
|
||||
def test_cmd_chat_tui_continue_uses_latest_tui_session(monkeypatch, main_mod):
|
||||
calls = []
|
||||
captured = {}
|
||||
|
||||
@@ -40,9 +51,7 @@ def test_cmd_chat_tui_continue_uses_latest_tui_session(monkeypatch):
|
||||
assert captured["resume"] == "20260408_235959_a1b2c3"
|
||||
|
||||
|
||||
def test_cmd_chat_tui_continue_falls_back_to_latest_cli_session(monkeypatch):
|
||||
import hermes_cli.main as main_mod
|
||||
|
||||
def test_cmd_chat_tui_continue_falls_back_to_latest_cli_session(monkeypatch, main_mod):
|
||||
calls = []
|
||||
captured = {}
|
||||
|
||||
@@ -69,9 +78,7 @@ def test_cmd_chat_tui_continue_falls_back_to_latest_cli_session(monkeypatch):
|
||||
assert captured["resume"] == "20260408_235959_d4e5f6"
|
||||
|
||||
|
||||
def test_cmd_chat_tui_resume_resolves_title_before_launch(monkeypatch):
|
||||
import hermes_cli.main as main_mod
|
||||
|
||||
def test_cmd_chat_tui_resume_resolves_title_before_launch(monkeypatch, main_mod):
|
||||
captured = {}
|
||||
|
||||
def fake_launch(resume_session_id=None, tui_dev=False):
|
||||
|
||||
@@ -1122,6 +1122,7 @@ class TestStatusRemoteGateway:
|
||||
assert data["gateway_running"] is True
|
||||
assert data["gateway_pid"] == 999
|
||||
assert data["gateway_state"] == "running"
|
||||
assert data["gateway_health_url"] == "http://gw:8642"
|
||||
|
||||
def test_status_remote_probe_not_attempted_when_local_pid_found(self, monkeypatch):
|
||||
"""When local PID check succeeds, the remote probe is never called."""
|
||||
@@ -1158,6 +1159,7 @@ class TestStatusRemoteGateway:
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["gateway_running"] is False
|
||||
assert data["gateway_health_url"] is None
|
||||
|
||||
def test_status_remote_running_null_pid(self, monkeypatch):
|
||||
"""Remote gateway running but PID not in response — pid should be None."""
|
||||
|
||||
@@ -73,6 +73,50 @@ def _build_encrypted_rtp_packet(secret_key, opus_payload, ssrc=100, seq=1, times
|
||||
return header + ciphertext + nonce_counter
|
||||
|
||||
|
||||
def _build_padded_rtp_packet(
|
||||
secret_key, opus_payload, pad_len, ssrc=100, seq=1, timestamp=960,
|
||||
declared_pad_len=None, ext_words=0,
|
||||
):
|
||||
"""Build a NaCl-encrypted RTP packet with the P bit set and padding appended.
|
||||
|
||||
Per RFC 3550 §5.1, the last padding byte declares how many trailing bytes
|
||||
(including itself) to discard. ``pad_len`` is the actual padding appended;
|
||||
``declared_pad_len`` lets a test forge a mismatched declared length to
|
||||
exercise the validation path. ``ext_words`` > 0 also sets the X bit and
|
||||
prepends a synthetic extension block (4-byte preamble in cleartext header,
|
||||
ext_words*4 bytes of encrypted extension data prepended to the payload).
|
||||
"""
|
||||
if pad_len < 1:
|
||||
raise ValueError("pad_len must be >= 1 (last byte includes itself)")
|
||||
declared = pad_len if declared_pad_len is None else declared_pad_len
|
||||
if declared < 0 or declared > 255:
|
||||
raise ValueError("declared_pad_len must fit in one byte")
|
||||
|
||||
has_extension = ext_words > 0
|
||||
first_byte = 0xA0 | (0x10 if has_extension else 0) # V=2, P=1, [X=?], CC=0
|
||||
fixed_header = struct.pack(">BBHII", first_byte, 0x78, seq, timestamp, ssrc)
|
||||
if has_extension:
|
||||
# 4-byte extension preamble: 2 bytes "defined by profile" + 2 bytes length-in-words
|
||||
ext_preamble = struct.pack(">HH", 0xBEDE, ext_words)
|
||||
header = fixed_header + ext_preamble
|
||||
ext_data = b"\xab" * (ext_words * 4)
|
||||
else:
|
||||
header = fixed_header
|
||||
ext_data = b""
|
||||
|
||||
padding = b"\x00" * (pad_len - 1) + bytes([declared])
|
||||
plaintext = ext_data + opus_payload + padding
|
||||
|
||||
box = nacl.secret.Aead(secret_key)
|
||||
nonce_counter = struct.pack(">I", seq)
|
||||
full_nonce = nonce_counter + b"\x00" * 20
|
||||
|
||||
enc_msg = box.encrypt(plaintext, header, full_nonce)
|
||||
ciphertext = enc_msg.ciphertext
|
||||
|
||||
return header + ciphertext + nonce_counter
|
||||
|
||||
|
||||
def _make_voice_receiver(secret_key, dave_session=None, bot_ssrc=9999,
|
||||
allowed_user_ids=None, members=None):
|
||||
"""Create a VoiceReceiver with real secret key."""
|
||||
@@ -212,6 +256,113 @@ class TestRealNaClWithDAVE:
|
||||
assert len(receiver._buffers.get(100, b"")) == 0
|
||||
|
||||
|
||||
class TestRTPPaddingStrip:
|
||||
"""RFC 3550 §5.1 — strip RTP padding before DAVE/Opus decode."""
|
||||
|
||||
def test_padded_packet_stripped_and_buffered(self):
|
||||
"""P bit set → trailing padding stripped → opus payload decoded."""
|
||||
key = _make_secret_key()
|
||||
opus_silence = b"\xf8\xff\xfe"
|
||||
receiver = _make_voice_receiver(key)
|
||||
|
||||
# 5 bytes of padding (4 zeros + count byte = 5)
|
||||
packet = _build_padded_rtp_packet(key, opus_silence, pad_len=5, ssrc=100)
|
||||
receiver._on_packet(packet)
|
||||
|
||||
assert 100 in receiver._buffers
|
||||
assert len(receiver._buffers[100]) > 0
|
||||
|
||||
def test_padded_packet_matches_unpadded_output(self):
|
||||
"""Same opus payload with/without padding → same decoded PCM."""
|
||||
key = _make_secret_key()
|
||||
opus_silence = b"\xf8\xff\xfe"
|
||||
|
||||
recv_plain = _make_voice_receiver(key)
|
||||
recv_plain._on_packet(
|
||||
_build_encrypted_rtp_packet(key, opus_silence, ssrc=100)
|
||||
)
|
||||
|
||||
recv_padded = _make_voice_receiver(key)
|
||||
recv_padded._on_packet(
|
||||
_build_padded_rtp_packet(key, opus_silence, pad_len=7, ssrc=100)
|
||||
)
|
||||
|
||||
assert bytes(recv_plain._buffers[100]) == bytes(recv_padded._buffers[100])
|
||||
|
||||
def test_padding_with_dave_passthrough(self):
|
||||
"""Padding stripped before DAVE → passthrough buffers cleanly."""
|
||||
key = _make_secret_key()
|
||||
opus_silence = b"\xf8\xff\xfe"
|
||||
dave = MagicMock() # SSRC unmapped → DAVE skipped, passthrough used
|
||||
receiver = _make_voice_receiver(key, dave_session=dave)
|
||||
|
||||
packet = _build_padded_rtp_packet(key, opus_silence, pad_len=4, ssrc=100)
|
||||
receiver._on_packet(packet)
|
||||
|
||||
dave.decrypt.assert_not_called()
|
||||
assert 100 in receiver._buffers
|
||||
assert len(receiver._buffers[100]) > 0
|
||||
|
||||
def test_invalid_padding_length_zero_dropped(self):
|
||||
"""Declared pad_len=0 is invalid (RFC requires count includes itself)."""
|
||||
key = _make_secret_key()
|
||||
opus_silence = b"\xf8\xff\xfe"
|
||||
receiver = _make_voice_receiver(key)
|
||||
|
||||
packet = _build_padded_rtp_packet(
|
||||
key, opus_silence, pad_len=4, declared_pad_len=0, ssrc=100
|
||||
)
|
||||
receiver._on_packet(packet)
|
||||
|
||||
assert len(receiver._buffers.get(100, b"")) == 0
|
||||
|
||||
def test_invalid_padding_length_overflow_dropped(self):
|
||||
"""Declared pad_len > payload size → packet dropped."""
|
||||
key = _make_secret_key()
|
||||
opus_silence = b"\xf8\xff\xfe"
|
||||
receiver = _make_voice_receiver(key)
|
||||
|
||||
packet = _build_padded_rtp_packet(
|
||||
key, opus_silence, pad_len=4, declared_pad_len=255, ssrc=100
|
||||
)
|
||||
receiver._on_packet(packet)
|
||||
|
||||
assert len(receiver._buffers.get(100, b"")) == 0
|
||||
|
||||
def test_padding_consuming_entire_payload_dropped(self):
|
||||
"""Padding consumes entire payload → no opus data → dropped."""
|
||||
key = _make_secret_key()
|
||||
receiver = _make_voice_receiver(key)
|
||||
|
||||
# Empty opus payload, 6 bytes of padding (count byte declares 6)
|
||||
packet = _build_padded_rtp_packet(key, b"", pad_len=6, ssrc=100)
|
||||
receiver._on_packet(packet)
|
||||
|
||||
assert len(receiver._buffers.get(100, b"")) == 0
|
||||
|
||||
def test_padding_with_extension_stripped_correctly(self):
|
||||
"""X+P bits both set → strip extension from start, padding from end."""
|
||||
key = _make_secret_key()
|
||||
opus_silence = b"\xf8\xff\xfe"
|
||||
|
||||
# Same opus payload sent two ways: plain, and with both ext+padding
|
||||
recv_plain = _make_voice_receiver(key)
|
||||
recv_plain._on_packet(
|
||||
_build_encrypted_rtp_packet(key, opus_silence, ssrc=100)
|
||||
)
|
||||
|
||||
recv_ext_pad = _make_voice_receiver(key)
|
||||
recv_ext_pad._on_packet(
|
||||
_build_padded_rtp_packet(
|
||||
key, opus_silence, pad_len=5, ext_words=2, ssrc=100
|
||||
)
|
||||
)
|
||||
|
||||
# Both must yield identical decoded PCM — ext data and padding both
|
||||
# stripped before opus decode.
|
||||
assert bytes(recv_plain._buffers[100]) == bytes(recv_ext_pad._buffers[100])
|
||||
|
||||
|
||||
class TestFullVoiceFlow:
|
||||
"""End-to-end: encrypt → receive → buffer → silence detect → complete."""
|
||||
|
||||
|
||||
186
tests/run_agent/test_create_openai_client_reuse.py
Normal file
186
tests/run_agent/test_create_openai_client_reuse.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""Regression guardrail: sequential _create_openai_client calls must not
|
||||
share a closed transport across invocations.
|
||||
|
||||
This is the behavioral twin of test_create_openai_client_kwargs_isolation.py.
|
||||
That test pins "don't mutate input kwargs" at the syntactic level — it catches
|
||||
#10933 specifically because the bug mutated ``client_kwargs`` in place. This
|
||||
test pins the user-visible invariant at the behavioral level: no matter HOW a
|
||||
future keepalive / transport reimplementation plumbs sockets in, the Nth call
|
||||
to ``_create_openai_client`` must not hand back a client wrapping a
|
||||
now-closed httpx transport from an earlier call.
|
||||
|
||||
AlexKucera's Discord report (2026-04-16): after ``hermes update`` pulled
|
||||
#10933, the first chat on a session worked, every subsequent chat failed
|
||||
with ``APIConnectionError('Connection error.')`` whose cause was
|
||||
``RuntimeError: Cannot send a request, as the client has been closed``.
|
||||
That is the exact scenario this test reproduces at object level without a
|
||||
network, so it runs in CI on every PR.
|
||||
"""
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from run_agent import AIAgent
|
||||
|
||||
|
||||
def _make_agent():
|
||||
return AIAgent(
|
||||
model="test/model",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
)
|
||||
|
||||
|
||||
def _make_fake_openai_factory(constructed):
|
||||
"""Return a fake ``OpenAI`` class that records every constructed instance
|
||||
along with whatever ``http_client`` it was handed (or ``None`` if the
|
||||
caller did not inject one).
|
||||
|
||||
The fake also forwards ``.close()`` calls down to the http_client if one
|
||||
is present, mirroring what the real OpenAI SDK does during teardown and
|
||||
what would expose the #10933 bug.
|
||||
"""
|
||||
|
||||
class _FakeOpenAI:
|
||||
def __init__(self, **kwargs):
|
||||
self._kwargs = kwargs
|
||||
self._http_client = kwargs.get("http_client")
|
||||
self._closed = False
|
||||
constructed.append(self)
|
||||
|
||||
def close(self):
|
||||
self._closed = True
|
||||
hc = self._http_client
|
||||
if hc is not None and hasattr(hc, "close"):
|
||||
try:
|
||||
hc.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _FakeOpenAI
|
||||
|
||||
|
||||
def test_second_create_does_not_wrap_closed_transport_from_first():
|
||||
"""Back-to-back _create_openai_client calls on the same _client_kwargs
|
||||
must not hand call N a closed http_client from call N-1.
|
||||
|
||||
The bug class: call 1 injects an httpx.Client into self._client_kwargs,
|
||||
client 1 closes (SDK teardown), its http_client closes with it, call 2
|
||||
reads the SAME now-closed http_client from self._client_kwargs and wraps
|
||||
it. Every request through client 2 then fails.
|
||||
"""
|
||||
agent = _make_agent()
|
||||
constructed: list = []
|
||||
fake_openai = _make_fake_openai_factory(constructed)
|
||||
|
||||
# Seed a baseline kwargs dict resembling real runtime state.
|
||||
agent._client_kwargs = {
|
||||
"api_key": "test-key-value",
|
||||
"base_url": "https://api.example.com/v1",
|
||||
}
|
||||
|
||||
with patch("run_agent.OpenAI", fake_openai):
|
||||
# Call 1 — what _replace_primary_openai_client does at init/rebuild.
|
||||
client_a = agent._create_openai_client(
|
||||
agent._client_kwargs, reason="initial", shared=True
|
||||
)
|
||||
# Simulate the SDK teardown that follows a rebuild: the old client's
|
||||
# close() is invoked, which closes its underlying http_client if one
|
||||
# was injected. This is exactly what _replace_primary_openai_client
|
||||
# does via _close_openai_client after a successful rebuild.
|
||||
client_a.close()
|
||||
|
||||
# Call 2 — the rebuild path. This is where #10933 crashed on the
|
||||
# next real request.
|
||||
client_b = agent._create_openai_client(
|
||||
agent._client_kwargs, reason="rebuild", shared=True
|
||||
)
|
||||
|
||||
assert len(constructed) == 2, f"expected 2 OpenAI constructions, got {len(constructed)}"
|
||||
assert constructed[0] is client_a
|
||||
assert constructed[1] is client_b
|
||||
|
||||
hc_a = constructed[0]._http_client
|
||||
hc_b = constructed[1]._http_client
|
||||
|
||||
# If the implementation does not inject http_client at all, we're safely
|
||||
# past the bug class — nothing to share, nothing to close. That's fine.
|
||||
if hc_a is None and hc_b is None:
|
||||
return
|
||||
|
||||
# If ANY http_client is injected, the two calls MUST NOT share the same
|
||||
# object, because call 1's object was closed between calls.
|
||||
if hc_a is not None and hc_b is not None:
|
||||
assert hc_a is not hc_b, (
|
||||
"Regression of #10933: _create_openai_client handed the same "
|
||||
"http_client to two sequential constructions. After the first "
|
||||
"client is closed (normal SDK teardown on rebuild), the second "
|
||||
"wraps a closed transport and every subsequent chat raises "
|
||||
"'Cannot send a request, as the client has been closed'."
|
||||
)
|
||||
|
||||
# And whatever http_client the LATEST call handed out must not be closed
|
||||
# already. This catches implementations that cache the injected client on
|
||||
# ``self`` (under any attribute name) and rebuild the SDK client around
|
||||
# it even after the previous SDK close closed the cached transport.
|
||||
if hc_b is not None:
|
||||
is_closed_attr = getattr(hc_b, "is_closed", None)
|
||||
if is_closed_attr is not None:
|
||||
assert not is_closed_attr, (
|
||||
"Regression of #10933: second _create_openai_client returned "
|
||||
"a client whose http_client is already closed. New chats on "
|
||||
"this session will fail with 'Cannot send a request, as the "
|
||||
"client has been closed'."
|
||||
)
|
||||
|
||||
|
||||
def test_replace_primary_openai_client_survives_repeated_rebuilds():
|
||||
"""Full rebuild path: exercise _replace_primary_openai_client three times
|
||||
back-to-back and confirm every resulting ``self.client`` is a fresh,
|
||||
usable construction rather than a wrapper around a previously-closed
|
||||
transport.
|
||||
|
||||
_replace_primary_openai_client is the real rebuild entrypoint — it is
|
||||
what runs on 401 credential refresh, pool rotation, and model switch.
|
||||
If a future keepalive tweak stores state on ``self`` between calls,
|
||||
this test is what notices.
|
||||
"""
|
||||
agent = _make_agent()
|
||||
constructed: list = []
|
||||
fake_openai = _make_fake_openai_factory(constructed)
|
||||
|
||||
agent._client_kwargs = {
|
||||
"api_key": "test-key-value",
|
||||
"base_url": "https://api.example.com/v1",
|
||||
}
|
||||
|
||||
with patch("run_agent.OpenAI", fake_openai):
|
||||
# Seed the initial client so _replace has something to tear down.
|
||||
agent.client = agent._create_openai_client(
|
||||
agent._client_kwargs, reason="seed", shared=True
|
||||
)
|
||||
# Three rebuilds in a row. Each one must install a fresh live client.
|
||||
for label in ("rebuild_1", "rebuild_2", "rebuild_3"):
|
||||
ok = agent._replace_primary_openai_client(reason=label)
|
||||
assert ok, f"rebuild {label} returned False"
|
||||
cur = agent.client
|
||||
assert not cur._closed, (
|
||||
f"after rebuild {label}, self.client is already closed — "
|
||||
"this breaks the very next chat turn"
|
||||
)
|
||||
hc = cur._http_client
|
||||
if hc is not None:
|
||||
is_closed_attr = getattr(hc, "is_closed", None)
|
||||
if is_closed_attr is not None:
|
||||
assert not is_closed_attr, (
|
||||
f"after rebuild {label}, self.client.http_client is "
|
||||
"closed — reproduces #10933 (AlexKucera report, "
|
||||
"Discord 2026-04-16)"
|
||||
)
|
||||
|
||||
# All four constructions (seed + 3 rebuilds) should be distinct objects.
|
||||
# If two are the same, the rebuild is cacheing the SDK client across
|
||||
# teardown, which also reproduces the bug class.
|
||||
assert len({id(c) for c in constructed}) == len(constructed), (
|
||||
"Some _create_openai_client calls returned the same object across "
|
||||
"a teardown — rebuild is not producing fresh clients"
|
||||
)
|
||||
137
tests/run_agent/test_sequential_chats_live.py
Normal file
137
tests/run_agent/test_sequential_chats_live.py
Normal file
@@ -0,0 +1,137 @@
|
||||
"""Live regression guardrail for the keepalive/transport bug class (#10933).
|
||||
|
||||
AlexKucera reported on Discord (2026-04-16) that after ``hermes update`` pulled
|
||||
#10933, the FIRST chat in a session worked and EVERY subsequent chat failed
|
||||
with ``APIConnectionError('Connection error.')`` whose cause was
|
||||
``RuntimeError: Cannot send a request, as the client has been closed``.
|
||||
|
||||
The companion ``test_create_openai_client_reuse.py`` pins this contract at
|
||||
object level with mocked ``OpenAI``. This file runs the same shape of
|
||||
reproduction against a real provider so we have a true end-to-end smoke test
|
||||
for any future keepalive / transport plumbing.
|
||||
|
||||
Opt-in — not part of default CI:
|
||||
HERMES_LIVE_TESTS=1 pytest tests/run_agent/test_sequential_chats_live.py -v
|
||||
|
||||
Requires ``OPENROUTER_API_KEY`` to be set (or sourced via ~/.hermes/.env).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# Load ~/.hermes/.env so live runs pick up OPENROUTER_API_KEY without
|
||||
# needing the runner to shell-source it first. Silent if the file is absent.
|
||||
def _load_user_env() -> None:
|
||||
env_file = Path.home() / ".hermes" / ".env"
|
||||
if not env_file.exists():
|
||||
return
|
||||
for raw in env_file.read_text().splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
v = v.strip().strip('"').strip("'")
|
||||
# Don't clobber an already-set env var — lets the caller override.
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
|
||||
_load_user_env()
|
||||
|
||||
|
||||
LIVE = os.environ.get("HERMES_LIVE_TESTS") == "1"
|
||||
OR_KEY = os.environ.get("OPENROUTER_API_KEY", "")
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(not LIVE, reason="live-only — set HERMES_LIVE_TESTS=1"),
|
||||
pytest.mark.skipif(not OR_KEY, reason="OPENROUTER_API_KEY not configured"),
|
||||
]
|
||||
|
||||
# Cheap, fast, tool-capable. Swap if it ever goes dark.
|
||||
LIVE_MODEL = "google/gemini-2.5-flash"
|
||||
|
||||
|
||||
def _make_live_agent():
|
||||
from run_agent import AIAgent
|
||||
|
||||
return AIAgent(
|
||||
model=LIVE_MODEL,
|
||||
provider="openrouter",
|
||||
api_key=OR_KEY,
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
max_iterations=3,
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
# All toolsets off so the agent just produces a single text reply
|
||||
# per turn — we want to test the HTTP client lifecycle, not tools.
|
||||
disabled_toolsets=["*"],
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_error_reply(reply: str) -> tuple[bool, str]:
|
||||
"""AIAgent returns an error-sentinel string (not an exception) when the
|
||||
underlying API call fails past retries. A naive ``assert reply and
|
||||
reply.strip()`` misses this because the sentinel is truthy. This
|
||||
checker enumerates the known-bad shapes so the live test actually
|
||||
catches #10933 instead of rubber-stamping the error response.
|
||||
"""
|
||||
lowered = reply.lower().strip()
|
||||
bad_substrings = (
|
||||
"api call failed",
|
||||
"connection error",
|
||||
"client has been closed",
|
||||
"cannot send a request",
|
||||
"max retries",
|
||||
)
|
||||
for marker in bad_substrings:
|
||||
if marker in lowered:
|
||||
return True, marker
|
||||
return False, ""
|
||||
|
||||
|
||||
def _assert_healthy_reply(reply, turn_label: str) -> None:
|
||||
assert reply and reply.strip(), f"{turn_label} returned empty: {reply!r}"
|
||||
is_err, marker = _looks_like_error_reply(reply)
|
||||
assert not is_err, (
|
||||
f"{turn_label} returned an error-sentinel string instead of a real "
|
||||
f"model reply — matched marker {marker!r}. This is the exact shape "
|
||||
f"of #10933 (AlexKucera Discord report, 2026-04-16): the agent's "
|
||||
f"retry loop burned three attempts against a closed httpx transport "
|
||||
f"and surfaced 'API call failed after 3 retries: Connection error.' "
|
||||
f"to the user. Reply was: {reply!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_three_sequential_chats_across_client_rebuild():
|
||||
"""Reproduces AlexKucera's exact failure shape end-to-end.
|
||||
|
||||
Turn 1 always worked under #10933. Turn 2 was the one that failed
|
||||
because the shared httpx transport had been torn down between turns.
|
||||
Turn 3 is here as extra insurance against any lazy-init shape where
|
||||
the failure only shows up on call N>=3.
|
||||
|
||||
We also deliberately trigger ``_replace_primary_openai_client`` between
|
||||
turn 2 and turn 3 — that is the real rebuild entrypoint (401 refresh,
|
||||
credential rotation, model switch) and is the path that actually
|
||||
stored the closed transport into ``self._client_kwargs`` in #10933.
|
||||
"""
|
||||
agent = _make_live_agent()
|
||||
|
||||
r1 = agent.chat("Respond with only the word: ONE")
|
||||
_assert_healthy_reply(r1, "turn 1")
|
||||
|
||||
r2 = agent.chat("Respond with only the word: TWO")
|
||||
_assert_healthy_reply(r2, "turn 2")
|
||||
|
||||
# Force a client rebuild through the real path — mimics 401 refresh /
|
||||
# credential rotation / model switch lifecycle.
|
||||
rebuilt = agent._replace_primary_openai_client(reason="regression_test_rebuild")
|
||||
assert rebuilt, "rebuild via _replace_primary_openai_client returned False"
|
||||
|
||||
r3 = agent.chat("Respond with only the word: THREE")
|
||||
_assert_healthy_reply(r3, "turn 3 (post-rebuild)")
|
||||
@@ -231,6 +231,9 @@ def test_config_set_personality_resets_history_and_returns_info(monkeypatch):
|
||||
monkeypatch.setattr(server, "_session_info", lambda agent: {"model": getattr(agent, "model", "?")})
|
||||
monkeypatch.setattr(server, "_restart_slash_worker", lambda session: None)
|
||||
monkeypatch.setattr(server, "_emit", lambda *args: emits.append(args))
|
||||
# _write_config_key writes to ~/.hermes/config.yaml — races with other
|
||||
# xdist workers that touch the same file. Stub it out.
|
||||
monkeypatch.setattr(server, "_write_config_key", lambda path, value: None)
|
||||
|
||||
resp = server.handle_request(
|
||||
{"id": "1", "method": "config.set", "params": {"session_id": "sid", "key": "personality", "value": "helpful"}}
|
||||
|
||||
@@ -165,7 +165,7 @@ def test_session_resume_returns_hydrated_messages(server, monkeypatch):
|
||||
def reopen_session(self, _sid):
|
||||
return None
|
||||
|
||||
def get_messages(self, _sid):
|
||||
def get_messages_as_conversation(self, _sid):
|
||||
return [
|
||||
{"role": "user", "content": "hello"},
|
||||
{"role": "assistant", "content": "yo"},
|
||||
@@ -193,7 +193,7 @@ def test_session_resume_returns_hydrated_messages(server, monkeypatch):
|
||||
assert resp["result"]["messages"] == [
|
||||
{"role": "user", "text": "hello"},
|
||||
{"role": "assistant", "text": "yo"},
|
||||
{"role": "tool", "text": "searched"},
|
||||
{"role": "tool", "name": "tool", "context": ""},
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user