refactor(tests): re-architect tests + fix CI failures (#5946)
* refactor: re-architect tests to mirror the codebase
* Update tests.yml
* fix: add missing tool_error imports after registry refactor
* fix(tests): replace patch.dict with monkeypatch to prevent env var leaks under xdist
patch.dict(os.environ) can leak TERMINAL_ENV across xdist workers,
causing test_code_execution tests to hit the Modal remote path.
* fix(tests): fix update_check and telegram xdist failures
- test_update_check: replace patch("hermes_cli.banner.os.getenv") with
monkeypatch.setenv("HERMES_HOME") — banner.py no longer imports os
directly, it uses get_hermes_home() from hermes_constants.
- test_telegram_conflict/approval_buttons: provide real exception classes
for telegram.error mock (NetworkError, TimedOut, BadRequest) so the
except clause in connect() doesn't fail with "catching classes that do
not inherit from BaseException" when xdist pollutes sys.modules.
* fix(tests): accept unavailable_models kwarg in _prompt_model_selection mock
This commit is contained in:
Siddharth Balyan
@@ -18,10 +18,18 @@ import pytest
|
||||
import json
|
||||
import os
|
||||
|
||||
# Force local terminal backend for ALL tests in this file.
|
||||
# Under xdist, another test may leak TERMINAL_ENV=modal/docker, sending
|
||||
# execute_code down the remote path → modal.exception.AuthError.
|
||||
os.environ["TERMINAL_ENV"] = "local"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _force_local_terminal(monkeypatch):
|
||||
"""Re-set TERMINAL_ENV=local before every test.
|
||||
|
||||
The module-level assignment above covers import time, but under xdist
|
||||
another worker can overwrite os.environ between tests. monkeypatch
|
||||
ensures each test starts (and ends) with the correct value.
|
||||
"""
|
||||
monkeypatch.setenv("TERMINAL_ENV", "local")
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
|
||||
@@ -91,7 +91,11 @@ def _install_fake_tools_package():
|
||||
def register(self, **kwargs):
|
||||
return None
|
||||
|
||||
sys.modules["tools.registry"] = types.SimpleNamespace(registry=_Registry())
|
||||
from tools.registry import tool_error
|
||||
|
||||
sys.modules["tools.registry"] = types.SimpleNamespace(
|
||||
registry=_Registry(), tool_error=tool_error,
|
||||
)
|
||||
|
||||
class _DummyEnvironment:
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
||||
178
tests/tools/test_managed_server_tool_support.py
Normal file
178
tests/tools/test_managed_server_tool_support.py
Normal file
@@ -0,0 +1,178 @@
|
||||
"""
|
||||
Tests for ManagedServer / tool-parser integration.
|
||||
|
||||
Validates that:
|
||||
1. The installed atroposlib API still matches Hermes's expectations
|
||||
2. Hermes's parser registry remains compatible with ManagedServer parsing
|
||||
3. HermesAgentBaseEnv wires the selected parser into ServerManager correctly
|
||||
|
||||
These tests verify the contract between hermes-agent's environments/ code
|
||||
and atroposlib's ManagedServer. They detect API incompatibilities early.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
try:
|
||||
import atroposlib # noqa: F401
|
||||
except ImportError:
|
||||
pytest.skip("atroposlib not installed", allow_module_level=True)
|
||||
|
||||
|
||||
class TestManagedServerAPI:
|
||||
"""Test that ManagedServer's API matches what hermes-agent expects."""
|
||||
|
||||
def test_managed_server_init_signature(self):
|
||||
"""ManagedServer should accept tool_call_parser parameter."""
|
||||
from atroposlib.envs.server_handling.managed_server import ManagedServer
|
||||
|
||||
sig = inspect.signature(ManagedServer.__init__)
|
||||
params = list(sig.parameters.keys())
|
||||
|
||||
# Core params that must exist
|
||||
assert "self" in params
|
||||
assert "server" in params
|
||||
assert "tokenizer" in params
|
||||
assert "track_tree" in params
|
||||
|
||||
# tool_call_parser — required for tool_call_support branch
|
||||
# If this fails, atroposlib hasn't been updated to tool_call_support
|
||||
has_tool_parser = "tool_call_parser" in params
|
||||
if not has_tool_parser:
|
||||
pytest.skip(
|
||||
"ManagedServer does not have tool_call_parser param — "
|
||||
"baseline atroposlib (pre tool_call_support branch)"
|
||||
)
|
||||
|
||||
def test_server_manager_managed_server_signature(self):
|
||||
"""ServerManager.managed_server() should accept tool_call_parser."""
|
||||
from atroposlib.envs.server_handling.server_manager import ServerManager
|
||||
|
||||
sig = inspect.signature(ServerManager.managed_server)
|
||||
params = list(sig.parameters.keys())
|
||||
|
||||
assert "self" in params
|
||||
assert "tokenizer" in params
|
||||
|
||||
has_tool_parser = "tool_call_parser" in params
|
||||
if not has_tool_parser:
|
||||
pytest.skip(
|
||||
"ServerManager.managed_server() does not have tool_call_parser param — "
|
||||
"baseline atroposlib (pre tool_call_support branch)"
|
||||
)
|
||||
|
||||
def test_managed_server_chat_template_kwargs(self):
|
||||
"""ManagedServer should have CHAT_TEMPLATE_KWARGS for forwarding tools/thinking."""
|
||||
from atroposlib.envs.server_handling.managed_server import ManagedServer
|
||||
|
||||
if not hasattr(ManagedServer, "CHAT_TEMPLATE_KWARGS"):
|
||||
pytest.skip(
|
||||
"ManagedServer does not have CHAT_TEMPLATE_KWARGS — "
|
||||
"baseline atroposlib (pre tool_call_support branch)"
|
||||
)
|
||||
|
||||
kwargs = ManagedServer.CHAT_TEMPLATE_KWARGS
|
||||
assert "tools" in kwargs, "tools must be in CHAT_TEMPLATE_KWARGS"
|
||||
|
||||
def test_no_get_logprobs_method(self):
|
||||
"""get_logprobs should be removed in tool_call_support branch."""
|
||||
from atroposlib.envs.server_handling.managed_server import ManagedServer
|
||||
|
||||
# In baseline, get_logprobs exists. In tool_call_support, it's removed.
|
||||
# We just note the state — not a hard fail either way.
|
||||
has_get_logprobs = hasattr(ManagedServer, "get_logprobs")
|
||||
if has_get_logprobs:
|
||||
pytest.skip(
|
||||
"ManagedServer still has get_logprobs — baseline atroposlib"
|
||||
)
|
||||
|
||||
|
||||
class TestParserCompatibility:
|
||||
"""Test that hermes-agent's parsers match ManagedServer's expectations."""
|
||||
|
||||
def test_parser_parse_returns_correct_format(self):
|
||||
"""
|
||||
ManagedServer expects parser.parse(text) -> (content, tool_calls)
|
||||
where tool_calls is a list of objects with .id, .function.name, .function.arguments
|
||||
"""
|
||||
from environments.tool_call_parsers import get_parser
|
||||
|
||||
parser = get_parser("hermes")
|
||||
text = '<tool_call>{"name": "terminal", "arguments": {"command": "ls"}}</tool_call>'
|
||||
content, tool_calls = parser.parse(text)
|
||||
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
|
||||
tc = tool_calls[0]
|
||||
# ManagedServer accesses these attrs directly
|
||||
assert hasattr(tc, "id")
|
||||
assert hasattr(tc, "function")
|
||||
assert hasattr(tc.function, "name")
|
||||
assert hasattr(tc.function, "arguments")
|
||||
|
||||
def test_parser_no_tools_returns_none(self):
|
||||
"""ManagedServer checks `if parsed_tool_calls:` — None should be falsy."""
|
||||
from environments.tool_call_parsers import get_parser
|
||||
|
||||
parser = get_parser("hermes")
|
||||
content, tool_calls = parser.parse("Just text, no tools")
|
||||
assert tool_calls is None
|
||||
|
||||
def test_parser_content_is_string_or_none(self):
|
||||
"""ManagedServer uses `parsed_content or ""` — must be str or None."""
|
||||
from environments.tool_call_parsers import get_parser
|
||||
|
||||
parser = get_parser("hermes")
|
||||
|
||||
# With tool calls
|
||||
text = '<tool_call>{"name": "terminal", "arguments": {"command": "ls"}}</tool_call>'
|
||||
content, _ = parser.parse(text)
|
||||
assert content is None or isinstance(content, str)
|
||||
|
||||
# Without tool calls
|
||||
content2, _ = parser.parse("Just text")
|
||||
assert isinstance(content2, str)
|
||||
|
||||
|
||||
class TestBaseEnvCompatibility:
|
||||
"""Test that hermes_base_env.py's tool-parser wiring matches the current API."""
|
||||
|
||||
def test_hermes_base_env_sets_server_manager_tool_parser(self):
|
||||
"""Hermes wires parser selection through ServerManager.tool_parser."""
|
||||
import ast
|
||||
|
||||
base_env_path = Path(__file__).parent.parent / "environments" / "hermes_base_env.py"
|
||||
source = base_env_path.read_text()
|
||||
tree = ast.parse(source)
|
||||
|
||||
found_assignment = False
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Attribute) and target.attr == "tool_parser":
|
||||
parent = target.value
|
||||
if (
|
||||
isinstance(parent, ast.Attribute)
|
||||
and parent.attr == "server"
|
||||
and isinstance(parent.value, ast.Name)
|
||||
and parent.value.id == "self"
|
||||
):
|
||||
found_assignment = True
|
||||
|
||||
assert found_assignment, (
|
||||
"hermes_base_env.py should set self.server.tool_parser from config.tool_call_parser"
|
||||
)
|
||||
|
||||
def test_hermes_base_env_uses_config_tool_call_parser(self):
|
||||
"""Verify hermes_base_env uses the config field rather than a local parser instance."""
|
||||
base_env_path = Path(__file__).parent.parent / "environments" / "hermes_base_env.py"
|
||||
source = base_env_path.read_text()
|
||||
|
||||
assert 'tool_call_parser: str = Field(' in source
|
||||
assert 'self.server.tool_parser = config.tool_call_parser' in source
|
||||
@@ -12,8 +12,6 @@ Covers the bugs discovered while setting up TBLite evaluation:
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure repo root is importable
|
||||
@@ -64,89 +62,72 @@ class TestToolResolution:
|
||||
class TestCwdHandling:
|
||||
"""Verify host paths are sanitized for container backends."""
|
||||
|
||||
def test_home_path_replaced_for_modal(self):
|
||||
def test_home_path_replaced_for_modal(self, monkeypatch):
|
||||
"""TERMINAL_CWD=/home/user/... should be replaced with /root for modal."""
|
||||
with patch.dict(os.environ, {
|
||||
"TERMINAL_ENV": "modal",
|
||||
"TERMINAL_CWD": "/home/dakota/github/hermes-agent",
|
||||
}):
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/root", (
|
||||
f"Expected /root, got {config['cwd']}. "
|
||||
"/home/ paths should be replaced for modal backend."
|
||||
)
|
||||
monkeypatch.setenv("TERMINAL_ENV", "modal")
|
||||
monkeypatch.setenv("TERMINAL_CWD", "/home/dakota/github/hermes-agent")
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/root", (
|
||||
f"Expected /root, got {config['cwd']}. "
|
||||
"/home/ paths should be replaced for modal backend."
|
||||
)
|
||||
|
||||
def test_users_path_replaced_for_docker_by_default(self):
|
||||
def test_users_path_replaced_for_docker_by_default(self, monkeypatch):
|
||||
"""Docker should keep host paths out of the sandbox unless explicitly enabled."""
|
||||
with patch.dict(os.environ, {
|
||||
"TERMINAL_ENV": "docker",
|
||||
"TERMINAL_CWD": "/Users/someone/projects",
|
||||
}):
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/root", (
|
||||
f"Expected /root, got {config['cwd']}. "
|
||||
"Host paths should be discarded for docker backend by default."
|
||||
)
|
||||
assert config["host_cwd"] is None
|
||||
assert config["docker_mount_cwd_to_workspace"] is False
|
||||
monkeypatch.setenv("TERMINAL_ENV", "docker")
|
||||
monkeypatch.setenv("TERMINAL_CWD", "/Users/someone/projects")
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/root", (
|
||||
f"Expected /root, got {config['cwd']}. "
|
||||
"Host paths should be discarded for docker backend by default."
|
||||
)
|
||||
assert config["host_cwd"] is None
|
||||
assert config["docker_mount_cwd_to_workspace"] is False
|
||||
|
||||
def test_users_path_maps_to_workspace_for_docker_when_enabled(self):
|
||||
def test_users_path_maps_to_workspace_for_docker_when_enabled(self, monkeypatch):
|
||||
"""Docker should map the host cwd into /workspace only when explicitly enabled."""
|
||||
with patch.dict(os.environ, {
|
||||
"TERMINAL_ENV": "docker",
|
||||
"TERMINAL_CWD": "/Users/someone/projects",
|
||||
"TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE": "true",
|
||||
}):
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/workspace"
|
||||
assert config["host_cwd"] == "/Users/someone/projects"
|
||||
assert config["docker_mount_cwd_to_workspace"] is True
|
||||
monkeypatch.setenv("TERMINAL_ENV", "docker")
|
||||
monkeypatch.setenv("TERMINAL_CWD", "/Users/someone/projects")
|
||||
monkeypatch.setenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "true")
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/workspace"
|
||||
assert config["host_cwd"] == "/Users/someone/projects"
|
||||
assert config["docker_mount_cwd_to_workspace"] is True
|
||||
|
||||
def test_windows_path_replaced_for_modal(self):
|
||||
def test_windows_path_replaced_for_modal(self, monkeypatch):
|
||||
"""TERMINAL_CWD=C:\\Users\\... should be replaced for modal."""
|
||||
with patch.dict(os.environ, {
|
||||
"TERMINAL_ENV": "modal",
|
||||
"TERMINAL_CWD": "C:\\Users\\someone\\projects",
|
||||
}):
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/root"
|
||||
monkeypatch.setenv("TERMINAL_ENV", "modal")
|
||||
monkeypatch.setenv("TERMINAL_CWD", "C:\\Users\\someone\\projects")
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/root"
|
||||
|
||||
def test_default_cwd_is_root_for_container_backends(self):
|
||||
@pytest.mark.parametrize("backend", ["modal", "docker", "singularity", "daytona"])
|
||||
def test_default_cwd_is_root_for_container_backends(self, backend, monkeypatch):
|
||||
"""Container backends should default to /root, not ~."""
|
||||
for backend in ("modal", "docker", "singularity", "daytona"):
|
||||
with patch.dict(os.environ, {"TERMINAL_ENV": backend}, clear=False):
|
||||
# Remove TERMINAL_CWD so it uses default
|
||||
env = os.environ.copy()
|
||||
env.pop("TERMINAL_CWD", None)
|
||||
env.pop("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", None)
|
||||
with patch.dict(os.environ, env, clear=True):
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/root", (
|
||||
f"Backend {backend}: expected /root default, got {config['cwd']}"
|
||||
)
|
||||
monkeypatch.setenv("TERMINAL_ENV", backend)
|
||||
monkeypatch.delenv("TERMINAL_CWD", raising=False)
|
||||
monkeypatch.delenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", raising=False)
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/root", (
|
||||
f"Backend {backend}: expected /root default, got {config['cwd']}"
|
||||
)
|
||||
|
||||
def test_docker_default_cwd_maps_current_directory_when_enabled(self):
|
||||
def test_docker_default_cwd_maps_current_directory_when_enabled(self, monkeypatch):
|
||||
"""Docker should use /workspace when cwd mounting is explicitly enabled."""
|
||||
with patch("tools.terminal_tool.os.getcwd", return_value="/home/user/project"):
|
||||
with patch.dict(os.environ, {
|
||||
"TERMINAL_ENV": "docker",
|
||||
"TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE": "true",
|
||||
}, clear=False):
|
||||
env = os.environ.copy()
|
||||
env.pop("TERMINAL_CWD", None)
|
||||
with patch.dict(os.environ, env, clear=True):
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/workspace"
|
||||
assert config["host_cwd"] == "/home/user/project"
|
||||
monkeypatch.setattr("tools.terminal_tool.os.getcwd", lambda: "/home/user/project")
|
||||
monkeypatch.setenv("TERMINAL_ENV", "docker")
|
||||
monkeypatch.setenv("TERMINAL_DOCKER_MOUNT_CWD_TO_WORKSPACE", "true")
|
||||
monkeypatch.delenv("TERMINAL_CWD", raising=False)
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/workspace"
|
||||
assert config["host_cwd"] == "/home/user/project"
|
||||
|
||||
def test_local_backend_uses_getcwd(self):
|
||||
def test_local_backend_uses_getcwd(self, monkeypatch):
|
||||
"""Local backend should use os.getcwd(), not /root."""
|
||||
with patch.dict(os.environ, {"TERMINAL_ENV": "local"}, clear=False):
|
||||
env = os.environ.copy()
|
||||
env.pop("TERMINAL_CWD", None)
|
||||
with patch.dict(os.environ, env, clear=True):
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == os.getcwd()
|
||||
monkeypatch.setenv("TERMINAL_ENV", "local")
|
||||
monkeypatch.delenv("TERMINAL_CWD", raising=False)
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == os.getcwd()
|
||||
|
||||
def test_create_environment_passes_docker_host_cwd_and_flag(self, monkeypatch):
|
||||
"""Docker host cwd and mount flag should reach DockerEnvironment."""
|
||||
@@ -173,18 +154,16 @@ class TestCwdHandling:
|
||||
assert captured["host_cwd"] == "/home/user/project"
|
||||
assert captured["auto_mount_cwd"] is True
|
||||
|
||||
def test_ssh_preserves_home_paths(self):
|
||||
def test_ssh_preserves_home_paths(self, monkeypatch):
|
||||
"""SSH backend should NOT replace /home/ paths (they're valid remotely)."""
|
||||
with patch.dict(os.environ, {
|
||||
"TERMINAL_ENV": "ssh",
|
||||
"TERMINAL_CWD": "/home/remote-user/work",
|
||||
"TERMINAL_SSH_HOST": "example.com",
|
||||
"TERMINAL_SSH_USER": "user",
|
||||
}):
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/home/remote-user/work", (
|
||||
"SSH backend should preserve /home/ paths"
|
||||
)
|
||||
monkeypatch.setenv("TERMINAL_ENV", "ssh")
|
||||
monkeypatch.setenv("TERMINAL_CWD", "/home/remote-user/work")
|
||||
monkeypatch.setenv("TERMINAL_SSH_HOST", "example.com")
|
||||
monkeypatch.setenv("TERMINAL_SSH_USER", "user")
|
||||
config = _tt_mod._get_env_config()
|
||||
assert config["cwd"] == "/home/remote-user/work", (
|
||||
"SSH backend should preserve /home/ paths"
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
@@ -194,12 +173,8 @@ class TestCwdHandling:
|
||||
class TestEphemeralDiskCheck:
|
||||
"""Verify ephemeral_disk is only passed when modal supports it."""
|
||||
|
||||
def test_ephemeral_disk_skipped_when_unsupported(self):
|
||||
def test_ephemeral_disk_skipped_when_unsupported(self, monkeypatch):
|
||||
"""If modal.Sandbox.create doesn't have ephemeral_disk param, skip it."""
|
||||
# Mock the modal import and Sandbox.create signature
|
||||
mock_modal = MagicMock()
|
||||
mock_sandbox_create = MagicMock()
|
||||
# Simulate a signature WITHOUT ephemeral_disk
|
||||
import inspect
|
||||
mock_params = {
|
||||
"args": inspect.Parameter("args", inspect.Parameter.VAR_POSITIONAL),
|
||||
@@ -208,26 +183,25 @@ class TestEphemeralDiskCheck:
|
||||
"cpu": inspect.Parameter("cpu", inspect.Parameter.KEYWORD_ONLY),
|
||||
"memory": inspect.Parameter("memory", inspect.Parameter.KEYWORD_ONLY),
|
||||
}
|
||||
mock_sig = inspect.Signature(parameters=list(mock_params.values()))
|
||||
|
||||
with patch.dict(os.environ, {"TERMINAL_ENV": "modal"}):
|
||||
config = _tt_mod._get_env_config()
|
||||
# The config has container_disk default of 51200
|
||||
disk = config.get("container_disk", 51200)
|
||||
assert disk > 0, "disk should default to > 0"
|
||||
monkeypatch.setenv("TERMINAL_ENV", "modal")
|
||||
config = _tt_mod._get_env_config()
|
||||
# The config has container_disk default of 51200
|
||||
disk = config.get("container_disk", 51200)
|
||||
assert disk > 0, "disk should default to > 0"
|
||||
|
||||
# Simulate the version check logic from terminal_tool.py
|
||||
sandbox_kwargs = {}
|
||||
if disk > 0:
|
||||
try:
|
||||
if "ephemeral_disk" in mock_params:
|
||||
sandbox_kwargs["ephemeral_disk"] = disk
|
||||
except Exception:
|
||||
pass
|
||||
# Simulate the version check logic from terminal_tool.py
|
||||
sandbox_kwargs = {}
|
||||
if disk > 0:
|
||||
try:
|
||||
if "ephemeral_disk" in mock_params:
|
||||
sandbox_kwargs["ephemeral_disk"] = disk
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
assert "ephemeral_disk" not in sandbox_kwargs, (
|
||||
"ephemeral_disk should not be set when Sandbox.create doesn't support it"
|
||||
)
|
||||
assert "ephemeral_disk" not in sandbox_kwargs, (
|
||||
"ephemeral_disk should not be set when Sandbox.create doesn't support it"
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
|
||||
274
tests/tools/test_tool_call_parsers.py
Normal file
274
tests/tools/test_tool_call_parsers.py
Normal file
@@ -0,0 +1,274 @@
|
||||
"""
|
||||
Tests for environments/tool_call_parsers/ — client-side tool call parsers.
|
||||
|
||||
These parsers extract structured tool_calls from raw model output text.
|
||||
Used in Phase 2 (VLLM/generate) where the server returns raw tokens.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure repo root is importable
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
try:
|
||||
from environments.tool_call_parsers import (
|
||||
ParseResult,
|
||||
ToolCallParser,
|
||||
get_parser,
|
||||
list_parsers,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("atroposlib not installed", allow_module_level=True)
|
||||
|
||||
|
||||
# ─── Registry tests ─────────────────────────────────────────────────────
|
||||
|
||||
class TestParserRegistry:
|
||||
def test_list_parsers_returns_nonempty(self):
|
||||
parsers = list_parsers()
|
||||
assert len(parsers) > 0
|
||||
|
||||
def test_hermes_parser_registered(self):
|
||||
parsers = list_parsers()
|
||||
assert "hermes" in parsers
|
||||
|
||||
def test_get_parser_returns_instance(self):
|
||||
parser = get_parser("hermes")
|
||||
assert isinstance(parser, ToolCallParser)
|
||||
|
||||
def test_get_parser_unknown_raises(self):
|
||||
with pytest.raises(KeyError):
|
||||
get_parser("nonexistent_parser_xyz")
|
||||
|
||||
def test_all_registered_parsers_instantiate(self):
|
||||
"""Every registered parser should be importable and instantiable."""
|
||||
for name in list_parsers():
|
||||
parser = get_parser(name)
|
||||
assert isinstance(parser, ToolCallParser)
|
||||
assert hasattr(parser, "parse")
|
||||
|
||||
|
||||
# ─── Hermes parser tests ────────────────────────────────────────────────
|
||||
|
||||
class TestHermesParser:
|
||||
@pytest.fixture
|
||||
def parser(self):
|
||||
return get_parser("hermes")
|
||||
|
||||
def test_no_tool_call(self, parser):
|
||||
text = "Hello, I can help you with that."
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert content == text
|
||||
assert tool_calls is None
|
||||
|
||||
def test_single_tool_call(self, parser):
|
||||
text = '<tool_call>{"name": "terminal", "arguments": {"command": "ls -la"}}</tool_call>'
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function.name == "terminal"
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
assert args["command"] == "ls -la"
|
||||
|
||||
def test_tool_call_with_surrounding_text(self, parser):
|
||||
text = 'Let me check that for you.\n<tool_call>{"name": "terminal", "arguments": {"command": "pwd"}}</tool_call>'
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function.name == "terminal"
|
||||
# Content should have the surrounding text
|
||||
if content is not None:
|
||||
assert "check that" in content or content.strip() != ""
|
||||
|
||||
def test_multiple_tool_calls(self, parser):
|
||||
text = (
|
||||
'<tool_call>{"name": "terminal", "arguments": {"command": "ls"}}</tool_call>\n'
|
||||
'<tool_call>{"name": "read_file", "arguments": {"path": "test.py"}}</tool_call>'
|
||||
)
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 2
|
||||
names = {tc.function.name for tc in tool_calls}
|
||||
assert "terminal" in names
|
||||
assert "read_file" in names
|
||||
|
||||
def test_tool_call_ids_are_unique(self, parser):
|
||||
text = (
|
||||
'<tool_call>{"name": "terminal", "arguments": {"command": "ls"}}</tool_call>\n'
|
||||
'<tool_call>{"name": "terminal", "arguments": {"command": "pwd"}}</tool_call>'
|
||||
)
|
||||
_, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
ids = [tc.id for tc in tool_calls]
|
||||
assert len(ids) == len(set(ids)), "Tool call IDs must be unique"
|
||||
|
||||
def test_empty_string(self, parser):
|
||||
content, tool_calls = parser.parse("")
|
||||
assert tool_calls is None
|
||||
|
||||
def test_malformed_json_in_tool_call(self, parser):
|
||||
text = '<tool_call>not valid json</tool_call>'
|
||||
content, tool_calls = parser.parse(text)
|
||||
# Should either return None tool_calls or handle gracefully
|
||||
# (implementation may vary — some parsers return error tool calls)
|
||||
|
||||
def test_truncated_tool_call(self, parser):
|
||||
"""Test handling of unclosed tool_call tag (model truncated mid-generation)."""
|
||||
text = '<tool_call>{"name": "terminal", "arguments": {"command": "ls -la"}'
|
||||
content, tool_calls = parser.parse(text)
|
||||
# Parser should handle truncated output gracefully
|
||||
# Either parse it successfully or return None
|
||||
|
||||
|
||||
# ─── Parse result contract tests (applies to ALL parsers) ───────────────
|
||||
|
||||
class TestParseResultContract:
|
||||
"""Ensure all parsers conform to the ParseResult contract."""
|
||||
|
||||
@pytest.fixture(params=["hermes"]) # Add more as needed
|
||||
def parser(self, request):
|
||||
return get_parser(request.param)
|
||||
|
||||
def test_returns_tuple_of_two(self, parser):
|
||||
result = parser.parse("hello world")
|
||||
assert isinstance(result, tuple)
|
||||
assert len(result) == 2
|
||||
|
||||
def test_no_tools_returns_none_tool_calls(self, parser):
|
||||
content, tool_calls = parser.parse("Just plain text, no tools.")
|
||||
assert tool_calls is None
|
||||
assert content is not None
|
||||
|
||||
def test_tool_calls_are_proper_objects(self, parser):
|
||||
"""When tool calls are found, they should be ChatCompletionMessageToolCall objects."""
|
||||
# Use hermes format since that's universal
|
||||
text = '<tool_call>{"name": "terminal", "arguments": {"command": "echo hi"}}</tool_call>'
|
||||
content, tool_calls = parser.parse(text)
|
||||
if tool_calls is not None:
|
||||
for tc in tool_calls:
|
||||
assert hasattr(tc, "id")
|
||||
assert hasattr(tc, "function")
|
||||
assert hasattr(tc.function, "name")
|
||||
assert hasattr(tc.function, "arguments")
|
||||
assert tc.id is not None
|
||||
assert isinstance(tc.function.name, str)
|
||||
assert isinstance(tc.function.arguments, str)
|
||||
|
||||
|
||||
# ─── DeepSeek V3 parser tests ───────────────────────────────────────────
|
||||
|
||||
class TestDeepSeekV3Parser:
|
||||
@pytest.fixture
|
||||
def parser(self):
|
||||
return get_parser("deepseek_v3")
|
||||
|
||||
def test_no_tool_call(self, parser):
|
||||
text = "Hello, how can I help you?"
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert content == text
|
||||
assert tool_calls is None
|
||||
|
||||
def test_single_tool_call(self, parser):
|
||||
text = (
|
||||
'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\n'
|
||||
'```json\n{"city": "London"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>'
|
||||
)
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
assert args["city"] == "London"
|
||||
|
||||
def test_multiple_tool_calls(self, parser):
|
||||
text = (
|
||||
'<｜tool▁calls▁begin｜>'
|
||||
'<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\n'
|
||||
'```json\n{"city": "London"}\n```<｜tool▁call▁end｜>'
|
||||
'<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_time\n'
|
||||
'```json\n{"timezone": "UTC"}\n```<｜tool▁call▁end｜>'
|
||||
'<｜tool▁calls▁end｜>'
|
||||
)
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 2, f"Expected 2 tool calls, got {len(tool_calls)}"
|
||||
names = [tc.function.name for tc in tool_calls]
|
||||
assert "get_weather" in names
|
||||
assert "get_time" in names
|
||||
|
||||
def test_tool_call_with_preceding_text(self, parser):
|
||||
text = (
|
||||
'Let me check that for you.\n'
|
||||
'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>terminal\n'
|
||||
'```json\n{"command": "ls"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>'
|
||||
)
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
|
||||
|
||||
# ─── Mistral parser tests ───────────────────────────────────────────────
|
||||
|
||||
class TestMistralParser:
|
||||
@pytest.fixture
|
||||
def parser(self):
|
||||
return get_parser("mistral")
|
||||
|
||||
def test_no_tool_call(self, parser):
|
||||
text = "Hello, how can I help you?"
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert content == text
|
||||
assert tool_calls is None
|
||||
|
||||
def test_pre_v11_single_tool_call(self, parser):
|
||||
text = '[TOOL_CALLS] [{"name": "func", "arguments": {"key": "val"}}]'
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function.name == "func"
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
assert args["key"] == "val"
|
||||
|
||||
def test_pre_v11_nested_json(self, parser):
|
||||
text = '[TOOL_CALLS] [{"name": "func", "arguments": {"nested": {"deep": true}}}]'
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function.name == "func"
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
assert args["nested"]["deep"] is True
|
||||
|
||||
def test_v11_single_tool_call(self, parser):
|
||||
text = '[TOOL_CALLS]get_weather{"city": "London"}'
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
assert args["city"] == "London"
|
||||
|
||||
def test_v11_multiple_tool_calls(self, parser):
|
||||
text = '[TOOL_CALLS]func1{"a": 1}[TOOL_CALLS]func2{"b": 2}'
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 2
|
||||
names = [tc.function.name for tc in tool_calls]
|
||||
assert "func1" in names
|
||||
assert "func2" in names
|
||||
|
||||
def test_preceding_text_preserved(self, parser):
|
||||
text = 'Hello[TOOL_CALLS]func{"a": 1}'
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert content == "Hello"
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function.name == "func"
|
||||
|
||||
def test_malformed_json_fallback(self, parser):
|
||||
text = "[TOOL_CALLS] not valid json"
|
||||
content, tool_calls = parser.parse(text)
|
||||
assert tool_calls is None
|
||||
Reference in New Issue
Block a user