common/hermes
2
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Harden setup provider flows

Browse Source 
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Carlos
2026-04-09 13:48:36 -05:00
committed by Teknium
parent 45034b746f
commit 38ccd9eb95
8 changed files with 354 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/hermes_cli/test_api_key_providers.py
View File

@@ -633,6 +633,7 @@ class TestHasAnyProviderConfigured:
hermes_home.mkdir()
monkeypatch.setattr(config_module, "get_env_path", lambda: hermes_home / ".env")
monkeypatch.setattr(config_module, "get_hermes_home", lambda: hermes_home)
monkeypatch.setattr("hermes_cli.copilot_auth.resolve_copilot_token", lambda: ("", ""))
# Clear all provider env vars so earlier checks don't short-circuit
_all_vars = {"OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY",
"ANTHROPIC_TOKEN", "OPENAI_BASE_URL"}
@@ -727,6 +728,7 @@ class TestHasAnyProviderConfigured:
monkeypatch.setattr(config_module, "get_env_path", lambda: hermes_home / ".env")
monkeypatch.setattr(config_module, "get_hermes_home", lambda: hermes_home)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
monkeypatch.setattr("hermes_cli.copilot_auth.resolve_copilot_token", lambda: ("", ""))
_all_vars = {"OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY",
"ANTHROPIC_TOKEN", "OPENAI_BASE_URL"}
for pconfig in PROVIDER_REGISTRY.values():

57
tests/hermes_cli/test_setup.py
View File

@@ -142,6 +142,31 @@ def test_setup_custom_providers_synced(tmp_path, monkeypatch):
assert reloaded.get("custom_providers") == [{"name": "Local", "base_url": "http://localhost:8080/v1"}]
def test_setup_syncs_custom_provider_removal_from_disk(tmp_path, monkeypatch):
"""Removing the last custom provider in model setup should persist."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
config = load_config()
config["custom_providers"] = [{"name": "Local", "base_url": "http://localhost:8080/v1"}]
save_config(config)
def fake_select():
cfg = load_config()
cfg["model"] = {"provider": "openrouter", "default": "anthropic/claude-opus-4.6"}
cfg["custom_providers"] = []
save_config(cfg)
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
setup_model_provider(config)
save_config(config)
reloaded = load_config()
assert reloaded.get("custom_providers") == []
def test_setup_cancel_preserves_existing_config(tmp_path, monkeypatch):
"""When the user cancels provider selection, existing config is preserved."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
@@ -201,6 +226,38 @@ def test_setup_keyboard_interrupt_gracefully_handled(tmp_path, monkeypatch):
setup_model_provider(config)
def test_select_provider_and_model_warns_if_named_custom_provider_disappears(
tmp_path, monkeypatch, capsys
):
"""If a saved custom provider is deleted mid-selection, show a warning instead of silently doing nothing."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
cfg = load_config()
cfg["custom_providers"] = [{"name": "Local", "base_url": "http://localhost:8080/v1"}]
save_config(cfg)
def fake_prompt_provider_choice(choices, default=0):
current = load_config()
current["custom_providers"] = []
save_config(current)
return next(i for i, label in enumerate(choices) if label.startswith("Local (localhost:8080/v1)"))
monkeypatch.setattr("hermes_cli.auth.resolve_provider", lambda provider: None)
monkeypatch.setattr("hermes_cli.main._prompt_provider_choice", fake_prompt_provider_choice)
monkeypatch.setattr(
"hermes_cli.main._model_flow_named_custom",
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("named custom flow should not run")),
)
from hermes_cli.main import select_provider_and_model
select_provider_and_model()
out = capsys.readouterr().out
assert "selected saved custom provider is no longer available" in out
def test_codex_setup_uses_runtime_access_token_for_live_model_list(tmp_path, monkeypatch):
"""Codex model list fetching uses the runtime access token."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))

33
tests/hermes_cli/test_setup_model_provider.py
View File

@@ -230,6 +230,39 @@ def test_setup_same_provider_fallback_can_add_another_credential(tmp_path, monke
assert config.get("credential_pool_strategies", {}).get("openrouter") == "fill_first"
def test_setup_same_provider_single_credential_keeps_existing_rotation_strategy(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
save_env_value("OPENROUTER_API_KEY", "or-key")
_write_model_config("openrouter", "", "anthropic/claude-opus-4.6")
config = load_config()
config["credential_pool_strategies"] = {"openrouter": "round_robin"}
save_config(config)
class _Entry:
def __init__(self, label):
self.label = label
class _Pool:
def entries(self):
return [_Entry("primary")]
def fake_select():
pass
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
_stub_tts(monkeypatch)
monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "")
monkeypatch.setattr("agent.credential_pool.load_pool", lambda provider: _Pool())
monkeypatch.setattr("agent.auxiliary_client.get_available_vision_backends", lambda: [])
setup_model_provider(config)
assert config.get("credential_pool_strategies", {}).get("openrouter") == "round_robin"
def test_setup_pool_step_shows_manual_vs_auto_detected_counts(tmp_path, monkeypatch, capsys):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)

109
tests/hermes_cli/test_setup_noninteractive.py
View File

@@ -4,6 +4,7 @@ from argparse import Namespace
from unittest.mock import MagicMock, patch
import pytest
from hermes_cli.config import DEFAULT_CONFIG, load_config, save_config
def _make_setup_args(**overrides):
@@ -34,6 +35,36 @@ def _make_chat_args(**overrides):
class TestNonInteractiveSetup:
"""Verify setup paths exit cleanly in headless/non-interactive environments."""
def test_cmd_setup_allows_noninteractive_flag_without_tty(self):
"""The CLI entrypoint should not block --non-interactive before setup.py handles it."""
from hermes_cli.main import cmd_setup
args = _make_setup_args(non_interactive=True)
with (
patch("hermes_cli.setup.run_setup_wizard") as mock_run_setup,
patch("sys.stdin") as mock_stdin,
):
mock_stdin.isatty.return_value = False
cmd_setup(args)
mock_run_setup.assert_called_once_with(args)
def test_cmd_setup_defers_no_tty_handling_to_setup_wizard(self):
"""Bare `hermes setup` should reach setup.py, which prints headless guidance."""
from hermes_cli.main import cmd_setup
args = _make_setup_args(non_interactive=False)
with (
patch("hermes_cli.setup.run_setup_wizard") as mock_run_setup,
patch("sys.stdin") as mock_stdin,
):
mock_stdin.isatty.return_value = False
cmd_setup(args)
mock_run_setup.assert_called_once_with(args)
def test_non_interactive_flag_skips_wizard(self, capsys):
"""--non-interactive should print guidance and not enter the wizard."""
from hermes_cli.setup import run_setup_wizard
@@ -72,6 +103,26 @@ class TestNonInteractiveSetup:
out = capsys.readouterr().out
assert "hermes config set model.provider custom" in out
def test_reset_flag_rewrites_config_before_noninteractive_exit(self, tmp_path, monkeypatch, capsys):
"""--reset should rewrite config.yaml even when the wizard cannot run interactively."""
from hermes_cli.setup import run_setup_wizard
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
cfg = load_config()
cfg["model"] = {"provider": "custom", "base_url": "http://localhost:8080/v1", "default": "llama3"}
cfg["agent"]["max_turns"] = 12
save_config(cfg)
args = _make_setup_args(non_interactive=True, reset=True)
run_setup_wizard(args)
reloaded = load_config()
assert reloaded["model"] == DEFAULT_CONFIG["model"]
assert reloaded["agent"]["max_turns"] == DEFAULT_CONFIG["agent"]["max_turns"]
out = capsys.readouterr().out
assert "Configuration reset to defaults." in out
def test_chat_first_run_headless_skips_setup_prompt(self, capsys):
"""Bare `hermes` should not prompt for input when no provider exists and stdin is headless."""
from hermes_cli.main import cmd_chat
@@ -117,7 +168,7 @@ class TestNonInteractiveSetup:
side_effect=lambda key: "sk-test" if key == "OPENROUTER_API_KEY" else "",
),
patch("hermes_cli.auth.get_active_provider", return_value=None),
patch.object(setup_mod, "prompt_choice", return_value=4),
patch.object(setup_mod, "prompt_choice", return_value=3),
patch.object(
setup_mod,
"SETUP_SECTIONS",
@@ -137,3 +188,59 @@ class TestNonInteractiveSetup:
terminal_section.assert_called_once_with(config)
tts_section.assert_not_called()
def test_returning_user_menu_does_not_show_separator_rows(self, tmp_path):
"""Returning-user menu should only show selectable actions."""
from hermes_cli import setup as setup_mod
args = _make_setup_args()
captured = {}
def fake_prompt_choice(question, choices, default=0):
captured["question"] = question
captured["choices"] = list(choices)
return len(choices) - 1
with (
patch.object(setup_mod, "ensure_hermes_home"),
patch.object(setup_mod, "load_config", return_value={}),
patch.object(setup_mod, "get_hermes_home", return_value=tmp_path),
patch.object(setup_mod, "is_interactive_stdin", return_value=True),
patch.object(
setup_mod,
"get_env_value",
side_effect=lambda key: "sk-test" if key == "OPENROUTER_API_KEY" else "",
),
patch("hermes_cli.auth.get_active_provider", return_value=None),
patch.object(setup_mod, "prompt_choice", side_effect=fake_prompt_choice),
):
setup_mod.run_setup_wizard(args)
assert captured["question"] == "What would you like to do?"
assert "---" not in captured["choices"]
assert captured["choices"] == [
"Quick Setup - configure missing items only",
"Full Setup - reconfigure everything",
"Model & Provider",
"Terminal Backend",
"Messaging Platforms (Gateway)",
"Tools",
"Agent Settings",
"Exit",
]
def test_main_accepts_tts_setup_section(self, monkeypatch):
"""`hermes setup tts` should parse and dispatch like other setup sections."""
from hermes_cli import main as main_mod
received = {}
def fake_cmd_setup(args):
received["section"] = args.section
monkeypatch.setattr(main_mod, "cmd_setup", fake_cmd_setup)
monkeypatch.setattr("sys.argv", ["hermes", "setup", "tts"])
main_mod.main()
assert received["section"] == "tts"

106
tests/hermes_cli/test_terminal_menu_fallbacks.py Normal file
View File

@@ -0,0 +1,106 @@
"""Regression tests for numbered fallbacks when TerminalMenu cannot initialize."""
import subprocess
import sys
import types
from hermes_cli.config import load_config, save_config
class _BrokenTerminalMenu:
def __init__(self, *args, **kwargs):
raise subprocess.CalledProcessError(2, ["tput", "clear"])
def test_prompt_model_selection_falls_back_on_terminalmenu_runtime_error(monkeypatch):
from hermes_cli.auth import _prompt_model_selection
monkeypatch.setitem(
sys.modules,
"simple_term_menu",
types.SimpleNamespace(TerminalMenu=_BrokenTerminalMenu),
)
responses = iter(["2"])
monkeypatch.setattr("builtins.input", lambda _prompt="": next(responses))
selected = _prompt_model_selection(["model-a", "model-b"])
assert selected == "model-b"
def test_prompt_reasoning_effort_falls_back_on_terminalmenu_runtime_error(monkeypatch):
from hermes_cli.main import _prompt_reasoning_effort_selection
monkeypatch.setitem(
sys.modules,
"simple_term_menu",
types.SimpleNamespace(TerminalMenu=_BrokenTerminalMenu),
)
responses = iter(["3"])
monkeypatch.setattr("builtins.input", lambda _prompt="": next(responses))
selected = _prompt_reasoning_effort_selection(["low", "medium", "high"], current_effort="")
assert selected == "high"
def test_remove_custom_provider_falls_back_on_terminalmenu_runtime_error(tmp_path, monkeypatch):
from hermes_cli.main import _remove_custom_provider
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setitem(
sys.modules,
"simple_term_menu",
types.SimpleNamespace(TerminalMenu=_BrokenTerminalMenu),
)
cfg = load_config()
cfg["custom_providers"] = [
{"name": "Local A", "base_url": "http://localhost:8001/v1"},
{"name": "Local B", "base_url": "http://localhost:8002/v1"},
]
save_config(cfg)
responses = iter(["1"])
monkeypatch.setattr("builtins.input", lambda _prompt="": next(responses))
_remove_custom_provider(cfg)
reloaded = load_config()
assert reloaded["custom_providers"] == [
{"name": "Local B", "base_url": "http://localhost:8002/v1"},
]
def test_named_custom_provider_model_picker_falls_back_on_terminalmenu_runtime_error(tmp_path, monkeypatch):
from hermes_cli.main import _model_flow_named_custom
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setitem(
sys.modules,
"simple_term_menu",
types.SimpleNamespace(TerminalMenu=_BrokenTerminalMenu),
)
monkeypatch.setattr("hermes_cli.models.fetch_api_models", lambda *args, **kwargs: ["model-a", "model-b"])
monkeypatch.setattr("hermes_cli.auth.deactivate_provider", lambda: None)
cfg = load_config()
save_config(cfg)
responses = iter(["2"])
monkeypatch.setattr("builtins.input", lambda _prompt="": next(responses))
_model_flow_named_custom(
cfg,
{
"name": "Local",
"base_url": "http://localhost:8000/v1",
"api_key": "",
"model": "",
},
)
reloaded = load_config()
assert reloaded["model"]["provider"] == "custom"
assert reloaded["model"]["base_url"] == "http://localhost:8000/v1"
assert reloaded["model"]["default"] == "model-b"