From 7368854398dd4dc375c49e5f1df982a9c1833224 Mon Sep 17 00:00:00 2001 From: Carlos Date: Thu, 9 Apr 2026 15:11:58 -0500 Subject: [PATCH] Refresh OpenRouter model catalog Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- hermes_cli/main.py | 4 +- hermes_cli/models.py | 101 +++++++++++++++++++--- tests/hermes_cli/test_model_validation.py | 18 +++- tests/hermes_cli/test_models.py | 97 ++++++++++++++++----- 4 files changed, 180 insertions(+), 40 deletions(-) diff --git a/hermes_cli/main.py b/hermes_cli/main.py index 2b919e15a..949f4f808 100644 --- a/hermes_cli/main.py +++ b/hermes_cli/main.py @@ -1142,10 +1142,10 @@ def _model_flow_openrouter(config, current_model=""): print() from hermes_cli.models import model_ids, get_pricing_for_provider - openrouter_models = model_ids() + openrouter_models = model_ids(force_refresh=True) # Fetch live pricing (non-blocking — returns empty dict on failure) - pricing = get_pricing_for_provider("openrouter") + pricing = get_pricing_for_provider("openrouter", force_refresh=True) selected = _prompt_model_selection(openrouter_models, current_model=current_model, pricing=pricing) if selected: diff --git a/hermes_cli/models.py b/hermes_cli/models.py index ac73fa211..32d08e39f 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -24,18 +24,19 @@ COPILOT_REASONING_EFFORTS_O_SERIES = ["low", "medium", "high"] GITHUB_MODELS_BASE_URL = COPILOT_BASE_URL GITHUB_MODELS_CATALOG_URL = COPILOT_MODELS_URL +# Fallback OpenRouter snapshot used when the live catalog is unavailable. # (model_id, display description shown in menus) OPENROUTER_MODELS: list[tuple[str, str]] = [ ("anthropic/claude-opus-4.6", "recommended"), ("anthropic/claude-sonnet-4.6", ""), - ("qwen/qwen3.6-plus:free", "free"), + ("qwen/qwen3.6-plus", ""), ("anthropic/claude-sonnet-4.5", ""), ("anthropic/claude-haiku-4.5", ""), ("openai/gpt-5.4", ""), ("openai/gpt-5.4-mini", ""), ("xiaomi/mimo-v2-pro", ""), ("openai/gpt-5.3-codex", ""), - ("google/gemini-3-pro-preview", ""), + ("google/gemini-3-pro-image-preview", ""), ("google/gemini-3-flash-preview", ""), ("google/gemini-3.1-pro-preview", ""), ("google/gemini-3.1-flash-lite-preview", ""), @@ -47,7 +48,7 @@ OPENROUTER_MODELS: list[tuple[str, str]] = [ ("z-ai/glm-5.1", ""), ("z-ai/glm-5-turbo", ""), ("moonshotai/kimi-k2.5", ""), - ("x-ai/grok-4.20-beta", ""), + ("x-ai/grok-4.20", ""), ("nvidia/nemotron-3-super-120b-a12b", ""), ("nvidia/nemotron-3-super-120b-a12b:free", "free"), ("arcee-ai/trinity-large-preview:free", "free"), @@ -56,6 +57,8 @@ OPENROUTER_MODELS: list[tuple[str, str]] = [ ("openai/gpt-5.4-nano", ""), ] +_openrouter_catalog_cache: list[tuple[str, str]] | None = None + _PROVIDER_MODELS: dict[str, list[str]] = { "nous": [ "anthropic/claude-opus-4.6", @@ -530,15 +533,79 @@ _PROVIDER_ALIASES = { } -def model_ids() -> list[str]: +def _openrouter_model_is_free(pricing: Any) -> bool: + """Return True when both prompt and completion pricing are zero.""" + if not isinstance(pricing, dict): + return False + try: + return float(pricing.get("prompt", "0")) == 0 and float(pricing.get("completion", "0")) == 0 + except (TypeError, ValueError): + return False + + +def fetch_openrouter_models( + timeout: float = 8.0, + *, + force_refresh: bool = False, +) -> list[tuple[str, str]]: + """Return the curated OpenRouter picker list, refreshed from the live catalog when possible.""" + global _openrouter_catalog_cache + + if _openrouter_catalog_cache is not None and not force_refresh: + return list(_openrouter_catalog_cache) + + fallback = list(OPENROUTER_MODELS) + preferred_ids = [mid for mid, _ in fallback] + + try: + req = urllib.request.Request( + "https://openrouter.ai/api/v1/models", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + payload = json.loads(resp.read().decode()) + except Exception: + return list(_openrouter_catalog_cache or fallback) + + live_items = payload.get("data", []) + if not isinstance(live_items, list): + return list(_openrouter_catalog_cache or fallback) + + live_by_id: dict[str, dict[str, Any]] = {} + for item in live_items: + if not isinstance(item, dict): + continue + mid = str(item.get("id") or "").strip() + if not mid: + continue + live_by_id[mid] = item + + curated: list[tuple[str, str]] = [] + for preferred_id in preferred_ids: + live_item = live_by_id.get(preferred_id) + if live_item is None: + continue + desc = "free" if _openrouter_model_is_free(live_item.get("pricing")) else "" + curated.append((preferred_id, desc)) + + if not curated: + return list(_openrouter_catalog_cache or fallback) + + first_id, _ = curated[0] + curated[0] = (first_id, "recommended") + _openrouter_catalog_cache = curated + return list(curated) + + +def model_ids(*, force_refresh: bool = False) -> list[str]: """Return just the OpenRouter model-id strings.""" - return [mid for mid, _ in OPENROUTER_MODELS] + return [mid for mid, _ in fetch_openrouter_models(force_refresh=force_refresh)] -def menu_labels() -> list[str]: +def menu_labels(*, force_refresh: bool = False) -> list[str]: """Return display labels like 'anthropic/claude-opus-4.6 (recommended)'.""" labels = [] - for mid, desc in OPENROUTER_MODELS: + for mid, desc in fetch_openrouter_models(force_refresh=force_refresh): labels.append(f"{mid} ({desc})" if desc else mid) return labels @@ -727,13 +794,14 @@ def _resolve_nous_pricing_credentials() -> tuple[str, str]: return ("", "") -def get_pricing_for_provider(provider: str) -> dict[str, dict[str, str]]: +def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]: """Return live pricing for providers that support it (openrouter, nous).""" normalized = normalize_provider(provider) if normalized == "openrouter": return fetch_models_with_pricing( api_key=_resolve_openrouter_api_key(), base_url="https://openrouter.ai/api", + force_refresh=force_refresh, ) if normalized == "nous": api_key, base_url = _resolve_nous_pricing_credentials() @@ -746,6 +814,7 @@ def get_pricing_for_provider(provider: str) -> dict[str, dict[str, str]]: return fetch_models_with_pricing( api_key=api_key, base_url=stripped, + force_refresh=force_refresh, ) return {} @@ -854,7 +923,11 @@ def _get_custom_base_url() -> str: return "" -def curated_models_for_provider(provider: Optional[str]) -> list[tuple[str, str]]: +def curated_models_for_provider( + provider: Optional[str], + *, + force_refresh: bool = False, +) -> list[tuple[str, str]]: """Return ``(model_id, description)`` tuples for a provider's model list. Tries to fetch the live model list from the provider's API first, @@ -863,7 +936,7 @@ def curated_models_for_provider(provider: Optional[str]) -> list[tuple[str, str] """ normalized = normalize_provider(provider) if normalized == "openrouter": - return list(OPENROUTER_MODELS) + return fetch_openrouter_models(force_refresh=force_refresh) # Try live API first (Codex, Nous, etc. all support /models) live = provider_model_ids(normalized) @@ -982,12 +1055,12 @@ def _find_openrouter_slug(model_name: str) -> Optional[str]: return None # Exact match (already has provider/ prefix) - for mid, _ in OPENROUTER_MODELS: + for mid in model_ids(): if name_lower == mid.lower(): return mid # Try matching just the model part (after the /) - for mid, _ in OPENROUTER_MODELS: + for mid in model_ids(): if "/" in mid: _, model_part = mid.split("/", 1) if name_lower == model_part.lower(): @@ -1101,7 +1174,7 @@ def _resolve_copilot_catalog_api_key() -> str: return "" -def provider_model_ids(provider: Optional[str]) -> list[str]: +def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False) -> list[str]: """Return the best known model catalog for a provider. Tries live API endpoints for providers that support them (Codex, Nous), @@ -1109,7 +1182,7 @@ def provider_model_ids(provider: Optional[str]) -> list[str]: """ normalized = normalize_provider(provider) if normalized == "openrouter": - return model_ids() + return model_ids(force_refresh=force_refresh) if normalized == "openai-codex": from hermes_cli.codex_models import get_codex_model_ids diff --git a/tests/hermes_cli/test_model_validation.py b/tests/hermes_cli/test_model_validation.py index 3a50df014..af1d89ae8 100644 --- a/tests/hermes_cli/test_model_validation.py +++ b/tests/hermes_cli/test_model_validation.py @@ -124,7 +124,14 @@ class TestParseModelInput: class TestCuratedModelsForProvider: def test_openrouter_returns_curated_list(self): - models = curated_models_for_provider("openrouter") + with patch( + "hermes_cli.models.fetch_openrouter_models", + return_value=[ + ("anthropic/claude-opus-4.6", "recommended"), + ("qwen/qwen3.6-plus", ""), + ], + ): + models = curated_models_for_provider("openrouter") assert len(models) > 0 assert any("claude" in m[0] for m in models) @@ -169,7 +176,14 @@ class TestProviderLabel: class TestProviderModelIds: def test_openrouter_returns_curated_list(self): - ids = provider_model_ids("openrouter") + with patch( + "hermes_cli.models.fetch_openrouter_models", + return_value=[ + ("anthropic/claude-opus-4.6", "recommended"), + ("qwen/qwen3.6-plus", ""), + ], + ): + ids = provider_model_ids("openrouter") assert len(ids) > 0 assert all("/" in mid for mid in ids) diff --git a/tests/hermes_cli/test_models.py b/tests/hermes_cli/test_models.py index 776256f0f..ee92eb672 100644 --- a/tests/hermes_cli/test_models.py +++ b/tests/hermes_cli/test_models.py @@ -3,7 +3,7 @@ from unittest.mock import patch, MagicMock from hermes_cli.models import ( - OPENROUTER_MODELS, menu_labels, model_ids, detect_provider_for_model, + OPENROUTER_MODELS, fetch_openrouter_models, menu_labels, model_ids, detect_provider_for_model, filter_nous_free_models, _NOUS_ALLOWED_FREE_MODELS, is_nous_free_tier, partition_nous_models_by_tier, check_nous_free_tier, clear_nous_free_tier_cache, @@ -11,43 +11,57 @@ from hermes_cli.models import ( ) import hermes_cli.models as _models_mod +LIVE_OPENROUTER_MODELS = [ + ("anthropic/claude-opus-4.6", "recommended"), + ("qwen/qwen3.6-plus", ""), + ("nvidia/nemotron-3-super-120b-a12b:free", "free"), +] + class TestModelIds: def test_returns_non_empty_list(self): - ids = model_ids() + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + ids = model_ids() assert isinstance(ids, list) assert len(ids) > 0 - def test_ids_match_models_list(self): - ids = model_ids() - expected = [mid for mid, _ in OPENROUTER_MODELS] + def test_ids_match_fetched_catalog(self): + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + ids = model_ids() + expected = [mid for mid, _ in LIVE_OPENROUTER_MODELS] assert ids == expected def test_all_ids_contain_provider_slash(self): """Model IDs should follow the provider/model format.""" - for mid in model_ids(): - assert "/" in mid, f"Model ID '{mid}' missing provider/ prefix" + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + for mid in model_ids(): + assert "/" in mid, f"Model ID '{mid}' missing provider/ prefix" def test_no_duplicate_ids(self): - ids = model_ids() + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + ids = model_ids() assert len(ids) == len(set(ids)), "Duplicate model IDs found" class TestMenuLabels: def test_same_length_as_model_ids(self): - assert len(menu_labels()) == len(model_ids()) + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + assert len(menu_labels()) == len(model_ids()) def test_first_label_marked_recommended(self): - labels = menu_labels() + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + labels = menu_labels() assert "recommended" in labels[0].lower() def test_each_label_contains_its_model_id(self): - for label, mid in zip(menu_labels(), model_ids()): - assert mid in label, f"Label '{label}' doesn't contain model ID '{mid}'" + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + for label, mid in zip(menu_labels(), model_ids()): + assert mid in label, f"Label '{label}' doesn't contain model ID '{mid}'" def test_non_recommended_labels_have_no_tag(self): """Only the first model should have (recommended).""" - labels = menu_labels() + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + labels = menu_labels() for label in labels[1:]: assert "recommended" not in label.lower(), f"Unexpected 'recommended' in '{label}'" @@ -65,30 +79,65 @@ class TestOpenRouterModels: assert len(OPENROUTER_MODELS) >= 5 +class TestFetchOpenRouterModels: + def test_live_fetch_recomputes_free_tags(self, monkeypatch): + class _Resp: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self): + return b'{"data":[{"id":"anthropic/claude-opus-4.6","pricing":{"prompt":"0.000015","completion":"0.000075"}},{"id":"qwen/qwen3.6-plus","pricing":{"prompt":"0.000000325","completion":"0.00000195"}},{"id":"nvidia/nemotron-3-super-120b-a12b:free","pricing":{"prompt":"0","completion":"0"}}]}' + + monkeypatch.setattr(_models_mod, "_openrouter_catalog_cache", None) + with patch("hermes_cli.models.urllib.request.urlopen", return_value=_Resp()): + models = fetch_openrouter_models(force_refresh=True) + + assert models == [ + ("anthropic/claude-opus-4.6", "recommended"), + ("qwen/qwen3.6-plus", ""), + ("nvidia/nemotron-3-super-120b-a12b:free", "free"), + ] + + def test_falls_back_to_static_snapshot_on_fetch_failure(self, monkeypatch): + monkeypatch.setattr(_models_mod, "_openrouter_catalog_cache", None) + with patch("hermes_cli.models.urllib.request.urlopen", side_effect=OSError("boom")): + models = fetch_openrouter_models(force_refresh=True) + + assert models == OPENROUTER_MODELS + + class TestFindOpenrouterSlug: def test_exact_match(self): from hermes_cli.models import _find_openrouter_slug - assert _find_openrouter_slug("anthropic/claude-opus-4.6") == "anthropic/claude-opus-4.6" + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + assert _find_openrouter_slug("anthropic/claude-opus-4.6") == "anthropic/claude-opus-4.6" def test_bare_name_match(self): from hermes_cli.models import _find_openrouter_slug - result = _find_openrouter_slug("claude-opus-4.6") + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + result = _find_openrouter_slug("claude-opus-4.6") assert result == "anthropic/claude-opus-4.6" def test_case_insensitive(self): from hermes_cli.models import _find_openrouter_slug - result = _find_openrouter_slug("Anthropic/Claude-Opus-4.6") + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + result = _find_openrouter_slug("Anthropic/Claude-Opus-4.6") assert result is not None def test_unknown_returns_none(self): from hermes_cli.models import _find_openrouter_slug - assert _find_openrouter_slug("totally-fake-model-xyz") is None + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + assert _find_openrouter_slug("totally-fake-model-xyz") is None class TestDetectProviderForModel: def test_anthropic_model_detected(self): """claude-opus-4-6 should resolve to anthropic provider.""" - result = detect_provider_for_model("claude-opus-4-6", "openai-codex") + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + result = detect_provider_for_model("claude-opus-4-6", "openai-codex") assert result is not None assert result[0] == "anthropic" @@ -105,7 +154,8 @@ class TestDetectProviderForModel: def test_openrouter_slug_match(self): """Models in the OpenRouter catalog should be found.""" - result = detect_provider_for_model("anthropic/claude-opus-4.6", "openai-codex") + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + result = detect_provider_for_model("anthropic/claude-opus-4.6", "openai-codex") assert result is not None assert result[0] == "openrouter" assert result[1] == "anthropic/claude-opus-4.6" @@ -119,18 +169,21 @@ class TestDetectProviderForModel: ): monkeypatch.delenv(env_var, raising=False) """Bare model names should get mapped to full OpenRouter slugs.""" - result = detect_provider_for_model("claude-opus-4.6", "openai-codex") + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + result = detect_provider_for_model("claude-opus-4.6", "openai-codex") assert result is not None # Should find it on OpenRouter with full slug assert result[1] == "anthropic/claude-opus-4.6" def test_unknown_model_returns_none(self): """Completely unknown model names should return None.""" - assert detect_provider_for_model("nonexistent-model-xyz", "openai-codex") is None + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + assert detect_provider_for_model("nonexistent-model-xyz", "openai-codex") is None def test_aggregator_not_suggested(self): """nous/openrouter should never be auto-suggested as target provider.""" - result = detect_provider_for_model("claude-opus-4-6", "openai-codex") + with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS): + result = detect_provider_for_model("claude-opus-4-6", "openai-codex") assert result is not None assert result[0] not in ("nous",) # nous has claude models but shouldn't be suggested