feat(openrouter): add response caching support (#19132)
Enable OpenRouter's response caching feature (beta) via X-OpenRouter-Cache
headers. When enabled, identical API requests return cached responses for
free (zero billing), reducing both latency and cost.
Configuration via config.yaml:
openrouter:
response_cache: true # default: on
response_cache_ttl: 300 # 1-86400 seconds
Changes:
- Add openrouter config section to DEFAULT_CONFIG (response_cache + TTL)
- Add build_or_headers() in auxiliary_client.py that builds attribution
headers plus optional cache headers based on config
- Replace inline _OR_HEADERS dicts with build_or_headers() at all 5 sites:
run_agent.py __init__, _apply_client_headers_for_base_url(), and
auxiliary_client.py _try_openrouter() + _to_async_client()
- Add _check_openrouter_cache_status() method to AIAgent that reads
X-OpenRouter-Cache-Status from streaming response headers and logs
HIT/MISS status
- Document in cli-config.yaml.example
- Add 28 tests (22 unit + 6 integration)
Ref: https://openrouter.ai/docs/guides/features/response-caching
This commit is contained in:
40
run_agent.py
40
run_agent.py
@@ -1258,6 +1258,10 @@ class AIAgent:
|
||||
# after each API call. Accessed by /usage slash command.
|
||||
self._rate_limit_state: Optional["RateLimitState"] = None
|
||||
|
||||
# OpenRouter response cache hit counter — incremented when
|
||||
# X-OpenRouter-Cache-Status: HIT is seen in streaming response headers.
|
||||
self._or_cache_hits: int = 0
|
||||
|
||||
# Centralized logging — agent.log (INFO+) and errors.log (WARNING+)
|
||||
# both live under ~/.hermes/logs/. Idempotent, so gateway mode
|
||||
# (which creates a new AIAgent per message) won't duplicate handlers.
|
||||
@@ -1421,11 +1425,8 @@ class AIAgent:
|
||||
client_kwargs["args"] = self.acp_args
|
||||
effective_base = base_url
|
||||
if base_url_host_matches(effective_base, "openrouter.ai"):
|
||||
client_kwargs["default_headers"] = {
|
||||
"HTTP-Referer": "https://hermes-agent.nousresearch.com",
|
||||
"X-OpenRouter-Title": "Hermes Agent",
|
||||
"X-OpenRouter-Categories": "productivity,cli-agent",
|
||||
}
|
||||
from agent.auxiliary_client import build_or_headers
|
||||
client_kwargs["default_headers"] = build_or_headers()
|
||||
elif base_url_host_matches(effective_base, "api.routermint.com"):
|
||||
client_kwargs["default_headers"] = _routermint_headers()
|
||||
elif base_url_host_matches(effective_base, "api.githubcopilot.com"):
|
||||
@@ -4580,6 +4581,28 @@ class AIAgent:
|
||||
"""Return the last captured RateLimitState, or None."""
|
||||
return self._rate_limit_state
|
||||
|
||||
def _check_openrouter_cache_status(self, http_response: Any) -> None:
|
||||
"""Read X-OpenRouter-Cache-Status from response headers and log it.
|
||||
|
||||
Increments ``_or_cache_hits`` on HIT so callers can report savings.
|
||||
"""
|
||||
if http_response is None:
|
||||
return
|
||||
headers = getattr(http_response, "headers", None)
|
||||
if not headers:
|
||||
return
|
||||
try:
|
||||
status = headers.get("x-openrouter-cache-status")
|
||||
if not status:
|
||||
return
|
||||
if status.upper() == "HIT":
|
||||
self._or_cache_hits += 1
|
||||
logger.info("OpenRouter response cache HIT (total: %d)", self._or_cache_hits)
|
||||
else:
|
||||
logger.debug("OpenRouter response cache %s", status.upper())
|
||||
except Exception:
|
||||
pass # Never let header parsing break the agent loop
|
||||
|
||||
def get_activity_summary(self) -> dict:
|
||||
"""Return a snapshot of the agent's current activity for diagnostics.
|
||||
|
||||
@@ -6157,10 +6180,10 @@ class AIAgent:
|
||||
return True
|
||||
|
||||
def _apply_client_headers_for_base_url(self, base_url: str) -> None:
|
||||
from agent.auxiliary_client import _AI_GATEWAY_HEADERS, _OR_HEADERS
|
||||
from agent.auxiliary_client import _AI_GATEWAY_HEADERS, build_or_headers
|
||||
|
||||
if base_url_host_matches(base_url, "openrouter.ai"):
|
||||
self._client_kwargs["default_headers"] = dict(_OR_HEADERS)
|
||||
self._client_kwargs["default_headers"] = build_or_headers()
|
||||
elif base_url_host_matches(base_url, "ai-gateway.vercel.sh"):
|
||||
self._client_kwargs["default_headers"] = dict(_AI_GATEWAY_HEADERS)
|
||||
elif base_url_host_matches(base_url, "api.routermint.com"):
|
||||
@@ -6780,6 +6803,9 @@ class AIAgent:
|
||||
# response via .response before any chunks are consumed.
|
||||
self._capture_rate_limits(getattr(stream, "response", None))
|
||||
|
||||
# Log OpenRouter response cache status when present.
|
||||
self._check_openrouter_cache_status(getattr(stream, "response", None))
|
||||
|
||||
content_parts: list = []
|
||||
tool_calls_acc: dict = {}
|
||||
tool_gen_notified: set = set()
|
||||
|
||||
Reference in New Issue
Block a user