common/hermes
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: add managed tool gateway and Nous subscription support

Browse Source 
- add managed modal and gateway-backed tool integrations\n- improve CLI setup, auth, and configuration for subscriber flows\n- expand tests and docs for managed tool support
This commit is contained in:
Robin Fernandes
2026-03-26 15:27:27 -07:00
parent cbf195e806
commit 95dc9aaa75
44 changed files with 4567 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
tools/browser_providers/browserbase.py
View File

@@ -2,14 +2,57 @@
import logging
import os
import threading
import uuid
from typing import Dict
from typing import Any, Dict, Optional
import requests
from tools.browser_providers.base import CloudBrowserProvider
from tools.managed_tool_gateway import resolve_managed_tool_gateway
logger = logging.getLogger(__name__)
_pending_create_keys: Dict[str, str] = {}
_pending_create_keys_lock = threading.Lock()
def _get_or_create_pending_create_key(task_id: str) -> str:
with _pending_create_keys_lock:
existing = _pending_create_keys.get(task_id)
if existing:
return existing
created = f"browserbase-session-create:{uuid.uuid4().hex}"
_pending_create_keys[task_id] = created
return created
def _clear_pending_create_key(task_id: str) -> None:
with _pending_create_keys_lock:
_pending_create_keys.pop(task_id, None)
def _should_preserve_pending_create_key(response: requests.Response) -> bool:
if response.status_code >= 500:
return True
if response.status_code != 409:
return False
try:
payload = response.json()
except Exception:
return False
if not isinstance(payload, dict):
return False
error = payload.get("error")
if not isinstance(error, dict):
return False
message = str(error.get("message") or "").lower()
return "already in progress" in message
class BrowserbaseProvider(CloudBrowserProvider):
@@ -19,28 +62,46 @@ class BrowserbaseProvider(CloudBrowserProvider):
return "Browserbase"
def is_configured(self) -> bool:
return bool(
os.environ.get("BROWSERBASE_API_KEY")
and os.environ.get("BROWSERBASE_PROJECT_ID")
)
return self._get_config_or_none() is not None
# ------------------------------------------------------------------
# Session lifecycle
# ------------------------------------------------------------------
def _get_config(self) -> Dict[str, str]:
def _get_config_or_none(self) -> Optional[Dict[str, Any]]:
api_key = os.environ.get("BROWSERBASE_API_KEY")
project_id = os.environ.get("BROWSERBASE_PROJECT_ID")
if not api_key or not project_id:
if api_key and project_id:
return {
"api_key": api_key,
"project_id": project_id,
"base_url": os.environ.get("BROWSERBASE_BASE_URL", "https://api.browserbase.com").rstrip("/"),
"managed_mode": False,
}
managed = resolve_managed_tool_gateway("browserbase")
if managed is None:
return None
return {
"api_key": managed.nous_user_token,
"project_id": "managed",
"base_url": managed.gateway_origin.rstrip("/"),
"managed_mode": True,
}
def _get_config(self) -> Dict[str, Any]:
config = self._get_config_or_none()
if config is None:
raise ValueError(
"BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID environment "
"variables are required. Get your credentials at "
"https://browserbase.com"
"Browserbase requires either direct BROWSERBASE_API_KEY/BROWSERBASE_PROJECT_ID credentials "
"or a managed Browserbase gateway configuration."
)
return {"api_key": api_key, "project_id": project_id}
return config
def create_session(self, task_id: str) -> Dict[str, object]:
config = self._get_config()
managed_mode = bool(config.get("managed_mode"))
# Optional env-var knobs
enable_proxies = os.environ.get("BROWSERBASE_PROXIES", "true").lower() != "false"
@@ -80,8 +141,11 @@ class BrowserbaseProvider(CloudBrowserProvider):
"Content-Type": "application/json",
"X-BB-API-Key": config["api_key"],
}
if managed_mode:
headers["X-Idempotency-Key"] = _get_or_create_pending_create_key(task_id)
response = requests.post(
"https://api.browserbase.com/v1/sessions",
f"{config['base_url']}/v1/sessions",
headers=headers,
json=session_config,
timeout=30,
@@ -91,7 +155,7 @@ class BrowserbaseProvider(CloudBrowserProvider):
keepalive_fallback = False
# Handle 402 — paid features unavailable
if response.status_code == 402:
if response.status_code == 402 and not managed_mode:
if enable_keep_alive:
keepalive_fallback = True
logger.warning(
@@ -100,7 +164,7 @@ class BrowserbaseProvider(CloudBrowserProvider):
)
session_config.pop("keepAlive", None)
response = requests.post(
"https://api.browserbase.com/v1/sessions",
f"{config['base_url']}/v1/sessions",
headers=headers,
json=session_config,
timeout=30,
@@ -114,20 +178,25 @@ class BrowserbaseProvider(CloudBrowserProvider):
)
session_config.pop("proxies", None)
response = requests.post(
"https://api.browserbase.com/v1/sessions",
f"{config['base_url']}/v1/sessions",
headers=headers,
json=session_config,
timeout=30,
)
if not response.ok:
if managed_mode and not _should_preserve_pending_create_key(response):
_clear_pending_create_key(task_id)
raise RuntimeError(
f"Failed to create Browserbase session: "
f"{response.status_code} {response.text}"
)
session_data = response.json()
if managed_mode:
_clear_pending_create_key(task_id)
session_name = f"hermes_{task_id}_{uuid.uuid4().hex[:8]}"
external_call_id = response.headers.get("x-external-call-id") if managed_mode else None
if enable_proxies and not proxies_fallback:
features_enabled["proxies"] = True
@@ -146,6 +215,7 @@ class BrowserbaseProvider(CloudBrowserProvider):
"bb_session_id": session_data["id"],
"cdp_url": session_data["connectUrl"],
"features": features_enabled,
"external_call_id": external_call_id,
}
def close_session(self, session_id: str) -> bool:
@@ -157,7 +227,7 @@ class BrowserbaseProvider(CloudBrowserProvider):
try:
response = requests.post(
f"https://api.browserbase.com/v1/sessions/{session_id}",
f"{config['base_url']}/v1/sessions/{session_id}",
headers={
"X-BB-API-Key": config["api_key"],
"Content-Type": "application/json",
@@ -184,20 +254,19 @@ class BrowserbaseProvider(CloudBrowserProvider):
return False
def emergency_cleanup(self, session_id: str) -> None:
api_key = os.environ.get("BROWSERBASE_API_KEY")
project_id = os.environ.get("BROWSERBASE_PROJECT_ID")
if not api_key or not project_id:
config = self._get_config_or_none()
if config is None:
logger.warning("Cannot emergency-cleanup Browserbase session %s — missing credentials", session_id)
return
try:
requests.post(
f"https://api.browserbase.com/v1/sessions/{session_id}",
f"{config['base_url']}/v1/sessions/{session_id}",
headers={
"X-BB-API-Key": api_key,
"X-BB-API-Key": config["api_key"],
"Content-Type": "application/json",
},
json={
"projectId": project_id,
"projectId": config["project_id"],
"status": "REQUEST_RELEASE",
},
timeout=5,

40
tools/browser_tool.py
View File

@@ -78,6 +78,7 @@ except Exception:
from tools.browser_providers.base import CloudBrowserProvider
from tools.browser_providers.browserbase import BrowserbaseProvider
from tools.browser_providers.browser_use import BrowserUseProvider
from tools.tool_backend_helpers import normalize_browser_cloud_provider
logger = logging.getLogger(__name__)
@@ -235,7 +236,9 @@ def _get_cloud_provider() -> Optional[CloudBrowserProvider]:
"""Return the configured cloud browser provider, or None for local mode.
Reads ``config["browser"]["cloud_provider"]`` once and caches the result
for the process lifetime. If unset → local mode (None).
for the process lifetime. An explicit ``local`` provider disables cloud
fallback. If unset, fall back to Browserbase when direct or managed
Browserbase credentials are available.
"""
global _cached_cloud_provider, _cloud_provider_resolved
if _cloud_provider_resolved:
@@ -249,14 +252,45 @@ def _get_cloud_provider() -> Optional[CloudBrowserProvider]:
import yaml
with open(config_path) as f:
cfg = yaml.safe_load(f) or {}
provider_key = cfg.get("browser", {}).get("cloud_provider")
browser_cfg = cfg.get("browser", {})
provider_key = None
if isinstance(browser_cfg, dict) and "cloud_provider" in browser_cfg:
provider_key = normalize_browser_cloud_provider(
browser_cfg.get("cloud_provider")
)
if provider_key == "local":
_cached_cloud_provider = None
return None
if provider_key and provider_key in _PROVIDER_REGISTRY:
_cached_cloud_provider = _PROVIDER_REGISTRY[provider_key]()
except Exception as e:
logger.debug("Could not read cloud_provider from config: %s", e)
if _cached_cloud_provider is None:
fallback_provider = BrowserbaseProvider()
if fallback_provider.is_configured():
_cached_cloud_provider = fallback_provider
return _cached_cloud_provider
def _get_browserbase_config_or_none() -> Optional[Dict[str, Any]]:
"""Return Browserbase direct or managed config, or None when unavailable."""
return BrowserbaseProvider()._get_config_or_none()
def _get_browserbase_config() -> Dict[str, Any]:
"""Return Browserbase config or raise when neither direct nor managed mode is available."""
return BrowserbaseProvider()._get_config()
def _is_local_mode() -> bool:
"""Return True when the browser tool will use a local browser backend."""
if _get_cdp_override():
return False
return _get_cloud_provider() is None
def _socket_safe_tmpdir() -> str:
"""Return a short temp directory path suitable for Unix domain sockets.
@@ -1845,7 +1879,7 @@ if __name__ == "__main__":
print(" Install: npm install -g agent-browser && agent-browser install --with-deps")
if _cp is not None and not _cp.is_configured():
print(f" - {_cp.provider_name()} credentials not configured")
print(" Tip: remove cloud_provider from config to use free local mode instead")
print(" Tip: set browser.cloud_provider to 'local' to use free local mode instead")
print("\n📋 Available Browser Tools:")
for schema in BROWSER_TOOL_SCHEMAS:

3
tools/code_execution_tool.py
View File

@@ -757,7 +757,8 @@ def build_execute_code_schema(enabled_sandbox_tools: set = None) -> dict:
f"Available via `from hermes_tools import ...`:\n\n"
f"{tool_lines}\n\n"
"Limits: 5-minute timeout, 50KB stdout cap, max 50 tool calls per script. "
"terminal() is foreground-only (no background or pty).\n\n"
"terminal() is foreground-only (no background or pty). "
"If the session uses a cloud sandbox backend, treat it as resumable task state rather than a durable always-on machine.\n\n"
"Print your final result to stdout. Use Python stdlib (json, re, math, csv, "
"datetime, collections, etc.) for processing between tool calls.\n\n"
"Also available (no import needed — built into hermes_tools):\n"

282
tools/environments/managed_modal.py Normal file
View File

@@ -0,0 +1,282 @@
"""Managed Modal environment backed by tool-gateway."""
from __future__ import annotations
import json
import logging
import os
import requests
import time
import uuid
from typing import Any, Dict, Optional
from tools.environments.base import BaseEnvironment
from tools.interrupt import is_interrupted
from tools.managed_tool_gateway import resolve_managed_tool_gateway
logger = logging.getLogger(__name__)
def _request_timeout_env(name: str, default: float) -> float:
try:
value = float(os.getenv(name, str(default)))
return value if value > 0 else default
except (TypeError, ValueError):
return default
class ManagedModalEnvironment(BaseEnvironment):
"""Gateway-owned Modal sandbox with Hermes-compatible execute/cleanup."""
_CONNECT_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CONNECT_TIMEOUT_SECONDS", 1.0)
_POLL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_POLL_READ_TIMEOUT_SECONDS", 5.0)
_CANCEL_READ_TIMEOUT_SECONDS = _request_timeout_env("TERMINAL_MANAGED_MODAL_CANCEL_READ_TIMEOUT_SECONDS", 5.0)
def __init__(
self,
image: str,
cwd: str = "/root",
timeout: int = 60,
modal_sandbox_kwargs: Optional[Dict[str, Any]] = None,
persistent_filesystem: bool = True,
task_id: str = "default",
):
super().__init__(cwd=cwd, timeout=timeout)
gateway = resolve_managed_tool_gateway("modal")
if gateway is None:
raise ValueError("Managed Modal requires a configured tool gateway and Nous user token")
self._gateway_origin = gateway.gateway_origin.rstrip("/")
self._nous_user_token = gateway.nous_user_token
self._task_id = task_id
self._persistent = persistent_filesystem
self._image = image
self._sandbox_kwargs = dict(modal_sandbox_kwargs or {})
self._create_idempotency_key = str(uuid.uuid4())
self._sandbox_id = self._create_sandbox()
def execute(self, command: str, cwd: str = "", *,
timeout: int | None = None,
stdin_data: str | None = None) -> dict:
exec_command, sudo_stdin = self._prepare_command(command)
# When a sudo password is present, inject it via a shell-level pipe
# (same approach as the direct ModalEnvironment) since the gateway
# cannot pipe subprocess stdin directly.
if sudo_stdin is not None:
import shlex
exec_command = (
f"printf '%s\\n' {shlex.quote(sudo_stdin.rstrip())} | {exec_command}"
)
exec_cwd = cwd or self.cwd
effective_timeout = timeout or self.timeout
exec_id = str(uuid.uuid4())
payload: Dict[str, Any] = {
"execId": exec_id,
"command": exec_command,
"cwd": exec_cwd,
"timeoutMs": int(effective_timeout * 1000),
}
if stdin_data is not None:
payload["stdinData"] = stdin_data
try:
response = self._request(
"POST",
f"/v1/sandboxes/{self._sandbox_id}/execs",
json=payload,
timeout=10,
)
except Exception as exc:
return {
"output": f"Managed Modal exec failed: {exc}",
"returncode": 1,
}
if response.status_code >= 400:
return {
"output": self._format_error("Managed Modal exec failed", response),
"returncode": 1,
}
body = response.json()
status = body.get("status")
if status in {"completed", "failed", "cancelled", "timeout"}:
return {
"output": body.get("output", ""),
"returncode": body.get("returncode", 1),
}
if body.get("execId") != exec_id:
return {
"output": "Managed Modal exec start did not return the expected exec id",
"returncode": 1,
}
poll_interval = 0.25
deadline = time.monotonic() + effective_timeout + 10
while time.monotonic() < deadline:
if is_interrupted():
self._cancel_exec(exec_id)
return {
"output": "[Command interrupted - Modal sandbox exec cancelled]",
"returncode": 130,
}
try:
status_response = self._request(
"GET",
f"/v1/sandboxes/{self._sandbox_id}/execs/{exec_id}",
timeout=(self._CONNECT_TIMEOUT_SECONDS, self._POLL_READ_TIMEOUT_SECONDS),
)
except Exception as exc:
return {
"output": f"Managed Modal exec poll failed: {exc}",
"returncode": 1,
}
if status_response.status_code == 404:
return {
"output": "Managed Modal exec not found",
"returncode": 1,
}
if status_response.status_code >= 400:
return {
"output": self._format_error("Managed Modal exec poll failed", status_response),
"returncode": 1,
}
status_body = status_response.json()
status = status_body.get("status")
if status in {"completed", "failed", "cancelled", "timeout"}:
return {
"output": status_body.get("output", ""),
"returncode": status_body.get("returncode", 1),
}
time.sleep(poll_interval)
self._cancel_exec(exec_id)
return {
"output": f"Managed Modal exec timed out after {effective_timeout}s",
"returncode": 124,
}
def cleanup(self):
if not getattr(self, "_sandbox_id", None):
return
try:
self._request(
"POST",
f"/v1/sandboxes/{self._sandbox_id}/terminate",
json={
"snapshotBeforeTerminate": self._persistent,
},
timeout=60,
)
except Exception as exc:
logger.warning("Managed Modal cleanup failed: %s", exc)
finally:
self._sandbox_id = None
def _create_sandbox(self) -> str:
cpu = self._coerce_number(self._sandbox_kwargs.get("cpu"), 1)
memory = self._coerce_number(
self._sandbox_kwargs.get("memoryMiB", self._sandbox_kwargs.get("memory")),
5120,
)
disk = self._coerce_number(
self._sandbox_kwargs.get("ephemeral_disk", self._sandbox_kwargs.get("diskMiB")),
None,
)
create_payload = {
"image": self._image,
"cwd": self.cwd,
"cpu": cpu,
"memoryMiB": memory,
"timeoutMs": 3_600_000,
"idleTimeoutMs": max(300_000, int(self.timeout * 1000)),
"persistentFilesystem": self._persistent,
"logicalKey": self._task_id,
}
if disk is not None:
create_payload["diskMiB"] = disk
response = self._request(
"POST",
"/v1/sandboxes",
json=create_payload,
timeout=60,
extra_headers={
"x-idempotency-key": self._create_idempotency_key,
},
)
if response.status_code >= 400:
raise RuntimeError(self._format_error("Managed Modal create failed", response))
body = response.json()
sandbox_id = body.get("id")
if not isinstance(sandbox_id, str) or not sandbox_id:
raise RuntimeError("Managed Modal create did not return a sandbox id")
return sandbox_id
def _request(self, method: str, path: str, *,
json: Dict[str, Any] | None = None,
timeout: int = 30,
extra_headers: Dict[str, str] | None = None) -> requests.Response:
headers = {
"Authorization": f"Bearer {self._nous_user_token}",
"Content-Type": "application/json",
}
if extra_headers:
headers.update(extra_headers)
return requests.request(
method,
f"{self._gateway_origin}{path}",
headers=headers,
json=json,
timeout=timeout,
)
def _cancel_exec(self, exec_id: str) -> None:
try:
self._request(
"POST",
f"/v1/sandboxes/{self._sandbox_id}/execs/{exec_id}/cancel",
timeout=(self._CONNECT_TIMEOUT_SECONDS, self._CANCEL_READ_TIMEOUT_SECONDS),
)
except Exception as exc:
logger.warning("Managed Modal exec cancel failed: %s", exc)
@staticmethod
def _coerce_number(value: Any, default: float) -> float:
try:
if value is None:
return default
return float(value)
except (TypeError, ValueError):
return default
@staticmethod
def _format_error(prefix: str, response: requests.Response) -> str:
try:
payload = response.json()
if isinstance(payload, dict):
message = payload.get("error") or payload.get("message") or payload.get("code")
if isinstance(message, str) and message:
return f"{prefix}: {message}"
return f"{prefix}: {json.dumps(payload, ensure_ascii=False)}"
except Exception:
pass
text = response.text.strip()
if text:
return f"{prefix}: {text}"
return f"{prefix}: HTTP {response.status_code}"

149
tools/environments/modal.py
View File

@@ -20,6 +20,7 @@ from tools.interrupt import is_interrupted
logger = logging.getLogger(__name__)
_SNAPSHOT_STORE = get_hermes_home() / "modal_snapshots.json"
_DIRECT_SNAPSHOT_NAMESPACE = "direct"
def _load_snapshots() -> Dict[str, str]:
@@ -38,12 +39,72 @@ def _save_snapshots(data: Dict[str, str]) -> None:
_SNAPSHOT_STORE.write_text(json.dumps(data, indent=2))
class _AsyncWorker:
"""Background thread with its own event loop for async-safe swe-rex calls.
def _direct_snapshot_key(task_id: str) -> str:
return f"{_DIRECT_SNAPSHOT_NAMESPACE}:{task_id}"
Allows sync code to submit async coroutines and block for results,
even when called from inside another running event loop (e.g. Atropos).
"""
def _get_snapshot_restore_candidate(task_id: str) -> tuple[str | None, bool]:
"""Return a snapshot id for direct Modal restore and whether the key is legacy."""
snapshots = _load_snapshots()
namespaced_key = _direct_snapshot_key(task_id)
snapshot_id = snapshots.get(namespaced_key)
if isinstance(snapshot_id, str) and snapshot_id:
return snapshot_id, False
legacy_snapshot_id = snapshots.get(task_id)
if isinstance(legacy_snapshot_id, str) and legacy_snapshot_id:
return legacy_snapshot_id, True
return None, False
def _store_direct_snapshot(task_id: str, snapshot_id: str) -> None:
"""Persist the direct Modal snapshot id under the direct namespace."""
snapshots = _load_snapshots()
snapshots[_direct_snapshot_key(task_id)] = snapshot_id
snapshots.pop(task_id, None)
_save_snapshots(snapshots)
def _delete_direct_snapshot(task_id: str, snapshot_id: str | None = None) -> None:
"""Remove direct Modal snapshot entries for a task, including legacy keys."""
snapshots = _load_snapshots()
updated = False
for key in (_direct_snapshot_key(task_id), task_id):
value = snapshots.get(key)
if value is None:
continue
if snapshot_id is None or value == snapshot_id:
snapshots.pop(key, None)
updated = True
if updated:
_save_snapshots(snapshots)
def _resolve_modal_image(image_spec: Any) -> Any:
"""Convert registry references or snapshot ids into Modal image objects."""
import modal as _modal
if not isinstance(image_spec, str):
return image_spec
if image_spec.startswith("im-"):
return _modal.Image.from_id(image_spec)
return _modal.Image.from_registry(
image_spec,
setup_dockerfile_commands=[
"RUN rm -rf /usr/local/lib/python*/site-packages/pip* 2>/dev/null; "
"python -m ensurepip --upgrade --default-pip 2>/dev/null || true",
],
)
class _AsyncWorker:
"""Background thread with its own event loop for async-safe swe-rex calls."""
def __init__(self):
self._loop: Optional[asyncio.AbstractEventLoop] = None
@@ -101,42 +162,20 @@ class ModalEnvironment(BaseEnvironment):
sandbox_kwargs = dict(modal_sandbox_kwargs or {})
# If persistent, try to restore from a previous snapshot
restored_image = None
restored_snapshot_id = None
restored_from_legacy_key = False
if self._persistent:
snapshot_id = _load_snapshots().get(self._task_id)
if snapshot_id:
try:
import modal
restored_image = modal.Image.from_id(snapshot_id)
logger.info("Modal: restoring from snapshot %s", snapshot_id[:20])
except Exception as e:
logger.warning("Modal: failed to restore snapshot, using base image: %s", e)
restored_image = None
restored_snapshot_id, restored_from_legacy_key = _get_snapshot_restore_candidate(self._task_id)
if restored_snapshot_id:
logger.info("Modal: restoring from snapshot %s", restored_snapshot_id[:20])
effective_image = restored_image if restored_image else image
# Pre-build a modal.Image with pip fix for Modal's legacy image builder.
# Some task images have broken pip; fix via ensurepip before Modal uses it.
import modal as _modal
if isinstance(effective_image, str):
effective_image = _modal.Image.from_registry(
effective_image,
setup_dockerfile_commands=[
"RUN rm -rf /usr/local/lib/python*/site-packages/pip* 2>/dev/null; "
"python -m ensurepip --upgrade --default-pip 2>/dev/null || true",
],
)
# Start the async worker thread and create the deployment on it
# so all gRPC channels are bound to the worker's event loop.
self._worker.start()
from swerex.deployment.modal import ModalDeployment
async def _create_and_start():
async def _create_and_start(image_spec: Any):
deployment = ModalDeployment(
image=effective_image,
image=image_spec,
startup_timeout=180.0,
runtime_timeout=3600.0,
deployment_timeout=3600.0,
@@ -146,7 +185,30 @@ class ModalEnvironment(BaseEnvironment):
await deployment.start()
return deployment
self._deployment = self._worker.run_coroutine(_create_and_start())
try:
target_image_spec = restored_snapshot_id or image
try:
effective_image = _resolve_modal_image(target_image_spec)
self._deployment = self._worker.run_coroutine(_create_and_start(effective_image))
except Exception as exc:
if not restored_snapshot_id:
raise
logger.warning(
"Modal: failed to restore snapshot %s, retrying with base image: %s",
restored_snapshot_id[:20],
exc,
)
_delete_direct_snapshot(self._task_id, restored_snapshot_id)
base_image = _resolve_modal_image(image)
self._deployment = self._worker.run_coroutine(_create_and_start(base_image))
else:
if restored_snapshot_id and restored_from_legacy_key:
_store_direct_snapshot(self._task_id, restored_snapshot_id)
logger.info("Modal: migrated legacy snapshot entry for task %s", self._task_id)
except Exception:
self._worker.stop()
raise
def execute(self, command: str, cwd: str = "", *,
timeout: int | None = None,
@@ -160,7 +222,7 @@ class ModalEnvironment(BaseEnvironment):
exec_command, sudo_stdin = self._prepare_command(command)
# Modal sandboxes execute commands via the Modal SDK and cannot pipe
# subprocess stdin directly the way a local Popen can. When a sudo
# subprocess stdin directly the way a local Popen can. When a sudo
# password is present, use a shell-level pipe from printf so that the
# password feeds sudo -S without appearing as an echo argument embedded
# in the shell string.
@@ -175,7 +237,6 @@ class ModalEnvironment(BaseEnvironment):
effective_cwd = cwd or self.cwd
effective_timeout = timeout or self.timeout
# Run in a background thread so we can poll for interrupts
result_holder = {"value": None, "error": None}
def _run():
@@ -191,6 +252,7 @@ class ModalEnvironment(BaseEnvironment):
merge_output_streams=True,
)
)
output = self._worker.run_coroutine(_do_execute())
result_holder["value"] = {
"output": output.stdout,
@@ -227,7 +289,7 @@ class ModalEnvironment(BaseEnvironment):
if self._persistent:
try:
sandbox = getattr(self._deployment, '_sandbox', None)
sandbox = getattr(self._deployment, "_sandbox", None)
if sandbox:
async def _snapshot():
img = await sandbox.snapshot_filesystem.aio()
@@ -239,11 +301,12 @@ class ModalEnvironment(BaseEnvironment):
snapshot_id = None
if snapshot_id:
snapshots = _load_snapshots()
snapshots[self._task_id] = snapshot_id
_save_snapshots(snapshots)
logger.info("Modal: saved filesystem snapshot %s for task %s",
snapshot_id[:20], self._task_id)
_store_direct_snapshot(self._task_id, snapshot_id)
logger.info(
"Modal: saved filesystem snapshot %s for task %s",
snapshot_id[:20],
self._task_id,
)
except Exception as e:
logger.warning("Modal: filesystem snapshot failed: %s", e)

159
tools/image_generation_tool.py
View File

@@ -32,9 +32,13 @@ import json
import logging
import os
import datetime
import threading
import uuid
from typing import Dict, Any, Optional, Union
from urllib.parse import urlencode
import fal_client
from tools.debug_helpers import DebugSession
from tools.managed_tool_gateway import resolve_managed_tool_gateway
logger = logging.getLogger(__name__)
@@ -77,6 +81,137 @@ VALID_OUTPUT_FORMATS = ["jpeg", "png"]
VALID_ACCELERATION_MODES = ["none", "regular", "high"]
_debug = DebugSession("image_tools", env_var="IMAGE_TOOLS_DEBUG")
_managed_fal_client = None
_managed_fal_client_config = None
_managed_fal_client_lock = threading.Lock()
def _resolve_managed_fal_gateway():
"""Return managed fal-queue gateway config when direct FAL credentials are absent."""
if os.getenv("FAL_KEY"):
return None
return resolve_managed_tool_gateway("fal-queue")
def _normalize_fal_queue_url_format(queue_run_origin: str) -> str:
normalized_origin = str(queue_run_origin or "").strip().rstrip("/")
if not normalized_origin:
raise ValueError("Managed FAL queue origin is required")
return f"{normalized_origin}/"
class _ManagedFalSyncClient:
"""Small per-instance wrapper around fal_client.SyncClient for managed queue hosts."""
def __init__(self, *, key: str, queue_run_origin: str):
sync_client_class = getattr(fal_client, "SyncClient", None)
if sync_client_class is None:
raise RuntimeError("fal_client.SyncClient is required for managed FAL gateway mode")
client_module = getattr(fal_client, "client", None)
if client_module is None:
raise RuntimeError("fal_client.client is required for managed FAL gateway mode")
self._queue_url_format = _normalize_fal_queue_url_format(queue_run_origin)
self._sync_client = sync_client_class(key=key)
self._http_client = getattr(self._sync_client, "_client", None)
self._maybe_retry_request = getattr(client_module, "_maybe_retry_request", None)
self._raise_for_status = getattr(client_module, "_raise_for_status", None)
self._request_handle_class = getattr(client_module, "SyncRequestHandle", None)
self._add_hint_header = getattr(client_module, "add_hint_header", None)
self._add_priority_header = getattr(client_module, "add_priority_header", None)
self._add_timeout_header = getattr(client_module, "add_timeout_header", None)
if self._http_client is None:
raise RuntimeError("fal_client.SyncClient._client is required for managed FAL gateway mode")
if self._maybe_retry_request is None or self._raise_for_status is None:
raise RuntimeError("fal_client.client request helpers are required for managed FAL gateway mode")
if self._request_handle_class is None:
raise RuntimeError("fal_client.client.SyncRequestHandle is required for managed FAL gateway mode")
def submit(
self,
application: str,
arguments: Dict[str, Any],
*,
path: str = "",
hint: Optional[str] = None,
webhook_url: Optional[str] = None,
priority: Any = None,
headers: Optional[Dict[str, str]] = None,
start_timeout: Optional[Union[int, float]] = None,
):
url = self._queue_url_format + application
if path:
url += "/" + path.lstrip("/")
if webhook_url is not None:
url += "?" + urlencode({"fal_webhook": webhook_url})
request_headers = dict(headers or {})
if hint is not None and self._add_hint_header is not None:
self._add_hint_header(hint, request_headers)
if priority is not None:
if self._add_priority_header is None:
raise RuntimeError("fal_client.client.add_priority_header is required for priority requests")
self._add_priority_header(priority, request_headers)
if start_timeout is not None:
if self._add_timeout_header is None:
raise RuntimeError("fal_client.client.add_timeout_header is required for timeout requests")
self._add_timeout_header(start_timeout, request_headers)
response = self._maybe_retry_request(
self._http_client,
"POST",
url,
json=arguments,
timeout=getattr(self._sync_client, "default_timeout", 120.0),
headers=request_headers,
)
self._raise_for_status(response)
data = response.json()
return self._request_handle_class(
request_id=data["request_id"],
response_url=data["response_url"],
status_url=data["status_url"],
cancel_url=data["cancel_url"],
client=self._http_client,
)
def _get_managed_fal_client(managed_gateway):
"""Reuse the managed FAL client so its internal httpx.Client is not leaked per call."""
global _managed_fal_client, _managed_fal_client_config
client_config = (
managed_gateway.gateway_origin.rstrip("/"),
managed_gateway.nous_user_token,
)
with _managed_fal_client_lock:
if _managed_fal_client is not None and _managed_fal_client_config == client_config:
return _managed_fal_client
_managed_fal_client = _ManagedFalSyncClient(
key=managed_gateway.nous_user_token,
queue_run_origin=managed_gateway.gateway_origin,
)
_managed_fal_client_config = client_config
return _managed_fal_client
def _submit_fal_request(model: str, arguments: Dict[str, Any]):
"""Submit a FAL request using direct credentials or the managed queue gateway."""
request_headers = {"x-idempotency-key": str(uuid.uuid4())}
managed_gateway = _resolve_managed_fal_gateway()
if managed_gateway is None:
return fal_client.submit(model, arguments=arguments, headers=request_headers)
managed_client = _get_managed_fal_client(managed_gateway)
return managed_client.submit(
model,
arguments=arguments,
headers=request_headers,
)
def _validate_parameters(
@@ -186,9 +321,9 @@ def _upscale_image(image_url: str, original_prompt: str) -> Dict[str, Any]:
# The async API (submit_async) caches a global httpx.AsyncClient via
# @cached_property, which breaks when asyncio.run() destroys the loop
# between calls (gateway thread-pool pattern).
handler = fal_client.submit(
handler = _submit_fal_request(
UPSCALER_MODEL,
arguments=upscaler_arguments
arguments=upscaler_arguments,
)
# Get the upscaled result (sync — blocks until done)
@@ -280,8 +415,10 @@ def image_generate_tool(
raise ValueError("Prompt is required and must be a non-empty string")
# Check API key availability
if not os.getenv("FAL_KEY"):
raise ValueError("FAL_KEY environment variable not set")
if not (os.getenv("FAL_KEY") or _resolve_managed_fal_gateway()):
raise ValueError(
"FAL_KEY environment variable not set and managed FAL gateway is unavailable"
)
# Validate other parameters
validated_params = _validate_parameters(
@@ -312,9 +449,9 @@ def image_generate_tool(
logger.info(" Guidance: %s", validated_params['guidance_scale'])
# Submit request to FAL.ai using sync API (avoids cached event loop issues)
handler = fal_client.submit(
handler = _submit_fal_request(
DEFAULT_MODEL,
arguments=arguments
arguments=arguments,
)
# Get the result (sync — blocks until done)
@@ -379,10 +516,12 @@ def image_generate_tool(
error_msg = f"Error generating image: {str(e)}"
logger.error("%s", error_msg, exc_info=True)
# Prepare error response - minimal format
# Include error details so callers can diagnose failures
response_data = {
"success": False,
"image": None
"image": None,
"error": str(e),
"error_type": type(e).__name__,
}
debug_call_data["error"] = error_msg
@@ -400,7 +539,7 @@ def check_fal_api_key() -> bool:
Returns:
bool: True if API key is set, False otherwise
"""
return bool(os.getenv("FAL_KEY"))
return bool(os.getenv("FAL_KEY") or _resolve_managed_fal_gateway())
def check_image_generation_requirements() -> bool:
@@ -556,7 +695,7 @@ registry.register(
schema=IMAGE_GENERATE_SCHEMA,
handler=_handle_image_generate,
check_fn=check_image_generation_requirements,
requires_env=["FAL_KEY"],
requires_env=[],
is_async=False, # Switched to sync fal_client API to fix "Event loop is closed" in gateway
emoji="🎨",
)

160
tools/managed_tool_gateway.py Normal file
View File

@@ -0,0 +1,160 @@
"""Generic managed-tool gateway helpers for Nous-hosted vendor passthroughs."""
from __future__ import annotations
import json
import os
from datetime import datetime, timezone
from dataclasses import dataclass
from typing import Callable, Optional
from hermes_cli.config import get_hermes_home
_DEFAULT_TOOL_GATEWAY_DOMAIN = "nousresearch.com"
_DEFAULT_TOOL_GATEWAY_SCHEME = "https"
_NOUS_ACCESS_TOKEN_REFRESH_SKEW_SECONDS = 120
@dataclass(frozen=True)
class ManagedToolGatewayConfig:
vendor: str
gateway_origin: str
nous_user_token: str
managed_mode: bool
def auth_json_path():
"""Return the Hermes auth store path, respecting HERMES_HOME overrides."""
return get_hermes_home() / "auth.json"
def _read_nous_provider_state() -> Optional[dict]:
try:
path = auth_json_path()
if not path.is_file():
return None
data = json.loads(path.read_text())
providers = data.get("providers", {})
if not isinstance(providers, dict):
return None
nous_provider = providers.get("nous", {})
if isinstance(nous_provider, dict):
return nous_provider
except Exception:
pass
return None
def _parse_timestamp(value: object) -> Optional[datetime]:
if not isinstance(value, str) or not value.strip():
return None
normalized = value.strip()
if normalized.endswith("Z"):
normalized = normalized[:-1] + "+00:00"
try:
parsed = datetime.fromisoformat(normalized)
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def _access_token_is_expiring(expires_at: object, skew_seconds: int) -> bool:
expires = _parse_timestamp(expires_at)
if expires is None:
return True
remaining = (expires - datetime.now(timezone.utc)).total_seconds()
return remaining <= max(0, int(skew_seconds))
def read_nous_access_token() -> Optional[str]:
"""Read a Nous Subscriber OAuth access token from auth store or env override."""
explicit = os.getenv("TOOL_GATEWAY_USER_TOKEN")
if isinstance(explicit, str) and explicit.strip():
return explicit.strip()
nous_provider = _read_nous_provider_state() or {}
access_token = nous_provider.get("access_token")
cached_token = access_token.strip() if isinstance(access_token, str) and access_token.strip() else None
if cached_token and not _access_token_is_expiring(
nous_provider.get("expires_at"),
_NOUS_ACCESS_TOKEN_REFRESH_SKEW_SECONDS,
):
return cached_token
try:
from hermes_cli.auth import resolve_nous_access_token
refreshed_token = resolve_nous_access_token(
refresh_skew_seconds=_NOUS_ACCESS_TOKEN_REFRESH_SKEW_SECONDS,
)
if isinstance(refreshed_token, str) and refreshed_token.strip():
return refreshed_token.strip()
except Exception:
pass
return cached_token
def get_tool_gateway_scheme() -> str:
"""Return configured shared gateway URL scheme."""
scheme = os.getenv("TOOL_GATEWAY_SCHEME", "").strip().lower()
if not scheme:
return _DEFAULT_TOOL_GATEWAY_SCHEME
if scheme in {"http", "https"}:
return scheme
raise ValueError("TOOL_GATEWAY_SCHEME must be 'http' or 'https'")
def build_vendor_gateway_url(vendor: str) -> str:
"""Return the gateway origin for a specific vendor."""
vendor_key = f"{vendor.upper().replace('-', '_')}_GATEWAY_URL"
explicit_vendor_url = os.getenv(vendor_key, "").strip().rstrip("/")
if explicit_vendor_url:
return explicit_vendor_url
shared_scheme = get_tool_gateway_scheme()
shared_domain = os.getenv("TOOL_GATEWAY_DOMAIN", "").strip().strip("/")
if shared_domain:
return f"{shared_scheme}://{vendor}-gateway.{shared_domain}"
return f"{shared_scheme}://{vendor}-gateway.{_DEFAULT_TOOL_GATEWAY_DOMAIN}"
def resolve_managed_tool_gateway(
vendor: str,
gateway_builder: Optional[Callable[[str], str]] = None,
token_reader: Optional[Callable[[], Optional[str]]] = None,
) -> Optional[ManagedToolGatewayConfig]:
"""Resolve shared managed-tool gateway config for a vendor."""
resolved_gateway_builder = gateway_builder or build_vendor_gateway_url
resolved_token_reader = token_reader or read_nous_access_token
gateway_origin = resolved_gateway_builder(vendor)
nous_user_token = resolved_token_reader()
if not gateway_origin or not nous_user_token:
return None
return ManagedToolGatewayConfig(
vendor=vendor,
gateway_origin=gateway_origin,
nous_user_token=nous_user_token,
managed_mode=True,
)
def is_managed_tool_gateway_ready(
vendor: str,
gateway_builder: Optional[Callable[[str], str]] = None,
token_reader: Optional[Callable[[], Optional[str]]] = None,
) -> bool:
"""Return True when gateway URL and Nous access token are available."""
return resolve_managed_tool_gateway(
vendor,
gateway_builder=gateway_builder,
token_reader=token_reader,
) is not None

107
tools/terminal_tool.py
View File

@@ -3,12 +3,12 @@
Terminal Tool Module
A terminal tool that executes commands in local, Docker, Modal, SSH, Singularity, and Daytona environments.
Supports local execution, Docker containers, and Modal cloud sandboxes.
Supports local execution, containerized backends, and Modal cloud sandboxes, including managed gateway mode.
Environment Selection (via TERMINAL_ENV environment variable):
- "local": Execute directly on the host machine (default, fastest)
- "docker": Execute in Docker containers (isolated, requires Docker)
- "modal": Execute in Modal cloud sandboxes (scalable, requires Modal account)
- "modal": Execute in Modal cloud sandboxes (direct Modal or managed gateway)
Features:
- Multiple execution backends (local, docker, modal)
@@ -16,6 +16,10 @@ Features:
- VM/container lifecycle management
- Automatic cleanup after inactivity
Cloud sandbox note:
- Persistent filesystems preserve working state across sandbox recreation
- Persistent filesystems do NOT guarantee the same live sandbox or long-running processes survive cleanup, idle reaping, or Hermes exit
Usage:
from terminal_tool import terminal_tool
@@ -50,12 +54,18 @@ logger = logging.getLogger(__name__)
from tools.interrupt import is_interrupted, _interrupt_event # noqa: F401 — re-exported
def ensure_minisweagent_on_path(_repo_root: Path | None = None) -> None:
"""Backward-compatible no-op after minisweagent_path.py removal."""
return
# =============================================================================
# Custom Singularity Environment with more space
# =============================================================================
# Singularity helpers (scratch dir, SIF cache) now live in tools/environments/singularity.py
from tools.environments.singularity import _get_scratch_dir
from tools.tool_backend_helpers import has_direct_modal_credentials, normalize_modal_mode
# Disk usage warning threshold (in GB)
@@ -361,10 +371,12 @@ from tools.environments.singularity import SingularityEnvironment as _Singularit
from tools.environments.ssh import SSHEnvironment as _SSHEnvironment
from tools.environments.docker import DockerEnvironment as _DockerEnvironment
from tools.environments.modal import ModalEnvironment as _ModalEnvironment
from tools.environments.managed_modal import ManagedModalEnvironment as _ManagedModalEnvironment
from tools.managed_tool_gateway import is_managed_tool_gateway_ready
# Tool description for LLM
TERMINAL_TOOL_DESCRIPTION = """Execute shell commands on a Linux environment. Filesystem persists between calls.
TERMINAL_TOOL_DESCRIPTION = """Execute shell commands on a Linux environment. Filesystem usually persists between calls.
Do NOT use cat/head/tail to read files — use read_file instead.
Do NOT use grep/rg/find to search — use search_files instead.
@@ -380,6 +392,7 @@ Working directory: Use 'workdir' for per-command cwd.
PTY mode: Set pty=true for interactive CLI tools (Codex, Claude Code, Python REPL).
Do NOT use vim/nano/interactive tools without pty=true — they hang without a pseudo-terminal. Pipe git output to cat if it might page.
Important: cloud sandboxes may be cleaned up, idled out, or recreated between turns. Persistent filesystem means files can resume later; it does NOT guarantee a continuously running machine or surviving background processes. Use terminal sandboxes for task work, not durable hosting.
"""
# Global state for environment lifecycle management
@@ -493,6 +506,7 @@ def _get_env_config() -> Dict[str, Any]:
return {
"env_type": env_type,
"modal_mode": normalize_modal_mode(os.getenv("TERMINAL_MODAL_MODE", "auto")),
"docker_image": os.getenv("TERMINAL_DOCKER_IMAGE", default_image),
"docker_forward_env": _parse_env_var("TERMINAL_DOCKER_FORWARD_ENV", "[]", json.loads, "valid JSON"),
"singularity_image": os.getenv("TERMINAL_SINGULARITY_IMAGE", f"docker://{default_image}"),
@@ -525,6 +539,27 @@ def _get_env_config() -> Dict[str, Any]:
}
def _get_modal_backend_state(modal_mode: object | None) -> Dict[str, Any]:
"""Resolve direct vs managed Modal backend selection."""
normalized_mode = normalize_modal_mode(modal_mode)
has_direct = has_direct_modal_credentials()
managed_ready = is_managed_tool_gateway_ready("modal")
if normalized_mode == "managed":
selected_backend = "managed" if managed_ready else None
elif normalized_mode == "direct":
selected_backend = "direct" if has_direct else None
else:
selected_backend = "direct" if has_direct else "managed" if managed_ready else None
return {
"mode": normalized_mode,
"has_direct": has_direct,
"managed_ready": managed_ready,
"selected_backend": selected_backend,
}
def _create_environment(env_type: str, image: str, cwd: str, timeout: int,
ssh_config: dict = None, container_config: dict = None,
local_config: dict = None,
@@ -590,7 +625,29 @@ def _create_environment(env_type: str, image: str, cwd: str, timeout: int,
sandbox_kwargs["ephemeral_disk"] = disk
except Exception:
pass
modal_state = _get_modal_backend_state(cc.get("modal_mode"))
if modal_state["selected_backend"] == "managed":
return _ManagedModalEnvironment(
image=image, cwd=cwd, timeout=timeout,
modal_sandbox_kwargs=sandbox_kwargs,
persistent_filesystem=persistent, task_id=task_id,
)
if modal_state["selected_backend"] != "direct":
if modal_state["mode"] == "managed":
raise ValueError(
"Modal backend is configured for managed mode, but the managed tool gateway is unavailable."
)
if modal_state["mode"] == "direct":
raise ValueError(
"Modal backend is configured for direct mode, but no direct Modal credentials/config were found."
)
raise ValueError(
"Modal backend selected but no direct Modal credentials/config or managed tool gateway was found."
)
return _ModalEnvironment(
image=image, cwd=cwd, timeout=timeout,
modal_sandbox_kwargs=sandbox_kwargs,
@@ -956,6 +1013,7 @@ def terminal_tool(
"container_memory": config.get("container_memory", 5120),
"container_disk": config.get("container_disk", 51200),
"container_persistent": config.get("container_persistent", True),
"modal_mode": config.get("modal_mode", "auto"),
"docker_volumes": config.get("docker_volumes", []),
"docker_mount_cwd_to_workspace": config.get("docker_mount_cwd_to_workspace", False),
}
@@ -1173,10 +1231,14 @@ def terminal_tool(
}, ensure_ascii=False)
except Exception as e:
import traceback
tb_str = traceback.format_exc()
logger.error("terminal_tool exception:\n%s", tb_str)
return json.dumps({
"output": "",
"exit_code": -1,
"error": f"Failed to execute command: {str(e)}",
"traceback": tb_str,
"status": "error"
}, ensure_ascii=False)
@@ -1216,18 +1278,35 @@ def check_terminal_requirements() -> bool:
return True
elif env_type == "modal":
modal_state = _get_modal_backend_state(config.get("modal_mode"))
if modal_state["selected_backend"] == "managed":
return True
if modal_state["selected_backend"] != "direct":
if modal_state["mode"] == "managed":
logger.error(
"Modal backend selected with TERMINAL_MODAL_MODE=managed, but the managed "
"tool gateway is unavailable. Configure the managed gateway or choose "
"TERMINAL_MODAL_MODE=direct/auto."
)
elif modal_state["mode"] == "direct":
logger.error(
"Modal backend selected with TERMINAL_MODAL_MODE=direct, but no direct "
"Modal credentials/config were found. Configure Modal or choose "
"TERMINAL_MODAL_MODE=managed/auto."
)
else:
logger.error(
"Modal backend selected but no direct Modal credentials/config or managed "
"tool gateway was found. Configure Modal, set up the managed gateway, "
"or choose a different TERMINAL_ENV."
)
return False
if importlib.util.find_spec("swerex") is None:
logger.error("swe-rex is required for modal terminal backend: pip install 'swe-rex[modal]'")
return False
has_token = os.getenv("MODAL_TOKEN_ID") is not None
has_config = Path.home().joinpath(".modal.toml").exists()
if not (has_token or has_config):
logger.error(
"Modal backend selected but no MODAL_TOKEN_ID environment variable "
"or ~/.modal.toml config file was found. Configure Modal or choose "
"a different TERMINAL_ENV."
)
logger.error("swe-rex is required for direct modal terminal backend: pip install 'swe-rex[modal]'")
return False
return True
elif env_type == "daytona":

41
tools/tool_backend_helpers.py Normal file
View File

@@ -0,0 +1,41 @@
"""Shared helpers for tool backend selection."""
from __future__ import annotations
import os
from pathlib import Path
_DEFAULT_BROWSER_PROVIDER = "local"
_DEFAULT_MODAL_MODE = "auto"
_VALID_MODAL_MODES = {"auto", "direct", "managed"}
def normalize_browser_cloud_provider(value: object | None) -> str:
"""Return a normalized browser provider key."""
provider = str(value or _DEFAULT_BROWSER_PROVIDER).strip().lower()
return provider or _DEFAULT_BROWSER_PROVIDER
def normalize_modal_mode(value: object | None) -> str:
"""Return a normalized modal execution mode."""
mode = str(value or _DEFAULT_MODAL_MODE).strip().lower()
if mode in _VALID_MODAL_MODES:
return mode
return _DEFAULT_MODAL_MODE
def has_direct_modal_credentials() -> bool:
"""Return True when direct Modal credentials/config are available."""
return bool(
(os.getenv("MODAL_TOKEN_ID") and os.getenv("MODAL_TOKEN_SECRET"))
or (Path.home() / ".modal.toml").exists()
)
def resolve_openai_audio_api_key() -> str:
"""Prefer the voice-tools key, but fall back to the normal OpenAI key."""
return (
os.getenv("VOICE_TOOLS_OPENAI_KEY", "")
or os.getenv("OPENAI_API_KEY", "")
).strip()

123
tools/transcription_tools.py
View File

@@ -31,6 +31,10 @@ import subprocess
import tempfile
from pathlib import Path
from typing import Optional, Dict, Any
from urllib.parse import urljoin
from tools.managed_tool_gateway import resolve_managed_tool_gateway
from tools.tool_backend_helpers import resolve_openai_audio_api_key
from hermes_constants import get_hermes_home
@@ -41,8 +45,17 @@ logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
import importlib.util as _ilu
_HAS_FASTER_WHISPER = _ilu.find_spec("faster_whisper") is not None
_HAS_OPENAI = _ilu.find_spec("openai") is not None
def _safe_find_spec(module_name: str) -> bool:
try:
return _ilu.find_spec(module_name) is not None
except (ImportError, ValueError):
return module_name in globals() or module_name in os.sys.modules
_HAS_FASTER_WHISPER = _safe_find_spec("faster_whisper")
_HAS_OPENAI = _safe_find_spec("openai")
# ---------------------------------------------------------------------------
# Constants
@@ -116,9 +129,9 @@ def is_stt_enabled(stt_config: Optional[dict] = None) -> bool:
return bool(enabled)
def _resolve_openai_api_key() -> str:
"""Prefer the voice-tools key, but fall back to the normal OpenAI key."""
return os.getenv("VOICE_TOOLS_OPENAI_KEY", "") or os.getenv("OPENAI_API_KEY", "")
def _has_openai_audio_backend() -> bool:
"""Return True when OpenAI audio can use direct credentials or the managed gateway."""
return bool(resolve_openai_audio_api_key() or resolve_managed_tool_gateway("openai-audio"))
def _find_binary(binary_name: str) -> Optional[str]:
@@ -210,7 +223,7 @@ def _get_provider(stt_config: dict) -> str:
return "none"
if provider == "openai":
if _HAS_OPENAI and _resolve_openai_api_key():
if _HAS_OPENAI and _has_openai_audio_backend():
return "openai"
logger.warning(
"STT provider 'openai' configured but no API key available"
@@ -228,7 +241,7 @@ def _get_provider(stt_config: dict) -> str:
if _HAS_OPENAI and os.getenv("GROQ_API_KEY"):
logger.info("No local STT available, using Groq Whisper API")
return "groq"
if _HAS_OPENAI and _resolve_openai_api_key():
if _HAS_OPENAI and _has_openai_audio_backend():
logger.info("No local STT available, using OpenAI Whisper API")
return "openai"
return "none"
@@ -404,19 +417,23 @@ def _transcribe_groq(file_path: str, model_name: str) -> Dict[str, Any]:
try:
from openai import OpenAI, APIError, APIConnectionError, APITimeoutError
client = OpenAI(api_key=api_key, base_url=GROQ_BASE_URL, timeout=30, max_retries=0)
try:
with open(file_path, "rb") as audio_file:
transcription = client.audio.transcriptions.create(
model=model_name,
file=audio_file,
response_format="text",
)
with open(file_path, "rb") as audio_file:
transcription = client.audio.transcriptions.create(
model=model_name,
file=audio_file,
response_format="text",
)
transcript_text = str(transcription).strip()
logger.info("Transcribed %s via Groq API (%s, %d chars)",
Path(file_path).name, model_name, len(transcript_text))
transcript_text = str(transcription).strip()
logger.info("Transcribed %s via Groq API (%s, %d chars)",
Path(file_path).name, model_name, len(transcript_text))
return {"success": True, "transcript": transcript_text, "provider": "groq"}
return {"success": True, "transcript": transcript_text, "provider": "groq"}
finally:
close = getattr(client, "close", None)
if callable(close):
close()
except PermissionError:
return {"success": False, "transcript": "", "error": f"Permission denied: {file_path}"}
@@ -437,12 +454,13 @@ def _transcribe_groq(file_path: str, model_name: str) -> Dict[str, Any]:
def _transcribe_openai(file_path: str, model_name: str) -> Dict[str, Any]:
"""Transcribe using OpenAI Whisper API (paid)."""
api_key = _resolve_openai_api_key()
if not api_key:
try:
api_key, base_url = _resolve_openai_audio_client_config()
except ValueError as exc:
return {
"success": False,
"transcript": "",
"error": "Neither VOICE_TOOLS_OPENAI_KEY nor OPENAI_API_KEY is set",
"error": str(exc),
}
if not _HAS_OPENAI:
@@ -455,20 +473,24 @@ def _transcribe_openai(file_path: str, model_name: str) -> Dict[str, Any]:
try:
from openai import OpenAI, APIError, APIConnectionError, APITimeoutError
client = OpenAI(api_key=api_key, base_url=OPENAI_BASE_URL, timeout=30, max_retries=0)
client = OpenAI(api_key=api_key, base_url=base_url, timeout=30, max_retries=0)
try:
with open(file_path, "rb") as audio_file:
transcription = client.audio.transcriptions.create(
model=model_name,
file=audio_file,
response_format="text" if model_name == "whisper-1" else "json",
)
with open(file_path, "rb") as audio_file:
transcription = client.audio.transcriptions.create(
model=model_name,
file=audio_file,
response_format="text",
)
transcript_text = _extract_transcript_text(transcription)
logger.info("Transcribed %s via OpenAI API (%s, %d chars)",
Path(file_path).name, model_name, len(transcript_text))
transcript_text = str(transcription).strip()
logger.info("Transcribed %s via OpenAI API (%s, %d chars)",
Path(file_path).name, model_name, len(transcript_text))
return {"success": True, "transcript": transcript_text, "provider": "openai"}
return {"success": True, "transcript": transcript_text, "provider": "openai"}
finally:
close = getattr(client, "close", None)
if callable(close):
close()
except PermissionError:
return {"success": False, "transcript": "", "error": f"Permission denied: {file_path}"}
@@ -554,3 +576,38 @@ def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, A
"or OPENAI_API_KEY for the OpenAI Whisper API."
),
}
def _resolve_openai_audio_client_config() -> tuple[str, str]:
"""Return direct OpenAI audio config or a managed gateway fallback."""
direct_api_key = resolve_openai_audio_api_key()
if direct_api_key:
return direct_api_key, OPENAI_BASE_URL
managed_gateway = resolve_managed_tool_gateway("openai-audio")
if managed_gateway is None:
raise ValueError(
"Neither VOICE_TOOLS_OPENAI_KEY nor OPENAI_API_KEY is set, and the managed OpenAI audio gateway is unavailable"
)
return managed_gateway.nous_user_token, urljoin(
f"{managed_gateway.gateway_origin.rstrip('/')}/", "v1"
)
def _extract_transcript_text(transcription: Any) -> str:
"""Normalize text and JSON transcription responses to a plain string."""
if isinstance(transcription, str):
return transcription.strip()
if hasattr(transcription, "text"):
value = getattr(transcription, "text")
if isinstance(value, str):
return value.strip()
if isinstance(transcription, dict):
value = transcription.get("text")
if isinstance(value, str):
return value.strip()
return str(transcription).strip()

62
tools/tts_tool.py
View File

@@ -32,11 +32,15 @@ import shutil
import subprocess
import tempfile
import threading
import uuid
from pathlib import Path
from hermes_constants import get_hermes_home
from typing import Callable, Dict, Any, Optional
from urllib.parse import urljoin
logger = logging.getLogger(__name__)
from tools.managed_tool_gateway import resolve_managed_tool_gateway
from tools.tool_backend_helpers import resolve_openai_audio_api_key
# ---------------------------------------------------------------------------
# Lazy imports -- providers are imported only when actually used to avoid
@@ -74,6 +78,7 @@ DEFAULT_ELEVENLABS_MODEL_ID = "eleven_multilingual_v2"
DEFAULT_ELEVENLABS_STREAMING_MODEL_ID = "eleven_flash_v2_5"
DEFAULT_OPENAI_MODEL = "gpt-4o-mini-tts"
DEFAULT_OPENAI_VOICE = "alloy"
DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"
DEFAULT_OUTPUT_DIR = str(get_hermes_home() / "audio_cache")
MAX_TEXT_LENGTH = 4000
@@ -233,14 +238,12 @@ def _generate_openai_tts(text: str, output_path: str, tts_config: Dict[str, Any]
Returns:
Path to the saved audio file.
"""
api_key = os.getenv("VOICE_TOOLS_OPENAI_KEY", "")
if not api_key:
raise ValueError("VOICE_TOOLS_OPENAI_KEY not set. Get one at https://platform.openai.com/api-keys")
api_key, base_url = _resolve_openai_audio_client_config()
oai_config = tts_config.get("openai", {})
model = oai_config.get("model", DEFAULT_OPENAI_MODEL)
voice = oai_config.get("voice", DEFAULT_OPENAI_VOICE)
base_url = oai_config.get("base_url", "https://api.openai.com/v1")
base_url = oai_config.get("base_url", base_url)
# Determine response format from extension
if output_path.endswith(".ogg"):
@@ -250,15 +253,21 @@ def _generate_openai_tts(text: str, output_path: str, tts_config: Dict[str, Any]
OpenAIClient = _import_openai_client()
client = OpenAIClient(api_key=api_key, base_url=base_url)
response = client.audio.speech.create(
model=model,
voice=voice,
input=text,
response_format=response_format,
)
try:
response = client.audio.speech.create(
model=model,
voice=voice,
input=text,
response_format=response_format,
extra_headers={"x-idempotency-key": str(uuid.uuid4())},
)
response.stream_to_file(output_path)
return output_path
response.stream_to_file(output_path)
return output_path
finally:
close = getattr(client, "close", None)
if callable(close):
close()
# ===========================================================================
@@ -539,7 +548,7 @@ def check_tts_requirements() -> bool:
pass
try:
_import_openai_client()
if os.getenv("VOICE_TOOLS_OPENAI_KEY"):
if _has_openai_audio_backend():
return True
except ImportError:
pass
@@ -548,6 +557,28 @@ def check_tts_requirements() -> bool:
return False
def _resolve_openai_audio_client_config() -> tuple[str, str]:
"""Return direct OpenAI audio config or a managed gateway fallback."""
direct_api_key = resolve_openai_audio_api_key()
if direct_api_key:
return direct_api_key, DEFAULT_OPENAI_BASE_URL
managed_gateway = resolve_managed_tool_gateway("openai-audio")
if managed_gateway is None:
raise ValueError(
"Neither VOICE_TOOLS_OPENAI_KEY nor OPENAI_API_KEY is set, and the managed OpenAI audio gateway is unavailable"
)
return managed_gateway.nous_user_token, urljoin(
f"{managed_gateway.gateway_origin.rstrip('/')}/", "v1"
)
def _has_openai_audio_backend() -> bool:
"""Return True when OpenAI audio can use direct credentials or the managed gateway."""
return bool(resolve_openai_audio_api_key() or resolve_managed_tool_gateway("openai-audio"))
# ===========================================================================
# Streaming TTS: sentence-by-sentence pipeline for ElevenLabs
# ===========================================================================
@@ -802,7 +833,10 @@ if __name__ == "__main__":
print(f" ElevenLabs: {'installed' if _check(_import_elevenlabs, 'el') else 'not installed (pip install elevenlabs)'}")
print(f" API Key: {'set' if os.getenv('ELEVENLABS_API_KEY') else 'not set'}")
print(f" OpenAI: {'installed' if _check(_import_openai_client, 'oai') else 'not installed'}")
print(f" API Key: {'set' if os.getenv('VOICE_TOOLS_OPENAI_KEY') else 'not set (VOICE_TOOLS_OPENAI_KEY)'}")
print(
" API Key: "
f"{'set' if resolve_openai_audio_api_key() else 'not set (VOICE_TOOLS_OPENAI_KEY or OPENAI_API_KEY)'}"
)
print(f" ffmpeg: {'✅ found' if _has_ffmpeg() else '❌ not found (needed for Telegram Opus)'}")
print(f"\n Output dir: {DEFAULT_OUTPUT_DIR}")

490
tools/web_tools.py
View File

@@ -4,15 +4,18 @@ Standalone Web Tools Module
This module provides generic web tools that work with multiple backend providers.
Backend is selected during ``hermes tools`` setup (web.backend in config.yaml).
When available, Hermes can route Firecrawl calls through a Nous-hosted tool-gateway
for Nous Subscribers only.
Available tools:
- web_search_tool: Search the web for information
- web_extract_tool: Extract content from specific web pages
- web_crawl_tool: Crawl websites with specific instructions (Firecrawl only)
- web_crawl_tool: Crawl websites with specific instructions
Backend compatibility:
- Firecrawl: https://docs.firecrawl.dev/introduction (search, extract, crawl)
- Firecrawl: https://docs.firecrawl.dev/introduction (search, extract, crawl; direct or derived firecrawl-gateway.<domain> for Nous Subscribers)
- Parallel: https://docs.parallel.ai (search, extract)
- Tavily: https://tavily.com (search, extract, crawl)
LLM Processing:
- Uses OpenRouter API with Gemini 3 Flash Preview for intelligent content extraction
@@ -44,8 +47,13 @@ import asyncio
from typing import List, Dict, Any, Optional
import httpx
from firecrawl import Firecrawl
from agent.auxiliary_client import async_call_llm
from agent.auxiliary_client import get_async_text_auxiliary_client
from tools.debug_helpers import DebugSession
from tools.managed_tool_gateway import (
build_vendor_gateway_url,
read_nous_access_token as _read_nous_access_token,
resolve_managed_tool_gateway,
)
from tools.url_safety import is_safe_url
from tools.website_policy import check_website_access
@@ -78,10 +86,13 @@ def _get_backend() -> str:
return configured
# Fallback for manual / legacy config — use whichever key is present.
has_firecrawl = _has_env("FIRECRAWL_API_KEY") or _has_env("FIRECRAWL_API_URL")
has_firecrawl = (
_has_env("FIRECRAWL_API_KEY")
or _has_env("FIRECRAWL_API_URL")
or _is_tool_gateway_ready()
)
has_parallel = _has_env("PARALLEL_API_KEY")
has_tavily = _has_env("TAVILY_API_KEY")
if has_tavily and not has_firecrawl and not has_parallel:
return "tavily"
if has_parallel and not has_firecrawl:
@@ -90,35 +101,100 @@ def _get_backend() -> str:
# Default to firecrawl (backward compat, or when both are set)
return "firecrawl"
def _is_backend_available(backend: str) -> bool:
"""Return True when the selected backend is currently usable."""
if backend == "parallel":
return _has_env("PARALLEL_API_KEY")
if backend == "firecrawl":
return check_firecrawl_api_key()
if backend == "tavily":
return _has_env("TAVILY_API_KEY")
return False
# ─── Firecrawl Client ────────────────────────────────────────────────────────
_firecrawl_client = None
_firecrawl_client_config = None
def _get_direct_firecrawl_config() -> Optional[tuple[Dict[str, str], tuple[str, Optional[str], Optional[str]]]]:
"""Return explicit direct Firecrawl kwargs + cache key, or None when unset."""
api_key = os.getenv("FIRECRAWL_API_KEY", "").strip()
api_url = os.getenv("FIRECRAWL_API_URL", "").strip().rstrip("/")
if not api_key and not api_url:
return None
kwargs: Dict[str, str] = {}
if api_key:
kwargs["api_key"] = api_key
if api_url:
kwargs["api_url"] = api_url
return kwargs, ("direct", api_url or None, api_key or None)
def _get_firecrawl_gateway_url() -> str:
"""Return configured Firecrawl gateway URL."""
return build_vendor_gateway_url("firecrawl")
def _is_tool_gateway_ready() -> bool:
"""Return True when gateway URL and a Nous Subscriber token are available."""
return resolve_managed_tool_gateway("firecrawl", token_reader=_read_nous_access_token) is not None
def _has_direct_firecrawl_config() -> bool:
"""Return True when direct Firecrawl config is explicitly configured."""
return _get_direct_firecrawl_config() is not None
def _raise_web_backend_configuration_error() -> None:
"""Raise a clear error for unsupported web backend configuration."""
raise ValueError(
"Web tools are not configured. "
"Set FIRECRAWL_API_KEY for cloud Firecrawl, set FIRECRAWL_API_URL for a self-hosted Firecrawl instance, "
"or, if you are a Nous Subscriber, login to Nous (`hermes model`) and provide "
"FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN."
)
def _get_firecrawl_client():
"""Get or create the Firecrawl client (lazy initialization).
"""Get or create Firecrawl client.
Uses the cloud API by default (requires FIRECRAWL_API_KEY).
Set FIRECRAWL_API_URL to point at a self-hosted instance instead —
in that case the API key is optional (set USE_DB_AUTHENTICATION=false
on your Firecrawl server to disable auth entirely).
Direct Firecrawl takes precedence when explicitly configured. Otherwise
Hermes falls back to the Firecrawl tool-gateway for logged-in Nous Subscribers.
"""
global _firecrawl_client
if _firecrawl_client is None:
api_key = os.getenv("FIRECRAWL_API_KEY")
api_url = os.getenv("FIRECRAWL_API_URL")
if not api_key and not api_url:
logger.error("Firecrawl client initialization failed: missing configuration.")
raise ValueError(
"Firecrawl client not configured. "
"Set FIRECRAWL_API_KEY (cloud) or FIRECRAWL_API_URL (self-hosted). "
"This tool requires Firecrawl to be available."
)
kwargs = {}
if api_key:
kwargs["api_key"] = api_key
if api_url:
kwargs["api_url"] = api_url
_firecrawl_client = Firecrawl(**kwargs)
global _firecrawl_client, _firecrawl_client_config
direct_config = _get_direct_firecrawl_config()
if direct_config is not None:
kwargs, client_config = direct_config
else:
managed_gateway = resolve_managed_tool_gateway(
"firecrawl",
token_reader=_read_nous_access_token,
)
if managed_gateway is None:
logger.error("Firecrawl client initialization failed: missing direct config and tool-gateway auth.")
_raise_web_backend_configuration_error()
kwargs = {
"api_key": managed_gateway.nous_user_token,
"api_url": managed_gateway.gateway_origin,
}
client_config = (
"tool-gateway",
kwargs["api_url"],
managed_gateway.nous_user_token,
)
if _firecrawl_client is not None and _firecrawl_client_config == client_config:
return _firecrawl_client
_firecrawl_client = Firecrawl(**kwargs)
_firecrawl_client_config = client_config
return _firecrawl_client
# ─── Parallel Client ─────────────────────────────────────────────────────────
@@ -243,10 +319,112 @@ def _normalize_tavily_documents(response: dict, fallback_url: str = "") -> List[
return documents
def _to_plain_object(value: Any) -> Any:
"""Convert SDK objects to plain python data structures when possible."""
if value is None:
return None
if isinstance(value, (dict, list, str, int, float, bool)):
return value
if hasattr(value, "model_dump"):
try:
return value.model_dump()
except Exception:
pass
if hasattr(value, "__dict__"):
try:
return {k: v for k, v in value.__dict__.items() if not k.startswith("_")}
except Exception:
pass
return value
def _normalize_result_list(values: Any) -> List[Dict[str, Any]]:
"""Normalize mixed SDK/list payloads into a list of dicts."""
if not isinstance(values, list):
return []
normalized: List[Dict[str, Any]] = []
for item in values:
plain = _to_plain_object(item)
if isinstance(plain, dict):
normalized.append(plain)
return normalized
def _extract_web_search_results(response: Any) -> List[Dict[str, Any]]:
"""Extract Firecrawl search results across SDK/direct/gateway response shapes."""
response_plain = _to_plain_object(response)
if isinstance(response_plain, dict):
data = response_plain.get("data")
if isinstance(data, list):
return _normalize_result_list(data)
if isinstance(data, dict):
data_web = _normalize_result_list(data.get("web"))
if data_web:
return data_web
data_results = _normalize_result_list(data.get("results"))
if data_results:
return data_results
top_web = _normalize_result_list(response_plain.get("web"))
if top_web:
return top_web
top_results = _normalize_result_list(response_plain.get("results"))
if top_results:
return top_results
if hasattr(response, "web"):
return _normalize_result_list(getattr(response, "web", []))
return []
def _extract_scrape_payload(scrape_result: Any) -> Dict[str, Any]:
"""Normalize Firecrawl scrape payload shape across SDK and gateway variants."""
result_plain = _to_plain_object(scrape_result)
if not isinstance(result_plain, dict):
return {}
nested = result_plain.get("data")
if isinstance(nested, dict):
return nested
return result_plain
DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION = 5000
# Allow per-task override via env var
DEFAULT_SUMMARIZER_MODEL = os.getenv("AUXILIARY_WEB_EXTRACT_MODEL", "").strip() or None
def _is_nous_auxiliary_client(client: Any) -> bool:
"""Return True when the resolved auxiliary backend is Nous Portal."""
base_url = str(getattr(client, "base_url", "") or "").lower()
return "nousresearch.com" in base_url
def _resolve_web_extract_auxiliary(model: Optional[str] = None) -> tuple[Optional[Any], Optional[str], Dict[str, Any]]:
"""Resolve the current web-extract auxiliary client, model, and extra body."""
client, default_model = get_async_text_auxiliary_client("web_extract")
configured_model = os.getenv("AUXILIARY_WEB_EXTRACT_MODEL", "").strip()
effective_model = model or configured_model or default_model
extra_body: Dict[str, Any] = {}
if client is not None and _is_nous_auxiliary_client(client):
from agent.auxiliary_client import get_auxiliary_extra_body
extra_body = get_auxiliary_extra_body() or {"tags": ["product=hermes-agent"]}
return client, effective_model, extra_body
def _get_default_summarizer_model() -> Optional[str]:
"""Return the current default model for web extraction summarization."""
_, model, _ = _resolve_web_extract_auxiliary()
return model
_debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG")
@@ -255,7 +433,7 @@ async def process_content_with_llm(
content: str,
url: str = "",
title: str = "",
model: str = DEFAULT_SUMMARIZER_MODEL,
model: Optional[str] = None,
min_length: int = DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION
) -> Optional[str]:
"""
@@ -338,7 +516,7 @@ async def process_content_with_llm(
async def _call_summarizer_llm(
content: str,
context_str: str,
model: str,
model: Optional[str],
max_tokens: int = 20000,
is_chunk: bool = False,
chunk_info: str = ""
@@ -404,22 +582,22 @@ Create a markdown summary that captures all key information in a well-organized,
for attempt in range(max_retries):
try:
call_kwargs = {
"task": "web_extract",
"messages": [
aux_client, effective_model, extra_body = _resolve_web_extract_auxiliary(model)
if aux_client is None or not effective_model:
logger.warning("No auxiliary model available for web content processing")
return None
from agent.auxiliary_client import auxiliary_max_tokens_param
response = await aux_client.chat.completions.create(
model=effective_model,
messages=[
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
],
"temperature": 0.1,
"max_tokens": max_tokens,
}
if model:
call_kwargs["model"] = model
response = await async_call_llm(**call_kwargs)
temperature=0.1,
**auxiliary_max_tokens_param(max_tokens),
**({} if not extra_body else {"extra_body": extra_body}),
)
return response.choices[0].message.content.strip()
except RuntimeError:
logger.warning("No auxiliary model available for web content processing")
return None
except Exception as api_error:
last_error = api_error
if attempt < max_retries - 1:
@@ -436,7 +614,7 @@ Create a markdown summary that captures all key information in a well-organized,
async def _process_large_content_chunked(
content: str,
context_str: str,
model: str,
model: Optional[str],
chunk_size: int,
max_output_size: int
) -> Optional[str]:
@@ -523,18 +701,25 @@ Synthesize these into ONE cohesive, comprehensive summary that:
Create a single, unified markdown summary."""
try:
call_kwargs = {
"task": "web_extract",
"messages": [
aux_client, effective_model, extra_body = _resolve_web_extract_auxiliary(model)
if aux_client is None or not effective_model:
logger.warning("No auxiliary model for synthesis, concatenating summaries")
fallback = "\n\n".join(summaries)
if len(fallback) > max_output_size:
fallback = fallback[:max_output_size] + "\n\n[... truncated ...]"
return fallback
from agent.auxiliary_client import auxiliary_max_tokens_param
response = await aux_client.chat.completions.create(
model=effective_model,
messages=[
{"role": "system", "content": "You synthesize multiple summaries into one cohesive, comprehensive summary. Be thorough but concise."},
{"role": "user", "content": synthesis_prompt}
],
"temperature": 0.1,
"max_tokens": 20000,
}
if model:
call_kwargs["model"] = model
response = await async_call_llm(**call_kwargs)
temperature=0.1,
**auxiliary_max_tokens_param(20000),
**({} if not extra_body else {"extra_body": extra_body}),
)
final_summary = response.choices[0].message.content.strip()
# Enforce hard cap
@@ -750,35 +935,7 @@ def web_search_tool(query: str, limit: int = 5) -> str:
limit=limit
)
# The response is a SearchData object with web, news, and images attributes
# When not scraping, the results are directly in these attributes
web_results = []
# Check if response has web attribute (SearchData object)
if hasattr(response, 'web'):
# Response is a SearchData object with web attribute
if response.web:
# Convert each SearchResultWeb object to dict
for result in response.web:
if hasattr(result, 'model_dump'):
# Pydantic model - use model_dump
web_results.append(result.model_dump())
elif hasattr(result, '__dict__'):
# Regular object - use __dict__
web_results.append(result.__dict__)
elif isinstance(result, dict):
# Already a dict
web_results.append(result)
elif hasattr(response, 'model_dump'):
# Response has model_dump method - use it to get dict
response_dict = response.model_dump()
if 'web' in response_dict and response_dict['web']:
web_results = response_dict['web']
elif isinstance(response, dict):
# Response is already a dictionary
if 'web' in response and response['web']:
web_results = response['web']
web_results = _extract_web_search_results(response)
results_count = len(web_results)
logger.info("Found %d search results", results_count)
@@ -807,11 +964,11 @@ def web_search_tool(query: str, limit: int = 5) -> str:
except Exception as e:
error_msg = f"Error searching web: {str(e)}"
logger.debug("%s", error_msg)
debug_call_data["error"] = error_msg
_debug.log_call("web_search_tool", debug_call_data)
_debug.save()
return json.dumps({"error": error_msg}, ensure_ascii=False)
@@ -819,7 +976,7 @@ async def web_extract_tool(
urls: List[str],
format: str = None,
use_llm_processing: bool = True,
model: str = DEFAULT_SUMMARIZER_MODEL,
model: Optional[str] = None,
min_length: int = DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION
) -> str:
"""
@@ -832,7 +989,7 @@ async def web_extract_tool(
urls (List[str]): List of URLs to extract content from
format (str): Desired output format ("markdown" or "html", optional)
use_llm_processing (bool): Whether to process content with LLM for summarization (default: True)
model (str): The model to use for LLM processing (default: google/gemini-3-flash-preview)
model (Optional[str]): The model to use for LLM processing (defaults to current auxiliary backend model)
min_length (int): Minimum content length to trigger LLM processing (default: 5000)
Returns:
@@ -929,39 +1086,11 @@ async def web_extract_tool(
formats=formats
)
# Process the result - properly handle object serialization
metadata = {}
scrape_payload = _extract_scrape_payload(scrape_result)
metadata = scrape_payload.get("metadata", {})
title = ""
content_markdown = None
content_html = None
# Extract data from the scrape result
if hasattr(scrape_result, 'model_dump'):
# Pydantic model - use model_dump to get dict
result_dict = scrape_result.model_dump()
content_markdown = result_dict.get('markdown')
content_html = result_dict.get('html')
metadata = result_dict.get('metadata', {})
elif hasattr(scrape_result, '__dict__'):
# Regular object with attributes
content_markdown = getattr(scrape_result, 'markdown', None)
content_html = getattr(scrape_result, 'html', None)
# Handle metadata - convert to dict if it's an object
metadata_obj = getattr(scrape_result, 'metadata', {})
if hasattr(metadata_obj, 'model_dump'):
metadata = metadata_obj.model_dump()
elif hasattr(metadata_obj, '__dict__'):
metadata = metadata_obj.__dict__
elif isinstance(metadata_obj, dict):
metadata = metadata_obj
else:
metadata = {}
elif isinstance(scrape_result, dict):
# Already a dictionary
content_markdown = scrape_result.get('markdown')
content_html = scrape_result.get('html')
metadata = scrape_result.get('metadata', {})
content_markdown = scrape_payload.get("markdown")
content_html = scrape_payload.get("html")
# Ensure metadata is a dict (not an object)
if not isinstance(metadata, dict):
@@ -1019,9 +1148,11 @@ async def web_extract_tool(
debug_call_data["pages_extracted"] = pages_extracted
debug_call_data["original_response_size"] = len(json.dumps(response))
effective_model = model or _get_default_summarizer_model()
auxiliary_available = check_auxiliary_model()
# Process each result with LLM if enabled
if use_llm_processing:
if use_llm_processing and auxiliary_available:
logger.info("Processing extracted content with LLM (parallel)...")
debug_call_data["processing_applied"].append("llm_processing")
@@ -1039,7 +1170,7 @@ async def web_extract_tool(
# Process content with LLM
processed = await process_content_with_llm(
raw_content, url, title, model, min_length
raw_content, url, title, effective_model, min_length
)
if processed:
@@ -1055,7 +1186,7 @@ async def web_extract_tool(
"original_size": original_size,
"processed_size": processed_size,
"compression_ratio": compression_ratio,
"model_used": model
"model_used": effective_model
}
return result, metrics, "processed"
else:
@@ -1087,6 +1218,9 @@ async def web_extract_tool(
else:
logger.warning("%s (no content to process)", url)
else:
if use_llm_processing and not auxiliary_available:
logger.warning("LLM processing requested but no auxiliary model available, returning raw content")
debug_call_data["processing_applied"].append("llm_processing_unavailable")
# Print summary of extracted pages for debugging (original behavior)
for result in response.get('results', []):
url = result.get('url', 'Unknown URL')
@@ -1141,7 +1275,7 @@ async def web_crawl_tool(
instructions: str = None,
depth: str = "basic",
use_llm_processing: bool = True,
model: str = DEFAULT_SUMMARIZER_MODEL,
model: Optional[str] = None,
min_length: int = DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION
) -> str:
"""
@@ -1155,7 +1289,7 @@ async def web_crawl_tool(
instructions (str): Instructions for what to crawl/extract using LLM intelligence (optional)
depth (str): Depth of extraction ("basic" or "advanced", default: "basic")
use_llm_processing (bool): Whether to process content with LLM for summarization (default: True)
model (str): The model to use for LLM processing (default: google/gemini-3-flash-preview)
model (Optional[str]): The model to use for LLM processing (defaults to current auxiliary backend model)
min_length (int): Minimum content length to trigger LLM processing (default: 5000)
Returns:
@@ -1185,6 +1319,8 @@ async def web_crawl_tool(
}
try:
effective_model = model or _get_default_summarizer_model()
auxiliary_available = check_auxiliary_model()
backend = _get_backend()
# Tavily supports crawl via its /crawl endpoint
@@ -1229,7 +1365,7 @@ async def web_crawl_tool(
debug_call_data["original_response_size"] = len(json.dumps(response))
# Process each result with LLM if enabled
if use_llm_processing:
if use_llm_processing and auxiliary_available:
logger.info("Processing crawled content with LLM (parallel)...")
debug_call_data["processing_applied"].append("llm_processing")
@@ -1240,12 +1376,12 @@ async def web_crawl_tool(
if not content:
return result, None, "no_content"
original_size = len(content)
processed = await process_content_with_llm(content, page_url, title, model, min_length)
processed = await process_content_with_llm(content, page_url, title, effective_model, min_length)
if processed:
result['raw_content'] = content
result['content'] = processed
metrics = {"url": page_url, "original_size": original_size, "processed_size": len(processed),
"compression_ratio": len(processed) / original_size if original_size else 1.0, "model_used": model}
"compression_ratio": len(processed) / original_size if original_size else 1.0, "model_used": effective_model}
return result, metrics, "processed"
metrics = {"url": page_url, "original_size": original_size, "processed_size": original_size,
"compression_ratio": 1.0, "model_used": None, "reason": "content_too_short"}
@@ -1258,6 +1394,10 @@ async def web_crawl_tool(
debug_call_data["compression_metrics"].append(metrics)
debug_call_data["pages_processed_with_llm"] += 1
if use_llm_processing and not auxiliary_available:
logger.warning("LLM processing requested but no auxiliary model available, returning raw content")
debug_call_data["processing_applied"].append("llm_processing_unavailable")
trimmed_results = [{"url": r.get("url", ""), "title": r.get("title", ""), "content": r.get("content", ""), "error": r.get("error"),
**({ "blocked_by_policy": r["blocked_by_policy"]} if "blocked_by_policy" in r else {})} for r in response.get("results", [])]
result_json = json.dumps({"results": trimmed_results}, indent=2, ensure_ascii=False)
@@ -1267,10 +1407,12 @@ async def web_crawl_tool(
_debug.save()
return cleaned_result
# web_crawl requires Firecrawl — Parallel has no crawl API
if not (os.getenv("FIRECRAWL_API_KEY") or os.getenv("FIRECRAWL_API_URL")):
# web_crawl requires Firecrawl or the Firecrawl tool-gateway — Parallel has no crawl API
if not check_firecrawl_api_key():
return json.dumps({
"error": "web_crawl requires Firecrawl. Set FIRECRAWL_API_KEY, "
"error": "web_crawl requires Firecrawl. Set FIRECRAWL_API_KEY, FIRECRAWL_API_URL, "
"or, if you are a Nous Subscriber, login to Nous and use FIRECRAWL_GATEWAY_URL, "
"or TOOL_GATEWAY_DOMAIN, "
"or use web_search + web_extract instead.",
"success": False,
}, ensure_ascii=False)
@@ -1431,7 +1573,7 @@ async def web_crawl_tool(
debug_call_data["original_response_size"] = len(json.dumps(response))
# Process each result with LLM if enabled
if use_llm_processing:
if use_llm_processing and auxiliary_available:
logger.info("Processing crawled content with LLM (parallel)...")
debug_call_data["processing_applied"].append("llm_processing")
@@ -1449,7 +1591,7 @@ async def web_crawl_tool(
# Process content with LLM
processed = await process_content_with_llm(
content, page_url, title, model, min_length
content, page_url, title, effective_model, min_length
)
if processed:
@@ -1465,7 +1607,7 @@ async def web_crawl_tool(
"original_size": original_size,
"processed_size": processed_size,
"compression_ratio": compression_ratio,
"model_used": model
"model_used": effective_model
}
return result, metrics, "processed"
else:
@@ -1497,6 +1639,9 @@ async def web_crawl_tool(
else:
logger.warning("%s (no content to process)", page_url)
else:
if use_llm_processing and not auxiliary_available:
logger.warning("LLM processing requested but no auxiliary model available, returning raw content")
debug_call_data["processing_applied"].append("llm_processing_unavailable")
# Print summary of crawled pages for debugging (original behavior)
for result in response.get('results', []):
page_url = result.get('url', 'Unknown URL')
@@ -1540,38 +1685,34 @@ async def web_crawl_tool(
return json.dumps({"error": error_msg}, ensure_ascii=False)
# Convenience function to check if API key is available
# Convenience function to check Firecrawl credentials
def check_firecrawl_api_key() -> bool:
"""
Check if the Firecrawl API key is available in environment variables.
Check whether the Firecrawl backend is available.
Availability is true when either:
1) direct Firecrawl config (`FIRECRAWL_API_KEY` or `FIRECRAWL_API_URL`), or
2) Firecrawl gateway origin + Nous Subscriber access token
(fallback when direct Firecrawl is not configured).
Returns:
bool: True if API key is set, False otherwise
bool: True if direct Firecrawl or the tool-gateway can be used.
"""
return bool(os.getenv("FIRECRAWL_API_KEY"))
return _has_direct_firecrawl_config() or _is_tool_gateway_ready()
def check_web_api_key() -> bool:
"""Check if any web backend API key is available (Parallel, Firecrawl, or Tavily)."""
return bool(
os.getenv("PARALLEL_API_KEY")
or os.getenv("FIRECRAWL_API_KEY")
or os.getenv("FIRECRAWL_API_URL")
or os.getenv("TAVILY_API_KEY")
)
"""Check whether the configured web backend is available."""
configured = _load_web_config().get("backend", "").lower().strip()
if configured in ("parallel", "firecrawl", "tavily"):
return _is_backend_available(configured)
return any(_is_backend_available(backend) for backend in ("parallel", "firecrawl", "tavily"))
def check_auxiliary_model() -> bool:
"""Check if an auxiliary text model is available for LLM content processing."""
try:
from agent.auxiliary_client import resolve_provider_client
for p in ("openrouter", "nous", "custom", "codex"):
client, _ = resolve_provider_client(p)
if client is not None:
return True
return False
except Exception:
return False
client, _, _ = _resolve_web_extract_auxiliary()
return client is not None
def get_debug_session_info() -> Dict[str, Any]:
@@ -1588,7 +1729,11 @@ if __name__ == "__main__":
# Check if API keys are available
web_available = check_web_api_key()
tool_gateway_available = _is_tool_gateway_ready()
firecrawl_key_available = bool(os.getenv("FIRECRAWL_API_KEY", "").strip())
firecrawl_url_available = bool(os.getenv("FIRECRAWL_API_URL", "").strip())
nous_available = check_auxiliary_model()
default_summarizer_model = _get_default_summarizer_model()
if web_available:
backend = _get_backend()
@@ -1598,17 +1743,28 @@ if __name__ == "__main__":
elif backend == "tavily":
print(" Using Tavily API (https://tavily.com)")
else:
print(" Using Firecrawl API (https://firecrawl.dev)")
if firecrawl_url_available:
print(f" Using self-hosted Firecrawl: {os.getenv('FIRECRAWL_API_URL').strip().rstrip('/')}")
elif firecrawl_key_available:
print(" Using direct Firecrawl cloud API")
elif tool_gateway_available:
print(f" Using Firecrawl tool-gateway: {_get_firecrawl_gateway_url()}")
else:
print(" Firecrawl backend selected but not configured")
else:
print("❌ No web search backend configured")
print("Set PARALLEL_API_KEY, TAVILY_API_KEY, or FIRECRAWL_API_KEY")
print(
"Set PARALLEL_API_KEY, TAVILY_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL, "
"or, if you are a Nous Subscriber, login to Nous and use "
"FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN"
)
if not nous_available:
print("❌ No auxiliary model available for LLM content processing")
print("Set OPENROUTER_API_KEY, configure Nous Portal, or set OPENAI_BASE_URL + OPENAI_API_KEY")
print("⚠️ Without an auxiliary model, LLM content processing will be disabled")
else:
print(f"✅ Auxiliary model available: {DEFAULT_SUMMARIZER_MODEL}")
print(f"✅ Auxiliary model available: {default_summarizer_model}")
if not web_available:
exit(1)
@@ -1616,7 +1772,7 @@ if __name__ == "__main__":
print("🛠️ Web tools ready for use!")
if nous_available:
print(f"🧠 LLM content processing available with {DEFAULT_SUMMARIZER_MODEL}")
print(f"🧠 LLM content processing available with {default_summarizer_model}")
print(f" Default min length for processing: {DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION} chars")
# Show debug mode status
@@ -1711,7 +1867,16 @@ registry.register(
schema=WEB_SEARCH_SCHEMA,
handler=lambda args, **kw: web_search_tool(args.get("query", ""), limit=5),
check_fn=check_web_api_key,
requires_env=["PARALLEL_API_KEY", "FIRECRAWL_API_KEY", "TAVILY_API_KEY"],
requires_env=[
"PARALLEL_API_KEY",
"TAVILY_API_KEY",
"FIRECRAWL_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_SCHEME",
"TOOL_GATEWAY_USER_TOKEN",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
],
emoji="🔍",
)
registry.register(
@@ -1721,7 +1886,16 @@ registry.register(
handler=lambda args, **kw: web_extract_tool(
args.get("urls", [])[:5] if isinstance(args.get("urls"), list) else [], "markdown"),
check_fn=check_web_api_key,
requires_env=["PARALLEL_API_KEY", "FIRECRAWL_API_KEY", "TAVILY_API_KEY"],
requires_env=[
"PARALLEL_API_KEY",
"TAVILY_API_KEY",
"FIRECRAWL_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_SCHEME",
"TOOL_GATEWAY_USER_TOKEN",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
],
is_async=True,
emoji="📄",
)