feat(onboarding): contextual first-touch hints for /busy and /verbose (#16046)
Instead of a blocking first-run questionnaire, show a one-time hint the first time the user hits each behavior fork: 1. First message while the agent is working — appends a hint to the busy-ack explaining the /busy queue vs /busy interrupt knob, phrased to match the mode that was just applied (don't tell a queue-mode user to switch to queue). 2. First tool that runs for >= 30s in the noisiest progress mode (tool_progress: all) — prints a hint about /verbose to cycle display modes (all -> new -> off -> verbose). Gated on /verbose actually being usable on the surface: always shown on CLI; on gateway only shown when display.tool_progress_command is enabled. Each hint is latched in config.yaml under onboarding.seen.<flag>, so it fires exactly once per install across CLI, gateway, and cron, then never again. Users can wipe the section to re-see hints. New: - agent/onboarding.py — is_seen / mark_seen / hint strings, shared by both CLI and gateway. - onboarding.seen in DEFAULT_CONFIG (hermes_cli/config.py) and in load_cli_config defaults (cli.py). No _config_version bump — deep merge handles new keys. Wired: - gateway/run.py: _handle_active_session_busy_message appends the hint after building the ack. progress_callback tracks tool.completed duration and queues the tool-progress hint into the progress bubble. - cli.py: CLI input loop appends the busy-input hint on the first busy Enter; _on_tool_progress appends the tool-progress hint on the first >=30s tool completion. In-memory CLI_CONFIG is also updated so subsequent fires in the same process are suppressed immediately. All writes go through atomic_yaml_write and are wrapped in try/except so onboarding can never break the input/busy-ack paths.
This commit is contained in:
Teknium
164
tests/agent/test_onboarding.py
Normal file
164
tests/agent/test_onboarding.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""Tests for agent/onboarding.py — contextual first-touch hint helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import yaml
|
||||
import pytest
|
||||
|
||||
from agent.onboarding import (
|
||||
BUSY_INPUT_FLAG,
|
||||
TOOL_PROGRESS_FLAG,
|
||||
busy_input_hint_cli,
|
||||
busy_input_hint_gateway,
|
||||
is_seen,
|
||||
mark_seen,
|
||||
tool_progress_hint_cli,
|
||||
tool_progress_hint_gateway,
|
||||
)
|
||||
|
||||
|
||||
class TestIsSeen:
|
||||
def test_empty_config_unseen(self):
|
||||
assert is_seen({}, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_missing_onboarding_unseen(self):
|
||||
assert is_seen({"display": {}}, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_onboarding_not_dict_unseen(self):
|
||||
assert is_seen({"onboarding": "nope"}, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_seen_dict_missing_flag(self):
|
||||
assert is_seen({"onboarding": {"seen": {}}}, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_seen_flag_true(self):
|
||||
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: True}}}
|
||||
assert is_seen(cfg, BUSY_INPUT_FLAG) is True
|
||||
|
||||
def test_seen_flag_falsy(self):
|
||||
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: False}}}
|
||||
assert is_seen(cfg, BUSY_INPUT_FLAG) is False
|
||||
|
||||
def test_other_flags_isolated(self):
|
||||
cfg = {"onboarding": {"seen": {BUSY_INPUT_FLAG: True}}}
|
||||
assert is_seen(cfg, TOOL_PROGRESS_FLAG) is False
|
||||
|
||||
|
||||
class TestMarkSeen:
|
||||
def test_creates_missing_file_and_sets_flag(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
def test_preserves_other_config(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({
|
||||
"model": {"default": "claude-sonnet-4.6"},
|
||||
"display": {"skin": "default"},
|
||||
}))
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
|
||||
assert loaded["model"]["default"] == "claude-sonnet-4.6"
|
||||
assert loaded["display"]["skin"] == "default"
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
def test_preserves_other_seen_flags(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({
|
||||
"onboarding": {"seen": {TOOL_PROGRESS_FLAG: True}},
|
||||
}))
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
|
||||
assert loaded["onboarding"]["seen"][TOOL_PROGRESS_FLAG] is True
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
def test_idempotent(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
mark_seen(cfg_path, BUSY_INPUT_FLAG)
|
||||
first = cfg_path.read_text()
|
||||
|
||||
# Second call must be a no-op on-disk content (file may be touched,
|
||||
# but the YAML contents should be identical).
|
||||
mark_seen(cfg_path, BUSY_INPUT_FLAG)
|
||||
second = cfg_path.read_text()
|
||||
|
||||
assert yaml.safe_load(first) == yaml.safe_load(second)
|
||||
|
||||
def test_handles_non_dict_onboarding(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({"onboarding": "corrupted"}))
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
def test_handles_non_dict_seen(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
cfg_path.write_text(yaml.safe_dump({"onboarding": {"seen": "corrupted"}}))
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
assert loaded["onboarding"]["seen"][BUSY_INPUT_FLAG] is True
|
||||
|
||||
|
||||
class TestHintMessages:
|
||||
def test_busy_input_hint_gateway_interrupt(self):
|
||||
msg = busy_input_hint_gateway("interrupt")
|
||||
assert "/busy queue" in msg
|
||||
assert "interrupted" in msg.lower()
|
||||
|
||||
def test_busy_input_hint_gateway_queue(self):
|
||||
msg = busy_input_hint_gateway("queue")
|
||||
assert "/busy interrupt" in msg
|
||||
assert "queued" in msg.lower()
|
||||
|
||||
def test_busy_input_hint_cli_interrupt(self):
|
||||
msg = busy_input_hint_cli("interrupt")
|
||||
assert "/busy queue" in msg
|
||||
|
||||
def test_busy_input_hint_cli_queue(self):
|
||||
msg = busy_input_hint_cli("queue")
|
||||
assert "/busy interrupt" in msg
|
||||
|
||||
def test_tool_progress_hints_mention_verbose(self):
|
||||
assert "/verbose" in tool_progress_hint_gateway()
|
||||
assert "/verbose" in tool_progress_hint_cli()
|
||||
|
||||
def test_hints_are_not_empty(self):
|
||||
for hint in (
|
||||
busy_input_hint_gateway("queue"),
|
||||
busy_input_hint_gateway("interrupt"),
|
||||
busy_input_hint_cli("queue"),
|
||||
busy_input_hint_cli("interrupt"),
|
||||
tool_progress_hint_gateway(),
|
||||
tool_progress_hint_cli(),
|
||||
):
|
||||
assert hint.strip()
|
||||
|
||||
|
||||
class TestRoundTrip:
|
||||
"""After mark_seen, is_seen on the re-loaded config must return True."""
|
||||
|
||||
def test_mark_then_is_seen(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
|
||||
assert mark_seen(cfg_path, BUSY_INPUT_FLAG) is True
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
|
||||
assert is_seen(loaded, BUSY_INPUT_FLAG) is True
|
||||
assert is_seen(loaded, TOOL_PROGRESS_FLAG) is False
|
||||
|
||||
def test_mark_both_flags_independently(self, tmp_path):
|
||||
cfg_path = tmp_path / "config.yaml"
|
||||
|
||||
mark_seen(cfg_path, BUSY_INPUT_FLAG)
|
||||
mark_seen(cfg_path, TOOL_PROGRESS_FLAG)
|
||||
loaded = yaml.safe_load(cfg_path.read_text())
|
||||
|
||||
assert is_seen(loaded, BUSY_INPUT_FLAG) is True
|
||||
assert is_seen(loaded, TOOL_PROGRESS_FLAG) is True
|
||||
@@ -349,3 +349,121 @@ class TestBusySessionAck:
|
||||
|
||||
result = await runner._handle_active_session_busy_message(event, sk)
|
||||
assert result is False # not handled, let default path try
|
||||
|
||||
|
||||
class TestBusySessionOnboardingHint:
|
||||
"""First-touch hint appended to the busy-ack the first time it fires."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_busy_ack_appends_interrupt_hint(self, tmp_path, monkeypatch):
|
||||
"""First busy-while-running message gets an extra hint about /busy."""
|
||||
import gateway.run as _gr
|
||||
|
||||
monkeypatch.setattr(_gr, "_hermes_home", tmp_path)
|
||||
# mark_seen imports utils.atomic_yaml_write; make sure it resolves
|
||||
# against a writable dir by pointing _hermes_home at tmp_path.
|
||||
monkeypatch.setattr(_gr, "_load_gateway_config", lambda: {})
|
||||
|
||||
runner, _sentinel = _make_runner()
|
||||
runner._busy_input_mode = "interrupt"
|
||||
adapter = _make_adapter()
|
||||
|
||||
event = _make_event(text="ping")
|
||||
sk = build_session_key(event.source)
|
||||
|
||||
agent = MagicMock()
|
||||
agent.get_activity_summary.return_value = {
|
||||
"api_call_count": 3, "max_iterations": 60,
|
||||
"current_tool": None, "last_activity_ts": time.time(),
|
||||
"last_activity_desc": "api", "seconds_since_activity": 0.1,
|
||||
}
|
||||
runner._running_agents[sk] = agent
|
||||
runner._running_agents_ts[sk] = time.time() - 5
|
||||
runner.adapters[event.source.platform] = adapter
|
||||
|
||||
await runner._handle_active_session_busy_message(event, sk)
|
||||
|
||||
call_kwargs = adapter._send_with_retry.call_args
|
||||
content = call_kwargs.kwargs.get("content", "")
|
||||
|
||||
# Normal ack body
|
||||
assert "Interrupting" in content
|
||||
# First-touch hint appended
|
||||
assert "First-time tip" in content
|
||||
assert "/busy queue" in content
|
||||
|
||||
# The flag is now persisted to tmp_path/config.yaml
|
||||
import yaml
|
||||
cfg = yaml.safe_load((tmp_path / "config.yaml").read_text())
|
||||
assert cfg["onboarding"]["seen"]["busy_input_prompt"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_second_busy_ack_omits_hint(self, tmp_path, monkeypatch):
|
||||
"""Once the flag is marked, the hint never appears again."""
|
||||
import gateway.run as _gr
|
||||
import yaml
|
||||
|
||||
monkeypatch.setattr(_gr, "_hermes_home", tmp_path)
|
||||
# Pre-populate the config so is_seen() returns True from the start.
|
||||
(tmp_path / "config.yaml").write_text(yaml.safe_dump({
|
||||
"onboarding": {"seen": {"busy_input_prompt": True}},
|
||||
}))
|
||||
monkeypatch.setattr(
|
||||
_gr, "_load_gateway_config",
|
||||
lambda: yaml.safe_load((tmp_path / "config.yaml").read_text()),
|
||||
)
|
||||
|
||||
runner, _sentinel = _make_runner()
|
||||
runner._busy_input_mode = "interrupt"
|
||||
adapter = _make_adapter()
|
||||
|
||||
event = _make_event(text="ping again")
|
||||
sk = build_session_key(event.source)
|
||||
|
||||
agent = MagicMock()
|
||||
agent.get_activity_summary.return_value = {
|
||||
"api_call_count": 3, "max_iterations": 60,
|
||||
"current_tool": None, "last_activity_ts": time.time(),
|
||||
"last_activity_desc": "api", "seconds_since_activity": 0.1,
|
||||
}
|
||||
runner._running_agents[sk] = agent
|
||||
runner._running_agents_ts[sk] = time.time() - 5
|
||||
runner.adapters[event.source.platform] = adapter
|
||||
|
||||
await runner._handle_active_session_busy_message(event, sk)
|
||||
|
||||
call_kwargs = adapter._send_with_retry.call_args
|
||||
content = call_kwargs.kwargs.get("content", "")
|
||||
|
||||
assert "Interrupting" in content
|
||||
assert "First-time tip" not in content
|
||||
assert "/busy queue" not in content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_mode_hint_points_to_interrupt(self, tmp_path, monkeypatch):
|
||||
"""In queue mode the hint should suggest /busy interrupt, not /busy queue."""
|
||||
import gateway.run as _gr
|
||||
|
||||
monkeypatch.setattr(_gr, "_hermes_home", tmp_path)
|
||||
monkeypatch.setattr(_gr, "_load_gateway_config", lambda: {})
|
||||
|
||||
runner, _sentinel = _make_runner()
|
||||
runner._busy_input_mode = "queue"
|
||||
adapter = _make_adapter()
|
||||
|
||||
event = _make_event(text="queue me")
|
||||
sk = build_session_key(event.source)
|
||||
runner.adapters[event.source.platform] = adapter
|
||||
|
||||
agent = MagicMock()
|
||||
runner._running_agents[sk] = agent
|
||||
|
||||
with patch("gateway.run.merge_pending_message_event"):
|
||||
await runner._handle_active_session_busy_message(event, sk)
|
||||
|
||||
content = adapter._send_with_retry.call_args.kwargs.get("content", "")
|
||||
assert "Queued for the next turn" in content
|
||||
assert "First-time tip" in content
|
||||
assert "/busy interrupt" in content
|
||||
# Must NOT tell the user to /busy queue when they're already on queue.
|
||||
assert "/busy queue" not in content
|
||||
|
||||
Reference in New Issue
Block a user