## Summary `read_skill` now returns an on-disk skill's body the same way context injection renders it: headed with the skill's directory, with markdown links to bundled files (`references/`, `scripts/`, `assets/`, `examples/`) rewritten to absolute paths. Why: the skill menu (scent) lists `local/<name>: <description>` and tells the model to read one with `read_skill(id)`. For a local skill that tool returned the raw SKILL.md body (`## <name>` plus content). A body names its bundled files relatively, for example `[layouts.md](references/layouts.md)`, and nothing in the result said where the skill lives. A skill mounted through `skillForge.localDirs` sits nowhere near the agent home, so the model had to guess. Every Raven-Design deck run in the local trajectories hit this, on two different models: - glm-5.3-flash: searched `~/.raven` twice with no match, then read `deck-to-pptx/references/*.md` from another checkout of the repo. - glm-5.3-flash: found them with a `find` over the source checkout. - step-5-preview: built a path from the agent-home skills dir plus `../` segments, got "File not found", ran a `find` over the whole home directory (hit its 20s budget), and read the other checkout's copy again. The injection path (`LocalSkillCatalog.load_skills_for_context`) already handled exactly this, with existence-checked rewriting and a `**Skill directory**` header. That per-skill rendering moves unchanged into a module-level `render_skill_body`, which both routes now call, so the fetched and the injected body read the same. Hub skills are untouched. ## Type - [x] Fix - [ ] Feature - [ ] Docs - [ ] CI / tooling - [ ] Refactor - [ ] Other ## Verification - `uv run pytest tests/test_skill_hub_tools.py tests/test_skill_ref_resolution.py tests/test_skill_forge_loader_parity.py tests/test_skill_router_local_source.py tests/test_agent_loop_skill_tools.py tests/test_segments.py tests/test_context_scent.py tests/test_trajectory_replay.py tests/test_agent_loop_injected_skill_ids.py -q`: 265 passed. - The new test `test_a_local_body_names_its_directory_and_links_its_references_absolute` fails with the `read_skill` change reverted and passes with it. - `uv run ruff check` and `uv run ruff format --check` on the three changed files: clean. - On a macOS install, `ReadSkillTool` over a `SkillRegistry` built from the Raven-Design product's own `skillForge.localDirs`: - before: `local/deck-to-pptx` came back headed `## deck-to-pptx` with 3 links still relative (`references/layouts.md`, `references/gates.md`), and `local/design-editorial-and-presentations` with 2; - after: both are headed with their real directories (`agents/raven-design/skills/deck-to-pptx` and `plugins-dist/design-engine/raven_design/skills/...`), with 4 and 2 absolute links, all existing on disk, and none left relative. - [x] Relevant tests pass locally - [x] Relevant lint / type checks pass locally - [ ] User-facing docs or screenshots are updated when needed ## Risk The model-visible header of a `read_skill` result for a local skill changes from `## <name>` to `### Skill: <name>` plus a skill-directory line, the same shape the injected bodies already carry. Links are rewritten only when the target exists, so no new broken paths are introduced. Rollback is reverting this commit. - [x] Security impact considered - [x] Backward compatibility considered - [x] Rollback path is clear for risky changes ## Related Issues N/A --------- Co-authored-by: Claude (claude-opus-5-5) <noreply@anthropic.com>
435 lines
18 KiB
Python
435 lines
18 KiB
Python
"""Unit tests for the read_skill / use_skill agent tools."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
from raven.agent.tools.skill_hub import (
|
|
ReadSkillTool,
|
|
UseSkillTool,
|
|
split_qualified_id,
|
|
)
|
|
|
|
|
|
class _FakeRegistry:
|
|
"""Minimal SkillRegistry stand-in, keyed the way the real one is.
|
|
|
|
Entries are keyed by *physical layer* — ``workspace`` / ``builtin`` /
|
|
``external`` / ``everos`` — never by the ``local`` router namespace, which
|
|
SkillRegistry does not know. ``source=None`` resolves to the layer-priority
|
|
winner. Keying a fixture under ``("local", name)`` would make a broken
|
|
namespace passthrough look green.
|
|
"""
|
|
|
|
_LAYER_PRIORITY = ("workspace", "external", "builtin", "everos")
|
|
|
|
def __init__(self, metas: dict[tuple[str, str], object]) -> None:
|
|
assert not any(layer == "local" for layer, _ in metas), "'local' is a router namespace, not a registry layer"
|
|
self._metas = metas
|
|
self.invalidated: list[str] = []
|
|
|
|
def get(self, name: str, source: str | None = None):
|
|
if source is not None:
|
|
return self._metas.get((source, name))
|
|
candidates = [(layer, m) for (layer, key), m in self._metas.items() if key == name]
|
|
if not candidates:
|
|
return None
|
|
candidates.sort(key=lambda c: self._LAYER_PRIORITY.index(c[0]))
|
|
return candidates[0][1]
|
|
|
|
def invalidate_source(self, source: str) -> None:
|
|
self.invalidated.append(source)
|
|
|
|
|
|
class _FakeClient:
|
|
"""SkillHubClient stand-in capturing calls and returning canned data."""
|
|
|
|
def __init__(self, *, get_result=None, install_result=None, raises=False):
|
|
self._get_result = get_result or {}
|
|
self._install_result = install_result or {}
|
|
self._raises = raises
|
|
self.get_calls: list[str] = []
|
|
self.install_calls: list[str] = []
|
|
self.install_meta: list[dict | None] = []
|
|
|
|
async def get(self, skill_id: str):
|
|
self.get_calls.append(skill_id)
|
|
if self._raises:
|
|
raise RuntimeError("boom")
|
|
return self._get_result
|
|
|
|
async def install(self, skill_id: str, *, prefetched_meta=None):
|
|
self.install_calls.append(skill_id)
|
|
self.install_meta.append(prefetched_meta)
|
|
if self._raises:
|
|
raise RuntimeError("boom")
|
|
return self._install_result
|
|
|
|
|
|
def _meta(tmp: Path, name: str, content: str, *, with_scripts: bool) -> object:
|
|
skill_dir = tmp / name
|
|
skill_dir.mkdir(parents=True, exist_ok=True)
|
|
(skill_dir / "SKILL.md").write_text(content, encoding="utf-8")
|
|
if with_scripts:
|
|
(skill_dir / "scripts").mkdir(exist_ok=True)
|
|
return SimpleNamespace(name=name, content=content, path=skill_dir / "SKILL.md")
|
|
|
|
|
|
class TestSplitQualifiedId:
|
|
def test_source_prefixed(self) -> None:
|
|
assert split_qualified_id("hub/my-skill") == ("hub", "my-skill")
|
|
assert split_qualified_id("local/x") == ("local", "x")
|
|
assert split_qualified_id("everos/abc123") == ("everos", "abc123")
|
|
|
|
def test_bare_id_assumed_hub(self) -> None:
|
|
assert split_qualified_id("abc123") == ("hub", "abc123")
|
|
|
|
|
|
class TestReadSkill:
|
|
async def test_missing_skill_id_errors(self) -> None:
|
|
assert (await ReadSkillTool().execute(skill_id=None)).startswith("Error")
|
|
assert (await ReadSkillTool().execute(skill_id="")).startswith("Error")
|
|
|
|
async def test_hub_body_fetched(self) -> None:
|
|
client = _FakeClient(
|
|
get_result={
|
|
"skill_md": "do the thing",
|
|
"name": "Foo",
|
|
"version": "v2",
|
|
"scenario_tags": ["a", "b"],
|
|
}
|
|
)
|
|
out = await ReadSkillTool(client=client).execute(skill_id="hub/foo")
|
|
assert client.get_calls == ["foo"]
|
|
assert "Foo" in out and "v2" in out and "do the thing" in out
|
|
assert "a, b" in out
|
|
|
|
async def test_hub_without_client_errors(self) -> None:
|
|
out = await ReadSkillTool(client=None).execute(skill_id="hub/foo")
|
|
assert out.startswith("Error") and "not configured" in out
|
|
|
|
async def test_hub_fetch_failure_is_isolated(self) -> None:
|
|
client = _FakeClient(raises=True)
|
|
out = await ReadSkillTool(client=client).execute(skill_id="hub/foo")
|
|
assert out.startswith("Error") and "failed to read" in out
|
|
|
|
async def test_local_body_from_registry(self, tmp_path: Path) -> None:
|
|
meta = _meta(tmp_path, "x", "local body", with_scripts=False)
|
|
reg = _FakeRegistry({("builtin", "x"): meta})
|
|
out = await ReadSkillTool(registry=reg).execute(skill_id="local/x")
|
|
assert "local body" in out
|
|
|
|
async def test_a_local_body_names_its_directory_and_links_its_references_absolute(self, tmp_path: Path) -> None:
|
|
"""The skill menu sends the model here, and a body only says `references/x.md`.
|
|
|
|
A mounted skill lives nowhere near the agent home, so a raw body left the
|
|
model to guess its directory: deck runs on two models each searched the
|
|
filesystem for the reference files, one reading another checkout's copy.
|
|
The fetched body reads the way the injected one does.
|
|
"""
|
|
from raven.memory_engine.skill_forge.catalog import render_skill_body
|
|
|
|
body = "Read [layouts](references/layouts.md) before placing a page."
|
|
meta = _meta(tmp_path, "deck", body, with_scripts=False)
|
|
skill_dir = tmp_path / "deck"
|
|
(skill_dir / "references").mkdir()
|
|
(skill_dir / "references" / "layouts.md").write_text("shapes", encoding="utf-8")
|
|
reg = _FakeRegistry({("builtin", "deck"): meta})
|
|
|
|
out = await ReadSkillTool(registry=reg).execute(skill_id="local/deck")
|
|
|
|
assert f"**Skill directory**: `{skill_dir}`" in out
|
|
assert f"[layouts]({skill_dir}/references/layouts.md)" in out
|
|
assert out == render_skill_body(meta), "one renderer for the fetched and the injected body"
|
|
|
|
async def test_the_blocklist_is_read_per_call(self, tmp_path: Path) -> None:
|
|
"""Enabling a skill stops the refusal without restarting the loop.
|
|
|
|
The tool's policy is built once when the loop wires its tools, so a
|
|
screen reading the construction-time list keeps refusing a skill the
|
|
operator has since switched back on -- with nothing in the product to
|
|
undo it.
|
|
"""
|
|
meta = _meta(tmp_path, "x", "local body", with_scripts=False)
|
|
reg = _FakeRegistry({("builtin", "x"): meta})
|
|
live: set[str] = {"x"}
|
|
tool = ReadSkillTool(registry=reg, blocklist=["x"], blocklist_reader=lambda: frozenset(live))
|
|
|
|
out = await tool.execute(skill_id="local/x")
|
|
assert out.startswith("Error") and "blocklist" in out
|
|
|
|
live.discard("x")
|
|
assert "local body" in await tool.execute(skill_id="local/x")
|
|
|
|
live.add("x")
|
|
assert (await tool.execute(skill_id="local/x")).startswith("Error")
|
|
|
|
async def test_local_missing_errors(self) -> None:
|
|
out = await ReadSkillTool(registry=_FakeRegistry({})).execute(
|
|
skill_id="local/nope",
|
|
)
|
|
assert out.startswith("Error")
|
|
|
|
|
|
class TestUseSkill:
|
|
async def test_missing_skill_id_errors(self) -> None:
|
|
assert (await UseSkillTool().execute(skill_id=None)).startswith("Error")
|
|
|
|
async def test_unknown_source_falls_back_to_an_on_disk_lookup(self) -> None:
|
|
"""The on-disk source's name is whatever ``memory.backend`` says, so an
|
|
unqualified prefix can no longer be checked against a hardcoded list:
|
|
anything but ``hub`` is looked up on disk and misses there."""
|
|
out = await UseSkillTool().execute(skill_id="weird/x")
|
|
assert out.startswith("Error") and "no weird skill" in out
|
|
|
|
async def test_local_with_scripts(self, tmp_path: Path) -> None:
|
|
meta = _meta(tmp_path, "x", "body here", with_scripts=True)
|
|
reg = _FakeRegistry({("builtin", "x"): meta})
|
|
out = await UseSkillTool(registry=reg).execute(skill_id="local/x")
|
|
assert "scripts_dir:" in out
|
|
assert str(tmp_path / "x" / "scripts") in out
|
|
assert "body here" in out
|
|
|
|
async def test_local_without_scripts_is_instruction_only(self, tmp_path: Path) -> None:
|
|
meta = _meta(tmp_path, "x", "just instructions", with_scripts=False)
|
|
reg = _FakeRegistry({("builtin", "x"): meta})
|
|
out = await UseSkillTool(registry=reg).execute(skill_id="local/x")
|
|
assert "no bundled scripts" in out
|
|
assert "scripts_dir:" not in out
|
|
|
|
async def test_everos_resolves_on_disk(self, tmp_path: Path) -> None:
|
|
meta = _meta(tmp_path, "sql123", "evolved skill", with_scripts=True)
|
|
reg = _FakeRegistry({("everos", "sql123"): meta})
|
|
out = await UseSkillTool(registry=reg).execute(skill_id="everos/sql123")
|
|
assert "scripts_dir:" in out and "evolved skill" in out
|
|
|
|
async def test_local_missing_errors(self) -> None:
|
|
out = await UseSkillTool(registry=_FakeRegistry({})).execute(
|
|
skill_id="local/nope",
|
|
)
|
|
assert out.startswith("Error")
|
|
|
|
async def test_local_id_reaches_a_builtin_via_the_real_registry(self, tmp_path: Path) -> None:
|
|
"""Pins the namespace translation against a real SkillRegistry.
|
|
|
|
No fake can catch this one: the bug was handing ``source="local"``
|
|
straight to ``SkillRegistry.get``, whose compound key only ever holds
|
|
physical layers (workspace / builtin / external), so every ``local/*``
|
|
id — the exact form the router prints in the ``# Skills`` brackets and
|
|
both tool descriptions tell the agent to copy — resolved to None.
|
|
"""
|
|
from raven.memory_engine.skill_local import SkillRegistry
|
|
|
|
workspace = tmp_path / "chanwork"
|
|
workspace.mkdir()
|
|
builtin = tmp_path / "builtin" / "demo"
|
|
builtin.mkdir(parents=True)
|
|
(builtin / "SKILL.md").write_text(
|
|
"---\nname: demo\ndescription: d\n---\n\nDEMO BODY\n",
|
|
encoding="utf-8",
|
|
)
|
|
registry = SkillRegistry(workspace, builtin_skills_dir=tmp_path / "builtin")
|
|
assert registry.get("demo").source == "builtin" # never "local"
|
|
|
|
for tool in (UseSkillTool(registry=registry), ReadSkillTool(registry=registry)):
|
|
out = await tool.execute(skill_id="local/demo")
|
|
assert not out.startswith("Error"), out
|
|
assert "DEMO BODY" in out
|
|
|
|
async def test_hub_install_and_registry_invalidation(self) -> None:
|
|
client = _FakeClient(
|
|
install_result={
|
|
"slug": "foo",
|
|
"version": "v1",
|
|
"scripts_dir": "/cache/foo/scripts",
|
|
"skill_md": "hub body",
|
|
}
|
|
)
|
|
reg = _FakeRegistry({})
|
|
out = await UseSkillTool(client=client, registry=reg).execute(
|
|
skill_id="hub/foo",
|
|
)
|
|
assert client.install_calls == ["foo"]
|
|
assert "scripts_dir: /cache/foo/scripts" in out
|
|
assert "hub body" in out
|
|
assert reg.invalidated == ["hub"]
|
|
|
|
async def test_hub_without_scripts(self) -> None:
|
|
client = _FakeClient(
|
|
install_result={
|
|
"slug": "foo",
|
|
"version": "v1",
|
|
"scripts_dir": None,
|
|
"skill_md": "pure instructions",
|
|
}
|
|
)
|
|
out = await UseSkillTool(client=client).execute(skill_id="hub/foo")
|
|
assert "no bundled scripts" in out and "pure instructions" in out
|
|
|
|
async def test_hub_without_client_errors(self) -> None:
|
|
out = await UseSkillTool(client=None).execute(skill_id="hub/foo")
|
|
assert out.startswith("Error") and "not configured" in out
|
|
|
|
async def test_hub_install_failure_is_isolated(self) -> None:
|
|
client = _FakeClient(raises=True)
|
|
out = await UseSkillTool(client=client).execute(skill_id="hub/foo")
|
|
assert out.startswith("Error") and "failed to install" in out
|
|
|
|
|
|
class TestUseSkillSafetyPolicy:
|
|
"""Install-time policy: min_safety and blocklist must gate use_skill."""
|
|
|
|
async def test_low_safety_hub_skill_refused(self) -> None:
|
|
client = _FakeClient(
|
|
get_result={"score_safety": 0.2, "slug": "tag-memory", "skill_md": "evil"},
|
|
install_result={"slug": "tag-memory", "version": "v1", "skill_md": "evil"},
|
|
)
|
|
out = await UseSkillTool(client=client, min_safety=0.7).execute(
|
|
skill_id="hub/tag-memory",
|
|
)
|
|
assert out.startswith("Error")
|
|
assert "score_safety" in out
|
|
assert client.install_calls == []
|
|
|
|
async def test_score_missing_allows_install(self) -> None:
|
|
client = _FakeClient(
|
|
get_result={"slug": "foo"},
|
|
install_result={"slug": "foo", "version": "v1", "skill_md": "ok"},
|
|
)
|
|
out = await UseSkillTool(client=client, min_safety=0.7).execute(
|
|
skill_id="hub/foo",
|
|
)
|
|
assert not out.startswith("Error")
|
|
assert client.install_calls == ["foo"]
|
|
|
|
async def test_blocklisted_hub_skill_refused_before_network(self) -> None:
|
|
client = _FakeClient(get_result={"slug": "tag-memory"})
|
|
out = await UseSkillTool(client=client, blocklist=["Tag-Memory"]).execute(
|
|
skill_id="hub/tag-memory",
|
|
)
|
|
assert out.startswith("Error") and "blocklist" in out
|
|
assert client.get_calls == []
|
|
assert client.install_calls == []
|
|
|
|
async def test_blocklisted_local_skill_refused(self, tmp_path: Path) -> None:
|
|
meta = _meta(tmp_path, "x", "body", with_scripts=True)
|
|
# Keyed under a real registry layer, not the "local" router namespace:
|
|
# `local/x` is a namespace-qualified id the router strips, and a fixture
|
|
# keyed under ("local", ...) would let a passthrough bug read as green.
|
|
reg = _FakeRegistry({("workspace", "x"): meta})
|
|
out = await UseSkillTool(registry=reg, blocklist=["x"]).execute(
|
|
skill_id="local/x",
|
|
)
|
|
assert out.startswith("Error") and "blocklist" in out
|
|
|
|
async def test_policy_pass_reuses_prefetched_meta(self) -> None:
|
|
client = _FakeClient(
|
|
get_result={"score_safety": 0.9, "slug": "foo"},
|
|
install_result={"slug": "foo", "version": "v1", "skill_md": "ok"},
|
|
)
|
|
out = await UseSkillTool(client=client, min_safety=0.7).execute(
|
|
skill_id="hub/foo",
|
|
)
|
|
assert not out.startswith("Error")
|
|
assert client.get_calls == ["foo"]
|
|
assert client.install_meta == [{"score_safety": 0.9, "slug": "foo"}]
|
|
|
|
async def test_external_path_in_body_refused(self) -> None:
|
|
client = _FakeClient(
|
|
get_result={
|
|
"score_safety": 0.9,
|
|
"slug": "tag-memory",
|
|
"skill_md": "python scripts/db.py --db ~/.openclaw/db.sqlite",
|
|
},
|
|
)
|
|
out = await UseSkillTool(client=client).execute(skill_id="hub/tag-memory")
|
|
assert out.startswith("Error") and "~/.openclaw" in out
|
|
assert client.install_calls == []
|
|
|
|
async def test_install_appends_audit_record(self, tmp_path: Path) -> None:
|
|
import json
|
|
|
|
audit = tmp_path / "installs.jsonl"
|
|
client = _FakeClient(
|
|
get_result={"score_safety": 0.9, "slug": "foo", "version": "v2"},
|
|
install_result={"slug": "foo", "version": "v2", "skill_md": "ok", "dir": "/cache/foo"},
|
|
)
|
|
out = await UseSkillTool(
|
|
client=client,
|
|
min_safety=0.7,
|
|
install_audit_path=audit,
|
|
).execute(skill_id="hub/foo")
|
|
assert not out.startswith("Error")
|
|
rec = json.loads(audit.read_text(encoding="utf-8").strip())
|
|
assert rec["slug"] == "foo"
|
|
assert rec["trigger"] == "use_skill"
|
|
|
|
async def test_install_writes_install_meta(self, tmp_path: Path) -> None:
|
|
import json
|
|
|
|
skill_dir = tmp_path / "foo@v2"
|
|
skill_dir.mkdir()
|
|
client = _FakeClient(
|
|
get_result={"score_safety": 0.9, "slug": "foo", "version": "v2"},
|
|
install_result={"slug": "foo", "version": "v2", "skill_md": "ok", "dir": str(skill_dir)},
|
|
)
|
|
out = await UseSkillTool(client=client).execute(skill_id="hub/foo")
|
|
assert not out.startswith("Error")
|
|
rec = json.loads((skill_dir / ".install-meta.json").read_text(encoding="utf-8"))
|
|
assert rec["slug"] == "foo"
|
|
assert rec["version"] == "v2"
|
|
assert rec["source"] == "hub"
|
|
assert rec["trigger"] == "use_skill"
|
|
assert rec["installed_at"]
|
|
|
|
|
|
class _FakeStdin:
|
|
def __init__(self, *, tty: bool) -> None:
|
|
self._tty = tty
|
|
|
|
def isatty(self) -> bool:
|
|
return self._tty
|
|
|
|
|
|
class TestUseSkillAutoInstall:
|
|
"""skillForge.autoInstall gates the Hub bundle download in use_skill."""
|
|
|
|
def _client(self) -> _FakeClient:
|
|
return _FakeClient(
|
|
get_result={"score_safety": 0.9, "slug": "foo", "skill_md": "ok"},
|
|
install_result={"slug": "foo", "version": "v1", "skill_md": "ok"},
|
|
)
|
|
|
|
async def test_off_skips_install_with_notice(self) -> None:
|
|
client = self._client()
|
|
out = await UseSkillTool(client=client, auto_install="off").execute(
|
|
skill_id="hub/foo",
|
|
)
|
|
assert out.startswith("Skill install skipped")
|
|
assert "'off'" in out
|
|
assert client.install_calls == []
|
|
|
|
async def test_prompt_without_tty_skips(self, monkeypatch) -> None:
|
|
import sys
|
|
|
|
monkeypatch.setattr(sys, "stdin", _FakeStdin(tty=False))
|
|
client = self._client()
|
|
out = await UseSkillTool(client=client, auto_install="prompt").execute(
|
|
skill_id="hub/foo",
|
|
)
|
|
assert out.startswith("Skill install skipped")
|
|
assert client.install_calls == []
|
|
|
|
async def test_prompt_tty_confirmation_installs(self, monkeypatch) -> None:
|
|
import sys
|
|
|
|
monkeypatch.setattr(sys, "stdin", _FakeStdin(tty=True))
|
|
monkeypatch.setattr("builtins.input", lambda _prompt="": "y")
|
|
client = self._client()
|
|
out = await UseSkillTool(client=client, auto_install="prompt").execute(
|
|
skill_id="hub/foo",
|
|
)
|
|
assert not out.startswith("Error") and not out.startswith("Skill install skipped")
|
|
assert client.install_calls == ["foo"]
|