Absorbs the GitHub release into the downstream trunk. A merge rather than a rebase so v0.1.11 becomes a real ancestor: the next sync then starts from v0.1.11 instead of the July fork point, and the eventual contribution back to GitHub can tell our commits from the upstream ones it already has. Notable conflict resolutions, 19 hunks over 14 files: * providers/litellm_provider.py takes upstream's wire_model delegation. Both sides had fixed the same gateway-prefix bug; upstream moved the logic into providers/wire.py and deleted _canonicalize_explicit_prefix, so keeping our call would reference a method that no longer exists. * cli/onboard_commands.py takes upstream's three-way file split. Our only change to that file was _NON_CREDENTIAL_CHANNEL_FIELDS, ported to its new home in cli/onboard_channels.py. * agent/loop/main.py keeps our stream reconnect loop and grafts upstream's terminal error-delta handling into it; takes upstream's _route_result_images helper, which also covers the no-vision case our inline version did not, and keeps our read_skill/use_skill reporting; takes upstream's ordering that routes before assembly, since assembly decides whether an attachment is inlined as a picture, while keeping our per-session model override. * agent/subagent/manager.py keeps our backend dispatch and spawn history, and threads the provider/model pair spawn snapshots into backend.run so a live /model switch reaches subagents. The SubagentBackend protocol gained the pair as optional keywords, travelling per call for the same reason workspace already does. Three breakages git merged cleanly and never flagged: * web_rpc/methods_config.py imported six EverOS symbols from onboard_commands, which upstream moved to onboard_everos. * tui_rpc/methods/turn.py passed a single directory to _resolve_path, whose third parameter we had changed to a tuple. Left as it was, an attachment was dropped in silence whenever restrict_to_workspace is on. * config/schema.py carried a second configuredness rule next to the one upstream routes through providers.auth, so routing bypassed the endpoints-first check and read a flat key that no request would use. It now delegates, with include_external for parity with list_providers. Verification: uv run ruff check passes; uv run pytest reports 6823 passed and 4 failed. All four fail on origin/main as well (one theme assertion, three channel QR cases) and are left for a separate fix. Upstream's own 53 changed test files pass, and so do our 85, which is what pins both sides' semantics through these resolutions. Co-authored-by: Claude (claude-opus-5) <noreply@anthropic.com>
268 lines
10 KiB
Python
268 lines
10 KiB
Python
"""Tests for raven.token_wise.cache_optimizer.CacheOptimizer."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
|
|
import pytest
|
|
|
|
from raven.token_wise.cache_optimizer import CacheOptimizer
|
|
|
|
# Anthropic models support cache_control per the provider registry.
|
|
ANTHROPIC_MODEL = "anthropic/claude-sonnet-4-5"
|
|
# A provider that does NOT accept `cache_control` blocks → strategy must be a no-op.
|
|
NON_CACHE_MODEL = "deepseek/deepseek-chat"
|
|
|
|
|
|
def _system_msg(text: str = "You are helpful.") -> dict:
|
|
return {"role": "system", "content": text}
|
|
|
|
|
|
def _user_msg(text: str = "hello") -> dict:
|
|
return {"role": "user", "content": text}
|
|
|
|
|
|
def _assistant_msg(text: str = "hi back") -> dict:
|
|
return {"role": "assistant", "content": text}
|
|
|
|
|
|
def _tool(name: str = "search") -> dict:
|
|
return {
|
|
"type": "function",
|
|
"function": {
|
|
"name": name,
|
|
"description": "x",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
|
|
|
|
def _has_cache_control(block: dict) -> bool:
|
|
return isinstance(block, dict) and block.get("cache_control") == {"type": "ephemeral"}
|
|
|
|
|
|
def _content_blocks_with_cache(msg: dict) -> int:
|
|
content = msg.get("content")
|
|
if isinstance(content, list):
|
|
return sum(1 for b in content if _has_cache_control(b))
|
|
return 0
|
|
|
|
|
|
def _count_breakpoints(messages: list[dict], tools: list[dict] | None) -> int:
|
|
n = sum(_content_blocks_with_cache(m) for m in messages)
|
|
if tools:
|
|
n += sum(1 for t in tools if _has_cache_control(t))
|
|
return n
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Behaviour
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def test_constructor_validates_max_breakpoints():
|
|
with pytest.raises(ValueError):
|
|
CacheOptimizer(max_breakpoints=0)
|
|
|
|
|
|
async def test_noop_for_non_cache_models():
|
|
opt = CacheOptimizer()
|
|
msgs = [_system_msg(), _user_msg()]
|
|
tools = [_tool()]
|
|
out_m, out_t, out_model = await opt.before_llm_call(msgs, tools, NON_CACHE_MODEL)
|
|
assert out_m is msgs
|
|
assert out_t is tools
|
|
assert out_model == NON_CACHE_MODEL
|
|
|
|
|
|
async def test_noop_when_model_empty():
|
|
opt = CacheOptimizer()
|
|
msgs = [_user_msg()]
|
|
out_m, _, _ = await opt.before_llm_call(msgs, None, "")
|
|
assert out_m is msgs
|
|
|
|
|
|
async def test_marks_tools_and_system_for_cache_capable_model():
|
|
opt = CacheOptimizer()
|
|
msgs = [_system_msg(), _user_msg()]
|
|
tools = [_tool("a"), _tool("b")]
|
|
out_m, out_t, _ = await opt.before_llm_call(msgs, tools, ANTHROPIC_MODEL)
|
|
|
|
# Last tool marked.
|
|
assert _has_cache_control(out_t[-1])
|
|
# Earlier tool NOT marked.
|
|
assert not _has_cache_control(out_t[0])
|
|
# System content has cache_control on a content block.
|
|
sys_msg = next(m for m in out_m if m["role"] == "system")
|
|
assert _content_blocks_with_cache(sys_msg) == 1
|
|
|
|
|
|
async def test_does_not_mutate_inputs():
|
|
"""Original messages and tools must remain pristine."""
|
|
opt = CacheOptimizer()
|
|
msgs = [_system_msg(), _user_msg(), _assistant_msg(), _user_msg("again")]
|
|
tools = [_tool("a"), _tool("b")]
|
|
msgs_before = copy.deepcopy(msgs)
|
|
tools_before = copy.deepcopy(tools)
|
|
|
|
await opt.before_llm_call(msgs, tools, ANTHROPIC_MODEL)
|
|
|
|
assert msgs == msgs_before, "input messages were mutated"
|
|
assert tools == tools_before, "input tools were mutated"
|
|
|
|
|
|
async def test_string_content_is_wrapped_into_text_block():
|
|
opt = CacheOptimizer()
|
|
msgs = [_system_msg("plain string system")]
|
|
out_m, _, _ = await opt.before_llm_call(msgs, None, ANTHROPIC_MODEL)
|
|
sys = out_m[0]
|
|
assert isinstance(sys["content"], list)
|
|
assert sys["content"][-1]["type"] == "text"
|
|
assert sys["content"][-1]["text"] == "plain string system"
|
|
assert sys["content"][-1]["cache_control"] == {"type": "ephemeral"}
|
|
|
|
|
|
async def test_list_content_marks_only_last_block():
|
|
opt = CacheOptimizer()
|
|
msgs = [
|
|
{
|
|
"role": "system",
|
|
"content": [
|
|
{"type": "text", "text": "first"},
|
|
{"type": "text", "text": "second"},
|
|
{"type": "text", "text": "third"},
|
|
],
|
|
}
|
|
]
|
|
out_m, _, _ = await opt.before_llm_call(msgs, None, ANTHROPIC_MODEL)
|
|
sys = out_m[0]
|
|
blocks = sys["content"]
|
|
assert "cache_control" not in blocks[0]
|
|
assert "cache_control" not in blocks[1]
|
|
assert _has_cache_control(blocks[2])
|
|
|
|
|
|
async def test_long_history_uses_more_breakpoints():
|
|
"""A long conversation should consume up to 4 breakpoints."""
|
|
opt = CacheOptimizer(max_breakpoints=4)
|
|
msgs = [_system_msg()]
|
|
# 5 prior turns to give the optimizer somewhere to place breakpoints.
|
|
for i in range(5):
|
|
msgs.append(_user_msg(f"q{i}"))
|
|
msgs.append(_assistant_msg(f"a{i}"))
|
|
msgs.append(_user_msg("current question"))
|
|
tools = [_tool()]
|
|
out_m, out_t, _ = await opt.before_llm_call(msgs, tools, ANTHROPIC_MODEL)
|
|
n = _count_breakpoints(out_m, out_t)
|
|
# tools(1) + system(1) + before-current-user(1) + mid(1) = 4
|
|
assert n == 4
|
|
|
|
|
|
async def test_max_breakpoints_is_respected():
|
|
"""With max_breakpoints=2 we never place more than 2 markers."""
|
|
opt = CacheOptimizer(max_breakpoints=2)
|
|
msgs = [_system_msg()]
|
|
for i in range(5):
|
|
msgs.append(_user_msg(f"q{i}"))
|
|
msgs.append(_assistant_msg(f"a{i}"))
|
|
msgs.append(_user_msg("current"))
|
|
tools = [_tool()]
|
|
out_m, out_t, _ = await opt.before_llm_call(msgs, tools, ANTHROPIC_MODEL)
|
|
n = _count_breakpoints(out_m, out_t)
|
|
# With budget=2 and tools present: tools(1) + system(1) = 2, no room for rolling tail.
|
|
assert n == 2
|
|
|
|
|
|
async def test_short_conversation_uses_fewer_breakpoints():
|
|
"""[system, user] with no tools → system + user (rolling tail marks the user msg too)."""
|
|
opt = CacheOptimizer(max_breakpoints=4)
|
|
msgs = [_system_msg(), _user_msg("hi")]
|
|
out_m, out_t, _ = await opt.before_llm_call(msgs, None, ANTHROPIC_MODEL)
|
|
# No tools: system(1) + rolling tail marks the one non-system msg(1) = 2
|
|
n = _count_breakpoints(out_m, out_t)
|
|
assert n == 2
|
|
|
|
|
|
async def test_no_system_no_tools_no_history():
|
|
"""Edge case: just one user message. Rolling tail marks it for next-call cache."""
|
|
opt = CacheOptimizer()
|
|
msgs = [_user_msg("hi")]
|
|
out_m, out_t, _ = await opt.before_llm_call(msgs, None, ANTHROPIC_MODEL)
|
|
# One non-system msg → 1 rolling breakpoint (prepares cache for next iteration).
|
|
assert _count_breakpoints(out_m, out_t) == 1
|
|
|
|
|
|
async def test_only_tools_no_messages():
|
|
opt = CacheOptimizer()
|
|
tools = [_tool()]
|
|
out_m, out_t, _ = await opt.before_llm_call([], tools, ANTHROPIC_MODEL)
|
|
assert out_m == []
|
|
assert _has_cache_control(out_t[-1])
|
|
|
|
|
|
async def test_tools_unchanged_when_no_tools_supplied():
|
|
opt = CacheOptimizer()
|
|
msgs = [_system_msg(), _user_msg()]
|
|
out_m, out_t, _ = await opt.before_llm_call(msgs, None, ANTHROPIC_MODEL)
|
|
assert out_t is None
|
|
|
|
|
|
async def test_idempotent_repeated_application():
|
|
"""Running the optimizer twice must not duplicate breakpoints."""
|
|
opt = CacheOptimizer()
|
|
msgs = [_system_msg(), _user_msg("hi")]
|
|
once_m, _, _ = await opt.before_llm_call(msgs, None, ANTHROPIC_MODEL)
|
|
twice_m, _, _ = await opt.before_llm_call(once_m, None, ANTHROPIC_MODEL)
|
|
# Same cache count; the marker is overwritten not duplicated.
|
|
assert _count_breakpoints(once_m, None) == _count_breakpoints(twice_m, None)
|
|
|
|
|
|
def test_cache_control_follows_the_wire_format_not_the_model_catalogue():
|
|
"""`cache_control` is Anthropic-shaped; the question is who accepts one.
|
|
|
|
LiteLLM's table carries a per-model `supports_prompt_caching`, and reading it
|
|
here looks like an upgrade: it says DeepSeek's and OpenAI's models cache,
|
|
which they do. But their caching is automatic and takes no breakpoints, so
|
|
acting on that flag stamped `cache_control` onto requests to APIs with
|
|
nowhere to put it -- and `ProviderSpec.supports_prompt_caching` already says
|
|
what it means, "this provider accepts the block".
|
|
|
|
The lesson is the naming: two flags spelled the same answer different
|
|
questions, and the more precise-looking one was the wrong one.
|
|
"""
|
|
from pathlib import Path
|
|
|
|
from raven.providers import capabilities
|
|
from raven.providers.registry import PROVIDERS
|
|
from raven.token_wise import cache_optimizer, system_and_tail_cache
|
|
|
|
lying = [s.name for s in PROVIDERS if s.supports_prompt_caching and s.via_driver == "openai"]
|
|
assert not lying, f"marked as accepting cache_control while speaking OpenAI's API: {lying}"
|
|
|
|
for module in (cache_optimizer, system_and_tail_cache):
|
|
source = Path(module.__file__).read_text(encoding="utf-8")
|
|
assert "capabilities import supports_prompt_caching" not in source, module.__name__
|
|
assert not hasattr(capabilities, "supports_prompt_caching"), "the misfounded lookup is back"
|
|
|
|
|
|
def test_a_gateway_alone_does_not_decide_caching_for_what_it_fronts():
|
|
"""Both halves are required: the wire has somewhere to put the field, and the
|
|
model's vendor is the one that reads it.
|
|
|
|
aihubmix / siliconflow / volcengine / custom speak an API with nowhere to
|
|
carry it, so nothing they front is marked. OpenRouter has somewhere to put
|
|
it and accepts it on every model it fronts -- and forwards it to vendors
|
|
that do not read it, which is what billed Gemini twice. So it is marked for
|
|
the Anthropic family and for nothing else.
|
|
"""
|
|
from raven.providers.litellm_provider import LiteLLMProvider
|
|
|
|
for slug in ("siliconflow", "aihubmix", "volcengine", "custom"):
|
|
provider = LiteLLMProvider(api_key="", default_model="x", provider_name=slug)
|
|
assert provider._supports_cache_control("anthropic/claude-fable-5") is False, slug
|
|
|
|
openrouter = LiteLLMProvider(api_key="", default_model="x", provider_name="openrouter")
|
|
assert openrouter._supports_cache_control("openrouter/anthropic/claude-fable-5") is True
|
|
for other in ("openrouter/google/gemini-3.5-flash", "openrouter/qwen/qwen3.7-max", "openrouter/anything"):
|
|
assert openrouter._supports_cache_control(other) is False, other
|