2026-08-18 17:30:49 +08:00
|
|
|
"""Coverage reporting and governance gates shared by CI and local development."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import json
|
|
|
|
|
import re
|
|
|
|
|
import shutil
|
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from typing import Any, Iterable
|
|
|
|
|
|
|
|
|
|
SCHEMA_VERSION = 1
|
|
|
|
|
DEFAULT_TOLERANCE = 0.05
|
|
|
|
|
# Glob magic so ** matches zero-or-more path components; the default pathspec
|
|
|
|
|
# raven/**/*.py drops top-level files like raven/__init__.py because plain * spans
|
|
|
|
|
# slashes and the middle separator then has nothing to match.
|
fix(tools): an array sent as its own JSON is still an array
## Summary
Reported as "deliver_files is broken". It is not: **the tool never ran.**
Every `deliver_files` call on one install was refused before dispatch -- five in a row in a single session -- and nothing was ever delivered:
```
Error: Invalid parameters for tool 'deliver_files': files should be array
```
What the model sent was the array, spelled as a string:
```json
{"files": "[{\"path\": \"...poster.png\", \"title\": \"...\"}]", "message": "..."}
```
`cast_params` already forgives that spelling for every scalar type -- `"3"` for an integer, `"true"` for a boolean -- and did not for arrays or objects, which are exactly the types a model is most likely to stringify.
## Type
- [x] Fix
- [ ] Feature
## The edge of the forgiving is the design
When the value is a string, try `json.loads`, and take the result **only if it is the type the schema asked for**. A string that is not JSON, or is JSON of the wrong shape, is returned unchanged so the validator still names the real problem -- the point is to widen what is accepted, not to swallow a type error.
**Nothing here can raise.** Turning "your argument has the wrong type" into a traceback would be worse than the refusal it replaces, so the parse is wrapped and failure falls through.
**A `type: "string"` field is never parsed.** JSON is ordinary text in a message body or a file's content, and replacing it would hand the tool something it never asked for. Two independent things prevent it today (the string branch returns before the parse is reached, and the parse only wins on a type match), which is why the case pinning it is a tripwire rather than a load-bearing assertion -- said so in its own docstring rather than left to look like coverage.
## Not just deliver_files
Six built-in tools declare an array or object parameter and were all exposed:
| tool | parameter |
|---|---|
| `deliver_files` | `files` |
| `ask_user` | `questions` |
| `image_generate` | `images` |
| `message` | `media` |
| `create_playbook` | `skills` |
| `video_generate` | `params` |
The same failure was already seen from the other side: `acp_client/acp_agent.py` records `update_task_state` and `operations should be array` from a real dispatch.
## Verification
```
make lint-python ruff check + ruff format --check, 1850 files
make lint-imports 9 contracts kept, 0 broken
pytest tests/ 19,699 passed, 7 failed (all pre-existing, see below)
```
- [x] Relevant tests pass locally
- [x] Relevant lint / type checks pass locally
- [ ] User-facing docs or screenshots -- no user-facing string
**End to end, through the real tool**, with the exact payload that failed five times in production:
```
files sent as: str -> [{"path": ".../poster.png", "title": "..."}]
returned: Delivered 1 file(s): poster.png (72 B).
still "should be array": False
registered in the store: True
```
Twelve cases, five mutations, each killing at least one:
| mutation | red |
|---|---|
| drop the whole branch (the old behaviour) | 3 |
| raise instead of falling through on a parse failure | 5 |
| accept the parse whatever type it yielded | 1 |
| do not re-cast the parsed value, so items are not cast | 1 |
| widen the tuple to include `string` | **survived -- unreachable** |
The last is worth stating rather than hiding: that mutation cannot change behaviour, because the `type == "string"` branch returns earlier and the type check would refuse the result anyway. I also tried moving the new block above that branch, alone and combined with widening the tuple, and neither reddens anything either. The invariant holds by construction; the case that documents it says so.
**Pre-existing failures, checked against the base.** 7 failed in the full run, all in `tests/test_security_network.py` (`test_a_redirect_to_a_dangerous_address_is_refused` and siblings). Running that file alone on this same tree gives 4 -- the count is order-dependent -- and the change touches only `raven/agent/tools/params.py` plus a new test file, neither of which `raven/security/network.py` imports. The failures are the box's resolver: it returns poisoned answers roughly half the time, which I measured separately (`en.wikipedia.org` came back as `2001::1` on 3 of 6 lookups). Not introduced here, and worth someone's attention on its own.
## Risk
- [x] Security impact considered -- widens what reaches a tool's own validation, not what bypasses it. A parsed value is still validated against the same schema, and a parse that yields the wrong type is refused exactly as before
- [x] Backward compatibility considered -- a call that worked before takes the identical path; the new branch is reached only by a `str` where an array or object is declared, which previously always failed
- [x] Rollback path is clear -- single-commit revert
## Related Issues
N/A
## Swept
`mr-review-patterns` pre-submit sweep, over `git diff origin/main...HEAD`.
- **1 (a write location moved)** -- nothing moved; a branch was added to an existing dispatch on `target_type`. Grepped every caller of `cast_params`: the registry is the only one, and it runs `validate_params` right after, which is the reader this affects.
- **2 (a value with several entry points)** -- the cast is the single funnel every tool argument passes through, which is why the fix belongs here and not in `deliver_files`.
- **3 (a guard)** -- checked both directions: what it wrongly lets through (JSON of the wrong type, non-JSON text -- both refused, with cases) and what it wrongly refuses (a real list, unaffected, with a case).
- **4 (tests)** -- five mutations above, one honestly reported as unreachable. Failing-ID sets diffed against the base rather than counts.
- **6 (stale text)** -- the module docstring enumerated what is coerced ("a string that should be an integer"); it now names arrays and objects too. Grepped the tracked tree for other statements of the rule: none.
- **9 (error handling)** -- the new path adds one way to end and it is caught: `json.loads` raises `ValueError` on bad text and `TypeError` on a non-string, and both fall through to the untouched value.
- **11** -- `git ls-files | grep -c node_modules` is 0; the commit-message gate was run, not eyeballed.
Not applicable, with reasons: **3b, 3c, 3d, 3e** (no tool schema produced, no state moved, one lane, no assertion lost its symbol); **4b, 4c, 4d** (no platform branch, no shell, no rendered number); **5, 5b** (no dependency; `json` is stdlib and already imported across this package); **6b, 6c** (no backup, no generated artifact); **7, 7b** (nothing on disk); **8** (no asset, no secret); **10** (no prompt text).
## Deliberately not fixed
The error text still ends with `[Analyze the error above and try a different approach.]`, and the model plainly did not -- it sent the same shape five times. A message naming the expected spelling (`files must be a JSON array, not a string containing one`) would give it something to act on. That is a change to the registry's error surface, affects every tool, and deserves its own argument.
## Moved from !594
Opened on a compliant source branch. AGENTS.md 2.1 asks for a 3-5 word snake_case description and the previous branch had seven; GitLab cannot change an existing merge request's source branch, so the change moves rather than the name.
Same commits, rebased onto `c2a9768f`. Both code findings from the review on !594 are already in them -- `RecursionError` caught with its regression case, and the implementation comment reduced to the two non-obvious reasons. Nothing else changed, so that review carries.
2026-09-09 15:42:36 +00:00
|
|
|
PRODUCTION_PATHSPECS = (
|
|
|
|
|
":(glob)raven/**/*.py",
|
ci(*): run unit tests as four shards and speed up the suite (#413)
## Summary
The unit job takes twelve minutes on one runner. The first cut of this
PR took the production waits out of the tests, which made the suite a
quarter faster on a laptop and moved CI by 14 s (pytest 720.95 s on
main, 706.57 s here): the runner is three to four times slower than a
laptop on CPU-bound tests, and two of the twelve minutes are the four
xdist workers importing every test module. So the PR now attacks the
wall clock from every side that CI's own numbers pointed at.
- `ci(*)`: the unit job runs as four shards. pytest gains `--shard K/N`
in `tests/conftest.py`, decided in `pytest_ignore_collect` so a shard
never imports the other shards' files; test files are dealt round-robin
over their sorted list so the slow families that sort together (the ppt
engine's) are spread out. Each shard writes coverage data and no report;
a `coverage` job downloads the four data files, combines them, writes
`coverage.json` and `coverage.xml`, and runs the gates that used to
follow the tests.
- `perf(ppt-engine)`: `menu()` caches its reading of a template, capped
at eight entries. One bind reads the bound template and then every other
bundled template for the borrow offer; with ten templates bundled, every
bind evicted what the next one needed and re-measured all of them with
freetype. The cap is 32 and a hit moves its entry to the back. On this
PR's first CI run the template-tool file alone was 257 s of the
over-ceiling list, its tests 10 to 26 s each.
- `test(*)`: an autouse fixture collapses the provider chat retry
ladder, the loop's LLM-error ladder and the memory store backoff to a
millisecond per step, and the steer hook grace to one poll. It keeps
each tuple's length because both ladders count attempts with `len()`;
the steps are a millisecond rather than zero because the store pipeline
waits with `wait_for(stopping.wait(), timeout=delay)`, and a zero
timeout cancels that wait before it runs. A test that proves a timing
property opts out with the new `production_timing` marker. Ten tests
spent 257 s of the suite's worker time in those waits; none asserted on
the time.
- `test(*)`: a wall-clock ceiling. Setup plus call over 2 s on a test
marked neither `slow` nor `production_timing` is listed at the end of
the run; `--wall-clock-ceiling-strict` fails an otherwise green session
on it. Warning only here; CI flips to strict once the remaining tail is
marked.
- `test(*)`: every test created two numbered temp directories, and
pytest picks each number by scanning the base temp dir; the fake home is
now a sibling of `tmp_path`. `MPLCONFIGDIR` points at a git-ignored dir
in the checkout, because the fake home changed per test and matplotlib
rebuilt its font list from the system fonts on every run.
- `refactor(agent)`: the outer LLM-error retry ladder default `(15, 30,
60)` lived in three places; it is one constant in
`raven/config/schema.py` now, read by both `RecoveryLimits` and
`_ladder`. Behaviour unchanged.
- `test(updates)`: the first sharded run turned up an order dependency.
`beta_channel` binds `ReleaseInfo` from `raven.updates.upgrade` at
import, the update-notice tests stand a fake in for that module through
`sys.modules`, and `_refresh` imports `beta_channel` lazily; a worker
that imported `beta_channel` for the first time inside that test got an
`ImportError` swallowed as a failed refresh. The test module now imports
`beta_channel` at the top.
- `ci(*)`: the PR diff gate named every `.py` under `plugins-dist` as
production code while coverage measures only `raven` and `raven_everos`,
so a change to the ppt or design engine could never appear in the report
and the gate refused every such PR on sight; the first sharded run hit
it on `menu.py`. The pathspec now names `plugins-dist/everos-memory`,
and a test holds every production pathspec to a package some `--cov`
flag in the Makefile measures.
- Two slow tests were not waits: the sandbox second-cancel test
cancelled one tick too early (two pending cancels coalesce into one
`CancelledError`) and now waits, under a one second timeout, for the
kill to land; the two thousand-session dispatch tests wrote a span per
refused enqueue through a store that opens its file per span, so tracing
is off for them. Follow-up on the product side: `raven/tracing/store.py`
should reuse its handle.
The suite has 30 upper-bound assertions of the form `assert elapsed <
X`; shortened delays can only make those looser.
## Type
- [ ] Fix
- [ ] Feature
- [ ] Docs
- [x] CI / tooling
- [ ] Refactor
- [ ] Other
## Verification
Whole suite, `uv run --frozen --all-extras pytest -q` (macOS, 4
workers), 22051 passed / 43 skipped each time:
| tree | wall clock | unmarked tests over 2 s |
|---|---|---|
| main before this branch | 372.82 s | (no ceiling yet) |
| after the waits were taken out | 279.65 s | 63 |
| this branch | 224.25 s | 39 |
The shards partition the suite exactly (`uv run --frozen --all-extras
pytest --collect-only -q -n 0 --shard K/4`, K = 1..4):
| whole | shard 1 | shard 2 | shard 3 | shard 4 | union | ids in two
shards |
|---|---|---|---|---|---|---|
| 22095 | 5058 | 5555 | 5955 | 5527 | 22095 | 0 |
196 / 195 / 194 / 194 test files per shard. A shard collects in 7 s
where the whole suite takes 23 s on the same machine. `--ignore` still
works under `--shard`: with four `--ignore` flags on shard 1, the
ignored file that shard owns is gone from its collection (5034 ids
against 5058).
Combining loses nothing. Two small runs with `--cov=raven
--cov=raven_everos --cov-branch --cov-report=` and `COVERAGE_FILE` set,
then `make coverage-combine COVERAGE_DATA_DIR=<dir>`, against one `make
coverage` of the whole suite on main:
| report | files | statements | branches |
|---|---|---|---|
| one run of the whole suite | 704 | 82634 | 25548 |
| two shards combined | 704 | 82636 | 25548 |
Same file set; the two extra statements are the new constant and its
import. A source package's never-imported files reach the combined
report through the shard that imported it, as they reached the single
run.
The menu cache, the three files that bind templates (`uv run --frozen
--all-extras pytest -q tests/test_ppt_engine_template_tool.py
tests/test_ppt_engine_outline_checks.py
tests/test_ppt_engine_default_templates.py`): 101 s of worker time in
the last profile, now `196 passed in 11.84s` with no test over 2.9 s.
The files that call `menu()` directly: `219 passed in 8.06s`.
The font cache, the formula render test at `-n 0`: 8.91 s cold, 0.69 s
on the next run. Seven home-sensitive files: `484 passed in 5.30s`.
Ceiling guard, both modes, with the ceiling forced low so it has
something to list:
```
uv run --frozen --all-extras pytest -q -n 0 --wall-clock-ceiling=0.01 tests/test_import_cycle_budget.py tests/test_updates_install_guard.py
uv run --frozen --all-extras pytest -q -n 2 --wall-clock-ceiling=0.01 --wall-clock-ceiling-strict tests/test_updates_install_guard.py
```
First: 28 passed, five listed, exit 0. Second: 24 passed, four listed,
exit 1 (the controller flips the status; workers do not).
`make lint-python` clean, `make lint-imports` passes. The CI run on this
PR is the measurement that matters: four `unit` shards and one `coverage
gates` job.
- [x] Relevant tests pass locally
- [x] Relevant lint / type checks pass locally
- [ ] User-facing docs or screenshots are updated when needed (none: CI
plumbing, a cache cap, test setup)
## Risk
- The coverage gates now run in a job after the four shards. A red shard
skips the gates, a deliberate trade: combining three of four data files
would report a coverage drop that is not one, and the PR is red either
way. The gate scripts and their thresholds are unchanged.
- The diff gate no longer sees `plugins-dist/ppt-engine` or
`plugins-dist/design-engine` at all, as it could never judge them;
measuring them (`--cov=raven_ppt --cov=raven_design`, a new ratchet
baseline) is the follow-up that would bring them under the gate.
- `--shard` is opt-in; a plain `pytest` run collects everything as
before. Shard data artifacts are kept one day, the combined reports 30
days as before.
- No product behaviour change: the ladder default is the same tuple read
from one place; the menu cap only changes how long the cache keeps
entries.
- Test-only risk: a test that implicitly relied on a production-length
retry delay would see different timing. The full suite passed with the
fixture on, and a single test can opt out with `production_timing`.
- Rollback: revert the PR. The unit job goes back to one runner with the
gates inline; both markers are additive.
- [x] Security impact considered (no security surface touched)
- [x] Backward compatibility considered
- [x] Rollback path is clear for risky changes
## Related Issues
N/A
---------
Co-authored-by: Claude (claude-fable-5-1) <noreply@anthropic.com>
2026-09-15 01:00:49 -07:00
|
|
|
# The everos memory plugin beside the host wheel is production code too;
|
|
|
|
|
# without this a change there dodges the diff gate. Only that plugin: the
|
|
|
|
|
# gate can judge what coverage measures (--cov=raven --cov=raven_everos), and
|
|
|
|
|
# a changed file in an unmeasured tree would fail every PR that touches it.
|
|
|
|
|
":(glob)plugins-dist/everos-memory/**/*.py",
|
feat(plugins): add the everos-cloud-memory plugin and its host wiring (#883)
## Summary
Adds `everos-cloud-memory`, a separately distributed memory plugin for
EverOS Cloud, and wires the host so a second memory plugin is a
first-class choice.
The plugin (`plugins-dist/everos-cloud-memory`) is a thin REST client
over httpx: one API key (the config slice
`plugins.config.everos-cloud-memory.api_key`, or
`EVEROS_CLOUD_API_KEY`), owner ids from the host's `ServiceLocator`
(never from the slice), every add in `agent` mode, and a flush only when
a caller marks the conversation over (`metadata.flush` /
`metadata.is_final`: the sub-agent handoff and the importer's final
batch) and once at stop. It ships its own onboard screen (env or on-file
key used without a prompt; 401 / 403 / unreachable get their own
sentences with Re-enter or Skip), `delete` answers False (the cloud
deletes by scope, nothing calls it), and `health()` classifies seven
answers for `raven doctor` and the settings card.
Host changes, each guarded so `memory.backend` `everos` and `null`
behave exactly as before:
- `raven onboard` step 4 asks "Which memory backend?" when more than one
plugin contributes a screen (preselects the current backend, `Off`
clears it, cancel exits 1 and writes nothing); with one plugin the step
is byte-identical to today.
- The memory page reads whichever backend `memory.backend` names: the
cloud slice's key goes out as a Bearer header and the `/health` probe is
skipped for the cloud; a missing distribution is named in the page's
note.
- `settings.everos` reports `reason: "other_backend"` for any non-EverOS
backend, so the Model page hides the three EverOS-only roles (embedding
stays: knowledge bases use that pin); `settings.everosSet` refuses those
three sections for another backend; `settings.set` admits
`plugins.config.<id>.<field>` against the plugin's manifest through the
same admission door channels use; new `settings.everosCloud` feeds the
card (key source, endpoint, health).
- Provider-key and main-model saves no longer restart the local EverOS
when another backend is active.
- ui-web: an EverOS Cloud card (status chip, key row, endpoint) and the
`settings.everosCloud` data behind it. No page mounts the card: it drew
on Model settings until the owner ruled that a memory backend's endpoint
and credential answer neither question that page asks, and where it
belongs is the Memory section's to say once that section has a design.
Until then the key is entered through `raven onboard` or
`EVEROS_CLOUD_API_KEY`. The fixture gate pins the four optional fields
the offline library never sends.
- Gates: the plugin boundary test scans both memory plugins, coverage
and the release workflow build the second wheel, deptry's ignore list
names the plugin beside `raven_everos`, and a conftest fixture keeps the
second plugin out of entry-point discovery for the existing onboarding
tests (they describe a one-plugin install).
A second topic rides along, because it is what makes the hosted backend
reachable from an agent product at all. Every product under `agents/`
pinned its memory at the stock EverOS address and only `raven-design`
read the host's, so the other four addressed whatever answered on 18791
-- the wizard moves a managed server when the port is taken, and a
server the user runs themselves is wherever they put it.
`inherit_memory_backend` in `raven/config/product_render.py` now carries
`memory.backend`, the endpoint, and (only when the product holds none
itself) the credential from the host into every render, beside the host
denials and plugin opt-outs that already travel there; `raven-design`'s
private copy is deleted and the five product configs stop naming an
address. Identity never travels, which is what keeps five products apart
inside one store. `memory.backend: null` travels like any other value,
because that null is how an operator turns long-term memory off and a
product that kept writing would be the one surface ignoring them; the
product's own default stands only when the host carries no key at all.
Which store memories live in is not a product's choice either: the host
reads a sub-agent's memories back through its own backend, so a product
writing anywhere else left that record empty with nothing reporting it.
`owned` is forced off whenever the local backend is in force, because a
product infers ownership from a slice naming no root -- the shape all
five ship -- and so answered a host that was merely not serving by
starting a second server against a root of its own, stranding every
memory written so far.
Decisions worth a reviewer's eye: the fake cloud
(`tests/_everos_cloud_fake.py`) honours or refuses every field the
backend can send and serves both an in-process transport and an HTTP
face for the real-host acceptance; the live tests
(`tests/integration/test_everos_cloud_real_cloud.py`) skip without
`EVEROS_CLOUD_API_KEY`; run with a real key on 2026-10-09 all six pass
against api.evermind.ai (round trip, get by session, empty flush, 401
wording, probe with and without a key, and extraction without a flush
within two minutes), so the documentation and the service agree on
everything the plugin sends. Design, acceptance cases and plan are under
`docs/specs/2026-10-08-everos-cloud-memory-*.md` and `docs/plans/`.
## Type
- [ ] Fix
- [x] Feature
- [ ] Docs
- [ ] CI / tooling
- [ ] Refactor
- [ ] Other
## Verification
- [x] Relevant tests pass locally
- [x] Relevant lint / type checks pass locally
- [x] User-facing docs or screenshots are updated when needed
Rebased onto main `182dacac0` on 2026-10-13. What that rebase and the
four commits on top were verified with:
- Full suite, `uv run --frozen --python 3.12 --extra dev pytest -q` ->
**29748 passed, 8 failed** in 4m10s. All eight fail identically with
these four commits' sources reverted in-tree at the same base, so none
of them is this branch's: the two `test_the_products_tool_face_*` cases
in the code and oncall launchers (an extra `web_search` in the visible
tool set), `test_agent_loop_token_budget`,
`test_rpc_files::test_a_host_without_libreoffice_says_so`,
`test_ppt_engine_image_search`, `test_shell_background`, and the two
`test_simulation_*` cases.
- Real-host before and after for the inheritance, driving the launcher
the host spawns (`agents/raven-research/run.py`) against an isolated
`RAVEN_HOME` whose EverOS address is `19701`, not the stock port.
Before: the product rendered `base_url: http://localhost:18791` with no
`owned` key, ignored the host, and started an EverOS of its own -- an
orphaned `everos server start --root <its own state root>/everos`
holding 18791, which is the stranding this change exists to stop. After:
it rendered `{"base_url": "http://localhost:19701", "owned": false}` and
said so itself, "the EverOS you manage is not running at
http://localhost:19701 ... Raven does not start or stop it", with `ps`
and `lsof` confirming nothing was started. On the cloud the product
rendered `memory.backend: everos-cloud` with the host's key and left the
out-of-force local slice untouched, and `raven doctor` over that
rendered config answered `everos-cloud: ok https://api.evermind.ai`, so
the credential reached the service. Identity stayed `raven-research-ng`
throughout, and a fingerprint of the operator's own EverOS root was
identical before and after.
- The first CI round on this head found one that the local gates did
not: `tests/test_import_cycle_budget.py`. `inherit_memory_backend` read
the backend-to-distribution pairing from `raven.core.plugin_stack`, and
that edge from `raven.config` into `raven.core` -- which imports
`raven.config` back -- took mutual pairs from 13 to 14 and packages
inside a cycle from 18 to 22. `make lint-imports` keeps ten contracts
and none of them covers that pair, so it was green throughout. The
pairing now lives in `raven/config/raven.py` beside the two fields it
joins (`MemoryConfig.backend` names a contribution,
`PluginsConfig.config` is keyed by the distribution), which `raven.core`
may import and which imports nothing back; `plugin_stack` re-exports the
two names its own callers read from it. An intermediate commit put it in
`raven.contracts` instead, which broke the cycle but took the papers
ceiling from 3700 to 3719 lines -- `tests/test_kernel_budget.py` says a
new paper is a reviewed change to that ceiling, so it moved again rather
than raising it.
- The review round on this head raised one non-blocking finding, fixed
rather than carried forward: the inheritance read `memory.backend` with
`.get()`, which collapses an explicit `null` into an absent key, so a
host with long-term memory off left every packaged agent on its own
shipped backend and still writing. Reproduced by calling the function
directly before the change and after, now read on presence, and pinned
by a case whose mutation (restoring the `.get()`) turns it and nothing
else red.
- One ordering problem found while checking the above, pre-existing and
not fixed here: `tests/test_rpc_settings.py` run before
`tests/test_cli_onboard_commands.py` leaves
`raven.home._current_config_path` pointing at a dead tmp config, so the
onboard cases write somewhere else. With main's own copies of both files
and main's conftest in the tree it is 57 failures; with this branch's,
19. Which cases turn red changes run to run as xdist distributes them
differently.
- `cd ui-web && npx vitest run` -> 876 files, 3728 passed, 0 failed;
`npm run lint` -> 0 errors (4 warnings, all in files this change does
not touch); `npm run gen:check` -> in sync (228 methods); `npm run
type-check` -> clean; `cd ui-tui && npm run lint:i18n && npm run
lint:rpc` -> in sync
- `make lint-python` / `lint-imports` (10 contracts kept) / `lint-types`
/ `lint-deps` -> all pass; `uv lock --check` -> exit 0 with both sides'
new packages present; `scripts/check_source_language.py` and
`scripts/check_large_files.py` on `github/main..HEAD` -> exit 0;
commitlint over every commit -> ok
- Semantic-conflict audit of the rebase, beyond what the suite answers:
21 of the branch's 66 files are also in main's 499, and every one of
main's changes in them is additive and elsewhere in the file. The one
that could have broken silently is main's new `vision.model` /
`vision.provider` simple keys meeting this branch's new
`plugins.config.<id>.<field>` branch in `settings_set`; nothing in the
suite exercises `vision.*` through that method, so it was run directly
-> `{'applied': True}` and `{"model": "..."}` on disk. The subtraction
decisions were re-checked against the new base: no new caller of a
memory backend's `delete`, main's new onboard cases are about TLS and
not the memory step, `inherit_everos_address` exists nowhere, and
`DEFAULT_EVEROS_BASE_URL` is still defined.
- Mutation runs on the new code, restored afterwards: dropping `owned:
false`, letting the host's credential overwrite the product's own, not
writing the host's backend name, and carrying every memory slice instead
of the one in force each turn their own cases red; making the cloud card
render nothing turns 4 of its 5 new cases red and leaves the fifth
("draws nothing when not selected") green.
From the earlier round, on base `6d509a8c5`:
- `uv run --frozen --python 3.12 --extra dev pytest -p no:cacheprovider
-q tests/test_everos_cloud_fake.py tests/test_everos_cloud_backend.py
tests/test_everos_cloud_backend_contract.py
tests/test_everos_cloud_onboard.py tests/test_everos_cloud_discover.py
tests/test_cli_onboard_commands.py tests/test_rpc_memory.py
tests/test_rpc_settings.py tests/test_plugin_boundary.py
tests/test_rpc_schema_match.py tests/test_rpc_registration.py
tests/test_release_plugin_list.py
tests/integration/test_everos_cloud_real_cloud.py` -> 1001 passed, 6
skipped (the six live tests, no key)
- `cd ui-web && npx vitest run` -> 214 files, 3211 passed; `npm run
lint` -> 0 errors (4 warnings, all in files this change does not touch);
`npm run gen:check` -> in sync (207 methods); `cd ui-tui && npm run
lint:rpc && npm run lint:i18n` -> in sync
- `make lint` (ruff, ruff format, ui tsc, deptry) -> ok; `make
lint-types` -> All checks passed; `make build-ui` -> boot snapshots
match; `scripts/check_large_files.py` and
`scripts/check_source_language.py` on `6d509a8c5..HEAD` -> exit 0;
commitlint and `scripts/check_commit_messages.py` on each commit -> ok
- Mutation runs per change (restored afterwards, `git diff` clean): a
host import of the plugin turns the boundary test red; `import
raven_everos` in the plugin turns the import-isolation test red;
removing the chooser's cancel exit, the Bearer header, the restart
guard, the slice-admission guard, the Roles filter and the fake's two
fixes each turn their own test red
- Real-host acceptance (64 cases,
`docs/specs/2026-10-08-everos-cloud-memory-acceptance.md`) run against a
fake EverOS Cloud: the wizard in tmux (chooser, key typed / from env /
on file, 401 / 403 / unreachable wording, back, Off, Ctrl-C,
non-interactive, base-tree transcript byte-identical with one plugin),
`raven agent` turns (user and agent-track recall, agent-mode adds, 4 s
recall bound, a 300 KB message dropped with a warning, a sub-agent trace
stored under its own ids, flushed and read back, one exit flush per
unflushed session), `raven import run` (3 adds then one flush; a hung
flush retried 4x and the exit sweep defeated), `raven doctor` under
seven cloud conditions (seven sentences, exit 2 only for the five
faults) and with the distribution hidden, and the web page through
playwright-cli (memory page counts and the fake's episode, empty and
refused stores, the Model page hiding the three EverOS-only roles while
the embedding row stays, the cloud card rotating a key and showing the
401 wording, three refused settings.set frames over the gateway
WebSocket, the local everos / null / mem0 backends drawing the page as
before, and a provider-key save that restarts the local EverOS on the
base tree and does not on this branch). Evidence and the per-case
verdicts are kept outside the repo. The live service was reached with a
real key after the acceptance run: six tests, six passes, the service's
replies archived with the evidence.
## Risk
- [x] Security impact considered
- [x] Backward compatibility considered
- [x] Rollback path is clear for risky changes
User-visible changes: the wizard's step 4 chooser (only when two memory
plugins are installed); Model settings hides the three EverOS-only role
rows when `memory.backend` is not `everos`/`null` and draws no cloud
card; the memory page sends a Bearer header for the cloud backend.
Two behaviour changes for agent products, both on the `agents/` lane
only. An agent product now follows the host's `memory.backend` and
endpoint instead of its own, which is a fix for the four products that
never followed the host and a change for `raven-design`, which followed
the endpoint but not the backend. And a product no longer starts an
EverOS server of its own: where it used to answer a host that was not
serving by starting one against an empty root of its own, it now reports
long-term memory as unavailable, which is what the host does. Rollback
for this part is the one commit; the product configs it edits are
shipped files, not user config. The plugin is not installed by default
(its wheel is built by the release workflow; nothing changes for an
install without it). Rollback: revert the squash commit; the config
schema gains no new key (the slice is a plugin slice), so an older raven
reads a config written by this one.
Credentials: the cloud key is written only by `set_plugin_config_fields`
into the plugin's slice; the settings page's key row goes through
`settings.set` with a manifest-declared field, and
`settings.everosCloud` reports `api_key_set` / `key_source`, never the
value.
## Related Issues
N/A
---------
Co-authored-by: gloryfromca <23442919+gloryfromca@users.noreply.github.com>
Co-authored-by: Claude (claude-fable-5-1) <noreply@anthropic.com>
2026-10-14 11:08:11 +08:00
|
|
|
":(glob)plugins-dist/everos-cloud-memory/**/*.py",
|
feat(cli): onboard spends one real search on the web_search key
## Summary
The onboard wizard stored the web_search vendor's key without checking it, so a mistyped or revoked key surfaced as the tool's first failure inside a conversation. Step 5 now spends one real query on the key right after the prompt and before anything is written.
- `WebSearchTool.probe()` runs the same provider request `execute()` uses, asking for one result, and returns `(ok, detail)`. The detail carries vendor and HTTP status only. Exception text is never echoed: SerpApi puts the key in the request URL, and an unusable proxy URL can carry its own credentials, so that branch reports the exception class alone.
- Interactive step 5 loops the key prompt through the probe. On failure the user picks re-enter, retry, or continue anyway. A stored key kept with Enter is probed too, since it may have been revoked. An empty slot is not probed, since leaving web_search off is a choice, not a failure.
- The flag path (`--search-api-key`, `--serper-api-key`) writes first and then probes, warning on failure without dropping the write, because a scripted run has nobody to re-enter the key. Only the active search vendor's key is probed there.
- The page reader is not probed. Jina works with no key, and a keyed reader whose key does not resolve is replaced by Jina at registration. A keyed reader with a wrong key still fails at call time; that is unchanged and out of scope here.
- All wizard strings reuse existing zh catalog entries; README step 5 and the `--search-api-key` help mention the check.
Deliberately not done, for the reviewer: the web settings page writes the same key slot and does not probe; a key supplied only through the vendor's env var is not probed; step 5 does not feed the wizard's end-of-run warnings list. A vendor that answers 200 with zero results passes, since only HTTP status and a Firecrawl refusal count as failure.
## Type
- [ ] Fix
- [x] Feature
- [ ] Docs
- [ ] CI / tooling
- [ ] Refactor
- [ ] Other
## Verification
```
uv run pytest tests/test_agent_tools_web_providers.py tests/test_cli_onboard_commands.py \
tests/test_i18n_boundary.py tests/test_agent_loop_web_tools.py 410 passed
make lint-python clean
make check-source-language exit 0
uv run python scripts/check_commit_messages.py origin/main..HEAD exit 0
```
Revert-the-fix check: with the onboard change reverted, the five new step 5 tests error out on the missing probe seam. No test reaches the network: an autouse fixture in the onboard tests replaces the probe, and the provider tests patch the HTTP client. `make lint-tui` fails on a stale generated i18n file that is identical on `main`; `make lint-bridge` needs `tsc`, which this box lacks.
Not verified: a live run against a real vendor key.
- [x] Relevant tests pass locally
- [x] Relevant lint / type checks pass locally
- [x] User-facing docs or screenshots are updated when needed
## Risk
- [x] Security impact considered
- [x] Backward compatibility considered
- [x] Rollback path is clear for risky changes
User-visible change: onboarding spends one search query per key check against the chosen vendor, and a rejected key adds one prompt. The key still goes only to its own vendor's endpoint, through the configured `tools.web.proxy`. Nothing in config changes shape. Rollback is reverting the commit; no data migration.
## Related Issues
N/A
2026-09-11 02:35:44 +00:00
|
|
|
# Scaffold templates are copied out by the scaffolder, never imported by
|
|
|
|
|
# the runtime; they are data with a .py spelling, not production code.
|
|
|
|
|
":(glob,exclude)raven/templates/agents_scaffold/**",
|
fix(tools): an array sent as its own JSON is still an array
## Summary
Reported as "deliver_files is broken". It is not: **the tool never ran.**
Every `deliver_files` call on one install was refused before dispatch -- five in a row in a single session -- and nothing was ever delivered:
```
Error: Invalid parameters for tool 'deliver_files': files should be array
```
What the model sent was the array, spelled as a string:
```json
{"files": "[{\"path\": \"...poster.png\", \"title\": \"...\"}]", "message": "..."}
```
`cast_params` already forgives that spelling for every scalar type -- `"3"` for an integer, `"true"` for a boolean -- and did not for arrays or objects, which are exactly the types a model is most likely to stringify.
## Type
- [x] Fix
- [ ] Feature
## The edge of the forgiving is the design
When the value is a string, try `json.loads`, and take the result **only if it is the type the schema asked for**. A string that is not JSON, or is JSON of the wrong shape, is returned unchanged so the validator still names the real problem -- the point is to widen what is accepted, not to swallow a type error.
**Nothing here can raise.** Turning "your argument has the wrong type" into a traceback would be worse than the refusal it replaces, so the parse is wrapped and failure falls through.
**A `type: "string"` field is never parsed.** JSON is ordinary text in a message body or a file's content, and replacing it would hand the tool something it never asked for. Two independent things prevent it today (the string branch returns before the parse is reached, and the parse only wins on a type match), which is why the case pinning it is a tripwire rather than a load-bearing assertion -- said so in its own docstring rather than left to look like coverage.
## Not just deliver_files
Six built-in tools declare an array or object parameter and were all exposed:
| tool | parameter |
|---|---|
| `deliver_files` | `files` |
| `ask_user` | `questions` |
| `image_generate` | `images` |
| `message` | `media` |
| `create_playbook` | `skills` |
| `video_generate` | `params` |
The same failure was already seen from the other side: `acp_client/acp_agent.py` records `update_task_state` and `operations should be array` from a real dispatch.
## Verification
```
make lint-python ruff check + ruff format --check, 1850 files
make lint-imports 9 contracts kept, 0 broken
pytest tests/ 19,699 passed, 7 failed (all pre-existing, see below)
```
- [x] Relevant tests pass locally
- [x] Relevant lint / type checks pass locally
- [ ] User-facing docs or screenshots -- no user-facing string
**End to end, through the real tool**, with the exact payload that failed five times in production:
```
files sent as: str -> [{"path": ".../poster.png", "title": "..."}]
returned: Delivered 1 file(s): poster.png (72 B).
still "should be array": False
registered in the store: True
```
Twelve cases, five mutations, each killing at least one:
| mutation | red |
|---|---|
| drop the whole branch (the old behaviour) | 3 |
| raise instead of falling through on a parse failure | 5 |
| accept the parse whatever type it yielded | 1 |
| do not re-cast the parsed value, so items are not cast | 1 |
| widen the tuple to include `string` | **survived -- unreachable** |
The last is worth stating rather than hiding: that mutation cannot change behaviour, because the `type == "string"` branch returns earlier and the type check would refuse the result anyway. I also tried moving the new block above that branch, alone and combined with widening the tuple, and neither reddens anything either. The invariant holds by construction; the case that documents it says so.
**Pre-existing failures, checked against the base.** 7 failed in the full run, all in `tests/test_security_network.py` (`test_a_redirect_to_a_dangerous_address_is_refused` and siblings). Running that file alone on this same tree gives 4 -- the count is order-dependent -- and the change touches only `raven/agent/tools/params.py` plus a new test file, neither of which `raven/security/network.py` imports. The failures are the box's resolver: it returns poisoned answers roughly half the time, which I measured separately (`en.wikipedia.org` came back as `2001::1` on 3 of 6 lookups). Not introduced here, and worth someone's attention on its own.
## Risk
- [x] Security impact considered -- widens what reaches a tool's own validation, not what bypasses it. A parsed value is still validated against the same schema, and a parse that yields the wrong type is refused exactly as before
- [x] Backward compatibility considered -- a call that worked before takes the identical path; the new branch is reached only by a `str` where an array or object is declared, which previously always failed
- [x] Rollback path is clear -- single-commit revert
## Related Issues
N/A
## Swept
`mr-review-patterns` pre-submit sweep, over `git diff origin/main...HEAD`.
- **1 (a write location moved)** -- nothing moved; a branch was added to an existing dispatch on `target_type`. Grepped every caller of `cast_params`: the registry is the only one, and it runs `validate_params` right after, which is the reader this affects.
- **2 (a value with several entry points)** -- the cast is the single funnel every tool argument passes through, which is why the fix belongs here and not in `deliver_files`.
- **3 (a guard)** -- checked both directions: what it wrongly lets through (JSON of the wrong type, non-JSON text -- both refused, with cases) and what it wrongly refuses (a real list, unaffected, with a case).
- **4 (tests)** -- five mutations above, one honestly reported as unreachable. Failing-ID sets diffed against the base rather than counts.
- **6 (stale text)** -- the module docstring enumerated what is coerced ("a string that should be an integer"); it now names arrays and objects too. Grepped the tracked tree for other statements of the rule: none.
- **9 (error handling)** -- the new path adds one way to end and it is caught: `json.loads` raises `ValueError` on bad text and `TypeError` on a non-string, and both fall through to the untouched value.
- **11** -- `git ls-files | grep -c node_modules` is 0; the commit-message gate was run, not eyeballed.
Not applicable, with reasons: **3b, 3c, 3d, 3e** (no tool schema produced, no state moved, one lane, no assertion lost its symbol); **4b, 4c, 4d** (no platform branch, no shell, no rendered number); **5, 5b** (no dependency; `json` is stdlib and already imported across this package); **6b, 6c** (no backup, no generated artifact); **7, 7b** (nothing on disk); **8** (no asset, no secret); **10** (no prompt text).
## Deliberately not fixed
The error text still ends with `[Analyze the error above and try a different approach.]`, and the model plainly did not -- it sent the same shape five times. A message naming the expected spelling (`files must be a JSON array, not a string containing one`) would give it something to act on. That is a change to the registry's error surface, affects every tool, and deserves its own argument.
## Moved from !594
Opened on a compliant source branch. AGENTS.md 2.1 asks for a 3-5 word snake_case description and the previous branch had seven; GitLab cannot change an existing merge request's source branch, so the change moves rather than the name.
Same commits, rebased onto `c2a9768f`. Both code findings from the review on !594 are already in them -- `RecursionError` caught with its regression case, and the implementation comment reduced to the two non-obvious reasons. Nothing else changed, so that review carries.
2026-09-09 15:42:36 +00:00
|
|
|
)
|
2026-08-18 17:30:49 +08:00
|
|
|
DEFAULT_DIFF_THRESHOLD = 90.0
|
|
|
|
|
OMITTED_PATHS = {
|
|
|
|
|
"raven/__main__.py",
|
fix(tools): an array sent as its own JSON is still an array
## Summary
Reported as "deliver_files is broken". It is not: **the tool never ran.**
Every `deliver_files` call on one install was refused before dispatch -- five in a row in a single session -- and nothing was ever delivered:
```
Error: Invalid parameters for tool 'deliver_files': files should be array
```
What the model sent was the array, spelled as a string:
```json
{"files": "[{\"path\": \"...poster.png\", \"title\": \"...\"}]", "message": "..."}
```
`cast_params` already forgives that spelling for every scalar type -- `"3"` for an integer, `"true"` for a boolean -- and did not for arrays or objects, which are exactly the types a model is most likely to stringify.
## Type
- [x] Fix
- [ ] Feature
## The edge of the forgiving is the design
When the value is a string, try `json.loads`, and take the result **only if it is the type the schema asked for**. A string that is not JSON, or is JSON of the wrong shape, is returned unchanged so the validator still names the real problem -- the point is to widen what is accepted, not to swallow a type error.
**Nothing here can raise.** Turning "your argument has the wrong type" into a traceback would be worse than the refusal it replaces, so the parse is wrapped and failure falls through.
**A `type: "string"` field is never parsed.** JSON is ordinary text in a message body or a file's content, and replacing it would hand the tool something it never asked for. Two independent things prevent it today (the string branch returns before the parse is reached, and the parse only wins on a type match), which is why the case pinning it is a tripwire rather than a load-bearing assertion -- said so in its own docstring rather than left to look like coverage.
## Not just deliver_files
Six built-in tools declare an array or object parameter and were all exposed:
| tool | parameter |
|---|---|
| `deliver_files` | `files` |
| `ask_user` | `questions` |
| `image_generate` | `images` |
| `message` | `media` |
| `create_playbook` | `skills` |
| `video_generate` | `params` |
The same failure was already seen from the other side: `acp_client/acp_agent.py` records `update_task_state` and `operations should be array` from a real dispatch.
## Verification
```
make lint-python ruff check + ruff format --check, 1850 files
make lint-imports 9 contracts kept, 0 broken
pytest tests/ 19,699 passed, 7 failed (all pre-existing, see below)
```
- [x] Relevant tests pass locally
- [x] Relevant lint / type checks pass locally
- [ ] User-facing docs or screenshots -- no user-facing string
**End to end, through the real tool**, with the exact payload that failed five times in production:
```
files sent as: str -> [{"path": ".../poster.png", "title": "..."}]
returned: Delivered 1 file(s): poster.png (72 B).
still "should be array": False
registered in the store: True
```
Twelve cases, five mutations, each killing at least one:
| mutation | red |
|---|---|
| drop the whole branch (the old behaviour) | 3 |
| raise instead of falling through on a parse failure | 5 |
| accept the parse whatever type it yielded | 1 |
| do not re-cast the parsed value, so items are not cast | 1 |
| widen the tuple to include `string` | **survived -- unreachable** |
The last is worth stating rather than hiding: that mutation cannot change behaviour, because the `type == "string"` branch returns earlier and the type check would refuse the result anyway. I also tried moving the new block above that branch, alone and combined with widening the tuple, and neither reddens anything either. The invariant holds by construction; the case that documents it says so.
**Pre-existing failures, checked against the base.** 7 failed in the full run, all in `tests/test_security_network.py` (`test_a_redirect_to_a_dangerous_address_is_refused` and siblings). Running that file alone on this same tree gives 4 -- the count is order-dependent -- and the change touches only `raven/agent/tools/params.py` plus a new test file, neither of which `raven/security/network.py` imports. The failures are the box's resolver: it returns poisoned answers roughly half the time, which I measured separately (`en.wikipedia.org` came back as `2001::1` on 3 of 6 lookups). Not introduced here, and worth someone's attention on its own.
## Risk
- [x] Security impact considered -- widens what reaches a tool's own validation, not what bypasses it. A parsed value is still validated against the same schema, and a parse that yields the wrong type is refused exactly as before
- [x] Backward compatibility considered -- a call that worked before takes the identical path; the new branch is reached only by a `str` where an array or object is declared, which previously always failed
- [x] Rollback path is clear -- single-commit revert
## Related Issues
N/A
## Swept
`mr-review-patterns` pre-submit sweep, over `git diff origin/main...HEAD`.
- **1 (a write location moved)** -- nothing moved; a branch was added to an existing dispatch on `target_type`. Grepped every caller of `cast_params`: the registry is the only one, and it runs `validate_params` right after, which is the reader this affects.
- **2 (a value with several entry points)** -- the cast is the single funnel every tool argument passes through, which is why the fix belongs here and not in `deliver_files`.
- **3 (a guard)** -- checked both directions: what it wrongly lets through (JSON of the wrong type, non-JSON text -- both refused, with cases) and what it wrongly refuses (a real list, unaffected, with a case).
- **4 (tests)** -- five mutations above, one honestly reported as unreachable. Failing-ID sets diffed against the base rather than counts.
- **6 (stale text)** -- the module docstring enumerated what is coerced ("a string that should be an integer"); it now names arrays and objects too. Grepped the tracked tree for other statements of the rule: none.
- **9 (error handling)** -- the new path adds one way to end and it is caught: `json.loads` raises `ValueError` on bad text and `TypeError` on a non-string, and both fall through to the untouched value.
- **11** -- `git ls-files | grep -c node_modules` is 0; the commit-message gate was run, not eyeballed.
Not applicable, with reasons: **3b, 3c, 3d, 3e** (no tool schema produced, no state moved, one lane, no assertion lost its symbol); **4b, 4c, 4d** (no platform branch, no shell, no rendered number); **5, 5b** (no dependency; `json` is stdlib and already imported across this package); **6b, 6c** (no backup, no generated artifact); **7, 7b** (nothing on disk); **8** (no asset, no secret); **10** (no prompt text).
## Deliberately not fixed
The error text still ends with `[Analyze the error above and try a different approach.]`, and the model plainly did not -- it sent the same shape five times. A message naming the expected spelling (`files must be a JSON array, not a string containing one`) would give it something to act on. That is a change to the registry's error surface, affects every tool, and deserves its own argument.
## Moved from !594
Opened on a compliant source branch. AGENTS.md 2.1 asks for a 3-5 word snake_case description and the previous branch had seven; GitLab cannot change an existing merge request's source branch, so the change moves rather than the name.
Same commits, rebased onto `c2a9768f`. Both code findings from the review on !594 are already in them -- `RecursionError` caught with its regression case, and the implementation comment reduced to the two non-obvious reasons. Nothing else changed, so that review carries.
2026-09-09 15:42:36 +00:00
|
|
|
"evolver/__main__.py",
|
2026-08-18 17:30:49 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class Metrics:
|
|
|
|
|
statements: int
|
|
|
|
|
covered_lines: int
|
|
|
|
|
branches: int
|
|
|
|
|
covered_branches: int
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def line_percent(self) -> float:
|
|
|
|
|
return _percent(self.covered_lines, self.statements)
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def branch_percent(self) -> float:
|
|
|
|
|
return _percent(self.covered_branches, self.branches)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class DiffCoverage:
|
|
|
|
|
executable: frozenset[tuple[str, int]]
|
|
|
|
|
uncovered: frozenset[tuple[str, int]]
|
|
|
|
|
zero_coverage_files: tuple[str, ...]
|
|
|
|
|
unmeasured_files: tuple[str, ...]
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def percent(self) -> float:
|
|
|
|
|
return _percent(len(self.executable) - len(self.uncovered), len(self.executable))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _percent(covered: int, total: int) -> float:
|
|
|
|
|
return 100.0 if total == 0 else covered * 100.0 / total
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _metrics(summary: dict[str, Any]) -> Metrics:
|
|
|
|
|
return Metrics(
|
|
|
|
|
statements=int(summary["num_statements"]),
|
|
|
|
|
covered_lines=int(summary["covered_lines"]),
|
|
|
|
|
branches=int(summary["num_branches"]),
|
|
|
|
|
covered_branches=int(summary["covered_branches"]),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_coverage(path: Path) -> dict[str, Any]:
|
|
|
|
|
with path.open(encoding="utf-8") as handle:
|
|
|
|
|
report = json.load(handle)
|
|
|
|
|
if not isinstance(report.get("files"), dict) or not isinstance(report.get("totals"), dict):
|
|
|
|
|
raise ValueError(f"{path} is not a coverage.py JSON report")
|
|
|
|
|
return report
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def print_summary(report: dict[str, Any], limit: int) -> None:
|
|
|
|
|
totals = _metrics(report["totals"])
|
|
|
|
|
print(f"Line coverage: {totals.line_percent:.2f}% ({totals.covered_lines}/{totals.statements})")
|
|
|
|
|
print(f"Branch coverage: {totals.branch_percent:.2f}% ({totals.covered_branches}/{totals.branches})")
|
|
|
|
|
print(f"Lowest {limit} production files by combined coverage:")
|
|
|
|
|
ranked = sorted(
|
|
|
|
|
(
|
|
|
|
|
(float(data["summary"]["percent_covered"]), name, _metrics(data["summary"]))
|
|
|
|
|
for name, data in report["files"].items()
|
|
|
|
|
if int(data["summary"]["num_statements"]) > 0
|
|
|
|
|
),
|
|
|
|
|
key=lambda item: (item[0], item[1]),
|
|
|
|
|
)
|
|
|
|
|
for combined, name, metrics in ranked[:limit]:
|
|
|
|
|
print(f" {combined:6.2f}% line {metrics.line_percent:6.2f}% branch {metrics.branch_percent:6.2f}% {name}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_baseline(report: dict[str, Any], commit: str) -> dict[str, Any]:
|
|
|
|
|
files = {
|
|
|
|
|
name: _metrics_payload(_metrics(data["summary"]))
|
|
|
|
|
for name, data in sorted(report["files"].items())
|
|
|
|
|
if int(data["summary"]["num_statements"]) > 0
|
|
|
|
|
}
|
|
|
|
|
return {
|
|
|
|
|
"schema_version": SCHEMA_VERSION,
|
|
|
|
|
"source": "raven",
|
|
|
|
|
"python": f"{sys.version_info.major}.{sys.version_info.minor}",
|
|
|
|
|
"reference_commit": commit,
|
|
|
|
|
"totals": _metrics_payload(_metrics(report["totals"])),
|
|
|
|
|
"files": files,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _metrics_payload(metrics: Metrics) -> dict[str, int | float]:
|
|
|
|
|
return {
|
|
|
|
|
"statements": metrics.statements,
|
|
|
|
|
"covered_lines": metrics.covered_lines,
|
|
|
|
|
"line_percent": round(metrics.line_percent, 6),
|
|
|
|
|
"branches": metrics.branches,
|
|
|
|
|
"covered_branches": metrics.covered_branches,
|
|
|
|
|
"branch_percent": round(metrics.branch_percent, 6),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def write_baseline(report: dict[str, Any], output: Path, commit: str) -> None:
|
|
|
|
|
output.write_text(json.dumps(create_baseline(report, commit), indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
|
print(f"Wrote coverage baseline candidate to {output}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check_baseline_update(current: dict[str, Any], previous: dict[str, Any]) -> bool:
|
|
|
|
|
if current.get("schema_version") != SCHEMA_VERSION or previous.get("schema_version") != SCHEMA_VERSION:
|
|
|
|
|
raise ValueError("current and target-branch baselines must use the supported schema")
|
|
|
|
|
passed = True
|
|
|
|
|
for metric in ("line_percent", "branch_percent"):
|
|
|
|
|
value = float(current["totals"][metric])
|
|
|
|
|
target = float(previous["totals"][metric])
|
|
|
|
|
delta = value - target
|
|
|
|
|
print(f"Baseline {metric}: proposed={value:.6f}% target={target:.6f}% delta={delta:+.6f}pp")
|
|
|
|
|
if value < target:
|
|
|
|
|
passed = False
|
|
|
|
|
if passed:
|
|
|
|
|
print("Coverage baseline update is monotonic.")
|
|
|
|
|
else:
|
|
|
|
|
print("Coverage baseline files may not lower line or branch coverage.")
|
|
|
|
|
return passed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check_ratchet(report: dict[str, Any], baseline: dict[str, Any], tolerance: float, limit: int) -> bool:
|
|
|
|
|
if baseline.get("schema_version") != SCHEMA_VERSION:
|
|
|
|
|
raise ValueError(f"unsupported baseline schema: {baseline.get('schema_version')!r}")
|
|
|
|
|
current = _metrics(report["totals"])
|
|
|
|
|
expected = baseline["totals"]
|
|
|
|
|
checks = (
|
|
|
|
|
("line", current.line_percent, float(expected["line_percent"])),
|
|
|
|
|
("branch", current.branch_percent, float(expected["branch_percent"])),
|
|
|
|
|
)
|
|
|
|
|
failed = [(name, value, target) for name, value, target in checks if value + tolerance < target]
|
|
|
|
|
for name, value, target in checks:
|
|
|
|
|
delta = value - target
|
|
|
|
|
print(f"Ratchet {name}: current={value:.2f}% baseline={target:.2f}% delta={delta:+.2f}pp")
|
|
|
|
|
if not failed:
|
|
|
|
|
print(f"Coverage ratchet passed with {tolerance:.2f}pp tolerance.")
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
print("Largest per-file coverage declines:")
|
|
|
|
|
declines = _file_declines(report, baseline)
|
|
|
|
|
for decline, name, line_delta, branch_delta in declines[:limit]:
|
|
|
|
|
print(f" {decline:6.2f}pp line {line_delta:+7.2f}pp branch {branch_delta:+7.2f}pp {name}")
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _file_declines(report: dict[str, Any], baseline: dict[str, Any]) -> list[tuple[float, str, float, float]]:
|
|
|
|
|
declines: list[tuple[float, str, float, float]] = []
|
|
|
|
|
current_files = report["files"]
|
|
|
|
|
for name, expected in baseline["files"].items():
|
|
|
|
|
data = current_files.get(name)
|
|
|
|
|
if data is None:
|
|
|
|
|
continue
|
|
|
|
|
current = _metrics(data["summary"])
|
|
|
|
|
line_delta = current.line_percent - float(expected["line_percent"])
|
|
|
|
|
branch_delta = current.branch_percent - float(expected["branch_percent"])
|
|
|
|
|
decline = max(-line_delta, -branch_delta, 0.0)
|
|
|
|
|
if decline > 0:
|
|
|
|
|
declines.append((decline, name, line_delta, branch_delta))
|
|
|
|
|
return sorted(declines, key=lambda item: (-item[0], item[1]))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parse_changed_lines(diff: str) -> dict[str, set[int]]:
|
|
|
|
|
"""Return new-side hunk lines; deleted lines never enter the denominator."""
|
|
|
|
|
changed: dict[str, set[int]] = {}
|
|
|
|
|
path: str | None = None
|
|
|
|
|
for line in diff.splitlines():
|
|
|
|
|
if line.startswith("+++ "):
|
|
|
|
|
# Reset on a non-b/ target (e.g. +++ /dev/null) so a deleted file's
|
|
|
|
|
# hunks are not misattributed to the previously seen path.
|
|
|
|
|
path = line[6:] if line.startswith("+++ b/") else None
|
|
|
|
|
if path is not None:
|
|
|
|
|
changed.setdefault(path, set())
|
|
|
|
|
continue
|
|
|
|
|
match = _HUNK_RE.match(line)
|
|
|
|
|
if path is None or match is None:
|
|
|
|
|
continue
|
|
|
|
|
start = int(match.group(1))
|
|
|
|
|
count = int(match.group(2) or "1")
|
|
|
|
|
changed[path].update(range(start, start + count))
|
|
|
|
|
return changed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def calculate_diff_coverage(
|
|
|
|
|
report: dict[str, Any], changed: dict[str, set[int]], added_files: Iterable[str] = ()
|
|
|
|
|
) -> DiffCoverage:
|
|
|
|
|
"""Intersect changed lines with coverage.py statements to ignore comments and blanks."""
|
|
|
|
|
executable: set[tuple[str, int]] = set()
|
|
|
|
|
uncovered: set[tuple[str, int]] = set()
|
|
|
|
|
zero_coverage_files: list[str] = []
|
|
|
|
|
unmeasured_files: list[str] = []
|
|
|
|
|
added = set(added_files)
|
|
|
|
|
for name, lines in changed.items():
|
|
|
|
|
data = report["files"].get(name)
|
|
|
|
|
if data is None:
|
|
|
|
|
if name not in OMITTED_PATHS:
|
|
|
|
|
unmeasured_files.append(name)
|
|
|
|
|
continue
|
|
|
|
|
executed = set(data["executed_lines"])
|
|
|
|
|
missing = set(data["missing_lines"])
|
|
|
|
|
statements = executed | missing
|
|
|
|
|
for line in lines & statements:
|
|
|
|
|
item = (name, line)
|
|
|
|
|
executable.add(item)
|
|
|
|
|
if line in missing:
|
|
|
|
|
uncovered.add(item)
|
|
|
|
|
if name in added and statements and not executed:
|
|
|
|
|
zero_coverage_files.append(name)
|
|
|
|
|
return DiffCoverage(
|
|
|
|
|
frozenset(executable),
|
|
|
|
|
frozenset(uncovered),
|
|
|
|
|
tuple(sorted(zero_coverage_files)),
|
|
|
|
|
tuple(sorted(unmeasured_files)),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _git(*args: str) -> str:
|
|
|
|
|
executable = shutil.which("git")
|
|
|
|
|
if executable is None:
|
|
|
|
|
raise OSError("git is required for coverage diff checks")
|
|
|
|
|
completed = subprocess.run([executable, *args], check=True, capture_output=True, text=True)
|
|
|
|
|
return completed.stdout
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_baseline_from_ref(base_ref: str, path: Path) -> dict[str, Any] | None:
|
|
|
|
|
_git("rev-parse", "--verify", f"{base_ref}^{{commit}}")
|
|
|
|
|
try:
|
|
|
|
|
content = _git("show", f"{base_ref}:{path.as_posix()}")
|
|
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
|
return None
|
|
|
|
|
return json.loads(content)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def git_diff_inputs(base_ref: str, head: str | None) -> tuple[dict[str, set[int]], set[str]]:
|
|
|
|
|
"""Resolve the merge base and collect Python changes for CI or the local worktree."""
|
|
|
|
|
merge_base = _git("merge-base", base_ref, "HEAD").strip()
|
|
|
|
|
end = head or ""
|
|
|
|
|
range_args = [merge_base, end] if end else [merge_base]
|
fix(tools): an array sent as its own JSON is still an array
## Summary
Reported as "deliver_files is broken". It is not: **the tool never ran.**
Every `deliver_files` call on one install was refused before dispatch -- five in a row in a single session -- and nothing was ever delivered:
```
Error: Invalid parameters for tool 'deliver_files': files should be array
```
What the model sent was the array, spelled as a string:
```json
{"files": "[{\"path\": \"...poster.png\", \"title\": \"...\"}]", "message": "..."}
```
`cast_params` already forgives that spelling for every scalar type -- `"3"` for an integer, `"true"` for a boolean -- and did not for arrays or objects, which are exactly the types a model is most likely to stringify.
## Type
- [x] Fix
- [ ] Feature
## The edge of the forgiving is the design
When the value is a string, try `json.loads`, and take the result **only if it is the type the schema asked for**. A string that is not JSON, or is JSON of the wrong shape, is returned unchanged so the validator still names the real problem -- the point is to widen what is accepted, not to swallow a type error.
**Nothing here can raise.** Turning "your argument has the wrong type" into a traceback would be worse than the refusal it replaces, so the parse is wrapped and failure falls through.
**A `type: "string"` field is never parsed.** JSON is ordinary text in a message body or a file's content, and replacing it would hand the tool something it never asked for. Two independent things prevent it today (the string branch returns before the parse is reached, and the parse only wins on a type match), which is why the case pinning it is a tripwire rather than a load-bearing assertion -- said so in its own docstring rather than left to look like coverage.
## Not just deliver_files
Six built-in tools declare an array or object parameter and were all exposed:
| tool | parameter |
|---|---|
| `deliver_files` | `files` |
| `ask_user` | `questions` |
| `image_generate` | `images` |
| `message` | `media` |
| `create_playbook` | `skills` |
| `video_generate` | `params` |
The same failure was already seen from the other side: `acp_client/acp_agent.py` records `update_task_state` and `operations should be array` from a real dispatch.
## Verification
```
make lint-python ruff check + ruff format --check, 1850 files
make lint-imports 9 contracts kept, 0 broken
pytest tests/ 19,699 passed, 7 failed (all pre-existing, see below)
```
- [x] Relevant tests pass locally
- [x] Relevant lint / type checks pass locally
- [ ] User-facing docs or screenshots -- no user-facing string
**End to end, through the real tool**, with the exact payload that failed five times in production:
```
files sent as: str -> [{"path": ".../poster.png", "title": "..."}]
returned: Delivered 1 file(s): poster.png (72 B).
still "should be array": False
registered in the store: True
```
Twelve cases, five mutations, each killing at least one:
| mutation | red |
|---|---|
| drop the whole branch (the old behaviour) | 3 |
| raise instead of falling through on a parse failure | 5 |
| accept the parse whatever type it yielded | 1 |
| do not re-cast the parsed value, so items are not cast | 1 |
| widen the tuple to include `string` | **survived -- unreachable** |
The last is worth stating rather than hiding: that mutation cannot change behaviour, because the `type == "string"` branch returns earlier and the type check would refuse the result anyway. I also tried moving the new block above that branch, alone and combined with widening the tuple, and neither reddens anything either. The invariant holds by construction; the case that documents it says so.
**Pre-existing failures, checked against the base.** 7 failed in the full run, all in `tests/test_security_network.py` (`test_a_redirect_to_a_dangerous_address_is_refused` and siblings). Running that file alone on this same tree gives 4 -- the count is order-dependent -- and the change touches only `raven/agent/tools/params.py` plus a new test file, neither of which `raven/security/network.py` imports. The failures are the box's resolver: it returns poisoned answers roughly half the time, which I measured separately (`en.wikipedia.org` came back as `2001::1` on 3 of 6 lookups). Not introduced here, and worth someone's attention on its own.
## Risk
- [x] Security impact considered -- widens what reaches a tool's own validation, not what bypasses it. A parsed value is still validated against the same schema, and a parse that yields the wrong type is refused exactly as before
- [x] Backward compatibility considered -- a call that worked before takes the identical path; the new branch is reached only by a `str` where an array or object is declared, which previously always failed
- [x] Rollback path is clear -- single-commit revert
## Related Issues
N/A
## Swept
`mr-review-patterns` pre-submit sweep, over `git diff origin/main...HEAD`.
- **1 (a write location moved)** -- nothing moved; a branch was added to an existing dispatch on `target_type`. Grepped every caller of `cast_params`: the registry is the only one, and it runs `validate_params` right after, which is the reader this affects.
- **2 (a value with several entry points)** -- the cast is the single funnel every tool argument passes through, which is why the fix belongs here and not in `deliver_files`.
- **3 (a guard)** -- checked both directions: what it wrongly lets through (JSON of the wrong type, non-JSON text -- both refused, with cases) and what it wrongly refuses (a real list, unaffected, with a case).
- **4 (tests)** -- five mutations above, one honestly reported as unreachable. Failing-ID sets diffed against the base rather than counts.
- **6 (stale text)** -- the module docstring enumerated what is coerced ("a string that should be an integer"); it now names arrays and objects too. Grepped the tracked tree for other statements of the rule: none.
- **9 (error handling)** -- the new path adds one way to end and it is caught: `json.loads` raises `ValueError` on bad text and `TypeError` on a non-string, and both fall through to the untouched value.
- **11** -- `git ls-files | grep -c node_modules` is 0; the commit-message gate was run, not eyeballed.
Not applicable, with reasons: **3b, 3c, 3d, 3e** (no tool schema produced, no state moved, one lane, no assertion lost its symbol); **4b, 4c, 4d** (no platform branch, no shell, no rendered number); **5, 5b** (no dependency; `json` is stdlib and already imported across this package); **6b, 6c** (no backup, no generated artifact); **7, 7b** (nothing on disk); **8** (no asset, no secret); **10** (no prompt text).
## Deliberately not fixed
The error text still ends with `[Analyze the error above and try a different approach.]`, and the model plainly did not -- it sent the same shape five times. A message naming the expected spelling (`files must be a JSON array, not a string containing one`) would give it something to act on. That is a change to the registry's error surface, affects every tool, and deserves its own argument.
## Moved from !594
Opened on a compliant source branch. AGENTS.md 2.1 asks for a 3-5 word snake_case description and the previous branch had seven; GitLab cannot change an existing merge request's source branch, so the change moves rather than the name.
Same commits, rebased onto `c2a9768f`. Both code findings from the review on !594 are already in them -- `RecursionError` caught with its regression case, and the implementation comment reduced to the two non-obvious reasons. Nothing else changed, so that review carries.
2026-09-09 15:42:36 +00:00
|
|
|
diff = _git("diff", "--unified=0", "--no-color", "--diff-filter=ACMR", *range_args, "--", *PRODUCTION_PATHSPECS)
|
|
|
|
|
names = _git("diff", "--name-only", "--diff-filter=A", *range_args, "--", *PRODUCTION_PATHSPECS)
|
2026-08-18 17:30:49 +08:00
|
|
|
return parse_changed_lines(diff), {name for name in names.splitlines() if name}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check_diff(report: dict[str, Any], base_ref: str, head: str | None, threshold: float) -> bool:
|
|
|
|
|
changed, added = git_diff_inputs(base_ref, head)
|
|
|
|
|
result = calculate_diff_coverage(report, changed, added)
|
|
|
|
|
covered = len(result.executable) - len(result.uncovered)
|
|
|
|
|
print(f"Diff coverage: {result.percent:.2f}% ({covered}/{len(result.executable)} executable changed lines)")
|
|
|
|
|
if result.uncovered:
|
|
|
|
|
print("Uncovered changed lines:")
|
|
|
|
|
for name, line in sorted(result.uncovered):
|
|
|
|
|
print(f" {name}:{line}")
|
|
|
|
|
if result.zero_coverage_files:
|
|
|
|
|
print("New production files with 0% line coverage:")
|
|
|
|
|
for name in result.zero_coverage_files:
|
|
|
|
|
print(f" {name}")
|
|
|
|
|
if result.unmeasured_files:
|
|
|
|
|
print("Changed production files missing from the coverage report:")
|
|
|
|
|
for name in result.unmeasured_files:
|
|
|
|
|
print(f" {name}")
|
|
|
|
|
passed = result.percent + 1e-9 >= threshold and not result.zero_coverage_files and not result.unmeasured_files
|
|
|
|
|
if passed:
|
|
|
|
|
print(f"Diff coverage passed the {threshold:.2f}% threshold.")
|
|
|
|
|
else:
|
|
|
|
|
print(f"Diff coverage failed the {threshold:.2f}% threshold.")
|
|
|
|
|
return passed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _parser() -> argparse.ArgumentParser:
|
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
|
|
|
parser.add_argument("--coverage-json", type=Path, default=Path("coverage.json"))
|
|
|
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
|
|
|
|
|
|
summary = subparsers.add_parser("summary")
|
|
|
|
|
summary.add_argument("--limit", type=int, default=10)
|
|
|
|
|
|
|
|
|
|
baseline = subparsers.add_parser("baseline")
|
|
|
|
|
baseline.add_argument("--output", type=Path, default=Path("coverage-baseline-candidate.json"))
|
|
|
|
|
baseline.add_argument("--commit", default="HEAD")
|
|
|
|
|
|
|
|
|
|
baseline_check = subparsers.add_parser("baseline-check")
|
|
|
|
|
baseline_check.add_argument("--baseline", type=Path, default=Path(".github/coverage-baseline.json"))
|
|
|
|
|
baseline_check.add_argument("--base-ref", required=True)
|
|
|
|
|
|
|
|
|
|
ratchet = subparsers.add_parser("ratchet")
|
|
|
|
|
ratchet.add_argument("--baseline", type=Path, default=Path(".github/coverage-baseline.json"))
|
|
|
|
|
ratchet.add_argument("--tolerance", type=float, default=DEFAULT_TOLERANCE)
|
|
|
|
|
ratchet.add_argument("--limit", type=int, default=10)
|
|
|
|
|
|
|
|
|
|
diff = subparsers.add_parser("diff")
|
|
|
|
|
diff.add_argument("--base-ref", required=True)
|
|
|
|
|
diff.add_argument("--head", default=None)
|
|
|
|
|
diff.add_argument("--threshold", type=float, default=DEFAULT_DIFF_THRESHOLD)
|
|
|
|
|
return parser
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
|
|
|
args = _parser().parse_args(argv)
|
|
|
|
|
try:
|
|
|
|
|
if args.command == "baseline-check":
|
|
|
|
|
with args.baseline.open(encoding="utf-8") as handle:
|
|
|
|
|
current_baseline = json.load(handle)
|
|
|
|
|
previous_baseline = load_baseline_from_ref(args.base_ref, args.baseline)
|
|
|
|
|
if previous_baseline is None:
|
|
|
|
|
print(f"No coverage baseline exists on {args.base_ref}; accepting the initial bootstrap baseline.")
|
|
|
|
|
return 0
|
|
|
|
|
return 0 if check_baseline_update(current_baseline, previous_baseline) else 1
|
|
|
|
|
|
|
|
|
|
report = load_coverage(args.coverage_json)
|
|
|
|
|
if args.command == "summary":
|
|
|
|
|
print_summary(report, args.limit)
|
|
|
|
|
return 0
|
|
|
|
|
if args.command == "baseline":
|
|
|
|
|
commit = _git("rev-parse", args.commit).strip()
|
|
|
|
|
write_baseline(report, args.output, commit)
|
|
|
|
|
return 0
|
|
|
|
|
if args.command == "ratchet":
|
|
|
|
|
with args.baseline.open(encoding="utf-8") as handle:
|
|
|
|
|
baseline = json.load(handle)
|
|
|
|
|
return 0 if check_ratchet(report, baseline, args.tolerance, args.limit) else 1
|
|
|
|
|
if args.command == "diff":
|
|
|
|
|
return 0 if check_diff(report, args.base_ref, args.head, args.threshold) else 1
|
|
|
|
|
except (OSError, ValueError, KeyError, json.JSONDecodeError, subprocess.CalledProcessError) as exc:
|
|
|
|
|
print(f"coverage gate error: {exc}", file=sys.stderr)
|
|
|
|
|
return 2
|
|
|
|
|
raise AssertionError(f"unhandled command: {args.command}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
raise SystemExit(main())
|