296 lines
11 KiB
Python
296 lines
11 KiB
Python
|
|
#!/usr/bin/env python3
|
||
|
|
"""Auto-tick docs/SUBMISSION_CHECKLIST.md from real evidence.
|
||
|
|
|
||
|
|
Reads the agent's append-only SQLite event log, its run report, and committed
|
||
|
|
configs, then flips each proof-artifact checkbox ``[ ]``/``[x]`` based on whether
|
||
|
|
the evidence actually exists — never by hand. It also regenerates an
|
||
|
|
auto-generated evidence table at the end of the file.
|
||
|
|
|
||
|
|
Honest by construction: an item ticks only when its artifact is present. Items
|
||
|
|
that genuinely require a live run (a real on-chain registration tx, a real
|
||
|
|
confirmed swap) report ``pending (live)`` in paper mode rather than a false tick.
|
||
|
|
|
||
|
|
Stdlib only. Read-only except for rewriting the checklist file.
|
||
|
|
|
||
|
|
Usage:
|
||
|
|
python3 python-lab/scripts/tick_checklist.py [--db PATH] [--report PATH] [--check]
|
||
|
|
|
||
|
|
``--check`` exits non-zero if any required item is unticked (for CI), without
|
||
|
|
writing.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import sqlite3
|
||
|
|
import sys
|
||
|
|
from typing import Any, Callable, Optional
|
||
|
|
|
||
|
|
# --- repo paths ---------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
def repo_root() -> str:
|
||
|
|
here = os.path.dirname(os.path.abspath(__file__))
|
||
|
|
return os.path.normpath(os.path.join(here, "..", ".."))
|
||
|
|
|
||
|
|
|
||
|
|
ROOT = repo_root()
|
||
|
|
DEFAULT_DB = os.path.join(ROOT, "data", "guardrail_alpha.db")
|
||
|
|
DEFAULT_REPORT = os.path.join(ROOT, "data", "run_report.json")
|
||
|
|
CHECKLIST = os.path.join(ROOT, "docs", "SUBMISSION_CHECKLIST.md")
|
||
|
|
UNIVERSE = os.path.join(ROOT, "configs", "eligible_assets.bsc.json")
|
||
|
|
SUBMISSION_MD = os.path.join(ROOT, "data", "exports", "submission.md")
|
||
|
|
|
||
|
|
AUTO_SECTION_HEADER = "## Evidence (auto-generated)"
|
||
|
|
|
||
|
|
|
||
|
|
# --- evidence gathering -------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class Evidence:
|
||
|
|
"""The facts we can establish from the agent's artifacts."""
|
||
|
|
|
||
|
|
def __init__(self, db: str, report_path: str) -> None:
|
||
|
|
self.event_types: set[str] = set()
|
||
|
|
self.report: dict[str, Any] = {}
|
||
|
|
self.published: dict[str, Any] = {}
|
||
|
|
self.enabled_assets = 0
|
||
|
|
self.submission_md = os.path.isfile(SUBMISSION_MD)
|
||
|
|
# Load the report first so events can be scoped to its run_id (the
|
||
|
|
# kill-switch demo shares the DB and would otherwise win "latest").
|
||
|
|
self._load_report(report_path)
|
||
|
|
self._load_events(db)
|
||
|
|
self._load_universe()
|
||
|
|
|
||
|
|
def _load_events(self, db: str) -> None:
|
||
|
|
if not os.path.isfile(db):
|
||
|
|
return
|
||
|
|
try:
|
||
|
|
conn = sqlite3.connect(db)
|
||
|
|
self.event_types = {
|
||
|
|
row[0] for row in conn.execute("SELECT DISTINCT event_type FROM events")
|
||
|
|
}
|
||
|
|
# Prefer the published report tied to the primary run; fall back to
|
||
|
|
# the most recent one if no run_id match (or no run report).
|
||
|
|
primary_run = self.report.get("run_id")
|
||
|
|
rows = conn.execute(
|
||
|
|
"SELECT payload_json FROM events WHERE event_type='agent_report_published' "
|
||
|
|
"ORDER BY timestamp DESC"
|
||
|
|
).fetchall()
|
||
|
|
conn.close()
|
||
|
|
published = [json.loads(r[0]) for r in rows]
|
||
|
|
self.published = next(
|
||
|
|
(p for p in published if p.get("run_id") == primary_run),
|
||
|
|
published[0] if published else {},
|
||
|
|
)
|
||
|
|
except (sqlite3.Error, json.JSONDecodeError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def _load_report(self, report_path: str) -> None:
|
||
|
|
try:
|
||
|
|
with open(report_path, "r", encoding="utf-8") as fh:
|
||
|
|
self.report = json.load(fh)
|
||
|
|
except (OSError, json.JSONDecodeError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def _load_universe(self) -> None:
|
||
|
|
try:
|
||
|
|
with open(UNIVERSE, "r", encoding="utf-8") as fh:
|
||
|
|
assets = json.load(fh)
|
||
|
|
self.enabled_assets = sum(1 for a in assets if a.get("enabled", True))
|
||
|
|
except (OSError, json.JSONDecodeError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def field(self, key: str) -> Optional[Any]:
|
||
|
|
"""Look a field up in the published-report payload, then the run report."""
|
||
|
|
if key in self.published:
|
||
|
|
return self.published[key]
|
||
|
|
return self.report.get(key)
|
||
|
|
|
||
|
|
def has(self, *event_types: str) -> bool:
|
||
|
|
return all(t in self.event_types for t in event_types)
|
||
|
|
|
||
|
|
def any_of(self, *event_types: str) -> bool:
|
||
|
|
return any(t in self.event_types for t in event_types)
|
||
|
|
|
||
|
|
|
||
|
|
# --- checklist items ----------------------------------------------------------
|
||
|
|
|
||
|
|
# Each item: a keyword that uniquely identifies its line in the checklist, and a
|
||
|
|
# predicate returning (ticked, note). A note prefixed "pending" never ticks.
|
||
|
|
Predicate = Callable[[Evidence], "tuple[bool, str]"]
|
||
|
|
|
||
|
|
|
||
|
|
def _is_hex64(v: Any) -> bool:
|
||
|
|
return isinstance(v, str) and bool(re.fullmatch(r"[0-9a-fA-F]{64}", v))
|
||
|
|
|
||
|
|
|
||
|
|
def policy_hash(e: Evidence) -> tuple[bool, str]:
|
||
|
|
h = e.field("policy_hash")
|
||
|
|
return (_is_hex64(h), f"policy_hash {h[:12]}…" if _is_hex64(h) else "no policy_hash")
|
||
|
|
|
||
|
|
|
||
|
|
def agent_identity(e: Evidence) -> tuple[bool, str]:
|
||
|
|
aid = e.field("agent_id")
|
||
|
|
ok = isinstance(aid, str) and len(aid) >= 16
|
||
|
|
return (ok, f"agent_id {aid[:12]}…" if ok else "no agent_id")
|
||
|
|
|
||
|
|
|
||
|
|
def registration(e: Evidence) -> tuple[bool, str]:
|
||
|
|
tx = e.field("registration_tx")
|
||
|
|
live = str(e.field("mode")) == "live"
|
||
|
|
if isinstance(tx, str) and tx and live:
|
||
|
|
return (True, f"on-chain registration_tx {tx[:12]}…")
|
||
|
|
if isinstance(tx, str) and tx:
|
||
|
|
# Paper/mock registration ran, but it is not a real on-chain tx.
|
||
|
|
return (False, "pending (live): paper mock registration — run scripts/go_live.sh")
|
||
|
|
return (False, "pending (live): no registration_tx — run scripts/go_live.sh")
|
||
|
|
|
||
|
|
|
||
|
|
def eligible_assets(e: Evidence) -> tuple[bool, str]:
|
||
|
|
n = e.enabled_assets
|
||
|
|
return (n >= 1, f"{n} enabled eligible asset(s)")
|
||
|
|
|
||
|
|
|
||
|
|
def cmc_data(e: Evidence) -> tuple[bool, str]:
|
||
|
|
ok = e.any_of("market_snapshot_received", "regime_classified")
|
||
|
|
return (ok, "market_snapshot_received / regime_classified in log" if ok else "no CMC events")
|
||
|
|
|
||
|
|
|
||
|
|
def risk_examples(e: Evidence) -> tuple[bool, str]:
|
||
|
|
ok = e.has("risk_approved") and e.any_of("risk_rejected", "risk_clipped")
|
||
|
|
return (ok, "risk_approved + risk_rejected/clipped in log" if ok else "missing approval or rejection")
|
||
|
|
|
||
|
|
|
||
|
|
def twak_trade(e: Evidence) -> tuple[bool, str]:
|
||
|
|
ok = e.has("twak_quote_received", "tx_confirmed")
|
||
|
|
note = "twak_quote_received + tx_confirmed in log"
|
||
|
|
if ok and str(e.field("mode")) == "paper":
|
||
|
|
note += " (paper: mock tx)"
|
||
|
|
return (ok, note if ok else "no quote/confirmed pair")
|
||
|
|
|
||
|
|
|
||
|
|
def kill_switch(e: Evidence) -> tuple[bool, str]:
|
||
|
|
ok = "kill_switch_triggered" in e.event_types or e.field("kill_switch") is True
|
||
|
|
return (ok, "kill_switch_triggered in log" if ok else "no kill-switch event — run scripts/kill_switch.sh")
|
||
|
|
|
||
|
|
|
||
|
|
def run_report(e: Evidence) -> tuple[bool, str]:
|
||
|
|
ok = bool(e.report) and "run_id" in e.report
|
||
|
|
return (ok, f"run_report.json present (run {str(e.report.get('run_id',''))[:8]}…)" if ok else "no run report")
|
||
|
|
|
||
|
|
|
||
|
|
def submission_md(e: Evidence) -> tuple[bool, str]:
|
||
|
|
return (e.submission_md, "data/exports/submission.md present" if e.submission_md else "not exported")
|
||
|
|
|
||
|
|
|
||
|
|
def proof_page(e: Evidence) -> tuple[bool, str]:
|
||
|
|
ok = bool(e.field("agent_id")) and _is_hex64(e.field("policy_hash"))
|
||
|
|
return (ok, "proof fields (agent_id + policy_hash) available" if ok else "proof data incomplete")
|
||
|
|
|
||
|
|
|
||
|
|
ITEMS: list[tuple[str, Predicate]] = [
|
||
|
|
("Policy hash generated", policy_hash),
|
||
|
|
("Agent identity", agent_identity),
|
||
|
|
("Competition registration", registration),
|
||
|
|
("Eligible BSC assets", eligible_assets),
|
||
|
|
("CMC data visible", cmc_data),
|
||
|
|
("Risk approval and rejection", risk_examples),
|
||
|
|
("TWAK quote", twak_trade),
|
||
|
|
("Kill-switch behavior", kill_switch),
|
||
|
|
("Daily report", run_report),
|
||
|
|
("Submission Markdown", submission_md),
|
||
|
|
("Dashboard proof page", proof_page),
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
# --- checklist rewriting ------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
def evaluate(e: Evidence) -> list[tuple[str, bool, str]]:
|
||
|
|
return [(kw, *pred(e)) for kw, pred in ITEMS]
|
||
|
|
|
||
|
|
|
||
|
|
def rewrite(text: str, results: list[tuple[str, bool, str]]) -> str:
|
||
|
|
"""Flip checkboxes in place and regenerate the auto evidence section."""
|
||
|
|
by_keyword = {kw: (ticked, note) for kw, ticked, note in results}
|
||
|
|
|
||
|
|
lines = text.splitlines()
|
||
|
|
out: list[str] = []
|
||
|
|
for line in lines:
|
||
|
|
stripped = line.lstrip()
|
||
|
|
if stripped.startswith("- [ ]") or stripped.startswith("- [x]"):
|
||
|
|
match = next((kw for kw in by_keyword if kw in line), None)
|
||
|
|
if match:
|
||
|
|
ticked, _ = by_keyword[match]
|
||
|
|
box = "[x]" if ticked else "[ ]"
|
||
|
|
line = re.sub(r"- \[[ x]\]", f"- {box}", line, count=1)
|
||
|
|
out.append(line)
|
||
|
|
|
||
|
|
body = "\n".join(out)
|
||
|
|
# Drop any prior auto-generated section, then append a fresh one.
|
||
|
|
idx = body.find(AUTO_SECTION_HEADER)
|
||
|
|
if idx != -1:
|
||
|
|
body = body[:idx].rstrip() + "\n"
|
||
|
|
body = body.rstrip() + "\n\n" + render_evidence_table(results) + "\n"
|
||
|
|
return body
|
||
|
|
|
||
|
|
|
||
|
|
def render_evidence_table(results: list[tuple[str, bool, str]]) -> str:
|
||
|
|
ticked = sum(1 for _, t, _ in results if t)
|
||
|
|
lines = [
|
||
|
|
AUTO_SECTION_HEADER,
|
||
|
|
"",
|
||
|
|
f"Generated by `python-lab/scripts/tick_checklist.py` from real artifacts. "
|
||
|
|
f"**{ticked}/{len(results)} ticked.**",
|
||
|
|
"",
|
||
|
|
"| Item | Status | Evidence |",
|
||
|
|
"|---|---|---|",
|
||
|
|
]
|
||
|
|
for kw, t, note in results:
|
||
|
|
mark = "✅" if t else ("🟡" if note.startswith("pending") else "⬜")
|
||
|
|
lines.append(f"| {kw} | {mark} | {note} |")
|
||
|
|
return "\n".join(lines)
|
||
|
|
|
||
|
|
|
||
|
|
def main(argv: Optional[list[str]] = None) -> int:
|
||
|
|
parser = argparse.ArgumentParser(description="Auto-tick the submission checklist from evidence.")
|
||
|
|
parser.add_argument("--db", default=DEFAULT_DB)
|
||
|
|
parser.add_argument("--report", default=DEFAULT_REPORT)
|
||
|
|
parser.add_argument("--check", action="store_true", help="Exit non-zero if any item is unticked; do not write.")
|
||
|
|
args = parser.parse_args(argv)
|
||
|
|
|
||
|
|
evidence = Evidence(args.db, args.report)
|
||
|
|
results = evaluate(evidence)
|
||
|
|
ticked = sum(1 for _, t, _ in results if t)
|
||
|
|
|
||
|
|
for kw, t, note in results:
|
||
|
|
mark = "PASS" if t else ("PEND" if note.startswith("pending") else "MISS")
|
||
|
|
print(f"[{mark}] {kw}: {note}")
|
||
|
|
print(f"\n{ticked}/{len(results)} ticked")
|
||
|
|
|
||
|
|
if args.check:
|
||
|
|
missing = [kw for kw, t, note in results if not t and not note.startswith("pending")]
|
||
|
|
if missing:
|
||
|
|
print(f"unticked (non-pending): {', '.join(missing)}", file=sys.stderr)
|
||
|
|
return 1
|
||
|
|
return 0
|
||
|
|
|
||
|
|
try:
|
||
|
|
with open(CHECKLIST, "r", encoding="utf-8") as fh:
|
||
|
|
text = fh.read()
|
||
|
|
except OSError as err:
|
||
|
|
print(f"error: cannot read {CHECKLIST}: {err}", file=sys.stderr)
|
||
|
|
return 2
|
||
|
|
with open(CHECKLIST, "w", encoding="utf-8") as fh:
|
||
|
|
fh.write(rewrite(text, results))
|
||
|
|
print(f"\nupdated {os.path.relpath(CHECKLIST, ROOT)}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(main())
|