- Rust engine: real bootstrap/wiring preflight (guardrail-agent), execution layer (retry/router/pre-trade/post-trade/reconciliation), guardrail-doctor diagnostics with allowlist-drift guard. - guardrail-cli: extracted pure helpers into util.rs (split in progress). - 6 Track-2 skills + ensemble; python-lab analytics (regime/drawdown/montecarlo/dossier/ensemble/journal/seed); clients (ts/py/go/mcp/langchain/web-lite/proof-verifier); alert relay; scenarios; helm/k8s/compose; docs. - dashboard/vercel.json + docs/VERCEL_DEPLOY.md for Vercel deployment. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
105 lines
3.0 KiB
Python
105 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Backtest-validate the Track-2 strategy skill examples.
|
|
|
|
Loads the JSON example files shipped with the
|
|
``cmc-regime-routed-alpha`` skill, runs lightweight validation on each, prints
|
|
a per-example PASS/ISSUES report plus a summary, and writes a flat CSV to
|
|
``data/exports/skill_validation.csv`` (columns: example, issues_count, issues).
|
|
|
|
Validation issues are reported, not fatal: the script exits nonzero only if a
|
|
required input cannot be read (i.e. the examples directory is missing).
|
|
|
|
Run from the repository root:
|
|
|
|
python3 python-lab/scripts/validate_skill.py
|
|
|
|
or from python-lab/:
|
|
|
|
python3 scripts/validate_skill.py
|
|
|
|
Standard-library only.
|
|
"""
|
|
|
|
import csv
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Make ``guardrail_lab`` importable regardless of the current directory.
|
|
_LAB_ROOT = Path(__file__).resolve().parent.parent
|
|
if str(_LAB_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(_LAB_ROOT))
|
|
|
|
from guardrail_lab.skill import load_skill_examples, validate_example # noqa: E402
|
|
|
|
DEFAULT_DIR = "skills/cmc-regime-routed-alpha/examples"
|
|
DEFAULT_OUT = "data/exports/skill_validation.csv"
|
|
|
|
CSV_COLUMNS = ("example", "issues_count", "issues")
|
|
|
|
|
|
def _example_name(example: dict, fallback: str) -> str:
|
|
"""Pick a stable display name for an example."""
|
|
source = example.get("_source")
|
|
if isinstance(source, str) and source:
|
|
return source
|
|
return fallback
|
|
|
|
|
|
def _print_report(rows: list[dict]) -> None:
|
|
"""Print a per-example PASS/ISSUES report."""
|
|
for row in rows:
|
|
if row["issues"]:
|
|
print(f"ISSUES {row['example']}")
|
|
for issue in row["issues"]:
|
|
print(f" - {issue}")
|
|
else:
|
|
print(f"PASS {row['example']}")
|
|
|
|
|
|
def main() -> int:
|
|
skill_dir = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_DIR
|
|
|
|
base = Path(skill_dir)
|
|
if not base.is_dir():
|
|
print(
|
|
f"validate_skill: examples directory not found: {skill_dir}",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
|
|
examples = load_skill_examples(skill_dir)
|
|
|
|
rows: list[dict] = []
|
|
for index, example in enumerate(examples):
|
|
issues = validate_example(example)
|
|
rows.append(
|
|
{
|
|
"example": _example_name(example, f"example_{index}"),
|
|
"issues": issues,
|
|
}
|
|
)
|
|
|
|
out_path = Path(DEFAULT_OUT)
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with out_path.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.writer(handle)
|
|
writer.writerow(CSV_COLUMNS)
|
|
for row in rows:
|
|
writer.writerow(
|
|
[row["example"], len(row["issues"]), "; ".join(row["issues"])]
|
|
)
|
|
|
|
_print_report(rows)
|
|
|
|
total = len(rows)
|
|
passed = sum(1 for row in rows if not row["issues"])
|
|
failed = total - passed
|
|
print()
|
|
print(f"validate_skill: {passed}/{total} examples passed, {failed} with issues")
|
|
print(f"validate_skill: wrote report to {out_path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|