Batch 3 of the expansion-v2 roadmap, all deterministic/offline/std-only with honest capability boundaries. crates/lawsynth-reduce (new): deterministic symmetry & separability detection. Numerical-derivative probe (finite differences replace AI-Feynman's trained NN gradients) over a reconstructed Cartesian grid; additive/multiplicative separability + difference/sum/product/ratio symmetries. Every detection is a HYPOTHESIS verified by reconstruction residual, never a proof; non-grid data is honestly rejected (NotReconstructed). 17 tests. crates/lawsynth-score: two-part MDL objective (L_data + L_model in nats) and O(n log n) pareto_front_2d (sort-then-sweep), verified equal to the O(n^2) front across 12 deterministic sets. +5 tests; crate fmt-normalized to config. python/lawsynth export.py (new): to_sympy/to_torch/to_jax + dep-free numeric_derivatives anchor. Constants+params hoisted into one trainable vector so autograd reaches every coefficient (Neural-ODE vector field). Backends absent here degrade gracefully (MissingDependencyError subclasses ImportError). Faithful to native simulate at 1.1e-16. 10 tests + 6 honest skips. benchmarks: SRBench Feynman (15: 12 static algebraic relations as honest capability-boundaries, 3 dynamics-framed ODEs executed at R2~=1.0) and black-box (4 systems, predictive R2 + parsimony, no ground truth) families. Fixes a pre-existing _runner.py failure where Strogatz cases leaked into the legacy contract suite. 31 benchmark tests. specs/structural-reductions: boundary spec (detection-is-a-hypothesis contract).
264 lines
9.3 KiB
Python
264 lines
9.3 KiB
Python
"""Benchmark suite orchestration: generate, execute, score, and report.
|
|
|
|
The runner walks every ``benchmark.toml`` under ``benchmarks/`` and dispatches
|
|
each case to the correct execution path:
|
|
|
|
* native supported cases run through the compiled ``lawsynth`` CLI binary;
|
|
* the public Python SDK boundary case runs through the installed package;
|
|
* dynamics/equation ``capability-boundary`` cases generate their deterministic
|
|
dataset and are reported honestly as boundaries;
|
|
* causal/regime/uncertainty contract cases assert an unavailable public API.
|
|
|
|
Only real ``supported`` regressions make the suite fail.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from _common import read_config, repository_root, write_dataset
|
|
from _engine import EngineUnavailable, ensure_binary, run_native_case
|
|
from _families import is_family_executed, run_family_case
|
|
from _scoring import (
|
|
BOUNDARY,
|
|
FAILED,
|
|
PASSED,
|
|
REGRESSION,
|
|
boundary_result,
|
|
classify_native,
|
|
declared_status,
|
|
)
|
|
|
|
# Statuses that make the whole suite fail with a non-zero exit code.
|
|
FAILING_STATUSES = frozenset({REGRESSION, FAILED})
|
|
|
|
# The SRBench-style families (Strogatz dynamics, Feynman exact-recovery, and the
|
|
# no-ground-truth black-box set) have their own dedicated harness and reporter
|
|
# (``_srbench`` / ``srbench_report``) and their own tests. They use a distinct
|
|
# case schema (``[system]`` generators, not ``[generation]``), so the legacy
|
|
# contract-suite runner deliberately excludes them rather than mis-dispatching
|
|
# them through the native-CLI path.
|
|
SRBENCH_FAMILIES = frozenset({"strogatz", "feynman", "blackbox"})
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CaseOutcome:
|
|
case_id: str
|
|
family: str
|
|
declared: str
|
|
result: dict[str, Any]
|
|
|
|
@property
|
|
def status(self) -> str:
|
|
return str(self.result["status"])
|
|
|
|
|
|
def discover_cases(benchmarks_dir: Path) -> list[Path]:
|
|
"""Return every legacy contract-suite case directory, sorted for determinism.
|
|
|
|
Cases belonging to an SRBench-style family (see :data:`SRBENCH_FAMILIES`) are
|
|
excluded: they are executed and scored by the dedicated ``srbench_report``
|
|
harness, not by this contract-suite runner.
|
|
"""
|
|
cases = []
|
|
for toml in benchmarks_dir.rglob("benchmark.toml"):
|
|
relative = toml.parent.relative_to(benchmarks_dir)
|
|
if relative.parts and relative.parts[0] in SRBENCH_FAMILIES:
|
|
continue
|
|
cases.append(toml.parent)
|
|
return sorted(cases, key=lambda path: str(path))
|
|
|
|
|
|
def case_id(benchmarks_dir: Path, case_dir: Path) -> str:
|
|
return case_dir.relative_to(benchmarks_dir).as_posix()
|
|
|
|
|
|
def _is_contract_case(config: dict[str, Any]) -> bool:
|
|
return "family" in config and config.get("status") == "capability_boundary"
|
|
|
|
|
|
def _workflow(config: dict[str, Any]) -> str:
|
|
workload = config.get("workload")
|
|
return str(workload["workflow"]) if workload else "native-cli"
|
|
|
|
|
|
def _run_python_sdk(case_dir: Path) -> dict[str, Any]:
|
|
from _performance import run_workload, score_workload
|
|
|
|
result = run_workload(case_dir, case_dir / ".benchmark-run")
|
|
scored = score_workload(case_dir, result)
|
|
status = PASSED if scored["passed"] else REGRESSION
|
|
return {
|
|
"status": status,
|
|
"passed": scored["passed"],
|
|
"operation": result.get("operation"),
|
|
"elapsed_ns": result.get("elapsed_ns"),
|
|
}
|
|
|
|
|
|
def _run_contract(case_dir: Path) -> dict[str, Any]:
|
|
from _capability_contract import execute
|
|
|
|
outcome = execute(case_dir)
|
|
return {
|
|
"status": BOUNDARY,
|
|
"passed": False,
|
|
"reason": outcome["error"],
|
|
"feature": outcome["feature"],
|
|
"rows": outcome["rows"],
|
|
"missing_public_operations": outcome["missing_public_operations"],
|
|
}
|
|
|
|
|
|
def run_case(
|
|
benchmarks_dir: Path,
|
|
case_dir: Path,
|
|
workdir: Path,
|
|
binary: Path | None,
|
|
) -> CaseOutcome:
|
|
"""Execute a single benchmark case and return its classified outcome."""
|
|
config = read_config(case_dir)
|
|
identifier = case_id(benchmarks_dir, case_dir)
|
|
family = identifier.split("/", 1)[0]
|
|
|
|
if config.get("suite") == "srbench":
|
|
from _srbench import run_srbench_case
|
|
|
|
declared = str(config["capability"]["status"])
|
|
result = run_srbench_case(case_dir, workdir, binary, check_determinism=False)
|
|
return CaseOutcome(identifier, family, declared, result)
|
|
|
|
if is_family_executed(config):
|
|
if binary is None:
|
|
return CaseOutcome(
|
|
identifier,
|
|
family,
|
|
"executed",
|
|
{"status": FAILED, "passed": False, "reason": "CLI binary unavailable"},
|
|
)
|
|
return CaseOutcome(identifier, family, "executed", run_family_case(case_dir, workdir, binary))
|
|
|
|
if _is_contract_case(config):
|
|
return CaseOutcome(identifier, family, "capability_boundary", _run_contract(case_dir))
|
|
|
|
declared = declared_status(case_dir)
|
|
|
|
if declared != "supported":
|
|
reason = str(config["capability"]["reason"])
|
|
dataset = write_dataset(case_dir, workdir) # prove deterministic generation
|
|
return CaseOutcome(identifier, family, declared, boundary_result(case_dir, reason, dataset))
|
|
|
|
workflow = _workflow(config)
|
|
if workflow == "python-sdk":
|
|
return CaseOutcome(identifier, family, declared, _run_python_sdk(case_dir))
|
|
|
|
if binary is None:
|
|
return CaseOutcome(
|
|
identifier,
|
|
family,
|
|
declared,
|
|
{"status": FAILED, "passed": False, "reason": "CLI binary unavailable"},
|
|
)
|
|
run = run_native_case(case_dir, workdir, binary)
|
|
return CaseOutcome(identifier, family, declared, classify_native(case_dir, run))
|
|
|
|
|
|
def run_suite(
|
|
benchmarks_dir: Path,
|
|
workroot: Path,
|
|
*,
|
|
family_filter: str | None = None,
|
|
allow_build: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Run every (optionally filtered) case and build a deterministic summary."""
|
|
root = repository_root(benchmarks_dir)
|
|
cases = discover_cases(benchmarks_dir)
|
|
if family_filter:
|
|
cases = [c for c in cases if case_id(benchmarks_dir, c).startswith(f"{family_filter}/")]
|
|
|
|
binary: Path | None
|
|
binary_note: str
|
|
try:
|
|
binary = ensure_binary(root, allow_build=allow_build)
|
|
try:
|
|
binary_note = str(binary.relative_to(root))
|
|
except ValueError:
|
|
binary_note = str(binary)
|
|
except EngineUnavailable as error:
|
|
binary = None
|
|
binary_note = f"unavailable: {error}"
|
|
|
|
outcomes: list[CaseOutcome] = []
|
|
for case_dir in cases:
|
|
workdir = workroot / case_id(benchmarks_dir, case_dir).replace("/", "__")
|
|
outcomes.append(run_case(benchmarks_dir, case_dir, workdir, binary))
|
|
|
|
return _summarize(outcomes, binary_note)
|
|
|
|
|
|
def _summarize(outcomes: list[CaseOutcome], binary_note: str) -> dict[str, Any]:
|
|
families: dict[str, dict[str, int]] = {}
|
|
cases_report: list[dict[str, Any]] = []
|
|
counts: dict[str, int] = {}
|
|
for outcome in sorted(outcomes, key=lambda o: o.case_id):
|
|
counts[outcome.status] = counts.get(outcome.status, 0) + 1
|
|
family = families.setdefault(outcome.family, {})
|
|
family[outcome.status] = family.get(outcome.status, 0) + 1
|
|
cases_report.append(
|
|
{
|
|
"id": outcome.case_id,
|
|
"family": outcome.family,
|
|
"declared": outcome.declared,
|
|
"status": outcome.status,
|
|
"result": outcome.result,
|
|
}
|
|
)
|
|
regressions = [o.case_id for o in outcomes if o.status in FAILING_STATUSES]
|
|
return {
|
|
"binary": binary_note,
|
|
"total": len(outcomes),
|
|
"counts": counts,
|
|
"families": families,
|
|
"regressions": sorted(regressions),
|
|
"passed_suite": not regressions,
|
|
"cases": cases_report,
|
|
}
|
|
|
|
|
|
def render_text(summary: dict[str, Any]) -> str:
|
|
"""Render a deterministic human-readable report."""
|
|
lines = ["LawSynth benchmark suite", f"binary: {summary['binary']}", ""]
|
|
for family in sorted(summary["families"]):
|
|
breakdown = summary["families"][family]
|
|
parts = ", ".join(f"{status}={breakdown[status]}" for status in sorted(breakdown))
|
|
lines.append(f"[{family}] {parts}")
|
|
lines.append("")
|
|
for case in summary["cases"]:
|
|
marker = {PASSED: "PASS", BOUNDARY: "BNDRY", REGRESSION: "REGR", FAILED: "FAIL"}.get(
|
|
case["status"], case["status"].upper()
|
|
)
|
|
detail = ""
|
|
vector = case["result"].get("score_vector")
|
|
if vector:
|
|
error = vector.get("trajectory_error")
|
|
fit = vector.get("fit_train")
|
|
detail = f" mse={fit} traj_err={error}"
|
|
lines.append(f" {marker:<5} {case['id']}{detail}")
|
|
lines.append("")
|
|
total = summary["total"]
|
|
counts = ", ".join(f"{status}={summary['counts'][status]}" for status in sorted(summary["counts"]))
|
|
lines.append(f"total={total} ({counts})")
|
|
lines.append("SUITE PASS" if summary["passed_suite"] else f"SUITE FAIL: {summary['regressions']}")
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def write_reports(summary: dict[str, Any], json_path: Path | None) -> None:
|
|
if json_path is not None:
|
|
json_path.parent.mkdir(parents=True, exist_ok=True)
|
|
json_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
sys.stdout.write(render_text(summary))
|