lawsynth/benchmarks/_runner.py
caelum0x 2335e9556d feat: v2-A engine breadth — structural reductions, MDL/Pareto, differentiable export, SRBench families
Batch 3 of the expansion-v2 roadmap, all deterministic/offline/std-only with
honest capability boundaries.

crates/lawsynth-reduce (new): deterministic symmetry & separability detection.
Numerical-derivative probe (finite differences replace AI-Feynman's trained NN
gradients) over a reconstructed Cartesian grid; additive/multiplicative
separability + difference/sum/product/ratio symmetries. Every detection is a
HYPOTHESIS verified by reconstruction residual, never a proof; non-grid data is
honestly rejected (NotReconstructed). 17 tests.

crates/lawsynth-score: two-part MDL objective (L_data + L_model in nats) and
O(n log n) pareto_front_2d (sort-then-sweep), verified equal to the O(n^2)
front across 12 deterministic sets. +5 tests; crate fmt-normalized to config.

python/lawsynth export.py (new): to_sympy/to_torch/to_jax + dep-free
numeric_derivatives anchor. Constants+params hoisted into one trainable vector
so autograd reaches every coefficient (Neural-ODE vector field). Backends
absent here degrade gracefully (MissingDependencyError subclasses ImportError).
Faithful to native simulate at 1.1e-16. 10 tests + 6 honest skips.

benchmarks: SRBench Feynman (15: 12 static algebraic relations as honest
capability-boundaries, 3 dynamics-framed ODEs executed at R2~=1.0) and
black-box (4 systems, predictive R2 + parsimony, no ground truth) families.
Fixes a pre-existing _runner.py failure where Strogatz cases leaked into the
legacy contract suite. 31 benchmark tests.

specs/structural-reductions: boundary spec (detection-is-a-hypothesis contract).
2026-08-18 15:04:43 +03:00

264 lines
9.3 KiB
Python

"""Benchmark suite orchestration: generate, execute, score, and report.
The runner walks every ``benchmark.toml`` under ``benchmarks/`` and dispatches
each case to the correct execution path:
* native supported cases run through the compiled ``lawsynth`` CLI binary;
* the public Python SDK boundary case runs through the installed package;
* dynamics/equation ``capability-boundary`` cases generate their deterministic
dataset and are reported honestly as boundaries;
* causal/regime/uncertainty contract cases assert an unavailable public API.
Only real ``supported`` regressions make the suite fail.
"""
from __future__ import annotations
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from _common import read_config, repository_root, write_dataset
from _engine import EngineUnavailable, ensure_binary, run_native_case
from _families import is_family_executed, run_family_case
from _scoring import (
BOUNDARY,
FAILED,
PASSED,
REGRESSION,
boundary_result,
classify_native,
declared_status,
)
# Statuses that make the whole suite fail with a non-zero exit code.
FAILING_STATUSES = frozenset({REGRESSION, FAILED})
# The SRBench-style families (Strogatz dynamics, Feynman exact-recovery, and the
# no-ground-truth black-box set) have their own dedicated harness and reporter
# (``_srbench`` / ``srbench_report``) and their own tests. They use a distinct
# case schema (``[system]`` generators, not ``[generation]``), so the legacy
# contract-suite runner deliberately excludes them rather than mis-dispatching
# them through the native-CLI path.
SRBENCH_FAMILIES = frozenset({"strogatz", "feynman", "blackbox"})
@dataclass(frozen=True)
class CaseOutcome:
case_id: str
family: str
declared: str
result: dict[str, Any]
@property
def status(self) -> str:
return str(self.result["status"])
def discover_cases(benchmarks_dir: Path) -> list[Path]:
"""Return every legacy contract-suite case directory, sorted for determinism.
Cases belonging to an SRBench-style family (see :data:`SRBENCH_FAMILIES`) are
excluded: they are executed and scored by the dedicated ``srbench_report``
harness, not by this contract-suite runner.
"""
cases = []
for toml in benchmarks_dir.rglob("benchmark.toml"):
relative = toml.parent.relative_to(benchmarks_dir)
if relative.parts and relative.parts[0] in SRBENCH_FAMILIES:
continue
cases.append(toml.parent)
return sorted(cases, key=lambda path: str(path))
def case_id(benchmarks_dir: Path, case_dir: Path) -> str:
return case_dir.relative_to(benchmarks_dir).as_posix()
def _is_contract_case(config: dict[str, Any]) -> bool:
return "family" in config and config.get("status") == "capability_boundary"
def _workflow(config: dict[str, Any]) -> str:
workload = config.get("workload")
return str(workload["workflow"]) if workload else "native-cli"
def _run_python_sdk(case_dir: Path) -> dict[str, Any]:
from _performance import run_workload, score_workload
result = run_workload(case_dir, case_dir / ".benchmark-run")
scored = score_workload(case_dir, result)
status = PASSED if scored["passed"] else REGRESSION
return {
"status": status,
"passed": scored["passed"],
"operation": result.get("operation"),
"elapsed_ns": result.get("elapsed_ns"),
}
def _run_contract(case_dir: Path) -> dict[str, Any]:
from _capability_contract import execute
outcome = execute(case_dir)
return {
"status": BOUNDARY,
"passed": False,
"reason": outcome["error"],
"feature": outcome["feature"],
"rows": outcome["rows"],
"missing_public_operations": outcome["missing_public_operations"],
}
def run_case(
benchmarks_dir: Path,
case_dir: Path,
workdir: Path,
binary: Path | None,
) -> CaseOutcome:
"""Execute a single benchmark case and return its classified outcome."""
config = read_config(case_dir)
identifier = case_id(benchmarks_dir, case_dir)
family = identifier.split("/", 1)[0]
if config.get("suite") == "srbench":
from _srbench import run_srbench_case
declared = str(config["capability"]["status"])
result = run_srbench_case(case_dir, workdir, binary, check_determinism=False)
return CaseOutcome(identifier, family, declared, result)
if is_family_executed(config):
if binary is None:
return CaseOutcome(
identifier,
family,
"executed",
{"status": FAILED, "passed": False, "reason": "CLI binary unavailable"},
)
return CaseOutcome(identifier, family, "executed", run_family_case(case_dir, workdir, binary))
if _is_contract_case(config):
return CaseOutcome(identifier, family, "capability_boundary", _run_contract(case_dir))
declared = declared_status(case_dir)
if declared != "supported":
reason = str(config["capability"]["reason"])
dataset = write_dataset(case_dir, workdir) # prove deterministic generation
return CaseOutcome(identifier, family, declared, boundary_result(case_dir, reason, dataset))
workflow = _workflow(config)
if workflow == "python-sdk":
return CaseOutcome(identifier, family, declared, _run_python_sdk(case_dir))
if binary is None:
return CaseOutcome(
identifier,
family,
declared,
{"status": FAILED, "passed": False, "reason": "CLI binary unavailable"},
)
run = run_native_case(case_dir, workdir, binary)
return CaseOutcome(identifier, family, declared, classify_native(case_dir, run))
def run_suite(
benchmarks_dir: Path,
workroot: Path,
*,
family_filter: str | None = None,
allow_build: bool = False,
) -> dict[str, Any]:
"""Run every (optionally filtered) case and build a deterministic summary."""
root = repository_root(benchmarks_dir)
cases = discover_cases(benchmarks_dir)
if family_filter:
cases = [c for c in cases if case_id(benchmarks_dir, c).startswith(f"{family_filter}/")]
binary: Path | None
binary_note: str
try:
binary = ensure_binary(root, allow_build=allow_build)
try:
binary_note = str(binary.relative_to(root))
except ValueError:
binary_note = str(binary)
except EngineUnavailable as error:
binary = None
binary_note = f"unavailable: {error}"
outcomes: list[CaseOutcome] = []
for case_dir in cases:
workdir = workroot / case_id(benchmarks_dir, case_dir).replace("/", "__")
outcomes.append(run_case(benchmarks_dir, case_dir, workdir, binary))
return _summarize(outcomes, binary_note)
def _summarize(outcomes: list[CaseOutcome], binary_note: str) -> dict[str, Any]:
families: dict[str, dict[str, int]] = {}
cases_report: list[dict[str, Any]] = []
counts: dict[str, int] = {}
for outcome in sorted(outcomes, key=lambda o: o.case_id):
counts[outcome.status] = counts.get(outcome.status, 0) + 1
family = families.setdefault(outcome.family, {})
family[outcome.status] = family.get(outcome.status, 0) + 1
cases_report.append(
{
"id": outcome.case_id,
"family": outcome.family,
"declared": outcome.declared,
"status": outcome.status,
"result": outcome.result,
}
)
regressions = [o.case_id for o in outcomes if o.status in FAILING_STATUSES]
return {
"binary": binary_note,
"total": len(outcomes),
"counts": counts,
"families": families,
"regressions": sorted(regressions),
"passed_suite": not regressions,
"cases": cases_report,
}
def render_text(summary: dict[str, Any]) -> str:
"""Render a deterministic human-readable report."""
lines = ["LawSynth benchmark suite", f"binary: {summary['binary']}", ""]
for family in sorted(summary["families"]):
breakdown = summary["families"][family]
parts = ", ".join(f"{status}={breakdown[status]}" for status in sorted(breakdown))
lines.append(f"[{family}] {parts}")
lines.append("")
for case in summary["cases"]:
marker = {PASSED: "PASS", BOUNDARY: "BNDRY", REGRESSION: "REGR", FAILED: "FAIL"}.get(
case["status"], case["status"].upper()
)
detail = ""
vector = case["result"].get("score_vector")
if vector:
error = vector.get("trajectory_error")
fit = vector.get("fit_train")
detail = f" mse={fit} traj_err={error}"
lines.append(f" {marker:<5} {case['id']}{detail}")
lines.append("")
total = summary["total"]
counts = ", ".join(f"{status}={summary['counts'][status]}" for status in sorted(summary["counts"]))
lines.append(f"total={total} ({counts})")
lines.append("SUITE PASS" if summary["passed_suite"] else f"SUITE FAIL: {summary['regressions']}")
return "\n".join(lines) + "\n"
def write_reports(summary: dict[str, Any], json_path: Path | None) -> None:
if json_path is not None:
json_path.parent.mkdir(parents=True, exist_ok=True)
json_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
sys.stdout.write(render_text(summary))