lawsynth/benchmarks/tests/test_srbench.py
caelum0x 2335e9556d feat: v2-A engine breadth — structural reductions, MDL/Pareto, differentiable export, SRBench families
Batch 3 of the expansion-v2 roadmap, all deterministic/offline/std-only with
honest capability boundaries.

crates/lawsynth-reduce (new): deterministic symmetry & separability detection.
Numerical-derivative probe (finite differences replace AI-Feynman's trained NN
gradients) over a reconstructed Cartesian grid; additive/multiplicative
separability + difference/sum/product/ratio symmetries. Every detection is a
HYPOTHESIS verified by reconstruction residual, never a proof; non-grid data is
honestly rejected (NotReconstructed). 17 tests.

crates/lawsynth-score: two-part MDL objective (L_data + L_model in nats) and
O(n log n) pareto_front_2d (sort-then-sweep), verified equal to the O(n^2)
front across 12 deterministic sets. +5 tests; crate fmt-normalized to config.

python/lawsynth export.py (new): to_sympy/to_torch/to_jax + dep-free
numeric_derivatives anchor. Constants+params hoisted into one trainable vector
so autograd reaches every coefficient (Neural-ODE vector field). Backends
absent here degrade gracefully (MissingDependencyError subclasses ImportError).
Faithful to native simulate at 1.1e-16. 10 tests + 6 honest skips.

benchmarks: SRBench Feynman (15: 12 static algebraic relations as honest
capability-boundaries, 3 dynamics-framed ODEs executed at R2~=1.0) and
black-box (4 systems, predictive R2 + parsimony, no ground truth) families.
Fixes a pre-existing _runner.py failure where Strogatz cases leaked into the
legacy contract suite. 31 benchmark tests.

specs/structural-reductions: boundary spec (detection-is-a-hypothesis contract).
2026-08-18 15:04:43 +03:00

86 lines
3.6 KiB
Python

"""Tests for the SRBench-style credibility benchmark harness.
These drive the real compiled ``lawsynth`` CLI on the Strogatz ODE family. If the
binary cannot be produced they skip with an explicit message (never a silent
pass), matching the rest of the benchmark suite.
"""
from __future__ import annotations
import tempfile
from pathlib import Path
import pytest
from _engine import ensure_binary
from _srbench import read_config, run_srbench_case
from _srbench_data import write_dataset
import srbench_report
ROOT = Path(__file__).resolve().parents[2]
STROGATZ = ROOT / "benchmarks" / "strogatz"
def _binary() -> Path:
try:
return ensure_binary(ROOT, allow_build=True)
except Exception as exc: # pragma: no cover - environment dependent
pytest.skip(f"lawsynth CLI unavailable: {exc}")
def test_strogatz_data_generation_is_byte_stable() -> None:
"""The generated observation CSV must be identical across runs (determinism)."""
case = STROGATZ / "damped-oscillator"
config = read_config(case)
with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b:
ds_a = write_dataset(config, Path(a)).read_bytes()
ds_b = write_dataset(config, Path(b)).read_bytes()
assert ds_a == ds_b and len(ds_a) > 0
@pytest.mark.parametrize("case_name", ["damped-oscillator", "lotka-volterra", "van-der-pol"])
def test_strogatz_case_executes_scores_and_is_deterministic(case_name: str) -> None:
binary = _binary()
case = STROGATZ / case_name
with tempfile.TemporaryDirectory() as tmp:
result = run_srbench_case(case, Path(tmp), binary, check_determinism=True)
assert result["status"] in {"passed", "capability-boundary"}, result
if result["status"] == "passed":
assert result["passed"] is True
# Determinism is the headline metric: byte-identical worlds on replay.
assert result["determinism"] is True
sv = result["score_vector"]
assert sv["fit_train"] is not None
assert sv["training_time_ns"] is not None
def test_srbench_report_runs_and_reports_determinism() -> None:
_binary() # ensure a binary exists (or skip)
report = srbench_report.build_report("strogatz", allow_build=True)
assert report["binary_available"] is True
assert report["total"] >= 3
# No supported regressions on the checked-in Strogatz cases.
assert report["supported_regressions"] == []
det = [r["determinism"] for r in report["rows"] if r["determinism"] is not None]
assert det and all(det), "every executed case must be byte-identical on replay"
print(srbench_report.render_table(report))
def test_full_report_covers_all_three_families() -> None:
"""The default report must span strogatz, feynman, and blackbox honestly."""
_binary()
report = srbench_report.build_report(None, allow_build=True)
assert report["binary_available"] is True
families = {r["family"] for r in report["rows"] if r["family"]}
# Boundary rows carry family too (from the outcome), so all three appear.
assert {"strogatz", "feynman", "blackbox"} <= families
# Honest split: static Feynman cases are boundaries, not fake recoveries.
boundaries = [r for r in report["rows"] if r["status"] == "capability-boundary"]
assert boundaries, "static Feynman cases must be reported as boundaries"
assert all(r["family"] == "feynman" for r in boundaries)
# No supported case regresses; every executed case is deterministic.
assert report["supported_regressions"] == []
det = [r["determinism"] for r in report["rows"] if r["determinism"] is not None]
assert det and all(det)
print(srbench_report.render_table(report))