Batch 3 of the expansion-v2 roadmap, all deterministic/offline/std-only with honest capability boundaries. crates/lawsynth-reduce (new): deterministic symmetry & separability detection. Numerical-derivative probe (finite differences replace AI-Feynman's trained NN gradients) over a reconstructed Cartesian grid; additive/multiplicative separability + difference/sum/product/ratio symmetries. Every detection is a HYPOTHESIS verified by reconstruction residual, never a proof; non-grid data is honestly rejected (NotReconstructed). 17 tests. crates/lawsynth-score: two-part MDL objective (L_data + L_model in nats) and O(n log n) pareto_front_2d (sort-then-sweep), verified equal to the O(n^2) front across 12 deterministic sets. +5 tests; crate fmt-normalized to config. python/lawsynth export.py (new): to_sympy/to_torch/to_jax + dep-free numeric_derivatives anchor. Constants+params hoisted into one trainable vector so autograd reaches every coefficient (Neural-ODE vector field). Backends absent here degrade gracefully (MissingDependencyError subclasses ImportError). Faithful to native simulate at 1.1e-16. 10 tests + 6 honest skips. benchmarks: SRBench Feynman (15: 12 static algebraic relations as honest capability-boundaries, 3 dynamics-framed ODEs executed at R2~=1.0) and black-box (4 systems, predictive R2 + parsimony, no ground truth) families. Fixes a pre-existing _runner.py failure where Strogatz cases leaked into the legacy contract suite. 31 benchmark tests. specs/structural-reductions: boundary spec (detection-is-a-hypothesis contract).
86 lines
3.6 KiB
Python
86 lines
3.6 KiB
Python
"""Tests for the SRBench-style credibility benchmark harness.
|
|
|
|
These drive the real compiled ``lawsynth`` CLI on the Strogatz ODE family. If the
|
|
binary cannot be produced they skip with an explicit message (never a silent
|
|
pass), matching the rest of the benchmark suite.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from _engine import ensure_binary
|
|
from _srbench import read_config, run_srbench_case
|
|
from _srbench_data import write_dataset
|
|
import srbench_report
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
STROGATZ = ROOT / "benchmarks" / "strogatz"
|
|
|
|
|
|
def _binary() -> Path:
|
|
try:
|
|
return ensure_binary(ROOT, allow_build=True)
|
|
except Exception as exc: # pragma: no cover - environment dependent
|
|
pytest.skip(f"lawsynth CLI unavailable: {exc}")
|
|
|
|
|
|
def test_strogatz_data_generation_is_byte_stable() -> None:
|
|
"""The generated observation CSV must be identical across runs (determinism)."""
|
|
case = STROGATZ / "damped-oscillator"
|
|
config = read_config(case)
|
|
with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b:
|
|
ds_a = write_dataset(config, Path(a)).read_bytes()
|
|
ds_b = write_dataset(config, Path(b)).read_bytes()
|
|
assert ds_a == ds_b and len(ds_a) > 0
|
|
|
|
|
|
@pytest.mark.parametrize("case_name", ["damped-oscillator", "lotka-volterra", "van-der-pol"])
|
|
def test_strogatz_case_executes_scores_and_is_deterministic(case_name: str) -> None:
|
|
binary = _binary()
|
|
case = STROGATZ / case_name
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
result = run_srbench_case(case, Path(tmp), binary, check_determinism=True)
|
|
assert result["status"] in {"passed", "capability-boundary"}, result
|
|
if result["status"] == "passed":
|
|
assert result["passed"] is True
|
|
# Determinism is the headline metric: byte-identical worlds on replay.
|
|
assert result["determinism"] is True
|
|
sv = result["score_vector"]
|
|
assert sv["fit_train"] is not None
|
|
assert sv["training_time_ns"] is not None
|
|
|
|
|
|
def test_srbench_report_runs_and_reports_determinism() -> None:
|
|
_binary() # ensure a binary exists (or skip)
|
|
report = srbench_report.build_report("strogatz", allow_build=True)
|
|
assert report["binary_available"] is True
|
|
assert report["total"] >= 3
|
|
# No supported regressions on the checked-in Strogatz cases.
|
|
assert report["supported_regressions"] == []
|
|
det = [r["determinism"] for r in report["rows"] if r["determinism"] is not None]
|
|
assert det and all(det), "every executed case must be byte-identical on replay"
|
|
print(srbench_report.render_table(report))
|
|
|
|
|
|
def test_full_report_covers_all_three_families() -> None:
|
|
"""The default report must span strogatz, feynman, and blackbox honestly."""
|
|
_binary()
|
|
report = srbench_report.build_report(None, allow_build=True)
|
|
assert report["binary_available"] is True
|
|
families = {r["family"] for r in report["rows"] if r["family"]}
|
|
# Boundary rows carry family too (from the outcome), so all three appear.
|
|
assert {"strogatz", "feynman", "blackbox"} <= families
|
|
# Honest split: static Feynman cases are boundaries, not fake recoveries.
|
|
boundaries = [r for r in report["rows"] if r["status"] == "capability-boundary"]
|
|
assert boundaries, "static Feynman cases must be reported as boundaries"
|
|
assert all(r["family"] == "feynman" for r in boundaries)
|
|
# No supported case regresses; every executed case is deterministic.
|
|
assert report["supported_regressions"] == []
|
|
det = [r["determinism"] for r in report["rows"] if r["determinism"] is not None]
|
|
assert det and all(det)
|
|
print(srbench_report.render_table(report))
|