lawsynth/benchmarks/tests/test_execution.py
caelum0x 13d696183d feat(product): ship report/explain/compare/forecast, Study SDK, Studio screens
Product depth across all three surfaces (PRODUCT.md captures the vision):

CLI — new lawsynth-report crate (self-contained HTML reports with inline SVG
trajectory + phase-portrait charts) and five product commands: report, explain
(plain-language laws), compare (world diff), forecast (what-if interventions),
library (local world index). Demonstrated end-to-end on a damped oscillator.

SDK — high-level Study workflow (from_csv -> discover -> explain -> simulate/
forecast -> report -> save/load), stdlib HTML report generator, and notebook
rich _repr_html_ display. Quickstart example recovers the oscillator exactly
(R2=1.0) and writes a 30KB report.

Studio — 5 interactive screens (discovery canvas, equation explorer, regime
timeline, uncertainty lens, world lab) composing the shared TS packages through
a reactive state-store; typecheck-clean.

Also: benchmark execution deepened 9 -> 18 real cases (causal + regime families
now scored against ground truth); cross-service integration tests (gateway->
artifact proxy, scheduler<->worker lifecycle over real sockets).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-08-17 21:05:39 +03:00

141 lines
6.5 KiB
Python

"""End-to-end execution and scoring tests through the real CLI binary."""
from __future__ import annotations
import tempfile
from pathlib import Path
from _runner import run_case, run_suite
from _scoring import BOUNDARY, PASSED, REGRESSION
def _run_single(benchmarks_dir: Path, binary: Path, case: str):
case_dir = benchmarks_dir / case
with tempfile.TemporaryDirectory() as workdir:
return run_case(benchmarks_dir, case_dir, Path(workdir), binary)
def test_supported_ode_small_executes_and_scores(benchmarks_dir: Path, cli_binary: Path) -> None:
outcome = _run_single(benchmarks_dir, cli_binary, "dynamics/ode-small")
assert outcome.declared == "supported"
assert outcome.status == PASSED, outcome.result
vector = outcome.result["score_vector"]
# The engine must return real, finite measurements, not fabricated ones.
assert vector["fit_train"] is not None
assert vector["complexity_nodes"] is not None
assert vector["trajectory_error"] is not None
assert vector["simulation_failure_rate"] == 0.0
assert outcome.result["returncode"] == 0
assert outcome.result["inspect_returncode"] == 0
def test_supported_end_to_end_simulates(benchmarks_dir: Path, cli_binary: Path) -> None:
outcome = _run_single(benchmarks_dir, cli_binary, "performance/end-to-end")
assert outcome.status == PASSED, outcome.result
assert outcome.result["simulate_returncode"] == 0
assert outcome.result["score_vector"]["trajectory_error"] is not None
def test_capability_boundary_is_not_a_hard_failure(benchmarks_dir: Path, cli_binary: Path) -> None:
# dynamics/delay declares a hyphenated capability-boundary; it must be
# reported as a boundary and must still generate its deterministic dataset.
case_dir = benchmarks_dir / "dynamics/delay"
with tempfile.TemporaryDirectory() as workdir:
outcome = run_case(benchmarks_dir, case_dir, Path(workdir), cli_binary)
assert outcome.declared == "capability-boundary"
assert outcome.status == BOUNDARY
assert outcome.status != REGRESSION
assert outcome.result["dataset"] is not None
# The boundary case still produces its deterministic dataset on disk.
assert Path(outcome.result["dataset"]).is_file()
def test_contract_boundary_reports_missing_public_api(
benchmarks_dir: Path, cli_binary: Path
) -> None:
# The uncertainty family remains an honest SDK-surface boundary: the CLI
# exposes no calibrated-interval or coverage output to score.
outcome = _run_single(benchmarks_dir, cli_binary, "uncertainty/parameter-coverage")
assert outcome.status == BOUNDARY
assert outcome.result["missing_public_operations"]
assert "uncertainty" in outcome.result["feature"] or "coverage" in outcome.result["feature"]
def test_causal_family_executes_and_scores(benchmarks_dir: Path, cli_binary: Path) -> None:
# Promoted from an SDK-surface boundary: `discover --causal` now emits a
# real dependency-hypothesis edge count that is scored end to end.
outcome = _run_single(benchmarks_dir, cli_binary, "causal/linear")
assert outcome.declared == "executed"
assert outcome.status == PASSED, outcome.result
assert outcome.result["signal"] == "dependency_edges"
assert outcome.result["signal_value"] >= outcome.result["expected_minimum"]
assert outcome.result["discover_returncode"] == 0
assert outcome.result["inspect_returncode"] == 0
vector = outcome.result["score_vector"]
assert vector["fit_train"] is not None
assert vector["dependency_edges"] is not None
assert vector["frontier_size"] is not None
def test_regime_family_executes_and_scores(benchmarks_dir: Path, cli_binary: Path) -> None:
# Promoted from an SDK-surface boundary: `discover --regimes` now emits a
# real regime-segment count. A single change point implies >= 2 segments.
outcome = _run_single(benchmarks_dir, cli_binary, "regime/change-point")
assert outcome.declared == "executed"
assert outcome.status == PASSED, outcome.result
assert outcome.result["signal"] == "regime_segments"
assert outcome.result["signal_value"] >= 2
assert outcome.result["score_vector"]["regime_segments"] is not None
def test_regime_family_regression_when_signal_missing(monkeypatch, benchmarks_dir: Path) -> None:
# A family case that stops emitting its structural signal is a real
# regression that fails the suite, never a silently-passing boundary.
import _families
case_dir = benchmarks_dir / "regime/change-point"
config = _families.read_config(case_dir)
empty = _families.FamilyRun(discover_returncode=0, inspect_returncode=0)
scored = _families.score_family(config, empty)
assert scored["status"] == REGRESSION
assert scored["passed"] is False
def test_dynamics_family_suite_passes(benchmarks_dir: Path, cli_binary: Path) -> None:
with tempfile.TemporaryDirectory() as workroot:
summary = run_suite(benchmarks_dir, Path(workroot), family_filter="dynamics")
# Two supported dynamics cases must pass; boundaries must not fail the suite.
assert summary["passed_suite"], summary["regressions"]
assert summary["counts"].get(PASSED) == 2
assert summary["counts"].get(BOUNDARY) == 4
assert summary["regressions"] == []
def test_causal_family_suite_all_execute(benchmarks_dir: Path, cli_binary: Path) -> None:
with tempfile.TemporaryDirectory() as workroot:
summary = run_suite(benchmarks_dir, Path(workroot), family_filter="causal")
# All five causal cases now execute-and-score; none are boundaries.
assert summary["passed_suite"], summary["regressions"]
assert summary["counts"].get(PASSED) == 5
assert summary["counts"].get(BOUNDARY) is None
def test_regime_family_suite_all_execute(benchmarks_dir: Path, cli_binary: Path) -> None:
with tempfile.TemporaryDirectory() as workroot:
summary = run_suite(benchmarks_dir, Path(workroot), family_filter="regime")
assert summary["passed_suite"], summary["regressions"]
assert summary["counts"].get(PASSED) == 4
assert summary["counts"].get(BOUNDARY) is None
def test_full_suite_executes_eighteen_and_bounds_twelve(
benchmarks_dir: Path, cli_binary: Path
) -> None:
# Baseline was 9 executed; causal (5) + regime (4) promotions raise it to 18.
with tempfile.TemporaryDirectory() as workroot:
summary = run_suite(benchmarks_dir, Path(workroot))
assert summary["passed_suite"], summary["regressions"]
assert summary["counts"].get(PASSED) == 18
assert summary["counts"].get(BOUNDARY) == 12
assert summary["total"] == 30