Four expansion-v2 items (finished after a session-limit interruption): Weak/integral-form discovery (crates/lawsynth-weakform, specs/weak-form) — the top noise-robustness technique from PySINDy: integrate candidates against compactly- supported test functions, NO differentiation of noisy data. ~23x lower coefficient error than finite-difference strong-form under noise. 20 tests, deterministic. Implicit/rational dynamics (crates/lawsynth-implicit, specs/implicit-dynamics) — SINDy-PI alternating-LHS nullspace solve; recovers rational laws (Michaelis-Menten) explicit regression can't express. Honest degree-2 identifiability limit documented (multi-dimensional nullspace); residual-floor + derivative-preferring tie-break for canonical selection. 10 tests. Optimizer breadth (lawsynth-sparse) — FROLS (orthogonal-least-squares, ERR ordering), SSR (stepwise), trapping (stability-biased); --solver frols|ssr|trapping. 24 tests; default path byte-identical. SRBench harness (benchmarks/) — 18 Strogatz ODE systems + scoring + a determinism column (byte-identical on replay — a metric no stochastic competitor can report). 3/3 Strogatz recovered at R²=1.0. srbench_report.py + 5 tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
470 lines
16 KiB
Python
470 lines
16 KiB
Python
"""Execute and score the SRBench-style credibility families through the CLI.
|
|
|
|
This module drives the *real* compiled ``lawsynth`` binary and scores the
|
|
SRBench metric set adapted to a dynamics engine:
|
|
|
|
* **symbolic recovery** — parse the discovered laws from ``explain`` and compare
|
|
the recovered term structure against the known governing terms;
|
|
* **trajectory R^2** — simulate the discovered world and measure fit against the
|
|
deterministically generated reference trajectory;
|
|
* **complexity** — the AST node count reported by ``discover``;
|
|
* **training time** — wall-clock of the ``discover`` invocation;
|
|
* **determinism** — run ``discover`` twice and assert byte-identical worlds, a
|
|
metric no stochastic competitor can report.
|
|
|
|
It never fabricates a recovery result. A ``capability-boundary`` case (static
|
|
algebraic Feynman regression the dynamics CLI does not perform) generates its
|
|
deterministic dataset and is reported honestly as a boundary.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
import re
|
|
import subprocess
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from _srbench_data import ground_truth_trajectory, write_dataset
|
|
|
|
# Classification labels shared with the rest of the harness.
|
|
PASSED = "passed"
|
|
BOUNDARY = "capability-boundary"
|
|
REGRESSION = "regression"
|
|
FAILED = "failed"
|
|
|
|
_SUMMARY = re.compile(r"mse=([0-9eE.+-]+),\s*complexity=([0-9]+)")
|
|
_LAW = re.compile(r"^\s*d([A-Za-z_][A-Za-z0-9_]*)/dt\s*=\s*(.+?)\s*$")
|
|
|
|
# Ordering of recovery quality; a case passes when achieved >= expected.
|
|
_RECOVERY_RANK = {"none": 0, "missed": 1, "partial": 2, "core": 3, "exact": 4}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Parsing discovered laws into canonical term signatures.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _canonical_term(factors: list[str]) -> str:
|
|
"""Canonicalise a product of factors into a stable monomial signature."""
|
|
atoms = [factor.strip() for factor in factors if factor.strip()]
|
|
if not atoms:
|
|
return "1"
|
|
powers: dict[str, int] = {}
|
|
for atom in atoms:
|
|
powers[atom] = powers.get(atom, 0) + 1
|
|
parts = []
|
|
for atom in sorted(powers):
|
|
exponent = powers[atom]
|
|
parts.append(atom if exponent == 1 else f"{atom}^{exponent}")
|
|
return "*".join(parts)
|
|
|
|
|
|
def _split_terms(expression: str) -> list[str]:
|
|
"""Split an RHS expression on additive boundaries (`` + ``)."""
|
|
return [piece.strip() for piece in expression.split(" + ") if piece.strip()]
|
|
|
|
|
|
def _term_signature(term: str) -> str | None:
|
|
"""Return the canonical signature of a single ``coef * f1 * f2`` term."""
|
|
pieces = [piece.strip() for piece in term.split("*")]
|
|
factors: list[str] = []
|
|
for piece in pieces:
|
|
# Drop the leading numeric coefficient(s); keep symbolic factors.
|
|
try:
|
|
float(piece)
|
|
continue
|
|
except ValueError:
|
|
factors.append(piece)
|
|
if not factors:
|
|
# A pure constant term (e.g. "1.5") contributes the bias signature,
|
|
# unless the whole RHS is a literal zero.
|
|
try:
|
|
if float(term) == 0.0:
|
|
return None
|
|
except ValueError:
|
|
pass
|
|
return "1"
|
|
return _canonical_term(factors)
|
|
|
|
|
|
def parse_laws(explain_text: str) -> dict[str, set[str]]:
|
|
"""Parse the ``explain`` output into ``{state: {term_signature, ...}}``."""
|
|
laws: dict[str, set[str]] = {}
|
|
for line in explain_text.splitlines():
|
|
match = _LAW.match(line)
|
|
if match is None:
|
|
continue
|
|
state, expression = match.group(1), match.group(2)
|
|
signatures: set[str] = set()
|
|
for term in _split_terms(expression):
|
|
signature = _term_signature(term)
|
|
if signature is not None:
|
|
signatures.add(signature)
|
|
laws[state] = signatures
|
|
return laws
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Symbolic recovery scoring.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _classify_law(recovered: set[str], expected: set[str]) -> str:
|
|
if recovered == expected:
|
|
return "exact"
|
|
if expected and expected.issubset(recovered):
|
|
return "core"
|
|
if recovered & expected:
|
|
return "partial"
|
|
return "missed"
|
|
|
|
|
|
def score_recovery(
|
|
discovered: dict[str, set[str]], expected_laws: list[dict[str, Any]]
|
|
) -> dict[str, Any]:
|
|
"""Compare discovered term structure against the known governing terms."""
|
|
per_law: list[dict[str, Any]] = []
|
|
worst = "exact"
|
|
exact_count = 0
|
|
for law in expected_laws:
|
|
state = str(law["state"])
|
|
expected = {str(term) for term in law["terms"]}
|
|
recovered = discovered.get(state, set())
|
|
level = _classify_law(recovered, expected)
|
|
if level == "exact":
|
|
exact_count += 1
|
|
if _RECOVERY_RANK[level] < _RECOVERY_RANK[worst]:
|
|
worst = level
|
|
per_law.append(
|
|
{
|
|
"state": state,
|
|
"level": level,
|
|
"expected": sorted(expected),
|
|
"recovered": sorted(recovered),
|
|
"spurious": sorted(recovered - expected),
|
|
"missing": sorted(expected - recovered),
|
|
}
|
|
)
|
|
total = len(expected_laws)
|
|
return {
|
|
"level": worst if expected_laws else "none",
|
|
"laws_exact": exact_count,
|
|
"laws_total": total,
|
|
"law_recovery_rate": (exact_count / total) if total else None,
|
|
"system_recovered": bool(expected_laws) and worst == "exact",
|
|
"per_law": per_law,
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Trajectory R^2.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def _r2(predicted: list[float], truth: list[float]) -> float | None:
|
|
length = min(len(predicted), len(truth))
|
|
if length < 2:
|
|
return None
|
|
predicted, truth = predicted[:length], truth[:length]
|
|
if any(not math.isfinite(value) for value in predicted):
|
|
return float("-inf")
|
|
mean = sum(truth) / length
|
|
ss_tot = sum((value - mean) ** 2 for value in truth)
|
|
ss_res = sum((p - t) ** 2 for p, t in zip(predicted, truth, strict=True))
|
|
if ss_tot == 0.0:
|
|
return 1.0 if ss_res == 0.0 else float("-inf")
|
|
return 1.0 - ss_res / ss_tot
|
|
|
|
|
|
def _parse_trajectory(text: str) -> dict[str, list[float]]:
|
|
lines = [line for line in text.splitlines() if line.strip()]
|
|
if not lines:
|
|
return {}
|
|
header = lines[0].split(",")
|
|
columns: dict[str, list[float]] = {name: [] for name in header}
|
|
for line in lines[1:]:
|
|
cells = line.split(",")
|
|
if len(cells) != len(header):
|
|
continue
|
|
for name, cell in zip(header, cells, strict=True):
|
|
try:
|
|
columns[name].append(float(cell))
|
|
except ValueError:
|
|
columns[name].append(float("nan"))
|
|
columns.pop("time", None)
|
|
return columns
|
|
|
|
|
|
def trajectory_r2(config: dict[str, Any], trajectory: dict[str, list[float]]) -> float | None:
|
|
"""Mean R^2 of the simulated states against the generated reference."""
|
|
if not trajectory:
|
|
return None
|
|
truth = ground_truth_trajectory(config)
|
|
shared = [name for name in trajectory if name in truth]
|
|
scores = [
|
|
value
|
|
for name in shared
|
|
if (value := _r2(trajectory[name], truth[name])) is not None
|
|
]
|
|
if not scores:
|
|
return None
|
|
if any(math.isinf(value) for value in scores):
|
|
return float("-inf")
|
|
return sum(scores) / len(scores)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# CLI invocation.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DiscoverRun:
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
world: Path
|
|
elapsed_ns: int
|
|
mean_squared_error: float | None
|
|
complexity: int | None
|
|
|
|
|
|
def _discover_args(config: dict[str, Any], dataset: Path, world: Path) -> list[str]:
|
|
discovery = config["discovery"]
|
|
states = ",".join(str(name) for name in config["system"]["states"])
|
|
args = ["discover", str(dataset), "--time", "time", "--state", states, "--output", str(world)]
|
|
if "preset" in discovery:
|
|
args.extend(["--preset", str(discovery["preset"])])
|
|
args.extend(["--degree", str(discovery.get("degree", 2))])
|
|
args.extend(["--threshold", str(discovery.get("threshold", 0.05))])
|
|
if discovery.get("trigonometric"):
|
|
args.append("--trigonometric")
|
|
if discovery.get("rational"):
|
|
args.append("--rational")
|
|
return args
|
|
|
|
|
|
def _run(binary: Path, args: list[str], cwd: Path) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run([str(binary), *args], cwd=cwd, text=True, capture_output=True, check=False)
|
|
|
|
|
|
def run_discover(binary: Path, config: dict[str, Any], dataset: Path, world: Path, root: Path) -> DiscoverRun:
|
|
started = time.perf_counter_ns()
|
|
completed = _run(binary, _discover_args(config, dataset, world), root)
|
|
elapsed = time.perf_counter_ns() - started
|
|
mse: float | None = None
|
|
complexity: int | None = None
|
|
match = _SUMMARY.search(completed.stdout)
|
|
if match is not None:
|
|
mse, complexity = float(match.group(1)), int(match.group(2))
|
|
return DiscoverRun(
|
|
returncode=completed.returncode,
|
|
stdout=completed.stdout,
|
|
stderr=completed.stderr,
|
|
world=world,
|
|
elapsed_ns=elapsed,
|
|
mean_squared_error=mse,
|
|
complexity=complexity,
|
|
)
|
|
|
|
|
|
def _simulation_plan(config: dict[str, Any]) -> dict[str, Any] | None:
|
|
expect = config.get("expect", {})
|
|
if not expect.get("simulate", True):
|
|
return None
|
|
system = config["system"]
|
|
initial = {str(k): float(v) for k, v in system["initial"].items()}
|
|
simulation = config.get("simulation")
|
|
if simulation is not None:
|
|
return {
|
|
"start": float(simulation["start"]),
|
|
"end": float(simulation["end"]),
|
|
"step": float(simulation["step"]),
|
|
"initial": initial,
|
|
}
|
|
step = float(system["step"])
|
|
samples = int(system["samples"])
|
|
return {"start": 0.0, "end": step * (samples - 1), "step": step, "initial": initial}
|
|
|
|
|
|
def run_simulate(binary: Path, plan: dict[str, Any], world: Path, root: Path) -> tuple[int, dict[str, list[float]], str]:
|
|
args = [
|
|
"simulate", str(world),
|
|
"--start", f"{plan['start']:.12g}",
|
|
"--end", f"{plan['end']:.12g}",
|
|
"--step", f"{plan['step']:.12g}",
|
|
]
|
|
for name, value in plan["initial"].items():
|
|
args.extend(["--initial", f"{name}={value:.12g}"])
|
|
completed = _run(binary, args, root)
|
|
trajectory = _parse_trajectory(completed.stdout) if completed.returncode == 0 else {}
|
|
return completed.returncode, trajectory, completed.stderr
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Determinism check.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def determinism_check(binary: Path, config: dict[str, Any], dataset: Path, workdir: Path, root: Path) -> bool:
|
|
"""Discover twice and assert the produced world bundles are byte-identical."""
|
|
first = workdir / "determinism_a.lsworld"
|
|
second = workdir / "determinism_b.lsworld"
|
|
run_a = run_discover(binary, config, dataset, first, root)
|
|
run_b = run_discover(binary, config, dataset, second, root)
|
|
if run_a.returncode != 0 or run_b.returncode != 0:
|
|
return False
|
|
if not first.is_file() or not second.is_file():
|
|
return False
|
|
return first.read_bytes() == second.read_bytes()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Case orchestration.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def repository_root(case_dir: Path) -> Path:
|
|
for parent in (case_dir, *case_dir.parents):
|
|
if (parent / "Cargo.toml").is_file() and (parent / "crates").is_dir():
|
|
return parent
|
|
raise RuntimeError(f"could not find LawSynth repository from {case_dir}")
|
|
|
|
|
|
def read_config(case_dir: Path) -> dict[str, Any]:
|
|
import tomllib
|
|
|
|
with (case_dir / "benchmark.toml").open("rb") as handle:
|
|
return tomllib.load(handle)
|
|
|
|
|
|
def _boundary_outcome(config: dict[str, Any], dataset: Path) -> dict[str, Any]:
|
|
return {
|
|
"status": BOUNDARY,
|
|
"passed": False,
|
|
"family": str(config["family"]),
|
|
"reason": str(config["capability"]["reason"]),
|
|
"dataset": str(dataset),
|
|
"determinism": None,
|
|
}
|
|
|
|
|
|
def _score_vector(
|
|
run: DiscoverRun,
|
|
recovery: dict[str, Any] | None,
|
|
r2: float | None,
|
|
sim_failure: float | None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"fit_train": run.mean_squared_error,
|
|
"trajectory_r2": r2,
|
|
"complexity_nodes": run.complexity,
|
|
"simulation_failure_rate": sim_failure,
|
|
"symbolic_level": recovery["level"] if recovery else "none",
|
|
"law_recovery_rate": recovery["law_recovery_rate"] if recovery else None,
|
|
"system_recovered": recovery["system_recovered"] if recovery else None,
|
|
"training_time_ns": run.elapsed_ns,
|
|
}
|
|
|
|
|
|
def run_srbench_case(
|
|
case_dir: Path,
|
|
workdir: Path,
|
|
binary: Path | None,
|
|
*,
|
|
check_determinism: bool = True,
|
|
) -> dict[str, Any]:
|
|
"""Generate, execute, score, and classify one SRBench-family case."""
|
|
config = read_config(case_dir)
|
|
root = repository_root(case_dir)
|
|
workdir.mkdir(parents=True, exist_ok=True)
|
|
dataset = write_dataset(config, workdir)
|
|
|
|
if config["capability"]["status"] != "supported":
|
|
return _boundary_outcome(config, dataset)
|
|
|
|
if binary is None:
|
|
return {"status": FAILED, "passed": False, "reason": "CLI binary unavailable", "determinism": None}
|
|
|
|
world = workdir / "world.lsworld"
|
|
run = run_discover(binary, config, dataset, world, root)
|
|
if run.returncode != 0:
|
|
return {
|
|
"status": REGRESSION,
|
|
"passed": False,
|
|
"reason": "discover failed",
|
|
"returncode": run.returncode,
|
|
"stderr": run.stderr or None,
|
|
"determinism": None,
|
|
}
|
|
|
|
explain = _run(binary, ["explain", str(world)], root)
|
|
discovered = parse_laws(explain.stdout) if explain.returncode == 0 else {}
|
|
expected_laws = config.get("recovery", {}).get("law", [])
|
|
recovery = score_recovery(discovered, expected_laws) if expected_laws else None
|
|
|
|
plan = _simulation_plan(config)
|
|
simulate_rc: int | None = None
|
|
r2: float | None = None
|
|
sim_failure: float | None = None
|
|
simulate_stderr = ""
|
|
if plan is not None:
|
|
simulate_rc, trajectory, simulate_stderr = run_simulate(binary, plan, world, root)
|
|
r2 = trajectory_r2(config, trajectory)
|
|
diverged = simulate_rc != 0 or (r2 is not None and not math.isfinite(r2))
|
|
sim_failure = 1.0 if diverged else 0.0
|
|
|
|
determinism: bool | None = None
|
|
if check_determinism:
|
|
determinism = determinism_check(binary, config, dataset, workdir, root)
|
|
|
|
expect = config.get("expect", {})
|
|
passed = _passed(run, explain.returncode, simulate_rc, sim_failure, recovery, r2, expect, determinism)
|
|
status = PASSED if passed else REGRESSION
|
|
return {
|
|
"status": status,
|
|
"passed": passed,
|
|
"family": str(config["family"]),
|
|
"returncode": run.returncode,
|
|
"inspect_returncode": explain.returncode,
|
|
"simulate_returncode": simulate_rc,
|
|
"determinism": determinism,
|
|
"recovery": recovery,
|
|
"score_vector": _score_vector(run, recovery, r2, sim_failure),
|
|
"stderr": None if passed else (run.stderr or simulate_stderr or explain.stderr or None),
|
|
}
|
|
|
|
|
|
def _passed(
|
|
run: DiscoverRun,
|
|
explain_rc: int,
|
|
simulate_rc: int | None,
|
|
sim_failure: float | None,
|
|
recovery: dict[str, Any] | None,
|
|
r2: float | None,
|
|
expect: dict[str, Any],
|
|
determinism: bool | None,
|
|
) -> bool:
|
|
if run.returncode != 0 or explain_rc != 0:
|
|
return False
|
|
if run.mean_squared_error is None or not math.isfinite(run.mean_squared_error):
|
|
return False
|
|
if simulate_rc is not None and simulate_rc != 0:
|
|
return False
|
|
if sim_failure == 1.0:
|
|
return False
|
|
expected_level = str(expect.get("symbolic_recovery", "none"))
|
|
if expected_level != "none":
|
|
achieved = recovery["level"] if recovery else "none"
|
|
if _RECOVERY_RANK.get(achieved, 0) < _RECOVERY_RANK[expected_level]:
|
|
return False
|
|
r2_min = expect.get("r2_min")
|
|
if r2_min is not None and (r2 is None or r2 < float(r2_min)):
|
|
return False
|
|
if determinism is False:
|
|
return False
|
|
return True
|