lawsynth/tests/conformance/_conformance.py
caelum0x c8700e78f8 feat: §8.5 parameter refinement + §8.6 causal hypothesis + executable conformance suites
Discovery: opt-in joint parameter refinement (refine.rs, via lawsynth-opt
coordinate_minimize, guaranteed non-worsening trajectory MSE) and dependency/
causal hypothesis discovery (causal.rs, via lawsynth-causal Granger/independence/
time-order, framed as a hypothesis with its assumption set). CLI --refine/--causal.

Verification: wire tests/{conformance,end-to-end,cross-language} to execute against
the real compiled CLI — valid worlds round-trip, every bad-* asserted-rejected,
unbuilt native/TS legs honestly skipped. 26 pytest cases.

Rust workspace 342->352 tests; +26 system-verification cases. clippy/fmt clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-08-17 19:05:23 +03:00

172 lines
7.7 KiB
Python

#!/usr/bin/env python3
"""Executable native-CLI conformance runner for the LawSynth runtime.
The case files are deliberately declarative. This runner builds a real
stored-ZIP ``.lsworld`` archive using the documented binary-v1 layout and then
invokes the compiled Rust CLI. It never emulates simulation or acceptance
behavior: valid worlds must round-trip through ``inspect``/``simulate`` and
every ``bad-*`` fixture must be rejected by the engine with a non-zero exit and
the documented diagnostic.
"""
from __future__ import annotations
import csv
import hashlib
import json
import struct
import sys
import tempfile
import tomllib
import zipfile
from pathlib import Path
# ``tests/conformance/_conformance.py`` -> parents[1] is ``tests/`` where the
# shared engine helper lives. Insert it so the module works both under pytest
# and when a case ``run.py`` is executed standalone.
_TESTS = Path(__file__).resolve().parents[1]
if str(_TESTS) not in sys.path:
sys.path.insert(0, str(_TESTS))
from _engine import cli # noqa: E402
MANIFEST = b'{\n "format": "lawsynth-world",\n "format_version": "0.1.0",\n "world_encoding": "lawsynth-world-binary-v1"\n}\n'
_OUTCOME_FOR_MODE = {"valid": "accepted", "invalid": "rejected", "unsupported": "unsupported"}
def put_string(value: str) -> bytes:
encoded = value.encode("utf-8")
if len(encoded) > 65535:
raise ValueError("bundle string is too long")
return struct.pack("<H", len(encoded)) + encoded
def put_optional_string(value: str | None) -> bytes:
return b"\x00" if value is None else b"\x01" + put_string(value)
def put_expression(expression: dict[str, object]) -> bytes:
if "constant" in expression:
return b"\x00" + struct.pack("<d", float(expression["constant"]))
if "symbol" in expression:
return b"\x01" + put_string(str(expression["symbol"]))
if "unary" in expression:
operators = {"negate": 0, "exp": 1, "log": 2, "sin": 3, "cos": 4}
return bytes((2, operators[str(expression["unary"])])) + put_expression(expression["operand"]) # type: ignore[arg-type]
operators = {"add": 0, "subtract": 1, "multiply": 2, "divide": 3, "power": 4}
return bytes((3, operators[str(expression["binary"])])) + put_expression(expression["left"]) + put_expression(expression["right"]) # type: ignore[arg-type]
def encode_world(bundle: dict[str, object]) -> bytes:
kind = str(bundle["kind"])
output = bytearray(b"LSW1" if kind == "continuous" else b"LSD1")
variables = bundle["variables"]
parameters = bundle["parameters"]
laws = bundle["laws"]
output.extend(struct.pack("<I", len(variables))) # type: ignore[arg-type]
roles = {"state": 0, "control": 1, "exogenous": 2, "observed": 3, "latent": 4, "derived": 5}
for variable in variables: # type: ignore[union-attr]
output.extend(put_string(variable["id"])) # type: ignore[index]
output.append(roles[variable["role"]]) # type: ignore[index]
output.extend(put_optional_string(variable.get("unit"))) # type: ignore[union-attr]
output.extend(struct.pack("<I", len(parameters))) # type: ignore[arg-type]
for parameter in parameters: # type: ignore[union-attr]
output.extend(put_string(parameter["id"])) # type: ignore[index]
output.extend(struct.pack("<d", float(parameter["value"]))) # type: ignore[index]
output.extend(put_optional_string(parameter.get("unit"))) # type: ignore[union-attr]
output.extend(struct.pack("<I", len(laws))) # type: ignore[arg-type]
for law in laws: # type: ignore[union-attr]
output.extend(put_string(law["target"])) # type: ignore[index]
output.extend(put_expression(law["expression"])) # type: ignore[index]
return bytes(output)
def write_bundle(path: Path, world: bytes, checksum: bool = True, unsafe: bool = False) -> None:
entries = {
"manifest.json": MANIFEST,
"world/world.bin": world,
}
sums = "".join(f"{hashlib.sha256(value).hexdigest()} {name}\n" for name, value in entries.items())
entries["provenance/checksums.sha256"] = sums.encode()
if not checksum:
entries["provenance/checksums.sha256"] = b"0" * 64 + b" manifest.json\n"
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_STORED) as archive:
if unsafe:
archive.writestr("../world/world.bin", world)
else:
for name, value in entries.items():
archive.writestr(name, value)
def invalid_world(mutation: str) -> bytes:
if mutation == "unknown-expression-tag":
return b"LSW1" + struct.pack("<III", 0, 0, 1) + put_string("x") + b"\xff"
if mutation == "invalid-unit":
return (
b"LSW1"
+ struct.pack("<I", 1)
+ put_string("x")
+ b"\x00\x01"
+ put_string("definitely_not_a_unit")
+ struct.pack("<II", 0, 0)
)
raise ValueError(f"unknown invalid fixture mutation: {mutation}")
def check_columns(stdout: str, expected: dict[str, list[float]]) -> None:
rows = list(csv.DictReader(stdout.splitlines()))
assert rows, "simulation produced no CSV rows"
assert set(rows[0]) == set(expected), f"unexpected CSV columns: {rows[0].keys()}"
for name, values in expected.items():
actual = [float(row[name]) for row in rows]
assert len(actual) == len(values), f"{name}: wrong sample count"
for index, (left, right) in enumerate(zip(actual, values, strict=True)):
assert abs(left - right) <= 1e-12, f"{name}[{index}]: {left} != {right}"
def run_case(directory: Path) -> None:
"""Execute a single conformance case against the real engine and assert."""
with (directory / "case.toml").open("rb") as handle:
case = tomllib.load(handle)
input_data = json.loads((directory / "input.json").read_text())
expected = json.loads((directory / "expected.json").read_text())
assert case["case"]["id"] == directory.name == input_data["case_id"]
assert case["case"]["mode"] == input_data["mode"]
assert _OUTCOME_FOR_MODE[case["case"]["mode"]] == expected["outcome"]
if expected["outcome"] == "unsupported":
assert input_data["feature"] == expected["feature"]
print(f"{directory.name}: unsupported capability documented: {expected['feature']}")
return
with tempfile.TemporaryDirectory(prefix="lawsynth-conformance-") as temporary:
bundle = Path(temporary) / "fixture.lsworld"
if expected["outcome"] == "accepted":
write_bundle(bundle, encode_world(input_data["bundle"]))
inspected = cli("inspect", str(bundle))
assert inspected.returncode == 0, inspected.stderr
assert expected["inspect_contains"] in inspected.stdout
command = input_data["command"]
result = cli(command["operation"], str(bundle), *command["arguments"])
assert result.returncode == 0, result.stderr
check_columns(result.stdout, expected["columns"])
print(f"{directory.name}: native CLI accepted fixture")
return
mutation = input_data["fixture"]["mutation"]
if mutation == "not-a-zip":
bundle.write_bytes(b"not a ZIP archive")
else:
write_bundle(
bundle,
invalid_world(mutation) if mutation in {"unknown-expression-tag", "invalid-unit"} else b"LSW1" + struct.pack("<III", 0, 0, 0),
checksum=mutation != "checksum-mismatch",
unsafe=mutation == "unsafe-path",
)
result = cli("inspect", str(bundle))
assert result.returncode != 0, "invalid fixture unexpectedly loaded"
combined = result.stdout + result.stderr
assert expected["error_contains"] in combined, combined
print(f"{directory.name}: native CLI rejected fixture")