2026-08-16 22:04:11 +03:00

170 lines
7.9 KiB
Python

"""Shared executable contracts for LawSynth's supported performance surface.
The runners in this directory measure real work. They do not substitute a
model implementation: native bundle/ODE cases invoke the Rust CLI and SDK
cases exercise the installed source package directly.
"""
from __future__ import annotations
import csv
import hashlib
import json
import os
import struct
import subprocess
import sys
import tempfile
import time
import tomllib
import tracemalloc
import zipfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SDK = ROOT / "python" / "lawsynth" / "src"
MANIFEST = b'{\n "format": "lawsynth-world",\n "format_version": "0.1.0",\n "world_encoding": "lawsynth-world-binary-v1"\n}\n'
def _string(value: str) -> bytes:
encoded = value.encode()
return struct.pack("<H", len(encoded)) + encoded
def _expr(expression: dict[str, object]) -> bytes:
if "constant" in expression:
return b"\x00" + struct.pack("<d", float(expression["constant"]))
if "symbol" in expression:
return b"\x01" + _string(str(expression["symbol"]))
op = {"add": 0, "subtract": 1, "multiply": 2, "divide": 3, "power": 4}[str(expression["binary"])]
return bytes((3, op)) + _expr(expression["left"]) + _expr(expression["right"])
def write_decay_bundle(path: Path) -> None:
"""Write a documented binary-v1 archive for dx/dt = -rate*x."""
world = bytearray(b"LSW1")
world.extend(struct.pack("<I", 1))
world.extend(_string("x") + b"\x00\x00")
world.extend(struct.pack("<I", 1))
world.extend(_string("rate") + struct.pack("<d", 1.0) + b"\x00")
world.extend(struct.pack("<I", 1))
world.extend(_string("x"))
world.extend(_expr({"binary": "multiply", "left": {"constant": -1.0}, "right": {"binary": "multiply", "left": {"symbol": "rate"}, "right": {"symbol": "x"}}}))
entries = {"manifest.json": MANIFEST, "world/world.bin": bytes(world)}
entries["provenance/checksums.sha256"] = "".join(
f"{hashlib.sha256(value).hexdigest()} {name}\n" for name, value in entries.items()
).encode()
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_STORED) as archive:
for name, value in entries.items():
archive.writestr(name, value)
def cli(*arguments: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["cargo", "run", "--quiet", "-p", "lawsynth-cli", "--bin", "lawsynth", "--", *arguments],
cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False,
)
def source_env() -> dict[str, str]:
environment = dict(os.environ)
environment["PYTHONPATH"] = str(SDK) + os.pathsep + environment.get("PYTHONPATH", "")
return environment
def load_case(directory: Path) -> tuple[dict[str, object], dict[str, object], dict[str, object]]:
with (directory / "case.toml").open("rb") as handle:
case = tomllib.load(handle)
return case, json.loads((directory / "input.json").read_text()), json.loads((directory / "expected.json").read_text())
def elapsed(operation):
started = time.perf_counter()
value = operation()
return value, time.perf_counter() - started
def require_budget(seconds: float, limit: float, label: str) -> None:
assert seconds <= limit, f"{label} exceeded budget: {seconds:.6f}s > {limit:.6f}s"
def _sdk_workload(case_id: str, size: int) -> tuple[int, int]:
sys.path.insert(0, str(SDK))
from lawsynth import Dataset, DiscoveryConfig
from lawsynth.candidate import CandidateMetrics
from lawsynth.frontier import pareto_front
from lawsynth.intervention import Intervention
from lawsynth.scenario import Scenario
if case_id == "expression-throughput":
metrics = [CandidateMetrics(float(index % 23), index % 11) for index in range(size)]
return len(metrics), len(pareto_front(metrics))
if case_id in {"profile-million", "memory-budget"}:
dataset = Dataset.from_columns(range(size), {"x": (float(index) for index in range(size))})
return len(dataset.time), len(dataset.columns["x"])
if case_id == "event-latency":
events = tuple(Intervention(float(index), "rate", float(index % 3)) for index in range(size))
scenario = Scenario({"x": 1.0}, interventions=events)
return len(scenario.interventions), int(scenario.interventions[-1].time)
if case_id == "studio-paint":
# The studio is not a P1 runtime. Measure its supported data boundary:
# serialising real client-side world metadata for a rendering host.
payload = {"states": [f"x{index}" for index in range(size)], "config": DiscoveryConfig().__dict__ if hasattr(DiscoveryConfig(), "__dict__") else {"solver": "stlsq"}}
encoded = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode()
return len(payload["states"]), len(encoded)
if case_id == "import-time":
completed = subprocess.run([sys.executable, "-c", "import lawsynth; print(lawsynth.__version__)"], env=source_env(), text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
assert completed.returncode == 0, completed.stderr
assert completed.stdout.strip()
return 1, len(completed.stdout.strip())
raise AssertionError(f"unsupported SDK workload {case_id}")
def execute(directory: Path) -> None:
case, input_data, expected = load_case(directory)
specification = case["case"]
assert specification["id"] == directory.name == input_data["case_id"] == expected["case_id"]
limit = float(specification["max_seconds"])
size = int(input_data["size"])
case_id = directory.name
if case_id in {"bundle-open", "ode-simulation"}:
with tempfile.TemporaryDirectory(prefix="lawsynth-performance-") as temporary:
bundle = Path(temporary) / "decay.lsworld"
write_decay_bundle(bundle)
if case_id == "bundle-open":
result, duration = elapsed(lambda: cli("inspect", str(bundle)))
assert result.returncode == 0, result.stderr
assert "continuous world: 1 states" in result.stdout
samples = 1
else:
result, duration = elapsed(lambda: cli("simulate", str(bundle), "--initial", "x=1", "--start", "0", "--end", str(size / 1000), "--step", "0.001"))
assert result.returncode == 0, result.stderr
rows = list(csv.DictReader(result.stdout.splitlines()))
assert len(rows) == size + 1 and float(rows[-1]["x"]) < 1.0
samples = len(rows)
detail = len(result.stdout)
elif case_id == "cancellation-latency":
command = ["cargo", "test", "--quiet", "-p", "lawsynth-discovery", "honours_a_pre_cancelled_discovery_request"]
result, duration = elapsed(lambda: subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False))
assert result.returncode == 0, result.stdout + result.stderr
samples = 1
detail = len(result.stdout)
elif case_id == "parquet-load":
command = ["cargo", "test", "--quiet", "-p", "lawsynth-data", "reads_uncompressed_plain_numeric_pages"]
result, duration = elapsed(lambda: subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False))
assert result.returncode == 0, result.stdout + result.stderr
samples = 1
detail = len(result.stdout)
elif case_id == "memory-budget":
tracemalloc.start()
(samples, detail), duration = elapsed(lambda: _sdk_workload(case_id, size))
_, peak = tracemalloc.get_traced_memory()
tracemalloc.stop()
assert peak <= int(expected["max_peak_bytes"]), f"peak allocation {peak} exceeds budget"
else:
(samples, detail), duration = elapsed(lambda: _sdk_workload(case_id, size))
require_budget(duration, limit, case_id)
assert samples >= int(expected["minimum_samples"])
print(json.dumps({"case": case_id, "seconds": round(duration, 6), "samples": samples, "detail": detail}, sort_keys=True))