lawsynth/benchmarks/_performance.py
2026-08-16 22:04:11 +03:00

94 lines
3.7 KiB
Python

"""Executable performance harnesses for public LawSynth workflows.
The benchmark fixtures describe a workload rather than a machine-specific
number. Runs measure a real command (or the public Python SDK) with a
monotonic clock and persist their observations in the requested work
directory. Checked-in baselines are deliberately qualitative; absolute
timings belong to the machine and CI environment that produced them.
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
from typing import Any
from _common import invoke_discovery, read_config, repository_root, write_dataset, write_json
def _python_sdk_workload(case_dir: Path) -> dict[str, Any]:
"""Exercise immutable public SDK objects without fabricating native work."""
root = repository_root(case_dir)
source = root / "python" / "lawsynth" / "src"
if str(source) not in sys.path:
sys.path.insert(0, str(source))
from lawsynth import Dataset, DiscoveryConfig
from lawsynth.variable import Variable
started = time.perf_counter_ns()
times = tuple(index / 1_000.0 for index in range(2_000))
dataset = Dataset.from_columns(times, {"x": tuple(index / 2_000.0 for index in range(2_000))})
variable = Variable("x", unit="1")
config = DiscoveryConfig(polynomial_degree=3, threshold=0.05)
elapsed = time.perf_counter_ns() - started
return {
"returncode": 0,
"operation": "public-python-sdk-object-construction",
"rows": len(dataset.time),
"variable": variable.name,
"degree": config.polynomial_degree,
"elapsed_ns": elapsed,
}
def run_workload(case_dir: Path, workdir: Path) -> dict[str, Any]:
"""Run the configured public workload and attach an elapsed duration."""
workflow = read_config(case_dir)["workload"]["workflow"]
if workflow == "python-sdk":
return _python_sdk_workload(case_dir)
if workflow != "native-cli":
raise ValueError(f"unknown benchmark workflow {workflow!r}")
started = time.perf_counter_ns()
result = invoke_discovery(case_dir, workdir)
result["elapsed_ns"] = time.perf_counter_ns() - started
result["operation"] = "native-discover-inspect" + ("-simulate" if "simulate_returncode" in result else "")
return result
def score_workload(case_dir: Path, result: dict[str, Any]) -> dict[str, Any]:
"""Report a portable, correctness-gated result instead of a fake SLA."""
workflow = read_config(case_dir)["workload"]["workflow"]
passed = result.get("returncode") == 0
if workflow == "native-cli":
passed = passed and result.get("inspect_returncode") == 0
if "simulate_returncode" in result:
passed = passed and result["simulate_returncode"] == 0
return {
"status": "passed" if passed else "failed",
"passed": passed,
"elapsed_ns": result.get("elapsed_ns"),
"operation": result.get("operation"),
"machine_independent": True,
}
def script_main(case_dir: Path, mode: str) -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--workdir", type=Path, default=case_dir / ".benchmark-run")
arguments = parser.parse_args()
if mode == "generate":
print(write_dataset(case_dir, arguments.workdir))
return 0
result = run_workload(case_dir, arguments.workdir)
if mode == "run":
write_json(arguments.workdir / "result.json", result)
print(json.dumps(result, sort_keys=True))
return 0 if result.get("returncode") == 0 else 2
score = score_workload(case_dir, result)
write_json(arguments.workdir / "score.json", score)
print(json.dumps(score, sort_keys=True))
return 0 if score["passed"] else 2