94 lines
3.7 KiB
Python
94 lines
3.7 KiB
Python
"""Executable performance harnesses for public LawSynth workflows.
|
|
|
|
The benchmark fixtures describe a workload rather than a machine-specific
|
|
number. Runs measure a real command (or the public Python SDK) with a
|
|
monotonic clock and persist their observations in the requested work
|
|
directory. Checked-in baselines are deliberately qualitative; absolute
|
|
timings belong to the machine and CI environment that produced them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from _common import invoke_discovery, read_config, repository_root, write_dataset, write_json
|
|
|
|
|
|
def _python_sdk_workload(case_dir: Path) -> dict[str, Any]:
|
|
"""Exercise immutable public SDK objects without fabricating native work."""
|
|
root = repository_root(case_dir)
|
|
source = root / "python" / "lawsynth" / "src"
|
|
if str(source) not in sys.path:
|
|
sys.path.insert(0, str(source))
|
|
from lawsynth import Dataset, DiscoveryConfig
|
|
from lawsynth.variable import Variable
|
|
|
|
started = time.perf_counter_ns()
|
|
times = tuple(index / 1_000.0 for index in range(2_000))
|
|
dataset = Dataset.from_columns(times, {"x": tuple(index / 2_000.0 for index in range(2_000))})
|
|
variable = Variable("x", unit="1")
|
|
config = DiscoveryConfig(polynomial_degree=3, threshold=0.05)
|
|
elapsed = time.perf_counter_ns() - started
|
|
return {
|
|
"returncode": 0,
|
|
"operation": "public-python-sdk-object-construction",
|
|
"rows": len(dataset.time),
|
|
"variable": variable.name,
|
|
"degree": config.polynomial_degree,
|
|
"elapsed_ns": elapsed,
|
|
}
|
|
|
|
|
|
def run_workload(case_dir: Path, workdir: Path) -> dict[str, Any]:
|
|
"""Run the configured public workload and attach an elapsed duration."""
|
|
workflow = read_config(case_dir)["workload"]["workflow"]
|
|
if workflow == "python-sdk":
|
|
return _python_sdk_workload(case_dir)
|
|
if workflow != "native-cli":
|
|
raise ValueError(f"unknown benchmark workflow {workflow!r}")
|
|
started = time.perf_counter_ns()
|
|
result = invoke_discovery(case_dir, workdir)
|
|
result["elapsed_ns"] = time.perf_counter_ns() - started
|
|
result["operation"] = "native-discover-inspect" + ("-simulate" if "simulate_returncode" in result else "")
|
|
return result
|
|
|
|
|
|
def score_workload(case_dir: Path, result: dict[str, Any]) -> dict[str, Any]:
|
|
"""Report a portable, correctness-gated result instead of a fake SLA."""
|
|
workflow = read_config(case_dir)["workload"]["workflow"]
|
|
passed = result.get("returncode") == 0
|
|
if workflow == "native-cli":
|
|
passed = passed and result.get("inspect_returncode") == 0
|
|
if "simulate_returncode" in result:
|
|
passed = passed and result["simulate_returncode"] == 0
|
|
return {
|
|
"status": "passed" if passed else "failed",
|
|
"passed": passed,
|
|
"elapsed_ns": result.get("elapsed_ns"),
|
|
"operation": result.get("operation"),
|
|
"machine_independent": True,
|
|
}
|
|
|
|
|
|
def script_main(case_dir: Path, mode: str) -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--workdir", type=Path, default=case_dir / ".benchmark-run")
|
|
arguments = parser.parse_args()
|
|
if mode == "generate":
|
|
print(write_dataset(case_dir, arguments.workdir))
|
|
return 0
|
|
result = run_workload(case_dir, arguments.workdir)
|
|
if mode == "run":
|
|
write_json(arguments.workdir / "result.json", result)
|
|
print(json.dumps(result, sort_keys=True))
|
|
return 0 if result.get("returncode") == 0 else 2
|
|
score = score_workload(case_dir, result)
|
|
write_json(arguments.workdir / "score.json", score)
|
|
print(json.dumps(score, sort_keys=True))
|
|
return 0 if score["passed"] else 2
|