This repository has been archived on 2026-11-17. You can view files and clone it, but cannot push or open issues or pull requests.
gitgram-ai/aitimemachine/api-server/scripts/run-search-benchmark.ts
caelum0x 49b183818c Unify gitgram + aitimemachine into the monorepo (de-submodule) and merge vendored engines
Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as
opaque submodule gitlinks on GitHub) into fully-tracked first-party content.

AITM now merges its 7 vendored repos as first-party engines:
- memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts
- git: vendored C git (bundled binary) + gitnexus (in-process code intelligence)
- agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess)

Includes the full session's product build: Gitgram web/mobile/backend features,
AITM website/CLI/MCP, the unified memory layer, engine build tooling
(scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md),
and extensive test suites.

Build artifacts, node_modules, external/, and gemini-cli are gitignored.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-23 14:14:09 +03:00

219 lines
7.7 KiB
TypeScript

#!/usr/bin/env npx tsx
/**
* Search Relevance Benchmark Runner
*
* CLI tool to run the semantic search benchmark suite, produce a JSON report
* and a human-readable summary, and optionally compare against a previous run
* for regression detection.
*
* Usage:
* npx tsx scripts/run-search-benchmark.ts
*
* Environment variables:
* BENCH_BASE_URL - API server URL (default: http://localhost:3001/v1)
* BENCH_API_KEY - API key for authentication (required)
* BENCH_OUTPUT_JSON - Path to write JSON report (default: benchmark-report.json)
* BENCH_PREVIOUS_JSON - Path to previous run for regression comparison
* BENCH_CATEGORIES - Comma-separated category filter (code,checkpoint,conversation,multi)
* BENCH_DIFFICULTIES - Comma-separated difficulty filter (easy,medium,hard)
* BENCH_TIMEOUT - Request timeout in ms (default: 10000)
* BENCH_DEGRADE_THRESHOLD - Regression detection threshold (default: 0.1)
*
* Threshold overrides (set as env vars):
* BENCH_MIN_PRECISION_5 - Minimum Precision@5 (default: 0.6)
* BENCH_MIN_PRECISION_10 - Minimum Precision@10 (default: 0.5)
* BENCH_MIN_PRECISION_20 - Minimum Precision@20 (default: 0.4)
* BENCH_MIN_RECALL_10 - Minimum Recall@10 (default: 0.7)
* BENCH_MIN_NDCG - Minimum NDCG (default: 0.65)
* BENCH_MIN_MRR - Minimum MRR (default: 0.5)
* BENCH_MAX_P95_LATENCY - Maximum P95 latency in ms (default: 500)
*
* Exit codes:
* 0 - All thresholds passed
* 1 - One or more thresholds failed
* 2 - Configuration error
*/
import fs from "fs";
import path from "path";
import process from "process";
import {
runBenchmark,
detectRegressions,
formatReport,
formatRegressions,
DEFAULT_THRESHOLDS,
type BenchmarkConfig,
type BenchmarkReport,
type BenchmarkThresholds,
} from "../tests/benchmarks/search-benchmark.js";
import type { BenchmarkCategory, BenchmarkDifficulty } from "../tests/benchmarks/search-benchmark-data.js";
// ---------------------------------------------------------------------------
// Parse CLI / Env
// ---------------------------------------------------------------------------
function envFloat(key: string): number | undefined {
const val = process.env[key];
if (val === undefined) return undefined;
const num = parseFloat(val);
return Number.isFinite(num) ? num : undefined;
}
function envInt(key: string): number | undefined {
const val = process.env[key];
if (val === undefined) return undefined;
const num = parseInt(val, 10);
return Number.isFinite(num) ? num : undefined;
}
function envList(key: string): string[] | undefined {
const val = process.env[key];
if (!val) return undefined;
return val.split(",").map((s) => s.trim()).filter(Boolean);
}
async function main() {
const baseUrl = process.env.BENCH_BASE_URL || "http://localhost:3001/v1";
const apiKey = process.env.BENCH_API_KEY;
const outputJson = process.env.BENCH_OUTPUT_JSON || "benchmark-report.json";
const previousJson = process.env.BENCH_PREVIOUS_JSON;
const degradeThreshold = envFloat("BENCH_DEGRADE_THRESHOLD") ?? 0.1;
const requestTimeout = envInt("BENCH_TIMEOUT") ?? 10000;
if (!apiKey) {
console.error("Error: BENCH_API_KEY environment variable is required.");
console.error("");
console.error("Usage:");
console.error(" BENCH_API_KEY=your-key npx tsx scripts/run-search-benchmark.ts");
process.exit(2);
}
// Build thresholds from env or defaults
const thresholds: BenchmarkThresholds = {
precision_at_5: envFloat("BENCH_MIN_PRECISION_5") ?? DEFAULT_THRESHOLDS.precision_at_5,
precision_at_10: envFloat("BENCH_MIN_PRECISION_10") ?? DEFAULT_THRESHOLDS.precision_at_10,
precision_at_20: envFloat("BENCH_MIN_PRECISION_20") ?? DEFAULT_THRESHOLDS.precision_at_20,
recall_at_10: envFloat("BENCH_MIN_RECALL_10") ?? DEFAULT_THRESHOLDS.recall_at_10,
ndcg: envFloat("BENCH_MIN_NDCG") ?? DEFAULT_THRESHOLDS.ndcg,
mrr: envFloat("BENCH_MIN_MRR") ?? DEFAULT_THRESHOLDS.mrr,
p95_latency_ms: envFloat("BENCH_MAX_P95_LATENCY") ?? DEFAULT_THRESHOLDS.p95_latency_ms,
};
// Parse optional filters
const categories = envList("BENCH_CATEGORIES") as BenchmarkCategory[] | undefined;
const difficulties = envList("BENCH_DIFFICULTIES") as BenchmarkDifficulty[] | undefined;
// Validate category values
const validCategories = new Set<string>(["code", "checkpoint", "conversation", "multi"]);
if (categories) {
for (const c of categories) {
if (!validCategories.has(c)) {
console.error(`Error: Invalid category "${c}". Valid: ${[...validCategories].join(", ")}`);
process.exit(2);
}
}
}
const validDifficulties = new Set<string>(["easy", "medium", "hard"]);
if (difficulties) {
for (const d of difficulties) {
if (!validDifficulties.has(d)) {
console.error(`Error: Invalid difficulty "${d}". Valid: ${[...validDifficulties].join(", ")}`);
process.exit(2);
}
}
}
// Print header
console.log("Search Relevance Benchmark");
console.log("==========================");
console.log(`Base URL: ${baseUrl}`);
console.log(`Output: ${outputJson}`);
if (categories) console.log(`Categories: ${categories.join(", ")}`);
if (difficulties) console.log(`Difficulties: ${difficulties.join(", ")}`);
console.log(`Timeout: ${requestTimeout}ms`);
console.log("");
// Build config
const config: BenchmarkConfig = {
baseUrl,
apiKey,
thresholds,
categories,
difficulties,
requestTimeout,
};
// Run benchmark
console.log("Running benchmark...");
console.log("");
const report = await runBenchmark(config);
// Print human-readable report
console.log(formatReport(report));
// Write JSON report
const outputPath = path.resolve(process.cwd(), outputJson);
fs.writeFileSync(outputPath, JSON.stringify(report, null, 2));
console.log(`JSON report written to: ${outputPath}`);
// Regression detection
if (previousJson) {
const prevPath = path.resolve(process.cwd(), previousJson);
if (fs.existsSync(prevPath)) {
console.log("");
console.log(`Comparing against previous run: ${prevPath}`);
const previousReport = JSON.parse(
fs.readFileSync(prevPath, "utf-8")
) as BenchmarkReport;
const regressions = detectRegressions(report, previousReport, degradeThreshold);
if (regressions.length > 0) {
console.log("");
console.log(formatRegressions(regressions));
console.log("");
console.log(`${regressions.length} regression(s) detected (threshold: ${degradeThreshold})`);
} else {
console.log("No regressions detected.");
}
// Also include regression info in the report
const enrichedReport = {
...report,
regression_comparison: {
previous_timestamp: previousReport.timestamp,
degrade_threshold: degradeThreshold,
regressions,
},
};
fs.writeFileSync(outputPath, JSON.stringify(enrichedReport, null, 2));
} else {
console.warn(`Warning: Previous report not found at ${prevPath}, skipping regression check.`);
}
}
// Exit code based on pass/fail
if (!report.passed) {
console.log("");
console.error("BENCHMARK FAILED - one or more thresholds not met.");
const failedMetrics = report.metrics.filter((m) => !m.passed);
for (const fm of failedMetrics) {
const direction = fm.name.includes("Latency") ? "<=" : ">=";
console.error(` ${fm.name}: ${fm.value} (threshold ${direction} ${fm.threshold})`);
}
process.exit(1);
}
console.log("");
console.log("BENCHMARK PASSED - all thresholds met.");
}
main().catch((error) => {
console.error("Benchmark runner failed:", error);
process.exit(2);
});