Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as opaque submodule gitlinks on GitHub) into fully-tracked first-party content. AITM now merges its 7 vendored repos as first-party engines: - memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts - git: vendored C git (bundled binary) + gitnexus (in-process code intelligence) - agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess) Includes the full session's product build: Gitgram web/mobile/backend features, AITM website/CLI/MCP, the unified memory layer, engine build tooling (scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md), and extensive test suites. Build artifacts, node_modules, external/, and gemini-cli are gitignored. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
219 lines
7.7 KiB
TypeScript
219 lines
7.7 KiB
TypeScript
#!/usr/bin/env npx tsx
|
|
/**
|
|
* Search Relevance Benchmark Runner
|
|
*
|
|
* CLI tool to run the semantic search benchmark suite, produce a JSON report
|
|
* and a human-readable summary, and optionally compare against a previous run
|
|
* for regression detection.
|
|
*
|
|
* Usage:
|
|
* npx tsx scripts/run-search-benchmark.ts
|
|
*
|
|
* Environment variables:
|
|
* BENCH_BASE_URL - API server URL (default: http://localhost:3001/v1)
|
|
* BENCH_API_KEY - API key for authentication (required)
|
|
* BENCH_OUTPUT_JSON - Path to write JSON report (default: benchmark-report.json)
|
|
* BENCH_PREVIOUS_JSON - Path to previous run for regression comparison
|
|
* BENCH_CATEGORIES - Comma-separated category filter (code,checkpoint,conversation,multi)
|
|
* BENCH_DIFFICULTIES - Comma-separated difficulty filter (easy,medium,hard)
|
|
* BENCH_TIMEOUT - Request timeout in ms (default: 10000)
|
|
* BENCH_DEGRADE_THRESHOLD - Regression detection threshold (default: 0.1)
|
|
*
|
|
* Threshold overrides (set as env vars):
|
|
* BENCH_MIN_PRECISION_5 - Minimum Precision@5 (default: 0.6)
|
|
* BENCH_MIN_PRECISION_10 - Minimum Precision@10 (default: 0.5)
|
|
* BENCH_MIN_PRECISION_20 - Minimum Precision@20 (default: 0.4)
|
|
* BENCH_MIN_RECALL_10 - Minimum Recall@10 (default: 0.7)
|
|
* BENCH_MIN_NDCG - Minimum NDCG (default: 0.65)
|
|
* BENCH_MIN_MRR - Minimum MRR (default: 0.5)
|
|
* BENCH_MAX_P95_LATENCY - Maximum P95 latency in ms (default: 500)
|
|
*
|
|
* Exit codes:
|
|
* 0 - All thresholds passed
|
|
* 1 - One or more thresholds failed
|
|
* 2 - Configuration error
|
|
*/
|
|
|
|
import fs from "fs";
|
|
import path from "path";
|
|
import process from "process";
|
|
|
|
import {
|
|
runBenchmark,
|
|
detectRegressions,
|
|
formatReport,
|
|
formatRegressions,
|
|
DEFAULT_THRESHOLDS,
|
|
type BenchmarkConfig,
|
|
type BenchmarkReport,
|
|
type BenchmarkThresholds,
|
|
} from "../tests/benchmarks/search-benchmark.js";
|
|
import type { BenchmarkCategory, BenchmarkDifficulty } from "../tests/benchmarks/search-benchmark-data.js";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Parse CLI / Env
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function envFloat(key: string): number | undefined {
|
|
const val = process.env[key];
|
|
if (val === undefined) return undefined;
|
|
const num = parseFloat(val);
|
|
return Number.isFinite(num) ? num : undefined;
|
|
}
|
|
|
|
function envInt(key: string): number | undefined {
|
|
const val = process.env[key];
|
|
if (val === undefined) return undefined;
|
|
const num = parseInt(val, 10);
|
|
return Number.isFinite(num) ? num : undefined;
|
|
}
|
|
|
|
function envList(key: string): string[] | undefined {
|
|
const val = process.env[key];
|
|
if (!val) return undefined;
|
|
return val.split(",").map((s) => s.trim()).filter(Boolean);
|
|
}
|
|
|
|
async function main() {
|
|
const baseUrl = process.env.BENCH_BASE_URL || "http://localhost:3001/v1";
|
|
const apiKey = process.env.BENCH_API_KEY;
|
|
const outputJson = process.env.BENCH_OUTPUT_JSON || "benchmark-report.json";
|
|
const previousJson = process.env.BENCH_PREVIOUS_JSON;
|
|
const degradeThreshold = envFloat("BENCH_DEGRADE_THRESHOLD") ?? 0.1;
|
|
const requestTimeout = envInt("BENCH_TIMEOUT") ?? 10000;
|
|
|
|
if (!apiKey) {
|
|
console.error("Error: BENCH_API_KEY environment variable is required.");
|
|
console.error("");
|
|
console.error("Usage:");
|
|
console.error(" BENCH_API_KEY=your-key npx tsx scripts/run-search-benchmark.ts");
|
|
process.exit(2);
|
|
}
|
|
|
|
// Build thresholds from env or defaults
|
|
const thresholds: BenchmarkThresholds = {
|
|
precision_at_5: envFloat("BENCH_MIN_PRECISION_5") ?? DEFAULT_THRESHOLDS.precision_at_5,
|
|
precision_at_10: envFloat("BENCH_MIN_PRECISION_10") ?? DEFAULT_THRESHOLDS.precision_at_10,
|
|
precision_at_20: envFloat("BENCH_MIN_PRECISION_20") ?? DEFAULT_THRESHOLDS.precision_at_20,
|
|
recall_at_10: envFloat("BENCH_MIN_RECALL_10") ?? DEFAULT_THRESHOLDS.recall_at_10,
|
|
ndcg: envFloat("BENCH_MIN_NDCG") ?? DEFAULT_THRESHOLDS.ndcg,
|
|
mrr: envFloat("BENCH_MIN_MRR") ?? DEFAULT_THRESHOLDS.mrr,
|
|
p95_latency_ms: envFloat("BENCH_MAX_P95_LATENCY") ?? DEFAULT_THRESHOLDS.p95_latency_ms,
|
|
};
|
|
|
|
// Parse optional filters
|
|
const categories = envList("BENCH_CATEGORIES") as BenchmarkCategory[] | undefined;
|
|
const difficulties = envList("BENCH_DIFFICULTIES") as BenchmarkDifficulty[] | undefined;
|
|
|
|
// Validate category values
|
|
const validCategories = new Set<string>(["code", "checkpoint", "conversation", "multi"]);
|
|
if (categories) {
|
|
for (const c of categories) {
|
|
if (!validCategories.has(c)) {
|
|
console.error(`Error: Invalid category "${c}". Valid: ${[...validCategories].join(", ")}`);
|
|
process.exit(2);
|
|
}
|
|
}
|
|
}
|
|
|
|
const validDifficulties = new Set<string>(["easy", "medium", "hard"]);
|
|
if (difficulties) {
|
|
for (const d of difficulties) {
|
|
if (!validDifficulties.has(d)) {
|
|
console.error(`Error: Invalid difficulty "${d}". Valid: ${[...validDifficulties].join(", ")}`);
|
|
process.exit(2);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Print header
|
|
console.log("Search Relevance Benchmark");
|
|
console.log("==========================");
|
|
console.log(`Base URL: ${baseUrl}`);
|
|
console.log(`Output: ${outputJson}`);
|
|
if (categories) console.log(`Categories: ${categories.join(", ")}`);
|
|
if (difficulties) console.log(`Difficulties: ${difficulties.join(", ")}`);
|
|
console.log(`Timeout: ${requestTimeout}ms`);
|
|
console.log("");
|
|
|
|
// Build config
|
|
const config: BenchmarkConfig = {
|
|
baseUrl,
|
|
apiKey,
|
|
thresholds,
|
|
categories,
|
|
difficulties,
|
|
requestTimeout,
|
|
};
|
|
|
|
// Run benchmark
|
|
console.log("Running benchmark...");
|
|
console.log("");
|
|
|
|
const report = await runBenchmark(config);
|
|
|
|
// Print human-readable report
|
|
console.log(formatReport(report));
|
|
|
|
// Write JSON report
|
|
const outputPath = path.resolve(process.cwd(), outputJson);
|
|
fs.writeFileSync(outputPath, JSON.stringify(report, null, 2));
|
|
console.log(`JSON report written to: ${outputPath}`);
|
|
|
|
// Regression detection
|
|
if (previousJson) {
|
|
const prevPath = path.resolve(process.cwd(), previousJson);
|
|
if (fs.existsSync(prevPath)) {
|
|
console.log("");
|
|
console.log(`Comparing against previous run: ${prevPath}`);
|
|
const previousReport = JSON.parse(
|
|
fs.readFileSync(prevPath, "utf-8")
|
|
) as BenchmarkReport;
|
|
|
|
const regressions = detectRegressions(report, previousReport, degradeThreshold);
|
|
|
|
if (regressions.length > 0) {
|
|
console.log("");
|
|
console.log(formatRegressions(regressions));
|
|
console.log("");
|
|
console.log(`${regressions.length} regression(s) detected (threshold: ${degradeThreshold})`);
|
|
} else {
|
|
console.log("No regressions detected.");
|
|
}
|
|
|
|
// Also include regression info in the report
|
|
const enrichedReport = {
|
|
...report,
|
|
regression_comparison: {
|
|
previous_timestamp: previousReport.timestamp,
|
|
degrade_threshold: degradeThreshold,
|
|
regressions,
|
|
},
|
|
};
|
|
fs.writeFileSync(outputPath, JSON.stringify(enrichedReport, null, 2));
|
|
} else {
|
|
console.warn(`Warning: Previous report not found at ${prevPath}, skipping regression check.`);
|
|
}
|
|
}
|
|
|
|
// Exit code based on pass/fail
|
|
if (!report.passed) {
|
|
console.log("");
|
|
console.error("BENCHMARK FAILED - one or more thresholds not met.");
|
|
const failedMetrics = report.metrics.filter((m) => !m.passed);
|
|
for (const fm of failedMetrics) {
|
|
const direction = fm.name.includes("Latency") ? "<=" : ">=";
|
|
console.error(` ${fm.name}: ${fm.value} (threshold ${direction} ${fm.threshold})`);
|
|
}
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log("");
|
|
console.log("BENCHMARK PASSED - all thresholds met.");
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error("Benchmark runner failed:", error);
|
|
process.exit(2);
|
|
});
|