This repository has been archived on 2026-11-17. You can view files and clone it, but cannot push or open issues or pull requests.
caelum0x 49b183818c Unify gitgram + aitimemachine into the monorepo (de-submodule) and merge vendored engines
Convert gitgram/ and aitimemachine/ from embedded git repos (which showed as
opaque submodule gitlinks on GitHub) into fully-tracked first-party content.

AITM now merges its 7 vendored repos as first-party engines:
- memory: beads (Go task DAG) + context7 (docs SDK), unified into saved_contexts
- git: vendored C git (bundled binary) + gitnexus (in-process code intelligence)
- agentic: a2a (@a2a-js/sdk) + pal (AITM-managed Python subprocess)

Includes the full session's product build: Gitgram web/mobile/backend features,
AITM website/CLI/MCP, the unified memory layer, engine build tooling
(scripts/build-engines.sh, Dockerfile, CI), docs (README, ENGINES.md, ARCHITECTURE.md),
and extensive test suites.

Build artifacts, node_modules, external/, and gemini-cli are gitignored.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-23 14:14:09 +03:00

318 lines
7.8 KiB
TypeScript

/**
* Universal Search Evaluation Harness
* Tests search quality against known fixtures
*
* Usage:
* npx ts-node tests/eval/search-eval.ts
* npm run test:eval
*/
import { describe, it, expect, beforeAll } from "vitest";
// Test fixtures representing real search scenarios
interface SearchFixture {
id: string;
query: string;
expectedResults: {
source: string;
mustContain?: string[]; // At least one of these should appear in top results
mustNotContain?: string[]; // These should NOT appear in top results
minScore?: number;
};
category: "code" | "kb" | "artifact" | "context" | "mixed";
}
const FIXTURES: SearchFixture[] = [
// Code search
{
id: "code-function-name",
query: "createCheckpoint",
expectedResults: {
source: "code",
mustContain: ["checkpoint", "create"],
},
category: "code",
},
{
id: "code-snake-case",
query: "create_checkpoint",
expectedResults: {
source: "code",
mustContain: ["checkpoint", "createCheckpoint", "create_checkpoint"],
},
category: "code",
},
{
id: "code-camel-case",
query: "getUserById",
expectedResults: {
source: "code",
mustContain: ["user", "get", "getUser"],
},
category: "code",
},
// Knowledge base search
{
id: "kb-concept",
query: "time travel restore",
expectedResults: {
source: "knowledge_base",
mustContain: ["restore", "checkpoint", "time"],
},
category: "kb",
},
{
id: "kb-error-handling",
query: "error handling API",
expectedResults: {
source: "knowledge_base",
mustContain: ["error", "API", "handling"],
},
category: "kb",
},
// Artifact search
{
id: "artifact-export",
query: "session export JSON",
expectedResults: {
source: "artifact",
mustContain: ["export", "session", "JSON"],
},
category: "artifact",
},
// Context search
{
id: "context-conversation",
query: "implement feature",
expectedResults: {
source: "context",
mustContain: ["implement", "feature"],
},
category: "context",
},
// Mixed source queries
{
id: "mixed-restore",
query: "restore previous state",
expectedResults: {
source: "mixed",
mustContain: ["restore", "state", "previous", "checkpoint"],
},
category: "mixed",
},
{
id: "mixed-search",
query: "search embeddings semantic",
expectedResults: {
source: "mixed",
mustContain: ["search", "embedding", "semantic"],
},
category: "mixed",
},
];
// Evaluation metrics
interface EvalMetrics {
precision: number; // Relevant results / Retrieved results
recall: number; // Relevant results / Total relevant
mrr: number; // Mean Reciprocal Rank
ndcg: number; // Normalized Discounted Cumulative Gain
}
interface EvalResult {
fixtureId: string;
passed: boolean;
metrics: Partial<EvalMetrics>;
topResults: Array<{ title: string; score: number; source: string }>;
errors?: string[];
}
// Threshold for CI pass/fail
const THRESHOLDS = {
minPrecision: 0.8,
minRecall: 0.6,
minMRR: 0.5,
minPassRate: 0.8,
};
/**
* Mock search function - replace with actual API call in integration tests
*/
async function mockSearch(query: string): Promise<any[]> {
// This would be replaced with actual HTTP call to universal-search endpoint
return [];
}
/**
* Calculate precision for a single query
*/
function calculatePrecision(
results: any[],
fixture: SearchFixture,
k: number = 10
): number {
const topK = results.slice(0, k);
if (topK.length === 0) return 0;
const mustContain = fixture.expectedResults.mustContain || [];
let relevant = 0;
for (const result of topK) {
const text = `${result.title || ""} ${result.snippet || ""}`.toLowerCase();
const isRelevant = mustContain.some(term => text.includes(term.toLowerCase()));
if (isRelevant) relevant++;
}
return relevant / topK.length;
}
/**
* Calculate Mean Reciprocal Rank
*/
function calculateMRR(results: any[], fixture: SearchFixture): number {
const mustContain = fixture.expectedResults.mustContain || [];
for (let i = 0; i < results.length; i++) {
const text = `${results[i].title || ""} ${results[i].snippet || ""}`.toLowerCase();
const isRelevant = mustContain.some(term => text.includes(term.toLowerCase()));
if (isRelevant) {
return 1 / (i + 1);
}
}
return 0;
}
/**
* Run evaluation for a single fixture
*/
async function evaluateFixture(
fixture: SearchFixture,
searchFn: (query: string) => Promise<any[]>
): Promise<EvalResult> {
const errors: string[] = [];
try {
const results = await searchFn(fixture.query);
// Check for required terms
const mustContain = fixture.expectedResults.mustContain || [];
const topResults = results.slice(0, 5).map(r => ({
title: r.title,
score: r.score,
source: r.source,
}));
let foundTerms = 0;
for (const term of mustContain) {
const found = results.slice(0, 10).some(r => {
const text = `${r.title || ""} ${r.snippet || ""}`.toLowerCase();
return text.includes(term.toLowerCase());
});
if (found) foundTerms++;
}
const precision = calculatePrecision(results, fixture);
const mrr = calculateMRR(results, fixture);
// Check must not contain
const mustNotContain = fixture.expectedResults.mustNotContain || [];
for (const term of mustNotContain) {
const found = results.slice(0, 5).some(r => {
const text = `${r.title || ""} ${r.snippet || ""}`.toLowerCase();
return text.includes(term.toLowerCase());
});
if (found) {
errors.push(`Unwanted term "${term}" found in top results`);
}
}
const passed =
precision >= THRESHOLDS.minPrecision &&
mrr >= THRESHOLDS.minMRR &&
errors.length === 0;
return {
fixtureId: fixture.id,
passed,
metrics: { precision, mrr },
topResults,
errors: errors.length > 0 ? errors : undefined,
};
} catch (error) {
return {
fixtureId: fixture.id,
passed: false,
metrics: {},
topResults: [],
errors: [`Evaluation failed: ${(error as Error).message}`],
};
}
}
/**
* Run full evaluation suite
*/
export async function runEvaluation(
searchFn: (query: string) => Promise<any[]>,
fixtures: SearchFixture[] = FIXTURES
): Promise<{
passed: boolean;
passRate: number;
results: EvalResult[];
summary: {
total: number;
passed: number;
failed: number;
avgPrecision: number;
avgMRR: number;
};
}> {
const results: EvalResult[] = [];
for (const fixture of fixtures) {
const result = await evaluateFixture(fixture, searchFn);
results.push(result);
}
const passed = results.filter(r => r.passed).length;
const passRate = passed / results.length;
const avgPrecision = results
.filter(r => r.metrics.precision !== undefined)
.reduce((sum, r) => sum + (r.metrics.precision || 0), 0) / results.length;
const avgMRR = results
.filter(r => r.metrics.mrr !== undefined)
.reduce((sum, r) => sum + (r.metrics.mrr || 0), 0) / results.length;
return {
passed: passRate >= THRESHOLDS.minPassRate,
passRate,
results,
summary: {
total: results.length,
passed,
failed: results.length - passed,
avgPrecision,
avgMRR,
},
};
}
// Export for use in tests
export { FIXTURES, THRESHOLDS, SearchFixture, EvalResult };
// CLI runner
if (process.argv[1]?.includes("search-eval")) {
console.log("Search Evaluation Harness");
console.log("=========================");
console.log(`Fixtures: ${FIXTURES.length}`);
console.log(`Thresholds: Precision >= ${THRESHOLDS.minPrecision}, MRR >= ${THRESHOLDS.minMRR}`);
console.log("\nNote: Run with actual API for real evaluation");
}