/** * Accuracy harness for `openjev classify`'s default (lexical) engine. * * Loads examples/classify-eval.jsonl (synthetic, hand-labeled, built for this * repo — see examples/README.md for provenance) and runs each text through * the same `classifyOne` the CLI uses, grouped by its `labels` set. Reports * accuracy overall, split by the `shared_vocab` tag (texts that deliberately * reuse words from a *different* label's vocabulary than their true one — * the review finding this harness exists to measure), and swept across a * range of `--threshold` values so a real default can be picked from data * instead of guessed. * * Usage: node --experimental-strip-types scripts/classify-eval.mjs [--threshold 0.0] */ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { loadEngine, classifyOne, applyThreshold } from "./classify.mjs"; const __dirname = dirname(fileURLToPath(import.meta.url)); const DATA_PATH = join(__dirname, "..", "examples", "classify-eval.jsonl"); const QUESTION = "Which label fits this text best?"; function loadCases() { return readFileSync(DATA_PATH, "utf8") .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean) .map((line) => JSON.parse(line)); } /** Run every case through the engine once; threshold sweeps reuse these raw results. */ async function scoreAll(engine, cases) { const out = []; for (const c of cases) { const labels = c.labels.split(","); const result = await classifyOne(engine, c.text, labels, {}, QUESTION); out.push({ case: c, result }); } return out; } function summarize(scored, threshold) { const bySet = new Map(); const bySharedVocab = { true: { n: 0, correct: 0, abstain: 0 }, false: { n: 0, correct: 0, abstain: 0 } }; let n = 0; let correct = 0; let abstain = 0; let wrongConfident = 0; for (const { case: c, result } of scored) { const judged = applyThreshold(result, threshold); n += 1; const tag = c.shared_vocab ? "true" : "false"; bySharedVocab[tag].n += 1; const set = bySet.get(c.set) ?? { n: 0, correct: 0, abstain: 0 }; set.n += 1; bySet.set(c.set, set); if (!judged.label) { abstain += 1; bySharedVocab[tag].abstain += 1; set.abstain += 1; continue; } const isCorrect = judged.label === c.label; if (isCorrect) { correct += 1; bySharedVocab[tag].correct += 1; set.correct += 1; } else { wrongConfident += 1; } } // decided = rows where the engine committed to a label (not abstained) const decided = n - abstain; return { threshold, n, correct, abstain, wrongConfident, accuracyOverall: correct / n, accuracyOnDecided: decided > 0 ? correct / decided : 0, bySet: Object.fromEntries( [...bySet.entries()].map(([k, v]) => [k, { ...v, accuracy: v.correct / v.n }]), ), bySharedVocab: Object.fromEntries( Object.entries(bySharedVocab).map(([k, v]) => [ k, { ...v, accuracy: v.n ? v.correct / v.n : 0 }, ]), ), }; } function printReport(r, label) { console.log(`\n=== ${label} (threshold=${r.threshold ?? "none"}) ===`); console.log( `overall: ${r.n} cases, ${r.correct} correct, ${r.abstain} abstained, ${r.wrongConfident} wrong-and-confident`, ); console.log(` accuracy (of all cases): ${(r.accuracyOverall * 100).toFixed(1)}%`); console.log(` accuracy (of decided only): ${(r.accuracyOnDecided * 100).toFixed(1)}%`); console.log(" by set:"); for (const [set, v] of Object.entries(r.bySet)) { console.log(` ${set.padEnd(12)} n=${v.n} correct=${v.correct} abstain=${v.abstain} acc=${(v.accuracy * 100).toFixed(1)}%`); } console.log(" by shared_vocab tag:"); for (const [tag, v] of Object.entries(r.bySharedVocab)) { console.log( ` shared_vocab=${tag.padEnd(5)} n=${v.n} correct=${v.correct} abstain=${v.abstain} acc=${(v.accuracy * 100).toFixed(1)}%`, ); } } async function main() { const cases = loadCases(); const engine = await loadEngine(); const scored = await scoreAll(engine, cases); const argThreshold = process.argv.includes("--threshold") ? Number(process.argv[process.argv.indexOf("--threshold") + 1]) : null; if (argThreshold !== null) { printReport(summarize(scored, argThreshold), "single run"); return; } // Baseline: no --threshold at all (CLI default when the flag is omitted). printReport(summarize(scored, null), "baseline (no --threshold, CLI default)"); // Sweep a range of thresholds to find the best real default. console.log("\n=== threshold sweep (accuracy over ALL cases, abstains count as wrong) ==="); const sweep = []; for (let t = 0; t <= 0.9; t += 0.05) { const r = summarize(scored, Math.round(t * 100) / 100); sweep.push(r); console.log( ` threshold=${r.threshold.toFixed(2)} acc_overall=${(r.accuracyOverall * 100).toFixed(1)}% acc_decided=${(r.accuracyOnDecided * 100).toFixed(1)}% abstain=${r.abstain}/${r.n} wrong_confident=${r.wrongConfident}`, ); } const best = sweep.reduce((a, b) => (b.accuracyOverall > a.accuracyOverall ? b : a)); console.log(`\nbest-by-accuracy-overall: threshold=${best.threshold} acc_overall=${(best.accuracyOverall * 100).toFixed(1)}%`); } main();