umpirix/scripts/classify-eval.mjs
caelum0x 74e3b24f7a fix(classify): close lexicon gaps behind the lexical engine's shared-word mislabels
The classify CLI's lexical engine had zero lexicon coverage for "feature"
and "praise" (only "bug" was covered) and none at all for generic
sentiment (positive/negative/neutral), so bare-label runs of the CLI's
own README example either tied on everything or confidently mislabeled a
praise/feature row as a bug whenever a bug-shaped word like "crash"
appeared. Add two small, additive synonym groups to the existing lexicon.

Measured on a new 65-row synthetic gold set (examples/classify-eval.jsonl,
run via `npm run classify-eval`, scripts/classify-eval.mjs): 32.3% -> 49.2%
accuracy counting ties as wrong, 67.7% -> 71.1% on rows the engine
actually committed to; the sentiment set alone went 5.0% -> 35.0%. The
product-wide gold regression suite (`npm run gold`) still passes 61/61 —
every change is a new synonym entry, not an edit to an existing one.

README now documents the measured numbers, the --threshold tradeoff
(0.55 raises decided-row accuracy to 83.8% at the cost of more abstains),
and the real remaining limit: a bag-of-words engine can't fully resolve
a praise sentence that mentions "crash" from an actual bug report.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-11 14:26:12 +03:00

146 lines
5.2 KiB
JavaScript

/**
* Accuracy harness for `openjev classify`'s default (lexical) engine.
*
* Loads examples/classify-eval.jsonl (synthetic, hand-labeled, built for this
* repo — see examples/README.md for provenance) and runs each text through
* the same `classifyOne` the CLI uses, grouped by its `labels` set. Reports
* accuracy overall, split by the `shared_vocab` tag (texts that deliberately
* reuse words from a *different* label's vocabulary than their true one —
* the review finding this harness exists to measure), and swept across a
* range of `--threshold` values so a real default can be picked from data
* instead of guessed.
*
* Usage: node --experimental-strip-types scripts/classify-eval.mjs [--threshold 0.0]
*/
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
import { loadEngine, classifyOne, applyThreshold } from "./classify.mjs";
const __dirname = dirname(fileURLToPath(import.meta.url));
const DATA_PATH = join(__dirname, "..", "examples", "classify-eval.jsonl");
const QUESTION = "Which label fits this text best?";
function loadCases() {
return readFileSync(DATA_PATH, "utf8")
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => JSON.parse(line));
}
/** Run every case through the engine once; threshold sweeps reuse these raw results. */
async function scoreAll(engine, cases) {
const out = [];
for (const c of cases) {
const labels = c.labels.split(",");
const result = await classifyOne(engine, c.text, labels, {}, QUESTION);
out.push({ case: c, result });
}
return out;
}
function summarize(scored, threshold) {
const bySet = new Map();
const bySharedVocab = { true: { n: 0, correct: 0, abstain: 0 }, false: { n: 0, correct: 0, abstain: 0 } };
let n = 0;
let correct = 0;
let abstain = 0;
let wrongConfident = 0;
for (const { case: c, result } of scored) {
const judged = applyThreshold(result, threshold);
n += 1;
const tag = c.shared_vocab ? "true" : "false";
bySharedVocab[tag].n += 1;
const set = bySet.get(c.set) ?? { n: 0, correct: 0, abstain: 0 };
set.n += 1;
bySet.set(c.set, set);
if (!judged.label) {
abstain += 1;
bySharedVocab[tag].abstain += 1;
set.abstain += 1;
continue;
}
const isCorrect = judged.label === c.label;
if (isCorrect) {
correct += 1;
bySharedVocab[tag].correct += 1;
set.correct += 1;
} else {
wrongConfident += 1;
}
}
// decided = rows where the engine committed to a label (not abstained)
const decided = n - abstain;
return {
threshold,
n,
correct,
abstain,
wrongConfident,
accuracyOverall: correct / n,
accuracyOnDecided: decided > 0 ? correct / decided : 0,
bySet: Object.fromEntries(
[...bySet.entries()].map(([k, v]) => [k, { ...v, accuracy: v.correct / v.n }]),
),
bySharedVocab: Object.fromEntries(
Object.entries(bySharedVocab).map(([k, v]) => [
k,
{ ...v, accuracy: v.n ? v.correct / v.n : 0 },
]),
),
};
}
function printReport(r, label) {
console.log(`\n=== ${label} (threshold=${r.threshold ?? "none"}) ===`);
console.log(
`overall: ${r.n} cases, ${r.correct} correct, ${r.abstain} abstained, ${r.wrongConfident} wrong-and-confident`,
);
console.log(` accuracy (of all cases): ${(r.accuracyOverall * 100).toFixed(1)}%`);
console.log(` accuracy (of decided only): ${(r.accuracyOnDecided * 100).toFixed(1)}%`);
console.log(" by set:");
for (const [set, v] of Object.entries(r.bySet)) {
console.log(` ${set.padEnd(12)} n=${v.n} correct=${v.correct} abstain=${v.abstain} acc=${(v.accuracy * 100).toFixed(1)}%`);
}
console.log(" by shared_vocab tag:");
for (const [tag, v] of Object.entries(r.bySharedVocab)) {
console.log(
` shared_vocab=${tag.padEnd(5)} n=${v.n} correct=${v.correct} abstain=${v.abstain} acc=${(v.accuracy * 100).toFixed(1)}%`,
);
}
}
async function main() {
const cases = loadCases();
const engine = await loadEngine();
const scored = await scoreAll(engine, cases);
const argThreshold = process.argv.includes("--threshold")
? Number(process.argv[process.argv.indexOf("--threshold") + 1])
: null;
if (argThreshold !== null) {
printReport(summarize(scored, argThreshold), "single run");
return;
}
// Baseline: no --threshold at all (CLI default when the flag is omitted).
printReport(summarize(scored, null), "baseline (no --threshold, CLI default)");
// Sweep a range of thresholds to find the best real default.
console.log("\n=== threshold sweep (accuracy over ALL cases, abstains count as wrong) ===");
const sweep = [];
for (let t = 0; t <= 0.9; t += 0.05) {
const r = summarize(scored, Math.round(t * 100) / 100);
sweep.push(r);
console.log(
` threshold=${r.threshold.toFixed(2)} acc_overall=${(r.accuracyOverall * 100).toFixed(1)}% acc_decided=${(r.accuracyOnDecided * 100).toFixed(1)}% abstain=${r.abstain}/${r.n} wrong_confident=${r.wrongConfident}`,
);
}
const best = sweep.reduce((a, b) => (b.accuracyOverall > a.accuracyOverall ? b : a));
console.log(`\nbest-by-accuracy-overall: threshold=${best.threshold} acc_overall=${(best.accuracyOverall * 100).toFixed(1)}%`);
}
main();