The classify CLI's lexical engine had zero lexicon coverage for "feature" and "praise" (only "bug" was covered) and none at all for generic sentiment (positive/negative/neutral), so bare-label runs of the CLI's own README example either tied on everything or confidently mislabeled a praise/feature row as a bug whenever a bug-shaped word like "crash" appeared. Add two small, additive synonym groups to the existing lexicon. Measured on a new 65-row synthetic gold set (examples/classify-eval.jsonl, run via `npm run classify-eval`, scripts/classify-eval.mjs): 32.3% -> 49.2% accuracy counting ties as wrong, 67.7% -> 71.1% on rows the engine actually committed to; the sentiment set alone went 5.0% -> 35.0%. The product-wide gold regression suite (`npm run gold`) still passes 61/61 — every change is a new synonym entry, not an edit to an existing one. README now documents the measured numbers, the --threshold tradeoff (0.55 raises decided-row accuracy to 83.8% at the cost of more abstains), and the real remaining limit: a bag-of-words engine can't fully resolve a praise sentence that mentions "crash" from an actual bug report. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
146 lines
5.2 KiB
JavaScript
146 lines
5.2 KiB
JavaScript
/**
|
|
* Accuracy harness for `openjev classify`'s default (lexical) engine.
|
|
*
|
|
* Loads examples/classify-eval.jsonl (synthetic, hand-labeled, built for this
|
|
* repo — see examples/README.md for provenance) and runs each text through
|
|
* the same `classifyOne` the CLI uses, grouped by its `labels` set. Reports
|
|
* accuracy overall, split by the `shared_vocab` tag (texts that deliberately
|
|
* reuse words from a *different* label's vocabulary than their true one —
|
|
* the review finding this harness exists to measure), and swept across a
|
|
* range of `--threshold` values so a real default can be picked from data
|
|
* instead of guessed.
|
|
*
|
|
* Usage: node --experimental-strip-types scripts/classify-eval.mjs [--threshold 0.0]
|
|
*/
|
|
import { readFileSync } from "node:fs";
|
|
import { fileURLToPath } from "node:url";
|
|
import { dirname, join } from "node:path";
|
|
import { loadEngine, classifyOne, applyThreshold } from "./classify.mjs";
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const DATA_PATH = join(__dirname, "..", "examples", "classify-eval.jsonl");
|
|
const QUESTION = "Which label fits this text best?";
|
|
|
|
function loadCases() {
|
|
return readFileSync(DATA_PATH, "utf8")
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line));
|
|
}
|
|
|
|
/** Run every case through the engine once; threshold sweeps reuse these raw results. */
|
|
async function scoreAll(engine, cases) {
|
|
const out = [];
|
|
for (const c of cases) {
|
|
const labels = c.labels.split(",");
|
|
const result = await classifyOne(engine, c.text, labels, {}, QUESTION);
|
|
out.push({ case: c, result });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function summarize(scored, threshold) {
|
|
const bySet = new Map();
|
|
const bySharedVocab = { true: { n: 0, correct: 0, abstain: 0 }, false: { n: 0, correct: 0, abstain: 0 } };
|
|
let n = 0;
|
|
let correct = 0;
|
|
let abstain = 0;
|
|
let wrongConfident = 0;
|
|
for (const { case: c, result } of scored) {
|
|
const judged = applyThreshold(result, threshold);
|
|
n += 1;
|
|
const tag = c.shared_vocab ? "true" : "false";
|
|
bySharedVocab[tag].n += 1;
|
|
const set = bySet.get(c.set) ?? { n: 0, correct: 0, abstain: 0 };
|
|
set.n += 1;
|
|
bySet.set(c.set, set);
|
|
if (!judged.label) {
|
|
abstain += 1;
|
|
bySharedVocab[tag].abstain += 1;
|
|
set.abstain += 1;
|
|
continue;
|
|
}
|
|
const isCorrect = judged.label === c.label;
|
|
if (isCorrect) {
|
|
correct += 1;
|
|
bySharedVocab[tag].correct += 1;
|
|
set.correct += 1;
|
|
} else {
|
|
wrongConfident += 1;
|
|
}
|
|
}
|
|
// decided = rows where the engine committed to a label (not abstained)
|
|
const decided = n - abstain;
|
|
return {
|
|
threshold,
|
|
n,
|
|
correct,
|
|
abstain,
|
|
wrongConfident,
|
|
accuracyOverall: correct / n,
|
|
accuracyOnDecided: decided > 0 ? correct / decided : 0,
|
|
bySet: Object.fromEntries(
|
|
[...bySet.entries()].map(([k, v]) => [k, { ...v, accuracy: v.correct / v.n }]),
|
|
),
|
|
bySharedVocab: Object.fromEntries(
|
|
Object.entries(bySharedVocab).map(([k, v]) => [
|
|
k,
|
|
{ ...v, accuracy: v.n ? v.correct / v.n : 0 },
|
|
]),
|
|
),
|
|
};
|
|
}
|
|
|
|
function printReport(r, label) {
|
|
console.log(`\n=== ${label} (threshold=${r.threshold ?? "none"}) ===`);
|
|
console.log(
|
|
`overall: ${r.n} cases, ${r.correct} correct, ${r.abstain} abstained, ${r.wrongConfident} wrong-and-confident`,
|
|
);
|
|
console.log(` accuracy (of all cases): ${(r.accuracyOverall * 100).toFixed(1)}%`);
|
|
console.log(` accuracy (of decided only): ${(r.accuracyOnDecided * 100).toFixed(1)}%`);
|
|
console.log(" by set:");
|
|
for (const [set, v] of Object.entries(r.bySet)) {
|
|
console.log(` ${set.padEnd(12)} n=${v.n} correct=${v.correct} abstain=${v.abstain} acc=${(v.accuracy * 100).toFixed(1)}%`);
|
|
}
|
|
console.log(" by shared_vocab tag:");
|
|
for (const [tag, v] of Object.entries(r.bySharedVocab)) {
|
|
console.log(
|
|
` shared_vocab=${tag.padEnd(5)} n=${v.n} correct=${v.correct} abstain=${v.abstain} acc=${(v.accuracy * 100).toFixed(1)}%`,
|
|
);
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const cases = loadCases();
|
|
const engine = await loadEngine();
|
|
const scored = await scoreAll(engine, cases);
|
|
|
|
const argThreshold = process.argv.includes("--threshold")
|
|
? Number(process.argv[process.argv.indexOf("--threshold") + 1])
|
|
: null;
|
|
|
|
if (argThreshold !== null) {
|
|
printReport(summarize(scored, argThreshold), "single run");
|
|
return;
|
|
}
|
|
|
|
// Baseline: no --threshold at all (CLI default when the flag is omitted).
|
|
printReport(summarize(scored, null), "baseline (no --threshold, CLI default)");
|
|
|
|
// Sweep a range of thresholds to find the best real default.
|
|
console.log("\n=== threshold sweep (accuracy over ALL cases, abstains count as wrong) ===");
|
|
const sweep = [];
|
|
for (let t = 0; t <= 0.9; t += 0.05) {
|
|
const r = summarize(scored, Math.round(t * 100) / 100);
|
|
sweep.push(r);
|
|
console.log(
|
|
` threshold=${r.threshold.toFixed(2)} acc_overall=${(r.accuracyOverall * 100).toFixed(1)}% acc_decided=${(r.accuracyOnDecided * 100).toFixed(1)}% abstain=${r.abstain}/${r.n} wrong_confident=${r.wrongConfident}`,
|
|
);
|
|
}
|
|
const best = sweep.reduce((a, b) => (b.accuracyOverall > a.accuracyOverall ? b : a));
|
|
console.log(`\nbest-by-accuracy-overall: threshold=${best.threshold} acc_overall=${(best.accuracyOverall * 100).toFixed(1)}%`);
|
|
}
|
|
|
|
main();
|