umpirix/scripts/classify.mjs
caelum0x 74e3b24f7a fix(classify): close lexicon gaps behind the lexical engine's shared-word mislabels
The classify CLI's lexical engine had zero lexicon coverage for "feature"
and "praise" (only "bug" was covered) and none at all for generic
sentiment (positive/negative/neutral), so bare-label runs of the CLI's
own README example either tied on everything or confidently mislabeled a
praise/feature row as a bug whenever a bug-shaped word like "crash"
appeared. Add two small, additive synonym groups to the existing lexicon.

Measured on a new 65-row synthetic gold set (examples/classify-eval.jsonl,
run via `npm run classify-eval`, scripts/classify-eval.mjs): 32.3% -> 49.2%
accuracy counting ties as wrong, 67.7% -> 71.1% on rows the engine
actually committed to; the sentiment set alone went 5.0% -> 35.0%. The
product-wide gold regression suite (`npm run gold`) still passes 61/61 —
every change is a new synonym entry, not an edit to an existing one.

README now documents the measured numbers, the --threshold tradeoff
(0.55 raises decided-row accuracy to 83.8% at the cost of more abstains),
and the real remaining limit: a bag-of-words engine can't fully resolve
a praise sentence that mentions "crash" from an actual bug report.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-11 14:26:12 +03:00

383 lines
14 KiB
JavaScript

/**
* `openjev classify` — label text with no server, no key and no model download.
*
* Imports the engine straight from src/ (run with Node type stripping, like
* `npm run gold`). The embedding engine answers when the gte-small model file
* is installed (`npm run embed-download`); otherwise the lexical engine
* answers. Every row says which engine answered.
*
* openjev classify bug,feature,praise < feedback.txt
* openjev classify bug,feature,praise --file data.csv --column text
* openjev classify bug,feature,praise --file data.jsonl --field text --out results.csv
*
* Exit codes: 0 ok, 1 input/output error (missing file, missing column, bad
* JSON line), 2 usage error (bad labels, bad flag).
*/
import { readFileSync, writeFileSync } from "node:fs";
import { extname, resolve } from "node:path";
import { pathToFileURL } from "node:url";
export const CLASSIFY_HELP = `openjev classify — label text locally (no server, no key)
Usage:
openjev classify <labels> [options] < input.txt
<labels> Comma-separated labels, at least two: bug,feature,praise
Add a description after a colon to say when a label applies:
"bug:crash error broken,feature:request add option,praise:love great thanks"
Input (default: one text per line on stdin):
--file <path> Read a .txt (one text per line), .csv or .jsonl file
--column <name> CSV column holding the text (default: text)
--field <name> JSONL field holding the text (default: text)
Output (default: a tab-separated table on stdout):
--out <path> Write results to a .csv or .jsonl file
--json Print one JSON object per row
--threshold <0-1> Below this probability the label is left blank and the
row is marked for review
A row where the engine cannot tell the labels apart (the top two labels are
within 0.01 of each other) is always left blank and marked for review.
Other:
--question <text> The question asked about each text
(default: "Which label fits this text best?")
Each row gets: label, probability, engine (embedding or lexical).
`;
const DEFAULT_QUESTION = "Which label fits this text best?";
/** Top two probabilities closer than this count as a tie: no label is given. */
const TIE_MARGIN = 0.01;
class UsageError extends Error {}
class InputError extends Error {}
/** Parse argv into { labels, options }. Throws UsageError on anything malformed. */
export function parseArgs(argv) {
const options = { json: false, question: DEFAULT_QUESTION };
const positional = [];
const valueFlags = new Set(["--file", "--column", "--field", "--out", "--threshold", "--question"]);
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (arg === "--json") {
options.json = true;
} else if (valueFlags.has(arg)) {
const value = argv[++i];
if (value === undefined) throw new UsageError(`${arg} needs a value.`);
options[arg.slice(2)] = value;
} else if (arg.startsWith("--")) {
throw new UsageError(`Unknown option: ${arg}`);
} else {
positional.push(arg);
}
}
if (positional.length !== 1) {
throw new UsageError("Give the labels as one comma-separated argument, e.g. bug,feature,praise.");
}
// Each item is `name` or `name:description`; the description tells the engine when the label applies.
const descriptions = {};
const labels = positional[0].split(",").map((item) => {
const colon = item.indexOf(":");
const name = (colon === -1 ? item : item.slice(0, colon)).trim();
const description = colon === -1 ? "" : item.slice(colon + 1).trim();
if (name) descriptions[name] = description || null;
return name;
});
if (labels.some((label) => !label)) throw new UsageError("Labels must not be empty.");
if (labels.length < 2) throw new UsageError("Give at least two labels.");
if (new Set(labels).size !== labels.length) throw new UsageError("Labels must be unique.");
if (labels.includes("__none__")) throw new UsageError("`__none__` is a reserved label.");
let threshold = null;
if (options.threshold !== undefined) {
threshold = Number(options.threshold);
if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
throw new UsageError("--threshold must be a number between 0 and 1.");
}
}
if (options.out !== undefined && ![".csv", ".jsonl"].includes(extname(options.out).toLowerCase())) {
throw new UsageError("--out must end in .csv or .jsonl.");
}
if (!options.question.trim()) throw new UsageError("--question must not be empty.");
return { labels, descriptions, options: { ...options, threshold } };
}
/** RFC 4180 CSV: quoted fields, doubled quotes, commas and newlines inside quotes. */
export function parseCsv(text) {
const rows = [];
let row = [];
let field = "";
let quoted = false;
const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
for (let i = 0; i < src.length; i++) {
const ch = src[i];
if (quoted) {
if (ch === '"') {
if (src[i + 1] === '"') {
field += '"';
i++;
} else {
quoted = false;
}
} else {
field += ch;
}
} else if (ch === '"') {
quoted = true;
} else if (ch === ",") {
row.push(field);
field = "";
} else if (ch === "\n" || ch === "\r") {
if (ch === "\r" && src[i + 1] === "\n") i++;
row.push(field);
rows.push(row);
row = [];
field = "";
} else {
field += ch;
}
}
if (field !== "" || row.length > 0) {
row.push(field);
rows.push(row);
}
return rows.filter((cells) => cells.length > 1 || cells[0] !== "");
}
function csvCell(value) {
const text = value === null || value === undefined ? "" : String(value);
return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
}
function readFile(path) {
try {
return readFileSync(path, "utf8");
} catch (err) {
if (err?.code === "ENOENT") throw new InputError(`File not found: ${path}`);
throw new InputError(`Cannot read ${path}: ${err instanceof Error ? err.message : String(err)}`);
}
}
function readStdin() {
if (process.stdin.isTTY) {
throw new UsageError("No input. Pipe text on stdin (one per line) or pass --file.");
}
return readFileSync(0, "utf8");
}
function linesOf(text) {
return text
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
}
/**
* Load the input as records: `{ text, source }`, where `source` holds the
* original columns/fields (kept in --out files) for CSV and JSONL input.
*/
export function loadRecords(options, cwd) {
if (options.file === undefined) {
return { kind: "lines", columns: [], records: linesOf(readStdin()).map((text) => ({ text, source: {} })) };
}
const path = resolve(cwd, options.file);
const ext = extname(path).toLowerCase();
const raw = readFile(path);
if (ext === ".csv") {
const rows = parseCsv(raw);
if (rows.length === 0) throw new InputError(`${options.file} is empty.`);
const [header, ...body] = rows;
const column = options.column ?? "text";
const index = header.indexOf(column);
if (index === -1) {
throw new InputError(
`Column "${column}" not found in ${options.file}. Columns: ${header.join(", ")}. Use --column.`,
);
}
const records = body.map((cells) => ({
text: (cells[index] ?? "").trim(),
source: Object.fromEntries(header.map((name, i) => [name, cells[i] ?? ""])),
}));
return { kind: "csv", columns: header, records };
}
if (ext === ".jsonl" || ext === ".ndjson") {
const field = options.field ?? "text";
const records = linesOf(raw).map((line, i) => {
let value;
try {
value = JSON.parse(line);
} catch {
throw new InputError(`${options.file} line ${i + 1} is not valid JSON.`);
}
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new InputError(`${options.file} line ${i + 1} is not a JSON object.`);
}
if (!(field in value)) {
throw new InputError(`Field "${field}" not found on line ${i + 1} of ${options.file}. Use --field.`);
}
const text = typeof value[field] === "string" ? value[field] : JSON.stringify(value[field]);
return { text: text.trim(), source: value };
});
return { kind: "jsonl", columns: [], records };
}
return { kind: "lines", columns: [], records: linesOf(raw).map((text) => ({ text, source: {} })) };
}
/** Load the engine from src/. The embedding engine is used when its model is installed. */
export async function loadEngine() {
const [{ runLocal }, { runEmbed }, { embedCapability }, { validateRequest }] = await Promise.all([
import("../src/lib/jev/engine.ts"),
import("../src/lib/jev/embed/run.server.ts"),
import("../src/lib/jev/embed/encoder.server.ts"),
import("../src/lib/jev/validate.ts"),
]);
const capability = await embedCapability();
return { runLocal, runEmbed, validateRequest, capability };
}
/** Classify one text. Returns { label, probability, engine, probabilities }. */
export async function classifyOne(engine, text, labels, descriptions, question) {
const request = engine.validateRequest({
model: "system-one",
state: text,
questions: {
label: {
type: "choice",
instructions: question,
criteria: Object.fromEntries(labels.map((label) => [label, descriptions[label] ?? null])),
},
},
});
let response = null;
if (engine.capability.available) {
response = await engine.runEmbed(request, { capability: engine.capability }).catch(() => null);
}
const usedEmbedding = response !== null;
const answer = (response ?? engine.runLocal(request, "system-one")).answers.label;
const probability = answer.probabilities[answer.choice] ?? 0;
// A (near) tie for first place means the engine found nothing to tell the labels apart.
const ranked = Object.values(answer.probabilities).sort((a, b) => b - a);
const tied = ranked.length > 1 && ranked[0] - ranked[1] < TIE_MARGIN;
return {
tied,
label: answer.choice,
probability: Math.round(probability * 10000) / 10000,
engine: usedEmbedding ? "embedding" : "lexical",
probabilities: answer.probabilities,
};
}
/** Blank the label and mark the row for review when it is a tie or under the threshold. */
export function applyThreshold(result, threshold) {
const { tied, ...rest } = result;
const review = tied || (threshold !== null && rest.probability < threshold);
return { ...rest, label: review ? "" : rest.label, review };
}
function formatTable(rows) {
const lines = ["label\tprobability\tengine\treview\ttext"];
for (const row of rows) {
const text = row.text.replace(/\s+/g, " ");
lines.push([row.label, row.probability.toFixed(4), row.engine, row.review ? "yes" : "", text].join("\t"));
}
return `${lines.join("\n")}\n`;
}
function jsonRow(row) {
return {
text: row.text,
label: row.label,
probability: row.probability,
engine: row.engine,
review: row.review,
probabilities: row.probabilities,
};
}
function formatOutFile(path, kind, columns, rows) {
if (extname(path).toLowerCase() === ".jsonl") {
return `${rows.map((row) => JSON.stringify({ ...row.source, ...jsonRow(row) })).join("\n")}\n`;
}
// CSV: keep the input columns for CSV input, otherwise a single text column.
const added = ["label", "probability", "engine", "review"];
const base = kind === "csv" ? columns.filter((name) => !added.includes(name)) : ["text"];
const lines = [[...base, ...added].map(csvCell).join(",")];
for (const row of rows) {
const cells = base.map((name) => (kind === "csv" ? row.source[name] : row.text));
lines.push([...cells, row.label, row.probability, row.engine, row.review ? "yes" : ""].map(csvCell).join(","));
}
return `${lines.join("\n")}\n`;
}
/** Run the command. Returns the process exit code. */
export async function runClassify(argv, { cwd = process.cwd() } = {}) {
if (argv.length === 0 || argv.includes("--help") || argv.includes("-h")) {
process.stdout.write(CLASSIFY_HELP);
return argv.length === 0 ? 2 : 0;
}
try {
const { labels, descriptions, options } = parseArgs(argv);
const { kind, columns, records } = loadRecords(options, cwd);
if (records.length === 0) throw new InputError("No text to classify: the input is empty.");
const engine = await loadEngine();
const rows = [];
for (const record of records) {
const result = record.text
? await classifyOne(engine, record.text, labels, descriptions, options.question)
: { label: "", probability: 0, engine: "none", probabilities: {} };
const judged = record.text ? applyThreshold(result, options.threshold) : { ...result, review: true };
rows.push({ ...record, ...judged });
}
if (options.out !== undefined) {
const outPath = resolve(cwd, options.out);
try {
writeFileSync(outPath, formatOutFile(outPath, kind, columns, rows));
} catch (err) {
throw new InputError(`Cannot write ${options.out}: ${err instanceof Error ? err.message : String(err)}`);
}
}
if (options.json) {
process.stdout.write(`${rows.map((row) => JSON.stringify(jsonRow(row))).join("\n")}\n`);
} else if (options.out === undefined) {
process.stdout.write(formatTable(rows));
}
const engines = [...new Set(rows.map((row) => row.engine).filter((name) => name !== "none"))];
const review = rows.filter((row) => row.review).length;
const note =
engines.includes("lexical") && !engine.capability.available
? " (run `npm run embed-download` for the embedding engine)"
: "";
process.stderr.write(
`[openjev classify] ${rows.length} rows, engine: ${engines.join(", ") || "none"}${note}` +
(review > 0 || options.threshold !== null ? `, ${review} marked for review` : "") +
(options.out !== undefined ? `, wrote ${options.out}` : "") +
"\n",
);
return 0;
} catch (err) {
if (err instanceof UsageError) {
process.stderr.write(`[openjev classify] ${err.message}\n\n${CLASSIFY_HELP}`);
return 2;
}
if (err instanceof InputError) {
process.stderr.write(`[openjev classify] ${err.message}\n`);
return 1;
}
process.stderr.write(`[openjev classify] ${err instanceof Error ? err.message : String(err)}\n`);
return 1;
}
}
// Entry point when spawned by scripts/cli.mjs.
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
process.exit(await runClassify(process.argv.slice(2)));
}