The classify CLI's lexical engine had zero lexicon coverage for "feature" and "praise" (only "bug" was covered) and none at all for generic sentiment (positive/negative/neutral), so bare-label runs of the CLI's own README example either tied on everything or confidently mislabeled a praise/feature row as a bug whenever a bug-shaped word like "crash" appeared. Add two small, additive synonym groups to the existing lexicon. Measured on a new 65-row synthetic gold set (examples/classify-eval.jsonl, run via `npm run classify-eval`, scripts/classify-eval.mjs): 32.3% -> 49.2% accuracy counting ties as wrong, 67.7% -> 71.1% on rows the engine actually committed to; the sentiment set alone went 5.0% -> 35.0%. The product-wide gold regression suite (`npm run gold`) still passes 61/61 — every change is a new synonym entry, not an edit to an existing one. README now documents the measured numbers, the --threshold tradeoff (0.55 raises decided-row accuracy to 83.8% at the cost of more abstains), and the real remaining limit: a bag-of-words engine can't fully resolve a praise sentence that mentions "crash" from an actual bug report. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
383 lines
14 KiB
JavaScript
383 lines
14 KiB
JavaScript
/**
|
|
* `openjev classify` — label text with no server, no key and no model download.
|
|
*
|
|
* Imports the engine straight from src/ (run with Node type stripping, like
|
|
* `npm run gold`). The embedding engine answers when the gte-small model file
|
|
* is installed (`npm run embed-download`); otherwise the lexical engine
|
|
* answers. Every row says which engine answered.
|
|
*
|
|
* openjev classify bug,feature,praise < feedback.txt
|
|
* openjev classify bug,feature,praise --file data.csv --column text
|
|
* openjev classify bug,feature,praise --file data.jsonl --field text --out results.csv
|
|
*
|
|
* Exit codes: 0 ok, 1 input/output error (missing file, missing column, bad
|
|
* JSON line), 2 usage error (bad labels, bad flag).
|
|
*/
|
|
import { readFileSync, writeFileSync } from "node:fs";
|
|
import { extname, resolve } from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
|
|
export const CLASSIFY_HELP = `openjev classify — label text locally (no server, no key)
|
|
|
|
Usage:
|
|
openjev classify <labels> [options] < input.txt
|
|
|
|
<labels> Comma-separated labels, at least two: bug,feature,praise
|
|
Add a description after a colon to say when a label applies:
|
|
"bug:crash error broken,feature:request add option,praise:love great thanks"
|
|
|
|
Input (default: one text per line on stdin):
|
|
--file <path> Read a .txt (one text per line), .csv or .jsonl file
|
|
--column <name> CSV column holding the text (default: text)
|
|
--field <name> JSONL field holding the text (default: text)
|
|
|
|
Output (default: a tab-separated table on stdout):
|
|
--out <path> Write results to a .csv or .jsonl file
|
|
--json Print one JSON object per row
|
|
--threshold <0-1> Below this probability the label is left blank and the
|
|
row is marked for review
|
|
|
|
A row where the engine cannot tell the labels apart (the top two labels are
|
|
within 0.01 of each other) is always left blank and marked for review.
|
|
|
|
Other:
|
|
--question <text> The question asked about each text
|
|
(default: "Which label fits this text best?")
|
|
|
|
Each row gets: label, probability, engine (embedding or lexical).
|
|
`;
|
|
|
|
const DEFAULT_QUESTION = "Which label fits this text best?";
|
|
/** Top two probabilities closer than this count as a tie: no label is given. */
|
|
const TIE_MARGIN = 0.01;
|
|
|
|
class UsageError extends Error {}
|
|
class InputError extends Error {}
|
|
|
|
/** Parse argv into { labels, options }. Throws UsageError on anything malformed. */
|
|
export function parseArgs(argv) {
|
|
const options = { json: false, question: DEFAULT_QUESTION };
|
|
const positional = [];
|
|
const valueFlags = new Set(["--file", "--column", "--field", "--out", "--threshold", "--question"]);
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const arg = argv[i];
|
|
if (arg === "--json") {
|
|
options.json = true;
|
|
} else if (valueFlags.has(arg)) {
|
|
const value = argv[++i];
|
|
if (value === undefined) throw new UsageError(`${arg} needs a value.`);
|
|
options[arg.slice(2)] = value;
|
|
} else if (arg.startsWith("--")) {
|
|
throw new UsageError(`Unknown option: ${arg}`);
|
|
} else {
|
|
positional.push(arg);
|
|
}
|
|
}
|
|
if (positional.length !== 1) {
|
|
throw new UsageError("Give the labels as one comma-separated argument, e.g. bug,feature,praise.");
|
|
}
|
|
// Each item is `name` or `name:description`; the description tells the engine when the label applies.
|
|
const descriptions = {};
|
|
const labels = positional[0].split(",").map((item) => {
|
|
const colon = item.indexOf(":");
|
|
const name = (colon === -1 ? item : item.slice(0, colon)).trim();
|
|
const description = colon === -1 ? "" : item.slice(colon + 1).trim();
|
|
if (name) descriptions[name] = description || null;
|
|
return name;
|
|
});
|
|
if (labels.some((label) => !label)) throw new UsageError("Labels must not be empty.");
|
|
if (labels.length < 2) throw new UsageError("Give at least two labels.");
|
|
if (new Set(labels).size !== labels.length) throw new UsageError("Labels must be unique.");
|
|
if (labels.includes("__none__")) throw new UsageError("`__none__` is a reserved label.");
|
|
|
|
let threshold = null;
|
|
if (options.threshold !== undefined) {
|
|
threshold = Number(options.threshold);
|
|
if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
|
|
throw new UsageError("--threshold must be a number between 0 and 1.");
|
|
}
|
|
}
|
|
if (options.out !== undefined && ![".csv", ".jsonl"].includes(extname(options.out).toLowerCase())) {
|
|
throw new UsageError("--out must end in .csv or .jsonl.");
|
|
}
|
|
if (!options.question.trim()) throw new UsageError("--question must not be empty.");
|
|
return { labels, descriptions, options: { ...options, threshold } };
|
|
}
|
|
|
|
/** RFC 4180 CSV: quoted fields, doubled quotes, commas and newlines inside quotes. */
|
|
export function parseCsv(text) {
|
|
const rows = [];
|
|
let row = [];
|
|
let field = "";
|
|
let quoted = false;
|
|
const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
for (let i = 0; i < src.length; i++) {
|
|
const ch = src[i];
|
|
if (quoted) {
|
|
if (ch === '"') {
|
|
if (src[i + 1] === '"') {
|
|
field += '"';
|
|
i++;
|
|
} else {
|
|
quoted = false;
|
|
}
|
|
} else {
|
|
field += ch;
|
|
}
|
|
} else if (ch === '"') {
|
|
quoted = true;
|
|
} else if (ch === ",") {
|
|
row.push(field);
|
|
field = "";
|
|
} else if (ch === "\n" || ch === "\r") {
|
|
if (ch === "\r" && src[i + 1] === "\n") i++;
|
|
row.push(field);
|
|
rows.push(row);
|
|
row = [];
|
|
field = "";
|
|
} else {
|
|
field += ch;
|
|
}
|
|
}
|
|
if (field !== "" || row.length > 0) {
|
|
row.push(field);
|
|
rows.push(row);
|
|
}
|
|
return rows.filter((cells) => cells.length > 1 || cells[0] !== "");
|
|
}
|
|
|
|
function csvCell(value) {
|
|
const text = value === null || value === undefined ? "" : String(value);
|
|
return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
|
|
}
|
|
|
|
function readFile(path) {
|
|
try {
|
|
return readFileSync(path, "utf8");
|
|
} catch (err) {
|
|
if (err?.code === "ENOENT") throw new InputError(`File not found: ${path}`);
|
|
throw new InputError(`Cannot read ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
}
|
|
}
|
|
|
|
function readStdin() {
|
|
if (process.stdin.isTTY) {
|
|
throw new UsageError("No input. Pipe text on stdin (one per line) or pass --file.");
|
|
}
|
|
return readFileSync(0, "utf8");
|
|
}
|
|
|
|
function linesOf(text) {
|
|
return text
|
|
.split(/\r?\n/)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
/**
|
|
* Load the input as records: `{ text, source }`, where `source` holds the
|
|
* original columns/fields (kept in --out files) for CSV and JSONL input.
|
|
*/
|
|
export function loadRecords(options, cwd) {
|
|
if (options.file === undefined) {
|
|
return { kind: "lines", columns: [], records: linesOf(readStdin()).map((text) => ({ text, source: {} })) };
|
|
}
|
|
const path = resolve(cwd, options.file);
|
|
const ext = extname(path).toLowerCase();
|
|
const raw = readFile(path);
|
|
|
|
if (ext === ".csv") {
|
|
const rows = parseCsv(raw);
|
|
if (rows.length === 0) throw new InputError(`${options.file} is empty.`);
|
|
const [header, ...body] = rows;
|
|
const column = options.column ?? "text";
|
|
const index = header.indexOf(column);
|
|
if (index === -1) {
|
|
throw new InputError(
|
|
`Column "${column}" not found in ${options.file}. Columns: ${header.join(", ")}. Use --column.`,
|
|
);
|
|
}
|
|
const records = body.map((cells) => ({
|
|
text: (cells[index] ?? "").trim(),
|
|
source: Object.fromEntries(header.map((name, i) => [name, cells[i] ?? ""])),
|
|
}));
|
|
return { kind: "csv", columns: header, records };
|
|
}
|
|
|
|
if (ext === ".jsonl" || ext === ".ndjson") {
|
|
const field = options.field ?? "text";
|
|
const records = linesOf(raw).map((line, i) => {
|
|
let value;
|
|
try {
|
|
value = JSON.parse(line);
|
|
} catch {
|
|
throw new InputError(`${options.file} line ${i + 1} is not valid JSON.`);
|
|
}
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
throw new InputError(`${options.file} line ${i + 1} is not a JSON object.`);
|
|
}
|
|
if (!(field in value)) {
|
|
throw new InputError(`Field "${field}" not found on line ${i + 1} of ${options.file}. Use --field.`);
|
|
}
|
|
const text = typeof value[field] === "string" ? value[field] : JSON.stringify(value[field]);
|
|
return { text: text.trim(), source: value };
|
|
});
|
|
return { kind: "jsonl", columns: [], records };
|
|
}
|
|
|
|
return { kind: "lines", columns: [], records: linesOf(raw).map((text) => ({ text, source: {} })) };
|
|
}
|
|
|
|
/** Load the engine from src/. The embedding engine is used when its model is installed. */
|
|
export async function loadEngine() {
|
|
const [{ runLocal }, { runEmbed }, { embedCapability }, { validateRequest }] = await Promise.all([
|
|
import("../src/lib/jev/engine.ts"),
|
|
import("../src/lib/jev/embed/run.server.ts"),
|
|
import("../src/lib/jev/embed/encoder.server.ts"),
|
|
import("../src/lib/jev/validate.ts"),
|
|
]);
|
|
const capability = await embedCapability();
|
|
return { runLocal, runEmbed, validateRequest, capability };
|
|
}
|
|
|
|
/** Classify one text. Returns { label, probability, engine, probabilities }. */
|
|
export async function classifyOne(engine, text, labels, descriptions, question) {
|
|
const request = engine.validateRequest({
|
|
model: "system-one",
|
|
state: text,
|
|
questions: {
|
|
label: {
|
|
type: "choice",
|
|
instructions: question,
|
|
criteria: Object.fromEntries(labels.map((label) => [label, descriptions[label] ?? null])),
|
|
},
|
|
},
|
|
});
|
|
let response = null;
|
|
if (engine.capability.available) {
|
|
response = await engine.runEmbed(request, { capability: engine.capability }).catch(() => null);
|
|
}
|
|
const usedEmbedding = response !== null;
|
|
const answer = (response ?? engine.runLocal(request, "system-one")).answers.label;
|
|
const probability = answer.probabilities[answer.choice] ?? 0;
|
|
// A (near) tie for first place means the engine found nothing to tell the labels apart.
|
|
const ranked = Object.values(answer.probabilities).sort((a, b) => b - a);
|
|
const tied = ranked.length > 1 && ranked[0] - ranked[1] < TIE_MARGIN;
|
|
return {
|
|
tied,
|
|
label: answer.choice,
|
|
probability: Math.round(probability * 10000) / 10000,
|
|
engine: usedEmbedding ? "embedding" : "lexical",
|
|
probabilities: answer.probabilities,
|
|
};
|
|
}
|
|
|
|
/** Blank the label and mark the row for review when it is a tie or under the threshold. */
|
|
export function applyThreshold(result, threshold) {
|
|
const { tied, ...rest } = result;
|
|
const review = tied || (threshold !== null && rest.probability < threshold);
|
|
return { ...rest, label: review ? "" : rest.label, review };
|
|
}
|
|
|
|
function formatTable(rows) {
|
|
const lines = ["label\tprobability\tengine\treview\ttext"];
|
|
for (const row of rows) {
|
|
const text = row.text.replace(/\s+/g, " ");
|
|
lines.push([row.label, row.probability.toFixed(4), row.engine, row.review ? "yes" : "", text].join("\t"));
|
|
}
|
|
return `${lines.join("\n")}\n`;
|
|
}
|
|
|
|
function jsonRow(row) {
|
|
return {
|
|
text: row.text,
|
|
label: row.label,
|
|
probability: row.probability,
|
|
engine: row.engine,
|
|
review: row.review,
|
|
probabilities: row.probabilities,
|
|
};
|
|
}
|
|
|
|
function formatOutFile(path, kind, columns, rows) {
|
|
if (extname(path).toLowerCase() === ".jsonl") {
|
|
return `${rows.map((row) => JSON.stringify({ ...row.source, ...jsonRow(row) })).join("\n")}\n`;
|
|
}
|
|
// CSV: keep the input columns for CSV input, otherwise a single text column.
|
|
const added = ["label", "probability", "engine", "review"];
|
|
const base = kind === "csv" ? columns.filter((name) => !added.includes(name)) : ["text"];
|
|
const lines = [[...base, ...added].map(csvCell).join(",")];
|
|
for (const row of rows) {
|
|
const cells = base.map((name) => (kind === "csv" ? row.source[name] : row.text));
|
|
lines.push([...cells, row.label, row.probability, row.engine, row.review ? "yes" : ""].map(csvCell).join(","));
|
|
}
|
|
return `${lines.join("\n")}\n`;
|
|
}
|
|
|
|
/** Run the command. Returns the process exit code. */
|
|
export async function runClassify(argv, { cwd = process.cwd() } = {}) {
|
|
if (argv.length === 0 || argv.includes("--help") || argv.includes("-h")) {
|
|
process.stdout.write(CLASSIFY_HELP);
|
|
return argv.length === 0 ? 2 : 0;
|
|
}
|
|
try {
|
|
const { labels, descriptions, options } = parseArgs(argv);
|
|
const { kind, columns, records } = loadRecords(options, cwd);
|
|
if (records.length === 0) throw new InputError("No text to classify: the input is empty.");
|
|
|
|
const engine = await loadEngine();
|
|
const rows = [];
|
|
for (const record of records) {
|
|
const result = record.text
|
|
? await classifyOne(engine, record.text, labels, descriptions, options.question)
|
|
: { label: "", probability: 0, engine: "none", probabilities: {} };
|
|
const judged = record.text ? applyThreshold(result, options.threshold) : { ...result, review: true };
|
|
rows.push({ ...record, ...judged });
|
|
}
|
|
|
|
if (options.out !== undefined) {
|
|
const outPath = resolve(cwd, options.out);
|
|
try {
|
|
writeFileSync(outPath, formatOutFile(outPath, kind, columns, rows));
|
|
} catch (err) {
|
|
throw new InputError(`Cannot write ${options.out}: ${err instanceof Error ? err.message : String(err)}`);
|
|
}
|
|
}
|
|
if (options.json) {
|
|
process.stdout.write(`${rows.map((row) => JSON.stringify(jsonRow(row))).join("\n")}\n`);
|
|
} else if (options.out === undefined) {
|
|
process.stdout.write(formatTable(rows));
|
|
}
|
|
|
|
const engines = [...new Set(rows.map((row) => row.engine).filter((name) => name !== "none"))];
|
|
const review = rows.filter((row) => row.review).length;
|
|
const note =
|
|
engines.includes("lexical") && !engine.capability.available
|
|
? " (run `npm run embed-download` for the embedding engine)"
|
|
: "";
|
|
process.stderr.write(
|
|
`[openjev classify] ${rows.length} rows, engine: ${engines.join(", ") || "none"}${note}` +
|
|
(review > 0 || options.threshold !== null ? `, ${review} marked for review` : "") +
|
|
(options.out !== undefined ? `, wrote ${options.out}` : "") +
|
|
"\n",
|
|
);
|
|
return 0;
|
|
} catch (err) {
|
|
if (err instanceof UsageError) {
|
|
process.stderr.write(`[openjev classify] ${err.message}\n\n${CLASSIFY_HELP}`);
|
|
return 2;
|
|
}
|
|
if (err instanceof InputError) {
|
|
process.stderr.write(`[openjev classify] ${err.message}\n`);
|
|
return 1;
|
|
}
|
|
process.stderr.write(`[openjev classify] ${err instanceof Error ? err.message : String(err)}\n`);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// Entry point when spawned by scripts/cli.mjs.
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
process.exit(await runClassify(process.argv.slice(2)));
|
|
}
|