/** * `openjev classify` — label text with no server, no key and no model download. * * Imports the engine straight from src/ (run with Node type stripping, like * `npm run gold`). The embedding engine answers when the gte-small model file * is installed (`npm run embed-download`); otherwise the lexical engine * answers. Every row says which engine answered. * * openjev classify bug,feature,praise < feedback.txt * openjev classify bug,feature,praise --file data.csv --column text * openjev classify bug,feature,praise --file data.jsonl --field text --out results.csv * * Exit codes: 0 ok, 1 input/output error (missing file, missing column, bad * JSON line), 2 usage error (bad labels, bad flag). */ import { readFileSync, writeFileSync } from "node:fs"; import { extname, resolve } from "node:path"; import { pathToFileURL } from "node:url"; export const CLASSIFY_HELP = `openjev classify — label text locally (no server, no key) Usage: openjev classify [options] < input.txt Comma-separated labels, at least two: bug,feature,praise Add a description after a colon to say when a label applies: "bug:crash error broken,feature:request add option,praise:love great thanks" Input (default: one text per line on stdin): --file Read a .txt (one text per line), .csv or .jsonl file --column CSV column holding the text (default: text) --field JSONL field holding the text (default: text) Output (default: a tab-separated table on stdout): --out Write results to a .csv or .jsonl file --json Print one JSON object per row --threshold <0-1> Below this probability the label is left blank and the row is marked for review A row where the engine cannot tell the labels apart (the top two labels are within 0.01 of each other) is always left blank and marked for review. Other: --question The question asked about each text (default: "Which label fits this text best?") Each row gets: label, probability, engine (embedding or lexical). `; const DEFAULT_QUESTION = "Which label fits this text best?"; /** Top two probabilities closer than this count as a tie: no label is given. */ const TIE_MARGIN = 0.01; class UsageError extends Error {} class InputError extends Error {} /** Parse argv into { labels, options }. Throws UsageError on anything malformed. */ export function parseArgs(argv) { const options = { json: false, question: DEFAULT_QUESTION }; const positional = []; const valueFlags = new Set(["--file", "--column", "--field", "--out", "--threshold", "--question"]); for (let i = 0; i < argv.length; i++) { const arg = argv[i]; if (arg === "--json") { options.json = true; } else if (valueFlags.has(arg)) { const value = argv[++i]; if (value === undefined) throw new UsageError(`${arg} needs a value.`); options[arg.slice(2)] = value; } else if (arg.startsWith("--")) { throw new UsageError(`Unknown option: ${arg}`); } else { positional.push(arg); } } if (positional.length !== 1) { throw new UsageError("Give the labels as one comma-separated argument, e.g. bug,feature,praise."); } // Each item is `name` or `name:description`; the description tells the engine when the label applies. const descriptions = {}; const labels = positional[0].split(",").map((item) => { const colon = item.indexOf(":"); const name = (colon === -1 ? item : item.slice(0, colon)).trim(); const description = colon === -1 ? "" : item.slice(colon + 1).trim(); if (name) descriptions[name] = description || null; return name; }); if (labels.some((label) => !label)) throw new UsageError("Labels must not be empty."); if (labels.length < 2) throw new UsageError("Give at least two labels."); if (new Set(labels).size !== labels.length) throw new UsageError("Labels must be unique."); if (labels.includes("__none__")) throw new UsageError("`__none__` is a reserved label."); let threshold = null; if (options.threshold !== undefined) { threshold = Number(options.threshold); if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { throw new UsageError("--threshold must be a number between 0 and 1."); } } if (options.out !== undefined && ![".csv", ".jsonl"].includes(extname(options.out).toLowerCase())) { throw new UsageError("--out must end in .csv or .jsonl."); } if (!options.question.trim()) throw new UsageError("--question must not be empty."); return { labels, descriptions, options: { ...options, threshold } }; } /** RFC 4180 CSV: quoted fields, doubled quotes, commas and newlines inside quotes. */ export function parseCsv(text) { const rows = []; let row = []; let field = ""; let quoted = false; const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text; for (let i = 0; i < src.length; i++) { const ch = src[i]; if (quoted) { if (ch === '"') { if (src[i + 1] === '"') { field += '"'; i++; } else { quoted = false; } } else { field += ch; } } else if (ch === '"') { quoted = true; } else if (ch === ",") { row.push(field); field = ""; } else if (ch === "\n" || ch === "\r") { if (ch === "\r" && src[i + 1] === "\n") i++; row.push(field); rows.push(row); row = []; field = ""; } else { field += ch; } } if (field !== "" || row.length > 0) { row.push(field); rows.push(row); } return rows.filter((cells) => cells.length > 1 || cells[0] !== ""); } function csvCell(value) { const text = value === null || value === undefined ? "" : String(value); return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; } function readFile(path) { try { return readFileSync(path, "utf8"); } catch (err) { if (err?.code === "ENOENT") throw new InputError(`File not found: ${path}`); throw new InputError(`Cannot read ${path}: ${err instanceof Error ? err.message : String(err)}`); } } function readStdin() { if (process.stdin.isTTY) { throw new UsageError("No input. Pipe text on stdin (one per line) or pass --file."); } return readFileSync(0, "utf8"); } function linesOf(text) { return text .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean); } /** * Load the input as records: `{ text, source }`, where `source` holds the * original columns/fields (kept in --out files) for CSV and JSONL input. */ export function loadRecords(options, cwd) { if (options.file === undefined) { return { kind: "lines", columns: [], records: linesOf(readStdin()).map((text) => ({ text, source: {} })) }; } const path = resolve(cwd, options.file); const ext = extname(path).toLowerCase(); const raw = readFile(path); if (ext === ".csv") { const rows = parseCsv(raw); if (rows.length === 0) throw new InputError(`${options.file} is empty.`); const [header, ...body] = rows; const column = options.column ?? "text"; const index = header.indexOf(column); if (index === -1) { throw new InputError( `Column "${column}" not found in ${options.file}. Columns: ${header.join(", ")}. Use --column.`, ); } const records = body.map((cells) => ({ text: (cells[index] ?? "").trim(), source: Object.fromEntries(header.map((name, i) => [name, cells[i] ?? ""])), })); return { kind: "csv", columns: header, records }; } if (ext === ".jsonl" || ext === ".ndjson") { const field = options.field ?? "text"; const records = linesOf(raw).map((line, i) => { let value; try { value = JSON.parse(line); } catch { throw new InputError(`${options.file} line ${i + 1} is not valid JSON.`); } if (!value || typeof value !== "object" || Array.isArray(value)) { throw new InputError(`${options.file} line ${i + 1} is not a JSON object.`); } if (!(field in value)) { throw new InputError(`Field "${field}" not found on line ${i + 1} of ${options.file}. Use --field.`); } const text = typeof value[field] === "string" ? value[field] : JSON.stringify(value[field]); return { text: text.trim(), source: value }; }); return { kind: "jsonl", columns: [], records }; } return { kind: "lines", columns: [], records: linesOf(raw).map((text) => ({ text, source: {} })) }; } /** Load the engine from src/. The embedding engine is used when its model is installed. */ export async function loadEngine() { const [{ runLocal }, { runEmbed }, { embedCapability }, { validateRequest }] = await Promise.all([ import("../src/lib/jev/engine.ts"), import("../src/lib/jev/embed/run.server.ts"), import("../src/lib/jev/embed/encoder.server.ts"), import("../src/lib/jev/validate.ts"), ]); const capability = await embedCapability(); return { runLocal, runEmbed, validateRequest, capability }; } /** Classify one text. Returns { label, probability, engine, probabilities }. */ export async function classifyOne(engine, text, labels, descriptions, question) { const request = engine.validateRequest({ model: "system-one", state: text, questions: { label: { type: "choice", instructions: question, criteria: Object.fromEntries(labels.map((label) => [label, descriptions[label] ?? null])), }, }, }); let response = null; if (engine.capability.available) { response = await engine.runEmbed(request, { capability: engine.capability }).catch(() => null); } const usedEmbedding = response !== null; const answer = (response ?? engine.runLocal(request, "system-one")).answers.label; const probability = answer.probabilities[answer.choice] ?? 0; // A (near) tie for first place means the engine found nothing to tell the labels apart. const ranked = Object.values(answer.probabilities).sort((a, b) => b - a); const tied = ranked.length > 1 && ranked[0] - ranked[1] < TIE_MARGIN; return { tied, label: answer.choice, probability: Math.round(probability * 10000) / 10000, engine: usedEmbedding ? "embedding" : "lexical", probabilities: answer.probabilities, }; } /** Blank the label and mark the row for review when it is a tie or under the threshold. */ export function applyThreshold(result, threshold) { const { tied, ...rest } = result; const review = tied || (threshold !== null && rest.probability < threshold); return { ...rest, label: review ? "" : rest.label, review }; } function formatTable(rows) { const lines = ["label\tprobability\tengine\treview\ttext"]; for (const row of rows) { const text = row.text.replace(/\s+/g, " "); lines.push([row.label, row.probability.toFixed(4), row.engine, row.review ? "yes" : "", text].join("\t")); } return `${lines.join("\n")}\n`; } function jsonRow(row) { return { text: row.text, label: row.label, probability: row.probability, engine: row.engine, review: row.review, probabilities: row.probabilities, }; } function formatOutFile(path, kind, columns, rows) { if (extname(path).toLowerCase() === ".jsonl") { return `${rows.map((row) => JSON.stringify({ ...row.source, ...jsonRow(row) })).join("\n")}\n`; } // CSV: keep the input columns for CSV input, otherwise a single text column. const added = ["label", "probability", "engine", "review"]; const base = kind === "csv" ? columns.filter((name) => !added.includes(name)) : ["text"]; const lines = [[...base, ...added].map(csvCell).join(",")]; for (const row of rows) { const cells = base.map((name) => (kind === "csv" ? row.source[name] : row.text)); lines.push([...cells, row.label, row.probability, row.engine, row.review ? "yes" : ""].map(csvCell).join(",")); } return `${lines.join("\n")}\n`; } /** Run the command. Returns the process exit code. */ export async function runClassify(argv, { cwd = process.cwd() } = {}) { if (argv.length === 0 || argv.includes("--help") || argv.includes("-h")) { process.stdout.write(CLASSIFY_HELP); return argv.length === 0 ? 2 : 0; } try { const { labels, descriptions, options } = parseArgs(argv); const { kind, columns, records } = loadRecords(options, cwd); if (records.length === 0) throw new InputError("No text to classify: the input is empty."); const engine = await loadEngine(); const rows = []; for (const record of records) { const result = record.text ? await classifyOne(engine, record.text, labels, descriptions, options.question) : { label: "", probability: 0, engine: "none", probabilities: {} }; const judged = record.text ? applyThreshold(result, options.threshold) : { ...result, review: true }; rows.push({ ...record, ...judged }); } if (options.out !== undefined) { const outPath = resolve(cwd, options.out); try { writeFileSync(outPath, formatOutFile(outPath, kind, columns, rows)); } catch (err) { throw new InputError(`Cannot write ${options.out}: ${err instanceof Error ? err.message : String(err)}`); } } if (options.json) { process.stdout.write(`${rows.map((row) => JSON.stringify(jsonRow(row))).join("\n")}\n`); } else if (options.out === undefined) { process.stdout.write(formatTable(rows)); } const engines = [...new Set(rows.map((row) => row.engine).filter((name) => name !== "none"))]; const review = rows.filter((row) => row.review).length; const note = engines.includes("lexical") && !engine.capability.available ? " (run `npm run embed-download` for the embedding engine)" : ""; process.stderr.write( `[openjev classify] ${rows.length} rows, engine: ${engines.join(", ") || "none"}${note}` + (review > 0 || options.threshold !== null ? `, ${review} marked for review` : "") + (options.out !== undefined ? `, wrote ${options.out}` : "") + "\n", ); return 0; } catch (err) { if (err instanceof UsageError) { process.stderr.write(`[openjev classify] ${err.message}\n\n${CLASSIFY_HELP}`); return 2; } if (err instanceof InputError) { process.stderr.write(`[openjev classify] ${err.message}\n`); return 1; } process.stderr.write(`[openjev classify] ${err instanceof Error ? err.message : String(err)}\n`); return 1; } } // Entry point when spawned by scripts/cli.mjs. if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { process.exit(await runClassify(process.argv.slice(2))); }