umpirix/scripts/fit-temp.ts
caelum0x 4847e35451 OpenJev: open-source typed decision service
Typed decision/classification service with an in-process gte-small ONNX
embedding engine and lexical fallback, JS/Python SDKs, adapters, and the
otsone model family. Apache-2.0 (sub-packages keep their MIT licenses).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 00:12:36 +03:00

86 lines
2.7 KiB
TypeScript

/**
* Offline temperature-schedule search (refit tool — NOT in the hot path).
*
* Grid-searches the `adaptiveTemp` schedule (in engine.ts) for the band temps
* that minimize the RAW (pre-calibration) Expected Calibration Error over the
* gold suite, so the reported confidences track accuracy as closely as possible
* before the isotonic map is even applied. Prints the best schedule + its ECE.
*
* Run:
* OTS_CALIBRATION=identity node --experimental-strip-types scripts/fit-temp.ts
*
* The engine reads the candidate schedule from OTS_TEMP_SCHEDULE at call time,
* so we can sweep many candidates in one deterministic process. Paste the
* winning TEMP_SCHEDULE into engine.ts and re-run scripts/fit-calibration.ts
* (the calibration map must be refit for the chosen schedule) then `npm run gold`.
*/
process.env.OTS_CALIBRATION = "identity";
const { evalGold } = await import("../src/lib/jev/gold.ts");
const { buildReliability } = await import("../src/lib/jev/calibration.ts");
const breakpoints: Array<[number, number, number]> = [
[0.12, 0.35, 0.8],
[0.1, 0.3, 0.7],
[0.15, 0.4, 0.9],
];
const T1 = [0.55, 0.62, 0.7, 0.8];
const T2 = [0.4, 0.46, 0.52];
const T3 = [0.32, 0.36, 0.42];
const T4 = [0.22, 0.26, 0.3];
const BASE = [0.35, 0.4, 0.45];
function ece(schedule: unknown): number {
process.env.OTS_TEMP_SCHEDULE = JSON.stringify(schedule);
const r = evalGold("system-one");
return buildReliability(r.predictions).ece;
}
// Baseline: the prior ad-hoc schedule.
const baseline = {
base: 0.4,
bands: [
[0.12, 0.7],
[0.35, 0.48],
[0.8, 0.36],
[Number.MAX_VALUE, 0.28],
],
};
const baselineEce = ece(baseline);
let best = { ece: Infinity, schedule: baseline as unknown };
for (const [b1, b2, b3] of breakpoints) {
for (const base of BASE)
for (const t1 of T1)
for (const t2 of T2)
for (const t3 of T3)
for (const t4 of T4) {
if (!(t1 > t2 && t2 > t3 && t3 > t4)) continue;
const schedule = {
base,
bands: [
[b1, t1],
[b2, t2],
[b3, t3],
[Number.MAX_VALUE, t4],
],
};
const e = ece(schedule);
if (e < best.ece) best = { ece: e, schedule };
}
}
const s = best.schedule as { base: number; bands: Array<[number, number]> };
console.log(`// baseline raw ECE = ${baselineEce.toFixed(4)}`);
console.log(`// best raw ECE = ${best.ece.toFixed(4)}`);
console.log("const TEMP_SCHEDULE: TempSchedule = {");
console.log(` base: ${s.base},`);
console.log(" bands: [");
for (const [sp, t] of s.bands) {
const spTxt = sp >= Number.MAX_VALUE ? "Infinity" : sp;
console.log(` [${spTxt}, ${t}],`);
}
console.log(" ],");
console.log("};");