umpirix/migrations/0011_external_benchmarks.sql
caelum0x c70f3d047c feat: add production-integrated Benchmark Submissions subsystem
Geek Room hackathon ("a benchmarking platform for AI agents"): a real,
platform-integrated feature replacing the earlier, too-small otsone
eval-external CLI addition. A third party can now submit their own
typed-decision model (live HTTP endpoint or precomputed answers JSON)
through the real console UI or API, get scored against the same 93-case
gold suite by the same calibration engine every OpenJev model uses
(no second, divergent scoring implementation), and show up ranked
alongside OpenJev's own models on the real /leaderboard page.

- migrations/0011_external_benchmarks.sql: benchmark_submissions +
  benchmark_results tables.
- src/lib/jev/benchmark-scoring.ts: pure scoring logic (answers-payload
  parsing/validation, HTTP + answers-replay runners, per-question-type
  breakdown) reusing evalGoldWith/buildReliability directly.
- src/lib/jev/benchmark-submissions.server.ts: DB-backed wrapper
  (create/get/list/public-leaderboard-feed), same console auth +
  checkQuota pattern as every other /api/v1/* route.
- src/lib/jev/outbound-guard.server.ts: SSRF guard extracted from
  webhooks.server.ts (one implementation, not two) — validates +
  DNS-resolves submitted endpoint URLs before any outbound call.
- src/routes/api/v1/benchmark-submissions.ts (+ .leaderboard.ts): real
  API routes, auth + quota enforced.
- src/routes/submit-model.tsx: real submission form + history UI.
- src/routes/leaderboard.tsx + src/lib/jev/leaderboard.ts: external
  submissions render as "Community submitted" rows in the same gold-suite
  table, fetched client-side from the public leaderboard feed.
- Tests: benchmark-scoring.test.ts (9, incl. an end-to-end check that
  replaying system-one's own answers through this path reproduces
  evalGold() byte-for-byte) + outbound-guard.server.test.ts (6).

Verified end to end locally: npm run typecheck clean, npm test 138/138
(1 pre-existing skip), npm run gold unchanged (61/61), and a real
browser session (gstack browse) submitting a 93-case answers payload
through the actual /submit-model form — result card showed real
computed scores (accuracy 85.6%, ECE 0.096, matching evalGold() exactly)
and the submission appeared live on /leaderboard's gold-suite tab.
2026-11-07 04:22:53 +03:00

56 lines
2.6 KiB
SQL

-- External benchmark submissions (Geek Room hackathon capability): a third
-- party points their own typed-decision model at the shipped gold calibration
-- suite (src/lib/jev/gold.ts, 93 cases) and gets scored by the SAME engine
-- (buildReliability in calibration.ts) OpenJev's own models are scored by, so
-- the result is directly comparable on the real /leaderboard page.
--
-- One row per submission attempt; one row per (submission, qtype) in
-- benchmark_results, where qtype = 'overall' is the aggregate row used on the
-- leaderboard table and 'choice' | 'noul' | 'score' are the per-question-type
-- breakdown (mirrors otsone/laya_mlx/eval_external.py's by_type report).
--
-- Scoped to user_id (the console account whose API key submitted it), same
-- ownership pattern as eval_runs / webhooks / decisions.
create table if not exists benchmark_submissions (
id text primary key,
user_id text not null,
model_label text not null,
-- 'endpoint': we POST {state, questions} to endpoint_url per gold case and
-- read back {answers}. 'answers': the submitter precomputed every gold
-- case's answers offline and uploaded them as one JSON payload.
source_kind text not null check (source_kind in ('endpoint', 'answers')),
endpoint_url text,
answers_json jsonb,
status text not null default 'pending'
check (status in ('pending', 'running', 'completed', 'failed')),
error_message text,
case_count int not null default 0,
submitted_at timestamptz not null default now(),
completed_at timestamptz
);
create index if not exists benchmark_submissions_user_idx
on benchmark_submissions (user_id, submitted_at desc);
create index if not exists benchmark_submissions_status_idx
on benchmark_submissions (status, submitted_at desc);
create table if not exists benchmark_results (
id text primary key,
submission_id text not null references benchmark_submissions (id) on delete cascade,
qtype text not null check (qtype in ('overall', 'choice', 'noul', 'score')),
case_count int not null default 0,
accuracy double precision,
strict_pass_rate double precision,
avg_confidence double precision,
ece double precision,
adaptive_ece double precision,
mce double precision,
brier double precision,
nll double precision,
created_at timestamptz not null default now()
);
create unique index if not exists benchmark_results_submission_qtype_idx
on benchmark_results (submission_id, qtype);