umpirix/engine/gold.json
caelum0x a3803a1e5a feat(otsone): grow gold.json from 10 to 14 held-out cases
Adds 4 unambiguous new labeled cases (shipping-delay-clear,
double-charge-billing, newsletter-unsubscribe, bug-intermittent-workaround)
covering paraphrases of existing question types. Re-ran `otsone eval`
live against both checkpoints: otsone-mini (never trained on gold) scores
12/14 (0.8889 acc, ece 0.1253, up from 8/10/0.857/0.16) with all 4 new
cases passing; otsone-mini-v2 (gold folded into training on the old 10)
also scores 14/14 on the new cases, a genuine held-out signal since those
4 examples did not exist when v2 was trained. This is the real first step
toward a genuine train/holdout split identified as the actual blocker on
every eval number this pipeline produces, independent of GPU/EVREN access.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-11-09 18:33:44 +03:00

411 lines
11 KiB
JSON

[
{
"id": "shoes-returns",
"source": "TypeSafe Choice docs",
"state": "My running shoes arrived in the wrong size. Can I swap them for a size 10?",
"questions": {
"department": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {
"returns": "Exchanges, refunds, wrong or damaged items",
"shipping": "Delivery status, delays, lost packages",
"billing": "Charges, invoices, payment problems"
}
}
},
"expect": [
{
"choice": {
"id": "department",
"value": "returns",
"minP": 0.55
}
}
]
},
{
"id": "shoes-multi",
"source": "TypeSafe Choice docs",
"state": "Shoes arrived two weeks late and in the wrong size. Also I see two charges on my card. What are you going to do about this?",
"questions": {
"department": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {
"returns": "Exchanges, refunds, wrong or damaged items",
"shipping": "Delivery status, delays, lost packages",
"billing": "Charges, invoices, payment problems"
}
},
"return_reason": {
"type": "choice",
"instructions": "If the customer wants to return something, why?",
"criteria": {
"wrong_size": "The item doesn't fit",
"wrong_item": "A different product was delivered",
"damaged": "The item arrived broken or faulty",
"changed_mind": "The item is fine, the customer no longer wants it",
"other": "A return reason that fits none of the above"
}
},
"tone": {
"type": "choice",
"instructions": "What is the customer's tone?",
"criteria": {
"calm": null,
"frustrated": null,
"angry": null
}
}
},
"expect": [
{
"choice": {
"id": "department",
"value": "returns"
}
},
{
"choice": {
"id": "return_reason",
"value": "wrong_size",
"minP": 0.5
}
},
{
"choice": {
"id": "tone",
"value": "frustrated"
}
}
]
},
{
"id": "return-status-structured",
"source": "TypeSafe Choice structured criteria",
"state": "I sent the shoes back a week ago. When do I get my money?",
"questions": {
"return_topic": {
"type": "choice",
"instructions": {
"question": "Which returns topic is the customer asking about?",
"focus": "Classify the information the customer wants."
},
"criteria": {
"return_policy": {
"what": "Whether and how an item can be returned",
"not_for": "Progress of a return already sent",
"examples": [
"Can I return shoes I've worn once?",
"How long do I have to return an order?"
]
},
"return_status": {
"what": "Progress of a return already sent",
"not_for": "Whether and how an item can be returned",
"examples": [
"Has my return arrived yet?",
"When will my refund be paid?"
]
}
}
}
},
"expect": [
{
"choice": {
"id": "return_topic",
"value": "return_status",
"minP": 0.55
}
}
]
},
{
"id": "bug-cosmetic",
"source": "TypeSafe Score docs",
"state": "The export button is misaligned by a few pixels. Everything still works.",
"questions": {
"bug_severity": {
"type": "score",
"instructions": "How severe is the reported issue?",
"criteria": [
"Cosmetic; no impact to functionality",
"Broken or degraded feature, but workaround exists",
"Blocking issue; no workaround exists"
]
}
},
"expect": [
{
"score": {
"id": "bug_severity",
"min": 0,
"max": 0.45
}
}
]
},
{
"id": "bug-workaround",
"source": "TypeSafe Score docs",
"state": "The PDF export button does nothing in Safari. There is a workaround via CSV export.",
"questions": {
"bug_severity": {
"type": "score",
"instructions": "How severe is the reported issue?",
"criteria": [
"Cosmetic; no impact to functionality",
"Broken or degraded feature, but workaround exists",
"Blocking issue; no workaround exists"
]
}
},
"expect": [
{
"score": {
"id": "bug_severity",
"min": 0.7,
"max": 1.4
}
}
]
},
{
"id": "bug-blocking",
"source": "TypeSafe Score docs",
"state": "Nobody can log in. Every attempt returns a 500 error.",
"questions": {
"bug_severity": {
"type": "score",
"instructions": "How severe is the reported issue?",
"criteria": [
"Cosmetic; no impact to functionality",
"Broken or degraded feature, but workaround exists",
"Blocking issue; no workaround exists"
]
}
},
"expect": [
{
"score": {
"id": "bug_severity",
"min": 1.35,
"max": 2
}
}
]
},
{
"id": "human-escalation",
"source": "TypeSafe Noul docs",
"state": "I have asked three times now. Can I please just talk to a real person?",
"questions": {
"is_human_escalation": {
"type": "noul",
"instructions": "Is the customer asking for a human agent?"
},
"is_repeat_contact": {
"type": "noul",
"instructions": "Has the customer contacted support about this before?",
"criteria": {
"true": "Mentions a prior attempt, ticket, or that they have asked before",
"false": "No sign of any previous contact"
}
}
},
"expect": [
{
"noul": {
"id": "is_human_escalation",
"min": 0.75
}
},
{
"noul": {
"id": "is_repeat_contact",
"min": 0.62
}
}
]
},
{
"id": "refund-requested",
"source": "TypeSafe System One docs",
"state": {
"ticket_message": "My flight was cancelled. Can I get a refund?",
"refund_policy": "Cancelled flights are eligible for a full refund."
},
"questions": {
"refund_requested": {
"type": "noul",
"instructions": "Does `ticket_message` request a refund?"
},
"request_type": {
"type": "choice",
"instructions": "What is the main request in `ticket_message`?",
"criteria": {
"refund": "The customer wants money returned.",
"rebooking": "The customer wants a replacement flight.",
"information": "The customer is asking for information only."
}
}
},
"expect": [
{
"noul": {
"id": "refund_requested",
"min": 0.7
}
},
{
"choice": {
"id": "request_type",
"value": "refund",
"minP": 0.5
}
}
]
},
{
"id": "not-refund",
"source": "OpenJev calibration",
"state": "Thanks, just confirming the order shipped. No action needed.",
"questions": {
"refund_requested": {
"type": "noul",
"instructions": "Does this message request a refund?"
}
},
"expect": [
{
"noul": {
"id": "refund_requested",
"max": 0.4
}
}
]
},
{
"id": "spam",
"source": "OpenJev calibration",
"state": "Congratulations! You have been selected as a winner of a $2,500 crypto prize. Click here to claim within 24 hours.",
"questions": {
"is_spam": {
"type": "noul",
"instructions": "Is this unsolicited spam or a scam?",
"criteria": {
"true": "Prize, crypto, phishing, unsubscribe bait, fake urgency",
"false": "A legitimate customer or coworker message"
}
}
},
"expect": [
{
"noul": {
"id": "is_spam",
"min": 0.75
}
}
]
},
{
"id": "shipping-delay-clear",
"source": "OpenJev calibration 2026-11-09",
"state": "My package was supposed to arrive 5 days ago according to the tracking page, but it still just says 'in transit'. Where is it?",
"questions": {
"department": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {
"returns": "Exchanges, refunds, wrong or damaged items",
"shipping": "Delivery status, delays, lost packages",
"billing": "Charges, invoices, payment problems"
}
}
},
"expect": [
{
"choice": {
"id": "department",
"value": "shipping",
"minP": 0.55
}
}
]
},
{
"id": "double-charge-billing",
"source": "OpenJev calibration 2026-11-09",
"state": "I was charged twice for the same order. $45.99 shows up two times on my card statement from yesterday, same order number both times.",
"questions": {
"department": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {
"returns": "Exchanges, refunds, wrong or damaged items",
"shipping": "Delivery status, delays, lost packages",
"billing": "Charges, invoices, payment problems"
}
}
},
"expect": [
{
"choice": {
"id": "department",
"value": "billing",
"minP": 0.55
}
}
]
},
{
"id": "newsletter-unsubscribe",
"source": "OpenJev calibration 2026-11-09",
"state": "Hi, please take me off your mailing list. I don't want to receive the weekly product newsletter anymore, thanks.",
"questions": {
"is_spam": {
"type": "noul",
"instructions": "Is this unsolicited spam or a scam?",
"criteria": {
"true": "Prize, crypto, phishing, unsubscribe bait, fake urgency",
"false": "A legitimate customer or coworker message"
}
}
},
"expect": [
{
"noul": {
"id": "is_spam",
"max": 0.35
}
}
]
},
{
"id": "bug-intermittent-workaround",
"source": "OpenJev calibration 2026-11-09",
"state": "The search bar returns zero results for any query containing an apostrophe or hyphen, but a plain alphanumeric search still works fine.",
"questions": {
"bug_severity": {
"type": "score",
"instructions": "How severe is the reported issue?",
"criteria": [
"Cosmetic; no impact to functionality",
"Broken or degraded feature, but workaround exists",
"Blocking issue; no workaround exists"
]
}
},
"expect": [
{
"score": {
"id": "bug_severity",
"min": 0.7,
"max": 1.4
}
}
]
}
]