Typed decision/classification service with an in-process gte-small ONNX embedding engine and lexical fallback, JS/Python SDKs, adapters, and the otsone model family. Apache-2.0 (sub-packages keep their MIT licenses). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
171 lines
5.8 KiB
Python
171 lines
5.8 KiB
Python
"""Trust-layer fields (calibration, abstain / out-of-scope, auto model) and the shadow resource."""
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
import httpx2
|
|
import pytest
|
|
|
|
from openjev import AsyncTypeSafeClient, Choice, TypeSafeClient, TypeSafeError
|
|
from tests.conftest import ClientFactory
|
|
|
|
ANSWER: dict[str, Any] = {
|
|
"model": "auto",
|
|
"answers": {
|
|
"intent": {
|
|
"type": "choice",
|
|
"choice": "__none__",
|
|
"confidence": 0.2,
|
|
"probabilities": {"refund": 0.1, "status": 0.1, "__none__": 0.8},
|
|
"ood": 0.91,
|
|
"abstained": True,
|
|
"abstain_reason": "out_of_scope",
|
|
"calibrated": True,
|
|
}
|
|
},
|
|
"usage": {"input_tokens": 3, "output_tokens": 0},
|
|
"latency_ms": 4,
|
|
"calibration": {"id": "cal_abc123", "method": "isotonic", "questions": ["intent"]},
|
|
"escalation": {
|
|
"local_model": "otsone-1",
|
|
"escalated": [],
|
|
"kept": ["intent"],
|
|
"threshold": 0.6,
|
|
"provider": None,
|
|
"llm_model": None,
|
|
},
|
|
}
|
|
|
|
QUESTIONS = {
|
|
"intent": Choice(
|
|
instructions="Intent?",
|
|
criteria={"refund": None, "status": None},
|
|
abstain=True,
|
|
abstain_below=0.4,
|
|
)
|
|
}
|
|
|
|
|
|
async def _call(client: TypeSafeClient | AsyncTypeSafeClient, **kwargs: Any) -> Any:
|
|
result = client.system_one("weather on mars", QUESTIONS, **kwargs)
|
|
return await result if isinstance(client, AsyncTypeSafeClient) else result
|
|
|
|
|
|
async def test_trust_request_fields_and_answer_fields(clients: ClientFactory) -> None:
|
|
sent: list[dict[str, Any]] = []
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
sent.append(json.loads(request.content))
|
|
return httpx2.Response(200, json=ANSWER)
|
|
|
|
client = clients(handler)
|
|
result = await _call(
|
|
client, model="auto", calibration_id="cal_abc123", escalate_below=0.6
|
|
)
|
|
body = sent[0]
|
|
assert body["model"] == "auto"
|
|
assert body["calibration_id"] == "cal_abc123"
|
|
assert body["escalate_below"] == 0.6
|
|
assert "abstain" not in body
|
|
assert body["questions"]["intent"]["abstain"] is True
|
|
assert body["questions"]["intent"]["abstain_below"] == 0.4
|
|
answer = result.choices["intent"]
|
|
assert answer.abstained is True
|
|
assert answer.abstain_reason == "out_of_scope"
|
|
assert answer.ood == 0.91
|
|
assert result.calibration is not None and result.calibration.questions == (
|
|
"intent",
|
|
)
|
|
assert result.escalation is not None and result.escalation.kept == ("intent",)
|
|
|
|
|
|
async def test_trust_fields_are_validated_client_side(clients: ClientFactory) -> None:
|
|
client = clients(lambda _: httpx2.Response(200, json=ANSWER))
|
|
with pytest.raises(TypeSafeError):
|
|
await _call(client, calibration_id="not-a-calibration")
|
|
with pytest.raises(TypeSafeError):
|
|
await _call(client, abstain_below=1.5)
|
|
with pytest.raises(ValueError):
|
|
Choice.model_validate(
|
|
{"instructions": "x", "criteria": {"a": None}, "abstain_below": 2}
|
|
)
|
|
|
|
|
|
async def test_shadow_run_and_summary(clients: ClientFactory) -> None:
|
|
sent: list[httpx2.Request] = []
|
|
result: dict[str, Any] = {
|
|
"batch_id": "shb_1",
|
|
"incumbent": {"name": "humans", "source": "caller", "kind": "label"},
|
|
"calibration_id": "cal_abc123",
|
|
"ours": ANSWER,
|
|
"comparisons": [
|
|
{
|
|
"question_id": "intent",
|
|
"qtype": "choice",
|
|
"ours": "__none__",
|
|
"incumbent": "refund",
|
|
"agree": False,
|
|
"confidence": 0.2,
|
|
"ood": 0.91,
|
|
}
|
|
],
|
|
"agreement": 0,
|
|
}
|
|
summary: dict[str, Any] = {
|
|
"window_days": 7,
|
|
"threshold": 0.8,
|
|
"total": 1,
|
|
"agreed": 0,
|
|
"agreement": 0,
|
|
"incumbents": ["humans"],
|
|
"by_question": [],
|
|
"by_confidence": [],
|
|
"above_threshold": {"n": 0, "share": 0, "agreement": None},
|
|
"disagreements": [],
|
|
"savings": {
|
|
"incumbent_input_tokens": 0,
|
|
"incumbent_output_tokens": 0,
|
|
"incumbent_cost_usd": 0,
|
|
"projected_savings_usd": 0,
|
|
"price_in_usd_per_mtok": 0.4,
|
|
"price_out_usd_per_mtok": 1.6,
|
|
"price_source": "default",
|
|
},
|
|
}
|
|
|
|
def handler(request: httpx2.Request) -> httpx2.Response:
|
|
sent.append(request)
|
|
return httpx2.Response(
|
|
200, json={"data": result if request.method == "POST" else summary}
|
|
)
|
|
|
|
client = clients(handler)
|
|
incumbent = {"name": "humans", "kind": "label", "answers": {"intent": "refund"}}
|
|
if isinstance(client, AsyncTypeSafeClient):
|
|
run = await client.shadow.run(
|
|
"refund me", QUESTIONS, incumbent=incumbent, calibration_id="cal_abc123"
|
|
)
|
|
stats = await client.shadow.summary(days=7, threshold=0.8)
|
|
else:
|
|
run = client.shadow.run(
|
|
"refund me", QUESTIONS, incumbent=incumbent, calibration_id="cal_abc123"
|
|
)
|
|
stats = client.shadow.summary(days=7, threshold=0.8)
|
|
assert sent[0].url.path == "/v1/shadow"
|
|
assert json.loads(sent[0].content)["incumbent"]["kind"] == "label"
|
|
assert run.incumbent.kind == "label"
|
|
assert run.comparisons[0].agree is False
|
|
assert run.ours.answers["intent"].abstained is True
|
|
assert sent[1].method == "GET"
|
|
assert sent[1].url.query == b"days=7&threshold=0.8"
|
|
assert stats.total == 1 and stats.savings.price_source == "default"
|
|
|
|
|
|
async def test_shadow_rejects_unknown_incumbent_kind(clients: ClientFactory) -> None:
|
|
client = clients(lambda _: httpx2.Response(200, json={}))
|
|
with pytest.raises(TypeSafeError):
|
|
if isinstance(client, AsyncTypeSafeClient):
|
|
await client.shadow.run("x", QUESTIONS, incumbent={"kind": "oracle"})
|
|
else:
|
|
client.shadow.run("x", QUESTIONS, incumbent={"kind": "oracle"})
|