umpirix/sdk/python/tests/test_responses.py
caelum0x 4847e35451 OpenJev: open-source typed decision service
Typed decision/classification service with an in-process gte-small ONNX
embedding engine and lexical fallback, JS/Python SDKs, adapters, and the
otsone model family. Apache-2.0 (sub-packages keep their MIT licenses).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 00:12:36 +03:00

379 lines
12 KiB
Python

import copy
import pickle
from typing import Any, cast
import httpx2
import pytest
from pydantic import ValidationError
from pydantic_core import from_json
from typing_extensions import assert_type
from openjev import (
Answer,
AsyncTypeSafeClient,
ChoiceAnswer,
ListModelsResponse,
ModelMetadata,
NoulAnswer,
ScoreAnswer,
SystemOneResponse,
TypeSafeAPIResponseValidationError,
TypeSafeError,
Usage,
)
from tests.conftest import ClientFactory
from tests.helpers import system_one
from tests.test_clients import RESULT
@pytest.mark.parametrize(
"answers,field_path",
[
({}, "model"),
({"n": {"type": "noul"}}, "answers.n.noul"),
({"n": {"type": "noul", "noul": "0.5"}}, "answers.n.noul"),
(
{"c": {"type": "choice", "choice": "a", "probabilities": {}}},
"answers.c.confidence",
),
(
{"c": {"type": "choice", "confidence": 0.5, "probabilities": {}}},
"answers.c.choice",
),
(
{
"s": {
"type": "score",
"score": 1.0,
"confidence": 1.0,
"legend": [],
"probabilities": {},
}
},
"answers.s.legend",
),
(
{
"s": {
"type": "score",
"score": 1.0,
"confidence": 1.0,
"legend": {"x": "bad"},
"probabilities": {},
}
},
"answers.s.legend.x",
),
({"c": "not-a-mapping"}, "answers.c.type"),
],
)
async def test_malformed_response_raises_validation_error(
clients: ClientFactory, answers: dict[str, Any], field_path: str
) -> None:
body: dict[str, Any] = {
"usage": {"input_tokens": 1, "output_tokens": 1},
"answers": answers,
}
if field_path != "model":
body["model"] = "test"
def handler(request: httpx2.Request) -> httpx2.Response:
return httpx2.Response(
200, json=body, headers={"x-typesafe-request-id": "req-123"}
)
with pytest.raises(TypeSafeAPIResponseValidationError) as caught:
await system_one(
clients(handler),
state="x",
questions={"q": {"type": "noul", "instructions": "?"}},
)
assert caught.value.field_path == field_path
assert caught.value.status == 200
assert caught.value.request_id == "req-123"
assert caught.value.body == body
assert str(caught.value) == (
f"POST http://localhost:8080/v1/systemone: 200 Invalid response data at {field_path!r}. (request_id=req-123)"
)
@pytest.mark.parametrize("missing", ["name", "description", "release_date"])
def test_nested_missing_field_path(missing: str) -> None:
model = {"name": "test", "description": "Test model", "release_date": "2026-09-14"}
body = {
"models": [
model,
{name: value for name, value in model.items() if name != missing},
]
}
with pytest.raises(TypeSafeAPIResponseValidationError) as caught:
ListModelsResponse.from_http_response(httpx2.Response(200, json=body))
assert caught.value.field_path == f"models[1].{missing}"
assert str(caught.value) == f"200 Invalid response data at 'models[1].{missing}'."
async def test_response_carries_request_id(clients: ClientFactory) -> None:
result = await system_one(
clients(
lambda request: httpx2.Response(
200, json=RESULT, headers={"x-typesafe-request-id": "req-42"}
)
),
state="text",
questions={"q": {"type": "noul", "instructions": "?"}},
)
assert result.request_id == "req-42"
async def test_response_carries_raw_http_response(clients: ClientFactory) -> None:
result = await system_one(
clients(
lambda request: httpx2.Response(
200, json=RESULT, headers={"x-typesafe-request-id": "req-42"}
)
),
state="text",
questions={"q": {"type": "noul", "instructions": "?"}},
)
assert result.raw_http_response.status_code == 200
assert result.raw_http_response.headers["x-typesafe-request-id"] == "req-42"
assert result.raw_http_response.json() == RESULT
@pytest.mark.parametrize("resource", ["models", "system_one"])
async def test_response_serialization_excludes_http_metadata(
clients: ClientFactory, resource: str
) -> None:
body = (
{
"models": [
{
"name": "test",
"description": "Test model",
"release_date": "2026-09-14",
}
]
}
if resource == "models"
else RESULT
)
client = clients(
lambda request: httpx2.Response(
200, json=body, headers={"x-typesafe-request-id": "req-export"}
)
)
if resource == "models":
result = (
await client.models.list()
if isinstance(client, AsyncTypeSafeClient)
else client.models.list()
)
else:
result = await system_one(
client, state="text", questions={"q": {"type": "noul", "instructions": "?"}}
)
# Populating derived views must not add them to the serialized API payload.
if isinstance(result, SystemOneResponse):
assert result.choices
assert result.scores
assert result.request_id == "req-export"
assert set(result.model_dump()) == set(body)
encoded = result.model_dump_json()
assert from_json(encoded) == body
restored = type(result).model_validate_json(encoded)
assert restored == result
assert result.raw_http_response.json() == body
def test_copied_response_preserves_metadata() -> None:
result = SystemOneResponse.from_http_response(
httpx2.Response(200, json=RESULT, headers={"x-typesafe-request-id": "req-copy"})
)
assert result.scores
unpickled = pickle.loads(pickle.dumps(result))
for restored in (copy.copy(result), copy.deepcopy(result), unpickled):
assert restored is not result
assert restored == result
assert restored.request_id == "req-copy"
assert restored.raw_http_response.json() == RESULT
assert restored.scores["quality"] is restored.answers["quality"]
def test_missing_raw_raises_on_access() -> None:
result = SystemOneResponse(model="test", usage=Usage(), answers={})
with pytest.raises(TypeSafeError, match="raw HTTP response"):
_ = result.raw_http_response
async def test_missing_request_id_raises_on_access(clients: ClientFactory) -> None:
result = await system_one(
clients(lambda request: httpx2.Response(200, json=RESULT)),
state="text",
questions={"q": {"type": "noul", "instructions": "?"}},
)
with pytest.raises(TypeSafeError, match="request ID"):
_ = result.request_id
async def test_unknown_extra_fields_tolerated(clients: ClientFactory) -> None:
body = {
"model": "test",
"usage": {
"input_tokens": 1,
"output_tokens": 1,
"reasoning_tokens": 9,
"billing_units": 1,
},
"answers": {"spam": {"type": "noul", "noul": 0.9, "explanation": "spammy"}},
}
result = await system_one(
clients(lambda request: httpx2.Response(200, json=body)),
state="x",
questions={"q": {"type": "noul", "instructions": "?"}},
)
assert result.nouls["spam"].noul == 0.9
assert not hasattr(result.usage, "billing_units")
assert result.usage.model_dump() == {"input_tokens": 1, "output_tokens": 1}
assert result.raw_http_response.json() == body
async def test_unknown_answer_type_ignored(clients: ClientFactory) -> None:
body = {
"model": "test",
"usage": {"input_tokens": 1, "output_tokens": 1},
"answers": {
"spam": {"type": "noul", "noul": 0.9},
"mystery": {"type": "aurora", "value": 3},
},
}
result = await system_one(
clients(
lambda request: httpx2.Response(
200, json=body, headers={"x-typesafe-request-id": "req-9"}
)
),
state="text",
questions={"q": {"type": "noul", "instructions": "?"}},
)
# A future answer type this SDK version does not model is skipped, not raised.
assert set(result.answers) == {"spam"}
assert result.nouls["spam"].noul == 0.9
# The unknown answer is still available in the raw response.
assert result.raw_http_response.json()["answers"]["mystery"]["type"] == "aurora"
def test_response_preserves_nested_json() -> None:
original = ScoreAnswer(
score=0.0,
confidence=1.0,
legend={0: {"examples": ["a", {"note": None}]}},
probabilities={0: 1.0},
)
result = SystemOneResponse(
model="test",
usage=Usage(input_tokens=1, output_tokens=1),
answers={"q": original},
)
assert not result.choices
answer = result.scores["q"]
assert answer is original
assert answer is result.answers["q"]
assert isinstance(result.usage, Usage)
assert answer.probabilities == {0: 1.0}
entry = answer.legend[0]
assert isinstance(entry, dict)
assert entry["examples"] == ["a", {"note": None}]
examples = entry["examples"]
assert isinstance(examples, list)
assert isinstance(examples[1], dict)
exported = answer.model_dump()
assert exported == {
"type": "score",
"score": 0.0,
"confidence": 1.0,
"legend": {0: {"examples": ["a", {"note": None}]}},
"probabilities": {0: 1.0},
}
# model_dump returns an independent deep copy: mutating it leaves the model untouched.
exported["legend"][0]["examples"].append("new")
exported["probabilities"][0] = 0.5
assert examples == ["a", {"note": None}]
assert answer.probabilities == {0: 1.0}
def test_answer_attributes_and_dictionary_types() -> None:
noul = NoulAnswer(noul=0.98)
choice = ChoiceAnswer(
choice="billing", confidence=0.9, probabilities={"billing": 0.9, "support": 0.1}
)
assert_type(noul.noul, float)
assert_type(choice.choice, str)
assert_type(choice.confidence, float)
assert_type(choice.probabilities, dict[str, float])
assert noul.model_dump() == {"type": "noul", "noul": 0.98}
assert choice.model_dump() == {
"type": "choice",
"choice": "billing",
"confidence": 0.9,
"probabilities": {"billing": 0.9, "support": 0.1},
}
@pytest.mark.parametrize(
"model_type,kwargs",
[
(NoulAnswer, {"noul": 0.5}),
(ChoiceAnswer, {"choice": "a", "confidence": 1.0, "probabilities": {"a": 1.0}}),
(
ScoreAnswer,
{
"score": 0.0,
"confidence": 1.0,
"legend": {0: "bad"},
"probabilities": {0: 1.0},
},
),
(Usage, {}),
(SystemOneResponse, {"model": "test", "usage": Usage()}),
(
ModelMetadata,
{"name": "test", "description": "Test model", "release_date": "2026-09-14"},
),
(ListModelsResponse, {"models": ()}),
],
)
def test_public_response_types_ignore_unknown_fields(
model_type: Any, kwargs: dict[str, Any]
) -> None:
result = model_type(**kwargs, unexpected=True)
assert not hasattr(result, "unexpected")
assert "unexpected" not in result.model_dump()
@pytest.mark.parametrize(
"answer",
[
NoulAnswer(noul=0.98),
ChoiceAnswer(choice="billing", confidence=1.0, probabilities={"billing": 1.0}),
ScoreAnswer(
score=0.0, confidence=1.0, legend={0: "bad"}, probabilities={0: 1.0}
),
],
)
def test_answer_fields_are_frozen(answer: Answer) -> None:
with pytest.raises(ValidationError):
cast(Any, answer).type = "other"
for name in type(answer).model_fields:
with pytest.raises(ValidationError):
setattr(answer, name, getattr(answer, name))
@pytest.mark.parametrize("group", ["nouls", "choices", "scores"])
def test_answer_groups_are_cached_and_not_serialized(group: str) -> None:
result = SystemOneResponse(model="test", usage=Usage(), answers={})
cached = getattr(result, group)
# The derived view is memoized (stable identity) and never leaks into the serialized payload.
assert getattr(result, group) is cached
assert group not in result.model_dump()