umpirix/sdk/python/tests/test_clients.py
caelum0x 193da7cd3c fix(sdk/python): test_headers_timeout_and_logging asserted the wrong package name
importlib.metadata.version("openjev") raises PackageNotFoundError now that the
real distribution name is "openjev-client" (renamed in a0c6263). The test
imported the stdlib helper directly instead of going through the SDK's own
__version__ (openjev._version, which already has the openjev-client/openjev
fallback), so any fresh install installed under the real "openjev-client"
distribution name fails this test with zero relation to the behavior being
tested (the user-agent header format itself is unaffected -- it is still
"openjev/<version>", matching SDK_NAME).

Verified live: before the fix, `pytest tests/test_clients.py` failed 2/111
(sync + async) with PackageNotFoundError; after, 111/111 pass. Full suite:
681 passed, 54 skipped (unchanged skip count, no regressions). ruff and
pyrefly both clean on the touched file.
2026-11-10 00:20:19 +03:00

783 lines
27 KiB
Python

import asyncio
import logging
from typing import Any, cast
import httpx2
import pytest
from pydantic import ValidationError
from pydantic_core import from_json, to_json
from typing_extensions import assert_type
from openjev import (
AsyncTypeSafeClient,
Choice,
ChoiceAnswer,
JSONValue,
ModelMetadata,
Noul,
NoulAnswer,
QuestionModel,
Questions,
Score,
ScoreAnswer,
TypeSafeAPIConnectionError,
TypeSafeAPIError,
TypeSafeAPIResponseValidationError,
TypeSafeAPITimeoutError,
TypeSafeAuthenticationError,
TypeSafeBadRequestError,
TypeSafeClient,
TypeSafeError,
TypeSafeInternalServerError,
TypeSafeNotFoundError,
TypeSafePermissionDeniedError,
TypeSafeRateLimitError,
TypeSafeUnprocessableEntityError,
Usage,
__version__,
)
from tests.conftest import ClientFactory
from tests.helpers import TrackingTransport, models, system_one
RESULT = {
"model": "otsone-latest",
"usage": {"input_tokens": 12, "output_tokens": 3},
"answers": {
"spam": {"type": "noul", "noul": 0.98},
"tone": {
"type": "choice",
"choice": "friendly",
"confidence": 0.9,
"probabilities": {"friendly": 0.9, "hostile": 0.1},
},
"quality": {
"type": "score",
"score": 1.7,
"confidence": 0.8,
"legend": {"0": "bad", "1": "ok", "2": "great"},
"probabilities": {"0": 0.1, "1": 0.1, "2": 0.8},
},
},
}
CARD = {"name": "otsone-latest", "description": "Fast model", "release_date": "2026-08-01"}
@pytest.mark.parametrize("question_form", ["dataclass", "raw", "mixed"])
async def test_round_trip(clients: ClientFactory, question_form: str) -> None:
criteria: list[str | dict[str, JSONValue | None] | list[JSONValue | None]] = [
"bad",
"ok",
"great",
]
raw: dict[str, QuestionModel] = {
"spam": {"type": "noul", "instructions": "Spam?"},
"tone": {
"type": "choice",
"instructions": "Tone?",
"criteria": {"friendly": None, "hostile": None},
},
"quality": {"type": "score", "instructions": "Quality?", "criteria": criteria},
}
questions: Questions
if question_form == "raw":
questions = raw
else:
questions = {
"spam": Noul(instructions="Spam?")
if question_form == "dataclass"
else raw["spam"],
"tone": Choice(
instructions="Tone?", criteria={"friendly": None, "hostile": None}
),
"quality": Score(instructions="Quality?", criteria=criteria),
}
expected: dict[str, JSONValue | None] = {
"state": {"document": "Hello 🌍"},
"model": "otsone-latest",
"questions": {
"spam": {"type": "noul", "instructions": "Spam?"},
"tone": {
"type": "choice",
"instructions": "Tone?",
"criteria": {"friendly": None, "hostile": None},
},
"quality": {
"type": "score",
"instructions": "Quality?",
"criteria": ["bad", "ok", "great"],
},
},
}
def handler(request: httpx2.Request) -> httpx2.Response:
assert request.method == "POST"
assert str(request.url) == "http://localhost:8080/v1/systemone"
assert from_json(request.content) == expected
assert request.headers["content-type"] == "application/json"
return httpx2.Response(200, content=to_json(RESULT))
result = await system_one(
clients(handler),
state={"document": "Hello 🌍"},
questions=questions,
)
assert "nouls" not in result.__dict__
assert "choices" not in result.__dict__
assert "scores" not in result.__dict__
assert_type(result.nouls, dict[str, NoulAnswer])
assert_type(result.choices, dict[str, ChoiceAnswer])
assert_type(result.scores, dict[str, ScoreAnswer])
assert result.nouls is result.nouls
assert result.choices is result.choices
assert result.scores is result.scores
assert result.model == "otsone-latest"
assert result.usage == Usage(input_tokens=12, output_tokens=3)
assert set(result.nouls) == {"spam"}
assert set(result.choices) == {"tone"}
assert set(result.scores) == {"quality"}
assert result.nouls["spam"].noul == 0.98
assert result.choices["tone"].choice == "friendly"
assert result.choices["tone"].confidence == 0.9
assert result.choices["tone"].probabilities == {"friendly": 0.9, "hostile": 0.1}
assert result.scores["quality"].score == 1.7
assert result.scores["quality"].confidence == 0.8
assert result.scores["quality"].legend == {0: "bad", 1: "ok", 2: "great"}
assert result.scores["quality"].probabilities == {0: 0.1, 1: 0.1, 2: 0.8}
assert result.answers["quality"] is result.scores["quality"]
assert result.answers["spam"] is result.nouls["spam"]
assert result.answers["tone"] is result.choices["tone"]
assert set(result.answers) == {"spam", "tone", "quality"}
with pytest.raises(ValidationError):
cast(Any, result).model = "other"
async def test_extra_body_shallow_override(clients: ClientFactory) -> None:
expected = {
"state": "hi",
"model": "override-model",
"questions": {"q": {"type": "noul", "instructions": "?"}},
"beam_width": 4,
"nullable": None,
}
def handler(request: httpx2.Request) -> httpx2.Response:
assert from_json(request.content) == expected
return httpx2.Response(200, content=to_json(RESULT))
await system_one(
clients(handler),
state="hi",
questions={"q": {"type": "noul", "instructions": "?"}},
model="call-model",
extra_body={"model": "override-model", "beam_width": 4, "nullable": None},
)
async def test_unserializable_request_body_raises(clients: ClientFactory) -> None:
def handler(request: httpx2.Request) -> httpx2.Response:
pytest.fail("An unencodable request body reached the network")
with pytest.raises(TypeSafeError, match="could not be encoded as JSON"):
await system_one(
clients(handler),
state="x",
questions={"q": {"type": "noul", "instructions": "?"}},
extra_body=cast(Any, {"bad": object()}),
)
async def test_raw_question_passthrough(clients: ClientFactory) -> None:
# Explicit escape hatch for fields introduced by the API before this SDK models them.
questions = cast(
Questions,
{
"q": {
"type": "noul",
"instructions": "Spam?",
"weight": 3,
"nested": {"k": None},
},
"choice": {"type": "choice", "criteria": {"a": None}, "weight": 2},
"score": {"type": "score", "criteria": ["bad", "good"], "weight": 1},
},
)
expected = {"state": "hi", "model": "otsone-latest", "questions": questions}
def handler(request: httpx2.Request) -> httpx2.Response:
assert from_json(request.content) == expected
return httpx2.Response(200, content=to_json(RESULT))
await system_one(clients(handler), state="hi", questions=questions)
# Raw dictionary questions are passed through untouched: the SDK leaves their schema validation to
# the API. (Typed `Noul`/`Choice`/`Score` objects instead validate eagerly on construction.)
@pytest.mark.parametrize(
"question",
[
{"type": "noul", "instructions": 1},
{"type": "choice", "criteria": ["invalid", "shape"]},
],
)
async def test_question_schema_validation_is_left_to_api(
clients: ClientFactory, question: Any
) -> None:
def handler(request: httpx2.Request) -> httpx2.Response:
assert from_json(request.content)["questions"]["q"] == question
return httpx2.Response(422, json={"detail": "Invalid question"})
with pytest.raises(TypeSafeUnprocessableEntityError, match="Invalid question"):
await system_one(clients(handler), state="x", questions={"q": question})
async def test_rich_descriptions(clients: ClientFactory) -> None:
criteria = {"summary": "duplicated", "examples": ["charged twice"]}
criteria2 = {"summary": "not a duplicate", "examples": []}
expected = {
"state": "a ticket",
"model": "custom",
"questions": {
"duplicate": {
"type": "noul",
"instructions": {"question": "Duplicate?"},
"criteria": {"true": criteria},
},
"team": {
"type": "choice",
"instructions": "Team?",
"criteria": {"billing": criteria, "other": None},
},
"risk": {
"type": "score",
"instructions": "Risk?",
"criteria": [criteria, criteria2],
},
},
}
def handler(request: httpx2.Request) -> httpx2.Response:
assert from_json(request.content) == expected
return httpx2.Response(
200,
json={
"model": "custom",
"usage": {"input_tokens": 1, "output_tokens": 1},
"answers": {
"risk": {
"type": "score",
"score": 0,
"confidence": 1,
"legend": {"0": criteria, "1": criteria2},
"probabilities": {"0": 1, "1": 0},
},
},
},
)
result = await system_one(
clients(handler),
state="a ticket",
model="custom",
questions={
"duplicate": {
"type": "noul",
"instructions": {"question": "Duplicate?"},
"criteria": {
"true": {"summary": "duplicated", "examples": ["charged twice"]}
},
},
"team": Choice(
instructions="Team?",
criteria={
"billing": {"summary": "duplicated", "examples": ["charged twice"]},
"other": None,
},
),
"risk": Score(
instructions="Risk?",
criteria=[
{"summary": "duplicated", "examples": ["charged twice"]},
{"summary": "not a duplicate", "examples": []},
],
),
},
)
assert result.scores["risk"].legend[0] == criteria
async def test_models_shape(clients: ClientFactory) -> None:
def handler(request: httpx2.Request) -> httpx2.Response:
assert request.method == "GET"
assert request.url.path == "/v1/models"
assert request.content == b""
return httpx2.Response(200, content=to_json({"models": [CARD]}))
assert await models(clients(handler)) == (ModelMetadata(**CARD),)
async def test_models_ignore_unknown_fields(clients: ClientFactory) -> None:
card = {**CARD, "context_window": 128000, "pricing": None}
client = clients(
lambda request: httpx2.Response(200, content=to_json({"models": [card]}))
)
response = (
await client.models.list()
if isinstance(client, AsyncTypeSafeClient)
else client.models.list()
)
[model] = response.models
assert model.name == "otsone-latest"
# Unmodeled fields are dropped from the card but remain available via the raw response.
assert response.raw_http_response.json()["models"][0]["context_window"] == 128000
@pytest.mark.parametrize(
"body", [None, {}, {"models": "bad"}, {"models": [{"name": "x"}]}]
)
async def test_invalid_models_response(clients: ClientFactory, body: object) -> None:
with pytest.raises(TypeSafeAPIResponseValidationError):
await models(clients(lambda request: httpx2.Response(200, json=body)))
@pytest.mark.parametrize(
"questions,match",
[
({}, "At least one question"),
(
{"rating": Score(instructions="?", criteria=[])},
r'"rating" has 0 criteria; at least two scores are required',
),
],
)
async def test_validation_before_network(
clients: ClientFactory, questions: Any, match: str
) -> None:
def handler(request: httpx2.Request) -> httpx2.Response:
pytest.fail("Invalid questions reached the network")
with pytest.raises(TypeSafeError, match=match):
await system_one(clients(handler), state="x", questions=questions)
@pytest.mark.parametrize(
"status,error",
[
(400, TypeSafeBadRequestError),
(401, TypeSafeAuthenticationError),
(403, TypeSafePermissionDeniedError),
(404, TypeSafeNotFoundError),
(422, TypeSafeUnprocessableEntityError),
(429, TypeSafeRateLimitError),
(500, TypeSafeInternalServerError),
(503, TypeSafeInternalServerError),
(408, TypeSafeAPIError),
(409, TypeSafeAPIError),
(302, TypeSafeAPIError),
],
)
async def test_error_mapping(
clients: ClientFactory, status: int, error: type[TypeSafeAPIError]
) -> None:
body = {"detail": {"message": "Server explanation"}}
with pytest.raises(error) as caught:
await models(
clients(
lambda request: httpx2.Response(
status,
json=body,
headers={
"x-typesafe-request-id": "req_123",
"retry-after-ms": "125",
},
)
)
)
assert type(caught.value) is error
assert caught.value.status == status
assert caught.value.body == body
assert caught.value.request_id == "req_123"
assert (
str(caught.value)
== f"GET http://localhost:8080/v1/models: {status} Server explanation (request_id=req_123)"
)
assert caught.value.headers["retry-after-ms"] == "125"
if isinstance(caught.value, TypeSafeRateLimitError):
assert caught.value.retry_after_ms == 125
@pytest.mark.parametrize(
"body,message",
[
({"error": "error", "message": "message", "detail": "detail"}, "error"),
({"error": {"message": "nested error"}, "message": "message"}, "nested error"),
({"message": "message", "detail": "detail"}, "message"),
({"detail": "detail"}, "detail"),
({"detail": {"message": "nested detail"}}, "nested detail"),
(
{
"detail": [
{
"loc": ["body", "questions", "q", "score", "criteria", 0],
"msg": "Invalid",
},
{"msg": "Missing"},
{},
]
},
"questions.q.score.criteria.0: Invalid; Missing",
),
("plain text", "plain text"),
({"unexpected": True}, '{"unexpected":true}'),
],
)
async def test_error_messages(
clients: ClientFactory, body: object, message: str
) -> None:
content = body.encode() if isinstance(body, str) else to_json(body)
with pytest.raises(TypeSafeAPIError, match="400") as caught:
await models(clients(lambda request: httpx2.Response(400, content=content)))
assert str(caught.value) == f"GET http://localhost:8080/v1/models: 400 {message}"
@pytest.mark.parametrize(
"transport_error,sdk_error",
[
(httpx2.LocalProtocolError, TypeSafeAPIConnectionError),
(httpx2.ConnectError, TypeSafeAPIConnectionError),
(httpx2.ReadError, TypeSafeAPIConnectionError),
(httpx2.RemoteProtocolError, TypeSafeAPIConnectionError),
(httpx2.ConnectTimeout, TypeSafeAPITimeoutError),
(httpx2.ReadTimeout, TypeSafeAPITimeoutError),
],
)
async def test_transport_errors(
clients: ClientFactory,
transport_error: type[httpx2.RequestError],
sdk_error: type[TypeSafeError],
) -> None:
def handler(request: httpx2.Request) -> httpx2.Response:
raise transport_error("failed", request=request)
with pytest.raises(sdk_error) as caught:
await models(clients(handler), timeout=1.25)
assert type(caught.value) is sdk_error
assert isinstance(caught.value.__cause__, transport_error)
assert str(caught.value.__cause__) == "failed"
assert caught.value.__context__ is None
if isinstance(caught.value, TypeSafeAPITimeoutError):
assert caught.value.timeout == 1.25
@pytest.mark.parametrize(
"timeout", [None, 2.0, httpx2.Timeout(3.0, read=9.0), httpx2.Timeout(None)]
)
async def test_system_one_timeout_override(
clients: ClientFactory, timeout: float | httpx2.Timeout | None
) -> None:
requests: list[httpx2.Request] = []
def handler(request: httpx2.Request) -> httpx2.Response:
requests.append(request)
return httpx2.Response(200, json=RESULT)
client = clients(handler, timeout=7.0)
await system_one(
client, state="hello", questions={"q": Noul(instructions="?")}, timeout=timeout
)
await system_one(client, state="hello", questions={"q": Noul(instructions="?")})
expected = httpx2.Timeout(7.0 if timeout is None else timeout).as_dict()
assert requests[0].extensions["timeout"] == expected
assert requests[1].extensions["timeout"] == httpx2.Timeout(7.0).as_dict()
async def test_headers_timeout_and_logging(
clients: ClientFactory, caplog: pytest.LogCaptureFixture
) -> None:
protected = {
"authorization": "injected-secret",
"accept": "text/plain",
"user-agent": "wrong",
"x-openjev-sdk": "wrong",
"x-openjev-runtime": "wrong",
"x-typesafe-sdk": "wrong",
"x-typesafe-runtime": "wrong",
}
def handler(request: httpx2.Request) -> httpx2.Response:
assert str(request.url) == "https://example.test/prefix/v1/systemone"
assert request.headers["authorization"] == "Bearer test-key"
assert request.headers["accept"] == "application/json"
assert (
request.headers["user-agent"]
== f"openjev/{__version__}"
)
# New branded headers (primary).
assert request.headers["x-openjev-sdk"] == request.headers["user-agent"]
assert request.headers["x-openjev-runtime"].startswith("python/")
# Back-compat headers also sent.
assert request.headers["x-typesafe-sdk"] == request.headers["user-agent"]
assert request.headers["x-typesafe-runtime"].startswith("python/")
assert "x-openjev-retry-count" not in request.headers
assert "x-typesafe-retry-count" not in request.headers
assert request.headers["x-team"] == "call"
assert request.headers["x-default"] == "kept"
assert request.headers["content-type"] == "application/json"
assert request.extensions["timeout"] == {
"connect": 2.0,
"read": 2.0,
"write": 2.0,
"pool": 2.0,
}
return httpx2.Response(
200,
json=RESULT,
headers={
"set-cookie": "response-secret",
"x-typesafe-request-id": "req_log",
},
)
with caplog.at_level(logging.DEBUG, logger="openjev"):
client = clients(
handler,
base_url="https://example.test/prefix///",
timeout=7,
headers={
**protected,
"X-Team": "default",
"X-Default": "kept",
"X-API-Key": "key-secret",
"cookie": "cookie-secret",
},
)
await system_one(
client,
state="hello",
questions={"q": {"type": "noul", "instructions": "?"}},
timeout=2.0,
extra_headers={
**protected,
"x-team": "call",
"x-openjev-retry-count": "99",
"x-typesafe-retry-count": "99",
"content-type": "wrong",
},
)
for secret in (
"test-key",
"injected-secret",
"key-secret",
"cookie-secret",
"response-secret",
):
assert secret not in caplog.text
assert "req_log" in caplog.text
assert "hello" in caplog.text
async def test_http_client_settings(clients: ClientFactory) -> None:
requests: list[httpx2.Request] = []
async def async_hook(request: httpx2.Request) -> None:
requests.append(request)
def handler(request: httpx2.Request) -> httpx2.Response:
assert str(request.url).startswith("http://localhost:8080/v1/")
assert request.headers["authorization"] == "Bearer test-key"
assert request.headers["accept"] == "application/json"
assert request.headers["user-agent"].startswith("openjev/")
# New branded headers (primary).
assert request.headers["x-openjev-sdk"] == request.headers["user-agent"]
assert request.headers["x-openjev-runtime"].startswith("python/")
# Back-compat headers also sent.
assert request.headers["x-typesafe-sdk"] == request.headers["user-agent"]
assert request.headers["x-typesafe-runtime"].startswith("python/")
assert request.headers["x-http-default"] == "kept"
assert request.headers["x-sdk-default"] == "sdk"
assert request.headers["x-call"] == "call"
if request.method == "POST":
assert request.headers["content-type"] == "application/json"
return httpx2.Response(
200, json=RESULT if request.method == "POST" else {"models": []}
)
headers = {
"authorization": "wrong",
"accept": "text/plain",
"content-type": "text/plain",
"user-agent": "wrong",
"x-openjev-sdk": "wrong",
"x-openjev-runtime": "wrong",
"x-typesafe-sdk": "wrong",
"x-typesafe-runtime": "wrong",
"x-http-default": "kept",
"x-sdk-default": "http",
"x-call": "http",
}
http_client = (
httpx2.AsyncClient(
transport=httpx2.MockTransport(handler),
event_hooks={"request": [async_hook]},
headers=headers,
auth=("wrong", "credentials"),
base_url="https://wrong.test",
)
if clients.async_mode
else httpx2.Client(
transport=httpx2.MockTransport(handler),
event_hooks={"request": [requests.append]},
headers=headers,
auth=("wrong", "credentials"),
base_url="https://wrong.test",
)
)
original_headers = httpx2.Headers(http_client.headers)
original_base_url = http_client.base_url
client = clients(
handler,
http_client=http_client,
headers={"x-sdk-default": "sdk", "x-call": "sdk"},
)
assert client._http_client is http_client
assert await models(client, extra_headers={"x-call": "call"}) == ()
await system_one(
client,
state="x",
questions={"q": Noul(instructions="?")},
extra_headers={"x-call": "call"},
)
assert [request.method for request in requests] == ["GET", "POST"]
assert http_client.headers == original_headers
assert http_client.base_url == original_base_url
@pytest.mark.parametrize("use_context", [False, True])
@pytest.mark.parametrize("supply_http_client", [False, True])
async def test_supplied_network_resources_closed(
clients: ClientFactory, use_context: bool, supply_http_client: bool
) -> None:
def handler(request: httpx2.Request) -> httpx2.Response:
assert str(request.url) == "http://localhost:8080/v1/models"
assert request.headers["authorization"] == "Bearer test-key"
return httpx2.Response(200, json={"models": []})
transport = TrackingTransport(handler)
if supply_http_client:
http_client = (
httpx2.AsyncClient(transport=transport)
if clients.async_mode
else httpx2.Client(transport=transport)
)
client = clients(handler, http_client=http_client)
assert client._http_client is http_client
else:
client = clients(handler, transport=transport)
assert await models(client) == ()
if isinstance(client, AsyncTypeSafeClient):
if use_context:
async with client:
assert (await client.models.list()).models == ()
else:
await client.aclose()
await client.aclose()
assert transport.aclose_calls == 1
assert transport.close_calls == 0
else:
if use_context:
with client:
assert client.models.list().models == ()
else:
client.close()
client.close()
assert transport.close_calls == 1
assert transport.aclose_calls == 0
assert client._http_client.is_closed
with pytest.raises(RuntimeError, match="closed"):
await models(client)
async def test_owned_http_client_closed() -> None:
with TypeSafeClient(api_key="test") as client:
assert not client._http_client.is_closed
assert client._http_client.timeout == httpx2.Timeout(10.0)
assert client._http_client.is_closed
async with AsyncTypeSafeClient(api_key="test") as async_client:
assert not async_client._http_client.is_closed
assert async_client._http_client.timeout == httpx2.Timeout(10.0)
assert async_client._http_client.is_closed
@pytest.mark.parametrize("kind", [ValueError, asyncio.CancelledError])
async def test_exceptional_context_closes_http_client(
clients: ClientFactory, kind: type[BaseException]
) -> None:
failure = kind("original failure")
def handler(request: httpx2.Request) -> httpx2.Response:
raise failure
transport = TrackingTransport(handler)
http_client = (
httpx2.AsyncClient(transport=transport)
if clients.async_mode
else httpx2.Client(transport=transport)
)
client = clients(handler, http_client=http_client)
assert client._http_client is http_client
with pytest.raises(kind) as caught:
if isinstance(client, AsyncTypeSafeClient):
async with client:
await client.models.list()
else:
with client:
client.models.list()
assert caught.value is failure
assert transport.aclose_calls == int(clients.async_mode)
assert transport.close_calls == int(not clients.async_mode)
assert client._http_client.is_closed
async def test_task_cancellation_closes_context() -> None:
started = asyncio.Event()
async def handler(request: httpx2.Request) -> httpx2.Response:
started.set()
await asyncio.Event().wait()
raise AssertionError("Cancelled request returned")
transport = TrackingTransport(handler)
client = AsyncTypeSafeClient(
api_key="test", http_client=httpx2.AsyncClient(transport=transport)
)
async def run() -> None:
async with client:
await client.system_one("x", {"q": Noul(instructions="?")})
task = asyncio.create_task(run())
try:
await asyncio.wait_for(started.wait(), timeout=1)
finally:
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert transport.aclose_calls == 1
assert client._http_client.is_closed
async def test_cancellation_propagates() -> None:
attempts = 0
def handler(request: httpx2.Request) -> httpx2.Response:
nonlocal attempts
attempts += 1
raise asyncio.CancelledError
async with AsyncTypeSafeClient(
api_key="test",
http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)),
) as client:
with pytest.raises(asyncio.CancelledError):
await client.models.list()
assert attempts == 1