umpirix/adapters/python/tests/test_gemini_transports.py
caelum0x 4847e35451 OpenJev: open-source typed decision service
Typed decision/classification service with an in-process gte-small ONNX
embedding engine and lexical fallback, JS/Python SDKs, adapters, and the
otsone model family. Apache-2.0 (sub-packages keep their MIT licenses).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 00:12:36 +03:00

215 lines
8.2 KiB
Python

"""Exercise the Gemini Interactions transport through the SDK's HTTP parsing."""
import asyncio
import json
from typing import Any
import httpx
import pytest
from openjev import RetryPolicy, TypeSafeAPIConnectionError, TypeSafeAPITimeoutError, TypeSafeError
from openjev_adapters import (
AsyncSystemOneAdapterClient,
Noul,
SystemOneAdapterClient,
SystemOneResponse,
)
from openjev_adapters.providers.gemini import AsyncGeminiProvider, GeminiProvider
def _interaction_payload(text: str, *, status: str = "completed") -> dict[str, Any]:
return {
"id": "interaction-test",
"status": status,
"model": "gemini-3.8-flash",
"steps": [
{
"type": "model_output",
"content": [{"type": "text", "text": text}],
}
],
"usage": {
"total_input_tokens": 12,
"total_output_tokens": 7,
"total_tokens": 19,
},
}
@pytest.mark.parametrize("provider_class", [GeminiProvider, AsyncGeminiProvider])
@pytest.mark.parametrize("structured", [False, True])
def test_gemini_transport_preserves_corrections_and_usage(
monkeypatch: pytest.MonkeyPatch,
provider_class: type[GeminiProvider] | type[AsyncGeminiProvider],
structured: bool,
) -> None:
requests: list[dict[str, Any]] = []
malformed = '{"answers":'
def respond(request: httpx.Request) -> httpx.Response:
requests.append(json.loads(request.content))
text = malformed if len(requests) == 1 else '{"answers":{"positive":true}}'
return httpx.Response(200, request=request, json=_interaction_payload(text))
def send(client: httpx.Client, request: httpx.Request, **kwargs: Any) -> httpx.Response:
return respond(request)
async def send_async(client: httpx.AsyncClient, request: httpx.Request, **kwargs: Any) -> httpx.Response:
return respond(request)
monkeypatch.setattr(httpx.Client, "send", send)
monkeypatch.setattr(httpx.AsyncClient, "send", send_async)
provider = provider_class("gemini-3.8-flash")
questions = {"positive": Noul(instructions="The review is positive.")}
if isinstance(provider, AsyncGeminiProvider):
async def run() -> SystemOneResponse:
try:
client = AsyncSystemOneAdapterClient(
structured_outputs=structured,
llm_answer_mode="discrete",
n_retry_malformed_structure=1,
model=provider,
)
return await client.system_one("A delightful book.", questions)
finally:
await provider.aclose()
response = asyncio.run(run())
else:
with provider._client:
client = SystemOneAdapterClient(
structured_outputs=structured,
llm_answer_mode="discrete",
n_retry_malformed_structure=1,
model=provider,
)
response = client.system_one("A delightful book.", questions)
assert response.nouls["positive"].noul == 1.0
assert (
response.usage.input_tokens,
response.usage.output_tokens,
response.usage.input_tokens_total,
response.usage.output_tokens_total,
response.usage.n_retries,
response.usage.n_retries_malformed_structure,
) == (12, 7, 24, 14, 0, 1)
attempts = response.debug["llm_attempts"]
assert len(attempts) == 2
assert [attempt["request"] for attempt in attempts] == requests
assert requests[0]["store"] is False
assert requests[0]["system_instruction"].startswith("Evaluate every question")
if structured:
assert requests[0]["response_format"]["schema"]["$defs"]["TypeSafeAnswers"]["properties"]["positive"]
else:
assert "response_format" not in requests[0]
assert requests[-1]["input"][-2] == {
"type": "model_output",
"content": [{"type": "text", "text": malformed}],
}
assert requests[-1]["input"][-1]["type"] == "user_input"
assert "previous response did not match" in requests[-1]["input"][-1]["content"][0]["text"]
@pytest.mark.parametrize("provider_class", [GeminiProvider, AsyncGeminiProvider])
def test_gemini_incomplete_http_response_is_not_an_answer(
monkeypatch: pytest.MonkeyPatch,
provider_class: type[GeminiProvider] | type[AsyncGeminiProvider],
) -> None:
def respond(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
request=request,
json=_interaction_payload('{"answers":{"positive":true}}', status="incomplete"),
)
def send(client: httpx.Client, request: httpx.Request, **kwargs: Any) -> httpx.Response:
return respond(request)
async def send_async(client: httpx.AsyncClient, request: httpx.Request, **kwargs: Any) -> httpx.Response:
return respond(request)
monkeypatch.setattr(httpx.Client, "send", send)
monkeypatch.setattr(httpx.AsyncClient, "send", send_async)
provider = provider_class("gemini-3.8-flash")
questions = {"positive": Noul(instructions="The review is positive.")}
def evaluate() -> None:
if isinstance(provider, AsyncGeminiProvider):
async def run() -> None:
try:
client = AsyncSystemOneAdapterClient(
structured_outputs=True,
llm_answer_mode="discrete",
model=provider,
)
await client.system_one("A delightful book.", questions)
finally:
await provider.aclose()
asyncio.run(run())
else:
with provider._client:
client = SystemOneAdapterClient(structured_outputs=True, llm_answer_mode="discrete", model=provider)
client.system_one("A delightful book.", questions)
with pytest.raises(TypeSafeError, match="did not complete"):
evaluate()
@pytest.mark.parametrize("provider_class", [GeminiProvider, AsyncGeminiProvider])
@pytest.mark.parametrize("retry_budget", [0, 1])
@pytest.mark.parametrize(
"transport_error,expected_error",
[(httpx.ConnectError, TypeSafeAPIConnectionError), (httpx.ReadTimeout, TypeSafeAPITimeoutError)],
)
def test_gemini_transport_errors_obey_retry_budget(
monkeypatch: pytest.MonkeyPatch,
provider_class: type[GeminiProvider] | type[AsyncGeminiProvider],
retry_budget: int,
transport_error: type[httpx.RequestError],
expected_error: type[TypeSafeError],
) -> None:
requests: list[httpx.Request] = []
def send(client: httpx.Client, request: httpx.Request, **kwargs: Any) -> httpx.Response:
requests.append(request)
raise transport_error("unavailable", request=request)
async def send_async(client: httpx.AsyncClient, request: httpx.Request, **kwargs: Any) -> httpx.Response:
requests.append(request)
raise transport_error("unavailable", request=request)
monkeypatch.setattr(httpx.Client, "send", send)
monkeypatch.setattr(httpx.AsyncClient, "send", send_async)
retry = RetryPolicy(max_retries=retry_budget, backoff_initial=0)
questions = {"positive": Noul(instructions="The review is positive.")}
provider = provider_class("test-model")
def evaluate() -> None:
if isinstance(provider, AsyncGeminiProvider):
async def run() -> None:
try:
async with AsyncSystemOneAdapterClient(
structured_outputs=False, llm_answer_mode="discrete", model=provider, retry=retry
) as client:
await client.system_one("A delightful book.", questions)
finally:
await provider.aclose()
asyncio.run(run())
else:
try:
with SystemOneAdapterClient(structured_outputs=False, llm_answer_mode="discrete", model=provider, retry=retry) as client:
client.system_one("A delightful book.", questions)
finally:
provider.close()
with pytest.raises(expected_error) as raised:
evaluate()
assert len(requests) == retry_budget + 1
assert len(raised.value.debug["llm_attempts"]) == len(requests) # pyrefly: ignore[missing-attribute]