umpirix/adapters/python/tests/test_provider_requests.py
caelum0x 4847e35451 OpenJev: open-source typed decision service
Typed decision/classification service with an in-process gte-small ONNX
embedding engine and lexical fallback, JS/Python SDKs, adapters, and the
otsone model family. Apache-2.0 (sub-packages keep their MIT licenses).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 00:12:36 +03:00

257 lines
10 KiB
Python

"""Provider request-building and result-parsing, without any network call.
These cover the OpenAI- and Anthropic-specific request envelopes and response parsing
that the cassette-backed live tests would otherwise be the only ones to exercise.
"""
import asyncio
from types import SimpleNamespace
from typing import Any
import pytest
from anthropic.types import Message as AnthropicMessage
from openjev import TypeSafeError
from openjev_adapters import (
AsyncSystemOneAdapterClient,
Noul,
RetryPolicy,
SystemOneAdapterClient,
SystemOneResponse,
)
from openjev_adapters.providers import (
Message,
build_async_provider,
build_sync_provider,
)
from openjev_adapters.providers.anthropic import (
AnthropicProvider,
AsyncAnthropicProvider,
_request_kwargs,
)
from openjev_adapters.providers.anthropic import _result as anthropic_result
from openjev_adapters.providers.gemini import AsyncGeminiProvider, GeminiProvider
from openjev_adapters.providers.gemini import _request_kwargs as gemini_request_kwargs
from openjev_adapters.providers.gemini import _result as gemini_result
from openjev_adapters.providers.openai import _response_format
from openjev_adapters.providers.openai import _result as openai_result
SCHEMA = {"type": "object", "properties": {"answers": {"type": "object"}}}
MESSAGES = [
Message(role="system", content="system prompt"),
Message(role="user", content="the document"),
]
def test_openai_native_response_format_wraps_schema() -> None:
assert _response_format(SCHEMA, structured=True) == {
"type": "json_schema",
"json_schema": {"name": "evaluation", "schema": SCHEMA, "strict": True},
}
def test_openai_prompted_sends_no_response_format() -> None:
assert _response_format(SCHEMA, structured=False) is None
def test_openai_result_reads_content_and_usage() -> None:
response = SimpleNamespace(
choices=[SimpleNamespace(message=SimpleNamespace(content='{"answers": {}}'), finish_reason="stop")],
usage=SimpleNamespace(prompt_tokens=12, completion_tokens=3),
)
result = openai_result(response)
assert result.text == '{"answers": {}}'
assert (result.input_tokens, result.output_tokens) == (12, 3)
def test_anthropic_request_puts_schema_in_output_config_when_structured() -> None:
kwargs = _request_kwargs("claude-haiku-4-5", MESSAGES, SCHEMA, structured=True, max_tokens=4096)
assert kwargs["system"] == "system prompt"
assert kwargs["messages"] == [{"role": "user", "content": "the document"}]
assert kwargs["max_tokens"] > 0
assert kwargs["output_config"] == {"format": {"type": "json_schema", "schema": SCHEMA}}
def test_anthropic_request_omits_output_config_when_prompted() -> None:
kwargs = _request_kwargs("claude-haiku-4-5", MESSAGES, SCHEMA, structured=False, max_tokens=4096)
assert "output_config" not in kwargs
def test_gemini_request_puts_schema_in_response_format_when_structured() -> None:
kwargs = gemini_request_kwargs("gemini-3.8-flash", MESSAGES, SCHEMA, structured=True)
assert kwargs["system_instruction"] == "system prompt"
assert kwargs["input"] == [{"type": "user_input", "content": [{"type": "text", "text": "the document"}]}]
assert kwargs["store"] is False
assert kwargs["response_format"] == {
"type": "text",
"mime_type": "application/json",
"schema": SCHEMA,
}
def test_gemini_request_omits_response_format_when_prompted() -> None:
kwargs = gemini_request_kwargs("gemini-3.8-flash", MESSAGES, SCHEMA, structured=False)
assert "response_format" not in kwargs
assert kwargs["store"] is False
def test_gemini_request_sends_correction_turns_as_steps() -> None:
messages = [
*MESSAGES,
Message(role="assistant", content='{"answers":'),
Message(role="user", content="fix it"),
]
kwargs = gemini_request_kwargs("gemini-3.8-flash", messages, SCHEMA, structured=True)
assert kwargs["input"] == [
{"type": "user_input", "content": [{"type": "text", "text": "the document"}]},
{"type": "model_output", "content": [{"type": "text", "text": '{"answers":'}]},
{"type": "user_input", "content": [{"type": "text", "text": "fix it"}]},
]
def test_gemini_result_reads_output_text_and_usage() -> None:
response = SimpleNamespace(
status="completed",
output_text='{"answers": {}}',
usage=SimpleNamespace(total_input_tokens=20, total_output_tokens=5),
errors=None,
)
result = gemini_result(response)
assert result.text == '{"answers": {}}'
assert (result.input_tokens, result.output_tokens) == (20, 5)
@pytest.mark.parametrize("status", ["incomplete", "failed", "cancelled", "budget_exceeded"])
def test_gemini_incomplete_status_is_not_treated_as_an_answer(status: str) -> None:
response = SimpleNamespace(
status=status,
output_text='{"answers":{"positive":true}}',
usage=SimpleNamespace(total_input_tokens=20, total_output_tokens=5),
errors=None,
)
with pytest.raises(TypeSafeError, match=r"Gemini response did not complete"):
gemini_result(response)
def test_gemini_omitted_usage_is_not_treated_as_an_answer() -> None:
response = SimpleNamespace(status="completed", output_text='{"answers":{}}', usage=None, errors=None)
with pytest.raises(TypeSafeError, match="omitted usage"):
gemini_result(response)
def test_anthropic_result_joins_text_blocks_and_reads_usage() -> None:
response = SimpleNamespace(
stop_reason="end_turn",
content=[
SimpleNamespace(type="text", text='{"answers":'),
SimpleNamespace(type="thinking", text="ignored"),
SimpleNamespace(type="text", text=" {}}"),
],
usage=SimpleNamespace(input_tokens=20, output_tokens=5),
)
result = anthropic_result(response)
assert result.text == '{"answers": {}}'
assert (result.input_tokens, result.output_tokens) == (20, 5)
@pytest.mark.parametrize("provider_class", [AnthropicProvider, AsyncAnthropicProvider])
@pytest.mark.parametrize("structured", [False, True])
@pytest.mark.parametrize("truncated", [False, True])
def test_anthropic_output_limit(
monkeypatch: pytest.MonkeyPatch,
provider_class: type[AnthropicProvider] | type[AsyncAnthropicProvider],
structured: bool,
truncated: bool,
) -> None:
provider = provider_class("claude-haiku-4-5", max_tokens=8192)
requests: list[dict[str, Any]] = []
def create(**kwargs: Any) -> AnthropicMessage:
requests.append(kwargs)
return AnthropicMessage(
id="msg-test",
type="message",
role="assistant",
model=provider.model_name,
stop_reason="max_tokens" if truncated else "end_turn",
# Even syntactically valid JSON must not hide a truncated generation.
content=[{"type": "text", "text": '{"answers":{"positive":true}}'}],
usage={"input_tokens": 20, "output_tokens": 8192 if truncated else 10},
)
async def create_async(**kwargs: Any) -> AnthropicMessage:
return create(**kwargs)
retry = RetryPolicy(max_retries=2, backoff_initial=0)
questions = {"positive": Noul(instructions="The review is positive.")}
def evaluate() -> SystemOneResponse:
if isinstance(provider, AsyncAnthropicProvider):
monkeypatch.setattr(provider._client.messages, "create", create_async)
async def run() -> SystemOneResponse:
async with provider._client:
client = AsyncSystemOneAdapterClient(
structured_outputs=structured,
llm_answer_mode="discrete",
n_retry_malformed_structure=2,
retry=retry,
model=provider,
)
return await client.system_one("A delightful book.", questions)
return asyncio.run(run())
monkeypatch.setattr(provider._client.messages, "create", create)
with provider._client:
client = SystemOneAdapterClient(
structured_outputs=structured,
llm_answer_mode="discrete",
n_retry_malformed_structure=2,
retry=retry,
model=provider,
)
return client.system_one("A delightful book.", questions)
if truncated:
with pytest.raises(TypeSafeError, match=r"truncated.*Increase max_tokens") as raised:
evaluate()
debug = raised.value.debug # pyrefly: ignore[missing-attribute]
else:
response = evaluate()
assert response.nouls["positive"].noul == 1.0
debug = response.debug
assert len(requests) == 1
assert requests[0]["max_tokens"] == 8192
assert len(debug["llm_attempts"]) == 1
attempt = debug["llm_attempts"][0]
assert attempt["request"] == requests[0]
assert attempt["llm_response"]["content"][0]["text"] == '{"answers":{"positive":true}}'
assert attempt["debug_info"]["finish_reason"] == ("max_tokens" if truncated else "end_turn")
assert ("error" in attempt["debug_info"]) == truncated
@pytest.mark.parametrize("provider_class", [AnthropicProvider, AsyncAnthropicProvider])
@pytest.mark.parametrize("max_tokens", [0, -1])
def test_anthropic_rejects_nonpositive_output_limit(
provider_class: type[AnthropicProvider] | type[AsyncAnthropicProvider],
max_tokens: int,
) -> None:
with pytest.raises(ValueError, match="max_tokens must be > 0"):
provider_class("claude-haiku-4-5", max_tokens=max_tokens)
def test_build_providers_select_gemini() -> None:
sync_provider = build_sync_provider("gemini", "gemini-3.8-flash")
async_provider = build_async_provider("gemini", "gemini-3.8-flash")
try:
assert isinstance(sync_provider, GeminiProvider)
assert isinstance(async_provider, AsyncGeminiProvider)
finally:
sync_provider.close()
asyncio.run(async_provider.aclose())
def test_unknown_provider_is_rejected() -> None:
with pytest.raises(ValueError, match="Unknown provider"):
build_sync_provider("nope", "model") # type: ignore[arg-type]