umpirix/adapters/python/tests/test_provider_lifecycle.py
caelum0x 4847e35451 OpenJev: open-source typed decision service
Typed decision/classification service with an in-process gte-small ONNX
embedding engine and lexical fallback, JS/Python SDKs, adapters, and the
otsone model family. Apache-2.0 (sub-packages keep their MIT licenses).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 00:12:36 +03:00

491 lines
21 KiB
Python

"""Provider reuse and ownership, with real SDK pools and no model API calls."""
import asyncio
from collections.abc import AsyncIterator, Iterator
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FutureTimeoutError
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from threading import Barrier, Event
from typing import Any
from unittest.mock import DEFAULT, AsyncMock, Mock
import pytest
from openjev import TypeSafeError
from openjev_adapters import AsyncSystemOneAdapterClient, Noul, SystemOneAdapterClient
from openjev_adapters.providers import Message, ProviderResult
from openjev_adapters.providers.anthropic import (
AnthropicProvider,
AsyncAnthropicProvider,
)
from openjev_adapters.providers.base import record_request, render_messages
from openjev_adapters.providers.openai import AsyncOpenAIProvider, OpenAIProvider
from tests.test_client_with_fake_model import FakeAsyncProvider, FakeSyncProvider
QUESTIONS = {"positive": Noul(instructions="The review is positive.")}
RESULT = ProviderResult('{"answers":{"positive":true}}', 11, 7)
@dataclass
class Lifecycle:
asynchronous: bool
vendor: str
providers: list[Any] = field(default_factory=list)
def client(self, **kwargs: Any) -> Any:
options: dict[str, Any] = {
"structured_outputs": True,
"llm_answer_mode": "discrete",
"provider": self.vendor,
"model": "test-model",
}
options.update(kwargs)
client_class = AsyncSystemOneAdapterClient if self.asynchronous else SystemOneAdapterClient
return client_class(**options)
async def evaluate(self, client: Any, state: str = "Great book", **kwargs: Any) -> Any:
response = client.system_one(state, QUESTIONS, **kwargs)
return await response if self.asynchronous else response
async def close(self, client: Any) -> None:
if self.asynchronous:
await client.aclose()
else:
client.close()
@asynccontextmanager
async def context(self, client: Any) -> AsyncIterator[Any]:
if self.asynchronous:
async with client:
yield client
else:
with client:
yield client
@pytest.fixture(params=[False, True], ids=["sync", "async"])
def lifecycle(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, vendor: str) -> Iterator[Lifecycle]:
case = Lifecycle(request.param, vendor)
def watch(provider_class: Any) -> None:
original_init = provider_class.__init__
def initialize(provider: Any, *args: Any, **kwargs: Any) -> None:
original_init(provider, *args, **kwargs)
spy = AsyncMock if case.asynchronous else Mock
monkeypatch.setattr(provider._client, "close", spy(wraps=provider._client.close))
case.providers.append(provider)
def respond(provider: Any, messages: list[Message], **kwargs: Any) -> ProviderResult:
record_request({"messages": render_messages(messages)}, api="offline-test")
return RESULT
async def respond_async(provider: Any, messages: list[Message], **kwargs: Any) -> ProviderResult:
result = respond(provider, messages, **kwargs)
await asyncio.sleep(0)
return result
monkeypatch.setattr(provider_class, "__init__", initialize)
monkeypatch.setattr(provider_class, "request", respond_async if case.asynchronous else respond)
for provider_class in (AsyncOpenAIProvider, AsyncAnthropicProvider) if case.asynchronous else (OpenAIProvider, AnthropicProvider):
watch(provider_class)
yield case
# Retaining providers makes missed cleanup observable; release test pools even
# when an assertion fails or a close spy has deliberately been made to raise.
async def cleanup() -> None:
for provider in case.providers:
sdk = provider._client
if not sdk.is_closed():
result = type(sdk).close(sdk)
if case.asynchronous:
await result
asyncio.run(cleanup())
@pytest.fixture(params=["openai", "anthropic"])
def vendor(request: pytest.FixtureRequest) -> str:
return request.param
def test_reuses_owned_provider_and_closes_sdk_on_context_exit(lifecycle: Lifecycle) -> None:
async def run() -> None:
client = lifecycle.client()
assert lifecycle.providers == []
async with lifecycle.context(client):
for _ in range(3):
response = await lifecycle.evaluate(client)
assert response.nouls["positive"].noul == 1
assert len(lifecycle.providers) == 1
assert not lifecycle.providers[0]._client.is_closed()
sdk = lifecycle.providers[0]._client
assert sdk.is_closed()
await lifecycle.close(client)
sdk.close.assert_called_once()
if lifecycle.asynchronous:
sdk.close.assert_awaited_once()
with pytest.raises(RuntimeError, match="closed"):
await lifecycle.evaluate(client)
with pytest.raises(RuntimeError, match="closed"):
async with lifecycle.context(client):
pytest.fail("Reentered a closed client")
asyncio.run(run())
def test_cache_uses_resolved_provider_and_model_and_is_per_client(lifecycle: Lifecycle) -> None:
async def run() -> None:
async with lifecycle.context(lifecycle.client()) as client:
await lifecycle.evaluate(client)
await lifecycle.evaluate(client, model="test-model", provider=lifecycle.vendor)
await lifecycle.evaluate(client, model="another-model")
other_vendor = "anthropic" if lifecycle.vendor == "openai" else "openai"
await lifecycle.evaluate(client, provider=other_vendor)
assert len(lifecycle.providers) == 3
async with lifecycle.context(lifecycle.client()) as second:
await lifecycle.evaluate(second)
assert len(lifecycle.providers) == 4
assert all(not provider._client.is_closed() for provider in lifecycle.providers[:3])
assert all(provider._client.is_closed() for provider in lifecycle.providers)
asyncio.run(run())
@pytest.mark.parametrize("constructor_default", [False, True])
def test_injected_provider_is_borrowed(lifecycle: Lifecycle, constructor_default: bool) -> None:
async def run() -> None:
provider_class = (
(AsyncOpenAIProvider if lifecycle.asynchronous else OpenAIProvider)
if lifecycle.vendor == "openai"
else (AsyncAnthropicProvider if lifecycle.asynchronous else AnthropicProvider)
)
injected: Any = provider_class("test-model")
kwargs: dict[str, Any] = {} if constructor_default else {"model": injected}
client = lifecycle.client(model=injected) if constructor_default else lifecycle.client()
async with lifecycle.context(client):
await lifecycle.evaluate(client, **kwargs)
await lifecycle.evaluate(client, model="owned-model")
await lifecycle.evaluate(client, **kwargs)
assert not injected._client.is_closed()
injected._client.close.assert_not_called()
assert lifecycle.providers[1]._client.is_closed()
with pytest.raises(RuntimeError, match="closed"):
await lifecycle.evaluate(client, model=injected)
if lifecycle.asynchronous:
await injected.aclose()
else:
injected.close()
assert injected._client.is_closed()
asyncio.run(run())
def test_custom_provider_without_close_remains_supported(lifecycle: Lifecycle) -> None:
async def run() -> None:
provider_class = FakeAsyncProvider if lifecycle.asynchronous else FakeSyncProvider
provider = provider_class({"answers": {"positive": True}})
async with lifecycle.context(lifecycle.client(model=provider)) as client:
await lifecycle.evaluate(client)
assert len(provider.calls) == 1
assert lifecycle.providers == []
asyncio.run(run())
@pytest.mark.parametrize("failure", ["body", "request", "validation", "cancelled"])
def test_exceptional_exit_closes_owned_sdks(lifecycle: Lifecycle, monkeypatch: pytest.MonkeyPatch, failure: str) -> None:
error_type = {"body": ValueError, "request": TypeSafeError, "validation": ValueError, "cancelled": asyncio.CancelledError}[failure]
async def fail_in_context() -> None:
async with lifecycle.context(lifecycle.client()) as client:
if failure == "validation":
response = client.system_one("document", {})
if lifecycle.asynchronous:
await response
else:
await lifecycle.evaluate(client)
if failure == "request":
mock_class = AsyncMock if lifecycle.asynchronous else Mock
monkeypatch.setattr(lifecycle.providers[0], "request", mock_class(side_effect=TypeSafeError("failed")))
await lifecycle.evaluate(client)
else:
raise error_type("interrupted")
async def run() -> None:
with pytest.raises(error_type):
await fail_in_context()
assert len(lifecycle.providers) == 1
assert lifecycle.providers[0]._client.is_closed()
asyncio.run(run())
def test_cleanup_continues_after_failure(lifecycle: Lifecycle) -> None:
async def run() -> None:
client = lifecycle.client()
await lifecycle.evaluate(client)
await lifecycle.evaluate(client, model="another-model")
await lifecycle.evaluate(client, model="third-model")
first_error = RuntimeError("close failed")
lifecycle.providers[0]._client.close.side_effect = first_error
lifecycle.providers[1]._client.close.side_effect = ValueError("another close failed")
with pytest.raises(RuntimeError, match="close failed") as caught:
await lifecycle.close(client)
assert caught.value is first_error
assert lifecycle.providers[2]._client.is_closed()
for provider in lifecycle.providers:
provider._client.close.assert_called_once()
with pytest.raises(RuntimeError, match="closed"):
await lifecycle.evaluate(client)
lifecycle.providers[0]._client.close.side_effect = None
lifecycle.providers[1]._client.close.side_effect = None
await lifecycle.close(client)
await lifecycle.close(client)
assert all(provider._client.is_closed() for provider in lifecycle.providers)
assert [provider._client.close.call_count for provider in lifecycle.providers] == [2, 2, 1]
asyncio.run(run())
def test_close_before_first_use_does_not_construct_providers(lifecycle: Lifecycle) -> None:
async def run() -> None:
client = lifecycle.client()
await lifecycle.close(client)
await lifecycle.close(client)
with pytest.raises(RuntimeError, match="closed"):
await lifecycle.evaluate(client)
assert lifecycle.providers == []
asyncio.run(run())
def test_failed_construction_is_not_cached(lifecycle: Lifecycle, monkeypatch: pytest.MonkeyPatch) -> None:
async def run() -> None:
async with lifecycle.context(lifecycle.client()) as client:
factory = Mock(wraps=client._build_provider, side_effect=[ValueError("constructor failed"), DEFAULT])
monkeypatch.setattr(client, "_build_provider", factory)
with pytest.raises(ValueError, match="constructor failed"):
await lifecycle.evaluate(client)
assert lifecycle.providers == []
await lifecycle.evaluate(client)
await lifecycle.evaluate(client)
assert factory.call_count == 2
assert len(lifecycle.providers) == 1
asyncio.run(run())
def test_environment_is_captured_on_first_use(lifecycle: Lifecycle, monkeypatch: pytest.MonkeyPatch) -> None:
async def run() -> None:
prefix = lifecycle.vendor.upper()
client = lifecycle.client()
monkeypatch.setenv(f"{prefix}_API_KEY", "first-test-key")
monkeypatch.setenv(f"{prefix}_BASE_URL", "https://first.invalid/v1")
async with lifecycle.context(client):
await lifecycle.evaluate(client)
monkeypatch.setenv(f"{prefix}_API_KEY", "second-test-key")
monkeypatch.setenv(f"{prefix}_BASE_URL", "https://second.invalid/v1")
await lifecycle.evaluate(client)
assert len(lifecycle.providers) == 1
sdk = lifecycle.providers[0]._client
assert sdk.api_key == "first-test-key"
assert sdk.base_url.host == "first.invalid"
async with lifecycle.context(lifecycle.client()) as fresh:
await lifecycle.evaluate(fresh)
sdk = lifecycle.providers[1]._client
assert sdk.api_key == "second-test-key"
assert sdk.base_url.host == "second.invalid"
asyncio.run(run())
@pytest.mark.parametrize("cleanup_failure", [None, "before", "after"])
def test_async_cleanup_propagates_cancellation_after_remaining_cleanup(monkeypatch: pytest.MonkeyPatch, cleanup_failure: str | None) -> None:
async def run() -> None:
client = AsyncSystemOneAdapterClient(structured_outputs=True, llm_answer_mode="discrete", provider="openai")
first = FakeAsyncProvider(RESULT.text)
second = FakeAsyncProvider(RESULT.text)
third = FakeAsyncProvider(RESULT.text)
closing = asyncio.Event()
async def wait_for_cancellation() -> None:
closing.set()
await asyncio.Event().wait()
first_close = AsyncMock(side_effect=RuntimeError("first close failed") if cleanup_failure == "before" else None)
cancelled_close = AsyncMock(side_effect=wait_for_cancellation)
third_close = AsyncMock(side_effect=RuntimeError("last close failed") if cleanup_failure == "after" else None)
monkeypatch.setattr(first, "aclose", first_close, raising=False)
monkeypatch.setattr(second, "aclose", cancelled_close, raising=False)
monkeypatch.setattr(third, "aclose", third_close, raising=False)
monkeypatch.setattr(client, "_build_provider", Mock(side_effect=[first, second, third]))
await client.system_one("document", QUESTIONS, model="first")
await client.system_one("document", QUESTIONS, model="second")
await client.system_one("document", QUESTIONS, model="third")
task = asyncio.create_task(client.aclose())
await asyncio.wait_for(closing.wait(), timeout=10)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert task.cancelled()
first_close.assert_awaited_once()
cancelled_close.assert_awaited_once()
third_close.assert_awaited_once()
first_close.side_effect = None
cancelled_close.side_effect = None
third_close.side_effect = None
await client.aclose()
await client.aclose()
assert first_close.await_count == (2 if cleanup_failure == "before" else 1)
assert cancelled_close.await_count == 2
assert third_close.await_count == (2 if cleanup_failure == "after" else 1)
asyncio.run(run())
@pytest.mark.parametrize("fails", [False, True], ids=["success", "failure"])
def test_concurrent_close_waits_for_same_cleanup(lifecycle: Lifecycle, fails: bool) -> None:
async def run() -> None:
client = lifecycle.client()
await lifecycle.evaluate(client)
sdk = lifecycle.providers[0]._client
error = RuntimeError("close failed")
if lifecycle.asynchronous:
started = asyncio.Event()
release = asyncio.Event()
async def close_async() -> Any:
started.set()
await release.wait()
if fails:
raise error
return DEFAULT
sdk.close.side_effect = close_async
first = asyncio.create_task(client.aclose())
await asyncio.wait_for(started.wait(), timeout=10)
second = asyncio.create_task(client.aclose())
try:
await asyncio.sleep(0)
assert not second.done()
finally:
release.set()
results = list(await asyncio.gather(first, second, return_exceptions=True))
else:
started_sync = Event()
release_sync = Event()
def close_sync() -> Any:
started_sync.set()
assert release_sync.wait(timeout=10)
if fails:
raise error
return DEFAULT
sdk.close.side_effect = close_sync
with ThreadPoolExecutor(max_workers=2) as executor:
first_sync = executor.submit(client.close)
try:
assert started_sync.wait(timeout=10)
second_sync = executor.submit(client.close)
with pytest.raises(FutureTimeoutError):
second_sync.result(timeout=0.05)
finally:
release_sync.set()
results = [future.exception(timeout=10) for future in (first_sync, second_sync)]
assert results == ([error, error] if fails else [None, None])
sdk.close.assert_called_once()
assert sdk.is_closed() is not fails
asyncio.run(run())
def test_cancelling_close_waiter_does_not_interrupt_cleanup(monkeypatch: pytest.MonkeyPatch) -> None:
async def run() -> None:
client = AsyncSystemOneAdapterClient(structured_outputs=True, llm_answer_mode="discrete", provider="openai")
provider = FakeAsyncProvider(RESULT.text)
started = asyncio.Event()
release = asyncio.Event()
async def close() -> None:
started.set()
await release.wait()
close_mock = AsyncMock(side_effect=close)
monkeypatch.setattr(provider, "aclose", close_mock, raising=False)
monkeypatch.setattr(client, "_build_provider", Mock(return_value=provider))
await client.system_one("document", QUESTIONS, model="test")
owner = asyncio.create_task(client.aclose())
await asyncio.wait_for(started.wait(), timeout=10)
waiter = asyncio.create_task(client.aclose())
try:
await asyncio.sleep(0)
waiter.cancel()
with pytest.raises(asyncio.CancelledError):
await waiter
assert not owner.done()
another_waiter = asyncio.create_task(client.aclose())
await asyncio.sleep(0)
assert not another_waiter.done()
finally:
release.set()
await owner
await another_waiter
await client.aclose()
close_mock.assert_awaited_once()
asyncio.run(run())
def test_concurrent_first_use_reuses_pool_and_isolates_traces(lifecycle: Lifecycle, monkeypatch: pytest.MonkeyPatch) -> None:
async def run() -> None:
provider_class = (
(AsyncOpenAIProvider if lifecycle.asynchronous else OpenAIProvider)
if lifecycle.vendor == "openai"
else (AsyncAnthropicProvider if lifecycle.asynchronous else AnthropicProvider)
)
original_sync_request = (OpenAIProvider if lifecycle.vendor == "openai" else AnthropicProvider).request
original_async_request = (AsyncOpenAIProvider if lifecycle.vendor == "openai" else AsyncAnthropicProvider).request
request_barrier = Barrier(8)
all_started = asyncio.Event()
started = 0
def request(provider: Any, messages: list[Message], **kwargs: Any) -> ProviderResult:
request_barrier.wait(timeout=10)
return original_sync_request(provider, messages, **kwargs)
async def request_async(provider: Any, messages: list[Message], **kwargs: Any) -> ProviderResult:
nonlocal started
started += 1
if started == 8:
all_started.set()
await asyncio.wait_for(all_started.wait(), timeout=10)
return await original_async_request(provider, messages, **kwargs)
monkeypatch.setattr(provider_class, "request", request_async if lifecycle.asynchronous else request)
async with lifecycle.context(lifecycle.client()) as client:
if lifecycle.asynchronous:
responses = await asyncio.gather(*(lifecycle.evaluate(client, f"document-{i}") for i in range(8)))
else:
barrier = Barrier(8)
def evaluate(index: int) -> Any:
barrier.wait(timeout=10)
return client.system_one(f"document-{index}", QUESTIONS)
with ThreadPoolExecutor(max_workers=8) as executor:
responses = list(executor.map(evaluate, range(8)))
assert len(lifecycle.providers) == 1
for index, response in enumerate(responses):
attempts = response.debug["llm_attempts"]
assert len(attempts) == 1
assert f"document-{index}" in attempts[0]["messages"][1]["content"]
assert attempts[0]["request"]["messages"] == attempts[0]["messages"]
assert response.usage.input_tokens_total == 11
assert response.usage.n_retries == 0
asyncio.run(run())