Typed decision/classification service with an in-process gte-small ONNX embedding engine and lexical fallback, JS/Python SDKs, adapters, and the otsone model family. Apache-2.0 (sub-packages keep their MIT licenses). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
491 lines
21 KiB
Python
491 lines
21 KiB
Python
"""Provider reuse and ownership, with real SDK pools and no model API calls."""
|
|
|
|
import asyncio
|
|
from collections.abc import AsyncIterator, Iterator
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from concurrent.futures import TimeoutError as FutureTimeoutError
|
|
from contextlib import asynccontextmanager
|
|
from dataclasses import dataclass, field
|
|
from threading import Barrier, Event
|
|
from typing import Any
|
|
from unittest.mock import DEFAULT, AsyncMock, Mock
|
|
|
|
import pytest
|
|
from openjev import TypeSafeError
|
|
|
|
from openjev_adapters import AsyncSystemOneAdapterClient, Noul, SystemOneAdapterClient
|
|
from openjev_adapters.providers import Message, ProviderResult
|
|
from openjev_adapters.providers.anthropic import (
|
|
AnthropicProvider,
|
|
AsyncAnthropicProvider,
|
|
)
|
|
from openjev_adapters.providers.base import record_request, render_messages
|
|
from openjev_adapters.providers.openai import AsyncOpenAIProvider, OpenAIProvider
|
|
from tests.test_client_with_fake_model import FakeAsyncProvider, FakeSyncProvider
|
|
|
|
QUESTIONS = {"positive": Noul(instructions="The review is positive.")}
|
|
RESULT = ProviderResult('{"answers":{"positive":true}}', 11, 7)
|
|
|
|
|
|
@dataclass
|
|
class Lifecycle:
|
|
asynchronous: bool
|
|
vendor: str
|
|
providers: list[Any] = field(default_factory=list)
|
|
|
|
def client(self, **kwargs: Any) -> Any:
|
|
options: dict[str, Any] = {
|
|
"structured_outputs": True,
|
|
"llm_answer_mode": "discrete",
|
|
"provider": self.vendor,
|
|
"model": "test-model",
|
|
}
|
|
options.update(kwargs)
|
|
client_class = AsyncSystemOneAdapterClient if self.asynchronous else SystemOneAdapterClient
|
|
return client_class(**options)
|
|
|
|
async def evaluate(self, client: Any, state: str = "Great book", **kwargs: Any) -> Any:
|
|
response = client.system_one(state, QUESTIONS, **kwargs)
|
|
return await response if self.asynchronous else response
|
|
|
|
async def close(self, client: Any) -> None:
|
|
if self.asynchronous:
|
|
await client.aclose()
|
|
else:
|
|
client.close()
|
|
|
|
@asynccontextmanager
|
|
async def context(self, client: Any) -> AsyncIterator[Any]:
|
|
if self.asynchronous:
|
|
async with client:
|
|
yield client
|
|
else:
|
|
with client:
|
|
yield client
|
|
|
|
|
|
@pytest.fixture(params=[False, True], ids=["sync", "async"])
|
|
def lifecycle(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, vendor: str) -> Iterator[Lifecycle]:
|
|
case = Lifecycle(request.param, vendor)
|
|
|
|
def watch(provider_class: Any) -> None:
|
|
original_init = provider_class.__init__
|
|
|
|
def initialize(provider: Any, *args: Any, **kwargs: Any) -> None:
|
|
original_init(provider, *args, **kwargs)
|
|
spy = AsyncMock if case.asynchronous else Mock
|
|
monkeypatch.setattr(provider._client, "close", spy(wraps=provider._client.close))
|
|
case.providers.append(provider)
|
|
|
|
def respond(provider: Any, messages: list[Message], **kwargs: Any) -> ProviderResult:
|
|
record_request({"messages": render_messages(messages)}, api="offline-test")
|
|
return RESULT
|
|
|
|
async def respond_async(provider: Any, messages: list[Message], **kwargs: Any) -> ProviderResult:
|
|
result = respond(provider, messages, **kwargs)
|
|
await asyncio.sleep(0)
|
|
return result
|
|
|
|
monkeypatch.setattr(provider_class, "__init__", initialize)
|
|
monkeypatch.setattr(provider_class, "request", respond_async if case.asynchronous else respond)
|
|
|
|
for provider_class in (AsyncOpenAIProvider, AsyncAnthropicProvider) if case.asynchronous else (OpenAIProvider, AnthropicProvider):
|
|
watch(provider_class)
|
|
yield case
|
|
|
|
# Retaining providers makes missed cleanup observable; release test pools even
|
|
# when an assertion fails or a close spy has deliberately been made to raise.
|
|
async def cleanup() -> None:
|
|
for provider in case.providers:
|
|
sdk = provider._client
|
|
if not sdk.is_closed():
|
|
result = type(sdk).close(sdk)
|
|
if case.asynchronous:
|
|
await result
|
|
|
|
asyncio.run(cleanup())
|
|
|
|
|
|
@pytest.fixture(params=["openai", "anthropic"])
|
|
def vendor(request: pytest.FixtureRequest) -> str:
|
|
return request.param
|
|
|
|
|
|
def test_reuses_owned_provider_and_closes_sdk_on_context_exit(lifecycle: Lifecycle) -> None:
|
|
async def run() -> None:
|
|
client = lifecycle.client()
|
|
assert lifecycle.providers == []
|
|
async with lifecycle.context(client):
|
|
for _ in range(3):
|
|
response = await lifecycle.evaluate(client)
|
|
assert response.nouls["positive"].noul == 1
|
|
assert len(lifecycle.providers) == 1
|
|
assert not lifecycle.providers[0]._client.is_closed()
|
|
sdk = lifecycle.providers[0]._client
|
|
assert sdk.is_closed()
|
|
await lifecycle.close(client)
|
|
sdk.close.assert_called_once()
|
|
if lifecycle.asynchronous:
|
|
sdk.close.assert_awaited_once()
|
|
with pytest.raises(RuntimeError, match="closed"):
|
|
await lifecycle.evaluate(client)
|
|
with pytest.raises(RuntimeError, match="closed"):
|
|
async with lifecycle.context(client):
|
|
pytest.fail("Reentered a closed client")
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_cache_uses_resolved_provider_and_model_and_is_per_client(lifecycle: Lifecycle) -> None:
|
|
async def run() -> None:
|
|
async with lifecycle.context(lifecycle.client()) as client:
|
|
await lifecycle.evaluate(client)
|
|
await lifecycle.evaluate(client, model="test-model", provider=lifecycle.vendor)
|
|
await lifecycle.evaluate(client, model="another-model")
|
|
other_vendor = "anthropic" if lifecycle.vendor == "openai" else "openai"
|
|
await lifecycle.evaluate(client, provider=other_vendor)
|
|
assert len(lifecycle.providers) == 3
|
|
async with lifecycle.context(lifecycle.client()) as second:
|
|
await lifecycle.evaluate(second)
|
|
assert len(lifecycle.providers) == 4
|
|
assert all(not provider._client.is_closed() for provider in lifecycle.providers[:3])
|
|
assert all(provider._client.is_closed() for provider in lifecycle.providers)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("constructor_default", [False, True])
|
|
def test_injected_provider_is_borrowed(lifecycle: Lifecycle, constructor_default: bool) -> None:
|
|
async def run() -> None:
|
|
provider_class = (
|
|
(AsyncOpenAIProvider if lifecycle.asynchronous else OpenAIProvider)
|
|
if lifecycle.vendor == "openai"
|
|
else (AsyncAnthropicProvider if lifecycle.asynchronous else AnthropicProvider)
|
|
)
|
|
injected: Any = provider_class("test-model")
|
|
kwargs: dict[str, Any] = {} if constructor_default else {"model": injected}
|
|
client = lifecycle.client(model=injected) if constructor_default else lifecycle.client()
|
|
async with lifecycle.context(client):
|
|
await lifecycle.evaluate(client, **kwargs)
|
|
await lifecycle.evaluate(client, model="owned-model")
|
|
await lifecycle.evaluate(client, **kwargs)
|
|
assert not injected._client.is_closed()
|
|
injected._client.close.assert_not_called()
|
|
assert lifecycle.providers[1]._client.is_closed()
|
|
with pytest.raises(RuntimeError, match="closed"):
|
|
await lifecycle.evaluate(client, model=injected)
|
|
if lifecycle.asynchronous:
|
|
await injected.aclose()
|
|
else:
|
|
injected.close()
|
|
assert injected._client.is_closed()
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_custom_provider_without_close_remains_supported(lifecycle: Lifecycle) -> None:
|
|
async def run() -> None:
|
|
provider_class = FakeAsyncProvider if lifecycle.asynchronous else FakeSyncProvider
|
|
provider = provider_class({"answers": {"positive": True}})
|
|
async with lifecycle.context(lifecycle.client(model=provider)) as client:
|
|
await lifecycle.evaluate(client)
|
|
assert len(provider.calls) == 1
|
|
assert lifecycle.providers == []
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("failure", ["body", "request", "validation", "cancelled"])
|
|
def test_exceptional_exit_closes_owned_sdks(lifecycle: Lifecycle, monkeypatch: pytest.MonkeyPatch, failure: str) -> None:
|
|
error_type = {"body": ValueError, "request": TypeSafeError, "validation": ValueError, "cancelled": asyncio.CancelledError}[failure]
|
|
|
|
async def fail_in_context() -> None:
|
|
async with lifecycle.context(lifecycle.client()) as client:
|
|
if failure == "validation":
|
|
response = client.system_one("document", {})
|
|
if lifecycle.asynchronous:
|
|
await response
|
|
else:
|
|
await lifecycle.evaluate(client)
|
|
if failure == "request":
|
|
mock_class = AsyncMock if lifecycle.asynchronous else Mock
|
|
monkeypatch.setattr(lifecycle.providers[0], "request", mock_class(side_effect=TypeSafeError("failed")))
|
|
await lifecycle.evaluate(client)
|
|
else:
|
|
raise error_type("interrupted")
|
|
|
|
async def run() -> None:
|
|
with pytest.raises(error_type):
|
|
await fail_in_context()
|
|
assert len(lifecycle.providers) == 1
|
|
assert lifecycle.providers[0]._client.is_closed()
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_cleanup_continues_after_failure(lifecycle: Lifecycle) -> None:
|
|
async def run() -> None:
|
|
client = lifecycle.client()
|
|
await lifecycle.evaluate(client)
|
|
await lifecycle.evaluate(client, model="another-model")
|
|
await lifecycle.evaluate(client, model="third-model")
|
|
first_error = RuntimeError("close failed")
|
|
lifecycle.providers[0]._client.close.side_effect = first_error
|
|
lifecycle.providers[1]._client.close.side_effect = ValueError("another close failed")
|
|
with pytest.raises(RuntimeError, match="close failed") as caught:
|
|
await lifecycle.close(client)
|
|
assert caught.value is first_error
|
|
assert lifecycle.providers[2]._client.is_closed()
|
|
for provider in lifecycle.providers:
|
|
provider._client.close.assert_called_once()
|
|
with pytest.raises(RuntimeError, match="closed"):
|
|
await lifecycle.evaluate(client)
|
|
lifecycle.providers[0]._client.close.side_effect = None
|
|
lifecycle.providers[1]._client.close.side_effect = None
|
|
await lifecycle.close(client)
|
|
await lifecycle.close(client)
|
|
assert all(provider._client.is_closed() for provider in lifecycle.providers)
|
|
assert [provider._client.close.call_count for provider in lifecycle.providers] == [2, 2, 1]
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_close_before_first_use_does_not_construct_providers(lifecycle: Lifecycle) -> None:
|
|
async def run() -> None:
|
|
client = lifecycle.client()
|
|
await lifecycle.close(client)
|
|
await lifecycle.close(client)
|
|
with pytest.raises(RuntimeError, match="closed"):
|
|
await lifecycle.evaluate(client)
|
|
assert lifecycle.providers == []
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_failed_construction_is_not_cached(lifecycle: Lifecycle, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
async def run() -> None:
|
|
async with lifecycle.context(lifecycle.client()) as client:
|
|
factory = Mock(wraps=client._build_provider, side_effect=[ValueError("constructor failed"), DEFAULT])
|
|
monkeypatch.setattr(client, "_build_provider", factory)
|
|
with pytest.raises(ValueError, match="constructor failed"):
|
|
await lifecycle.evaluate(client)
|
|
assert lifecycle.providers == []
|
|
await lifecycle.evaluate(client)
|
|
await lifecycle.evaluate(client)
|
|
assert factory.call_count == 2
|
|
assert len(lifecycle.providers) == 1
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_environment_is_captured_on_first_use(lifecycle: Lifecycle, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
async def run() -> None:
|
|
prefix = lifecycle.vendor.upper()
|
|
client = lifecycle.client()
|
|
monkeypatch.setenv(f"{prefix}_API_KEY", "first-test-key")
|
|
monkeypatch.setenv(f"{prefix}_BASE_URL", "https://first.invalid/v1")
|
|
async with lifecycle.context(client):
|
|
await lifecycle.evaluate(client)
|
|
monkeypatch.setenv(f"{prefix}_API_KEY", "second-test-key")
|
|
monkeypatch.setenv(f"{prefix}_BASE_URL", "https://second.invalid/v1")
|
|
await lifecycle.evaluate(client)
|
|
assert len(lifecycle.providers) == 1
|
|
sdk = lifecycle.providers[0]._client
|
|
assert sdk.api_key == "first-test-key"
|
|
assert sdk.base_url.host == "first.invalid"
|
|
async with lifecycle.context(lifecycle.client()) as fresh:
|
|
await lifecycle.evaluate(fresh)
|
|
sdk = lifecycle.providers[1]._client
|
|
assert sdk.api_key == "second-test-key"
|
|
assert sdk.base_url.host == "second.invalid"
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("cleanup_failure", [None, "before", "after"])
|
|
def test_async_cleanup_propagates_cancellation_after_remaining_cleanup(monkeypatch: pytest.MonkeyPatch, cleanup_failure: str | None) -> None:
|
|
async def run() -> None:
|
|
client = AsyncSystemOneAdapterClient(structured_outputs=True, llm_answer_mode="discrete", provider="openai")
|
|
first = FakeAsyncProvider(RESULT.text)
|
|
second = FakeAsyncProvider(RESULT.text)
|
|
third = FakeAsyncProvider(RESULT.text)
|
|
closing = asyncio.Event()
|
|
|
|
async def wait_for_cancellation() -> None:
|
|
closing.set()
|
|
await asyncio.Event().wait()
|
|
|
|
first_close = AsyncMock(side_effect=RuntimeError("first close failed") if cleanup_failure == "before" else None)
|
|
cancelled_close = AsyncMock(side_effect=wait_for_cancellation)
|
|
third_close = AsyncMock(side_effect=RuntimeError("last close failed") if cleanup_failure == "after" else None)
|
|
monkeypatch.setattr(first, "aclose", first_close, raising=False)
|
|
monkeypatch.setattr(second, "aclose", cancelled_close, raising=False)
|
|
monkeypatch.setattr(third, "aclose", third_close, raising=False)
|
|
monkeypatch.setattr(client, "_build_provider", Mock(side_effect=[first, second, third]))
|
|
await client.system_one("document", QUESTIONS, model="first")
|
|
await client.system_one("document", QUESTIONS, model="second")
|
|
await client.system_one("document", QUESTIONS, model="third")
|
|
task = asyncio.create_task(client.aclose())
|
|
await asyncio.wait_for(closing.wait(), timeout=10)
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
assert task.cancelled()
|
|
first_close.assert_awaited_once()
|
|
cancelled_close.assert_awaited_once()
|
|
third_close.assert_awaited_once()
|
|
first_close.side_effect = None
|
|
cancelled_close.side_effect = None
|
|
third_close.side_effect = None
|
|
await client.aclose()
|
|
await client.aclose()
|
|
assert first_close.await_count == (2 if cleanup_failure == "before" else 1)
|
|
assert cancelled_close.await_count == 2
|
|
assert third_close.await_count == (2 if cleanup_failure == "after" else 1)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize("fails", [False, True], ids=["success", "failure"])
|
|
def test_concurrent_close_waits_for_same_cleanup(lifecycle: Lifecycle, fails: bool) -> None:
|
|
async def run() -> None:
|
|
client = lifecycle.client()
|
|
await lifecycle.evaluate(client)
|
|
sdk = lifecycle.providers[0]._client
|
|
error = RuntimeError("close failed")
|
|
if lifecycle.asynchronous:
|
|
started = asyncio.Event()
|
|
release = asyncio.Event()
|
|
|
|
async def close_async() -> Any:
|
|
started.set()
|
|
await release.wait()
|
|
if fails:
|
|
raise error
|
|
return DEFAULT
|
|
|
|
sdk.close.side_effect = close_async
|
|
first = asyncio.create_task(client.aclose())
|
|
await asyncio.wait_for(started.wait(), timeout=10)
|
|
second = asyncio.create_task(client.aclose())
|
|
try:
|
|
await asyncio.sleep(0)
|
|
assert not second.done()
|
|
finally:
|
|
release.set()
|
|
results = list(await asyncio.gather(first, second, return_exceptions=True))
|
|
else:
|
|
started_sync = Event()
|
|
release_sync = Event()
|
|
|
|
def close_sync() -> Any:
|
|
started_sync.set()
|
|
assert release_sync.wait(timeout=10)
|
|
if fails:
|
|
raise error
|
|
return DEFAULT
|
|
|
|
sdk.close.side_effect = close_sync
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
first_sync = executor.submit(client.close)
|
|
try:
|
|
assert started_sync.wait(timeout=10)
|
|
second_sync = executor.submit(client.close)
|
|
with pytest.raises(FutureTimeoutError):
|
|
second_sync.result(timeout=0.05)
|
|
finally:
|
|
release_sync.set()
|
|
results = [future.exception(timeout=10) for future in (first_sync, second_sync)]
|
|
assert results == ([error, error] if fails else [None, None])
|
|
sdk.close.assert_called_once()
|
|
assert sdk.is_closed() is not fails
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_cancelling_close_waiter_does_not_interrupt_cleanup(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
async def run() -> None:
|
|
client = AsyncSystemOneAdapterClient(structured_outputs=True, llm_answer_mode="discrete", provider="openai")
|
|
provider = FakeAsyncProvider(RESULT.text)
|
|
started = asyncio.Event()
|
|
release = asyncio.Event()
|
|
|
|
async def close() -> None:
|
|
started.set()
|
|
await release.wait()
|
|
|
|
close_mock = AsyncMock(side_effect=close)
|
|
monkeypatch.setattr(provider, "aclose", close_mock, raising=False)
|
|
monkeypatch.setattr(client, "_build_provider", Mock(return_value=provider))
|
|
await client.system_one("document", QUESTIONS, model="test")
|
|
owner = asyncio.create_task(client.aclose())
|
|
await asyncio.wait_for(started.wait(), timeout=10)
|
|
waiter = asyncio.create_task(client.aclose())
|
|
try:
|
|
await asyncio.sleep(0)
|
|
waiter.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await waiter
|
|
assert not owner.done()
|
|
another_waiter = asyncio.create_task(client.aclose())
|
|
await asyncio.sleep(0)
|
|
assert not another_waiter.done()
|
|
finally:
|
|
release.set()
|
|
await owner
|
|
await another_waiter
|
|
await client.aclose()
|
|
close_mock.assert_awaited_once()
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_concurrent_first_use_reuses_pool_and_isolates_traces(lifecycle: Lifecycle, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
async def run() -> None:
|
|
provider_class = (
|
|
(AsyncOpenAIProvider if lifecycle.asynchronous else OpenAIProvider)
|
|
if lifecycle.vendor == "openai"
|
|
else (AsyncAnthropicProvider if lifecycle.asynchronous else AnthropicProvider)
|
|
)
|
|
original_sync_request = (OpenAIProvider if lifecycle.vendor == "openai" else AnthropicProvider).request
|
|
original_async_request = (AsyncOpenAIProvider if lifecycle.vendor == "openai" else AsyncAnthropicProvider).request
|
|
request_barrier = Barrier(8)
|
|
all_started = asyncio.Event()
|
|
started = 0
|
|
|
|
def request(provider: Any, messages: list[Message], **kwargs: Any) -> ProviderResult:
|
|
request_barrier.wait(timeout=10)
|
|
return original_sync_request(provider, messages, **kwargs)
|
|
|
|
async def request_async(provider: Any, messages: list[Message], **kwargs: Any) -> ProviderResult:
|
|
nonlocal started
|
|
started += 1
|
|
if started == 8:
|
|
all_started.set()
|
|
await asyncio.wait_for(all_started.wait(), timeout=10)
|
|
return await original_async_request(provider, messages, **kwargs)
|
|
|
|
monkeypatch.setattr(provider_class, "request", request_async if lifecycle.asynchronous else request)
|
|
async with lifecycle.context(lifecycle.client()) as client:
|
|
if lifecycle.asynchronous:
|
|
responses = await asyncio.gather(*(lifecycle.evaluate(client, f"document-{i}") for i in range(8)))
|
|
else:
|
|
barrier = Barrier(8)
|
|
|
|
def evaluate(index: int) -> Any:
|
|
barrier.wait(timeout=10)
|
|
return client.system_one(f"document-{index}", QUESTIONS)
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
responses = list(executor.map(evaluate, range(8)))
|
|
assert len(lifecycle.providers) == 1
|
|
for index, response in enumerate(responses):
|
|
attempts = response.debug["llm_attempts"]
|
|
assert len(attempts) == 1
|
|
assert f"document-{index}" in attempts[0]["messages"][1]["content"]
|
|
assert attempts[0]["request"]["messages"] == attempts[0]["messages"]
|
|
assert response.usage.input_tokens_total == 11
|
|
assert response.usage.n_retries == 0
|
|
|
|
asyncio.run(run())
|