umpirix/adapters/python/tests/test_schema.py
caelum0x 4847e35451 OpenJev: open-source typed decision service
Typed decision/classification service with an in-process gte-small ONNX
embedding engine and lexical fallback, JS/Python SDKs, adapters, and the
otsone model family. Apache-2.0 (sub-packages keep their MIT licenses).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 00:12:36 +03:00

117 lines
4.9 KiB
Python

"""Provider schemas preserve caller-defined question IDs and choice labels."""
from typing import Any
import pytest
from pydantic import ValidationError
from pydantic_core import to_json
from openjev_adapters import Choice, Noul, Score
from openjev_adapters._schema import (
Question,
convert_question_collection_to_validated_api_question_models,
create_llm_output_model,
create_raw_output_schema,
)
from openjev_adapters._utils.probability_normalization import AnswerMode
SCHEMA_KEYWORDS = ["title", "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"]
FIELD_NAMES = [*SCHEMA_KEYWORDS, "model_dump", "model_config", "_private", "", "with spaces", "answer_0", "probability_0"]
@pytest.mark.parametrize(
"question",
[
{"type": "unknown"},
{"type": "noul", "instructions": 42},
{"type": "choice", "criteria": ["yes", "no"]},
{"type": "score", "criteria": {"0": "Bad.", "1": "Good."}},
],
)
def test_invalid_dictionary_questions_are_rejected(question: Any) -> None:
with pytest.raises(ValidationError):
convert_question_collection_to_validated_api_question_models({"answer": question})
def test_sdk_question_fields_are_revalidated() -> None:
question = Noul()
question.criteria = {"true": 42} # type: ignore[assignment]
with pytest.raises(ValidationError):
convert_question_collection_to_validated_api_question_models({"answer": question})
@pytest.mark.parametrize("answer_mode", ["probabilities", "discrete"])
def test_question_ids_preserve_arbitrary_names(answer_mode: AnswerMode) -> None:
questions = {key: Noul(instructions=f"Evaluate {key}.") for key in FIELD_NAMES}
output_model = create_llm_output_model(questions, answer_mode)
schema = create_raw_output_schema(output_model)
answers = schema["$defs"]["TypeSafeAnswers"]
assert schema["properties"]["answers"] == {"$ref": "#/$defs/TypeSafeAnswers"}
assert "Use these property names verbatim" in answers["description"]
assert set(answers["properties"]) == set(answers["required"]) == set(questions)
for key, answer in answers["properties"].items():
assert questions[key].instructions in answer["description"]
assert not set(answer) & set(SCHEMA_KEYWORDS)
payload = {"answers": dict.fromkeys(questions, True if answer_mode == "discrete" else 0.8)}
assert output_model.model_validate_json(to_json(payload)).model_dump() == payload
def test_probability_labels_preserve_arbitrary_names() -> None:
criteria = {key: f"The {key} option." for key in FIELD_NAMES}
output_model = create_llm_output_model({"level": Choice(criteria=criteria)}, "probabilities")
schema = create_raw_output_schema(output_model)
probabilities = schema["$defs"]["ProbabilityMap0"]
assert set(probabilities["properties"]) == set(probabilities["required"]) == set(criteria)
assert "title" not in probabilities
for key, probability in probabilities["properties"].items():
assert probability["description"] == criteria[key]
assert not set(probability) & set(SCHEMA_KEYWORDS)
payload = {"answers": {"level": dict.fromkeys(criteria, 1 / len(criteria))}}
assert output_model.model_validate_json(to_json(payload)).model_dump() == payload
with pytest.raises(ValidationError):
output_model.model_validate_json(to_json({"answers": {"level": dict.fromkeys(criteria, 2)}}))
@pytest.mark.parametrize(
"question,answer_mode,answer",
[
(Noul(), "discrete", "true"),
(Noul(), "discrete", 1),
(Noul(), "probabilities", "0.5"),
(Noul(), "probabilities", True),
(Noul(), "probabilities", -0.1),
(Noul(), "probabilities", 1.1),
(Noul(), "probabilities", float("nan")),
(Score(criteria=["Bad.", "Good."]), "discrete", 1.0),
(Score(criteria=["Bad.", "Good."]), "discrete", True),
(Score(criteria=["Bad.", "Good."]), "discrete", 2),
(Choice(criteria={"yes": None, "no": None}), "discrete", "maybe"),
(Choice(criteria={"yes": None, "no": None}), "probabilities", {"yes": 0.5}),
(Choice(criteria={"yes": None, "no": None}), "probabilities", {"yes": 0.5, "no": 0.5, "maybe": 0}),
],
)
def test_output_validation_preserves_types_bounds_and_allowed_values(question: Question, answer_mode: AnswerMode, answer: Any) -> None:
output_model = create_llm_output_model({"answer": question}, answer_mode)
with pytest.raises(ValidationError):
output_model.model_validate_json(to_json({"answers": {"answer": answer}}))
@pytest.mark.parametrize(
"payload",
[
{"answers": {"answer": 0.5}, "extra": 1},
{"answers": {"answer": 0.5, "extra": 1}},
{"answers": {"answer_0": 0.5}},
],
)
def test_output_rejects_extra_fields_and_internal_field_names(payload: dict[str, Any]) -> None:
output_model = create_llm_output_model({"answer": Noul()}, "probabilities")
with pytest.raises(ValidationError):
output_model.model_validate_json(to_json(payload))