235 lines
12 KiB
Python
235 lines
12 KiB
Python
|
|
"""The value judge: which runs show the owner's rules turned from failing to holding by the Curator's revisions, and
|
||
|
|
which of them were sedimented as planning or action that then acted."""
|
||
|
|
|
||
|
|
from experimental.simulation.value import candidates, cases, cultivated, judge, reason_key
|
||
|
|
|
||
|
|
RULE = "quote-sheet-correct"
|
||
|
|
SECOND = "two-questions-at-most"
|
||
|
|
FOR_RULE = "S4.2: the quote sheet lacks the party bands"
|
||
|
|
|
||
|
|
|
||
|
|
def drill(session, deck=True):
|
||
|
|
delivered = [{"name": "HL-Q-1011-2P.pptx"}] if deck else [{"name": "notes.md"}]
|
||
|
|
return {"session": session, "exchanges": [{"delivered": delivered}]}
|
||
|
|
|
||
|
|
|
||
|
|
def row(
|
||
|
|
target, count=1, *, intervention=False, kind="participant.result", session="family", scope="root", reason=FOR_RULE
|
||
|
|
):
|
||
|
|
return {
|
||
|
|
"scope": scope,
|
||
|
|
"kind": kind,
|
||
|
|
"target": target,
|
||
|
|
"count": count,
|
||
|
|
"intervention": intervention,
|
||
|
|
"session": session,
|
||
|
|
"summary": reason,
|
||
|
|
"reasons": {reason: count},
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def round_(number, *rows, deck=True, partial=False, waiting=()):
|
||
|
|
return {
|
||
|
|
"number": number,
|
||
|
|
"partial": partial,
|
||
|
|
"drills": [drill("family", deck), drill("student")],
|
||
|
|
"evaluation": {"items": [{"criterion": RULE, "result": "pass", "session": "family"}]},
|
||
|
|
"analysis": {"waiting_on_material": list(waiting)},
|
||
|
|
"mechanism_evidence": list(rows),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def moment(number, result):
|
||
|
|
return {"round": number, "result": result}
|
||
|
|
|
||
|
|
|
||
|
|
def entry(timeline, landings, severity="red_line", criterion=RULE):
|
||
|
|
return {
|
||
|
|
"criterion": criterion,
|
||
|
|
"severity": severity,
|
||
|
|
"rule": "S4: every quote is a complete quote sheet",
|
||
|
|
"timeline": timeline,
|
||
|
|
"sedimented_in": landings,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def landing(round_number, target="action.review", scope="root"):
|
||
|
|
return {"round": round_number, "scope": scope, "facet": target.split(".")[0], "target": target, "paths": []}
|
||
|
|
|
||
|
|
|
||
|
|
CLEAN = {"valid": True, "breaches": 0, "logs": 3, "findings": []}
|
||
|
|
|
||
|
|
|
||
|
|
def record(rounds, ledger, isolation=CLEAN):
|
||
|
|
return {
|
||
|
|
"run": {"name": "hoh-staged", "chain": "staged", "status": "finished"},
|
||
|
|
"rounds": rounds,
|
||
|
|
"ledger": ledger,
|
||
|
|
"isolation": isolation,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
FAILED_THEN_HELD = [moment(1, "fail"), moment(2, "pass"), moment(3, "pass")]
|
||
|
|
|
||
|
|
|
||
|
|
def two_rules(first=("action.review",), second="memory.prompt", severity="red_line"):
|
||
|
|
return [
|
||
|
|
entry(FAILED_THEN_HELD, [landing(1, target) for target in first], severity),
|
||
|
|
entry(FAILED_THEN_HELD, [landing(1, second)], "standard", SECOND),
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def test_two_cultivated_rules_with_one_held_by_an_intervening_mechanism_qualify_and_prompts_alone_do_not():
|
||
|
|
rounds = [round_(1), round_(2, row("action.review", 2, intervention=True)), round_(3, row("action.review"))]
|
||
|
|
verdict = judge(record(rounds, two_rules()))
|
||
|
|
assert verdict["verdict"] == "qualifies" and all(check["ok"] for check in verdict["checks"].values())
|
||
|
|
assert [(item["criterion"], item["failed_in"], item["held_in"]) for item in verdict["cultivated"]] == [
|
||
|
|
(RULE, [1], [2, 3]),
|
||
|
|
(SECOND, [1], [2, 3]),
|
||
|
|
]
|
||
|
|
(case,) = verdict["cases"]
|
||
|
|
assert (case["failed_in"], case["held_in"], case["mechanism_rows"], case["interventions"]) == ([1], [2, 3], 3, 2)
|
||
|
|
prompted = judge(record(rounds, two_rules(first=("memory.prompt",))))
|
||
|
|
assert prompted["verdict"] == "partial" and not prompted["checks"]["hard"]["ok"]
|
||
|
|
assert verdict["score"] > prompted["score"]
|
||
|
|
one = judge(record(rounds, two_rules()[:1]))
|
||
|
|
assert one["verdict"] == "partial" and not one["checks"]["cultivated"]["ok"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_red_line_missed_in_the_last_round_fails_the_core_but_one_standard_miss_is_tolerated():
|
||
|
|
rounds = [round_(1), round_(2, row("action.review", intervention=True)), round_(3)]
|
||
|
|
relapsed = [moment(1, "fail"), moment(2, "fail"), moment(3, "fail")]
|
||
|
|
red = judge(record(rounds, [*two_rules(), entry(relapsed, [], criterion="no-internal-chatter")]))
|
||
|
|
assert red["verdict"] == "partial" and "no-internal-chatter" in red["checks"]["core"]["detail"]
|
||
|
|
standard = judge(record(rounds, [*two_rules(), entry(relapsed, [], "standard", "deck-aesthetics")]))
|
||
|
|
assert standard["checks"]["core"]["ok"] and standard["verdict"] == "qualifies"
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_run_that_left_its_bounds_is_invalid_and_an_unscanned_one_cannot_qualify():
|
||
|
|
rounds = [round_(1), round_(2, row("action.review", 2, intervention=True)), round_(3, row("action.review"))]
|
||
|
|
ledger = two_rules()
|
||
|
|
breach = {"valid": False, "breaches": 1, "logs": 3, "findings": [{"kind": "broad_search", "path": "/"}]}
|
||
|
|
left = judge(record(rounds, ledger, breach))
|
||
|
|
assert left["verdict"] == "invalid" and "broad_search" in left["checks"]["isolated"]["detail"]
|
||
|
|
unscanned = judge(record(rounds, ledger, None))
|
||
|
|
assert unscanned["verdict"] == "partial" and not unscanned["checks"]["isolated"]["ok"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_only_rows_of_the_sedimented_target_in_the_rules_drills_count():
|
||
|
|
elsewhere = [
|
||
|
|
round_(1),
|
||
|
|
round_(2, row("capability.strategy", 50)),
|
||
|
|
round_(3, row("action.review", session="student")),
|
||
|
|
]
|
||
|
|
assert cases(record(elsewhere, [entry(FAILED_THEN_HELD, [landing(1, "planning.skills")])])) == ([], [])
|
||
|
|
assert cases(record(elsewhere, [entry(FAILED_THEN_HELD, [landing(1)])])) == ([], [])
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_child_harness_mechanism_counts_only_for_that_childs_sedimentation():
|
||
|
|
child = row("action.review", 2, intervention=True, scope="Raven-PPT")
|
||
|
|
rounds = [round_(1), round_(2, child), round_(3, child)]
|
||
|
|
assert cases(record(rounds, [entry(FAILED_THEN_HELD, [landing(1)])])) == ([], [])
|
||
|
|
(case,), _ = cases(record(rounds, [entry(FAILED_THEN_HELD, [landing(1, scope="Raven-PPT")])]))
|
||
|
|
assert case["sedimented"]["scope"] == "Raven-PPT" and case["interventions"] == 4
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_rule_sedimented_through_several_targets_is_one_case():
|
||
|
|
rounds = [
|
||
|
|
round_(1),
|
||
|
|
round_(2, row("action.review", intervention=True), row("planning.strategy", 4, intervention=True)),
|
||
|
|
round_(3),
|
||
|
|
]
|
||
|
|
(case,), _ = cases(record(rounds, [entry(FAILED_THEN_HELD, [landing(1), landing(1, "planning.strategy")])]))
|
||
|
|
assert case["targets"] == ["action.review", "planning.strategy"] and case["mechanism_rows"] == 5
|
||
|
|
|
||
|
|
|
||
|
|
def test_only_an_intervention_whose_reason_enforces_the_rule_counts():
|
||
|
|
other = row("action.review", 3, intervention=True, reason="G2: this message asks three questions")
|
||
|
|
rounds = [round_(1), round_(2, other), round_(3, other)]
|
||
|
|
ledger = [entry(FAILED_THEN_HELD, [landing(1)])]
|
||
|
|
assert cases(record(rounds, ledger)) == ([], [])
|
||
|
|
(key,) = candidates(record(rounds, ledger))
|
||
|
|
assert key == reason_key("root", "action.review", "G2: this message asks three questions")
|
||
|
|
attributed = {**record(rounds, ledger), "attribution": {"links": {key: [RULE]}}}
|
||
|
|
(case,), _ = cases(attributed)
|
||
|
|
assert case["interventions"] == 6
|
||
|
|
wrong = {**record(rounds, ledger), "attribution": {"links": {key: []}}}
|
||
|
|
assert cases(wrong) == ([], [])
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_failure_that_only_waited_on_material_is_not_the_before():
|
||
|
|
acted = row("action.review", intervention=True)
|
||
|
|
rounds = [round_(1, waiting=[RULE]), round_(2, acted), round_(3, acted)]
|
||
|
|
assert cases(record(rounds, [entry(FAILED_THEN_HELD, [landing(1)])])) == ([], [])
|
||
|
|
assert cultivated(record(rounds, [entry(FAILED_THEN_HELD, [landing(1)])])) == []
|
||
|
|
|
||
|
|
|
||
|
|
def test_prompt_changes_a_relapse_or_a_silent_mechanism_are_not_hard_cases():
|
||
|
|
rounds = [round_(1), round_(2, row("memory.prompt"), row("action.review")), round_(3, row("action.review"))]
|
||
|
|
relapse = [moment(1, "fail"), moment(2, "pass"), moment(3, "fail")]
|
||
|
|
assert cases(record(rounds, [entry(FAILED_THEN_HELD, [landing(1, "memory.prompt")])])) == ([], [])
|
||
|
|
assert cases(record(rounds, [entry(relapse, [landing(1)])])) == ([], [])
|
||
|
|
assert cultivated(record(rounds, [entry(relapse, [landing(1)])])) == []
|
||
|
|
silent = [round_(1), round_(2), round_(3)]
|
||
|
|
quiet = judge(record(silent, [entry(FAILED_THEN_HELD, [landing(1)])]))
|
||
|
|
assert quiet["cases"] == [] and [item["criterion"] for item in quiet["cultivated"]] == [RULE]
|
||
|
|
assert judge(record(silent, [entry(relapse, [landing(1)])]))["verdict"] == "none"
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_mechanism_that_never_intervened_is_no_hard_case_and_a_refused_tool_counts_for_action():
|
||
|
|
shaped = [round_(1), round_(2, row("action.tool_gates", kind="component.load")), round_(3)]
|
||
|
|
assert cases(record(shaped, [entry(FAILED_THEN_HELD, [landing(1, "action.tool_gates")])])) == ([], [])
|
||
|
|
refused = [*shaped[:2], round_(3, row("tool:quote_sheet", kind="tool.refused", intervention=True))]
|
||
|
|
(case,), _ = cases(record(refused, [entry(FAILED_THEN_HELD, [landing(1, "action.tool_gates")])]))
|
||
|
|
assert case["interventions"] == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_an_onboarding_mechanism_that_intervened_and_held_is_a_weaker_case():
|
||
|
|
rounds = [round_(1, row("planning.strategy", 2, intervention=True)), round_(2), round_(3)]
|
||
|
|
timeline = [moment(0, "unknown"), moment(1, "pass"), moment(2, "pass"), moment(3, "pass")]
|
||
|
|
found, held = cases(record(rounds, [entry(timeline, [landing(0, "planning.strategy")])]))
|
||
|
|
assert found == [] and held[0]["interventions"] == 2
|
||
|
|
assert judge(record(rounds, [entry(timeline, [landing(0, "planning.strategy")])]))["verdict"] == "partial"
|
||
|
|
# Shaping the Harness from the owner's materials is value too: with two rules cultivated by prompts, the onboarding
|
||
|
|
# mechanism that enforced a third rule meets the planning-or-action condition.
|
||
|
|
shaped = [
|
||
|
|
entry(timeline, [landing(0, "planning.strategy")], criterion="ai-identity-disclosed"),
|
||
|
|
*two_rules(("memory.prompt",)),
|
||
|
|
]
|
||
|
|
verdict = judge(record(rounds, shaped))
|
||
|
|
assert verdict["checks"]["hard"]["ok"] and verdict["verdict"] == "qualifies"
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_blocked_last_round_or_too_few_rounds_keeps_a_case_partial():
|
||
|
|
acted = row("action.review", intervention=True)
|
||
|
|
blocked = judge(record([round_(1), round_(2, acted), round_(3, acted, deck=False)], two_rules()))
|
||
|
|
assert blocked["verdict"] == "partial" and not blocked["checks"]["delivered"]["ok"]
|
||
|
|
short = judge(record([round_(1), round_(2, acted, partial=True)], [entry(FAILED_THEN_HELD[:1], [landing(0)])]))
|
||
|
|
assert short["verdict"] == "none" and not short["checks"]["rounds"]["ok"]
|
||
|
|
|
||
|
|
|
||
|
|
async def test_a_model_names_the_rules_each_intervention_reason_enforces_within_what_its_target_was_built_for():
|
||
|
|
from experimental.simulation.attribution import NAME, attribute
|
||
|
|
from raven.contracts.llm_provider import LLMResponse, ToolCallRequest
|
||
|
|
|
||
|
|
class Provider:
|
||
|
|
def __init__(self, *arguments):
|
||
|
|
self.responses = [LLMResponse(content=None, tool_calls=[ToolCallRequest(NAME, NAME, a)]) for a in arguments]
|
||
|
|
self.requests = []
|
||
|
|
|
||
|
|
async def chat_with_retry(self, **kwargs):
|
||
|
|
self.requests.append(kwargs)
|
||
|
|
return self.responses.pop(0)
|
||
|
|
|
||
|
|
reason = "This message asks three questions; keep at most two."
|
||
|
|
acted = row("action.review", 2, intervention=True, reason=reason)
|
||
|
|
run = record([round_(1), round_(2, acted), round_(3, acted)], [entry(FAILED_THEN_HELD, [landing(1)])])
|
||
|
|
provider = Provider({"links": [{"id": "r1", "rules": ["deck-aesthetics"]}]}, {"links": [{"id": "r1", "rules": []}]})
|
||
|
|
result = await attribute(run, provider)
|
||
|
|
assert result == {"links": {reason_key("root", "action.review", reason): []}, "interventions": 1}
|
||
|
|
packet = provider.requests[0]["messages"][1]["content"]
|
||
|
|
assert reason in packet and RULE in packet and "may_enforce" in packet
|
||
|
|
errors = [message["content"] for message in provider.requests[1]["messages"] if message["role"] == "tool"]
|
||
|
|
assert "may name only" in errors[0]
|
||
|
|
assert await attribute(record([round_(1)], []), provider) == {"links": {}, "interventions": 0}
|