bench/tests/test_verdict.py

"""The contamination gate.

A probe nobody reads is the same as no probe. These assert the gate does something
consequential — it takes the signal score away — rather than recording a number beside it.

    python3 -m unittest discover -s tests -v
"""

from __future__ import annotations

import sys
import unittest
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

from kbench.results import QualityResult, compute_verdict  # noqa: E402


def quality(tier: str, accuracy: float, task: str = "t") -> QualityResult:
    return QualityResult(task=task, tier=tier, metrics={"accuracy": accuracy})


class ContaminationGate(unittest.TestCase):
    def test_a_clean_probe_leaves_the_signal_score_readable(self):
        v = compute_verdict([quality("signal", 0.81), quality("canary", 0.0)], None)
        self.assertEqual(v["contamination"], "clean")
        self.assertEqual(v["signal_score"], 0.81)
        self.assertEqual(v["canary_score"], 0.0)

    def test_a_fired_probe_removes_the_signal_score_entirely(self):
        # The behaviour that matters. Anything reading verdict["signal_score"] — the results
        # table, a comparison, a published card — must get None rather than a memorised number
        # with a warning attached somewhere else.
        v = compute_verdict([quality("signal", 0.94), quality("canary", 0.67)], None)
        self.assertEqual(v["contamination"], "detected")
        self.assertIsNone(v["signal_score"])

    def test_the_withheld_score_is_kept_for_forensics(self):
        v = compute_verdict([quality("signal", 0.94), quality("canary", 0.67)], None)
        self.assertEqual(v["signal_score_unverified"], 0.94)

    def test_even_one_probe_in_many_is_enough_to_void_the_run(self):
        # Partial memorisation is still memorisation; there is no safe amount.
        v = compute_verdict(
            [quality("signal", 0.9), quality("canary", 0.0, "a"), quality("canary", 1.0, "b")],
            None,
        )
        self.assertEqual(v["contamination"], "detected")
        self.assertIsNone(v["signal_score"])

    def test_no_probe_is_reported_as_unverified_not_clean(self):
        # "Nobody checked" and "checked and clean" are different claims, and conflating them is
        # how an unverified number gets quoted as a verified one.
        v = compute_verdict([quality("signal", 0.81)], None)
        self.assertEqual(v["contamination"], "unverified")
        self.assertIsNone(v["canary_score"])
        self.assertEqual(v["signal_score"], 0.81, "an unverified run keeps its score")

    def test_a_failed_probe_does_not_count_as_a_clean_one(self):
        errored = QualityResult(task="c", tier="canary", metrics={}, error="judge timed out")
        v = compute_verdict([quality("signal", 0.81), errored], None)
        self.assertEqual(v["contamination"], "unverified")

    def test_contamination_does_not_touch_the_perf_half(self):
        # Memorisation says nothing about throughput; a contaminated run is still valid
        # serving-performance evidence, and discarding it would be its own error.
        from kbench.results import PerfPoint, PerfResult

        perf = PerfResult(
            engine="vllm",
            points=[
                PerfPoint(
                    concurrency=1,
                    input_tokens=512,
                    output_tokens=128,
                    n_requests=4,
                    completed=4,
                    failed=0,
                    duration_s=10.0,
                    output_tps_total=30.0,
                    output_tps_per_stream=30.0,
                )
            ],
        )
        v = compute_verdict([quality("signal", 0.9), quality("canary", 1.0)], perf)
        self.assertIsNone(v["signal_score"])
        self.assertEqual(v["single_stream_tps"], 30.0)
        self.assertTrue(v["interactive_viable"])


if __name__ == "__main__":
    unittest.main()


class ScoreCardsAreValidJson(unittest.TestCase):
    """A score card only Python can read is not a durable record.

    Regression: an ungraded rubric sample (no judge configured) reached the file as a bare
    `NaN`. Python's json emits and accepts it; JSON.parse, Go and serde all reject it. The
    first committed card was unparseable by the website meant to render it.
    """

    def test_non_finite_scores_serialize_as_null(self):
        from kbench.results import _json_safe

        nan, inf = float("nan"), float("inf")
        self.assertIsNone(_json_safe(nan))
        self.assertIsNone(_json_safe(inf))
        self.assertIsNone(_json_safe(-inf))
        self.assertEqual(_json_safe(0.5), 0.5)
        self.assertEqual(
            _json_safe({"a": [1.0, nan], "b": {"c": inf}}),
            {"a": [1.0, None], "b": {"c": None}},
        )

    def test_committed_cards_parse_under_strict_json(self):
        import json
        from pathlib import Path

        def strict(c):
            raise ValueError(f"non-JSON constant {c!r}")

        results = Path(__file__).resolve().parent.parent / "results"
        cards = sorted(results.glob("*.json")) if results.exists() else []
        if not cards:
            self.skipTest("no committed score cards yet")
        for card in cards:
            with self.subTest(card=card.name):
                json.loads(card.read_text(), parse_constant=strict)


class UngradedSamplesAreNotFailures(unittest.TestCase):
    def _normalizer(self):
        # kbench.run pulls in perf -> httpx. The data-only tests are meant to run on a bare
        # interpreter (that is what a fresh clone and the publish gate both have), so this
        # skips rather than errors when the serving deps are absent.
        try:
            from kbench.run import _normalize_sample_score
        except ImportError as exc:
            self.skipTest(f"serving deps unavailable: {exc}")
        return _normalize_sample_score

    def test_nan_score_is_marked_ungraded(self):
        _normalize_sample_score = self._normalizer()

        score, passed, extra = _normalize_sample_score(float("nan"), None)
        self.assertNotEqual(score, score, "NaN should be preserved in-memory")
        self.assertFalse(passed)
        self.assertTrue(extra.get("ungraded"), "an ungraded sample must say so")

    def test_ordinary_zero_is_a_failure_not_ungraded(self):
        _normalize_sample_score = self._normalizer()

        _, passed, extra = _normalize_sample_score(0.0, None)
        self.assertFalse(passed)
        self.assertNotIn("ungraded", extra)