Lumbridge Bench

tests/test_verdict.py

@@ -0,0 +1,158 @@
+"""The contamination gate.
+
+A probe nobody reads is the same as no probe. These assert the gate does something
+consequential — it takes the signal score away — rather than recording a number beside it.
+
+    python3 -m unittest discover -s tests -v
+"""
+
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from kbench.results import QualityResult, compute_verdict  # noqa: E402
+
+
+def quality(tier: str, accuracy: float, task: str = "t") -> QualityResult:
+    return QualityResult(task=task, tier=tier, metrics={"accuracy": accuracy})
+
+
+class ContaminationGate(unittest.TestCase):
+    def test_a_clean_probe_leaves_the_signal_score_readable(self):
+        v = compute_verdict([quality("signal", 0.81), quality("canary", 0.0)], None)
+        self.assertEqual(v["contamination"], "clean")
+        self.assertEqual(v["signal_score"], 0.81)
+        self.assertEqual(v["canary_score"], 0.0)
+
+    def test_a_fired_probe_removes_the_signal_score_entirely(self):
+        # The behaviour that matters. Anything reading verdict["signal_score"] — the results
+        # table, a comparison, a published card — must get None rather than a memorised number
+        # with a warning attached somewhere else.
+        v = compute_verdict([quality("signal", 0.94), quality("canary", 0.67)], None)
+        self.assertEqual(v["contamination"], "detected")
+        self.assertIsNone(v["signal_score"])
+
+    def test_the_withheld_score_is_kept_for_forensics(self):
+        v = compute_verdict([quality("signal", 0.94), quality("canary", 0.67)], None)
+        self.assertEqual(v["signal_score_unverified"], 0.94)
+
+    def test_even_one_probe_in_many_is_enough_to_void_the_run(self):
+        # Partial memorisation is still memorisation; there is no safe amount.
+        v = compute_verdict(
+            [quality("signal", 0.9), quality("canary", 0.0, "a"), quality("canary", 1.0, "b")],
+            None,
+        )
+        self.assertEqual(v["contamination"], "detected")
+        self.assertIsNone(v["signal_score"])
+
+    def test_no_probe_is_reported_as_unverified_not_clean(self):
+        # "Nobody checked" and "checked and clean" are different claims, and conflating them is
+        # how an unverified number gets quoted as a verified one.
+        v = compute_verdict([quality("signal", 0.81)], None)
+        self.assertEqual(v["contamination"], "unverified")
+        self.assertIsNone(v["canary_score"])
+        self.assertEqual(v["signal_score"], 0.81, "an unverified run keeps its score")
+
+    def test_a_failed_probe_does_not_count_as_a_clean_one(self):
+        errored = QualityResult(task="c", tier="canary", metrics={}, error="judge timed out")
+        v = compute_verdict([quality("signal", 0.81), errored], None)
+        self.assertEqual(v["contamination"], "unverified")
+
+    def test_contamination_does_not_touch_the_perf_half(self):
+        # Memorisation says nothing about throughput; a contaminated run is still valid
+        # serving-performance evidence, and discarding it would be its own error.
+        from kbench.results import PerfPoint, PerfResult
+
+        perf = PerfResult(
+            engine="vllm",
+            points=[
+                PerfPoint(
+                    concurrency=1,
+                    input_tokens=512,
+                    output_tokens=128,
+                    n_requests=4,
+                    completed=4,
+                    failed=0,
+                    duration_s=10.0,
+                    output_tps_total=30.0,
+                    output_tps_per_stream=30.0,
+                )
+            ],
+        )
+        v = compute_verdict([quality("signal", 0.9), quality("canary", 1.0)], perf)
+        self.assertIsNone(v["signal_score"])
+        self.assertEqual(v["single_stream_tps"], 30.0)
+        self.assertTrue(v["interactive_viable"])
+
+
+if __name__ == "__main__":
+    unittest.main()
+
+
+class ScoreCardsAreValidJson(unittest.TestCase):
+    """A score card only Python can read is not a durable record.
+
+    Regression: an ungraded rubric sample (no judge configured) reached the file as a bare
+    `NaN`. Python's json emits and accepts it; JSON.parse, Go and serde all reject it. The
+    first committed card was unparseable by the website meant to render it.
+    """
+
+    def test_non_finite_scores_serialize_as_null(self):
+        from kbench.results import _json_safe
+
+        nan, inf = float("nan"), float("inf")
+        self.assertIsNone(_json_safe(nan))
+        self.assertIsNone(_json_safe(inf))
+        self.assertIsNone(_json_safe(-inf))
+        self.assertEqual(_json_safe(0.5), 0.5)
+        self.assertEqual(
+            _json_safe({"a": [1.0, nan], "b": {"c": inf}}),
+            {"a": [1.0, None], "b": {"c": None}},
+        )
+
+    def test_committed_cards_parse_under_strict_json(self):
+        import json
+        from pathlib import Path
+
+        def strict(c):
+            raise ValueError(f"non-JSON constant {c!r}")
+
+        results = Path(__file__).resolve().parent.parent / "results"
+        cards = sorted(results.glob("*.json")) if results.exists() else []
+        if not cards:
+            self.skipTest("no committed score cards yet")
+        for card in cards:
+            with self.subTest(card=card.name):
+                json.loads(card.read_text(), parse_constant=strict)
+
+
+class UngradedSamplesAreNotFailures(unittest.TestCase):
+    def _normalizer(self):
+        # kbench.run pulls in perf -> httpx. The data-only tests are meant to run on a bare
+        # interpreter (that is what a fresh clone and the publish gate both have), so this
+        # skips rather than errors when the serving deps are absent.
+        try:
+            from kbench.run import _normalize_sample_score
+        except ImportError as exc:
+            self.skipTest(f"serving deps unavailable: {exc}")
+        return _normalize_sample_score
+
+    def test_nan_score_is_marked_ungraded(self):
+        _normalize_sample_score = self._normalizer()
+
+        score, passed, extra = _normalize_sample_score(float("nan"), None)
+        self.assertNotEqual(score, score, "NaN should be preserved in-memory")
+        self.assertFalse(passed)
+        self.assertTrue(extra.get("ungraded"), "an ungraded sample must say so")
+
+    def test_ordinary_zero_is_a_failure_not_ungraded(self):
+        _normalize_sample_score = self._normalizer()
+
+        _, passed, extra = _normalize_sample_score(0.0, None)
+        self.assertFalse(passed)
+        self.assertNotIn("ungraded", extra)