"""The contamination gate. A probe nobody reads is the same as no probe. These assert the gate does something consequential — it takes the signal score away — rather than recording a number beside it. python3 -m unittest discover -s tests -v """ from __future__ import annotations import sys import unittest from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) from kbench.results import QualityResult, compute_verdict # noqa: E402 def quality(tier: str, accuracy: float, task: str = "t") -> QualityResult: return QualityResult(task=task, tier=tier, metrics={"accuracy": accuracy}) class ContaminationGate(unittest.TestCase): def test_a_clean_probe_leaves_the_signal_score_readable(self): v = compute_verdict([quality("signal", 0.81), quality("canary", 0.0)], None) self.assertEqual(v["contamination"], "clean") self.assertEqual(v["signal_score"], 0.81) self.assertEqual(v["canary_score"], 0.0) def test_a_fired_probe_removes_the_signal_score_entirely(self): # The behaviour that matters. Anything reading verdict["signal_score"] — the results # table, a comparison, a published card — must get None rather than a memorised number # with a warning attached somewhere else. v = compute_verdict([quality("signal", 0.94), quality("canary", 0.67)], None) self.assertEqual(v["contamination"], "detected") self.assertIsNone(v["signal_score"]) def test_the_withheld_score_is_kept_for_forensics(self): v = compute_verdict([quality("signal", 0.94), quality("canary", 0.67)], None) self.assertEqual(v["signal_score_unverified"], 0.94) def test_even_one_probe_in_many_is_enough_to_void_the_run(self): # Partial memorisation is still memorisation; there is no safe amount. v = compute_verdict( [quality("signal", 0.9), quality("canary", 0.0, "a"), quality("canary", 1.0, "b")], None, ) self.assertEqual(v["contamination"], "detected") self.assertIsNone(v["signal_score"]) def test_no_probe_is_reported_as_unverified_not_clean(self): # "Nobody checked" and "checked and clean" are different claims, and conflating them is # how an unverified number gets quoted as a verified one. v = compute_verdict([quality("signal", 0.81)], None) self.assertEqual(v["contamination"], "unverified") self.assertIsNone(v["canary_score"]) self.assertEqual(v["signal_score"], 0.81, "an unverified run keeps its score") def test_a_failed_probe_does_not_count_as_a_clean_one(self): errored = QualityResult(task="c", tier="canary", metrics={}, error="judge timed out") v = compute_verdict([quality("signal", 0.81), errored], None) self.assertEqual(v["contamination"], "unverified") def test_contamination_does_not_touch_the_perf_half(self): # Memorisation says nothing about throughput; a contaminated run is still valid # serving-performance evidence, and discarding it would be its own error. from kbench.results import PerfPoint, PerfResult perf = PerfResult( engine="vllm", points=[ PerfPoint( concurrency=1, input_tokens=512, output_tokens=128, n_requests=4, completed=4, failed=0, duration_s=10.0, output_tps_total=30.0, output_tps_per_stream=30.0, ) ], ) v = compute_verdict([quality("signal", 0.9), quality("canary", 1.0)], perf) self.assertIsNone(v["signal_score"]) self.assertEqual(v["single_stream_tps"], 30.0) self.assertTrue(v["interactive_viable"]) if __name__ == "__main__": unittest.main() class ScoreCardsAreValidJson(unittest.TestCase): """A score card only Python can read is not a durable record. Regression: an ungraded rubric sample (no judge configured) reached the file as a bare `NaN`. Python's json emits and accepts it; JSON.parse, Go and serde all reject it. The first committed card was unparseable by the website meant to render it. """ def test_non_finite_scores_serialize_as_null(self): from kbench.results import _json_safe nan, inf = float("nan"), float("inf") self.assertIsNone(_json_safe(nan)) self.assertIsNone(_json_safe(inf)) self.assertIsNone(_json_safe(-inf)) self.assertEqual(_json_safe(0.5), 0.5) self.assertEqual( _json_safe({"a": [1.0, nan], "b": {"c": inf}}), {"a": [1.0, None], "b": {"c": None}}, ) def test_committed_cards_parse_under_strict_json(self): import json from pathlib import Path def strict(c): raise ValueError(f"non-JSON constant {c!r}") results = Path(__file__).resolve().parent.parent / "results" cards = sorted(results.glob("*.json")) if results.exists() else [] if not cards: self.skipTest("no committed score cards yet") for card in cards: with self.subTest(card=card.name): json.loads(card.read_text(), parse_constant=strict) class UngradedSamplesAreNotFailures(unittest.TestCase): def _normalizer(self): # kbench.run pulls in perf -> httpx. The data-only tests are meant to run on a bare # interpreter (that is what a fresh clone and the publish gate both have), so this # skips rather than errors when the serving deps are absent. try: from kbench.run import _normalize_sample_score except ImportError as exc: self.skipTest(f"serving deps unavailable: {exc}") return _normalize_sample_score def test_nan_score_is_marked_ungraded(self): _normalize_sample_score = self._normalizer() score, passed, extra = _normalize_sample_score(float("nan"), None) self.assertNotEqual(score, score, "NaN should be preserved in-memory") self.assertFalse(passed) self.assertTrue(extra.get("ungraded"), "an ungraded sample must say so") def test_ordinary_zero_is_a_failure_not_ungraded(self): _normalize_sample_score = self._normalizer() _, passed, extra = _normalize_sample_score(0.0, None) self.assertFalse(passed) self.assertNotIn("ungraded", extra)