lumbridge-public/bench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lumbridge Bench
CI / verify (push) Successful in 24s
Details
CI / deploy (push) Failing after 1m14s
Details

Browse Source
This commit is contained in:
Karti Tripathi
2026-08-04 00:44:07 -07:00
commit 006feee0f7
65 changed files with 13516 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_catalog.py
+198
View File
@@ -0,0 +1,198 @@
"""The catalog must describe things that exist.
This guards one specific failure that has now happened twice in this repo, in two different
shapes: a control that cannot fire. First the canary GUID that was never embedded in a prompt,
then the canary task that was never registered — in both cases every surface reported success
because nothing raised, and the thing simply never ran.
The pattern is the same each time: a component is *described* somewhere and *wired* nowhere,
and no test notices because nothing errors. So these tests check the wiring itself — that every
catalog entry resolves to a real task in a real module over a real data file.
Deliberately parses source rather than importing: `kbench.tasks.signal` imports `inspect_ai` at
module scope, and a structural check that only runs when a heavy optional dependency is
installed is a check that does not run.
python3 -m unittest discover -s tests -v
"""
from __future__ import annotations
import ast
import re
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
CATALOG_SRC = (ROOT / "kbench" / "tasks" / "__init__.py").read_text()
def catalog_entries() -> dict[str, dict[str, str]]:
"""Pull (name -> {field: literal}) out of the CATALOG without importing it."""
tree = ast.parse(CATALOG_SRC)
entries: dict[str, dict[str, str]] = {}
for node in ast.walk(tree):
if not isinstance(node, ast.AnnAssign) or getattr(node.target, "id", None) != "CATALOG":
continue
assert isinstance(node.value, ast.Dict)
for key, value in zip(node.value.keys, node.value.values):
assert isinstance(key, ast.Constant) and isinstance(value, ast.Call)
fields = {
kw.arg: kw.value.value
for kw in value.keywords
if isinstance(kw.value, ast.Constant)
}
entries[key.value] = fields
return entries
ENTRIES = catalog_entries()
class CatalogWiring(unittest.TestCase):
def test_the_catalog_was_parsed_at_all(self):
self.assertTrue(ENTRIES, "could not read CATALOG — this test is not testing anything")
def test_the_key_matches_the_spec_name(self):
for key, fields in ENTRIES.items():
with self.subTest(task=key):
self.assertEqual(fields.get("name"), key)
def test_every_local_task_resolves_to_a_defined_function(self):
# `module.py@func` must name a function that actually exists and is decorated @task.
for key, fields in ENTRIES.items():
ref = fields.get("inspect_task", "")
if "@" not in ref or ref.startswith("inspect_evals/"):
continue
module_path, func = ref.split("@", 1)
with self.subTest(task=key):
path = ROOT / module_path
self.assertTrue(path.exists(), f"{ref}: {module_path} does not exist")
src = path.read_text()
self.assertRegex(
src,
rf"@task\s*\ndef {re.escape(func)}\s*\(",
f"{ref}: no @task-decorated def {func}() in {module_path}",
)
def test_every_local_task_has_a_data_file_with_samples(self):
# A registered task over an empty family raises only at run time, after a target has
# been stood up and paid for.
from kbench.schema import read_records # local import: stdlib-only module
tier_for = {"signal": "private", "canary": "canary", "example": "public"}
for key, fields in ENTRIES.items():
ref = fields.get("inspect_task", "")
if "@" not in ref or ref.startswith("inspect_evals/"):
continue
tier = tier_for.get(fields.get("tier", ""))
if tier is None:
continue
family = ref.split("@", 1)[1]
path = ROOT / "data" / tier / f"{family}.jsonl"
with self.subTest(task=key):
# A missing tier DIRECTORY means the published tree, which ships neither
# the signal set nor the canary — expected, and not a wiring bug. A missing
# FILE inside a directory that exists is the real defect this guards: a task
# registered in the catalog with nothing behind it.
if not path.parent.exists():
self.skipTest(f"{tier}/ not present (expected in the published tree)")
self.assertTrue(path.exists(), f"{key}: no data file at {path}")
self.assertTrue(read_records(path), f"{key}: {path} has no samples")
def test_declared_sample_count_matches_the_file(self):
# A drifting count is how "we ran the whole set" quietly becomes false.
from kbench.schema import read_records
tier_for = {"signal": "private", "canary": "canary", "example": "public"}
for key, fields in ENTRIES.items():
ref = fields.get("inspect_task", "")
declared = fields.get("dataset_samples")
tier = tier_for.get(fields.get("tier", ""))
if "@" not in ref or declared is None or tier is None:
continue
family = ref.split("@", 1)[1]
path = ROOT / "data" / tier / f"{family}.jsonl"
if not path.exists():
continue
actual = len([r for _, r in read_records(path) if r["split"] == "test"])
with self.subTest(task=key):
self.assertEqual(
actual, declared, f"{key}: catalog says {declared} samples, file has {actual}"
)
class ContaminationIsWired(unittest.TestCase):
"""The gate is only real if the probe runs and something reads the result."""
def test_a_canary_tier_task_is_registered(self):
canary = [k for k, f in ENTRIES.items() if f.get("tier") == "canary"]
self.assertTrue(
canary,
"no canary-tier task in the CATALOG — the contamination gate would report "
"'unverified' forever, which is indistinguishable from having no probe at all",
)
def test_the_verdict_reads_the_canary_tier(self):
src = (ROOT / "kbench" / "results.py").read_text()
self.assertIn('tier == "canary"', src, "compute_verdict ignores the canary tier")
self.assertIn(
'verdict["signal_score"] = None',
src,
"a fired probe must remove the signal score, not merely annotate it",
)
def test_at_least_one_signal_family_exists(self):
signal = [k for k, f in ENTRIES.items() if f.get("tier") == "signal"]
self.assertTrue(signal, "no signal-tier family — the bench measures nothing of its own")
class SamplingResolution(unittest.TestCase):
"""Registry sampling overrides the default; the card and the run share one source."""
def setUp(self):
try:
from kbench.run import DEFAULT_SAMPLING, resolve_sampling
except ImportError as exc:
self.skipTest(f"serving deps unavailable: {exc}")
self.resolve, self.default = resolve_sampling, DEFAULT_SAMPLING
def test_default_is_not_greedy(self):
# Greedy decoding is the intuitive choice for reproducibility and makes reasoning
# models repeat forever -- a 90s probe ran past 12 minutes at temperature 0.
# Reproducibility comes from the seed instead.
self.assertGreater(self.default["temperature"], 0.0)
self.assertIn("seed", self.default)
self.assertIn("max_tokens", self.default)
def test_registry_values_win_over_defaults(self):
from kbench.registry import load_registry
target = load_registry().target("brain")
resolved = self.resolve(target)
for key, value in (target.serving.sampling or {}).items():
with self.subTest(key=key):
self.assertEqual(resolved[key], value)
def test_every_key_is_a_real_generate_config_field(self):
# A sampling key inspect does not know is not ignored -- it aborts the run. Worse,
# a key that IS accepted but silently dropped would look pinned and still sample.
try:
from inspect_ai.model import GenerateConfig
except ImportError as exc:
self.skipTest(f"inspect-ai unavailable: {exc}")
from kbench.registry import load_registry
fields = set(GenerateConfig.model_fields)
for target in load_registry().targets.values():
resolved = self.resolve(target)
for key in resolved:
with self.subTest(target=target.id, key=key):
self.assertIn(key, fields, f"{key} is not a GenerateConfig field")
if __name__ == "__main__":
unittest.main()
tests/test_compare.py
+356
View File
@@ -0,0 +1,356 @@
"""Tests for the run diff.
Written against `unittest` rather than pytest so they run with a bare interpreter — this repo
has no dev dependency group, and a test you cannot run is not a test. pytest collects
TestCase subclasses too, so adding it later changes nothing.
python3 -m unittest discover -s tests -v
"""
from __future__ import annotations
import json
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from kbench.compare import PERF_NOISE_FLOOR_PCT, compare, direction # noqa: E402
RESULTS = Path(__file__).resolve().parent.parent / "results"
def run(
*,
run_id="r",
model="m",
quant="nvfp4",
checkpoint=None,
serving=None,
host="your-node",
quality=None,
perf=None,
):
return {
"run_id": run_id,
"target": {
"model": model,
"quantization": quant,
"checkpoint": checkpoint,
"serving_config": serving or {"gpu-memory-utilization": "0.55"},
"engine": "vllm",
},
"host": {"id": host},
"quality": quality or [],
"perf": {"points": perf} if perf is not None else None,
}
def sample(sid, score, passed):
return {"sample_id": sid, "score": score, "passed": passed}
def point(concurrency, tps, ttft=None, accept=None, natural_stop=False):
return {
"concurrency": concurrency,
"output_tps_total": tps,
"output_tps_per_stream": tps / concurrency,
"ttft_p50_ms": ttft,
"spec_acceptance_rate": accept,
"natural_stop": natural_stop,
}
class TargetComparability(unittest.TestCase):
def test_identical_targets_are_flagged_as_variance_not_change(self):
c = compare(run(), run())
self.assertFalse(c.target_deltas)
self.assertTrue(any("variance" in w for w in c.warnings))
def test_one_changed_axis_is_attributable(self):
c = compare(run(quant="nvfp4"), run(quant="fp8"))
self.assertEqual([d.field for d in c.target_deltas], ["quantization"])
self.assertFalse(c.confounded)
def test_two_changed_axes_are_called_confounded(self):
# The failure this exists to prevent: changing quant AND host, then attributing the
# throughput difference to the quant.
c = compare(run(quant="nvfp4", host="your-node"), run(quant="fp8", host="other"))
self.assertTrue(c.confounded)
self.assertTrue(any("attributed" in w for w in c.warnings))
def test_reordered_serving_config_is_not_a_change(self):
a = run(serving={"a": "1", "b": "2"})
b = run(serving={"b": "2", "a": "1"})
self.assertFalse(compare(a, b).target_deltas)
class QualityDiff(unittest.TestCase):
def _with(self, samples, metrics=None):
return run(
quality=[{"task": "t", "metrics": metrics or {}, "samples": samples}]
)
def test_regressions_are_listed_before_fixes(self):
before = self._with([sample("a", 1.0, True), sample("b", 0.0, False)])
after = self._with([sample("a", 0.0, False), sample("b", 1.0, True)])
flips = compare(before, after).flips
self.assertEqual([f.became for f in flips], ["fail", "pass"])
self.assertEqual(flips[0].sample_id, "a")
def test_unchanged_samples_are_not_reported(self):
same = self._with([sample("a", 1.0, True)])
self.assertEqual(compare(same, same).flips, [])
def test_a_changed_eval_set_is_a_warning_not_a_flip(self):
# Silently comparing aggregates across different sample sets is the quiet way to be
# wrong, so it must never look like a normal diff.
before = self._with([sample("a", 1.0, True)])
after = self._with([sample("b", 1.0, True)])
c = compare(before, after)
self.assertEqual(c.flips, [])
self.assertEqual(c.only_in_before, ["t/a"])
self.assertEqual(c.only_in_after, ["t/b"])
self.assertTrue(any("eval set changed" in w for w in c.warnings))
def test_a_perf_only_baseline_says_so_instead_of_claiming_the_set_changed(self):
c = compare(run(), self._with([sample("a", 1.0, True)]))
self.assertTrue(any("no quality results" in w for w in c.warnings))
self.assertFalse(any("eval set changed" in w for w in c.warnings))
def test_sample_ids_are_scoped_to_their_task(self):
# The same id in two tasks is two different samples; collapsing them would diff
# unrelated things against each other.
before = run(quality=[
{"task": "x", "metrics": {}, "samples": [sample("1", 1.0, True)]},
{"task": "y", "metrics": {}, "samples": [sample("1", 0.0, False)]},
])
self.assertEqual(compare(before, before).flips, [])
class PerfDiff(unittest.TestCase):
def test_direction_accounts_for_metrics_where_lower_is_better(self):
c = compare(
run(perf=[point(1, 30.0, ttft=300.0)]),
run(perf=[point(1, 30.0, ttft=200.0)]),
)
deltas = {d.name: d for d in c.perf[1]}
# Latency fell, which is an improvement — the sign alone would say otherwise.
self.assertEqual(direction("ttft_p50_ms", deltas["ttft_p50_ms"]), "better")
def test_more_throughput_is_better(self):
c = compare(run(perf=[point(1, 20.0)]), run(perf=[point(1, 30.0)]))
deltas = {d.name: d for d in c.perf[1]}
self.assertEqual(direction("output_tps_total", deltas["output_tps_total"]), "better")
def test_movement_under_the_noise_floor_reads_as_flat(self):
small = 100.0 * (1 + (PERF_NOISE_FLOOR_PCT - 1) / 100)
c = compare(run(perf=[point(1, 100.0)]), run(perf=[point(1, small)]))
deltas = {d.name: d for d in c.perf[1]}
self.assertEqual(direction("output_tps_total", deltas["output_tps_total"]), "flat")
def test_a_different_sweep_is_warned_about(self):
c = compare(
run(perf=[point(1, 30.0), point(8, 180.0)]),
run(perf=[point(1, 30.0), point(32, 400.0)]),
)
self.assertEqual(sorted(c.perf), [1])
self.assertTrue(any("sweep differs" in w for w in c.warnings))
class SpeculativeDecoding(unittest.TestCase):
def test_acceptance_rate_is_diffed_and_higher_is_better(self):
# Throughput moves for many reasons; acceptance moves only because drafting got better
# or worse, which is the number an MTP comparison is actually about.
c = compare(
run(perf=[point(1, 30.0, accept=0.55)]),
run(perf=[point(1, 31.0, accept=0.80)]),
)
deltas = {d.name: d for d in c.perf[1]}
self.assertIn("spec_acceptance_rate", deltas)
self.assertEqual(direction("spec_acceptance_rate", deltas["spec_acceptance_rate"]), "better")
def test_a_target_without_drafting_yields_no_acceptance_delta(self):
c = compare(run(perf=[point(1, 30.0)]), run(perf=[point(1, 30.0, accept=0.8)]))
deltas = {d.name: d for d in c.perf[1]}
self.assertIsNone(deltas["spec_acceptance_rate"].absolute)
def test_mixing_pinned_and_natural_stop_is_warned_about(self):
# Forced continuation past EOS and natural stopping produce different token
# distributions; comparing their throughput directly is comparing two things.
c = compare(
run(perf=[point(1, 30.0)]),
run(perf=[point(1, 45.0, natural_stop=True)]),
)
self.assertTrue(any("not comparable across those two modes" in w for w in c.warnings))
def test_two_natural_stop_runs_are_not_warned_about(self):
c = compare(
run(perf=[point(1, 30.0, natural_stop=True)]),
run(perf=[point(1, 45.0, natural_stop=True)]),
)
self.assertFalse(any("two modes" in w for w in c.warnings))
class AgainstCommittedResults(unittest.TestCase):
"""The two real score cards in results/ — the diff must survive real data."""
def setUp(self):
files = sorted(RESULTS.glob("*.json"))
if len(files) < 2:
self.skipTest("needs two committed results")
self.a, self.b = (json.loads(f.read_text()) for f in files[:2])
def test_diffs_real_runs_without_error_and_finds_the_shared_sweep(self):
c = compare(self.a, self.b)
self.assertEqual(sorted(c.perf), [1, 8, 32])
# Same target, same host, same day: the tool must say so rather than imply a change.
self.assertTrue(any("variance" in w for w in c.warnings))
def test_neither_run_is_mutated(self):
before = json.dumps(self.a, sort_keys=True)
compare(self.a, self.b)
self.assertEqual(json.dumps(self.a, sort_keys=True), before)
if __name__ == "__main__":
unittest.main()
class EditedSamplesAreDetected(unittest.TestCase):
"""An edited sample keeps its id, so the add/remove diff sees nothing.
This is the most dangerous shape of eval drift: loosen a regex, and every id still
lines up while the scores now answer a different question.
"""
def _run(self, fingerprint, score=1.0):
return {
"target_id": "m@h",
"quality": [
{
"task": "agent_ops",
"tier": "signal",
"dataset_fingerprint": fingerprint,
"metrics": {"accuracy": score},
"samples": [
{"sample_id": "agent_ops/0001", "score": score, "passed": True}
],
}
],
"perf": None,
"verdict": {"contamination": "clean"},
}
def test_same_fingerprint_does_not_warn(self):
r = compare(self._run("sha256:aaaa"), self._run("sha256:aaaa"))
self.assertFalse([w for w in r.warnings if "edited" in w or "fingerprint" in w])
def test_changed_fingerprint_warns_even_when_ids_match(self):
r = compare(self._run("sha256:aaaa"), self._run("sha256:bbbb"))
self.assertFalse(r.only_in_before)
self.assertFalse(r.only_in_after)
self.assertTrue(
any("was edited between runs" in w for w in r.warnings),
f"an edit with matching ids went unreported: {r.warnings}",
)
def test_missing_fingerprint_warns(self):
r = compare(self._run(None), self._run("sha256:bbbb"))
self.assertTrue(any("no dataset fingerprint" in w for w in r.warnings))
class SamplingChangesAreDetected(unittest.TestCase):
"""Sampling is part of a quality result's identity.
Unpinned sampling moved this bench by 27% of its samples between identical runs --
more than most real regressions. Diffing across different sampling reports noise as
signal, so it has to be called out rather than silently compared.
"""
def _run(self, sampling, score=1.0):
return {
"target_id": "m@h",
"sampling": sampling,
"quality": [
{
"task": "agent_ops",
"tier": "signal",
"dataset_fingerprint": "sha256:aaaa",
"metrics": {"accuracy": score},
"samples": [
{"sample_id": "agent_ops/0001", "score": score, "passed": True}
],
}
],
"perf": None,
"verdict": {"contamination": "clean"},
}
PINNED = {"temperature": 0.6, "top_p": 0.95, "seed": 1}
def test_identical_sampling_does_not_warn(self):
r = compare(self._run(self.PINNED), self._run(self.PINNED))
self.assertFalse([w for w in r.warnings if "ampling" in w], r.warnings)
def test_changed_temperature_warns(self):
other = {**self.PINNED, "temperature": 1.0}
r = compare(self._run(self.PINNED), self._run(other))
self.assertTrue(
any("Sampling changed between runs" in w for w in r.warnings), r.warnings
)
self.assertTrue(any("temperature" in w for w in r.warnings), r.warnings)
def test_missing_sampling_block_warns(self):
r = compare(self._run(None), self._run(self.PINNED))
self.assertTrue(
any("does not record how the model was sampled" in w for w in r.warnings),
r.warnings,
)
class EpochsAreNotASamplingChange(unittest.TestCase):
"""More epochs is better precision on the same estimand, not a different measurement."""
def _run(self, sampling):
return {
"target_id": "m@h",
"sampling": sampling,
"quality": [
{
"task": "agent_ops",
"tier": "signal",
"dataset_fingerprint": "sha256:aaaa",
"metrics": {"accuracy": 1.0},
"samples": [
{"sample_id": "agent_ops/0001", "score": 1.0, "passed": True}
],
}
],
"perf": None,
"verdict": {"contamination": "clean"},
}
BASE = {"temperature": 0.6, "top_p": 0.95, "seed": 1, "epochs": 1}
def test_differing_epochs_does_not_claim_incomparability(self):
r = compare(self._run(self.BASE), self._run({**self.BASE, "epochs": 5}))
self.assertFalse(
any("not comparable across different sampling" in w for w in r.warnings),
f"epoch count reported as a sampling change: {r.warnings}",
)
self.assertTrue(any("Epochs differ" in w for w in r.warnings), r.warnings)
def test_a_real_sampling_change_alongside_epochs_still_warns(self):
other = {**self.BASE, "epochs": 5, "temperature": 1.0}
r = compare(self._run(self.BASE), self._run(other))
self.assertTrue(
any("not comparable across different sampling" in w for w in r.warnings),
r.warnings,
)
self.assertTrue(any("Epochs differ" in w for w in r.warnings), r.warnings)
# the incomparability warning must name temperature, not epochs
sampling_warn = next(w for w in r.warnings if "not comparable" in w)
self.assertIn("temperature", sampling_warn)
self.assertNotIn("epochs", sampling_warn)
tests/test_data.py
+219
View File
@@ -0,0 +1,219 @@
"""Tests over the eval data itself.
Data rots differently from code: nothing crashes when a sample is quietly malformed, an id is
reused, or a `test` sample creeps into a training split. The score just moves, and the move
looks exactly like a real result. These are the checks that fail loudly instead.
python3 -m unittest discover -s tests -v
"""
from __future__ import annotations
import re
import sys
import unittest
from collections import Counter
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from kbench.schema import read_records # noqa: E402
DATA = ROOT / "data"
FAMILIES = sorted(DATA.glob("*/*.jsonl"))
# Anything that must never reach a third-party judge or baseline model. The authoring rules
# say repo privacy is defence in depth, not the control — this is the control.
FORBIDDEN = [
(r"\b115\b", "customer identifier"),
(r"workstation|tailnet|cloud-[123]-|your-second-node|your-node", "internal host"),
(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", "IP address"),
(r"sk-[A-Za-z0-9]{16,}|ghp_[A-Za-z0-9]{16,}", "API key"),
(r"-----BEGIN [A-Z ]*PRIVATE KEY", "private key"),
]
class DataIntegrity(unittest.TestCase):
def test_every_family_parses_and_validates(self):
self.assertTrue(FAMILIES, "no data files found at all")
for path in FAMILIES:
with self.subTest(family=path.name):
self.assertTrue(read_records(path), f"{path.name} is empty")
def test_ids_are_unique_within_a_family(self):
# Per-sample regression tracking keys on the id. A duplicate silently makes one of the
# two invisible in every future comparison.
for path in FAMILIES:
ids = [r["id"] for _, r in read_records(path)]
dupes = [i for i, n in Counter(ids).items() if n > 1]
self.assertEqual(dupes, [], f"{path.name} reuses ids: {dupes}")
def test_no_two_samples_share_a_prompt(self):
# Unique ids are not enough. Re-running a seeder appends a second copy of every
# sample under fresh ids, which passes the id check, doubles the family, and silently
# double-weights whatever those samples measure. This has already happened once.
for path in FAMILIES:
inputs = [r["input"] for _, r in read_records(path)]
dupes = [i for i, n in Counter(inputs).items() if n > 1]
with self.subTest(family=path.name):
self.assertEqual(
[d[:60] for d in dupes],
[],
f"{path.name} contains the same prompt more than once",
)
def test_ids_match_their_family_and_file(self):
for path in FAMILIES:
for lineno, r in read_records(path):
with self.subTest(sample=r["id"]):
self.assertTrue(
r["id"].startswith(f"{r.get('family')}/"),
f"{path.name}:{lineno} id {r['id']} does not match family {r.get('family')}",
)
def test_regex_targets_actually_compile(self):
# An invalid regex target does not raise at author time; it fails at grading time,
# mid-run, after the model has already been paid for.
for path in FAMILIES:
for lineno, r in read_records(path):
if r["scorer"] != "regex":
continue
with self.subTest(sample=r["id"]):
try:
re.compile(r["target"])
except re.error as exc:
self.fail(f"{path.name}:{lineno} {r['id']} bad regex: {exc}")
def test_no_sensitive_content_reaches_a_third_party_model(self):
for path in FAMILIES:
for lineno, r in read_records(path):
haystack = " ".join(
str(r.get(k) or "") for k in ("input", "target", "rubric")
)
for pattern, label in FORBIDDEN:
with self.subTest(sample=r["id"], rule=label):
self.assertIsNone(
re.search(pattern, haystack, re.I),
f"{path.name}:{lineno} {r['id']} contains a {label}",
)
def test_rubric_samples_are_rare(self):
# Each rubric sample adds judge cost per run and judge drift between runs, which
# confounds exactly the checkpoint comparisons this bench exists to make.
for path in FAMILIES:
records = [r for _, r in read_records(path)]
rubrics = [r for r in records if r["scorer"] == "rubric"]
with self.subTest(family=path.name):
self.assertLessEqual(
len(rubrics),
max(1, len(records) // 4),
f"{path.name}: {len(rubrics)}/{len(records)} samples are judge-graded",
)
class ContaminationProbes(unittest.TestCase):
"""The canary tier is the one place where passing is bad."""
PATH = DATA / "canary" / "contamination.jsonl"
def setUp(self):
if not self.PATH.exists():
self.skipTest("no canary family")
self.records = [r for _, r in read_records(self.PATH)]
def _carriers(self):
"""Probes that put the GUID into the traffic. Their answer is in their own prompt."""
return [r for r in self.records if r["target"] in r["input"]]
def _detectors(self):
"""Probes that ask for the GUID cold. Only a trained-on model can answer."""
return [r for r in self.records if r["target"] not in r["input"]]
def test_all_probes_target_one_shared_canary(self):
# Carrier and detectors must chase the SAME string. A detector for a GUID no carrier
# ever transmitted is unanswerable by construction — it can never fire, so it proves
# nothing while looking like a control. That was the first version of this file.
targets = {r["target"] for r in self.records}
self.assertEqual(len(targets), 1, f"probes chase different canaries: {targets}")
def test_exactly_one_carrier_transmits_the_guid(self):
carriers = self._carriers()
self.assertEqual(
len(carriers), 1, "need exactly one carrier: the probe that leaks the GUID on purpose"
)
self.assertIn("LBCANARY-", carriers[0]["input"])
def test_at_least_two_detectors_withhold_the_guid(self):
detectors = self._detectors()
self.assertGreaterEqual(len(detectors), 2, "need at least two cold detectors")
for r in detectors:
with self.subTest(sample=r["id"]):
# A detector may name the canary, and must not contain the answer.
self.assertNotIn(r["target"], r["input"])
def test_a_detector_never_leaks_the_suffix_it_is_testing_for(self):
# A prefix hint is fine and useful; the tail must be absent or the probe grades itself.
for r in self._detectors():
suffix = r["target"].split("-", 2)[-1]
with self.subTest(sample=r["id"]):
self.assertNotIn(suffix, r["input"], "detector prompt contains the answer suffix")
def test_role_tags_agree_with_the_structural_role(self):
# Two sources of truth for the same fact: the `tags` field, which the task code reads
# to decide what to score, and whether the prompt contains its own answer, which is
# what actually makes a probe a carrier. If they drift, the task scores the wrong set
# and the disagreement is invisible at runtime.
for r in self.records:
tags = (r.get("metadata") or {}).get("tags", [])
structural = "carrier" if r["target"] in r["input"] else "detector"
with self.subTest(sample=r["id"]):
self.assertIn(
structural,
tags,
f"{r['id']} is structurally a {structural} but is tagged {tags}",
)
class CanaryTaskScoresOnlyDetectors(unittest.TestCase):
"""The carrier must be loaded and then not scored.
Regression: the task scored every probe including the carrier. A carrier states the GUID
in its own prompt and asks for it back, so it always passes — which pinned the canary at
>= 1/n for every model alive, reported CONTAMINATED, and nulled signal_score. The data
said `carrier` and carried a note reading "it always passes"; the task never looked.
"""
def setUp(self):
# The published tree ships neither family: the signal set would be contaminated by
# publication, and a published canary GUID would enter public corpora and fire
# against every model that ever read the repo. So both absences are expected here.
if not (DATA / "canary" / "contamination.jsonl").exists():
self.skipTest("no canary family (expected in the published tree)")
try:
from kbench.tasks.canary import canary_task
except ImportError as exc: # inspect-ai is not needed for the data-only tests
self.skipTest(f"inspect-ai unavailable: {exc}")
self.canary_task = canary_task
def test_carrier_is_excluded_from_the_scored_dataset(self):
task = self.canary_task("contamination")
scored = list(task.dataset)
self.assertTrue(scored, "canary task scored nothing")
for sample in scored:
with self.subTest(sample=sample.id):
self.assertNotIn(
str(sample.target),
str(sample.input),
f"{sample.id} contains its own answer, so it always passes; "
"carriers must not be scored",
)
def test_every_scored_probe_is_tagged_detector(self):
for sample in self.canary_task("contamination").dataset:
with self.subTest(sample=sample.id):
self.assertIn("detector", (sample.metadata or {}).get("tags", []))
if __name__ == "__main__":
unittest.main()
tests/test_specdec.py
+102
View File
@@ -0,0 +1,102 @@
"""Speculative-decoding counter parsing.
The parser reads a live Prometheus payload, so its failure modes are all "the server said
something slightly different than expected". Each of these is a shape a real /metrics endpoint
actually produces.
python3 -m unittest discover -s tests -v
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from kbench.specdec import SpecCounters, metrics_url, parse_metrics # noqa: E402
PAYLOAD = """\
# HELP vllm:spec_decode_num_draft_tokens_total Number of draft tokens.
# TYPE vllm:spec_decode_num_draft_tokens_total counter
vllm:spec_decode_num_draft_tokens_total{model_name="qwen"} 1000.0
# HELP vllm:spec_decode_num_accepted_tokens_total Number accepted.
# TYPE vllm:spec_decode_num_accepted_tokens_total counter
vllm:spec_decode_num_accepted_tokens_total{model_name="qwen"} 720.0
vllm:num_requests_running{model_name="qwen"} 4.0
"""
class Parsing(unittest.TestCase):
def test_reads_both_counters(self):
c = parse_metrics(PAYLOAD)
self.assertEqual((c.draft, c.accepted), (1000.0, 720.0))
self.assertEqual(c.acceptance_rate, 0.72)
def test_ignores_comments_and_unrelated_series(self):
c = parse_metrics(PAYLOAD)
self.assertEqual(c.draft, 1000.0, "an unrelated metric leaked into the total")
def test_sums_across_label_sets(self):
# A server hosting two models exposes one series each. Summing is correct, and taking
# the first would silently measure only one of them.
two = PAYLOAD + (
'vllm:spec_decode_num_draft_tokens_total{model_name="b"} 500.0\n'
'vllm:spec_decode_num_accepted_tokens_total{model_name="b"} 250.0\n'
)
c = parse_metrics(two)
self.assertEqual((c.draft, c.accepted), (1500.0, 970.0))
def test_no_speculative_counters_means_none_not_zero(self):
# An engine without speculative decoding must report "no data". Zero would render as a
# 0% acceptance rate, which reads as a broken draft model rather than an absent one.
self.assertIsNone(parse_metrics("vllm:num_requests_running 1.0\n"))
def test_a_missing_half_is_not_usable(self):
self.assertIsNone(
parse_metrics('vllm:spec_decode_num_draft_tokens_total{m="x"} 10.0\n')
)
def test_malformed_values_do_not_raise(self):
payload = (
'vllm:spec_decode_num_draft_tokens_total{m="x"} not-a-number\n'
'vllm:spec_decode_num_accepted_tokens_total{m="x"} 5.0\n'
)
self.assertIsNone(parse_metrics(payload))
def test_unlabelled_series_are_read(self):
payload = (
"vllm:spec_decode_num_draft_tokens_total 100.0\n"
"vllm:spec_decode_num_accepted_tokens_total 50.0\n"
)
self.assertEqual(parse_metrics(payload).acceptance_rate, 0.5)
class WindowedRate(unittest.TestCase):
def test_the_delta_is_what_gets_reported(self):
# Lifetime counters. Reporting the total would fold warm-up and every earlier
# concurrency level into this point's number.
before = SpecCounters(accepted=700.0, draft=1000.0)
after = SpecCounters(accepted=1600.0, draft=2000.0)
window = after - before
self.assertEqual(window.acceptance_rate, 0.9)
self.assertNotEqual(window.acceptance_rate, after.acceptance_rate)
def test_no_drafting_in_the_window_is_none(self):
same = SpecCounters(accepted=10.0, draft=10.0)
self.assertIsNone((same - same).acceptance_rate)
class MetricsUrl(unittest.TestCase):
def test_strips_the_openai_prefix(self):
# base_url points at the OpenAI-compatible surface; /metrics is at the server root.
self.assertEqual(metrics_url("http://host:8001/v1"), "http://host:8001/metrics")
self.assertEqual(metrics_url("http://host:8001/v1/"), "http://host:8001/metrics")
def test_leaves_a_bare_root_alone(self):
self.assertEqual(metrics_url("http://host:8001"), "http://host:8001/metrics")
if __name__ == "__main__":
unittest.main()
tests/test_sweep.py
+135
View File
@@ -0,0 +1,135 @@
"""Frontier reduction.
The claim a sweep makes is "you can stop considering this one", so the tests are mostly about
when that claim is NOT safe to make.
python3 -m unittest discover -s tests -v
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from kbench.sweep import SweepPoint, analyse, point_from_run # noqa: E402
def p(tid, quality=None, tps=None, contamination="clean", error=None):
return SweepPoint(
target_id=tid, quality=quality, throughput_tps=tps, contamination=contamination, error=error
)
class Domination(unittest.TestCase):
def test_worse_on_both_axes_is_dominated(self):
r = analyse([p("fast-good", 0.80, 200.0), p("slow-bad", 0.70, 100.0)])
self.assertEqual(r.best, ["fast-good"])
self.assertEqual(r.dominated["slow-bad"], "fast-good")
def test_a_genuine_trade_off_keeps_both_on_the_frontier(self):
# Higher quality at lower throughput is a decision the operator has to make with
# knowledge this tool does not have. Collapsing it to one winner would be inventing
# a preference.
r = analyse([p("accurate", 0.90, 100.0), p("quick", 0.70, 200.0)])
self.assertEqual(sorted(r.best), ["accurate", "quick"])
self.assertEqual(r.dominated, {})
def test_equal_quality_and_more_throughput_dominates(self):
r = analyse([p("a", 0.80, 200.0), p("b", 0.80, 150.0)])
self.assertEqual(r.best, ["a"])
self.assertIn("b", r.dominated)
def test_identical_points_do_not_dominate_each_other(self):
# Nothing is better anywhere, so neither may be dismissed — otherwise a tie would
# silently eliminate an option.
r = analyse([p("a", 0.80, 200.0), p("b", 0.80, 200.0)])
self.assertEqual(sorted(r.best), ["a", "b"])
self.assertEqual(r.dominated, {})
def test_the_frontier_is_ordered_fastest_first(self):
r = analyse([p("mid", 0.85, 150.0), p("fast", 0.70, 200.0), p("slow", 0.95, 100.0)])
self.assertEqual(r.best, ["fast", "mid", "slow"])
class WhenQualityCannotBeTraded(unittest.TestCase):
def test_a_contaminated_point_is_ranked_on_throughput_only(self):
# Its quality number is memorisation. Letting it win a quality comparison would launder
# a void score into a recommendation.
r = analyse([p("clean", 0.80, 100.0), p("dirty", 0.99, 90.0, contamination="detected")])
self.assertTrue(any("throughput alone" in w for w in r.warnings))
self.assertEqual(r.best, ["clean"])
self.assertIn("dirty", r.dominated)
def test_a_missing_quality_score_also_falls_back_to_throughput(self):
r = analyse([p("measured", 0.80, 100.0), p("unmeasured", None, 150.0)])
self.assertTrue(any("throughput alone" in w for w in r.warnings))
self.assertEqual(r.best, ["unmeasured"])
def test_quality_is_used_when_every_point_has_a_comparable_one(self):
r = analyse([p("a", 0.90, 100.0), p("b", 0.70, 120.0)])
self.assertFalse(any("throughput alone" in w for w in r.warnings))
self.assertEqual(sorted(r.best), ["a", "b"])
class Robustness(unittest.TestCase):
def test_a_failed_target_never_reaches_the_frontier(self):
r = analyse([p("ok", 0.80, 100.0), p("broken", error="server never came up")])
self.assertEqual(r.best, ["ok"])
self.assertEqual([f.target_id for f in r.failed], ["broken"])
self.assertNotIn("broken", r.dominated)
def test_all_failed_is_reported_rather_than_returning_an_empty_winner(self):
r = analyse([p("a", error="boom"), p("b", error="boom")])
self.assertEqual(r.best, [])
self.assertTrue(any("No target produced" in w for w in r.warnings))
def test_mixed_hosts_are_warned_about(self):
# A sweep is meant to hold the machine constant; two hosts means two causes.
r = analyse([p("a", 0.8, 100.0), p("b", 0.8, 200.0)], hosts=["your-node", "metal"])
self.assertTrue(any("more than one host" in w for w in r.warnings))
def test_one_host_is_not_warned_about(self):
r = analyse([p("a", 0.8, 100.0)], hosts=["your-node", "your-node"])
self.assertFalse(any("more than one host" in w for w in r.warnings))
class ReadingSavedRuns(unittest.TestCase):
def test_a_contaminated_run_contributes_no_quality(self):
# compute_verdict nulls signal_score when a probe fires; the sweep must inherit that
# rather than reaching for the withheld value.
run = {
"target_id": "t",
"verdict": {
"signal_score": None,
"signal_score_unverified": 0.99,
"peak_throughput_tps": 180.0,
"contamination": "detected",
},
}
point = point_from_run(run)
self.assertIsNone(point.quality)
self.assertFalse(point.quality_comparable)
self.assertTrue(point.usable, "throughput is still valid evidence")
def test_a_clean_run_is_fully_comparable(self):
run = {
"target_id": "t",
"verdict": {
"signal_score": 0.81,
"peak_throughput_tps": 180.0,
"single_stream_tps": 30.0,
"contamination": "clean",
},
}
point = point_from_run(run)
self.assertTrue(point.quality_comparable)
self.assertEqual(point.quality, 0.81)
self.assertEqual(point.single_stream_tps, 30.0)
if __name__ == "__main__":
unittest.main()
tests/test_verdict.py
+158
View File
@@ -0,0 +1,158 @@
"""The contamination gate.
A probe nobody reads is the same as no probe. These assert the gate does something
consequential — it takes the signal score away — rather than recording a number beside it.
python3 -m unittest discover -s tests -v
"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from kbench.results import QualityResult, compute_verdict # noqa: E402
def quality(tier: str, accuracy: float, task: str = "t") -> QualityResult:
return QualityResult(task=task, tier=tier, metrics={"accuracy": accuracy})
class ContaminationGate(unittest.TestCase):
def test_a_clean_probe_leaves_the_signal_score_readable(self):
v = compute_verdict([quality("signal", 0.81), quality("canary", 0.0)], None)
self.assertEqual(v["contamination"], "clean")
self.assertEqual(v["signal_score"], 0.81)
self.assertEqual(v["canary_score"], 0.0)
def test_a_fired_probe_removes_the_signal_score_entirely(self):
# The behaviour that matters. Anything reading verdict["signal_score"] — the results
# table, a comparison, a published card — must get None rather than a memorised number
# with a warning attached somewhere else.
v = compute_verdict([quality("signal", 0.94), quality("canary", 0.67)], None)
self.assertEqual(v["contamination"], "detected")
self.assertIsNone(v["signal_score"])
def test_the_withheld_score_is_kept_for_forensics(self):
v = compute_verdict([quality("signal", 0.94), quality("canary", 0.67)], None)
self.assertEqual(v["signal_score_unverified"], 0.94)
def test_even_one_probe_in_many_is_enough_to_void_the_run(self):
# Partial memorisation is still memorisation; there is no safe amount.
v = compute_verdict(
[quality("signal", 0.9), quality("canary", 0.0, "a"), quality("canary", 1.0, "b")],
None,
)
self.assertEqual(v["contamination"], "detected")
self.assertIsNone(v["signal_score"])
def test_no_probe_is_reported_as_unverified_not_clean(self):
# "Nobody checked" and "checked and clean" are different claims, and conflating them is
# how an unverified number gets quoted as a verified one.
v = compute_verdict([quality("signal", 0.81)], None)
self.assertEqual(v["contamination"], "unverified")
self.assertIsNone(v["canary_score"])
self.assertEqual(v["signal_score"], 0.81, "an unverified run keeps its score")
def test_a_failed_probe_does_not_count_as_a_clean_one(self):
errored = QualityResult(task="c", tier="canary", metrics={}, error="judge timed out")
v = compute_verdict([quality("signal", 0.81), errored], None)
self.assertEqual(v["contamination"], "unverified")
def test_contamination_does_not_touch_the_perf_half(self):
# Memorisation says nothing about throughput; a contaminated run is still valid
# serving-performance evidence, and discarding it would be its own error.
from kbench.results import PerfPoint, PerfResult
perf = PerfResult(
engine="vllm",
points=[
PerfPoint(
concurrency=1,
input_tokens=512,
output_tokens=128,
n_requests=4,
completed=4,
failed=0,
duration_s=10.0,
output_tps_total=30.0,
output_tps_per_stream=30.0,
)
],
)
v = compute_verdict([quality("signal", 0.9), quality("canary", 1.0)], perf)
self.assertIsNone(v["signal_score"])
self.assertEqual(v["single_stream_tps"], 30.0)
self.assertTrue(v["interactive_viable"])
if __name__ == "__main__":
unittest.main()
class ScoreCardsAreValidJson(unittest.TestCase):
"""A score card only Python can read is not a durable record.
Regression: an ungraded rubric sample (no judge configured) reached the file as a bare
`NaN`. Python's json emits and accepts it; JSON.parse, Go and serde all reject it. The
first committed card was unparseable by the website meant to render it.
"""
def test_non_finite_scores_serialize_as_null(self):
from kbench.results import _json_safe
nan, inf = float("nan"), float("inf")
self.assertIsNone(_json_safe(nan))
self.assertIsNone(_json_safe(inf))
self.assertIsNone(_json_safe(-inf))
self.assertEqual(_json_safe(0.5), 0.5)
self.assertEqual(
_json_safe({"a": [1.0, nan], "b": {"c": inf}}),
{"a": [1.0, None], "b": {"c": None}},
)
def test_committed_cards_parse_under_strict_json(self):
import json
from pathlib import Path
def strict(c):
raise ValueError(f"non-JSON constant {c!r}")
results = Path(__file__).resolve().parent.parent / "results"
cards = sorted(results.glob("*.json")) if results.exists() else []
if not cards:
self.skipTest("no committed score cards yet")
for card in cards:
with self.subTest(card=card.name):
json.loads(card.read_text(), parse_constant=strict)
class UngradedSamplesAreNotFailures(unittest.TestCase):
def _normalizer(self):
# kbench.run pulls in perf -> httpx. The data-only tests are meant to run on a bare
# interpreter (that is what a fresh clone and the publish gate both have), so this
# skips rather than errors when the serving deps are absent.
try:
from kbench.run import _normalize_sample_score
except ImportError as exc:
self.skipTest(f"serving deps unavailable: {exc}")
return _normalize_sample_score
def test_nan_score_is_marked_ungraded(self):
_normalize_sample_score = self._normalizer()
score, passed, extra = _normalize_sample_score(float("nan"), None)
self.assertNotEqual(score, score, "NaN should be preserved in-memory")
self.assertFalse(passed)
self.assertTrue(extra.get("ungraded"), "an ungraded sample must say so")
def test_ordinary_zero_is_a_failure_not_ungraded(self):
_normalize_sample_score = self._normalizer()
_, passed, extra = _normalize_sample_score(0.0, None)
self.assertFalse(passed)
self.assertNotIn("ungraded", extra)