Lumbridge Bench

kbench/tasks/canary.py

@@ -0,0 +1,109 @@
+"""Contamination probes.
+
+D3 specifies `data/canary/` as "GUID strings embedded in samples". The tier existed and was
+empty, and nothing loaded it — while `kbench add` stamped a `canary` GUID into every signal
+record that was never embedded in the prompt, never sent to a model, and never checked. A
+canary only works if it is *in the text that could be trained on*, so as written the control
+could not fire.
+
+**Scoring here is inverted, and that is the whole point.**
+
+A canary probe contains a GUID that exists nowhere else and asks the model to reproduce it.
+A model that has never seen this repo's data cannot possibly answer, so:
+
+    score 0.0  =  clean. The expected, healthy result.
+    score > 0  =  this model has seen our eval data. Every signal number for it is void.
+
+Read `canary_score` as a contamination alarm, never as a capability. It is reported separately
+from `signal_score` for exactly that reason.
+
+**What this actually detects.** The private set never leaves the repo, so the realistic leak
+vector is not a scraper — it is the authoring rules' own admission: prompts get sent to
+third-party judge and baseline models regardless of how private the repo is. If a provider
+trains on inference traffic, these GUIDs go with it. That is the leak this catches, and it is
+the one that would otherwise be invisible.
+
+A probe firing does not tell you *which* provider learned it. It tells you the set is burned
+and needs regenerating, which is the actionable part.
+"""
+
+from __future__ import annotations
+
+from inspect_ai import Task, task
+from inspect_ai.dataset import MemoryDataset
+from inspect_ai.solver import generate
+
+# Absolute for the same reason as signal.py: inspect-ai loads this file by path, outside
+# the package, so relative imports raise "beyond top-level package".
+from kbench.scorers import dispatching
+from kbench.tasks.signal import load_samples
+
+CANARY_TIER = "canary"
+
+
+def canary_task(family: str = "contamination") -> Task:
+    """Build the contamination probe task.
+
+    Probes are `split: test` like everything else — they are not training data and must never
+    be filtered out by the split discipline that protects the signal set.
+    """
+    samples = load_samples(family, tier=CANARY_TIER, splits=("test",))
+    if not samples:
+        raise ValueError(
+            f"canary family {family!r} is empty. A bench with no contamination probe cannot "
+            "tell a real score from a memorised one."
+        )
+
+    # Carriers must be loaded and then NOT scored. The carrier states the GUID in its own
+    # prompt and asks for it back, so every model repeats it — that is instruction-following,
+    # not memorisation. Scoring it pinned the canary at >= 1/n for every target alive, which
+    # reads as CONTAMINATED and nulls signal_score, voiding the whole quality half.
+    #
+    # The data already carried `tags: [contamination, carrier]` and a note saying "it always
+    # passes"; nothing read it. Hence the assertions below: this file now fails loudly if the
+    # split it depends on is missing, rather than silently scoring the wrong set.
+    carriers = [s for s in samples if "carrier" in (s.metadata or {}).get("tags", [])]
+    detectors = [s for s in samples if "detector" in (s.metadata or {}).get("tags", [])]
+
+    unlabelled = [s for s in samples if s not in carriers and s not in detectors]
+    if unlabelled:
+        raise ValueError(
+            f"canary family {family!r} has samples tagged neither 'carrier' nor 'detector': "
+            f"{[s.id for s in unlabelled]}. Every probe must declare its role, because the "
+            "two are scored differently."
+        )
+    if not carriers:
+        raise ValueError(
+            f"canary family {family!r} has no carrier. Without one the GUID never enters any "
+            "corpus, so the detectors are unanswerable by construction and would report "
+            "'clean' against a model that is in fact contaminated."
+        )
+    if not detectors:
+        raise ValueError(
+            f"canary family {family!r} has no detector. The carrier alone detects nothing."
+        )
+
+    return Task(
+        name=f"canary/{family}",
+        dataset=MemoryDataset(samples=detectors, name=f"canary-{family}"),
+        solver=generate(),
+        scorer=dispatching(),
+    )
+
+
+def interpret(score: float | None) -> str:
+    """Turn a canary score into the sentence a reader needs."""
+    if score is None:
+        return "no contamination probe was run — signal scores are unverified"
+    if score <= 0.0:
+        return "clean: the model could not reproduce any probe GUID"
+    return (
+        f"CONTAMINATED: the model reproduced {score:.0%} of the probe GUIDs. "
+        "Signal scores for this target are void; regenerate the eval set."
+    )
+
+
+@task
+def contamination() -> Task:
+    """Registered so `kbench` can run the probe like any other task."""
+    return canary_task()