Lumbridge Bench

kbench/tasks/__init__.py

@@ -0,0 +1,132 @@
+"""Task catalog.
+
+Two tiers, and the distinction is not cosmetic:
+
+REFERENCE -- public benchmarks pulled from inspect_evals. These are NOT here to
+  rank models. They exist to (a) prove the harness is wired correctly, (b) give
+  a calibration anchor -- if our number lands far from published values, the
+  harness is broken, not the model, and (c) let an outside reader locate this
+  bench against numbers they already know. A bench made only of private tasks
+  is unfalsifiable to everyone including its author. Reference scores are
+  displayed with a caveat: public, likely contaminated, calibration only.
+
+SIGNAL -- our own private tasks, derived from real workloads. This is the
+  actual product. These never become public: the moment they do they enter the
+  next training corpus and stop measuring anything. See docs/DECISIONS.md#d3.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class TaskSpec:
+    """One benchmark task and how to read its numbers."""
+
+    name: str
+    inspect_task: str
+    tier: str  # "reference" | "signal" | "canary" | "example"
+    # Which key in the scorer's metrics is the headline accuracy.
+    primary_metric: str
+    # Sample scores may be dict-valued (IFEval) rather than scalar. If so, this
+    # names the key that decides pass/fail for a single sample.
+    primary_sample_key: str | None = None
+    dataset_samples: int | None = None
+    judge_required: bool = False
+    # NLTK corpora the scorer needs. Checked (and fetched) before the eval
+    # starts: IFEval's scorer needs punkt_tab but only touches it at scoring
+    # time, so a missing corpus kills the run ~20 minutes of GPU time in.
+    # Preflight turns that into a two-second failure.
+    nltk_resources: tuple[str, ...] = ()
+    notes: str = ""
+
+
+CATALOG: dict[str, TaskSpec] = {
+    "ifeval": TaskSpec(
+        name="ifeval",
+        inspect_task="inspect_evals/ifeval",
+        tier="reference",
+        primary_metric="final_acc",
+        primary_sample_key="prompt_level_strict",
+        dataset_samples=541,
+        judge_required=False,
+        nltk_resources=("tokenizers/punkt_tab",),
+        notes=(
+            "Verifiable instruction following -- 'write exactly 3 paragraphs', "
+            "'do not use the letter e'. Chosen as the first reference eval "
+            "because its scorer is deterministic (no judge model, no cost, no "
+            "judge drift between runs) and because instruction adherence is "
+            "the property that actually decides whether a small local model "
+            "can hold a system prompt in production."
+        ),
+    ),
+    "agent_ops": TaskSpec(
+        name="agent_ops",
+        inspect_task="kbench/tasks/signal.py@agent_ops",
+        tier="signal",
+        primary_metric="accuracy",
+        dataset_samples=12,
+        # One rubric sample in twelve, so a judge is needed but the drift it
+        # introduces is bounded to a twelfth of the score.
+        judge_required=True,
+        notes=(
+            "Operating a Lumbridge Compute node: admission arithmetic against "
+            "both the declared budget and the observed pool, watchdog policy, "
+            "why a Scene naming ids rather than commands cannot introduce "
+            "code, transactional rollback, and process identity under PID "
+            "reuse. Derived from the decisions the MCP server exists to let an "
+            "agent make, which is what makes it unfakeable by a general "
+            "benchmark -- nobody else measures competence at running this."
+        ),
+    ),
+    "contamination": TaskSpec(
+        name="contamination",
+        inspect_task="kbench/tasks/canary.py@contamination",
+        # Its own tier because its score is an alarm, not a capability, and
+        # compute_verdict() reads it to decide whether the signal score is
+        # usable at all. Registering it is load-bearing: an unregistered probe
+        # never runs, so the gate reports "unverified" forever -- which is the
+        # same shape of bug as the canary that was never embedded in a prompt.
+        tier="canary",
+        primary_metric="accuracy",
+        dataset_samples=3,
+        judge_required=False,
+        notes=(
+            "Contamination probe. One carrier puts the GUID into the inference "
+            "traffic; two detectors ask for it cold. Scoring is inverted -- 0 "
+            "is the healthy result, and anything above 0 voids every quality "
+            "number for the target. See data/README.md."
+        ),
+    ),
+    "example": TaskSpec(
+        name="example",
+        inspect_task="kbench/tasks/signal.py@example",
+        # Tier "example" deliberately: it exercises the signal machinery but
+        # measures nothing, so it must not contribute to the signal score in
+        # compute_verdict(). Real families are registered with tier="signal".
+        tier="example",
+        primary_metric="accuracy",
+        dataset_samples=4,
+        judge_required=True,
+        notes=(
+            "Public demonstration of the signal-task format -- one sample per "
+            "scorer type. Lives in data/public/. Real signal families live in "
+            "data/private/ and never ship."
+        ),
+    ),
+}
+
+
+def get(name: str) -> TaskSpec:
+    if name not in CATALOG:
+        known = ", ".join(sorted(CATALOG))
+        raise KeyError(f"unknown task {name!r}. known tasks: {known}")
+    return CATALOG[name]
+
+
+def by_tier(tier: str) -> list[TaskSpec]:
+    return [t for t in CATALOG.values() if t.tier == tier]
+
+
+__all__ = ["CATALOG", "TaskSpec", "by_tier", "get"]

kbench/tasks/canary.py

@@ -0,0 +1,109 @@
+"""Contamination probes.
+
+D3 specifies `data/canary/` as "GUID strings embedded in samples". The tier existed and was
+empty, and nothing loaded it — while `kbench add` stamped a `canary` GUID into every signal
+record that was never embedded in the prompt, never sent to a model, and never checked. A
+canary only works if it is *in the text that could be trained on*, so as written the control
+could not fire.
+
+**Scoring here is inverted, and that is the whole point.**
+
+A canary probe contains a GUID that exists nowhere else and asks the model to reproduce it.
+A model that has never seen this repo's data cannot possibly answer, so:
+
+    score 0.0  =  clean. The expected, healthy result.
+    score > 0  =  this model has seen our eval data. Every signal number for it is void.
+
+Read `canary_score` as a contamination alarm, never as a capability. It is reported separately
+from `signal_score` for exactly that reason.
+
+**What this actually detects.** The private set never leaves the repo, so the realistic leak
+vector is not a scraper — it is the authoring rules' own admission: prompts get sent to
+third-party judge and baseline models regardless of how private the repo is. If a provider
+trains on inference traffic, these GUIDs go with it. That is the leak this catches, and it is
+the one that would otherwise be invisible.
+
+A probe firing does not tell you *which* provider learned it. It tells you the set is burned
+and needs regenerating, which is the actionable part.
+"""
+
+from __future__ import annotations
+
+from inspect_ai import Task, task
+from inspect_ai.dataset import MemoryDataset
+from inspect_ai.solver import generate
+
+# Absolute for the same reason as signal.py: inspect-ai loads this file by path, outside
+# the package, so relative imports raise "beyond top-level package".
+from kbench.scorers import dispatching
+from kbench.tasks.signal import load_samples
+
+CANARY_TIER = "canary"
+
+
+def canary_task(family: str = "contamination") -> Task:
+    """Build the contamination probe task.
+
+    Probes are `split: test` like everything else — they are not training data and must never
+    be filtered out by the split discipline that protects the signal set.
+    """
+    samples = load_samples(family, tier=CANARY_TIER, splits=("test",))
+    if not samples:
+        raise ValueError(
+            f"canary family {family!r} is empty. A bench with no contamination probe cannot "
+            "tell a real score from a memorised one."
+        )
+
+    # Carriers must be loaded and then NOT scored. The carrier states the GUID in its own
+    # prompt and asks for it back, so every model repeats it — that is instruction-following,
+    # not memorisation. Scoring it pinned the canary at >= 1/n for every target alive, which
+    # reads as CONTAMINATED and nulls signal_score, voiding the whole quality half.
+    #
+    # The data already carried `tags: [contamination, carrier]` and a note saying "it always
+    # passes"; nothing read it. Hence the assertions below: this file now fails loudly if the
+    # split it depends on is missing, rather than silently scoring the wrong set.
+    carriers = [s for s in samples if "carrier" in (s.metadata or {}).get("tags", [])]
+    detectors = [s for s in samples if "detector" in (s.metadata or {}).get("tags", [])]
+
+    unlabelled = [s for s in samples if s not in carriers and s not in detectors]
+    if unlabelled:
+        raise ValueError(
+            f"canary family {family!r} has samples tagged neither 'carrier' nor 'detector': "
+            f"{[s.id for s in unlabelled]}. Every probe must declare its role, because the "
+            "two are scored differently."
+        )
+    if not carriers:
+        raise ValueError(
+            f"canary family {family!r} has no carrier. Without one the GUID never enters any "
+            "corpus, so the detectors are unanswerable by construction and would report "
+            "'clean' against a model that is in fact contaminated."
+        )
+    if not detectors:
+        raise ValueError(
+            f"canary family {family!r} has no detector. The carrier alone detects nothing."
+        )
+
+    return Task(
+        name=f"canary/{family}",
+        dataset=MemoryDataset(samples=detectors, name=f"canary-{family}"),
+        solver=generate(),
+        scorer=dispatching(),
+    )
+
+
+def interpret(score: float | None) -> str:
+    """Turn a canary score into the sentence a reader needs."""
+    if score is None:
+        return "no contamination probe was run — signal scores are unverified"
+    if score <= 0.0:
+        return "clean: the model could not reproduce any probe GUID"
+    return (
+        f"CONTAMINATED: the model reproduced {score:.0%} of the probe GUIDs. "
+        "Signal scores for this target are void; regenerate the eval set."
+    )
+
+
+@task
+def contamination() -> Task:
+    """Registered so `kbench` can run the probe like any other task."""
+    return canary_task()

kbench/tasks/signal.py

@@ -0,0 +1,121 @@
+"""Signal-tier tasks: our own private evals, loaded from JSONL.
+
+One family (`agent_ops`, `repo_edit`, ...) becomes one Inspect Task. Samples
+within a family may be graded differently; `kbench.scorers.dispatching` handles
+that per sample.
+
+Split discipline is enforced here rather than left to convention: by default
+only `test` samples are evaluated. If a `train` sample ever reaches the eval
+path, fine-tuning on it silently invalidates every number that follows and
+there is no way to detect it after the fact.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from inspect_ai import Task, task
+from inspect_ai.dataset import MemoryDataset, Sample
+from inspect_ai.solver import generate
+
+# Absolute, not relative. inspect-ai loads a task file by path as a standalone module, so
+# it is not a member of the `kbench` package at load time and `from ..scorers` raises
+# "attempted relative import beyond top-level package". kbench is an installed package, so
+# the absolute form resolves under both import styles.
+from kbench.scorers import dispatching
+
+# The schema lives in kbench/schema.py, which imports nothing beyond the standard library —
+# validating a JSONL record must not require the eval framework. Re-exported here so existing
+# callers of signal.family_path / signal.validate_record keep working.
+from kbench.schema import (  # noqa: F401
+    DATA_DIR,
+    VALID_SCORERS,
+    VALID_SPLITS,
+    family_path,
+    validate_record,
+)
+
+
+
+
+def load_samples(
+    family: str,
+    tier: str = "private",
+    splits: tuple[str, ...] = ("test",),
+) -> list[Sample]:
+    path = family_path(family, tier)
+    if not path.exists():
+        raise FileNotFoundError(
+            f"no data file for family {family!r} at {path}. "
+            f"create samples with: kbench add {family}"
+        )
+
+    samples: list[Sample] = []
+    for lineno, line in enumerate(path.read_text().splitlines(), start=1):
+        line = line.strip()
+        if not line or line.startswith("//"):
+            continue
+        rec = json.loads(line)
+        validate_record(rec, f"{path.name}:{lineno}")
+        if rec["split"] not in splits:
+            continue
+        samples.append(
+            Sample(
+                id=rec["id"],
+                input=rec["input"],
+                target=rec.get("target") or "",
+                metadata={
+                    "scorer": rec["scorer"],
+                    "rubric": rec.get("rubric"),
+                    "family": rec.get("family", family),
+                    "split": rec["split"],
+                    **(rec.get("metadata") or {}),
+                },
+            )
+        )
+    return samples
+
+
+def signal_task(
+    family: str,
+    tier: str = "private",
+    splits: tuple[str, ...] = ("test",),
+    judge_model: str | None = None,
+) -> Task:
+    """Build an Inspect Task for one signal family."""
+    samples = load_samples(family, tier=tier, splits=splits)
+    if not samples:
+        raise ValueError(
+            f"family {family!r} has no samples in splits {splits}. "
+            f"refusing to run an empty eval -- it would report a score of 0/0."
+        )
+    return Task(
+        name=f"signal/{family}",
+        dataset=MemoryDataset(samples=samples, name=family),
+        solver=generate(),
+        scorer=dispatching(judge_model=judge_model),
+    )
+
+
+# --- registered example task -------------------------------------------------
+# Demonstrates the machinery against the shipped public examples. Real families
+# live in data/private/ and are registered the same way.
+
+
+@task
+def example() -> Task:
+    """Public example family -- shows the format, measures nothing important."""
+    return signal_task("example", tier="public", splits=("test",))
+
+
+@task
+def agent_ops() -> Task:
+    """Operating a Compute node: admission, watchdog policy, Scene semantics, process identity.
+
+    Only the `test` split runs, which is the split discipline this module enforces rather than
+    documents: a `train` sample reaching the eval path silently invalidates every number after
+    it, and there is no way to detect that afterwards.
+    """
+    return signal_task("agent_ops", tier="private", splits=("test",))