Lumbridge Bench

kbench/tasks/signal.py

@@ -0,0 +1,121 @@
+"""Signal-tier tasks: our own private evals, loaded from JSONL.
+
+One family (`agent_ops`, `repo_edit`, ...) becomes one Inspect Task. Samples
+within a family may be graded differently; `kbench.scorers.dispatching` handles
+that per sample.
+
+Split discipline is enforced here rather than left to convention: by default
+only `test` samples are evaluated. If a `train` sample ever reaches the eval
+path, fine-tuning on it silently invalidates every number that follows and
+there is no way to detect it after the fact.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from inspect_ai import Task, task
+from inspect_ai.dataset import MemoryDataset, Sample
+from inspect_ai.solver import generate
+
+# Absolute, not relative. inspect-ai loads a task file by path as a standalone module, so
+# it is not a member of the `kbench` package at load time and `from ..scorers` raises
+# "attempted relative import beyond top-level package". kbench is an installed package, so
+# the absolute form resolves under both import styles.
+from kbench.scorers import dispatching
+
+# The schema lives in kbench/schema.py, which imports nothing beyond the standard library —
+# validating a JSONL record must not require the eval framework. Re-exported here so existing
+# callers of signal.family_path / signal.validate_record keep working.
+from kbench.schema import (  # noqa: F401
+    DATA_DIR,
+    VALID_SCORERS,
+    VALID_SPLITS,
+    family_path,
+    validate_record,
+)
+
+
+
+
+def load_samples(
+    family: str,
+    tier: str = "private",
+    splits: tuple[str, ...] = ("test",),
+) -> list[Sample]:
+    path = family_path(family, tier)
+    if not path.exists():
+        raise FileNotFoundError(
+            f"no data file for family {family!r} at {path}. "
+            f"create samples with: kbench add {family}"
+        )
+
+    samples: list[Sample] = []
+    for lineno, line in enumerate(path.read_text().splitlines(), start=1):
+        line = line.strip()
+        if not line or line.startswith("//"):
+            continue
+        rec = json.loads(line)
+        validate_record(rec, f"{path.name}:{lineno}")
+        if rec["split"] not in splits:
+            continue
+        samples.append(
+            Sample(
+                id=rec["id"],
+                input=rec["input"],
+                target=rec.get("target") or "",
+                metadata={
+                    "scorer": rec["scorer"],
+                    "rubric": rec.get("rubric"),
+                    "family": rec.get("family", family),
+                    "split": rec["split"],
+                    **(rec.get("metadata") or {}),
+                },
+            )
+        )
+    return samples
+
+
+def signal_task(
+    family: str,
+    tier: str = "private",
+    splits: tuple[str, ...] = ("test",),
+    judge_model: str | None = None,
+) -> Task:
+    """Build an Inspect Task for one signal family."""
+    samples = load_samples(family, tier=tier, splits=splits)
+    if not samples:
+        raise ValueError(
+            f"family {family!r} has no samples in splits {splits}. "
+            f"refusing to run an empty eval -- it would report a score of 0/0."
+        )
+    return Task(
+        name=f"signal/{family}",
+        dataset=MemoryDataset(samples=samples, name=family),
+        solver=generate(),
+        scorer=dispatching(judge_model=judge_model),
+    )
+
+
+# --- registered example task -------------------------------------------------
+# Demonstrates the machinery against the shipped public examples. Real families
+# live in data/private/ and are registered the same way.
+
+
+@task
+def example() -> Task:
+    """Public example family -- shows the format, measures nothing important."""
+    return signal_task("example", tier="public", splits=("test",))
+
+
+@task
+def agent_ops() -> Task:
+    """Operating a Compute node: admission, watchdog policy, Scene semantics, process identity.
+
+    Only the `test` split runs, which is the split discipline this module enforces rather than
+    documents: a `train` sample reaching the eval path silently invalidates every number after
+    it, and there is no way to detect that afterwards.
+    """
+    return signal_task("agent_ops", tier="private", splits=("test",))