Lumbridge Bench

kbench/scorers/__init__.py

@@ -0,0 +1,95 @@
+"""Scorers for signal-tier tasks.
+
+Inspect attaches scorers to a Task, not to a sample. Our families are mixed --
+one `agent_ops` file legitimately contains samples graded by exact match, by
+substring, and by rubric. Splitting a family into one Task per scorer would
+fragment the results and make family-level scores meaningless.
+
+So we dispatch per sample instead, reading `scorer` from sample metadata.
+"""
+
+from __future__ import annotations
+
+import re
+
+from inspect_ai.scorer import (
+    CORRECT,
+    INCORRECT,
+    Score,
+    Scorer,
+    Target,
+    accuracy,
+    model_graded_qa,
+    scorer,
+    stderr,
+)
+from inspect_ai.solver import TaskState
+
+DEFAULT_RUBRIC = """You are grading a model's answer against a reference.
+
+Grade CORRECT if the answer is substantively equivalent to the reference --
+matching facts, decisions, and any required steps. Ignore differences in
+wording, formatting, or verbosity.
+
+Grade INCORRECT if it contradicts the reference, omits something the reference
+treats as required, or adds a claim the reference does not support.
+
+Be strict. A partially correct answer is INCORRECT."""
+
+
+def _text(state: TaskState) -> str:
+    return (state.output.completion or "").strip()
+
+
+@scorer(metrics=[accuracy(), stderr()])
+def dispatching(judge_model: str | None = None) -> Scorer:
+    """Grade each sample by the method named in its own metadata.
+
+    Falls back to `includes` when unspecified, which is the most forgiving
+    deterministic option -- a sample that silently scores zero because of a
+    missing metadata field would look like a model failure.
+    """
+    graded = model_graded_qa(instructions=DEFAULT_RUBRIC, model=judge_model)
+
+    async def score(state: TaskState, target: Target) -> Score:
+        method = (state.metadata or {}).get("scorer") or "includes"
+        answer = _text(state)
+        expected = target.text or ""
+
+        if method == "rubric":
+            rubric = (state.metadata or {}).get("rubric")
+            if rubric:
+                custom = model_graded_qa(instructions=rubric, model=judge_model)
+                return await custom(state, target)
+            return await graded(state, target)
+
+        if method == "exact":
+            ok = answer == expected.strip()
+        elif method == "regex":
+            try:
+                ok = re.search(expected, answer, re.IGNORECASE | re.DOTALL) is not None
+            except re.error as exc:
+                return Score(
+                    value=INCORRECT,
+                    answer=answer,
+                    explanation=f"invalid regex in sample target: {exc}",
+                )
+        elif method == "includes":
+            ok = expected.strip().lower() in answer.lower()
+        else:
+            return Score(
+                value=INCORRECT,
+                answer=answer,
+                explanation=f"unknown scorer {method!r} -- fix the sample, not the model",
+            )
+
+        return Score(
+            value=CORRECT if ok else INCORRECT,
+            answer=answer,
+            explanation=f"scorer={method}",
+        )
+
+    return score
+
+
+__all__ = ["DEFAULT_RUBRIC", "dispatching"]