Karti Tripathi
122 lines
4.1 KiB
Python
122 lines
4.1 KiB
Python
"""Signal-tier tasks: our own private evals, loaded from JSONL.
|
|
|
|
One family (`agent_ops`, `repo_edit`, ...) becomes one Inspect Task. Samples
|
|
within a family may be graded differently; `kbench.scorers.dispatching` handles
|
|
that per sample.
|
|
|
|
Split discipline is enforced here rather than left to convention: by default
|
|
only `test` samples are evaluated. If a `train` sample ever reaches the eval
|
|
path, fine-tuning on it silently invalidates every number that follows and
|
|
there is no way to detect it after the fact.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from inspect_ai import Task, task
|
|
from inspect_ai.dataset import MemoryDataset, Sample
|
|
from inspect_ai.solver import generate
|
|
|
|
# Absolute, not relative. inspect-ai loads a task file by path as a standalone module, so
|
|
# it is not a member of the `kbench` package at load time and `from ..scorers` raises
|
|
# "attempted relative import beyond top-level package". kbench is an installed package, so
|
|
# the absolute form resolves under both import styles.
|
|
from kbench.scorers import dispatching
|
|
|
|
# The schema lives in kbench/schema.py, which imports nothing beyond the standard library —
|
|
# validating a JSONL record must not require the eval framework. Re-exported here so existing
|
|
# callers of signal.family_path / signal.validate_record keep working.
|
|
from kbench.schema import ( # noqa: F401
|
|
DATA_DIR,
|
|
VALID_SCORERS,
|
|
VALID_SPLITS,
|
|
family_path,
|
|
validate_record,
|
|
)
|
|
|
|
|
|
|
|
|
|
def load_samples(
|
|
family: str,
|
|
tier: str = "private",
|
|
splits: tuple[str, ...] = ("test",),
|
|
) -> list[Sample]:
|
|
path = family_path(family, tier)
|
|
if not path.exists():
|
|
raise FileNotFoundError(
|
|
f"no data file for family {family!r} at {path}. "
|
|
f"create samples with: kbench add {family}"
|
|
)
|
|
|
|
samples: list[Sample] = []
|
|
for lineno, line in enumerate(path.read_text().splitlines(), start=1):
|
|
line = line.strip()
|
|
if not line or line.startswith("//"):
|
|
continue
|
|
rec = json.loads(line)
|
|
validate_record(rec, f"{path.name}:{lineno}")
|
|
if rec["split"] not in splits:
|
|
continue
|
|
samples.append(
|
|
Sample(
|
|
id=rec["id"],
|
|
input=rec["input"],
|
|
target=rec.get("target") or "",
|
|
metadata={
|
|
"scorer": rec["scorer"],
|
|
"rubric": rec.get("rubric"),
|
|
"family": rec.get("family", family),
|
|
"split": rec["split"],
|
|
**(rec.get("metadata") or {}),
|
|
},
|
|
)
|
|
)
|
|
return samples
|
|
|
|
|
|
def signal_task(
|
|
family: str,
|
|
tier: str = "private",
|
|
splits: tuple[str, ...] = ("test",),
|
|
judge_model: str | None = None,
|
|
) -> Task:
|
|
"""Build an Inspect Task for one signal family."""
|
|
samples = load_samples(family, tier=tier, splits=splits)
|
|
if not samples:
|
|
raise ValueError(
|
|
f"family {family!r} has no samples in splits {splits}. "
|
|
f"refusing to run an empty eval -- it would report a score of 0/0."
|
|
)
|
|
return Task(
|
|
name=f"signal/{family}",
|
|
dataset=MemoryDataset(samples=samples, name=family),
|
|
solver=generate(),
|
|
scorer=dispatching(judge_model=judge_model),
|
|
)
|
|
|
|
|
|
# --- registered example task -------------------------------------------------
|
|
# Demonstrates the machinery against the shipped public examples. Real families
|
|
# live in data/private/ and are registered the same way.
|
|
|
|
|
|
@task
|
|
def example() -> Task:
|
|
"""Public example family -- shows the format, measures nothing important."""
|
|
return signal_task("example", tier="public", splits=("test",))
|
|
|
|
|
|
@task
|
|
def agent_ops() -> Task:
|
|
"""Operating a Compute node: admission, watchdog policy, Scene semantics, process identity.
|
|
|
|
Only the `test` split runs, which is the split discipline this module enforces rather than
|
|
documents: a `train` sample reaching the eval path silently invalidates every number after
|
|
it, and there is no way to detect that afterwards.
|
|
"""
|
|
return signal_task("agent_ops", tier="private", splits=("test",))
|