lumbridge-public/bench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lumbridge Bench
CI / verify (push) Successful in 24s
Details
CI / deploy (push) Failing after 1m14s
Details

Browse Source
This commit is contained in:
Karti Tripathi
2026-08-04 00:44:07 -07:00
commit 006feee0f7
65 changed files with 13516 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kbench/__init__.py
View File
kbench/cli.py
+646
View File
@@ -0,0 +1,646 @@
"""kbench command line."""
from __future__ import annotations
import json
import shlex
import subprocess
from pathlib import Path
import typer
from rich.console import Console
from rich.table import Table
from . import tasks as task_catalog
from .compare import compare as compare_runs, direction
from .sweep import analyse, point_from_run
from .registry import load_registry
from .results import RESULTS_DIR, Run
from .run import build_run, run_perf, run_quality
app = typer.Typer(
add_completion=False,
help="Lumbridge Bench -- private eval and serving-performance evidence for self-hosted and API models",
)
console = Console()
def _parse_concurrencies(value: str) -> tuple[int, ...]:
return tuple(int(x) for x in value.split(",") if x.strip())
@app.command("targets")
def list_targets() -> None:
"""List benchmarkable targets."""
registry = load_registry()
table = Table(title="targets", header_style="bold")
for col in ("id", "tier", "host", "quant", "params", "engine", "aliases"):
table.add_column(col)
for t in registry.targets.values():
params = f"{t.params_b:g}B" if t.params_b else "-"
if t.active_params_b:
params += f" ({t.active_params_b:g}B active)"
table.add_row(
t.id, t.tier, t.host, t.quant or "-", params,
t.serving.engine, ", ".join(t.aliases) or "-",
)
console.print(table)
@app.command("tasks")
def list_tasks() -> None:
"""List benchmark tasks."""
table = Table(title="tasks", header_style="bold")
for col in ("name", "tier", "samples", "judge", "inspect task"):
table.add_column(col)
for spec in task_catalog.CATALOG.values():
table.add_row(
spec.name, spec.tier, str(spec.dataset_samples or "-"),
"yes" if spec.judge_required else "no", spec.inspect_task,
)
console.print(table)
console.print(
"\n[dim]reference = public, likely contaminated, calibration only\n"
"signal = private, ours, the actual measurement[/dim]"
)
@app.command("snapshot")
def snapshot(host: str) -> None:
"""Print the live serving flags on a host, for pasting into models.yaml.
Results are only comparable between targets with identical flags, so the
registry has to record what the server was ACTUALLY started with rather
than what we believe we started it with.
"""
registry = load_registry()
h = registry.host(host)
if not h.ssh:
raise typer.BadParameter(f"host {host!r} has no ssh address")
proc = subprocess.run(
["ssh", h.ssh, "ps -eo args | grep -E '[v]llm serve|[s]glang|[l]lama-server'"],
capture_output=True, text=True, timeout=30,
# A host with no serving process is an ordinary outcome, not an
# exception -- it is reported below.
check=False,
)
if proc.returncode != 0 or not proc.stdout.strip():
console.print(f"[yellow]no serving process found on {host}[/yellow]")
raise typer.Exit(1)
for line in proc.stdout.strip().splitlines():
parts = shlex.split(line)
console.print(f"\n[bold]{parts[0] if parts else '?'}[/bold]")
flags: dict[str, object] = {}
i = 0
while i < len(parts):
if parts[i].startswith("--"):
key = parts[i][2:].replace("-", "_")
if i + 1 < len(parts) and not parts[i + 1].startswith("--"):
flags[key] = parts[i + 1]
i += 2
else:
flags[key] = True
i += 1
else:
i += 1
for k, v in flags.items():
console.print(f" {k}: {v}")
def _load_prompts(path: Path | None) -> list[str] | None:
"""Read representative prompts: one per line, or a JSONL with an `input` field.
Measuring speculative decoding on the synthetic prompt is measuring the wrong thing — the
draft model's acceptance rate depends on how predictable the text is, and lorem with a
random nonce is not the text you serve.
"""
if path is None:
return None
lines = [ln.strip() for ln in path.read_text().splitlines() if ln.strip()]
prompts = []
for ln in lines:
if ln.startswith("{"):
try:
prompts.append(json.loads(ln)["input"])
continue
except (json.JSONDecodeError, KeyError):
pass
prompts.append(ln)
if not prompts:
raise typer.BadParameter(f"{path} contains no prompts")
return prompts
@app.command("perf")
def perf_cmd(
target: str,
concurrency: str = typer.Option("1,8,32", help="comma-separated concurrency levels"),
input_tokens: int = typer.Option(1024),
output_tokens: int = typer.Option(256),
save: bool = typer.Option(True, help="write a results JSON"),
prompt_file: Path | None = typer.Option(
None, help="representative prompts (one per line, or JSONL with an `input` field)"
),
natural_stop: bool = typer.Option(
False,
help="let the model stop on its own instead of pinning output length. Required for an "
"honest speculative-decoding measurement; weakens comparability across points.",
),
) -> None:
"""Run a serving performance sweep only (no quality tasks)."""
registry = load_registry()
t = registry.target(target)
result = run_perf(
t, _parse_concurrencies(concurrency), input_tokens, output_tokens,
prompts=_load_prompts(prompt_file), pin_output=not natural_stop,
)
run = build_run(registry, t, quality=[], perf=result)
_print_card(run)
if save:
path = run.save()
console.print(f"\n[green]saved[/green] {path.relative_to(Path.cwd())}")
@app.command("run")
def run_cmd(
target: str,
tasks: str = typer.Option("ifeval", help="comma-separated task names, or 'all'"),
limit: int | None = typer.Option(None, help="cap samples per task (recorded in output)"),
max_connections: int = typer.Option(16, help="concurrent requests to the model"),
epochs: int = typer.Option(
1,
help="run each sample N times and average. Sampling is not reproducible on a "
"batching server even with a fixed seed, so one epoch is a draw, not a rate.",
),
concurrency: str = typer.Option("1,8,32"),
input_tokens: int = typer.Option(1024),
output_tokens: int = typer.Option(256),
skip_perf: bool = typer.Option(False),
skip_quality: bool = typer.Option(False),
) -> None:
"""Full score card: quality tasks + perf sweep -> one results JSON."""
registry = load_registry()
t = registry.target(target)
names = list(task_catalog.CATALOG) if tasks == "all" else [
n.strip() for n in tasks.split(",") if n.strip()
]
specs = [task_catalog.get(n) for n in names]
quality = []
if not skip_quality:
quality = run_quality(
t, specs, limit=limit, max_connections=max_connections, epochs=epochs
)
perf = None
if not skip_perf:
perf = run_perf(t, _parse_concurrencies(concurrency), input_tokens, output_tokens)
run = build_run(registry, t, quality=quality, perf=perf)
_print_card(run)
path = run.save()
console.print(f"\n[green]saved[/green] {path.relative_to(Path.cwd())}")
@app.command("sweep")
def sweep_cmd(
targets: list[str] = typer.Argument(..., help="two or more target ids to compare"),
tasks: str = typer.Option("ifeval", help="comma-separated task names, or 'all'"),
limit: int | None = typer.Option(None, help="cap samples per task"),
max_connections: int = typer.Option(16),
concurrency: str = typer.Option("1,8,32"),
input_tokens: int = typer.Option(1024),
output_tokens: int = typer.Option(256),
skip_perf: bool = typer.Option(False),
skip_quality: bool = typer.Option(False),
) -> None:
"""Run the same tasks against several targets and reduce them to a frontier.
Sequential on purpose: on a unified-memory box two targets cannot both be resident, so
running them together would measure contention rather than either one. Each target still
produces an ordinary score card in results/, so compare and card keep working on them.
"""
if len(targets) < 2:
raise typer.BadParameter("a sweep needs at least two targets")
registry = load_registry()
names = list(task_catalog.CATALOG) if tasks == "all" else [
n.strip() for n in tasks.split(",") if n.strip()
]
specs = [task_catalog.get(n) for n in names]
points, hosts = [], []
for i, target_id in enumerate(targets, start=1):
console.rule(f"[bold]{i}/{len(targets)} {target_id}[/bold]")
try:
t = registry.target(target_id)
hosts.append(t.host)
quality = [] if skip_quality else run_quality(
t, specs, limit=limit, max_connections=max_connections
)
perf = None if skip_perf else run_perf(
t, _parse_concurrencies(concurrency), input_tokens, output_tokens
)
run = build_run(registry, t, quality=quality, perf=perf)
path = run.save()
console.print(f"[green]saved[/green] {path.name}")
points.append(point_from_run(run.to_dict(), str(path.name)))
except Exception as exc: # noqa: BLE001
# One bad target must not throw away the targets already measured — a sweep is
# hours of GPU time and re-running the good ones to reach the next is wasteful.
console.print(f"[red]failed[/red] {target_id}: {exc}")
points.append(
point_from_run({"target_id": target_id, "verdict": {}})
)
points[-1].error = str(exc)
result = analyse(points, hosts=hosts)
console.print()
console.rule("[bold]frontier[/bold]")
table = Table(header_style="bold")
for col in ("", "target", "signal", "peak tok/s", "tok/s @1", "verdict"):
table.add_column(col)
for point in sorted(
result.points, key=lambda x: (x.error is not None, -(x.throughput_tps or 0))
):
if point.error:
mark, verdict = "[red]![/red]", f"[red]{point.error[:44]}[/red]"
elif point.target_id in result.dominated:
mark = "[dim]·[/dim]"
verdict = f"[dim]beaten by {result.dominated[point.target_id]}[/dim]"
else:
mark, verdict = "[green]✓[/green]", "[green]worth running[/green]"
table.add_row(
mark,
point.target_id,
f"{point.quality:.3f}" if point.quality is not None else "-",
f"{point.throughput_tps:.1f}" if point.throughput_tps else "-",
f"{point.single_stream_tps:.1f}" if point.single_stream_tps else "-",
verdict,
)
console.print(table)
for warning in result.warnings:
console.print(f"[yellow]![/yellow] {warning}")
if result.best:
console.print(
f"\n[bold]{len(result.best)} of {len(result.points)} "
f"worth considering:[/bold] " + ", ".join(result.best)
)
if result.dominated:
console.print(
"[dim]The rest are beaten on every measured axis at once — not a close call, "
"a strictly worse option.[/dim]"
)
@app.command("card")
def card_cmd(path: Path) -> None:
"""Render a saved results JSON as a score card."""
data = Run.load(path)
_print_card_dict(data)
def _print_card(run: Run) -> None:
_print_card_dict(run.to_dict())
def _print_card_dict(data: dict) -> None:
tgt = data["target"]
console.print()
console.rule(f"[bold]{tgt.get('display_name', data['target_id'])}[/bold]")
console.print(
f"[dim]{data['target_id']} | {data['host'].get('hardware', '?')} | "
f"{data['timestamp']}[/dim]"
)
quality = data.get("quality") or []
if quality:
table = Table(title="quality", header_style="bold", title_justify="left")
for col in ("task", "tier", "n", "accuracy", "notes"):
table.add_column(col)
for q in quality:
if q.get("error"):
table.add_row(q["task"], q["tier"], "-", "[red]ERROR[/red]", q["error"][:60])
continue
acc = q["metrics"].get("accuracy")
limit = q["metrics"].get("_limit")
note = f"limited to {int(limit)} samples" if limit else ""
if q["tier"] == "reference":
note = (note + "; " if note else "") + "calibration only"
table.add_row(
q["task"], q["tier"], str(q["n_samples"]),
f"{acc:.3f}" if acc is not None else "-", note,
)
console.print(table)
perf = data.get("perf")
if perf and perf.get("points"):
table = Table(title="serving perf", header_style="bold", title_justify="left")
points = data["perf"].get("points") or []
# Only widen the table when there is something to put in the column — a permanent
# "accept: -" on every non-speculative target is noise.
show_accept = any(pt.get("spec_acceptance_rate") is not None for pt in points)
cols = ["conc", "tok/s total", "tok/s /stream", "TTFT p50", "TTFT p95", "TPOT p50"]
if show_accept:
cols.append("accept")
cols.append("failed")
for col in cols:
table.add_column(col, justify="right")
for p in perf["points"]:
table.add_row(
str(p["concurrency"]),
f"{p['output_tps_total']:.1f}",
f"{p['output_tps_per_stream']:.1f}",
f"{p['ttft_p50_ms']:.0f}ms" if p.get("ttft_p50_ms") else "-",
f"{p['ttft_p95_ms']:.0f}ms" if p.get("ttft_p95_ms") else "-",
f"{p['tpot_p50_ms']:.1f}ms" if p.get("tpot_p50_ms") else "-",
*(
[
f"{p['spec_acceptance_rate']:.0%}"
if p.get("spec_acceptance_rate") is not None
else "-"
]
if show_accept
else []
),
f"[red]{p['failed']}[/red]" if p["failed"] else "0",
)
console.print(table)
if any(pt.get("natural_stop") for pt in points):
spread = next(
(pt["output_token_stdev"] for pt in points if pt.get("output_token_stdev")), None
)
console.print(
" [dim]natural stop: output length was not pinned"
+ (f", ±{spread:.0f} tokens" if spread else "")
+ ". Required for an honest speculative-decoding number, and it means these"
" points are not directly comparable to pinned-length runs.[/dim]"
)
verdict = data.get("verdict") or {}
if verdict:
# Contamination first and loud. It is the one result that invalidates every other
# number on the card, so it cannot be a row someone scrolls past.
state = verdict.get("contamination")
if state == "detected":
console.print()
console.print(
"[bold white on red] CONTAMINATED [/bold white on red] "
f"the model reproduced {verdict.get('canary_score', 0):.0%} of the canary probes."
)
console.print(
" [red]Quality scores for this target are void — it has seen the eval set.[/red]"
)
if verdict.get("signal_score_unverified") is not None:
console.print(
f" [dim]withheld signal score: {verdict['signal_score_unverified']:.3f} "
"(recorded for forensics, not usable as a result)[/dim]"
)
console.print(" [dim]Regenerate the private set before trusting anything here.[/dim]")
elif state == "unverified":
console.print(
"\n[yellow]![/yellow] no contamination probe ran — quality scores are unverified"
)
console.print("\n[bold]verdict[/bold]")
for k, v in verdict.items():
if v is None or k in ("contamination", "signal_score_unverified"):
continue
console.print(f" {k}: {v}")
@app.command("add")
def add_cmd(
family: str,
input_text: str | None = typer.Option(None, "--input", help="prompt text"),
input_file: Path | None = typer.Option(None, help="read the prompt from a file"),
target: str | None = typer.Option(None, help="expected answer (or regex)"),
scorer: str = typer.Option("includes", help="exact | includes | regex | rubric"),
rubric: str | None = typer.Option(None, help="grading instructions for --scorer rubric"),
split: str = typer.Option("test", help="train | dev | test -- never change this later"),
tier: str = typer.Option("private", help="private | public"),
source: str = typer.Option("manual", help="where this sample came from"),
tags: str = typer.Option("", help="comma-separated"),
) -> None:
"""Append a sample to a family, with id, canary, and validation applied.
Do not hand-edit the JSONL. This assigns a stable non-reused id (per-sample
regression tracking keys on it) and stamps a canary GUID used later to
prove contamination.
"""
import uuid
from .tasks.signal import family_path, validate_record
if input_file:
input_text = input_file.read_text().strip()
if not input_text:
raise typer.BadParameter("provide --input or --input-file")
path = family_path(family, tier)
path.parent.mkdir(parents=True, exist_ok=True)
existing = [
json.loads(line)
for line in (path.read_text().splitlines() if path.exists() else [])
if line.strip() and not line.startswith("//")
]
# Highest existing number + 1, so ids are never reused even after deletions.
next_n = 1 + max(
(int(r["id"].rsplit("/", 1)[-1]) for r in existing if "/" in r.get("id", "")),
default=0,
)
record = {
"id": f"{family}/{next_n:04d}",
"family": family,
"split": split,
"canary": f"KBENCH-CANARY-{uuid.uuid4().hex[:8]}",
"input": input_text,
"target": target,
"scorer": scorer,
"rubric": rubric,
"metadata": {
"source": source,
"authored": __import__("datetime").date.today().isoformat(),
"tags": [t.strip() for t in tags.split(",") if t.strip()],
},
}
validate_record(record, "new sample")
with path.open("a") as f:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
console.print(f"[green]added[/green] {record['id']} -> {path}")
console.print(
"[yellow]scrub check:[/yellow] no keys, tokens, tailscale addresses, "
"or customer data in that sample?"
)
def _resolve_run(ref: str) -> Path:
"""Accept a path, a bare filename, or a run-id prefix.
Nobody types a full run id. Being strict about it just means every comparison starts with
an `ls`.
"""
direct = Path(ref)
if direct.is_file():
return direct
candidates = sorted(RESULTS_DIR.glob(f"*{ref}*.json"))
if not candidates:
raise typer.BadParameter(f"no result matching {ref!r} in {RESULTS_DIR}")
if len(candidates) > 1:
names = "\n ".join(c.name for c in candidates)
raise typer.BadParameter(f"{ref!r} matches several results:\n {names}")
return candidates[0]
def _fmt(value: float | None, places: int = 1) -> str:
return "-" if value is None else f"{value:.{places}f}"
def _delta_cell(d, places: int = 1) -> str:
"""A delta with its sign, its percent, and — crucially — whether that is good."""
if d.absolute is None:
return "[dim]-[/dim]"
verdict = direction(d.name.split(".")[-1], d)
colour = {"better": "green", "worse": "red", "flat": "dim"}[verdict]
pct = "" if d.percent is None else f" ({d.percent:+.1f}%)"
return f"[{colour}]{d.absolute:+.{places}f}{pct}[/{colour}]"
@app.command("compare")
def compare_cmd(
before: str = typer.Argument(..., help="Baseline run: path, filename, or run-id fragment"),
after: str = typer.Argument(..., help="Run to compare against the baseline"),
show_flips: int = typer.Option(20, help="How many changed samples to list (0 for all)"),
) -> None:
"""Diff two score cards: what changed, and whether the comparison is honest."""
before_path, after_path = _resolve_run(before), _resolve_run(after)
b, a = Run.load(before_path), Run.load(after_path)
result = compare_runs(b, a)
console.print()
console.rule("[bold]compare[/bold]")
console.print(f"[dim]before[/dim] {before_path.name}")
console.print(f"[dim]after [/dim] {after_path.name}\n")
# What actually differs about the two targets. Printed FIRST: without it the reader has
# no idea what the numbers below are attributable to.
if result.target_deltas or result.host_changed:
t = Table(title="what changed about the target", header_style="bold")
for col in ("axis", "before", "after"):
t.add_column(col, overflow="fold")
for d in result.target_deltas:
t.add_row(d.field, str(d.before), str(d.after))
if result.host_changed:
h = result.host_changed
t.add_row("[yellow]host[/yellow]", str(h.before), str(h.after))
console.print(t)
for warning in result.warnings:
console.print(f"[yellow]![/yellow] {warning}")
if result.warnings:
console.print()
if result.quality:
t = Table(title="quality", header_style="bold")
for col in ("metric", "before", "after", "delta"):
t.add_column(col)
for d in result.quality:
t.add_row(d.name, _fmt(d.before, 3), _fmt(d.after, 3), _delta_cell(d, 3))
console.print(t)
if result.flips:
shown = result.flips if show_flips == 0 else result.flips[:show_flips]
# When nothing about the measurement changed -- same target, same host, same eval
# set, same sampling -- a flipped sample did not regress or get fixed. It is the
# same question answered twice with different luck. Calling it "regressed" sends
# someone hunting a cause that does not exist, which is worse than saying nothing.
same_setup = (
not result.target_deltas
and not result.host_changed
and not any(
"edited between runs" in w or "Sampling changed" in w
for w in result.warnings
)
)
t = Table(
title=(
f"samples that changed ({len(result.flips)})"
+ (" — same setup, so this is instability, not change" if same_setup else "")
),
header_style="bold",
)
for col in ("task", "sample", "before", "after", ""):
t.add_column(col, overflow="fold")
for f in shown:
if same_setup:
mark = "[yellow]unstable[/yellow]"
else:
mark = {
"fail": "[red]regressed[/red]",
"pass": "[green]fixed[/green]",
"changed": "[dim]score[/dim]",
}[f.became]
t.add_row(f.task, f.sample_id, f"{f.before:.2f}", f"{f.after:.2f}", mark)
console.print(t)
if len(shown) < len(result.flips):
console.print(f"[dim]… {len(result.flips) - len(shown)} more; --show-flips 0 for all[/dim]")
elif result.quality:
console.print("[dim]no sample outcomes changed[/dim]")
if result.perf:
t = Table(title="serving performance", header_style="bold")
for col in ("concurrency", "metric", "before", "after", "delta"):
t.add_column(col)
for concurrency in sorted(result.perf):
for i, d in enumerate(result.perf[concurrency]):
t.add_row(
str(concurrency) if i == 0 else "",
d.name.split(".")[-1],
_fmt(d.before),
_fmt(d.after),
_delta_cell(d),
)
console.print(t)
if not result.quality and not result.perf:
console.print("[yellow]neither run carries quality or perf results[/yellow]")
@app.command("results")
def results_cmd() -> None:
"""List saved score cards."""
files = sorted(RESULTS_DIR.glob("*.json"))
if not files:
console.print("[dim]no results yet[/dim]")
return
table = Table(title="results", header_style="bold")
for col in ("date", "target", "signal", "reference", "tok/s @1", "peak tok/s", "file"):
table.add_column(col)
for f in files:
d = json.loads(f.read_text())
v = d.get("verdict") or {}
table.add_row(
d["timestamp"][:10],
d["target_id"],
f"{v['signal_score']:.3f}" if v.get("signal_score") is not None else "-",
f"{v['reference_score']:.3f}" if v.get("reference_score") is not None else "-",
f"{v['single_stream_tps']:.1f}" if v.get("single_stream_tps") else "-",
f"{v['peak_throughput_tps']:.1f}" if v.get("peak_throughput_tps") else "-",
f.name,
)
console.print(table)
if __name__ == "__main__":
app()
kbench/compare.py
+335
View File
@@ -0,0 +1,335 @@
"""Diff two runs.
This is the verb the bench exists for. A single score card tells you what one target did
once; it cannot tell you whether a quantisation cost you anything, whether a checkpoint
regressed, or whether a serving flag was worth it. Those are the questions people actually
have, and all of them are differences.
Two design decisions carry most of the weight here:
1. PER-SAMPLE DIFFS, NOT AGGREGATE DIFFS. "0.81 → 0.78" tells you something got worse and
nothing about what. Listing the samples that flipped tells you *which capability broke*,
which is the difference between a leaderboard and a debugging tool. The result schema
stores per-sample outcomes precisely so this is possible.
2. COMPARABILITY IS CHECKED, NOT ASSUMED. Two runs are only honestly comparable when the
thing you did not change actually did not change. Comparing a quant against a different
quant on a different host tells you nothing, but it renders just as confidently as a
clean comparison. So the diff states what differs about the targets themselves, and
warns when more than one axis moved at once.
Perf and quality are diffed independently: a run may have one, both, or neither, and a
target with no signal-tier score is still worth comparing on throughput.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
# Below this, a rate difference is not worth reading as a change. Applied to rates only: a
# single sample flipping is always worth showing.
#
# Calibrate this against your own hardware before trusting it. The two runs committed here are
# the same target on the same box on the same day, and throughput between them differs by ~10%
# — so on that machine, anything under 10% is indistinguishable from running it twice. The
# floor below is deliberately lower than that observed spread, because suppressing a real
# regression is worse than showing noise the reader can dismiss; the identical-target warning
# is what tells them which they are looking at.
PERF_NOISE_FLOOR_PCT = 5.0
# The axes that make a target a target. Comparing across more than one at a time produces a
# number that cannot be attributed to anything.
TARGET_AXES = ("model", "quantization", "checkpoint", "serving_config", "engine")
@dataclass
class FieldDelta:
"""One axis on which two targets differ."""
field: str
before: Any
after: Any
@dataclass
class SampleFlip:
"""A sample whose outcome changed between runs."""
sample_id: str
task: str
before: float
after: float
became: str # "pass" | "fail" | "changed"
@dataclass
class MetricDelta:
name: str
before: float | None
after: float | None
@property
def absolute(self) -> float | None:
if self.before is None or self.after is None:
return None
return self.after - self.before
@property
def percent(self) -> float | None:
if self.before in (None, 0) or self.after is None:
return None
return (self.after - self.before) / abs(self.before) * 100.0
@property
def significant(self) -> bool:
"""Whether a rate moved enough to be worth reading."""
pct = self.percent
return pct is not None and abs(pct) >= PERF_NOISE_FLOOR_PCT
@dataclass
class Comparison:
before_id: str
after_id: str
target_deltas: list[FieldDelta] = field(default_factory=list)
host_changed: FieldDelta | None = None
quality: list[MetricDelta] = field(default_factory=list)
flips: list[SampleFlip] = field(default_factory=list)
perf: dict[int, list[MetricDelta]] = field(default_factory=dict)
only_in_before: list[str] = field(default_factory=list)
only_in_after: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
@property
def confounded(self) -> bool:
"""More than one axis moved, so no difference below can be attributed to any of them."""
return len(self.target_deltas) + (1 if self.host_changed else 0) > 1
def _target_field(target: dict[str, Any], name: str) -> Any:
value = target.get(name)
# serving_config is a dict; render it deterministically so a reordered YAML load does not
# read as a change.
if isinstance(value, dict):
return tuple(sorted((str(k), str(v)) for k, v in value.items()))
return value
def _diff_targets(before: dict, after: dict) -> list[FieldDelta]:
deltas = []
for axis in TARGET_AXES:
b, a = _target_field(before, axis), _target_field(after, axis)
if b != a:
deltas.append(FieldDelta(axis, b, a))
return deltas
def _sample_index(run: dict) -> dict[tuple[str, str], dict]:
"""(task, sample_id) -> outcome. Keyed on both because ids are only unique within a task."""
index = {}
for quality in run.get("quality") or []:
task = quality.get("task", "?")
for sample in quality.get("samples") or []:
index[(task, sample["sample_id"])] = sample
return index
def _perf_index(run: dict) -> dict[int, dict]:
perf = run.get("perf") or {}
return {p["concurrency"]: p for p in perf.get("points") or []}
def compare(before: dict, after: dict) -> Comparison:
"""Diff two loaded result dicts. Neither is mutated."""
result = Comparison(
before_id=before.get("run_id", "?"),
after_id=after.get("run_id", "?"),
target_deltas=_diff_targets(before.get("target") or {}, after.get("target") or {}),
)
b_host = (before.get("host") or {}).get("id") or (before.get("host") or {}).get("name")
a_host = (after.get("host") or {}).get("id") or (after.get("host") or {}).get("name")
if b_host != a_host:
result.host_changed = FieldDelta("host", b_host, a_host)
# A contaminated run has no comparable quality half, and diffing against one produces a
# delta that looks exactly like a real regression or gain. Say so before anything else.
for label, run in (("before", before), ("after", after)):
state = (run.get("verdict") or {}).get("contamination")
if state == "detected":
result.warnings.append(
f"The {label} run is CONTAMINATED — it reproduced canary probes. Its quality "
"numbers are memorisation, not capability, and no quality delta below means "
"anything."
)
elif state == "unverified":
result.warnings.append(
f"The {label} run has no contamination probe, so its quality scores are "
"unverified."
)
if not result.target_deltas and not result.host_changed:
# Same target, same box: any difference is run-to-run variance, which is worth knowing
# because it sets the floor below which no other comparison means anything.
result.warnings.append(
"Identical target and host — this measures run-to-run variance, not a change."
)
if result.confounded:
moved = [d.field for d in result.target_deltas] + (["host"] if result.host_changed else [])
result.warnings.append(
f"{len(moved)} axes changed at once ({', '.join(moved)}) — no difference below "
"can be attributed to any one of them."
)
# ---- quality -----------------------------------------------------------------
b_metrics = {q["task"]: q.get("metrics", {}) for q in before.get("quality") or []}
a_metrics = {q["task"]: q.get("metrics", {}) for q in after.get("quality") or []}
for task in sorted(set(b_metrics) | set(a_metrics)):
keys = set(b_metrics.get(task, {})) | set(a_metrics.get(task, {}))
for key in sorted(keys):
result.quality.append(
MetricDelta(
f"{task}.{key}",
b_metrics.get(task, {}).get(key),
a_metrics.get(task, {}).get(key),
)
)
b_samples, a_samples = _sample_index(before), _sample_index(after)
for key in sorted(b_samples.keys() & a_samples.keys()):
b, a = b_samples[key], a_samples[key]
if b.get("passed") == a.get("passed") and b.get("score") == a.get("score"):
continue
if b.get("passed") and not a.get("passed"):
became = "fail"
elif a.get("passed") and not b.get("passed"):
became = "pass"
else:
became = "changed"
result.flips.append(
SampleFlip(key[1], key[0], b.get("score", 0.0), a.get("score", 0.0), became)
)
# Regressions first: a capability that broke is the reason anyone runs this.
result.flips.sort(key=lambda f: ({"fail": 0, "changed": 1, "pass": 2}[f.became], f.sample_id))
# A sample present in only one run is not a flip — it is a changed eval set, which
# silently moves an aggregate. Surfaced separately so it cannot be mistaken for signal.
result.only_in_before = sorted(f"{t}/{s}" for t, s in b_samples.keys() - a_samples.keys())
result.only_in_after = sorted(f"{t}/{s}" for t, s in a_samples.keys() - b_samples.keys())
if not b_samples or not a_samples:
# Not a changed eval set — one side simply never measured quality. Saying "100 samples
# added" here would be a confusing way to report "the baseline is perf-only".
if b_samples or a_samples:
missing = "before" if not b_samples else "after"
result.warnings.append(
f"The {missing} run carries no quality results, so only serving performance "
"is comparable."
)
elif result.only_in_before or result.only_in_after:
result.warnings.append(
f"The eval set changed: {len(result.only_in_before)} sample(s) gone, "
f"{len(result.only_in_after)} added. Aggregate scores are not comparable."
)
# An EDITED sample keeps its id, so the add/remove diff above sees nothing. Rewording a
# prompt or loosening a regex changes what the score means while every id still lines up,
# which is the most dangerous shape of eval drift: the comparison looks clean. The
# fingerprint hashes (id, input, target), so it moves when the content does.
b_fp = {q["task"]: q.get("dataset_fingerprint") for q in before.get("quality") or []}
a_fp = {q["task"]: q.get("dataset_fingerprint") for q in after.get("quality") or []}
for task in sorted(b_fp.keys() & a_fp.keys()):
bf, af = b_fp[task], a_fp[task]
if bf and af and bf != af:
result.warnings.append(
f"Task {task!r} was edited between runs ({bf} -> {af}): same sample ids, "
"different content. The scores measure different questions."
)
elif not bf or not af:
result.warnings.append(
f"Task {task!r} has no dataset fingerprint on one side, so an edit to the "
"sample text cannot be ruled out. Re-run to get a comparable pair."
)
# Sampling is part of the identity of a quality result. The same model at temperature
# 1 and at 0.6 is two different measurements, and unpinned sampling moved this bench's
# score by 27% of its samples between identical runs -- far more than most real
# regressions. A card that predates sampling being recorded cannot be ruled comparable
# either, so say so rather than diff it silently.
b_s, a_s = before.get("sampling") or {}, after.get("sampling") or {}
if b_samples and a_samples:
if not b_s or not a_s:
result.warnings.append(
"One run does not record how the model was sampled, so an unpinned "
"temperature cannot be ruled out. Quality deltas below may be noise."
)
else:
changed = sorted(
k for k in set(b_s) | set(a_s) if b_s.get(k) != a_s.get(k)
)
# Epoch count is not a sampling change. Both sides estimate the same quantity;
# more epochs just estimates it better. Reporting that as "not comparable"
# overstates it, and a warning that overstates is one people learn to skip --
# which costs you the ones that matter.
decode = [k for k in changed if k != "epochs"]
if decode:
result.warnings.append(
f"Sampling changed between runs ({', '.join(decode)}): "
f"{ {k: b_s.get(k) for k in decode} } -> { {k: a_s.get(k) for k in decode} }. "
"Quality scores are not comparable across different sampling."
)
if "epochs" in changed:
b_e, a_e = b_s.get("epochs", 1), a_s.get("epochs", 1)
result.warnings.append(
f"Epochs differ ({b_e} -> {a_e}). Both estimate the same score; the "
f"{'before' if (b_e or 1) < (a_e or 1) else 'after'} run is the noisier "
"of the two, so read small deltas with that in mind."
)
# ---- perf --------------------------------------------------------------------
b_perf, a_perf = _perf_index(before), _perf_index(after)
for concurrency in sorted(b_perf.keys() & a_perf.keys()):
b, a = b_perf[concurrency], a_perf[concurrency]
result.perf[concurrency] = [
MetricDelta(name, b.get(name), a.get(name))
for name in (
"output_tps_total",
"output_tps_per_stream",
"ttft_p50_ms",
"ttft_p95_ms",
"tpot_p50_ms",
# Higher is better, so the default direction rule is already right. Diffing it
# is the point of an MTP comparison: throughput can move for many reasons,
# acceptance moves only because drafting got better or worse.
"spec_acceptance_rate",
)
if b.get(name) is not None or a.get(name) is not None
]
modes = {bool(p.get("natural_stop")) for p in list(b_perf.values()) + list(a_perf.values())}
if len(modes) > 1:
result.warnings.append(
"One run pinned output length and the other let the model stop naturally. "
"Throughput is not comparable across those two modes."
)
dropped = sorted(b_perf.keys() - a_perf.keys())
added = sorted(a_perf.keys() - b_perf.keys())
if dropped or added:
result.warnings.append(
f"Concurrency sweep differs: {dropped or 'none'} dropped, {added or 'none'} added. "
"Peak throughput is only comparable across a shared sweep."
)
return result
# Metrics where a smaller number is the better one, so a fall is an improvement.
LOWER_IS_BETTER = ("ttft_p50_ms", "ttft_p95_ms", "tpot_p50_ms")
def direction(metric: str, delta: MetricDelta) -> str:
""""better" | "worse" | "flat" — meaning, not just sign."""
if delta.absolute is None or not delta.significant:
return "flat"
improved = delta.absolute < 0 if metric in LOWER_IS_BETTER else delta.absolute > 0
return "better" if improved else "worse"
kbench/perf.py
+319
View File
@@ -0,0 +1,319 @@
"""Serving performance measurement.
A self-contained async load generator rather than a wrapper around
`vllm bench serve`. Reasons:
* It works against ANY OpenAI-compatible endpoint -- vLLM, SGLang,
llama.cpp, ollama, or a submitted model served however we choose to serve
it. Submissions are the point of the site; we cannot assume vLLM.
* It needs nothing installed on the target box. The runner talks HTTP.
* We control the metric definitions, so numbers stay comparable across
engines rather than inheriting each engine's benchmarking conventions.
TWO MEASUREMENT TRAPS THIS CODE AVOIDS -- both silently produce numbers that
look great and mean nothing:
1. PREFIX CACHING. vLLM caches shared prompt prefixes. If every concurrent
request sends the same prompt, prefill is nearly free after the first and
throughput is wildly overstated. Every request here gets a unique nonce
prefix so each one actually does its own prefill.
2. VARIABLE OUTPUT LENGTH. If the model decides when to stop, concurrency
levels finish at different token counts and tok/s is not comparable
between runs. We send `ignore_eos` so every request emits exactly
`output_tokens` tokens.
"""
from __future__ import annotations
import asyncio
import json
import statistics
import time
import uuid
from dataclasses import dataclass
import httpx
from .results import PerfPoint, PerfResult
from .specdec import read_counters
# Filler vocabulary for synthetic prompts. Ordinary words rather than repeated
# junk so tokenization stays close to what real traffic looks like.
_FILLER = (
"the quick brown fox jumps over a lazy dog while distant thunder rolls "
"across open water and the harbour lights flicker against low cloud cover "
"as fishing boats return with the morning tide carrying nets and crates "
).split()
def _synthetic_prompt(approx_tokens: int, nonce: str) -> str:
"""Build a prompt of roughly `approx_tokens` tokens, unique per request.
The nonce leads so it lands in the first block and defeats prefix-cache
reuse across concurrent requests.
"""
# ~0.75 tokens per word for ordinary English text.
n_words = max(1, int(approx_tokens / 0.75))
body = " ".join(_FILLER[i % len(_FILLER)] for i in range(n_words))
return f"[{nonce}] {body}"
@dataclass
class RequestResult:
ok: bool
ttft_ms: float | None = None
total_s: float | None = None
output_tokens: int = 0
input_tokens: int = 0
error: str | None = None
@property
def tpot_ms(self) -> float | None:
"""Inter-token latency: decode time divided by tokens after the first."""
if self.ttft_ms is None or self.total_s is None or self.output_tokens < 2:
return None
decode_ms = self.total_s * 1000 - self.ttft_ms
return decode_ms / (self.output_tokens - 1)
async def _one_request(
client: httpx.AsyncClient,
base_url: str,
model: str,
input_tokens: int,
output_tokens: int,
prompt: str | None = None,
pin_output: bool = True,
) -> RequestResult:
nonce = uuid.uuid4().hex[:12]
# A supplied prompt still gets the nonce, because prefix-cache defeat is orthogonal to
# whether the text is representative — reusing one real prompt across a concurrency level
# would let the engine serve most of it from cache and report an inflated number.
content = f"[{nonce}] {prompt}" if prompt else _synthetic_prompt(input_tokens, nonce)
payload = {
"model": model,
"messages": [{"role": "user", "content": content}],
"max_tokens": output_tokens,
"temperature": 0.0,
"stream": True,
"stream_options": {"include_usage": True},
}
if pin_output:
# vLLM/SGLang extension: force exactly max_tokens of output so runs are
# comparable. Harmless (ignored) on engines that do not support it.
#
# Turned OFF for speculative-decoding measurement. Forcing generation past a natural
# stop produces degenerate continuation, and a draft model's acceptance rate on
# degenerate text is not its acceptance rate on real work — so pinning the length here
# buys comparability at the cost of measuring the wrong thing.
payload["ignore_eos"] = True
start = time.perf_counter()
ttft: float | None = None
counted = 0
usage_in = 0
usage_out = 0
try:
async with client.stream(
"POST", f"{base_url}/chat/completions", json=payload
) as resp:
if resp.status_code != 200:
body = (await resp.aread()).decode()[:200]
return RequestResult(ok=False, error=f"HTTP {resp.status_code}: {body}")
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
data = line[6:]
if data == "[DONE]":
break
try:
chunk = json.loads(data)
except json.JSONDecodeError:
continue
if usage := chunk.get("usage"):
usage_in = usage.get("prompt_tokens", 0)
usage_out = usage.get("completion_tokens", 0)
for choice in chunk.get("choices") or []:
delta = choice.get("delta") or {}
# Engines disagree on where reasoning text lands: vLLM with
# --reasoning-parser emits `reasoning`, others use
# `reasoning_content`. Count all of them -- reasoning tokens
# cost the same decode time and the user waits for them
# either way. Checking only `content` would report TTFT as
# the time to the *answer*, which on a reasoning model is
# thousands of tokens late.
if any(
isinstance(delta.get(k), str) and delta[k]
for k in ("content", "reasoning", "reasoning_content")
):
# The opening chunk carries role + empty content; it is
# the stream opening, not a token, and is excluded by
# the emptiness check above.
if ttft is None:
ttft = (time.perf_counter() - start) * 1000
counted += 1
except Exception as exc: # noqa: BLE001 - report, do not abort the sweep
return RequestResult(ok=False, error=f"{type(exc).__name__}: {exc}")
total = time.perf_counter() - start
return RequestResult(
ok=True,
ttft_ms=ttft,
total_s=total,
# Prefer server-reported usage; fall back to counted chunks.
output_tokens=usage_out or counted,
input_tokens=usage_in,
)
async def _sweep_point(
base_url: str,
model: str,
concurrency: int,
input_tokens: int,
output_tokens: int,
timeout_s: float,
prompts: list[str] | None = None,
pin_output: bool = True,
) -> PerfPoint:
limits = httpx.Limits(max_connections=concurrency + 8)
async with httpx.AsyncClient(timeout=timeout_s, limits=limits) as client:
# Warm up so the first measured request does not absorb graph capture,
# weight paging, or connection setup.
await _one_request(client, base_url, model, 32, 8)
# Bracket the measured window. These are lifetime counters, so the rate for THIS point
# is the delta — reading the total would fold in warm-up and every earlier point.
before = await read_counters(client, base_url)
start = time.perf_counter()
results = await asyncio.gather(
*(
_one_request(
client,
base_url,
model,
input_tokens,
output_tokens,
# Cycle rather than repeat: every concurrent request in a point gets a
# different prompt, so one unusually easy or hard example cannot set the
# whole level's number.
prompt=prompts[i % len(prompts)] if prompts else None,
pin_output=pin_output,
)
for i in range(concurrency)
)
)
wall = time.perf_counter() - start
after = await read_counters(client, base_url)
ok = [r for r in results if r.ok]
failed = [r for r in results if not r.ok]
total_out = sum(r.output_tokens for r in ok)
ttfts = [r.ttft_ms for r in ok if r.ttft_ms is not None]
tpots = [r.tpot_ms for r in ok if r.tpot_ms is not None]
per_stream = [
r.output_tokens / r.total_s for r in ok if r.total_s and r.output_tokens
]
def pct(values: list[float], p: float) -> float | None:
if not values:
return None
s = sorted(values)
idx = min(len(s) - 1, round(p * (len(s) - 1)))
return round(s[idx], 2)
# Prefill rate: input tokens processed per second, inferred from TTFT.
# At concurrency > 1 this is per-stream and includes queueing, so it reads
# low; treat the concurrency-1 value as the true prefill capability.
prefill = None
if ttfts and ok:
reported = [r.input_tokens for r in ok if r.input_tokens]
mean_in = statistics.mean(reported) if reported else input_tokens
mean_ttft_s = statistics.mean(ttfts) / 1000
if mean_ttft_s > 0:
prefill = round(mean_in / mean_ttft_s, 1)
# Delta across the measured window only. `None` when the engine exposes no counters, which
# is an ordinary outcome (no speculative decoding, or metrics disabled) and not an error.
spec = (after - before) if (before and after) else None
acceptance = spec.acceptance_rate if spec else None
drafted = int(spec.draft) if spec and spec.draft > 0 else None
# With a natural stop, requests no longer emit identical token counts, so the spread is
# itself a caveat on comparability and has to travel with the number.
out_counts = [r.output_tokens for r in ok if r.output_tokens]
stdev = (
round(statistics.stdev(out_counts), 2)
if not pin_output and len(out_counts) > 1
else None
)
return PerfPoint(
concurrency=concurrency,
input_tokens=input_tokens,
output_tokens=output_tokens,
n_requests=concurrency,
completed=len(ok),
failed=len(failed),
duration_s=round(wall, 3),
output_tps_total=round(total_out / wall, 2) if wall else 0.0,
output_tps_per_stream=round(statistics.mean(per_stream), 2) if per_stream else 0.0,
ttft_p50_ms=pct(ttfts, 0.50),
ttft_p95_ms=pct(ttfts, 0.95),
tpot_p50_ms=pct(tpots, 0.50),
prefill_tps=prefill,
spec_acceptance_rate=acceptance,
spec_draft_tokens=drafted,
natural_stop=not pin_output,
output_token_stdev=stdev,
error=failed[0].error if failed else None,
)
async def run_sweep(
base_url: str,
model: str,
concurrencies: tuple[int, ...] = (1, 8, 32),
input_tokens: int = 1024,
output_tokens: int = 256,
timeout_s: float = 600.0,
engine: str = "unknown",
progress: bool = True,
prompts: list[str] | None = None,
pin_output: bool = True,
) -> PerfResult:
"""Run a concurrency sweep and return a PerfResult.
`prompts` and `pin_output=False` together give the mode a speculative-decoding target needs:
representative text, stopping naturally. Both defaults stay as they were, so an ordinary
throughput sweep is unchanged and remains directly comparable to every result already
committed.
"""
points: list[PerfPoint] = []
for c in concurrencies:
if progress:
print(f" concurrency {c:>3} ...", end="", flush=True)
point = await _sweep_point(
base_url, model, c, input_tokens, output_tokens, timeout_s,
prompts=prompts, pin_output=pin_output,
)
points.append(point)
if progress:
status = f"{point.output_tps_total:>8.1f} tok/s total"
if point.spec_acceptance_rate is not None:
status += f" accept {point.spec_acceptance_rate:.0%}"
status += f" | {point.output_tps_per_stream:>6.1f} /stream"
if point.ttft_p50_ms is not None:
status += f" | TTFT p50 {point.ttft_p50_ms:>7.0f}ms"
if point.failed:
status += f" | {point.failed} FAILED"
print(status, flush=True)
return PerfResult(engine=engine, points=points)
kbench/registry/__init__.py
+167
View File
@@ -0,0 +1,167 @@
"""Model registry.
A *target* is not a model. It is a specific way of serving a specific model:
(model × host × quantization × serving config × checkpoint)
This is the central design decision of the bench and it cannot be retrofitted
later without invalidating every result already recorded. On unified-memory
boxes like the DGX Spark, serving flags move throughput more than a model swap
does -- `--enforce-eager` alone can cost 30%+, and `--gpu-memory-utilization`
determines how much KV cache exists, which sets the concurrency ceiling. A
registry keyed on model name alone produces numbers nobody can reproduce.
`checkpoint` is first-class for the same reason: once we fine-tune our own
models, we will compare `karti-7b@step2000` against `karti-7b@step4000` far
more often than we compare Qwen against Llama.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import yaml
REGISTRY_PATH = Path(__file__).parent / "models.yaml"
@dataclass(frozen=True)
class Host:
"""A machine that can serve models."""
id: str
ssh: str | None = None
hardware: str | None = None
memory_gb: int | None = None
memory_bandwidth_gbs: int | None = None
notes: str | None = None
@dataclass(frozen=True)
class Serving:
"""How a target is served. Part of the identity of a result."""
engine: str # vllm | sglang | llama.cpp | anthropic | openai | ...
model_name: str # value sent as "model" in the API request
base_url: str | None = None
service: str | None = None # inspect openai-api service prefix
max_model_len: int | None = None
flags: dict[str, Any] = field(default_factory=dict)
# How to sample this model during quality runs. Part of the identity of a result: the
# same model at temperature 1 and at temperature 0.6 is two different measurements.
#
# Not a global constant, because the right value is a property of the model. Greedy
# decoding looks like the obvious choice for reproducibility and is actively wrong for
# reasoning models -- Qwen3 in thinking mode degenerates into repetition at
# temperature 0, and a 2-sample probe that took two minutes ran past twelve without
# terminating. Reproducibility comes from pinning the seed, not from killing entropy.
sampling: dict[str, Any] = field(default_factory=dict)
@property
def is_local(self) -> bool:
return self.engine in {"vllm", "sglang", "llama.cpp", "ollama"}
@dataclass(frozen=True)
class Target:
"""One benchmarkable configuration."""
id: str
display_name: str
serving: Serving
host: str
family: str | None = None
params_b: float | None = None
active_params_b: float | None = None # MoE: params active per token
quant: str | None = None
checkpoint: str | None = None
tier: str = "candidate" # production | candidate | reference | baseline
aliases: list[str] = field(default_factory=list)
notes: str | None = None
@property
def inspect_model(self) -> str:
"""The model string Inspect uses to address this target."""
s = self.serving
if s.engine in {"anthropic", "openai", "google", "openrouter", "grok"}:
return f"{s.engine}/{s.model_name}"
if s.is_local:
if not s.service:
raise ValueError(f"target {self.id}: local serving requires a 'service'")
# Inspect convention: openai-api/<service>/<model>, with base_url and
# api key read from <SERVICE>_BASE_URL / <SERVICE>_API_KEY.
return f"openai-api/{s.service}/{s.model_name}"
raise ValueError(f"target {self.id}: unsupported engine {s.engine!r}")
@property
def env(self) -> dict[str, str]:
"""Environment variables Inspect needs to reach this target."""
s = self.serving
if not s.is_local or not s.service:
return {}
prefix = s.service.upper().replace("-", "_")
env = {f"{prefix}_API_KEY": os.environ.get(f"{prefix}_API_KEY", "no-key-required")}
if s.base_url:
env[f"{prefix}_BASE_URL"] = s.base_url
return env
def apply_env(self) -> None:
"""Export this target's env vars into the current process."""
os.environ.update(self.env)
@property
def slug(self) -> str:
"""Filesystem-safe identifier used in results filenames."""
return self.id.replace("/", "_").replace("@", "__")
@dataclass(frozen=True)
class Registry:
hosts: dict[str, Host]
targets: dict[str, Target]
def target(self, ident: str) -> Target:
"""Look up a target by id or alias."""
if ident in self.targets:
return self.targets[ident]
for t in self.targets.values():
if ident in t.aliases:
return t
known = ", ".join(sorted(self.targets))
raise KeyError(f"unknown target {ident!r}. known targets: {known}")
def host(self, ident: str) -> Host:
if ident not in self.hosts:
raise KeyError(f"unknown host {ident!r}")
return self.hosts[ident]
def by_tier(self, tier: str) -> list[Target]:
return [t for t in self.targets.values() if t.tier == tier]
def load_registry(path: Path | None = None) -> Registry:
raw = yaml.safe_load((path or REGISTRY_PATH).read_text())
hosts = {
hid: Host(id=hid, **(spec or {}))
for hid, spec in (raw.get("hosts") or {}).items()
}
targets: dict[str, Target] = {}
for spec in raw.get("targets") or []:
spec = dict(spec)
serving = Serving(**spec.pop("serving"))
target = Target(serving=serving, **spec)
if target.host not in hosts:
raise ValueError(f"target {target.id}: unknown host {target.host!r}")
if target.id in targets:
raise ValueError(f"duplicate target id {target.id!r}")
targets[target.id] = target
return Registry(hosts=hosts, targets=targets)
__all__ = ["Host", "Registry", "Serving", "Target", "load_registry"]
kbench/registry/models.yaml
+109
View File
@@ -0,0 +1,109 @@
# Lumbridge Bench target registry
#
# A target = (model x host x quant x serving config x checkpoint).
# See kbench/registry/__init__.py for why the key is this wide.
#
# `serving.flags` records the ACTUAL flags the server was started with. Snapshot
# them from a live host with: kbench snapshot <host>
# Results are only comparable between targets with identical flags.
hosts:
your-node:
ssh: your-node
hardware: NVIDIA GB10 Grace Blackwell (DGX Spark)
memory_gb: 121
memory_bandwidth_gbs: 273
notes: >
Unified LPDDR5X. Memory bandwidth is the binding constraint on decode
throughput, not compute -- large dense models are bandwidth-starved here
while MoE models with small active-param counts do comparatively well.
metal:
ssh: your-second-node
hardware: TBD
notes: Secondary inference box. Not yet benchmarked.
your-second-node:
ssh: root@your-second-node
hardware: TBD
notes: Storage origin box, 761GB. Not yet benchmarked.
api:
hardware: hosted
notes: Frontier APIs. Used as quality ceiling and as judge models.
targets:
# ---------------------------------------------------------------------
# PRODUCTION -- currently serving live traffic, so regressions here matter
# ---------------------------------------------------------------------
- id: qwen3.6-35b-a3b-nvfp4@your-node
display_name: Qwen3.6-35B-A3B (NVFP4)
family: qwen3.6
params_b: 35
active_params_b: 3
quant: nvfp4
host: your-node
tier: production
aliases: [brain, spark]
notes: >
Live behind the `brain` and `local-moe` aliases. This is the
model the bench exists to interrogate: it was deployed without
measurement.
Flags below were snapshotted from the live process on 2026-08-03 and had
drifted badly from what was recorded here: gpu_memory_utilization was
0.25 not 0.55, the MoE backend was flashinfer_cutlass not flashinfer_b12x,
and MTP speculative decoding was running but undocumented. Since target
identity IS the serving config, score cards taken against the old entry
would have been mislabeled. Re-snapshot before trusting a comparison.
FLAGS OF CONCERN, in the order worth testing:
(1) --max-num-seqs 4 caps concurrency at four sequences, so a throughput
sweep past 4 measures queueing, not the server.
(2) --gpu-memory-utilization 0.25 leaves ~75% of unified memory unused,
capping KV cache and therefore concurrency.
(3) --enforce-eager disables CUDA graphs, which costs throughput.
(4) MTP is on; report spec_acceptance_rate, since a low acceptance rate
means the draft tokens are wasted work.
serving:
engine: vllm
service: spark
base_url: http://your-node:8001/v1
model_name: brain
max_model_len: 65536
# Qwen3's documented values for thinking mode. Do NOT set temperature 0 here: this
# model degenerates into endless repetition under greedy decoding, and a two-sample
# probe that normally finishes in two minutes ran past twelve without terminating.
# The seed is what makes a re-run a re-run.
sampling:
temperature: 0.6
top_p: 0.95
top_k: 20
seed: 20260804
max_tokens: 4096
flags:
async_scheduling: true
enforce_eager: true
gpu_memory_utilization: 0.25
kv_cache_dtype: fp8
max_num_batched_tokens: 4096
max_num_seqs: 4
moe_backend: flashinfer_cutlass
reasoning_parser: qwen3
speculative_config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}'
tool_call_parser: qwen3_coder
enable_auto_tool_choice: true
# ---------------------------------------------------------------------
# BASELINE -- frontier APIs. Quality ceiling, and the judge for graded tasks.
# Uncomment once the corresponding API key is in .env.
# ---------------------------------------------------------------------
# - id: claude-opus-5@api
# display_name: Claude Opus 5
# family: claude
# host: api
# tier: baseline
# aliases: [opus]
# serving:
# engine: anthropic
# model_name: claude-opus-5
kbench/results.py
+254
View File
@@ -0,0 +1,254 @@
"""Result schema.
One run produces one JSON file in results/, committed to git. The commit
history IS the longitudinal record -- there is no database to migrate or rot.
Two properties of this schema are load-bearing and expensive to retrofit:
1. PER-SAMPLE OUTCOMES, not just aggregates. An aggregate tells you a
checkpoint got worse. Per-sample diffs tell you *which capability broke*.
That difference is what separates a leaderboard from a debugging tool, and
it is the entire reason this bench is useful for fine-tuning.
2. THE FULL TARGET IS SNAPSHOTTED INTO THE RESULT. Registry entries change.
A result that merely references a target id becomes unreproducible the
first time someone edits models.yaml. Every run carries its own copy of
the serving flags it actually ran against.
Output storage policy: we store the score plus a short excerpt per sample.
Full model outputs stay in the raw Inspect .eval log, which is gitignored --
they are large, and for signal-tier tasks they can echo sensitive content
from the prompts they were derived from. Deep debugging uses the local log;
git carries the shape of the failure, not the whole transcript.
"""
from __future__ import annotations
import json
import platform
from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
SCHEMA_VERSION = 1
RESULTS_DIR = Path(__file__).resolve().parent.parent / "results"
def _json_safe(value: Any) -> Any:
"""Map non-finite floats to None, recursively.
Python's json emits bare `NaN`/`Infinity`, which its own loader accepts and almost
nothing else does — JSON.parse, Go, and serde all reject them. A score card is meant
to be a portable, durable record, so a card that only Python can read is a broken one.
An ungraded sample surfaced as NaN here and made the first committed card unparseable
by the website that has to render it.
None means "no score", which is exactly what a non-finite score meant anyway.
"""
if isinstance(value, float):
# NaN != NaN, and inf fails the range check; both are unrepresentable in JSON.
return None if value != value or value in (float("inf"), float("-inf")) else value
if isinstance(value, dict):
return {k: _json_safe(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [_json_safe(v) for v in value]
return value
# How much of each model output to keep in the committed record.
EXCERPT_CHARS = 400
def utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
@dataclass
class SampleOutcome:
"""One sample's result. The unit of regression debugging."""
sample_id: str
score: float
passed: bool
excerpt: str | None = None # truncated model output; full text in .eval log
metadata: dict[str, Any] = field(default_factory=dict)
@dataclass
class QualityResult:
"""One task's scores against one target."""
task: str
tier: str # "reference" (public, contaminated, calibration only) | "signal"
dataset_version: str | None = None
# Content hash of the samples actually run. dataset_version is inspect's static
# task_version and does not move when the data changes; this does. Two cards are
# only comparable if their fingerprints match.
dataset_fingerprint: str | None = None
n_samples: int = 0
metrics: dict[str, float] = field(default_factory=dict)
samples: list[SampleOutcome] = field(default_factory=list)
duration_s: float | None = None
error: str | None = None
@dataclass
class PerfPoint:
"""Serving performance at one concurrency level."""
concurrency: int
input_tokens: int
output_tokens: int
n_requests: int
completed: int
failed: int
duration_s: float
# Aggregate throughput across all streams -- the number that matters for
# multi-user serving, and the one single-stream reviews miss entirely.
output_tps_total: float
# Per-stream decode rate -- what a single user actually feels.
output_tps_per_stream: float
ttft_p50_ms: float | None = None
ttft_p95_ms: float | None = None
tpot_p50_ms: float | None = None # time per output token, inter-token latency
prefill_tps: float | None = None
# Speculative decoding, when the engine reports it. Throughput alone cannot say whether
# drafting is working: a draft rejected almost every time still emits tokens, having paid
# for both passes. `None` means the engine exposed no counters, not that the rate was zero.
spec_acceptance_rate: float | None = None
spec_draft_tokens: int | None = None
# True when output length was NOT pinned with ignore_eos. Comparability across points is
# weaker in that mode, and a reader has to know which they are looking at.
natural_stop: bool = False
output_token_stdev: float | None = None
error: str | None = None
@dataclass
class PerfResult:
engine: str
points: list[PerfPoint] = field(default_factory=list)
peak_memory_gb: float | None = None
idle_memory_gb: float | None = None
notes: str | None = None
@dataclass
class Run:
"""A complete score card: one target, one point in time."""
run_id: str
timestamp: str
target_id: str
target: dict[str, Any] # full snapshot -- see module docstring
host: dict[str, Any]
quality: list[QualityResult] = field(default_factory=list)
perf: PerfResult | None = None
verdict: dict[str, Any] = field(default_factory=dict)
runner: dict[str, Any] = field(default_factory=dict)
# How the model was sampled. Two cards are only comparable if this matches: a score
# taken at temperature 1 is a draw from a distribution, and differs run to run by more
# than most real regressions do. Recorded rather than assumed, because the quality half
# sampled unpinned for its entire life and nothing in the card said so.
sampling: dict[str, Any] = field(default_factory=dict)
schema_version: int = SCHEMA_VERSION
# ---------------------------------------------------------------- io --
@property
def filename(self) -> str:
date = self.timestamp[:10]
slug = self.target.get("slug") or self.target_id.replace("@", "__")
return f"{date}__{slug}__{self.run_id[:8]}.json"
def to_dict(self) -> dict[str, Any]:
return _json_safe(asdict(self))
def save(self, results_dir: Path | None = None) -> Path:
d = results_dir or RESULTS_DIR
d.mkdir(parents=True, exist_ok=True)
path = d / self.filename
# allow_nan=False so this raises instead of emitting a token that is not JSON.
# to_dict has already mapped the non-finite values to null; if a new one appears,
# the write should fail loudly rather than commit an unparseable score card.
path.write_text(
json.dumps(self.to_dict(), indent=2, sort_keys=False, allow_nan=False) + "\n"
)
return path
@staticmethod
def load(path: Path) -> dict[str, Any]:
return json.loads(Path(path).read_text())
def runner_info() -> dict[str, Any]:
"""Provenance of the machine that drove the benchmark (not the one serving)."""
return {
"host": platform.node(),
"python": platform.python_version(),
"platform": platform.platform(),
}
def truncate(text: str | None, limit: int = EXCERPT_CHARS) -> str | None:
if text is None:
return None
text = text.strip()
return text if len(text) <= limit else text[:limit] + f"... [+{len(text) - limit} chars]"
def compute_verdict(quality: list[QualityResult], perf: PerfResult | None) -> dict[str, Any]:
"""Derive a self-hosting verdict from the raw numbers.
Deliberately simple and explicit rather than clever. The thresholds encode
what makes a model usable in our own stack, and they live in code (and
therefore in git history) so that a changed verdict is always traceable to
a changed rule rather than to drift.
"""
verdict: dict[str, Any] = {}
signal = [q for q in quality if q.tier == "signal" and q.error is None]
reference = [q for q in quality if q.tier == "reference" and q.error is None]
def mean_accuracy(results: list[QualityResult]) -> float | None:
vals = [q.metrics.get("accuracy") for q in results]
vals = [v for v in vals if v is not None]
return round(sum(vals) / len(vals), 4) if vals else None
verdict["signal_score"] = mean_accuracy(signal)
verdict["reference_score"] = mean_accuracy(reference)
# Contamination gate. A probe nobody reads is the same as no probe, so this does not just
# record a number — when a probe fires it takes `signal_score` away.
#
# Nulling it is deliberate. Anything downstream that reads `verdict["signal_score"]` —
# `kbench results`, a comparison, a published card — would otherwise happily render a
# memorised score next to a small warning, and the number is what people quote. The raw
# value is preserved under a name nobody reads by accident, so nothing is lost for
# forensics and nothing is quotable by mistake.
canary = [q for q in quality if q.tier == "canary" and q.error is None]
canary_score = mean_accuracy(canary)
verdict["canary_score"] = canary_score
if canary_score is None:
# No probe ran. The signal score stands, but it is unverified, and saying so is the
# difference between "clean" and "nobody checked".
verdict["contamination"] = "unverified"
elif canary_score > 0:
verdict["contamination"] = "detected"
verdict["signal_score_unverified"] = verdict["signal_score"]
verdict["signal_score"] = None
else:
verdict["contamination"] = "clean"
if perf and perf.points:
single = next((p for p in perf.points if p.concurrency == 1), None)
best = max(perf.points, key=lambda p: p.output_tps_total)
if single:
verdict["single_stream_tps"] = round(single.output_tps_per_stream, 1)
# Below ~15 tok/s a model reads slower than a person, which rules
# out anything interactive regardless of how good its answers are.
verdict["interactive_viable"] = single.output_tps_per_stream >= 15.0
verdict["peak_throughput_tps"] = round(best.output_tps_total, 1)
verdict["peak_throughput_concurrency"] = best.concurrency
return verdict
kbench/run.py
+358
View File
@@ -0,0 +1,358 @@
"""Run orchestration: target + tasks + perf sweep -> one committed score card."""
from __future__ import annotations
import asyncio
import hashlib
import uuid
from dataclasses import asdict
from pathlib import Path
from typing import Any
from . import tasks as task_catalog
from .perf import run_sweep
from .registry import Registry, Target
from .results import (
PerfResult,
QualityResult,
Run,
SampleOutcome,
compute_verdict,
runner_info,
truncate,
utc_now,
)
LOG_DIR = Path(__file__).resolve().parent.parent / "logs"
# Inspect's canonical scalar score values.
_CORRECTNESS = {"C": 1.0, "I": 0.0, "P": 0.5, "N": 0.0}
def _normalize_sample_score(value: Any, primary_key: str | None) -> tuple[float, bool, dict]:
"""Reduce a score of any shape to (score, passed, extra_metadata).
Inspect scorers return scalars, correctness letters, or dicts depending on
the eval. IFEval returns a dict; a simple match scorer returns "C"/"I".
Normalizing here keeps the results schema uniform, which is what makes
cross-task and cross-checkpoint comparison possible at all.
"""
extra: dict[str, Any] = {}
if isinstance(value, dict):
extra = dict(value)
if primary_key and primary_key in value:
inner = value[primary_key]
else:
# Fall back to the first bool/numeric entry so an unconfigured task
# still produces something rather than silently scoring zero.
inner = next(
(v for v in value.values() if isinstance(v, (bool, int, float))),
0.0,
)
return _normalize_sample_score(inner, None)[0], bool(inner), extra
if isinstance(value, bool):
return (1.0 if value else 0.0), value, extra
if isinstance(value, (int, float)):
# A non-finite score means the sample was never graded — a rubric sample with no
# judge configured arrives as NaN. `NaN > 0.0` is False, so it used to be recorded
# as an ordinary failure, which is a different claim: "the model got this wrong"
# rather than "nobody scored this". The mean already skips it; this makes the
# per-sample record say so too.
if isinstance(value, float) and value != value:
return value, False, {**extra, "ungraded": True}
return float(value), float(value) > 0.0, extra
if isinstance(value, str):
score = _CORRECTNESS.get(value.strip().upper()[:1], 0.0)
return score, score >= 1.0, {"raw": value}
return 0.0, False, {"raw": repr(value)}
def _quality_from_log(log: Any, spec: task_catalog.TaskSpec) -> QualityResult:
"""Extract a QualityResult (including per-sample outcomes) from an EvalLog."""
if log.status != "success":
err = getattr(log, "error", None)
return QualityResult(
task=spec.name,
tier=spec.tier,
error=str(err) if err else f"eval status: {log.status}",
)
metrics: dict[str, float] = {}
for scorer in (log.results.scores if log.results else []):
for key, metric in scorer.metrics.items():
metrics[key] = round(float(metric.value), 6)
# Promote the task's headline number to a uniform "accuracy" key so the
# leaderboard and verdict logic never need per-task special cases.
if spec.primary_metric in metrics:
metrics["accuracy"] = metrics[spec.primary_metric]
samples: list[SampleOutcome] = []
for s in (log.samples or []):
for scorer_name, score in (s.scores or {}).items():
value, passed, extra = _normalize_sample_score(
score.value, spec.primary_sample_key
)
samples.append(
SampleOutcome(
sample_id=str(s.id),
score=value,
passed=passed,
excerpt=truncate(getattr(s.output, "completion", None)),
metadata={
"scorer": scorer_name,
"epoch": s.epoch,
**extra,
},
)
)
stats = getattr(log, "stats", None)
duration = None
if stats and getattr(stats, "started_at", None) and getattr(stats, "completed_at", None):
try:
from datetime import datetime
duration = (
datetime.fromisoformat(stats.completed_at)
- datetime.fromisoformat(stats.started_at)
).total_seconds()
except Exception: # noqa: BLE001 - duration is nice-to-have, never fatal
duration = None
return QualityResult(
task=spec.name,
tier=spec.tier,
dataset_version=getattr(log.eval, "task_version", None) and str(log.eval.task_version),
dataset_fingerprint=_dataset_fingerprint(log.samples or []),
n_samples=len(log.samples or []),
metrics=metrics,
samples=samples,
duration_s=duration,
)
def _dataset_fingerprint(samples: list[Any]) -> str | None:
"""Content hash of the samples actually run.
`dataset_version` is inspect's static task_version -- it stays "2" whether the family
has twelve samples or a rewritten scorer. Editing a prompt, tightening a regex, or
dropping a sample therefore produced two cards that claimed the same dataset version
and were not comparable, with nothing to reveal it. Comparing those is the exact error
this bench exists to prevent, so the identity has to come from the content.
Hashed over (id, input, target) because those are what determine whether a score means
the same thing. Sorted, so sample ordering does not change the fingerprint.
"""
if not samples:
return None
h = hashlib.sha256()
for key in sorted(
f"{s.id}\x1f{s.input}\x1f{s.target}" for s in samples
):
h.update(key.encode())
h.update(b"\x1e")
return f"sha256:{h.hexdigest()[:16]}"
def preflight(specs: list[task_catalog.TaskSpec]) -> None:
"""Fetch scorer prerequisites before burning GPU time on a doomed run.
Scorers often touch their corpora only at scoring time, so a missing
dependency surfaces after the whole dataset has been generated. Checking
up front converts a 20-minute failure into a 2-second one.
"""
needed = {res for spec in specs for res in spec.nltk_resources}
if not needed:
return
import nltk
for resource in sorted(needed):
try:
nltk.data.find(resource)
except LookupError:
name = resource.rsplit("/", 1)[-1]
print(f"[preflight] fetching nltk resource {name}")
nltk.download(name, quiet=True)
nltk.data.find(resource) # raise loudly if it still is not there
# Fallback sampling for a target that declares none. Deliberately NOT greedy: temperature 0
# looks like the reproducible choice and breaks reasoning models, which repeat forever
# without it. Reproducibility comes from the fixed seed. A target should override this in
# the registry with the values its model card documents.
DEFAULT_SAMPLING = {
"temperature": 0.6,
"top_p": 0.95,
"seed": 20260804,
"max_tokens": 4096,
}
def resolve_sampling(target: Target) -> dict[str, Any]:
"""The sampling settings for a target, registry first, defaults second.
Returned rather than read inline so the score card records exactly what the run used:
the two cannot drift, because they are the same call.
"""
return {**DEFAULT_SAMPLING, **(target.serving.sampling or {})}
def run_quality(
target: Target,
specs: list[task_catalog.TaskSpec],
limit: int | None = None,
max_connections: int = 16,
log_dir: Path | None = None,
epochs: int = 1,
) -> list[QualityResult]:
"""Run quality tasks against a target via Inspect."""
from inspect_ai import eval as inspect_eval
preflight(specs)
target.apply_env()
results: list[QualityResult] = []
sampling = resolve_sampling(target)
print(f" sampling: {sampling} | epochs: {epochs}")
if epochs == 1:
print(
" NOTE: one epoch. Sampling is not reproducible on this server even with a\n"
" fixed seed -- continuous batching changes the logits -- so a single\n"
" epoch is one draw, not a rate. Use --epochs for a signal-tier number."
)
for spec in specs:
print(f"\n[quality] {spec.name} ({spec.tier}) -> {target.id}")
logs = inspect_eval(
tasks=spec.inspect_task,
model=target.inspect_model,
limit=limit,
epochs=epochs,
log_dir=str(log_dir or LOG_DIR),
display="plain",
# Greedy decoding. The perf harness has always pinned temperature 0; the
# quality half pinned nothing and inherited the server's default, so it
# sampled. Three runs over the same twelve samples scored 4, 5 and 7 of 11
# -- 27% of samples flipped -- and a score card that cannot reproduce its
# own number is not evidence, it is a draw from a distribution nobody
# recorded. Sampling also makes `compare` meaningless: a per-sample flip
# caused by temperature is indistinguishable from a real regression.
#
# These MUST go through config=GenerateConfig. inspect_eval takes **kwargs,
# so passing temperature= directly is accepted silently and does nothing --
# the run would have looked pinned and still sampled.
# Loose kwargs ARE the interface: inspect_eval collects everything it does
# not consume into a GenerateConfig itself, so `config=` is rejected as an
# unknown field and `max_connections` must ride along here rather than as a
# sibling argument. Verified by running it, not by reading the signature --
# these names do not appear in it.
max_connections=max_connections,
**sampling,
)
for log in logs:
qr = _quality_from_log(log, spec)
# Record the cap explicitly. A limited run that looks like a full
# run is the single easiest way to publish a misleading number.
if limit is not None:
qr.metrics["_limit"] = float(limit)
results.append(qr)
return results
def run_perf(
target: Target,
concurrencies: tuple[int, ...] = (1, 8, 32),
input_tokens: int = 1024,
output_tokens: int = 256,
prompts: list[str] | None = None,
pin_output: bool = True,
) -> PerfResult:
"""Run a serving performance sweep against a target."""
base_url = target.serving.base_url
if not base_url:
raise ValueError(f"target {target.id} has no base_url; cannot run perf sweep")
print(f"\n[perf] {target.id} @ {base_url}")
# A sweep past the server's own concurrency ceiling measures the queue, not the server.
# vLLM admits max_num_seqs sequences per step and queues the rest, so those points show
# flat total throughput and inflating TTFT — which reads as saturation and is really just
# waiting. your-node runs max_num_seqs=4 against a default sweep of 1,8,32, so two of three
# points were destined to be misread. Warn rather than refuse: measuring the queue is a
# legitimate thing to want, as long as nobody mistakes it for the engine's limit.
max_seqs = (target.serving.flags or {}).get("max_num_seqs")
if isinstance(max_seqs, int):
beyond = [c for c in concurrencies if c > max_seqs]
if beyond:
print(
f" WARNING: max_num_seqs={max_seqs}, so concurrency {beyond} exceeds what "
f"the server admits per step.\n"
f" Those points measure queueing, not serving capacity. "
f"Consider --concurrency 1,{max(2, max_seqs // 2)},{max_seqs}."
)
mode = []
if prompts:
mode.append(f"{len(prompts)} representative prompts")
if not pin_output:
mode.append("natural stop (output length not pinned)")
print(f" input={input_tokens} output={output_tokens} tokens"
+ (f" [{'; '.join(mode)}]" if mode else ""))
return asyncio.run(
run_sweep(
base_url=base_url,
model=target.serving.model_name,
concurrencies=concurrencies,
input_tokens=input_tokens,
output_tokens=output_tokens,
engine=target.serving.engine,
prompts=prompts,
pin_output=pin_output,
)
)
def _epochs_of(quality: list[QualityResult]) -> int:
"""How many times each sample was actually run, read back from the outcomes.
Taken from the results rather than the argument so the card cannot claim an averaging
it did not do -- the same reason resolve_sampling feeds the run and the card from one
call.
"""
per_sample: dict[str, int] = {}
for q in quality:
for s in q.samples:
per_sample[s.sample_id] = per_sample.get(s.sample_id, 0) + 1
return max(per_sample.values()) if per_sample else 1
def build_run(
registry: Registry,
target: Target,
quality: list[QualityResult],
perf: PerfResult | None,
) -> Run:
host = registry.host(target.host)
target_snapshot = asdict(target)
target_snapshot["slug"] = target.slug
target_snapshot["inspect_model"] = target.inspect_model
return Run(
run_id=uuid.uuid4().hex,
timestamp=utc_now(),
target_id=target.id,
target=target_snapshot,
host=asdict(host),
quality=quality,
perf=perf,
verdict=compute_verdict(quality, perf),
runner=runner_info(),
sampling={**resolve_sampling(target), "epochs": _epochs_of(quality)},
)
kbench/schema.py
+62
View File
@@ -0,0 +1,62 @@
"""The sample schema and its validation. Standard library only, on purpose.
This used to live in `kbench/tasks/signal.py`, which imports `inspect_ai` at module scope —
so checking whether a JSONL record was well-formed required the whole eval framework to be
installed. That is backwards: the schema is ours and the runner is swappable, so the thing
that defines what a sample *is* must not depend on the thing that happens to execute it.
Practically it means authoring tools, CI checks and editors can validate data without
resolving a heavy dependency tree, and that swapping the execution backend later touches
`tasks/`, not this file.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
DATA_DIR = Path(__file__).resolve().parent.parent / "data"
VALID_SPLITS = {"train", "dev", "test"}
VALID_SCORERS = {"exact", "includes", "regex", "rubric"}
def family_path(family: str, tier: str = "private") -> Path:
return DATA_DIR / tier / f"{family}.jsonl"
def validate_record(rec: dict[str, Any], where: str) -> None:
"""Fail loudly on a malformed sample.
A silently-skipped sample shrinks the eval set without changing the score's appearance,
which is the worst possible failure mode for a benchmark.
"""
for field in ("id", "input", "split", "scorer"):
if not rec.get(field):
raise ValueError(f"{where}: missing required field {field!r}")
if rec["split"] not in VALID_SPLITS:
raise ValueError(f"{where}: split must be one of {sorted(VALID_SPLITS)}")
if rec["scorer"] not in VALID_SCORERS:
raise ValueError(f"{where}: scorer must be one of {sorted(VALID_SCORERS)}")
if rec["scorer"] != "rubric" and not rec.get("target"):
raise ValueError(f"{where}: scorer {rec['scorer']!r} requires a target")
if rec["scorer"] == "rubric" and not (rec.get("rubric") or rec.get("target")):
raise ValueError(f"{where}: rubric scorer requires a rubric or a target")
def read_records(path: Path) -> list[tuple[int, dict[str, Any]]]:
"""Parse and validate a family file. Returns (line number, record) pairs.
Line numbers are carried so an error names the line a human has to open.
"""
import json
out: list[tuple[int, dict[str, Any]]] = []
for lineno, line in enumerate(path.read_text().splitlines(), start=1):
line = line.strip()
if not line or line.startswith("//"):
continue
rec = json.loads(line)
validate_record(rec, f"{path.name}:{lineno}")
out.append((lineno, rec))
return out
kbench/scorers/__init__.py
+95
View File
@@ -0,0 +1,95 @@
"""Scorers for signal-tier tasks.
Inspect attaches scorers to a Task, not to a sample. Our families are mixed --
one `agent_ops` file legitimately contains samples graded by exact match, by
substring, and by rubric. Splitting a family into one Task per scorer would
fragment the results and make family-level scores meaningless.
So we dispatch per sample instead, reading `scorer` from sample metadata.
"""
from __future__ import annotations
import re
from inspect_ai.scorer import (
CORRECT,
INCORRECT,
Score,
Scorer,
Target,
accuracy,
model_graded_qa,
scorer,
stderr,
)
from inspect_ai.solver import TaskState
DEFAULT_RUBRIC = """You are grading a model's answer against a reference.
Grade CORRECT if the answer is substantively equivalent to the reference --
matching facts, decisions, and any required steps. Ignore differences in
wording, formatting, or verbosity.
Grade INCORRECT if it contradicts the reference, omits something the reference
treats as required, or adds a claim the reference does not support.
Be strict. A partially correct answer is INCORRECT."""
def _text(state: TaskState) -> str:
return (state.output.completion or "").strip()
@scorer(metrics=[accuracy(), stderr()])
def dispatching(judge_model: str | None = None) -> Scorer:
"""Grade each sample by the method named in its own metadata.
Falls back to `includes` when unspecified, which is the most forgiving
deterministic option -- a sample that silently scores zero because of a
missing metadata field would look like a model failure.
"""
graded = model_graded_qa(instructions=DEFAULT_RUBRIC, model=judge_model)
async def score(state: TaskState, target: Target) -> Score:
method = (state.metadata or {}).get("scorer") or "includes"
answer = _text(state)
expected = target.text or ""
if method == "rubric":
rubric = (state.metadata or {}).get("rubric")
if rubric:
custom = model_graded_qa(instructions=rubric, model=judge_model)
return await custom(state, target)
return await graded(state, target)
if method == "exact":
ok = answer == expected.strip()
elif method == "regex":
try:
ok = re.search(expected, answer, re.IGNORECASE | re.DOTALL) is not None
except re.error as exc:
return Score(
value=INCORRECT,
answer=answer,
explanation=f"invalid regex in sample target: {exc}",
)
elif method == "includes":
ok = expected.strip().lower() in answer.lower()
else:
return Score(
value=INCORRECT,
answer=answer,
explanation=f"unknown scorer {method!r} -- fix the sample, not the model",
)
return Score(
value=CORRECT if ok else INCORRECT,
answer=answer,
explanation=f"scorer={method}",
)
return score
__all__ = ["DEFAULT_RUBRIC", "dispatching"]
kbench/specdec.py
+106
View File
@@ -0,0 +1,106 @@
"""Speculative-decoding metrics, read from the engine rather than inferred.
Throughput alone cannot tell you whether speculative decoding is working. A draft model that
is rejected almost every time still produces tokens — just slowly, having paid for the draft
pass as well as the verify pass — and on a synthetic benchmark that can even look like a
modest win. The number that says whether it is working is the **acceptance rate**: of the
tokens the draft proposed, how many survived verification.
vLLM exposes this on its Prometheus endpoint as counters, so a rate for one sweep point is a
delta across that point rather than the process-lifetime total. Reading the total instead would
mix in every request since the server started, including warm-up.
Parsing is pure standard library and `httpx` is only needed to fetch — the import is deferred
so the parser can be tested, and reasoned about, without the HTTP client installed. Same rule as
`kbench/schema.py`: the logic that decides what a number means must not depend on the machinery
that fetches it.
This is deliberately best-effort. An engine without these counters, or with metrics disabled,
gets `None` and the sweep continues — an absent metric must never fail a benchmark run.
Why it matters for this bench specifically: `perf.py` sends a random nonce prefix and
`ignore_eos`, both correct for defeating prefix caching and pinning output length, and both
adversarial to speculative decoding. Forced continuation past a natural stop is degenerate
text, and a draft model's acceptance on degenerate text is not its acceptance on real work. So
a speculative target measured with the synthetic prompt needs its acceptance rate reported
beside the throughput, or the throughput will be read as if it transfers.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING: # pragma: no cover
import httpx
# vLLM V1 names. The older V0 engine used vllm:spec_decode_* with the same meaning; both are
# accepted so a mixed fleet does not silently report nothing.
ACCEPTED = ("vllm:spec_decode_num_accepted_tokens_total",)
DRAFT = ("vllm:spec_decode_num_draft_tokens_total",)
_SAMPLE = re.compile(r"^(?P<name>[a-zA-Z_:][\w:]*)(?P<labels>\{[^}]*\})?\s+(?P<value>[-+0-9.eE]+)$")
@dataclass(frozen=True)
class SpecCounters:
accepted: float
draft: float
def __sub__(self, other: "SpecCounters") -> "SpecCounters":
return SpecCounters(self.accepted - other.accepted, self.draft - other.draft)
@property
def acceptance_rate(self) -> float | None:
"""Accepted / proposed over this window. None when nothing was proposed."""
if self.draft <= 0:
return None
return round(self.accepted / self.draft, 4)
def parse_metrics(text: str) -> SpecCounters | None:
"""Pull the speculative counters out of a Prometheus exposition payload."""
totals: dict[str, float] = {}
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
m = _SAMPLE.match(line)
if not m:
continue
name = m.group("name")
if name in ACCEPTED or name in DRAFT:
try:
# Counters are per-label-set (one per model); sum them. A single-model server
# has one series, and summing is still correct there.
totals[name] = totals.get(name, 0.0) + float(m.group("value"))
except ValueError:
continue
accepted = next((totals[n] for n in ACCEPTED if n in totals), None)
draft = next((totals[n] for n in DRAFT if n in totals), None)
if accepted is None or draft is None:
return None
return SpecCounters(accepted=accepted, draft=draft)
def metrics_url(base_url: str) -> str:
"""/metrics sits at the server root, not under the OpenAI /v1 prefix."""
root = base_url.rstrip("/")
for suffix in ("/v1", "/openai/v1"):
if root.endswith(suffix):
root = root[: -len(suffix)]
break
return f"{root}/metrics"
async def read_counters(client: "httpx.AsyncClient", base_url: str) -> SpecCounters | None:
"""Best-effort read. Any failure means "no data", never an exception into the sweep."""
try:
resp = await client.get(metrics_url(base_url), timeout=5.0)
if resp.status_code != 200:
return None
return parse_metrics(resp.text)
except Exception: # noqa: BLE001 - a missing metrics endpoint is an ordinary outcome
return None
kbench/sweep.py
+149
View File
@@ -0,0 +1,149 @@
"""Sweep a matrix of targets and reduce it to a decision.
`run` answers "what did this target do". `compare` answers "what changed between two". Neither
answers the question people actually start with, which is **which of these should I run**, and
that one is a matrix: the same model at three quantisations, or one quantisation under four sets
of serving flags, all on the box you actually own.
Three things make this more than a shell loop:
**It holds the box constant.** A sweep varies configuration and nothing else. Varying the host
too produces numbers with two causes, and the tempting reading — "fp8 is faster" — is then
unsupported by the data that appears to show it.
**It runs sequentially, and that is not a simplification.** On a unified-memory machine two
targets cannot both be resident; that constraint is the entire premise of the stack this bench
was built for. Running them concurrently would measure contention between them rather than
either one.
**It reduces to a frontier, not a leaderboard.** Self-hosting is a trade: quality against
throughput, under a memory ceiling. A single ranking hides that, so the output is the set of
targets that are not beaten on both axes at once — plus, explicitly, the ones that are, because
a target that is worse at everything is the one useful thing a matrix can tell you.
Every point is an ordinary score card saved to `results/`. There is no sweep-shaped result
format, so `compare`, `card` and the git history all keep working unchanged.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
@dataclass
class SweepPoint:
"""One target's outcome within a sweep."""
target_id: str
quality: float | None
throughput_tps: float | None
single_stream_tps: float | None = None
contamination: str | None = None
error: str | None = None
run_path: str | None = None
@property
def usable(self) -> bool:
"""Whether this point can take part in the trade-off at all."""
return self.error is None and self.throughput_tps is not None
@property
def quality_comparable(self) -> bool:
"""A contaminated or unmeasured quality score cannot be traded against anything."""
return self.quality is not None and self.contamination != "detected"
@dataclass
class SweepResult:
points: list[SweepPoint] = field(default_factory=list)
best: list[str] = field(default_factory=list)
dominated: dict[str, str] = field(default_factory=dict) # target -> what beats it
warnings: list[str] = field(default_factory=list)
@property
def failed(self) -> list[SweepPoint]:
return [p for p in self.points if p.error]
def _dominates(a: SweepPoint, b: SweepPoint, use_quality: bool) -> bool:
"""True when `a` is at least as good as `b` everywhere and better somewhere.
Strict domination is the only claim worth making from a small matrix. "Higher throughput
and slightly lower quality" is a trade the operator has to make with knowledge this tool
does not have — how much quality their workload can spare — so it is deliberately left to
them rather than collapsed into a score.
"""
if not (a.usable and b.usable):
return False
better_anywhere = a.throughput_tps > b.throughput_tps
at_least_equal = a.throughput_tps >= b.throughput_tps
if use_quality and a.quality_comparable and b.quality_comparable:
at_least_equal = at_least_equal and a.quality >= b.quality
better_anywhere = better_anywhere or a.quality > b.quality
return at_least_equal and better_anywhere
def analyse(points: list[SweepPoint], hosts: list[str] | None = None) -> SweepResult:
"""Reduce a set of measured targets to a frontier and a set of dominated options."""
result = SweepResult(points=list(points))
if hosts and len(set(hosts)) > 1:
result.warnings.append(
f"Targets ran on more than one host ({', '.join(sorted(set(hosts)))}). A sweep is "
"meant to hold the machine constant — these differences have two causes and "
"neither can be isolated."
)
usable = [p for p in points if p.usable]
if not usable:
result.warnings.append("No target produced a usable measurement.")
return result
with_quality = [p for p in usable if p.quality_comparable]
use_quality = len(with_quality) == len(usable)
if not use_quality:
missing = sorted(p.target_id for p in usable if not p.quality_comparable)
result.warnings.append(
f"Ranking on throughput alone: {', '.join(missing)} has no comparable quality score "
"(never measured, or voided by a contamination probe)."
)
for candidate in usable:
beaten_by = next(
(other.target_id for other in usable
if other.target_id != candidate.target_id and _dominates(other, candidate, use_quality)),
None,
)
if beaten_by:
result.dominated[candidate.target_id] = beaten_by
else:
result.best.append(candidate.target_id)
# Present the frontier the way the decision is made: fastest first, quality breaking ties.
order = {p.target_id: p for p in usable}
result.best.sort(
key=lambda t: (
-(order[t].throughput_tps or 0),
-(order[t].quality if order[t].quality_comparable else 0),
)
)
return result
def point_from_run(run: dict[str, Any], path: str | None = None) -> SweepPoint:
"""Read a saved score card into a sweep point.
Reads `signal_score` rather than any raw value: a contaminated run has had that nulled by
compute_verdict, which is exactly the behaviour a sweep should inherit — a memorised score
must not win a frontier.
"""
verdict = run.get("verdict") or {}
return SweepPoint(
target_id=run.get("target_id", "?"),
quality=verdict.get("signal_score"),
throughput_tps=verdict.get("peak_throughput_tps"),
single_stream_tps=verdict.get("single_stream_tps"),
contamination=verdict.get("contamination"),
run_path=path,
)
kbench/tasks/__init__.py
+132
View File
@@ -0,0 +1,132 @@
"""Task catalog.
Two tiers, and the distinction is not cosmetic:
REFERENCE -- public benchmarks pulled from inspect_evals. These are NOT here to
rank models. They exist to (a) prove the harness is wired correctly, (b) give
a calibration anchor -- if our number lands far from published values, the
harness is broken, not the model, and (c) let an outside reader locate this
bench against numbers they already know. A bench made only of private tasks
is unfalsifiable to everyone including its author. Reference scores are
displayed with a caveat: public, likely contaminated, calibration only.
SIGNAL -- our own private tasks, derived from real workloads. This is the
actual product. These never become public: the moment they do they enter the
next training corpus and stop measuring anything. See docs/DECISIONS.md#d3.
"""
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class TaskSpec:
"""One benchmark task and how to read its numbers."""
name: str
inspect_task: str
tier: str # "reference" | "signal" | "canary" | "example"
# Which key in the scorer's metrics is the headline accuracy.
primary_metric: str
# Sample scores may be dict-valued (IFEval) rather than scalar. If so, this
# names the key that decides pass/fail for a single sample.
primary_sample_key: str | None = None
dataset_samples: int | None = None
judge_required: bool = False
# NLTK corpora the scorer needs. Checked (and fetched) before the eval
# starts: IFEval's scorer needs punkt_tab but only touches it at scoring
# time, so a missing corpus kills the run ~20 minutes of GPU time in.
# Preflight turns that into a two-second failure.
nltk_resources: tuple[str, ...] = ()
notes: str = ""
CATALOG: dict[str, TaskSpec] = {
"ifeval": TaskSpec(
name="ifeval",
inspect_task="inspect_evals/ifeval",
tier="reference",
primary_metric="final_acc",
primary_sample_key="prompt_level_strict",
dataset_samples=541,
judge_required=False,
nltk_resources=("tokenizers/punkt_tab",),
notes=(
"Verifiable instruction following -- 'write exactly 3 paragraphs', "
"'do not use the letter e'. Chosen as the first reference eval "
"because its scorer is deterministic (no judge model, no cost, no "
"judge drift between runs) and because instruction adherence is "
"the property that actually decides whether a small local model "
"can hold a system prompt in production."
),
),
"agent_ops": TaskSpec(
name="agent_ops",
inspect_task="kbench/tasks/signal.py@agent_ops",
tier="signal",
primary_metric="accuracy",
dataset_samples=12,
# One rubric sample in twelve, so a judge is needed but the drift it
# introduces is bounded to a twelfth of the score.
judge_required=True,
notes=(
"Operating a Lumbridge Compute node: admission arithmetic against "
"both the declared budget and the observed pool, watchdog policy, "
"why a Scene naming ids rather than commands cannot introduce "
"code, transactional rollback, and process identity under PID "
"reuse. Derived from the decisions the MCP server exists to let an "
"agent make, which is what makes it unfakeable by a general "
"benchmark -- nobody else measures competence at running this."
),
),
"contamination": TaskSpec(
name="contamination",
inspect_task="kbench/tasks/canary.py@contamination",
# Its own tier because its score is an alarm, not a capability, and
# compute_verdict() reads it to decide whether the signal score is
# usable at all. Registering it is load-bearing: an unregistered probe
# never runs, so the gate reports "unverified" forever -- which is the
# same shape of bug as the canary that was never embedded in a prompt.
tier="canary",
primary_metric="accuracy",
dataset_samples=3,
judge_required=False,
notes=(
"Contamination probe. One carrier puts the GUID into the inference "
"traffic; two detectors ask for it cold. Scoring is inverted -- 0 "
"is the healthy result, and anything above 0 voids every quality "
"number for the target. See data/README.md."
),
),
"example": TaskSpec(
name="example",
inspect_task="kbench/tasks/signal.py@example",
# Tier "example" deliberately: it exercises the signal machinery but
# measures nothing, so it must not contribute to the signal score in
# compute_verdict(). Real families are registered with tier="signal".
tier="example",
primary_metric="accuracy",
dataset_samples=4,
judge_required=True,
notes=(
"Public demonstration of the signal-task format -- one sample per "
"scorer type. Lives in data/public/. Real signal families live in "
"data/private/ and never ship."
),
),
}
def get(name: str) -> TaskSpec:
if name not in CATALOG:
known = ", ".join(sorted(CATALOG))
raise KeyError(f"unknown task {name!r}. known tasks: {known}")
return CATALOG[name]
def by_tier(tier: str) -> list[TaskSpec]:
return [t for t in CATALOG.values() if t.tier == tier]
__all__ = ["CATALOG", "TaskSpec", "by_tier", "get"]
kbench/tasks/canary.py
+109
View File
@@ -0,0 +1,109 @@
"""Contamination probes.
D3 specifies `data/canary/` as "GUID strings embedded in samples". The tier existed and was
empty, and nothing loaded it — while `kbench add` stamped a `canary` GUID into every signal
record that was never embedded in the prompt, never sent to a model, and never checked. A
canary only works if it is *in the text that could be trained on*, so as written the control
could not fire.
**Scoring here is inverted, and that is the whole point.**
A canary probe contains a GUID that exists nowhere else and asks the model to reproduce it.
A model that has never seen this repo's data cannot possibly answer, so:
score 0.0 = clean. The expected, healthy result.
score > 0 = this model has seen our eval data. Every signal number for it is void.
Read `canary_score` as a contamination alarm, never as a capability. It is reported separately
from `signal_score` for exactly that reason.
**What this actually detects.** The private set never leaves the repo, so the realistic leak
vector is not a scraper — it is the authoring rules' own admission: prompts get sent to
third-party judge and baseline models regardless of how private the repo is. If a provider
trains on inference traffic, these GUIDs go with it. That is the leak this catches, and it is
the one that would otherwise be invisible.
A probe firing does not tell you *which* provider learned it. It tells you the set is burned
and needs regenerating, which is the actionable part.
"""
from __future__ import annotations
from inspect_ai import Task, task
from inspect_ai.dataset import MemoryDataset
from inspect_ai.solver import generate
# Absolute for the same reason as signal.py: inspect-ai loads this file by path, outside
# the package, so relative imports raise "beyond top-level package".
from kbench.scorers import dispatching
from kbench.tasks.signal import load_samples
CANARY_TIER = "canary"
def canary_task(family: str = "contamination") -> Task:
"""Build the contamination probe task.
Probes are `split: test` like everything else — they are not training data and must never
be filtered out by the split discipline that protects the signal set.
"""
samples = load_samples(family, tier=CANARY_TIER, splits=("test",))
if not samples:
raise ValueError(
f"canary family {family!r} is empty. A bench with no contamination probe cannot "
"tell a real score from a memorised one."
)
# Carriers must be loaded and then NOT scored. The carrier states the GUID in its own
# prompt and asks for it back, so every model repeats it — that is instruction-following,
# not memorisation. Scoring it pinned the canary at >= 1/n for every target alive, which
# reads as CONTAMINATED and nulls signal_score, voiding the whole quality half.
#
# The data already carried `tags: [contamination, carrier]` and a note saying "it always
# passes"; nothing read it. Hence the assertions below: this file now fails loudly if the
# split it depends on is missing, rather than silently scoring the wrong set.
carriers = [s for s in samples if "carrier" in (s.metadata or {}).get("tags", [])]
detectors = [s for s in samples if "detector" in (s.metadata or {}).get("tags", [])]
unlabelled = [s for s in samples if s not in carriers and s not in detectors]
if unlabelled:
raise ValueError(
f"canary family {family!r} has samples tagged neither 'carrier' nor 'detector': "
f"{[s.id for s in unlabelled]}. Every probe must declare its role, because the "
"two are scored differently."
)
if not carriers:
raise ValueError(
f"canary family {family!r} has no carrier. Without one the GUID never enters any "
"corpus, so the detectors are unanswerable by construction and would report "
"'clean' against a model that is in fact contaminated."
)
if not detectors:
raise ValueError(
f"canary family {family!r} has no detector. The carrier alone detects nothing."
)
return Task(
name=f"canary/{family}",
dataset=MemoryDataset(samples=detectors, name=f"canary-{family}"),
solver=generate(),
scorer=dispatching(),
)
def interpret(score: float | None) -> str:
"""Turn a canary score into the sentence a reader needs."""
if score is None:
return "no contamination probe was run — signal scores are unverified"
if score <= 0.0:
return "clean: the model could not reproduce any probe GUID"
return (
f"CONTAMINATED: the model reproduced {score:.0%} of the probe GUIDs. "
"Signal scores for this target are void; regenerate the eval set."
)
@task
def contamination() -> Task:
"""Registered so `kbench` can run the probe like any other task."""
return canary_task()
kbench/tasks/signal.py
+121
View File
@@ -0,0 +1,121 @@
"""Signal-tier tasks: our own private evals, loaded from JSONL.
One family (`agent_ops`, `repo_edit`, ...) becomes one Inspect Task. Samples
within a family may be graded differently; `kbench.scorers.dispatching` handles
that per sample.
Split discipline is enforced here rather than left to convention: by default
only `test` samples are evaluated. If a `train` sample ever reaches the eval
path, fine-tuning on it silently invalidates every number that follows and
there is no way to detect it after the fact.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from inspect_ai import Task, task
from inspect_ai.dataset import MemoryDataset, Sample
from inspect_ai.solver import generate
# Absolute, not relative. inspect-ai loads a task file by path as a standalone module, so
# it is not a member of the `kbench` package at load time and `from ..scorers` raises
# "attempted relative import beyond top-level package". kbench is an installed package, so
# the absolute form resolves under both import styles.
from kbench.scorers import dispatching
# The schema lives in kbench/schema.py, which imports nothing beyond the standard library —
# validating a JSONL record must not require the eval framework. Re-exported here so existing
# callers of signal.family_path / signal.validate_record keep working.
from kbench.schema import ( # noqa: F401
DATA_DIR,
VALID_SCORERS,
VALID_SPLITS,
family_path,
validate_record,
)
def load_samples(
family: str,
tier: str = "private",
splits: tuple[str, ...] = ("test",),
) -> list[Sample]:
path = family_path(family, tier)
if not path.exists():
raise FileNotFoundError(
f"no data file for family {family!r} at {path}. "
f"create samples with: kbench add {family}"
)
samples: list[Sample] = []
for lineno, line in enumerate(path.read_text().splitlines(), start=1):
line = line.strip()
if not line or line.startswith("//"):
continue
rec = json.loads(line)
validate_record(rec, f"{path.name}:{lineno}")
if rec["split"] not in splits:
continue
samples.append(
Sample(
id=rec["id"],
input=rec["input"],
target=rec.get("target") or "",
metadata={
"scorer": rec["scorer"],
"rubric": rec.get("rubric"),
"family": rec.get("family", family),
"split": rec["split"],
**(rec.get("metadata") or {}),
},
)
)
return samples
def signal_task(
family: str,
tier: str = "private",
splits: tuple[str, ...] = ("test",),
judge_model: str | None = None,
) -> Task:
"""Build an Inspect Task for one signal family."""
samples = load_samples(family, tier=tier, splits=splits)
if not samples:
raise ValueError(
f"family {family!r} has no samples in splits {splits}. "
f"refusing to run an empty eval -- it would report a score of 0/0."
)
return Task(
name=f"signal/{family}",
dataset=MemoryDataset(samples=samples, name=family),
solver=generate(),
scorer=dispatching(judge_model=judge_model),
)
# --- registered example task -------------------------------------------------
# Demonstrates the machinery against the shipped public examples. Real families
# live in data/private/ and are registered the same way.
@task
def example() -> Task:
"""Public example family -- shows the format, measures nothing important."""
return signal_task("example", tier="public", splits=("test",))
@task
def agent_ops() -> Task:
"""Operating a Compute node: admission, watchdog policy, Scene semantics, process identity.
Only the `test` split runs, which is the split discipline this module enforces rather than
documents: a `train` sample reaching the eval path silently invalidates every number after
it, and there is no way to detect that afterwards.
"""
return signal_task("agent_ops", tier="private", splits=("test",))