lumbridge-public/bench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lumbridge Bench
CI / verify (push) Successful in 24s
Details
CI / deploy (push) Failing after 1m14s
Details

Browse Source
This commit is contained in:
Karti Tripathi
2026-08-04 00:44:07 -07:00
commit 006feee0f7
65 changed files with 13516 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kbench/cli.py
+646
View File
@@ -0,0 +1,646 @@
"""kbench command line."""
from __future__ import annotations
import json
import shlex
import subprocess
from pathlib import Path
import typer
from rich.console import Console
from rich.table import Table
from . import tasks as task_catalog
from .compare import compare as compare_runs, direction
from .sweep import analyse, point_from_run
from .registry import load_registry
from .results import RESULTS_DIR, Run
from .run import build_run, run_perf, run_quality
app = typer.Typer(
add_completion=False,
help="Lumbridge Bench -- private eval and serving-performance evidence for self-hosted and API models",
)
console = Console()
def _parse_concurrencies(value: str) -> tuple[int, ...]:
return tuple(int(x) for x in value.split(",") if x.strip())
@app.command("targets")
def list_targets() -> None:
"""List benchmarkable targets."""
registry = load_registry()
table = Table(title="targets", header_style="bold")
for col in ("id", "tier", "host", "quant", "params", "engine", "aliases"):
table.add_column(col)
for t in registry.targets.values():
params = f"{t.params_b:g}B" if t.params_b else "-"
if t.active_params_b:
params += f" ({t.active_params_b:g}B active)"
table.add_row(
t.id, t.tier, t.host, t.quant or "-", params,
t.serving.engine, ", ".join(t.aliases) or "-",
)
console.print(table)
@app.command("tasks")
def list_tasks() -> None:
"""List benchmark tasks."""
table = Table(title="tasks", header_style="bold")
for col in ("name", "tier", "samples", "judge", "inspect task"):
table.add_column(col)
for spec in task_catalog.CATALOG.values():
table.add_row(
spec.name, spec.tier, str(spec.dataset_samples or "-"),
"yes" if spec.judge_required else "no", spec.inspect_task,
)
console.print(table)
console.print(
"\n[dim]reference = public, likely contaminated, calibration only\n"
"signal = private, ours, the actual measurement[/dim]"
)
@app.command("snapshot")
def snapshot(host: str) -> None:
"""Print the live serving flags on a host, for pasting into models.yaml.
Results are only comparable between targets with identical flags, so the
registry has to record what the server was ACTUALLY started with rather
than what we believe we started it with.
"""
registry = load_registry()
h = registry.host(host)
if not h.ssh:
raise typer.BadParameter(f"host {host!r} has no ssh address")
proc = subprocess.run(
["ssh", h.ssh, "ps -eo args | grep -E '[v]llm serve|[s]glang|[l]lama-server'"],
capture_output=True, text=True, timeout=30,
# A host with no serving process is an ordinary outcome, not an
# exception -- it is reported below.
check=False,
)
if proc.returncode != 0 or not proc.stdout.strip():
console.print(f"[yellow]no serving process found on {host}[/yellow]")
raise typer.Exit(1)
for line in proc.stdout.strip().splitlines():
parts = shlex.split(line)
console.print(f"\n[bold]{parts[0] if parts else '?'}[/bold]")
flags: dict[str, object] = {}
i = 0
while i < len(parts):
if parts[i].startswith("--"):
key = parts[i][2:].replace("-", "_")
if i + 1 < len(parts) and not parts[i + 1].startswith("--"):
flags[key] = parts[i + 1]
i += 2
else:
flags[key] = True
i += 1
else:
i += 1
for k, v in flags.items():
console.print(f" {k}: {v}")
def _load_prompts(path: Path | None) -> list[str] | None:
"""Read representative prompts: one per line, or a JSONL with an `input` field.
Measuring speculative decoding on the synthetic prompt is measuring the wrong thing — the
draft model's acceptance rate depends on how predictable the text is, and lorem with a
random nonce is not the text you serve.
"""
if path is None:
return None
lines = [ln.strip() for ln in path.read_text().splitlines() if ln.strip()]
prompts = []
for ln in lines:
if ln.startswith("{"):
try:
prompts.append(json.loads(ln)["input"])
continue
except (json.JSONDecodeError, KeyError):
pass
prompts.append(ln)
if not prompts:
raise typer.BadParameter(f"{path} contains no prompts")
return prompts
@app.command("perf")
def perf_cmd(
target: str,
concurrency: str = typer.Option("1,8,32", help="comma-separated concurrency levels"),
input_tokens: int = typer.Option(1024),
output_tokens: int = typer.Option(256),
save: bool = typer.Option(True, help="write a results JSON"),
prompt_file: Path | None = typer.Option(
None, help="representative prompts (one per line, or JSONL with an `input` field)"
),
natural_stop: bool = typer.Option(
False,
help="let the model stop on its own instead of pinning output length. Required for an "
"honest speculative-decoding measurement; weakens comparability across points.",
),
) -> None:
"""Run a serving performance sweep only (no quality tasks)."""
registry = load_registry()
t = registry.target(target)
result = run_perf(
t, _parse_concurrencies(concurrency), input_tokens, output_tokens,
prompts=_load_prompts(prompt_file), pin_output=not natural_stop,
)
run = build_run(registry, t, quality=[], perf=result)
_print_card(run)
if save:
path = run.save()
console.print(f"\n[green]saved[/green] {path.relative_to(Path.cwd())}")
@app.command("run")
def run_cmd(
target: str,
tasks: str = typer.Option("ifeval", help="comma-separated task names, or 'all'"),
limit: int | None = typer.Option(None, help="cap samples per task (recorded in output)"),
max_connections: int = typer.Option(16, help="concurrent requests to the model"),
epochs: int = typer.Option(
1,
help="run each sample N times and average. Sampling is not reproducible on a "
"batching server even with a fixed seed, so one epoch is a draw, not a rate.",
),
concurrency: str = typer.Option("1,8,32"),
input_tokens: int = typer.Option(1024),
output_tokens: int = typer.Option(256),
skip_perf: bool = typer.Option(False),
skip_quality: bool = typer.Option(False),
) -> None:
"""Full score card: quality tasks + perf sweep -> one results JSON."""
registry = load_registry()
t = registry.target(target)
names = list(task_catalog.CATALOG) if tasks == "all" else [
n.strip() for n in tasks.split(",") if n.strip()
]
specs = [task_catalog.get(n) for n in names]
quality = []
if not skip_quality:
quality = run_quality(
t, specs, limit=limit, max_connections=max_connections, epochs=epochs
)
perf = None
if not skip_perf:
perf = run_perf(t, _parse_concurrencies(concurrency), input_tokens, output_tokens)
run = build_run(registry, t, quality=quality, perf=perf)
_print_card(run)
path = run.save()
console.print(f"\n[green]saved[/green] {path.relative_to(Path.cwd())}")
@app.command("sweep")
def sweep_cmd(
targets: list[str] = typer.Argument(..., help="two or more target ids to compare"),
tasks: str = typer.Option("ifeval", help="comma-separated task names, or 'all'"),
limit: int | None = typer.Option(None, help="cap samples per task"),
max_connections: int = typer.Option(16),
concurrency: str = typer.Option("1,8,32"),
input_tokens: int = typer.Option(1024),
output_tokens: int = typer.Option(256),
skip_perf: bool = typer.Option(False),
skip_quality: bool = typer.Option(False),
) -> None:
"""Run the same tasks against several targets and reduce them to a frontier.
Sequential on purpose: on a unified-memory box two targets cannot both be resident, so
running them together would measure contention rather than either one. Each target still
produces an ordinary score card in results/, so compare and card keep working on them.
"""
if len(targets) < 2:
raise typer.BadParameter("a sweep needs at least two targets")
registry = load_registry()
names = list(task_catalog.CATALOG) if tasks == "all" else [
n.strip() for n in tasks.split(",") if n.strip()
]
specs = [task_catalog.get(n) for n in names]
points, hosts = [], []
for i, target_id in enumerate(targets, start=1):
console.rule(f"[bold]{i}/{len(targets)} {target_id}[/bold]")
try:
t = registry.target(target_id)
hosts.append(t.host)
quality = [] if skip_quality else run_quality(
t, specs, limit=limit, max_connections=max_connections
)
perf = None if skip_perf else run_perf(
t, _parse_concurrencies(concurrency), input_tokens, output_tokens
)
run = build_run(registry, t, quality=quality, perf=perf)
path = run.save()
console.print(f"[green]saved[/green] {path.name}")
points.append(point_from_run(run.to_dict(), str(path.name)))
except Exception as exc: # noqa: BLE001
# One bad target must not throw away the targets already measured — a sweep is
# hours of GPU time and re-running the good ones to reach the next is wasteful.
console.print(f"[red]failed[/red] {target_id}: {exc}")
points.append(
point_from_run({"target_id": target_id, "verdict": {}})
)
points[-1].error = str(exc)
result = analyse(points, hosts=hosts)
console.print()
console.rule("[bold]frontier[/bold]")
table = Table(header_style="bold")
for col in ("", "target", "signal", "peak tok/s", "tok/s @1", "verdict"):
table.add_column(col)
for point in sorted(
result.points, key=lambda x: (x.error is not None, -(x.throughput_tps or 0))
):
if point.error:
mark, verdict = "[red]![/red]", f"[red]{point.error[:44]}[/red]"
elif point.target_id in result.dominated:
mark = "[dim]·[/dim]"
verdict = f"[dim]beaten by {result.dominated[point.target_id]}[/dim]"
else:
mark, verdict = "[green]✓[/green]", "[green]worth running[/green]"
table.add_row(
mark,
point.target_id,
f"{point.quality:.3f}" if point.quality is not None else "-",
f"{point.throughput_tps:.1f}" if point.throughput_tps else "-",
f"{point.single_stream_tps:.1f}" if point.single_stream_tps else "-",
verdict,
)
console.print(table)
for warning in result.warnings:
console.print(f"[yellow]![/yellow] {warning}")
if result.best:
console.print(
f"\n[bold]{len(result.best)} of {len(result.points)} "
f"worth considering:[/bold] " + ", ".join(result.best)
)
if result.dominated:
console.print(
"[dim]The rest are beaten on every measured axis at once — not a close call, "
"a strictly worse option.[/dim]"
)
@app.command("card")
def card_cmd(path: Path) -> None:
"""Render a saved results JSON as a score card."""
data = Run.load(path)
_print_card_dict(data)
def _print_card(run: Run) -> None:
_print_card_dict(run.to_dict())
def _print_card_dict(data: dict) -> None:
tgt = data["target"]
console.print()
console.rule(f"[bold]{tgt.get('display_name', data['target_id'])}[/bold]")
console.print(
f"[dim]{data['target_id']} | {data['host'].get('hardware', '?')} | "
f"{data['timestamp']}[/dim]"
)
quality = data.get("quality") or []
if quality:
table = Table(title="quality", header_style="bold", title_justify="left")
for col in ("task", "tier", "n", "accuracy", "notes"):
table.add_column(col)
for q in quality:
if q.get("error"):
table.add_row(q["task"], q["tier"], "-", "[red]ERROR[/red]", q["error"][:60])
continue
acc = q["metrics"].get("accuracy")
limit = q["metrics"].get("_limit")
note = f"limited to {int(limit)} samples" if limit else ""
if q["tier"] == "reference":
note = (note + "; " if note else "") + "calibration only"
table.add_row(
q["task"], q["tier"], str(q["n_samples"]),
f"{acc:.3f}" if acc is not None else "-", note,
)
console.print(table)
perf = data.get("perf")
if perf and perf.get("points"):
table = Table(title="serving perf", header_style="bold", title_justify="left")
points = data["perf"].get("points") or []
# Only widen the table when there is something to put in the column — a permanent
# "accept: -" on every non-speculative target is noise.
show_accept = any(pt.get("spec_acceptance_rate") is not None for pt in points)
cols = ["conc", "tok/s total", "tok/s /stream", "TTFT p50", "TTFT p95", "TPOT p50"]
if show_accept:
cols.append("accept")
cols.append("failed")
for col in cols:
table.add_column(col, justify="right")
for p in perf["points"]:
table.add_row(
str(p["concurrency"]),
f"{p['output_tps_total']:.1f}",
f"{p['output_tps_per_stream']:.1f}",
f"{p['ttft_p50_ms']:.0f}ms" if p.get("ttft_p50_ms") else "-",
f"{p['ttft_p95_ms']:.0f}ms" if p.get("ttft_p95_ms") else "-",
f"{p['tpot_p50_ms']:.1f}ms" if p.get("tpot_p50_ms") else "-",
*(
[
f"{p['spec_acceptance_rate']:.0%}"
if p.get("spec_acceptance_rate") is not None
else "-"
]
if show_accept
else []
),
f"[red]{p['failed']}[/red]" if p["failed"] else "0",
)
console.print(table)
if any(pt.get("natural_stop") for pt in points):
spread = next(
(pt["output_token_stdev"] for pt in points if pt.get("output_token_stdev")), None
)
console.print(
" [dim]natural stop: output length was not pinned"
+ (f", ±{spread:.0f} tokens" if spread else "")
+ ". Required for an honest speculative-decoding number, and it means these"
" points are not directly comparable to pinned-length runs.[/dim]"
)
verdict = data.get("verdict") or {}
if verdict:
# Contamination first and loud. It is the one result that invalidates every other
# number on the card, so it cannot be a row someone scrolls past.
state = verdict.get("contamination")
if state == "detected":
console.print()
console.print(
"[bold white on red] CONTAMINATED [/bold white on red] "
f"the model reproduced {verdict.get('canary_score', 0):.0%} of the canary probes."
)
console.print(
" [red]Quality scores for this target are void — it has seen the eval set.[/red]"
)
if verdict.get("signal_score_unverified") is not None:
console.print(
f" [dim]withheld signal score: {verdict['signal_score_unverified']:.3f} "
"(recorded for forensics, not usable as a result)[/dim]"
)
console.print(" [dim]Regenerate the private set before trusting anything here.[/dim]")
elif state == "unverified":
console.print(
"\n[yellow]![/yellow] no contamination probe ran — quality scores are unverified"
)
console.print("\n[bold]verdict[/bold]")
for k, v in verdict.items():
if v is None or k in ("contamination", "signal_score_unverified"):
continue
console.print(f" {k}: {v}")
@app.command("add")
def add_cmd(
family: str,
input_text: str | None = typer.Option(None, "--input", help="prompt text"),
input_file: Path | None = typer.Option(None, help="read the prompt from a file"),
target: str | None = typer.Option(None, help="expected answer (or regex)"),
scorer: str = typer.Option("includes", help="exact | includes | regex | rubric"),
rubric: str | None = typer.Option(None, help="grading instructions for --scorer rubric"),
split: str = typer.Option("test", help="train | dev | test -- never change this later"),
tier: str = typer.Option("private", help="private | public"),
source: str = typer.Option("manual", help="where this sample came from"),
tags: str = typer.Option("", help="comma-separated"),
) -> None:
"""Append a sample to a family, with id, canary, and validation applied.
Do not hand-edit the JSONL. This assigns a stable non-reused id (per-sample
regression tracking keys on it) and stamps a canary GUID used later to
prove contamination.
"""
import uuid
from .tasks.signal import family_path, validate_record
if input_file:
input_text = input_file.read_text().strip()
if not input_text:
raise typer.BadParameter("provide --input or --input-file")
path = family_path(family, tier)
path.parent.mkdir(parents=True, exist_ok=True)
existing = [
json.loads(line)
for line in (path.read_text().splitlines() if path.exists() else [])
if line.strip() and not line.startswith("//")
]
# Highest existing number + 1, so ids are never reused even after deletions.
next_n = 1 + max(
(int(r["id"].rsplit("/", 1)[-1]) for r in existing if "/" in r.get("id", "")),
default=0,
)
record = {
"id": f"{family}/{next_n:04d}",
"family": family,
"split": split,
"canary": f"KBENCH-CANARY-{uuid.uuid4().hex[:8]}",
"input": input_text,
"target": target,
"scorer": scorer,
"rubric": rubric,
"metadata": {
"source": source,
"authored": __import__("datetime").date.today().isoformat(),
"tags": [t.strip() for t in tags.split(",") if t.strip()],
},
}
validate_record(record, "new sample")
with path.open("a") as f:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
console.print(f"[green]added[/green] {record['id']} -> {path}")
console.print(
"[yellow]scrub check:[/yellow] no keys, tokens, tailscale addresses, "
"or customer data in that sample?"
)
def _resolve_run(ref: str) -> Path:
"""Accept a path, a bare filename, or a run-id prefix.
Nobody types a full run id. Being strict about it just means every comparison starts with
an `ls`.
"""
direct = Path(ref)
if direct.is_file():
return direct
candidates = sorted(RESULTS_DIR.glob(f"*{ref}*.json"))
if not candidates:
raise typer.BadParameter(f"no result matching {ref!r} in {RESULTS_DIR}")
if len(candidates) > 1:
names = "\n ".join(c.name for c in candidates)
raise typer.BadParameter(f"{ref!r} matches several results:\n {names}")
return candidates[0]
def _fmt(value: float | None, places: int = 1) -> str:
return "-" if value is None else f"{value:.{places}f}"
def _delta_cell(d, places: int = 1) -> str:
"""A delta with its sign, its percent, and — crucially — whether that is good."""
if d.absolute is None:
return "[dim]-[/dim]"
verdict = direction(d.name.split(".")[-1], d)
colour = {"better": "green", "worse": "red", "flat": "dim"}[verdict]
pct = "" if d.percent is None else f" ({d.percent:+.1f}%)"
return f"[{colour}]{d.absolute:+.{places}f}{pct}[/{colour}]"
@app.command("compare")
def compare_cmd(
before: str = typer.Argument(..., help="Baseline run: path, filename, or run-id fragment"),
after: str = typer.Argument(..., help="Run to compare against the baseline"),
show_flips: int = typer.Option(20, help="How many changed samples to list (0 for all)"),
) -> None:
"""Diff two score cards: what changed, and whether the comparison is honest."""
before_path, after_path = _resolve_run(before), _resolve_run(after)
b, a = Run.load(before_path), Run.load(after_path)
result = compare_runs(b, a)
console.print()
console.rule("[bold]compare[/bold]")
console.print(f"[dim]before[/dim] {before_path.name}")
console.print(f"[dim]after [/dim] {after_path.name}\n")
# What actually differs about the two targets. Printed FIRST: without it the reader has
# no idea what the numbers below are attributable to.
if result.target_deltas or result.host_changed:
t = Table(title="what changed about the target", header_style="bold")
for col in ("axis", "before", "after"):
t.add_column(col, overflow="fold")
for d in result.target_deltas:
t.add_row(d.field, str(d.before), str(d.after))
if result.host_changed:
h = result.host_changed
t.add_row("[yellow]host[/yellow]", str(h.before), str(h.after))
console.print(t)
for warning in result.warnings:
console.print(f"[yellow]![/yellow] {warning}")
if result.warnings:
console.print()
if result.quality:
t = Table(title="quality", header_style="bold")
for col in ("metric", "before", "after", "delta"):
t.add_column(col)
for d in result.quality:
t.add_row(d.name, _fmt(d.before, 3), _fmt(d.after, 3), _delta_cell(d, 3))
console.print(t)
if result.flips:
shown = result.flips if show_flips == 0 else result.flips[:show_flips]
# When nothing about the measurement changed -- same target, same host, same eval
# set, same sampling -- a flipped sample did not regress or get fixed. It is the
# same question answered twice with different luck. Calling it "regressed" sends
# someone hunting a cause that does not exist, which is worse than saying nothing.
same_setup = (
not result.target_deltas
and not result.host_changed
and not any(
"edited between runs" in w or "Sampling changed" in w
for w in result.warnings
)
)
t = Table(
title=(
f"samples that changed ({len(result.flips)})"
+ (" — same setup, so this is instability, not change" if same_setup else "")
),
header_style="bold",
)
for col in ("task", "sample", "before", "after", ""):
t.add_column(col, overflow="fold")
for f in shown:
if same_setup:
mark = "[yellow]unstable[/yellow]"
else:
mark = {
"fail": "[red]regressed[/red]",
"pass": "[green]fixed[/green]",
"changed": "[dim]score[/dim]",
}[f.became]
t.add_row(f.task, f.sample_id, f"{f.before:.2f}", f"{f.after:.2f}", mark)
console.print(t)
if len(shown) < len(result.flips):
console.print(f"[dim]… {len(result.flips) - len(shown)} more; --show-flips 0 for all[/dim]")
elif result.quality:
console.print("[dim]no sample outcomes changed[/dim]")
if result.perf:
t = Table(title="serving performance", header_style="bold")
for col in ("concurrency", "metric", "before", "after", "delta"):
t.add_column(col)
for concurrency in sorted(result.perf):
for i, d in enumerate(result.perf[concurrency]):
t.add_row(
str(concurrency) if i == 0 else "",
d.name.split(".")[-1],
_fmt(d.before),
_fmt(d.after),
_delta_cell(d),
)
console.print(t)
if not result.quality and not result.perf:
console.print("[yellow]neither run carries quality or perf results[/yellow]")
@app.command("results")
def results_cmd() -> None:
"""List saved score cards."""
files = sorted(RESULTS_DIR.glob("*.json"))
if not files:
console.print("[dim]no results yet[/dim]")
return
table = Table(title="results", header_style="bold")
for col in ("date", "target", "signal", "reference", "tok/s @1", "peak tok/s", "file"):
table.add_column(col)
for f in files:
d = json.loads(f.read_text())
v = d.get("verdict") or {}
table.add_row(
d["timestamp"][:10],
d["target_id"],
f"{v['signal_score']:.3f}" if v.get("signal_score") is not None else "-",
f"{v['reference_score']:.3f}" if v.get("reference_score") is not None else "-",
f"{v['single_stream_tps']:.1f}" if v.get("single_stream_tps") else "-",
f"{v['peak_throughput_tps']:.1f}" if v.get("peak_throughput_tps") else "-",
f.name,
)
console.print(table)
if __name__ == "__main__":
app()