Karti Tripathi
647 lines
25 KiB
Python
647 lines
25 KiB
Python
"""kbench command line."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import shlex
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import typer
|
|
from rich.console import Console
|
|
from rich.table import Table
|
|
|
|
from . import tasks as task_catalog
|
|
from .compare import compare as compare_runs, direction
|
|
from .sweep import analyse, point_from_run
|
|
from .registry import load_registry
|
|
from .results import RESULTS_DIR, Run
|
|
from .run import build_run, run_perf, run_quality
|
|
|
|
app = typer.Typer(
|
|
add_completion=False,
|
|
help="Lumbridge Bench -- private eval and serving-performance evidence for self-hosted and API models",
|
|
)
|
|
console = Console()
|
|
|
|
|
|
def _parse_concurrencies(value: str) -> tuple[int, ...]:
|
|
return tuple(int(x) for x in value.split(",") if x.strip())
|
|
|
|
|
|
@app.command("targets")
|
|
def list_targets() -> None:
|
|
"""List benchmarkable targets."""
|
|
registry = load_registry()
|
|
table = Table(title="targets", header_style="bold")
|
|
for col in ("id", "tier", "host", "quant", "params", "engine", "aliases"):
|
|
table.add_column(col)
|
|
for t in registry.targets.values():
|
|
params = f"{t.params_b:g}B" if t.params_b else "-"
|
|
if t.active_params_b:
|
|
params += f" ({t.active_params_b:g}B active)"
|
|
table.add_row(
|
|
t.id, t.tier, t.host, t.quant or "-", params,
|
|
t.serving.engine, ", ".join(t.aliases) or "-",
|
|
)
|
|
console.print(table)
|
|
|
|
|
|
@app.command("tasks")
|
|
def list_tasks() -> None:
|
|
"""List benchmark tasks."""
|
|
table = Table(title="tasks", header_style="bold")
|
|
for col in ("name", "tier", "samples", "judge", "inspect task"):
|
|
table.add_column(col)
|
|
for spec in task_catalog.CATALOG.values():
|
|
table.add_row(
|
|
spec.name, spec.tier, str(spec.dataset_samples or "-"),
|
|
"yes" if spec.judge_required else "no", spec.inspect_task,
|
|
)
|
|
console.print(table)
|
|
console.print(
|
|
"\n[dim]reference = public, likely contaminated, calibration only\n"
|
|
"signal = private, ours, the actual measurement[/dim]"
|
|
)
|
|
|
|
|
|
@app.command("snapshot")
|
|
def snapshot(host: str) -> None:
|
|
"""Print the live serving flags on a host, for pasting into models.yaml.
|
|
|
|
Results are only comparable between targets with identical flags, so the
|
|
registry has to record what the server was ACTUALLY started with rather
|
|
than what we believe we started it with.
|
|
"""
|
|
registry = load_registry()
|
|
h = registry.host(host)
|
|
if not h.ssh:
|
|
raise typer.BadParameter(f"host {host!r} has no ssh address")
|
|
|
|
proc = subprocess.run(
|
|
["ssh", h.ssh, "ps -eo args | grep -E '[v]llm serve|[s]glang|[l]lama-server'"],
|
|
capture_output=True, text=True, timeout=30,
|
|
# A host with no serving process is an ordinary outcome, not an
|
|
# exception -- it is reported below.
|
|
check=False,
|
|
)
|
|
if proc.returncode != 0 or not proc.stdout.strip():
|
|
console.print(f"[yellow]no serving process found on {host}[/yellow]")
|
|
raise typer.Exit(1)
|
|
|
|
for line in proc.stdout.strip().splitlines():
|
|
parts = shlex.split(line)
|
|
console.print(f"\n[bold]{parts[0] if parts else '?'}[/bold]")
|
|
flags: dict[str, object] = {}
|
|
i = 0
|
|
while i < len(parts):
|
|
if parts[i].startswith("--"):
|
|
key = parts[i][2:].replace("-", "_")
|
|
if i + 1 < len(parts) and not parts[i + 1].startswith("--"):
|
|
flags[key] = parts[i + 1]
|
|
i += 2
|
|
else:
|
|
flags[key] = True
|
|
i += 1
|
|
else:
|
|
i += 1
|
|
for k, v in flags.items():
|
|
console.print(f" {k}: {v}")
|
|
|
|
|
|
|
|
def _load_prompts(path: Path | None) -> list[str] | None:
|
|
"""Read representative prompts: one per line, or a JSONL with an `input` field.
|
|
|
|
Measuring speculative decoding on the synthetic prompt is measuring the wrong thing — the
|
|
draft model's acceptance rate depends on how predictable the text is, and lorem with a
|
|
random nonce is not the text you serve.
|
|
"""
|
|
if path is None:
|
|
return None
|
|
lines = [ln.strip() for ln in path.read_text().splitlines() if ln.strip()]
|
|
prompts = []
|
|
for ln in lines:
|
|
if ln.startswith("{"):
|
|
try:
|
|
prompts.append(json.loads(ln)["input"])
|
|
continue
|
|
except (json.JSONDecodeError, KeyError):
|
|
pass
|
|
prompts.append(ln)
|
|
if not prompts:
|
|
raise typer.BadParameter(f"{path} contains no prompts")
|
|
return prompts
|
|
|
|
|
|
@app.command("perf")
|
|
def perf_cmd(
|
|
target: str,
|
|
concurrency: str = typer.Option("1,8,32", help="comma-separated concurrency levels"),
|
|
input_tokens: int = typer.Option(1024),
|
|
output_tokens: int = typer.Option(256),
|
|
save: bool = typer.Option(True, help="write a results JSON"),
|
|
prompt_file: Path | None = typer.Option(
|
|
None, help="representative prompts (one per line, or JSONL with an `input` field)"
|
|
),
|
|
natural_stop: bool = typer.Option(
|
|
False,
|
|
help="let the model stop on its own instead of pinning output length. Required for an "
|
|
"honest speculative-decoding measurement; weakens comparability across points.",
|
|
),
|
|
) -> None:
|
|
"""Run a serving performance sweep only (no quality tasks)."""
|
|
registry = load_registry()
|
|
t = registry.target(target)
|
|
result = run_perf(
|
|
t, _parse_concurrencies(concurrency), input_tokens, output_tokens,
|
|
prompts=_load_prompts(prompt_file), pin_output=not natural_stop,
|
|
)
|
|
run = build_run(registry, t, quality=[], perf=result)
|
|
_print_card(run)
|
|
if save:
|
|
path = run.save()
|
|
console.print(f"\n[green]saved[/green] {path.relative_to(Path.cwd())}")
|
|
|
|
|
|
@app.command("run")
|
|
def run_cmd(
|
|
target: str,
|
|
tasks: str = typer.Option("ifeval", help="comma-separated task names, or 'all'"),
|
|
limit: int | None = typer.Option(None, help="cap samples per task (recorded in output)"),
|
|
max_connections: int = typer.Option(16, help="concurrent requests to the model"),
|
|
epochs: int = typer.Option(
|
|
1,
|
|
help="run each sample N times and average. Sampling is not reproducible on a "
|
|
"batching server even with a fixed seed, so one epoch is a draw, not a rate.",
|
|
),
|
|
concurrency: str = typer.Option("1,8,32"),
|
|
input_tokens: int = typer.Option(1024),
|
|
output_tokens: int = typer.Option(256),
|
|
skip_perf: bool = typer.Option(False),
|
|
skip_quality: bool = typer.Option(False),
|
|
) -> None:
|
|
"""Full score card: quality tasks + perf sweep -> one results JSON."""
|
|
registry = load_registry()
|
|
t = registry.target(target)
|
|
|
|
names = list(task_catalog.CATALOG) if tasks == "all" else [
|
|
n.strip() for n in tasks.split(",") if n.strip()
|
|
]
|
|
specs = [task_catalog.get(n) for n in names]
|
|
|
|
quality = []
|
|
if not skip_quality:
|
|
quality = run_quality(
|
|
t, specs, limit=limit, max_connections=max_connections, epochs=epochs
|
|
)
|
|
|
|
perf = None
|
|
if not skip_perf:
|
|
perf = run_perf(t, _parse_concurrencies(concurrency), input_tokens, output_tokens)
|
|
|
|
run = build_run(registry, t, quality=quality, perf=perf)
|
|
_print_card(run)
|
|
path = run.save()
|
|
console.print(f"\n[green]saved[/green] {path.relative_to(Path.cwd())}")
|
|
|
|
|
|
@app.command("sweep")
|
|
def sweep_cmd(
|
|
targets: list[str] = typer.Argument(..., help="two or more target ids to compare"),
|
|
tasks: str = typer.Option("ifeval", help="comma-separated task names, or 'all'"),
|
|
limit: int | None = typer.Option(None, help="cap samples per task"),
|
|
max_connections: int = typer.Option(16),
|
|
concurrency: str = typer.Option("1,8,32"),
|
|
input_tokens: int = typer.Option(1024),
|
|
output_tokens: int = typer.Option(256),
|
|
skip_perf: bool = typer.Option(False),
|
|
skip_quality: bool = typer.Option(False),
|
|
) -> None:
|
|
"""Run the same tasks against several targets and reduce them to a frontier.
|
|
|
|
Sequential on purpose: on a unified-memory box two targets cannot both be resident, so
|
|
running them together would measure contention rather than either one. Each target still
|
|
produces an ordinary score card in results/, so compare and card keep working on them.
|
|
"""
|
|
if len(targets) < 2:
|
|
raise typer.BadParameter("a sweep needs at least two targets")
|
|
|
|
registry = load_registry()
|
|
names = list(task_catalog.CATALOG) if tasks == "all" else [
|
|
n.strip() for n in tasks.split(",") if n.strip()
|
|
]
|
|
specs = [task_catalog.get(n) for n in names]
|
|
|
|
points, hosts = [], []
|
|
for i, target_id in enumerate(targets, start=1):
|
|
console.rule(f"[bold]{i}/{len(targets)} {target_id}[/bold]")
|
|
try:
|
|
t = registry.target(target_id)
|
|
hosts.append(t.host)
|
|
quality = [] if skip_quality else run_quality(
|
|
t, specs, limit=limit, max_connections=max_connections
|
|
)
|
|
perf = None if skip_perf else run_perf(
|
|
t, _parse_concurrencies(concurrency), input_tokens, output_tokens
|
|
)
|
|
run = build_run(registry, t, quality=quality, perf=perf)
|
|
path = run.save()
|
|
console.print(f"[green]saved[/green] {path.name}")
|
|
points.append(point_from_run(run.to_dict(), str(path.name)))
|
|
except Exception as exc: # noqa: BLE001
|
|
# One bad target must not throw away the targets already measured — a sweep is
|
|
# hours of GPU time and re-running the good ones to reach the next is wasteful.
|
|
console.print(f"[red]failed[/red] {target_id}: {exc}")
|
|
points.append(
|
|
point_from_run({"target_id": target_id, "verdict": {}})
|
|
)
|
|
points[-1].error = str(exc)
|
|
|
|
result = analyse(points, hosts=hosts)
|
|
|
|
console.print()
|
|
console.rule("[bold]frontier[/bold]")
|
|
table = Table(header_style="bold")
|
|
for col in ("", "target", "signal", "peak tok/s", "tok/s @1", "verdict"):
|
|
table.add_column(col)
|
|
for point in sorted(
|
|
result.points, key=lambda x: (x.error is not None, -(x.throughput_tps or 0))
|
|
):
|
|
if point.error:
|
|
mark, verdict = "[red]![/red]", f"[red]{point.error[:44]}[/red]"
|
|
elif point.target_id in result.dominated:
|
|
mark = "[dim]·[/dim]"
|
|
verdict = f"[dim]beaten by {result.dominated[point.target_id]}[/dim]"
|
|
else:
|
|
mark, verdict = "[green]✓[/green]", "[green]worth running[/green]"
|
|
table.add_row(
|
|
mark,
|
|
point.target_id,
|
|
f"{point.quality:.3f}" if point.quality is not None else "-",
|
|
f"{point.throughput_tps:.1f}" if point.throughput_tps else "-",
|
|
f"{point.single_stream_tps:.1f}" if point.single_stream_tps else "-",
|
|
verdict,
|
|
)
|
|
console.print(table)
|
|
|
|
for warning in result.warnings:
|
|
console.print(f"[yellow]![/yellow] {warning}")
|
|
|
|
if result.best:
|
|
console.print(
|
|
f"\n[bold]{len(result.best)} of {len(result.points)} "
|
|
f"worth considering:[/bold] " + ", ".join(result.best)
|
|
)
|
|
if result.dominated:
|
|
console.print(
|
|
"[dim]The rest are beaten on every measured axis at once — not a close call, "
|
|
"a strictly worse option.[/dim]"
|
|
)
|
|
|
|
|
|
@app.command("card")
|
|
def card_cmd(path: Path) -> None:
|
|
"""Render a saved results JSON as a score card."""
|
|
data = Run.load(path)
|
|
_print_card_dict(data)
|
|
|
|
|
|
def _print_card(run: Run) -> None:
|
|
_print_card_dict(run.to_dict())
|
|
|
|
|
|
def _print_card_dict(data: dict) -> None:
|
|
tgt = data["target"]
|
|
console.print()
|
|
console.rule(f"[bold]{tgt.get('display_name', data['target_id'])}[/bold]")
|
|
console.print(
|
|
f"[dim]{data['target_id']} | {data['host'].get('hardware', '?')} | "
|
|
f"{data['timestamp']}[/dim]"
|
|
)
|
|
|
|
quality = data.get("quality") or []
|
|
if quality:
|
|
table = Table(title="quality", header_style="bold", title_justify="left")
|
|
for col in ("task", "tier", "n", "accuracy", "notes"):
|
|
table.add_column(col)
|
|
for q in quality:
|
|
if q.get("error"):
|
|
table.add_row(q["task"], q["tier"], "-", "[red]ERROR[/red]", q["error"][:60])
|
|
continue
|
|
acc = q["metrics"].get("accuracy")
|
|
limit = q["metrics"].get("_limit")
|
|
note = f"limited to {int(limit)} samples" if limit else ""
|
|
if q["tier"] == "reference":
|
|
note = (note + "; " if note else "") + "calibration only"
|
|
table.add_row(
|
|
q["task"], q["tier"], str(q["n_samples"]),
|
|
f"{acc:.3f}" if acc is not None else "-", note,
|
|
)
|
|
console.print(table)
|
|
|
|
perf = data.get("perf")
|
|
if perf and perf.get("points"):
|
|
table = Table(title="serving perf", header_style="bold", title_justify="left")
|
|
points = data["perf"].get("points") or []
|
|
# Only widen the table when there is something to put in the column — a permanent
|
|
# "accept: -" on every non-speculative target is noise.
|
|
show_accept = any(pt.get("spec_acceptance_rate") is not None for pt in points)
|
|
cols = ["conc", "tok/s total", "tok/s /stream", "TTFT p50", "TTFT p95", "TPOT p50"]
|
|
if show_accept:
|
|
cols.append("accept")
|
|
cols.append("failed")
|
|
for col in cols:
|
|
table.add_column(col, justify="right")
|
|
for p in perf["points"]:
|
|
table.add_row(
|
|
str(p["concurrency"]),
|
|
f"{p['output_tps_total']:.1f}",
|
|
f"{p['output_tps_per_stream']:.1f}",
|
|
f"{p['ttft_p50_ms']:.0f}ms" if p.get("ttft_p50_ms") else "-",
|
|
f"{p['ttft_p95_ms']:.0f}ms" if p.get("ttft_p95_ms") else "-",
|
|
f"{p['tpot_p50_ms']:.1f}ms" if p.get("tpot_p50_ms") else "-",
|
|
*(
|
|
[
|
|
f"{p['spec_acceptance_rate']:.0%}"
|
|
if p.get("spec_acceptance_rate") is not None
|
|
else "-"
|
|
]
|
|
if show_accept
|
|
else []
|
|
),
|
|
f"[red]{p['failed']}[/red]" if p["failed"] else "0",
|
|
)
|
|
console.print(table)
|
|
if any(pt.get("natural_stop") for pt in points):
|
|
spread = next(
|
|
(pt["output_token_stdev"] for pt in points if pt.get("output_token_stdev")), None
|
|
)
|
|
console.print(
|
|
" [dim]natural stop: output length was not pinned"
|
|
+ (f", ±{spread:.0f} tokens" if spread else "")
|
|
+ ". Required for an honest speculative-decoding number, and it means these"
|
|
" points are not directly comparable to pinned-length runs.[/dim]"
|
|
)
|
|
|
|
verdict = data.get("verdict") or {}
|
|
if verdict:
|
|
# Contamination first and loud. It is the one result that invalidates every other
|
|
# number on the card, so it cannot be a row someone scrolls past.
|
|
state = verdict.get("contamination")
|
|
if state == "detected":
|
|
console.print()
|
|
console.print(
|
|
"[bold white on red] CONTAMINATED [/bold white on red] "
|
|
f"the model reproduced {verdict.get('canary_score', 0):.0%} of the canary probes."
|
|
)
|
|
console.print(
|
|
" [red]Quality scores for this target are void — it has seen the eval set.[/red]"
|
|
)
|
|
if verdict.get("signal_score_unverified") is not None:
|
|
console.print(
|
|
f" [dim]withheld signal score: {verdict['signal_score_unverified']:.3f} "
|
|
"(recorded for forensics, not usable as a result)[/dim]"
|
|
)
|
|
console.print(" [dim]Regenerate the private set before trusting anything here.[/dim]")
|
|
elif state == "unverified":
|
|
console.print(
|
|
"\n[yellow]![/yellow] no contamination probe ran — quality scores are unverified"
|
|
)
|
|
|
|
console.print("\n[bold]verdict[/bold]")
|
|
for k, v in verdict.items():
|
|
if v is None or k in ("contamination", "signal_score_unverified"):
|
|
continue
|
|
console.print(f" {k}: {v}")
|
|
|
|
|
|
@app.command("add")
|
|
def add_cmd(
|
|
family: str,
|
|
input_text: str | None = typer.Option(None, "--input", help="prompt text"),
|
|
input_file: Path | None = typer.Option(None, help="read the prompt from a file"),
|
|
target: str | None = typer.Option(None, help="expected answer (or regex)"),
|
|
scorer: str = typer.Option("includes", help="exact | includes | regex | rubric"),
|
|
rubric: str | None = typer.Option(None, help="grading instructions for --scorer rubric"),
|
|
split: str = typer.Option("test", help="train | dev | test -- never change this later"),
|
|
tier: str = typer.Option("private", help="private | public"),
|
|
source: str = typer.Option("manual", help="where this sample came from"),
|
|
tags: str = typer.Option("", help="comma-separated"),
|
|
) -> None:
|
|
"""Append a sample to a family, with id, canary, and validation applied.
|
|
|
|
Do not hand-edit the JSONL. This assigns a stable non-reused id (per-sample
|
|
regression tracking keys on it) and stamps a canary GUID used later to
|
|
prove contamination.
|
|
"""
|
|
import uuid
|
|
|
|
from .tasks.signal import family_path, validate_record
|
|
|
|
if input_file:
|
|
input_text = input_file.read_text().strip()
|
|
if not input_text:
|
|
raise typer.BadParameter("provide --input or --input-file")
|
|
|
|
path = family_path(family, tier)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
existing = [
|
|
json.loads(line)
|
|
for line in (path.read_text().splitlines() if path.exists() else [])
|
|
if line.strip() and not line.startswith("//")
|
|
]
|
|
# Highest existing number + 1, so ids are never reused even after deletions.
|
|
next_n = 1 + max(
|
|
(int(r["id"].rsplit("/", 1)[-1]) for r in existing if "/" in r.get("id", "")),
|
|
default=0,
|
|
)
|
|
|
|
record = {
|
|
"id": f"{family}/{next_n:04d}",
|
|
"family": family,
|
|
"split": split,
|
|
"canary": f"KBENCH-CANARY-{uuid.uuid4().hex[:8]}",
|
|
"input": input_text,
|
|
"target": target,
|
|
"scorer": scorer,
|
|
"rubric": rubric,
|
|
"metadata": {
|
|
"source": source,
|
|
"authored": __import__("datetime").date.today().isoformat(),
|
|
"tags": [t.strip() for t in tags.split(",") if t.strip()],
|
|
},
|
|
}
|
|
validate_record(record, "new sample")
|
|
|
|
with path.open("a") as f:
|
|
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
|
|
console.print(f"[green]added[/green] {record['id']} -> {path}")
|
|
console.print(
|
|
"[yellow]scrub check:[/yellow] no keys, tokens, tailscale addresses, "
|
|
"or customer data in that sample?"
|
|
)
|
|
|
|
|
|
def _resolve_run(ref: str) -> Path:
|
|
"""Accept a path, a bare filename, or a run-id prefix.
|
|
|
|
Nobody types a full run id. Being strict about it just means every comparison starts with
|
|
an `ls`.
|
|
"""
|
|
direct = Path(ref)
|
|
if direct.is_file():
|
|
return direct
|
|
candidates = sorted(RESULTS_DIR.glob(f"*{ref}*.json"))
|
|
if not candidates:
|
|
raise typer.BadParameter(f"no result matching {ref!r} in {RESULTS_DIR}")
|
|
if len(candidates) > 1:
|
|
names = "\n ".join(c.name for c in candidates)
|
|
raise typer.BadParameter(f"{ref!r} matches several results:\n {names}")
|
|
return candidates[0]
|
|
|
|
|
|
def _fmt(value: float | None, places: int = 1) -> str:
|
|
return "-" if value is None else f"{value:.{places}f}"
|
|
|
|
|
|
def _delta_cell(d, places: int = 1) -> str:
|
|
"""A delta with its sign, its percent, and — crucially — whether that is good."""
|
|
if d.absolute is None:
|
|
return "[dim]-[/dim]"
|
|
verdict = direction(d.name.split(".")[-1], d)
|
|
colour = {"better": "green", "worse": "red", "flat": "dim"}[verdict]
|
|
pct = "" if d.percent is None else f" ({d.percent:+.1f}%)"
|
|
return f"[{colour}]{d.absolute:+.{places}f}{pct}[/{colour}]"
|
|
|
|
|
|
@app.command("compare")
|
|
def compare_cmd(
|
|
before: str = typer.Argument(..., help="Baseline run: path, filename, or run-id fragment"),
|
|
after: str = typer.Argument(..., help="Run to compare against the baseline"),
|
|
show_flips: int = typer.Option(20, help="How many changed samples to list (0 for all)"),
|
|
) -> None:
|
|
"""Diff two score cards: what changed, and whether the comparison is honest."""
|
|
before_path, after_path = _resolve_run(before), _resolve_run(after)
|
|
b, a = Run.load(before_path), Run.load(after_path)
|
|
result = compare_runs(b, a)
|
|
|
|
console.print()
|
|
console.rule("[bold]compare[/bold]")
|
|
console.print(f"[dim]before[/dim] {before_path.name}")
|
|
console.print(f"[dim]after [/dim] {after_path.name}\n")
|
|
|
|
# What actually differs about the two targets. Printed FIRST: without it the reader has
|
|
# no idea what the numbers below are attributable to.
|
|
if result.target_deltas or result.host_changed:
|
|
t = Table(title="what changed about the target", header_style="bold")
|
|
for col in ("axis", "before", "after"):
|
|
t.add_column(col, overflow="fold")
|
|
for d in result.target_deltas:
|
|
t.add_row(d.field, str(d.before), str(d.after))
|
|
if result.host_changed:
|
|
h = result.host_changed
|
|
t.add_row("[yellow]host[/yellow]", str(h.before), str(h.after))
|
|
console.print(t)
|
|
|
|
for warning in result.warnings:
|
|
console.print(f"[yellow]![/yellow] {warning}")
|
|
if result.warnings:
|
|
console.print()
|
|
|
|
if result.quality:
|
|
t = Table(title="quality", header_style="bold")
|
|
for col in ("metric", "before", "after", "delta"):
|
|
t.add_column(col)
|
|
for d in result.quality:
|
|
t.add_row(d.name, _fmt(d.before, 3), _fmt(d.after, 3), _delta_cell(d, 3))
|
|
console.print(t)
|
|
|
|
if result.flips:
|
|
shown = result.flips if show_flips == 0 else result.flips[:show_flips]
|
|
# When nothing about the measurement changed -- same target, same host, same eval
|
|
# set, same sampling -- a flipped sample did not regress or get fixed. It is the
|
|
# same question answered twice with different luck. Calling it "regressed" sends
|
|
# someone hunting a cause that does not exist, which is worse than saying nothing.
|
|
same_setup = (
|
|
not result.target_deltas
|
|
and not result.host_changed
|
|
and not any(
|
|
"edited between runs" in w or "Sampling changed" in w
|
|
for w in result.warnings
|
|
)
|
|
)
|
|
t = Table(
|
|
title=(
|
|
f"samples that changed ({len(result.flips)})"
|
|
+ (" — same setup, so this is instability, not change" if same_setup else "")
|
|
),
|
|
header_style="bold",
|
|
)
|
|
for col in ("task", "sample", "before", "after", ""):
|
|
t.add_column(col, overflow="fold")
|
|
for f in shown:
|
|
if same_setup:
|
|
mark = "[yellow]unstable[/yellow]"
|
|
else:
|
|
mark = {
|
|
"fail": "[red]regressed[/red]",
|
|
"pass": "[green]fixed[/green]",
|
|
"changed": "[dim]score[/dim]",
|
|
}[f.became]
|
|
t.add_row(f.task, f.sample_id, f"{f.before:.2f}", f"{f.after:.2f}", mark)
|
|
console.print(t)
|
|
if len(shown) < len(result.flips):
|
|
console.print(f"[dim]… {len(result.flips) - len(shown)} more; --show-flips 0 for all[/dim]")
|
|
elif result.quality:
|
|
console.print("[dim]no sample outcomes changed[/dim]")
|
|
|
|
if result.perf:
|
|
t = Table(title="serving performance", header_style="bold")
|
|
for col in ("concurrency", "metric", "before", "after", "delta"):
|
|
t.add_column(col)
|
|
for concurrency in sorted(result.perf):
|
|
for i, d in enumerate(result.perf[concurrency]):
|
|
t.add_row(
|
|
str(concurrency) if i == 0 else "",
|
|
d.name.split(".")[-1],
|
|
_fmt(d.before),
|
|
_fmt(d.after),
|
|
_delta_cell(d),
|
|
)
|
|
console.print(t)
|
|
|
|
if not result.quality and not result.perf:
|
|
console.print("[yellow]neither run carries quality or perf results[/yellow]")
|
|
|
|
|
|
@app.command("results")
|
|
def results_cmd() -> None:
|
|
"""List saved score cards."""
|
|
files = sorted(RESULTS_DIR.glob("*.json"))
|
|
if not files:
|
|
console.print("[dim]no results yet[/dim]")
|
|
return
|
|
table = Table(title="results", header_style="bold")
|
|
for col in ("date", "target", "signal", "reference", "tok/s @1", "peak tok/s", "file"):
|
|
table.add_column(col)
|
|
for f in files:
|
|
d = json.loads(f.read_text())
|
|
v = d.get("verdict") or {}
|
|
table.add_row(
|
|
d["timestamp"][:10],
|
|
d["target_id"],
|
|
f"{v['signal_score']:.3f}" if v.get("signal_score") is not None else "-",
|
|
f"{v['reference_score']:.3f}" if v.get("reference_score") is not None else "-",
|
|
f"{v['single_stream_tps']:.1f}" if v.get("single_stream_tps") else "-",
|
|
f"{v['peak_throughput_tps']:.1f}" if v.get("peak_throughput_tps") else "-",
|
|
f.name,
|
|
)
|
|
console.print(table)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app()
|