Lumbridge Bench
This commit is contained in:
Karti Tripathi
@@ -0,0 +1,335 @@
|
||||
"""Diff two runs.
|
||||
|
||||
This is the verb the bench exists for. A single score card tells you what one target did
|
||||
once; it cannot tell you whether a quantisation cost you anything, whether a checkpoint
|
||||
regressed, or whether a serving flag was worth it. Those are the questions people actually
|
||||
have, and all of them are differences.
|
||||
|
||||
Two design decisions carry most of the weight here:
|
||||
|
||||
1. PER-SAMPLE DIFFS, NOT AGGREGATE DIFFS. "0.81 → 0.78" tells you something got worse and
|
||||
nothing about what. Listing the samples that flipped tells you *which capability broke*,
|
||||
which is the difference between a leaderboard and a debugging tool. The result schema
|
||||
stores per-sample outcomes precisely so this is possible.
|
||||
|
||||
2. COMPARABILITY IS CHECKED, NOT ASSUMED. Two runs are only honestly comparable when the
|
||||
thing you did not change actually did not change. Comparing a quant against a different
|
||||
quant on a different host tells you nothing, but it renders just as confidently as a
|
||||
clean comparison. So the diff states what differs about the targets themselves, and
|
||||
warns when more than one axis moved at once.
|
||||
|
||||
Perf and quality are diffed independently: a run may have one, both, or neither, and a
|
||||
target with no signal-tier score is still worth comparing on throughput.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
# Below this, a rate difference is not worth reading as a change. Applied to rates only: a
|
||||
# single sample flipping is always worth showing.
|
||||
#
|
||||
# Calibrate this against your own hardware before trusting it. The two runs committed here are
|
||||
# the same target on the same box on the same day, and throughput between them differs by ~10%
|
||||
# — so on that machine, anything under 10% is indistinguishable from running it twice. The
|
||||
# floor below is deliberately lower than that observed spread, because suppressing a real
|
||||
# regression is worse than showing noise the reader can dismiss; the identical-target warning
|
||||
# is what tells them which they are looking at.
|
||||
PERF_NOISE_FLOOR_PCT = 5.0
|
||||
|
||||
# The axes that make a target a target. Comparing across more than one at a time produces a
|
||||
# number that cannot be attributed to anything.
|
||||
TARGET_AXES = ("model", "quantization", "checkpoint", "serving_config", "engine")
|
||||
|
||||
|
||||
@dataclass
|
||||
class FieldDelta:
|
||||
"""One axis on which two targets differ."""
|
||||
|
||||
field: str
|
||||
before: Any
|
||||
after: Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class SampleFlip:
|
||||
"""A sample whose outcome changed between runs."""
|
||||
|
||||
sample_id: str
|
||||
task: str
|
||||
before: float
|
||||
after: float
|
||||
became: str # "pass" | "fail" | "changed"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricDelta:
|
||||
name: str
|
||||
before: float | None
|
||||
after: float | None
|
||||
|
||||
@property
|
||||
def absolute(self) -> float | None:
|
||||
if self.before is None or self.after is None:
|
||||
return None
|
||||
return self.after - self.before
|
||||
|
||||
@property
|
||||
def percent(self) -> float | None:
|
||||
if self.before in (None, 0) or self.after is None:
|
||||
return None
|
||||
return (self.after - self.before) / abs(self.before) * 100.0
|
||||
|
||||
@property
|
||||
def significant(self) -> bool:
|
||||
"""Whether a rate moved enough to be worth reading."""
|
||||
pct = self.percent
|
||||
return pct is not None and abs(pct) >= PERF_NOISE_FLOOR_PCT
|
||||
|
||||
|
||||
@dataclass
|
||||
class Comparison:
|
||||
before_id: str
|
||||
after_id: str
|
||||
target_deltas: list[FieldDelta] = field(default_factory=list)
|
||||
host_changed: FieldDelta | None = None
|
||||
quality: list[MetricDelta] = field(default_factory=list)
|
||||
flips: list[SampleFlip] = field(default_factory=list)
|
||||
perf: dict[int, list[MetricDelta]] = field(default_factory=dict)
|
||||
only_in_before: list[str] = field(default_factory=list)
|
||||
only_in_after: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def confounded(self) -> bool:
|
||||
"""More than one axis moved, so no difference below can be attributed to any of them."""
|
||||
return len(self.target_deltas) + (1 if self.host_changed else 0) > 1
|
||||
|
||||
|
||||
def _target_field(target: dict[str, Any], name: str) -> Any:
|
||||
value = target.get(name)
|
||||
# serving_config is a dict; render it deterministically so a reordered YAML load does not
|
||||
# read as a change.
|
||||
if isinstance(value, dict):
|
||||
return tuple(sorted((str(k), str(v)) for k, v in value.items()))
|
||||
return value
|
||||
|
||||
|
||||
def _diff_targets(before: dict, after: dict) -> list[FieldDelta]:
|
||||
deltas = []
|
||||
for axis in TARGET_AXES:
|
||||
b, a = _target_field(before, axis), _target_field(after, axis)
|
||||
if b != a:
|
||||
deltas.append(FieldDelta(axis, b, a))
|
||||
return deltas
|
||||
|
||||
|
||||
def _sample_index(run: dict) -> dict[tuple[str, str], dict]:
|
||||
"""(task, sample_id) -> outcome. Keyed on both because ids are only unique within a task."""
|
||||
index = {}
|
||||
for quality in run.get("quality") or []:
|
||||
task = quality.get("task", "?")
|
||||
for sample in quality.get("samples") or []:
|
||||
index[(task, sample["sample_id"])] = sample
|
||||
return index
|
||||
|
||||
|
||||
def _perf_index(run: dict) -> dict[int, dict]:
|
||||
perf = run.get("perf") or {}
|
||||
return {p["concurrency"]: p for p in perf.get("points") or []}
|
||||
|
||||
|
||||
def compare(before: dict, after: dict) -> Comparison:
|
||||
"""Diff two loaded result dicts. Neither is mutated."""
|
||||
result = Comparison(
|
||||
before_id=before.get("run_id", "?"),
|
||||
after_id=after.get("run_id", "?"),
|
||||
target_deltas=_diff_targets(before.get("target") or {}, after.get("target") or {}),
|
||||
)
|
||||
|
||||
b_host = (before.get("host") or {}).get("id") or (before.get("host") or {}).get("name")
|
||||
a_host = (after.get("host") or {}).get("id") or (after.get("host") or {}).get("name")
|
||||
if b_host != a_host:
|
||||
result.host_changed = FieldDelta("host", b_host, a_host)
|
||||
|
||||
# A contaminated run has no comparable quality half, and diffing against one produces a
|
||||
# delta that looks exactly like a real regression or gain. Say so before anything else.
|
||||
for label, run in (("before", before), ("after", after)):
|
||||
state = (run.get("verdict") or {}).get("contamination")
|
||||
if state == "detected":
|
||||
result.warnings.append(
|
||||
f"The {label} run is CONTAMINATED — it reproduced canary probes. Its quality "
|
||||
"numbers are memorisation, not capability, and no quality delta below means "
|
||||
"anything."
|
||||
)
|
||||
elif state == "unverified":
|
||||
result.warnings.append(
|
||||
f"The {label} run has no contamination probe, so its quality scores are "
|
||||
"unverified."
|
||||
)
|
||||
|
||||
if not result.target_deltas and not result.host_changed:
|
||||
# Same target, same box: any difference is run-to-run variance, which is worth knowing
|
||||
# because it sets the floor below which no other comparison means anything.
|
||||
result.warnings.append(
|
||||
"Identical target and host — this measures run-to-run variance, not a change."
|
||||
)
|
||||
if result.confounded:
|
||||
moved = [d.field for d in result.target_deltas] + (["host"] if result.host_changed else [])
|
||||
result.warnings.append(
|
||||
f"{len(moved)} axes changed at once ({', '.join(moved)}) — no difference below "
|
||||
"can be attributed to any one of them."
|
||||
)
|
||||
|
||||
# ---- quality -----------------------------------------------------------------
|
||||
b_metrics = {q["task"]: q.get("metrics", {}) for q in before.get("quality") or []}
|
||||
a_metrics = {q["task"]: q.get("metrics", {}) for q in after.get("quality") or []}
|
||||
for task in sorted(set(b_metrics) | set(a_metrics)):
|
||||
keys = set(b_metrics.get(task, {})) | set(a_metrics.get(task, {}))
|
||||
for key in sorted(keys):
|
||||
result.quality.append(
|
||||
MetricDelta(
|
||||
f"{task}.{key}",
|
||||
b_metrics.get(task, {}).get(key),
|
||||
a_metrics.get(task, {}).get(key),
|
||||
)
|
||||
)
|
||||
|
||||
b_samples, a_samples = _sample_index(before), _sample_index(after)
|
||||
for key in sorted(b_samples.keys() & a_samples.keys()):
|
||||
b, a = b_samples[key], a_samples[key]
|
||||
if b.get("passed") == a.get("passed") and b.get("score") == a.get("score"):
|
||||
continue
|
||||
if b.get("passed") and not a.get("passed"):
|
||||
became = "fail"
|
||||
elif a.get("passed") and not b.get("passed"):
|
||||
became = "pass"
|
||||
else:
|
||||
became = "changed"
|
||||
result.flips.append(
|
||||
SampleFlip(key[1], key[0], b.get("score", 0.0), a.get("score", 0.0), became)
|
||||
)
|
||||
# Regressions first: a capability that broke is the reason anyone runs this.
|
||||
result.flips.sort(key=lambda f: ({"fail": 0, "changed": 1, "pass": 2}[f.became], f.sample_id))
|
||||
|
||||
# A sample present in only one run is not a flip — it is a changed eval set, which
|
||||
# silently moves an aggregate. Surfaced separately so it cannot be mistaken for signal.
|
||||
result.only_in_before = sorted(f"{t}/{s}" for t, s in b_samples.keys() - a_samples.keys())
|
||||
result.only_in_after = sorted(f"{t}/{s}" for t, s in a_samples.keys() - b_samples.keys())
|
||||
if not b_samples or not a_samples:
|
||||
# Not a changed eval set — one side simply never measured quality. Saying "100 samples
|
||||
# added" here would be a confusing way to report "the baseline is perf-only".
|
||||
if b_samples or a_samples:
|
||||
missing = "before" if not b_samples else "after"
|
||||
result.warnings.append(
|
||||
f"The {missing} run carries no quality results, so only serving performance "
|
||||
"is comparable."
|
||||
)
|
||||
elif result.only_in_before or result.only_in_after:
|
||||
result.warnings.append(
|
||||
f"The eval set changed: {len(result.only_in_before)} sample(s) gone, "
|
||||
f"{len(result.only_in_after)} added. Aggregate scores are not comparable."
|
||||
)
|
||||
|
||||
# An EDITED sample keeps its id, so the add/remove diff above sees nothing. Rewording a
|
||||
# prompt or loosening a regex changes what the score means while every id still lines up,
|
||||
# which is the most dangerous shape of eval drift: the comparison looks clean. The
|
||||
# fingerprint hashes (id, input, target), so it moves when the content does.
|
||||
b_fp = {q["task"]: q.get("dataset_fingerprint") for q in before.get("quality") or []}
|
||||
a_fp = {q["task"]: q.get("dataset_fingerprint") for q in after.get("quality") or []}
|
||||
for task in sorted(b_fp.keys() & a_fp.keys()):
|
||||
bf, af = b_fp[task], a_fp[task]
|
||||
if bf and af and bf != af:
|
||||
result.warnings.append(
|
||||
f"Task {task!r} was edited between runs ({bf} -> {af}): same sample ids, "
|
||||
"different content. The scores measure different questions."
|
||||
)
|
||||
elif not bf or not af:
|
||||
result.warnings.append(
|
||||
f"Task {task!r} has no dataset fingerprint on one side, so an edit to the "
|
||||
"sample text cannot be ruled out. Re-run to get a comparable pair."
|
||||
)
|
||||
|
||||
# Sampling is part of the identity of a quality result. The same model at temperature
|
||||
# 1 and at 0.6 is two different measurements, and unpinned sampling moved this bench's
|
||||
# score by 27% of its samples between identical runs -- far more than most real
|
||||
# regressions. A card that predates sampling being recorded cannot be ruled comparable
|
||||
# either, so say so rather than diff it silently.
|
||||
b_s, a_s = before.get("sampling") or {}, after.get("sampling") or {}
|
||||
if b_samples and a_samples:
|
||||
if not b_s or not a_s:
|
||||
result.warnings.append(
|
||||
"One run does not record how the model was sampled, so an unpinned "
|
||||
"temperature cannot be ruled out. Quality deltas below may be noise."
|
||||
)
|
||||
else:
|
||||
changed = sorted(
|
||||
k for k in set(b_s) | set(a_s) if b_s.get(k) != a_s.get(k)
|
||||
)
|
||||
# Epoch count is not a sampling change. Both sides estimate the same quantity;
|
||||
# more epochs just estimates it better. Reporting that as "not comparable"
|
||||
# overstates it, and a warning that overstates is one people learn to skip --
|
||||
# which costs you the ones that matter.
|
||||
decode = [k for k in changed if k != "epochs"]
|
||||
if decode:
|
||||
result.warnings.append(
|
||||
f"Sampling changed between runs ({', '.join(decode)}): "
|
||||
f"{ {k: b_s.get(k) for k in decode} } -> { {k: a_s.get(k) for k in decode} }. "
|
||||
"Quality scores are not comparable across different sampling."
|
||||
)
|
||||
if "epochs" in changed:
|
||||
b_e, a_e = b_s.get("epochs", 1), a_s.get("epochs", 1)
|
||||
result.warnings.append(
|
||||
f"Epochs differ ({b_e} -> {a_e}). Both estimate the same score; the "
|
||||
f"{'before' if (b_e or 1) < (a_e or 1) else 'after'} run is the noisier "
|
||||
"of the two, so read small deltas with that in mind."
|
||||
)
|
||||
|
||||
# ---- perf --------------------------------------------------------------------
|
||||
b_perf, a_perf = _perf_index(before), _perf_index(after)
|
||||
for concurrency in sorted(b_perf.keys() & a_perf.keys()):
|
||||
b, a = b_perf[concurrency], a_perf[concurrency]
|
||||
result.perf[concurrency] = [
|
||||
MetricDelta(name, b.get(name), a.get(name))
|
||||
for name in (
|
||||
"output_tps_total",
|
||||
"output_tps_per_stream",
|
||||
"ttft_p50_ms",
|
||||
"ttft_p95_ms",
|
||||
"tpot_p50_ms",
|
||||
# Higher is better, so the default direction rule is already right. Diffing it
|
||||
# is the point of an MTP comparison: throughput can move for many reasons,
|
||||
# acceptance moves only because drafting got better or worse.
|
||||
"spec_acceptance_rate",
|
||||
)
|
||||
if b.get(name) is not None or a.get(name) is not None
|
||||
]
|
||||
modes = {bool(p.get("natural_stop")) for p in list(b_perf.values()) + list(a_perf.values())}
|
||||
if len(modes) > 1:
|
||||
result.warnings.append(
|
||||
"One run pinned output length and the other let the model stop naturally. "
|
||||
"Throughput is not comparable across those two modes."
|
||||
)
|
||||
|
||||
dropped = sorted(b_perf.keys() - a_perf.keys())
|
||||
added = sorted(a_perf.keys() - b_perf.keys())
|
||||
if dropped or added:
|
||||
result.warnings.append(
|
||||
f"Concurrency sweep differs: {dropped or 'none'} dropped, {added or 'none'} added. "
|
||||
"Peak throughput is only comparable across a shared sweep."
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# Metrics where a smaller number is the better one, so a fall is an improvement.
|
||||
LOWER_IS_BETTER = ("ttft_p50_ms", "ttft_p95_ms", "tpot_p50_ms")
|
||||
|
||||
|
||||
def direction(metric: str, delta: MetricDelta) -> str:
|
||||
""""better" | "worse" | "flat" — meaning, not just sign."""
|
||||
if delta.absolute is None or not delta.significant:
|
||||
return "flat"
|
||||
improved = delta.absolute < 0 if metric in LOWER_IS_BETTER else delta.absolute > 0
|
||||
return "better" if improved else "worse"
|
||||
Reference in New Issue
Block a user