Karti Tripathi
336 lines
15 KiB
Python
336 lines
15 KiB
Python
"""Diff two runs.
|
|
|
|
This is the verb the bench exists for. A single score card tells you what one target did
|
|
once; it cannot tell you whether a quantisation cost you anything, whether a checkpoint
|
|
regressed, or whether a serving flag was worth it. Those are the questions people actually
|
|
have, and all of them are differences.
|
|
|
|
Two design decisions carry most of the weight here:
|
|
|
|
1. PER-SAMPLE DIFFS, NOT AGGREGATE DIFFS. "0.81 → 0.78" tells you something got worse and
|
|
nothing about what. Listing the samples that flipped tells you *which capability broke*,
|
|
which is the difference between a leaderboard and a debugging tool. The result schema
|
|
stores per-sample outcomes precisely so this is possible.
|
|
|
|
2. COMPARABILITY IS CHECKED, NOT ASSUMED. Two runs are only honestly comparable when the
|
|
thing you did not change actually did not change. Comparing a quant against a different
|
|
quant on a different host tells you nothing, but it renders just as confidently as a
|
|
clean comparison. So the diff states what differs about the targets themselves, and
|
|
warns when more than one axis moved at once.
|
|
|
|
Perf and quality are diffed independently: a run may have one, both, or neither, and a
|
|
target with no signal-tier score is still worth comparing on throughput.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
# Below this, a rate difference is not worth reading as a change. Applied to rates only: a
|
|
# single sample flipping is always worth showing.
|
|
#
|
|
# Calibrate this against your own hardware before trusting it. The two runs committed here are
|
|
# the same target on the same box on the same day, and throughput between them differs by ~10%
|
|
# — so on that machine, anything under 10% is indistinguishable from running it twice. The
|
|
# floor below is deliberately lower than that observed spread, because suppressing a real
|
|
# regression is worse than showing noise the reader can dismiss; the identical-target warning
|
|
# is what tells them which they are looking at.
|
|
PERF_NOISE_FLOOR_PCT = 5.0
|
|
|
|
# The axes that make a target a target. Comparing across more than one at a time produces a
|
|
# number that cannot be attributed to anything.
|
|
TARGET_AXES = ("model", "quantization", "checkpoint", "serving_config", "engine")
|
|
|
|
|
|
@dataclass
|
|
class FieldDelta:
|
|
"""One axis on which two targets differ."""
|
|
|
|
field: str
|
|
before: Any
|
|
after: Any
|
|
|
|
|
|
@dataclass
|
|
class SampleFlip:
|
|
"""A sample whose outcome changed between runs."""
|
|
|
|
sample_id: str
|
|
task: str
|
|
before: float
|
|
after: float
|
|
became: str # "pass" | "fail" | "changed"
|
|
|
|
|
|
@dataclass
|
|
class MetricDelta:
|
|
name: str
|
|
before: float | None
|
|
after: float | None
|
|
|
|
@property
|
|
def absolute(self) -> float | None:
|
|
if self.before is None or self.after is None:
|
|
return None
|
|
return self.after - self.before
|
|
|
|
@property
|
|
def percent(self) -> float | None:
|
|
if self.before in (None, 0) or self.after is None:
|
|
return None
|
|
return (self.after - self.before) / abs(self.before) * 100.0
|
|
|
|
@property
|
|
def significant(self) -> bool:
|
|
"""Whether a rate moved enough to be worth reading."""
|
|
pct = self.percent
|
|
return pct is not None and abs(pct) >= PERF_NOISE_FLOOR_PCT
|
|
|
|
|
|
@dataclass
|
|
class Comparison:
|
|
before_id: str
|
|
after_id: str
|
|
target_deltas: list[FieldDelta] = field(default_factory=list)
|
|
host_changed: FieldDelta | None = None
|
|
quality: list[MetricDelta] = field(default_factory=list)
|
|
flips: list[SampleFlip] = field(default_factory=list)
|
|
perf: dict[int, list[MetricDelta]] = field(default_factory=dict)
|
|
only_in_before: list[str] = field(default_factory=list)
|
|
only_in_after: list[str] = field(default_factory=list)
|
|
warnings: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def confounded(self) -> bool:
|
|
"""More than one axis moved, so no difference below can be attributed to any of them."""
|
|
return len(self.target_deltas) + (1 if self.host_changed else 0) > 1
|
|
|
|
|
|
def _target_field(target: dict[str, Any], name: str) -> Any:
|
|
value = target.get(name)
|
|
# serving_config is a dict; render it deterministically so a reordered YAML load does not
|
|
# read as a change.
|
|
if isinstance(value, dict):
|
|
return tuple(sorted((str(k), str(v)) for k, v in value.items()))
|
|
return value
|
|
|
|
|
|
def _diff_targets(before: dict, after: dict) -> list[FieldDelta]:
|
|
deltas = []
|
|
for axis in TARGET_AXES:
|
|
b, a = _target_field(before, axis), _target_field(after, axis)
|
|
if b != a:
|
|
deltas.append(FieldDelta(axis, b, a))
|
|
return deltas
|
|
|
|
|
|
def _sample_index(run: dict) -> dict[tuple[str, str], dict]:
|
|
"""(task, sample_id) -> outcome. Keyed on both because ids are only unique within a task."""
|
|
index = {}
|
|
for quality in run.get("quality") or []:
|
|
task = quality.get("task", "?")
|
|
for sample in quality.get("samples") or []:
|
|
index[(task, sample["sample_id"])] = sample
|
|
return index
|
|
|
|
|
|
def _perf_index(run: dict) -> dict[int, dict]:
|
|
perf = run.get("perf") or {}
|
|
return {p["concurrency"]: p for p in perf.get("points") or []}
|
|
|
|
|
|
def compare(before: dict, after: dict) -> Comparison:
|
|
"""Diff two loaded result dicts. Neither is mutated."""
|
|
result = Comparison(
|
|
before_id=before.get("run_id", "?"),
|
|
after_id=after.get("run_id", "?"),
|
|
target_deltas=_diff_targets(before.get("target") or {}, after.get("target") or {}),
|
|
)
|
|
|
|
b_host = (before.get("host") or {}).get("id") or (before.get("host") or {}).get("name")
|
|
a_host = (after.get("host") or {}).get("id") or (after.get("host") or {}).get("name")
|
|
if b_host != a_host:
|
|
result.host_changed = FieldDelta("host", b_host, a_host)
|
|
|
|
# A contaminated run has no comparable quality half, and diffing against one produces a
|
|
# delta that looks exactly like a real regression or gain. Say so before anything else.
|
|
for label, run in (("before", before), ("after", after)):
|
|
state = (run.get("verdict") or {}).get("contamination")
|
|
if state == "detected":
|
|
result.warnings.append(
|
|
f"The {label} run is CONTAMINATED — it reproduced canary probes. Its quality "
|
|
"numbers are memorisation, not capability, and no quality delta below means "
|
|
"anything."
|
|
)
|
|
elif state == "unverified":
|
|
result.warnings.append(
|
|
f"The {label} run has no contamination probe, so its quality scores are "
|
|
"unverified."
|
|
)
|
|
|
|
if not result.target_deltas and not result.host_changed:
|
|
# Same target, same box: any difference is run-to-run variance, which is worth knowing
|
|
# because it sets the floor below which no other comparison means anything.
|
|
result.warnings.append(
|
|
"Identical target and host — this measures run-to-run variance, not a change."
|
|
)
|
|
if result.confounded:
|
|
moved = [d.field for d in result.target_deltas] + (["host"] if result.host_changed else [])
|
|
result.warnings.append(
|
|
f"{len(moved)} axes changed at once ({', '.join(moved)}) — no difference below "
|
|
"can be attributed to any one of them."
|
|
)
|
|
|
|
# ---- quality -----------------------------------------------------------------
|
|
b_metrics = {q["task"]: q.get("metrics", {}) for q in before.get("quality") or []}
|
|
a_metrics = {q["task"]: q.get("metrics", {}) for q in after.get("quality") or []}
|
|
for task in sorted(set(b_metrics) | set(a_metrics)):
|
|
keys = set(b_metrics.get(task, {})) | set(a_metrics.get(task, {}))
|
|
for key in sorted(keys):
|
|
result.quality.append(
|
|
MetricDelta(
|
|
f"{task}.{key}",
|
|
b_metrics.get(task, {}).get(key),
|
|
a_metrics.get(task, {}).get(key),
|
|
)
|
|
)
|
|
|
|
b_samples, a_samples = _sample_index(before), _sample_index(after)
|
|
for key in sorted(b_samples.keys() & a_samples.keys()):
|
|
b, a = b_samples[key], a_samples[key]
|
|
if b.get("passed") == a.get("passed") and b.get("score") == a.get("score"):
|
|
continue
|
|
if b.get("passed") and not a.get("passed"):
|
|
became = "fail"
|
|
elif a.get("passed") and not b.get("passed"):
|
|
became = "pass"
|
|
else:
|
|
became = "changed"
|
|
result.flips.append(
|
|
SampleFlip(key[1], key[0], b.get("score", 0.0), a.get("score", 0.0), became)
|
|
)
|
|
# Regressions first: a capability that broke is the reason anyone runs this.
|
|
result.flips.sort(key=lambda f: ({"fail": 0, "changed": 1, "pass": 2}[f.became], f.sample_id))
|
|
|
|
# A sample present in only one run is not a flip — it is a changed eval set, which
|
|
# silently moves an aggregate. Surfaced separately so it cannot be mistaken for signal.
|
|
result.only_in_before = sorted(f"{t}/{s}" for t, s in b_samples.keys() - a_samples.keys())
|
|
result.only_in_after = sorted(f"{t}/{s}" for t, s in a_samples.keys() - b_samples.keys())
|
|
if not b_samples or not a_samples:
|
|
# Not a changed eval set — one side simply never measured quality. Saying "100 samples
|
|
# added" here would be a confusing way to report "the baseline is perf-only".
|
|
if b_samples or a_samples:
|
|
missing = "before" if not b_samples else "after"
|
|
result.warnings.append(
|
|
f"The {missing} run carries no quality results, so only serving performance "
|
|
"is comparable."
|
|
)
|
|
elif result.only_in_before or result.only_in_after:
|
|
result.warnings.append(
|
|
f"The eval set changed: {len(result.only_in_before)} sample(s) gone, "
|
|
f"{len(result.only_in_after)} added. Aggregate scores are not comparable."
|
|
)
|
|
|
|
# An EDITED sample keeps its id, so the add/remove diff above sees nothing. Rewording a
|
|
# prompt or loosening a regex changes what the score means while every id still lines up,
|
|
# which is the most dangerous shape of eval drift: the comparison looks clean. The
|
|
# fingerprint hashes (id, input, target), so it moves when the content does.
|
|
b_fp = {q["task"]: q.get("dataset_fingerprint") for q in before.get("quality") or []}
|
|
a_fp = {q["task"]: q.get("dataset_fingerprint") for q in after.get("quality") or []}
|
|
for task in sorted(b_fp.keys() & a_fp.keys()):
|
|
bf, af = b_fp[task], a_fp[task]
|
|
if bf and af and bf != af:
|
|
result.warnings.append(
|
|
f"Task {task!r} was edited between runs ({bf} -> {af}): same sample ids, "
|
|
"different content. The scores measure different questions."
|
|
)
|
|
elif not bf or not af:
|
|
result.warnings.append(
|
|
f"Task {task!r} has no dataset fingerprint on one side, so an edit to the "
|
|
"sample text cannot be ruled out. Re-run to get a comparable pair."
|
|
)
|
|
|
|
# Sampling is part of the identity of a quality result. The same model at temperature
|
|
# 1 and at 0.6 is two different measurements, and unpinned sampling moved this bench's
|
|
# score by 27% of its samples between identical runs -- far more than most real
|
|
# regressions. A card that predates sampling being recorded cannot be ruled comparable
|
|
# either, so say so rather than diff it silently.
|
|
b_s, a_s = before.get("sampling") or {}, after.get("sampling") or {}
|
|
if b_samples and a_samples:
|
|
if not b_s or not a_s:
|
|
result.warnings.append(
|
|
"One run does not record how the model was sampled, so an unpinned "
|
|
"temperature cannot be ruled out. Quality deltas below may be noise."
|
|
)
|
|
else:
|
|
changed = sorted(
|
|
k for k in set(b_s) | set(a_s) if b_s.get(k) != a_s.get(k)
|
|
)
|
|
# Epoch count is not a sampling change. Both sides estimate the same quantity;
|
|
# more epochs just estimates it better. Reporting that as "not comparable"
|
|
# overstates it, and a warning that overstates is one people learn to skip --
|
|
# which costs you the ones that matter.
|
|
decode = [k for k in changed if k != "epochs"]
|
|
if decode:
|
|
result.warnings.append(
|
|
f"Sampling changed between runs ({', '.join(decode)}): "
|
|
f"{ {k: b_s.get(k) for k in decode} } -> { {k: a_s.get(k) for k in decode} }. "
|
|
"Quality scores are not comparable across different sampling."
|
|
)
|
|
if "epochs" in changed:
|
|
b_e, a_e = b_s.get("epochs", 1), a_s.get("epochs", 1)
|
|
result.warnings.append(
|
|
f"Epochs differ ({b_e} -> {a_e}). Both estimate the same score; the "
|
|
f"{'before' if (b_e or 1) < (a_e or 1) else 'after'} run is the noisier "
|
|
"of the two, so read small deltas with that in mind."
|
|
)
|
|
|
|
# ---- perf --------------------------------------------------------------------
|
|
b_perf, a_perf = _perf_index(before), _perf_index(after)
|
|
for concurrency in sorted(b_perf.keys() & a_perf.keys()):
|
|
b, a = b_perf[concurrency], a_perf[concurrency]
|
|
result.perf[concurrency] = [
|
|
MetricDelta(name, b.get(name), a.get(name))
|
|
for name in (
|
|
"output_tps_total",
|
|
"output_tps_per_stream",
|
|
"ttft_p50_ms",
|
|
"ttft_p95_ms",
|
|
"tpot_p50_ms",
|
|
# Higher is better, so the default direction rule is already right. Diffing it
|
|
# is the point of an MTP comparison: throughput can move for many reasons,
|
|
# acceptance moves only because drafting got better or worse.
|
|
"spec_acceptance_rate",
|
|
)
|
|
if b.get(name) is not None or a.get(name) is not None
|
|
]
|
|
modes = {bool(p.get("natural_stop")) for p in list(b_perf.values()) + list(a_perf.values())}
|
|
if len(modes) > 1:
|
|
result.warnings.append(
|
|
"One run pinned output length and the other let the model stop naturally. "
|
|
"Throughput is not comparable across those two modes."
|
|
)
|
|
|
|
dropped = sorted(b_perf.keys() - a_perf.keys())
|
|
added = sorted(a_perf.keys() - b_perf.keys())
|
|
if dropped or added:
|
|
result.warnings.append(
|
|
f"Concurrency sweep differs: {dropped or 'none'} dropped, {added or 'none'} added. "
|
|
"Peak throughput is only comparable across a shared sweep."
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
# Metrics where a smaller number is the better one, so a fall is an improvement.
|
|
LOWER_IS_BETTER = ("ttft_p50_ms", "ttft_p95_ms", "tpot_p50_ms")
|
|
|
|
|
|
def direction(metric: str, delta: MetricDelta) -> str:
|
|
""""better" | "worse" | "flat" — meaning, not just sign."""
|
|
if delta.absolute is None or not delta.significant:
|
|
return "flat"
|
|
improved = delta.absolute < 0 if metric in LOWER_IS_BETTER else delta.absolute > 0
|
|
return "better" if improved else "worse"
|