Karti Tripathi
123 lines
3.3 KiB
TypeScript
123 lines
3.3 KiB
TypeScript
import fs from "node:fs";
|
|
import path from "node:path";
|
|
|
|
/**
|
|
* The leaderboard is a view over committed JSON, not a database.
|
|
*
|
|
* results/*.json is the record of every run and it lives in git, so the site
|
|
* has no state of its own and cannot drift from what was actually measured.
|
|
* Supabase enters only for submissions and auth, which are genuinely mutable.
|
|
*/
|
|
|
|
const RESULTS_DIR = path.join(process.cwd(), "..", "results");
|
|
|
|
export type SampleOutcome = {
|
|
sample_id: string;
|
|
score: number;
|
|
passed: boolean;
|
|
excerpt: string | null;
|
|
metadata: Record<string, unknown>;
|
|
};
|
|
|
|
export type QualityResult = {
|
|
task: string;
|
|
tier: "reference" | "signal" | "example";
|
|
dataset_version: string | null;
|
|
n_samples: number;
|
|
metrics: Record<string, number>;
|
|
samples: SampleOutcome[];
|
|
duration_s: number | null;
|
|
error: string | null;
|
|
};
|
|
|
|
export type PerfPoint = {
|
|
concurrency: number;
|
|
input_tokens: number;
|
|
output_tokens: number;
|
|
completed: number;
|
|
failed: number;
|
|
output_tps_total: number;
|
|
output_tps_per_stream: number;
|
|
ttft_p50_ms: number | null;
|
|
ttft_p95_ms: number | null;
|
|
tpot_p50_ms: number | null;
|
|
prefill_tps: number | null;
|
|
};
|
|
|
|
export type Run = {
|
|
run_id: string;
|
|
timestamp: string;
|
|
target_id: string;
|
|
target: {
|
|
display_name: string;
|
|
family: string | null;
|
|
params_b: number | null;
|
|
active_params_b: number | null;
|
|
quant: string | null;
|
|
checkpoint: string | null;
|
|
tier: string;
|
|
host: string;
|
|
slug: string;
|
|
notes: string | null;
|
|
serving: {
|
|
engine: string;
|
|
model_name: string;
|
|
max_model_len: number | null;
|
|
flags: Record<string, unknown>;
|
|
};
|
|
};
|
|
host: { id: string; hardware: string | null; memory_gb: number | null };
|
|
quality: QualityResult[];
|
|
perf: { engine: string; points: PerfPoint[] } | null;
|
|
verdict: {
|
|
signal_score: number | null;
|
|
reference_score: number | null;
|
|
single_stream_tps?: number;
|
|
interactive_viable?: boolean;
|
|
peak_throughput_tps?: number;
|
|
peak_throughput_concurrency?: number;
|
|
};
|
|
schema_version: number;
|
|
};
|
|
|
|
export function getRuns(): Run[] {
|
|
if (!fs.existsSync(RESULTS_DIR)) return [];
|
|
return fs
|
|
.readdirSync(RESULTS_DIR)
|
|
.filter((f) => f.endsWith(".json"))
|
|
.map((f) => JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), "utf8")) as Run)
|
|
.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
}
|
|
|
|
export function getRun(id: string): Run | undefined {
|
|
return getRuns().find((r) => r.run_id.startsWith(id));
|
|
}
|
|
|
|
/**
|
|
* Latest run per target — what the leaderboard ranks.
|
|
*
|
|
* Ranking every run would let a target dominate the board simply by being
|
|
* measured more often.
|
|
*/
|
|
export function latestPerTarget(runs: Run[]): Run[] {
|
|
const seen = new Map<string, Run>();
|
|
for (const run of runs) {
|
|
if (!seen.has(run.target_id)) seen.set(run.target_id, run);
|
|
}
|
|
return [...seen.values()];
|
|
}
|
|
|
|
/** Was any task in this run capped with --limit? Must never be hidden. */
|
|
export function limitOf(run: Run): number | null {
|
|
for (const q of run.quality) {
|
|
if (q.metrics?._limit) return q.metrics._limit;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function fmtParams(t: Run["target"]): string {
|
|
if (!t.params_b) return "—";
|
|
const base = `${t.params_b}B`;
|
|
return t.active_params_b ? `${base}·${t.active_params_b}B active` : base;
|
|
}
|