This commit is contained in:
Karti Tripathi
+495
@@ -0,0 +1,495 @@
|
||||
//! Reproducible evaluations for any OpenAI-compatible model server.
|
||||
//!
|
||||
//! Suites are declarative YAML. Results are append-only JSON artifacts suitable
|
||||
//! for CI, regression comparisons, and future publication to an eval registry.
|
||||
|
||||
use anyhow::{bail, Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use std::fs;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Suite {
|
||||
#[serde(rename = "apiVersion")]
|
||||
api_version: String,
|
||||
kind: String,
|
||||
metadata: Metadata,
|
||||
#[serde(default)]
|
||||
defaults: Defaults,
|
||||
cases: Vec<Case>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Metadata {
|
||||
name: String,
|
||||
#[serde(default)]
|
||||
version: u32,
|
||||
#[serde(default)]
|
||||
description: String,
|
||||
/// Part of the published suite manifest surface; parsed so a suite
|
||||
/// carrying tags loads, not read by the runner itself.
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)]
|
||||
tags: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct Defaults {
|
||||
#[serde(default = "default_max_tokens")]
|
||||
max_tokens: u32,
|
||||
#[serde(default = "default_temperature")]
|
||||
temperature: f64,
|
||||
#[serde(default = "default_repeat")]
|
||||
repeat: u32,
|
||||
#[serde(default)]
|
||||
system: String,
|
||||
}
|
||||
|
||||
fn default_max_tokens() -> u32 {
|
||||
128
|
||||
}
|
||||
fn default_temperature() -> f64 {
|
||||
0.0
|
||||
}
|
||||
fn default_repeat() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Case {
|
||||
id: String,
|
||||
#[serde(default)]
|
||||
category: String,
|
||||
prompt: String,
|
||||
#[serde(default)]
|
||||
max_tokens: Option<u32>,
|
||||
#[serde(default)]
|
||||
assertions: Vec<Assertion>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
enum Assertion {
|
||||
Exact { value: String },
|
||||
Contains { value: String },
|
||||
ContainsAny { values: Vec<String> },
|
||||
NotContains { value: String },
|
||||
MaxWords { value: usize },
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Artifact {
|
||||
schema_version: u32,
|
||||
run_id: String,
|
||||
suite: String,
|
||||
suite_version: u32,
|
||||
model: String,
|
||||
base_url: String,
|
||||
started_unix_ms: u128,
|
||||
summary: Summary,
|
||||
samples: Vec<Sample>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Summary {
|
||||
passed: usize,
|
||||
total: usize,
|
||||
score: f64,
|
||||
mean_ttft_ms: f64,
|
||||
p50_ttft_ms: f64,
|
||||
p95_ttft_ms: f64,
|
||||
mean_prefill_tps: f64,
|
||||
mean_decode_tps: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Sample {
|
||||
case_id: String,
|
||||
category: String,
|
||||
repetition: u32,
|
||||
passed: bool,
|
||||
assertion_results: Vec<bool>,
|
||||
output: String,
|
||||
reasoning: String,
|
||||
prompt_tokens: u64,
|
||||
completion_tokens: u64,
|
||||
ttft_ms: f64,
|
||||
total_ms: f64,
|
||||
prefill_tps: f64,
|
||||
decode_tps: f64,
|
||||
}
|
||||
|
||||
struct Completion {
|
||||
output: String,
|
||||
reasoning: String,
|
||||
prompt_tokens: u64,
|
||||
completion_tokens: u64,
|
||||
ttft_ms: f64,
|
||||
total_ms: f64,
|
||||
}
|
||||
|
||||
/// One line of `eval ls`. Suites themselves stay private — a caller has no
|
||||
/// business reaching into cases and assertions — but the catalogue is the
|
||||
/// useful part and is shared by the CLI and the MCP server.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SuiteSummary {
|
||||
pub name: String,
|
||||
pub version: u32,
|
||||
pub cases: usize,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
pub fn catalogue(root: &Path) -> Result<Vec<SuiteSummary>> {
|
||||
Ok(load_all(root)?
|
||||
.into_iter()
|
||||
.map(|s| SuiteSummary {
|
||||
name: s.metadata.name,
|
||||
version: s.metadata.version,
|
||||
cases: s.cases.len(),
|
||||
description: s.metadata.description,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn list(root: &Path) -> Result<()> {
|
||||
println!("{:<20} {:>4} {:>5} DESCRIPTION", "SUITE", "VER", "CASES");
|
||||
for s in catalogue(root)? {
|
||||
println!(
|
||||
"{:<20} {:>4} {:>5} {}",
|
||||
s.name, s.version, s.cases, s.description
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn run(
|
||||
root: &Path,
|
||||
name: &str,
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
repeat: Option<u32>,
|
||||
) -> Result<()> {
|
||||
let suite = load(root, name)?;
|
||||
// `kuda/v1` is the pre-rename contract; still accepted so an older suite keeps running.
|
||||
let known_contract = matches!(suite.api_version.as_str(), "lumbridge/v1" | "kuda/v1");
|
||||
if !known_contract || suite.kind != "EvalSuite" {
|
||||
bail!(
|
||||
"unsupported eval contract: {}/{}",
|
||||
suite.api_version,
|
||||
suite.kind
|
||||
);
|
||||
}
|
||||
let reps = repeat.unwrap_or(suite.defaults.repeat).max(1);
|
||||
println!(
|
||||
"eval '{}' v{} → {} ({})",
|
||||
suite.metadata.name, suite.metadata.version, model, base_url
|
||||
);
|
||||
let started = now_ms();
|
||||
let mut samples = Vec::new();
|
||||
for case in &suite.cases {
|
||||
for repetition in 1..=reps {
|
||||
print!(" {:<24} [{}/{}] ", case.id, repetition, reps);
|
||||
std::io::stdout().flush().ok();
|
||||
let c = stream_completion(base_url, model, &suite.defaults, case)?;
|
||||
let assertion_results: Vec<bool> = case
|
||||
.assertions
|
||||
.iter()
|
||||
.map(|a| evaluate(a, &c.output))
|
||||
.collect();
|
||||
let passed = assertion_results.iter().all(|v| *v);
|
||||
let decode_seconds = ((c.total_ms - c.ttft_ms) / 1000.0).max(0.001);
|
||||
let prefill_seconds = (c.ttft_ms / 1000.0).max(0.001);
|
||||
let sample = Sample {
|
||||
case_id: case.id.clone(),
|
||||
category: case.category.clone(),
|
||||
repetition,
|
||||
passed,
|
||||
assertion_results,
|
||||
output: c.output,
|
||||
reasoning: c.reasoning,
|
||||
prompt_tokens: c.prompt_tokens,
|
||||
completion_tokens: c.completion_tokens,
|
||||
ttft_ms: c.ttft_ms,
|
||||
total_ms: c.total_ms,
|
||||
prefill_tps: c.prompt_tokens as f64 / prefill_seconds,
|
||||
decode_tps: c.completion_tokens as f64 / decode_seconds,
|
||||
};
|
||||
println!(
|
||||
"{} ttft={:.0}ms decode={:.1}tok/s",
|
||||
if passed { "PASS" } else { "FAIL" },
|
||||
sample.ttft_ms,
|
||||
sample.decode_tps
|
||||
);
|
||||
samples.push(sample);
|
||||
}
|
||||
}
|
||||
let summary = summarize(&samples);
|
||||
let run_id = format!("{}-{}-{}", started, slug(name), slug(model));
|
||||
let artifact = Artifact {
|
||||
schema_version: 1,
|
||||
run_id: run_id.clone(),
|
||||
suite: suite.metadata.name,
|
||||
suite_version: suite.metadata.version,
|
||||
model: model.to_string(),
|
||||
base_url: base_url.to_string(),
|
||||
started_unix_ms: started,
|
||||
summary,
|
||||
samples,
|
||||
};
|
||||
let out_dir = root.join("eval-results");
|
||||
fs::create_dir_all(&out_dir)?;
|
||||
let out = out_dir.join(format!("{run_id}.json"));
|
||||
fs::write(&out, serde_json::to_string_pretty(&artifact)?)?;
|
||||
println!(
|
||||
"\nscore {:.1}% ({}/{}) · mean TTFT {:.0}ms · prefill≈{:.1}tok/s · decode {:.1}tok/s",
|
||||
artifact.summary.score * 100.0,
|
||||
artifact.summary.passed,
|
||||
artifact.summary.total,
|
||||
artifact.summary.mean_ttft_ms,
|
||||
artifact.summary.mean_prefill_tps,
|
||||
artifact.summary.mean_decode_tps
|
||||
);
|
||||
println!("result {}", out.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stream_completion(
|
||||
base_url: &str,
|
||||
model: &str,
|
||||
defaults: &Defaults,
|
||||
case: &Case,
|
||||
) -> Result<Completion> {
|
||||
let url = format!("{}/chat/completions", base_url.trim_end_matches('/'));
|
||||
let mut messages = Vec::new();
|
||||
if !defaults.system.is_empty() {
|
||||
messages.push(json!({"role":"system","content":defaults.system}));
|
||||
}
|
||||
messages.push(json!({"role":"user","content":case.prompt}));
|
||||
let request = json!({
|
||||
"model": model, "messages": messages, "stream": true,
|
||||
"stream_options": {"include_usage": true},
|
||||
"max_tokens": case.max_tokens.unwrap_or(defaults.max_tokens),
|
||||
"temperature": defaults.temperature,
|
||||
"chat_template_kwargs": {"enable_thinking": false}
|
||||
});
|
||||
let mut child = Command::new("curl")
|
||||
.args([
|
||||
"-sS",
|
||||
"-N",
|
||||
"-X",
|
||||
"POST",
|
||||
&url,
|
||||
"-H",
|
||||
"Content-Type: application/json",
|
||||
"--data-binary",
|
||||
"@-",
|
||||
])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.context("starting curl (required for eval HTTP streaming)")?;
|
||||
child
|
||||
.stdin
|
||||
.take()
|
||||
.unwrap()
|
||||
.write_all(request.to_string().as_bytes())?;
|
||||
let start = Instant::now();
|
||||
let mut first = None;
|
||||
let mut output = String::new();
|
||||
let mut reasoning = String::new();
|
||||
let mut prompt_tokens = 0;
|
||||
let mut completion_tokens = 0;
|
||||
for line in BufReader::new(child.stdout.take().unwrap()).lines() {
|
||||
let line = line?;
|
||||
let Some(data) = line.strip_prefix("data: ") else {
|
||||
continue;
|
||||
};
|
||||
if data == "[DONE]" {
|
||||
break;
|
||||
}
|
||||
let v: Value = serde_json::from_str(data).context("parsing streamed completion")?;
|
||||
if let Some(usage) = v.get("usage") {
|
||||
prompt_tokens = usage
|
||||
.get("prompt_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(prompt_tokens);
|
||||
completion_tokens = usage
|
||||
.get("completion_tokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(completion_tokens);
|
||||
}
|
||||
let delta = &v["choices"][0]["delta"];
|
||||
let content = delta.get("content").and_then(Value::as_str).unwrap_or("");
|
||||
let thought = delta
|
||||
.get("reasoning")
|
||||
.or_else(|| delta.get("reasoning_content"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("");
|
||||
if first.is_none() && (!content.is_empty() || !thought.is_empty()) {
|
||||
first = Some(start.elapsed());
|
||||
}
|
||||
output.push_str(content);
|
||||
reasoning.push_str(thought);
|
||||
}
|
||||
let status = child.wait()?;
|
||||
if !status.success() {
|
||||
bail!("completion request failed with {status}");
|
||||
}
|
||||
let total = start.elapsed().as_secs_f64() * 1000.0;
|
||||
Ok(Completion {
|
||||
output,
|
||||
reasoning,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
ttft_ms: first.map(|d| d.as_secs_f64() * 1000.0).unwrap_or(total),
|
||||
total_ms: total,
|
||||
})
|
||||
}
|
||||
|
||||
fn evaluate(a: &Assertion, output: &str) -> bool {
|
||||
let normalized = output.trim();
|
||||
match a {
|
||||
Assertion::Exact { value } => normalized.eq_ignore_ascii_case(value.trim()),
|
||||
Assertion::Contains { value } => normalized.to_lowercase().contains(&value.to_lowercase()),
|
||||
Assertion::ContainsAny { values } => values
|
||||
.iter()
|
||||
.any(|v| normalized.to_lowercase().contains(&v.to_lowercase())),
|
||||
Assertion::NotContains { value } => {
|
||||
!normalized.to_lowercase().contains(&value.to_lowercase())
|
||||
}
|
||||
Assertion::MaxWords { value } => normalized.split_whitespace().count() <= *value,
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize(samples: &[Sample]) -> Summary {
|
||||
let mut ttft: Vec<f64> = samples.iter().map(|s| s.ttft_ms).collect();
|
||||
ttft.sort_by(f64::total_cmp);
|
||||
let mean = |f: fn(&Sample) -> f64| {
|
||||
if samples.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
samples.iter().map(f).sum::<f64>() / samples.len() as f64
|
||||
}
|
||||
};
|
||||
let percentile = |p: f64| {
|
||||
if ttft.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
ttft[((ttft.len() - 1) as f64 * p).round() as usize]
|
||||
}
|
||||
};
|
||||
let passed = samples.iter().filter(|s| s.passed).count();
|
||||
Summary {
|
||||
passed,
|
||||
total: samples.len(),
|
||||
score: if samples.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
passed as f64 / samples.len() as f64
|
||||
},
|
||||
mean_ttft_ms: mean(|s| s.ttft_ms),
|
||||
p50_ttft_ms: percentile(0.50),
|
||||
p95_ttft_ms: percentile(0.95),
|
||||
mean_prefill_tps: mean(|s| s.prefill_tps),
|
||||
mean_decode_tps: mean(|s| s.decode_tps),
|
||||
}
|
||||
}
|
||||
|
||||
fn load(root: &Path, name: &str) -> Result<Suite> {
|
||||
load_all(root)?
|
||||
.into_iter()
|
||||
.find(|s| s.metadata.name == name)
|
||||
.with_context(|| format!("no eval suite named '{name}' in {}/evals", root.display()))
|
||||
}
|
||||
|
||||
fn load_all(root: &Path) -> Result<Vec<Suite>> {
|
||||
let dir = root.join("evals");
|
||||
let mut suites = Vec::new();
|
||||
if !dir.exists() {
|
||||
return Ok(suites);
|
||||
}
|
||||
for entry in fs::read_dir(&dir)? {
|
||||
let path: PathBuf = entry?.path();
|
||||
if path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(|n| n.ends_with(".eval.yaml"))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
let suite: Suite = serde_yaml::from_str(&fs::read_to_string(&path)?)
|
||||
.with_context(|| format!("parsing {}", path.display()))?;
|
||||
suites.push(suite);
|
||||
}
|
||||
}
|
||||
suites.sort_by(|a, b| a.metadata.name.cmp(&b.metadata.name));
|
||||
Ok(suites)
|
||||
}
|
||||
|
||||
fn now_ms() -> u128 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis()
|
||||
}
|
||||
fn slug(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphanumeric() {
|
||||
c.to_ascii_lowercase()
|
||||
} else {
|
||||
'-'
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn assertions_are_case_insensitive_and_composable() {
|
||||
let output = "Market risk remains, while company-specific risk falls.";
|
||||
assert!(evaluate(
|
||||
&Assertion::Contains {
|
||||
value: "MARKET RISK".into()
|
||||
},
|
||||
output
|
||||
));
|
||||
assert!(evaluate(
|
||||
&Assertion::ContainsAny {
|
||||
values: vec!["idiosyncratic".into(), "company-specific".into()],
|
||||
},
|
||||
output
|
||||
));
|
||||
assert!(evaluate(
|
||||
&Assertion::NotContains {
|
||||
value: "markdown".into()
|
||||
},
|
||||
output
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_trims_but_does_not_accept_explanation() {
|
||||
assert!(evaluate(&Assertion::Exact { value: "25".into() }, " 25\n"));
|
||||
assert!(!evaluate(&Assertion::Exact { value: "25".into() }, "25 GB"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn word_limit_counts_whitespace_tokens() {
|
||||
assert!(evaluate(
|
||||
&Assertion::MaxWords { value: 4 },
|
||||
"one two three four"
|
||||
));
|
||||
assert!(!evaluate(
|
||||
&Assertion::MaxWords { value: 3 },
|
||||
"one two three four"
|
||||
));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user