From 006feee0f7497e6be0d11dd97f894c4797496922 Mon Sep 17 00:00:00 2001 From: Karti Tripathi Date: Tue, 4 Aug 2026 00:44:07 -0700 Subject: [PATCH] Lumbridge Bench --- .gitea/workflows/ci.yml | 102 + .gitignore | 42 + AGENTS.md | 80 + LICENSE | 202 ++ Makefile | 37 + NOTICE | 16 + README.md | 106 + data/README.md | 104 + data/public/example.jsonl | 4 + deploy/bench-karti.service | 25 + deploy/lumbridge-bench.service | 25 + docs/DECISIONS.md | 200 ++ docs/DEPLOY.md | 106 + kbench/__init__.py | 0 kbench/cli.py | 646 ++++ kbench/compare.py | 335 ++ kbench/perf.py | 319 ++ kbench/registry/__init__.py | 167 + kbench/registry/models.yaml | 109 + kbench/results.py | 254 ++ kbench/run.py | 358 ++ kbench/schema.py | 62 + kbench/scorers/__init__.py | 95 + kbench/specdec.py | 106 + kbench/sweep.py | 149 + kbench/tasks/__init__.py | 132 + kbench/tasks/canary.py | 109 + kbench/tasks/signal.py | 121 + pyproject.toml | 51 + results/.gitkeep | 0 ...n3.6-35b-a3b-nvfp4__spark-1__6f89eb66.json | 119 + ...n3.6-35b-a3b-nvfp4__spark-1__b5a9f595.json | 1644 +++++++++ scripts/.gitkeep | 0 site/.env.example | 6 + site/app/api/card/[id]/route.tsx | 171 + site/app/api/health/route.ts | 13 + site/app/api/submit/route.ts | 73 + site/app/globals.css | 166 + site/app/icon.svg | 7 + site/app/layout.tsx | 88 + site/app/method/page.tsx | 142 + site/app/page.tsx | 280 ++ site/app/run/[id]/page.tsx | 219 ++ site/app/submit/page.tsx | 155 + site/assets/fonts/IBMPlexMono-Regular.ttf | Bin 0 -> 135580 bytes site/assets/fonts/IBMPlexMono-SemiBold.ttf | Bin 0 -> 140216 bytes site/assets/fonts/InstrumentSerif-Regular.ttf | Bin 0 -> 70012 bytes site/components/ScopeTrace.tsx | 134 + site/lib/model-ref.ts | 12 + site/lib/results.ts | 122 + site/next-env.d.ts | 6 + site/next.config.mjs | 17 + site/package-lock.json | 1676 +++++++++ site/package.json | 29 + site/postcss.config.mjs | 5 + site/tsconfig.json | 33 + supabase/migrations/0001_bench_schema.sql | 97 + .../migrations/0002_private_suggestions.sql | 5 + tests/test_catalog.py | 198 ++ tests/test_compare.py | 356 ++ tests/test_data.py | 219 ++ tests/test_specdec.py | 102 + tests/test_sweep.py | 135 + tests/test_verdict.py | 158 + uv.lock | 3067 +++++++++++++++++ 65 files changed, 13516 insertions(+) create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 NOTICE create mode 100644 README.md create mode 100644 data/README.md create mode 100644 data/public/example.jsonl create mode 100644 deploy/bench-karti.service create mode 100644 deploy/lumbridge-bench.service create mode 100644 docs/DECISIONS.md create mode 100644 docs/DEPLOY.md create mode 100644 kbench/__init__.py create mode 100644 kbench/cli.py create mode 100644 kbench/compare.py create mode 100644 kbench/perf.py create mode 100644 kbench/registry/__init__.py create mode 100644 kbench/registry/models.yaml create mode 100644 kbench/results.py create mode 100644 kbench/run.py create mode 100644 kbench/schema.py create mode 100644 kbench/scorers/__init__.py create mode 100644 kbench/specdec.py create mode 100644 kbench/sweep.py create mode 100644 kbench/tasks/__init__.py create mode 100644 kbench/tasks/canary.py create mode 100644 kbench/tasks/signal.py create mode 100644 pyproject.toml create mode 100644 results/.gitkeep create mode 100644 results/2026-07-28__qwen3.6-35b-a3b-nvfp4__spark-1__6f89eb66.json create mode 100644 results/2026-07-28__qwen3.6-35b-a3b-nvfp4__spark-1__b5a9f595.json create mode 100644 scripts/.gitkeep create mode 100644 site/.env.example create mode 100644 site/app/api/card/[id]/route.tsx create mode 100644 site/app/api/health/route.ts create mode 100644 site/app/api/submit/route.ts create mode 100644 site/app/globals.css create mode 100644 site/app/icon.svg create mode 100644 site/app/layout.tsx create mode 100644 site/app/method/page.tsx create mode 100644 site/app/page.tsx create mode 100644 site/app/run/[id]/page.tsx create mode 100644 site/app/submit/page.tsx create mode 100644 site/assets/fonts/IBMPlexMono-Regular.ttf create mode 100644 site/assets/fonts/IBMPlexMono-SemiBold.ttf create mode 100644 site/assets/fonts/InstrumentSerif-Regular.ttf create mode 100644 site/components/ScopeTrace.tsx create mode 100644 site/lib/model-ref.ts create mode 100644 site/lib/results.ts create mode 100644 site/next-env.d.ts create mode 100644 site/next.config.mjs create mode 100644 site/package-lock.json create mode 100644 site/package.json create mode 100644 site/postcss.config.mjs create mode 100644 site/tsconfig.json create mode 100644 supabase/migrations/0001_bench_schema.sql create mode 100644 supabase/migrations/0002_private_suggestions.sql create mode 100644 tests/test_catalog.py create mode 100644 tests/test_compare.py create mode 100644 tests/test_data.py create mode 100644 tests/test_specdec.py create mode 100644 tests/test_sweep.py create mode 100644 tests/test_verdict.py create mode 100644 uv.lock diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..0139f1d --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,102 @@ +# CI/CD — verify on every push, ship main when green. +# +# Runs on the act_runner registered on build-host (label `ubuntu-latest`, Docker-backed). +# Same shape as music.karti.ai: verify and deploy are two jobs in ONE workflow joined by +# `needs:`, because Gitea's cross-workflow triggers are less reliable than GitHub's. A red +# build cannot deploy. +# +# Requires one repo secret: CLOUD2_SSH_KEY — the private half of a deploy key whose public +# half is in ubuntu@web-host's authorized_keys. + +name: CI + +on: + push: + branches: ["**"] + pull_request: + branches: [main] + workflow_dispatch: + +env: + CLOUD2_HOST: 100.92.185.76 + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Harness imports and registry parses + # Catches a malformed models.yaml or a broken target before it reaches + # the site build, where the failure would be far less legible. + run: | + python3 -m pip install --quiet pyyaml + python3 - <<'PY' + import sys; sys.path.insert(0, '.') + from kbench.registry import load_registry + reg = load_registry() + for t in reg.targets.values(): + t.inspect_model # raises on an unserveable target + print(f"registry ok: {len(reg.targets)} targets, {len(reg.hosts)} hosts") + PY + + - name: Results files are valid and schema-consistent + run: | + python3 - <<'PY' + import json, pathlib, sys + bad = 0 + for p in sorted(pathlib.Path('results').glob('*.json')): + d = json.loads(p.read_text()) + for field in ('run_id', 'timestamp', 'target_id', 'target', 'schema_version'): + if field not in d: + print(f"::error::{p.name} missing {field}"); bad += 1 + print(f"checked results files, {bad} problem(s)") + sys.exit(1 if bad else 0) + PY + + - name: Build site + working-directory: site + run: | + npm install --no-audit --no-fund + npm run build + + deploy: + needs: verify + # Gitea populates the **github** context, not a `gitea` one — `gitea.ref` silently + # evaluates to nothing and the job is skipped with no error. + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Set up SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.CLOUD2_SSH_KEY }}" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + ssh-keyscan -H "$CLOUD2_HOST" >> ~/.ssh/known_hosts 2>/dev/null + + - name: Deploy to web-host + run: | + ssh -i ~/.ssh/deploy_key -o BatchMode=yes ubuntu@"$CLOUD2_HOST" bash -s <<'REMOTE' + set -euo pipefail + cd /home/ubuntu/workspace/bench.karti.ai + git fetch origin -q + git reset --hard origin/main -q + cd site + npm install --no-audit --no-fund + npm run build + sudo systemctl restart bench-karti + REMOTE + + - name: Verify the deploy + run: | + for i in $(seq 1 20); do + if curl -sf https://bench.karti.ai/api/health | grep -q '"ok":true'; then + echo "✓ bench.karti.ai is healthy" + exit 0 + fi + sleep 3 + done + echo "::error::bench.karti.ai did not come back healthy after deploy" + ssh -i ~/.ssh/deploy_key -o BatchMode=yes ubuntu@"$CLOUD2_HOST" \ + 'sudo journalctl -u bench-karti -n 40 --no-pager' || true + exit 1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1c2bd3b --- /dev/null +++ b/.gitignore @@ -0,0 +1,42 @@ +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +.env +.env.local + +# Raw inspect logs are large and machine-specific. We extract the parts we +# care about (per-sample outcomes) into results/*.json, which IS committed. +logs/ +*.eval + +# Website +node_modules/ +.next/ +out/ + +# NOTE: data/private/ is intentionally NOT ignored. This repo is private on +# gitea; the private eval set is the whole point and must be versioned. If +# this repo is ever made public, data/private/ must be split out FIRST. +# See docs/DECISIONS.md#d3. + +# Playwright MCP scratch output +.playwright-mcp/ + +# Secrets. +.env +.env.* +!.env.example +*.pem +*.key +*_rsa +*_ed25519 +id_ed25519* +.ssh/ +*secret* +*credentials* + +# Private eval data never belongs in the public repo. Publishing the signal set +# contaminates it; publishing the canary inverts it. See scripts/publish-bench.sh. +data/private/ +data/canary/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b7cb74f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,80 @@ +# Working guide — Lumbridge Bench + +Read `docs/DECISIONS.md` before changing anything structural. It records why +the shape is the shape; the code does not. + +## What this repo is + +A Lumbridge evidence lab that answers "should we self-host this model?" — quality on our real +work, plus serving performance on our own boxes, in one score card per +`(model × host × quant × serving config × checkpoint)` target. + +Built **on** Inspect AI as a dependency. It is not a fork and must not become +one (D1). Adding a benchmark means adding a task, not patching the harness. + +## Ground rules + +1. **Never commit anything to `data/private/` that was not scrubbed.** No API + keys, tokens, tailscale hostnames/IPs, customer data, or personal + information. Samples derived from real tool calls get sanitised at + authoring time. The content will be sent to third-party APIs (judges, + baselines) regardless of how private this repo is. + +2. **Never publish or export `data/private/`.** Not in the site build, not in + an artifact, not in a share card, not in a bug report. A public test set is + a dead test set (D3). + +3. **Never present a capped run as a full run.** `--limit` is recorded in the + result JSON and rendered on the card. Keep it that way. + +4. **Do not change `results/` schema fields in place.** Old result files must + stay readable. Bump `SCHEMA_VERSION` and add fields; never repurpose one. + +5. **Results are only comparable across identical serving flags.** Before + recording a run, confirm the target's flags with `kbench snapshot `. + If they drifted, update `models.yaml` first — a result carries its own + snapshot, so a stale registry silently produces incomparable numbers. + +## Adding a reference task + +Reference tasks come from `inspect_evals` unmodified. Add an entry to +`CATALOG` in `kbench/tasks/__init__.py` with its `primary_metric` (the headline +key in the scorer's metrics) and, if sample scores are dict-valued, +`primary_sample_key`. Do not modify the upstream task. + +## Adding a signal task + +Signal tasks are ours. Samples live in `data/private/.jsonl`, one JSON +object per line. See `data/README.md` for the format. Use `kbench add` rather +than hand-editing so the canary tag and split assignment are applied. + +Assign `split` at authoring time and never move a sample between splits — +`test` samples must never be seen during fine-tuning, or every number after +that point is a lie you cannot detect. + +## Perf measurement + +`kbench/perf.py` deliberately defeats prefix caching (unique nonce per +request) and fixes output length (`ignore_eos`). If you touch it, preserve +both — without them the numbers look better and mean nothing (D6). + +Concurrency-1 numbers describe what a single user feels. Aggregate throughput +at higher concurrency describes what the box can serve. Report both; they can +differ by more than 20x on the Spark. + +## Environment + +- Runner: EQ (this box). It only needs HTTP to the serving host. +- `your-node` serves vLLM on `:8001`, OpenAI-compatible. +- Inspect addresses it as `openai-api/spark/` with `SPARK_BASE_URL` / + `SPARK_API_KEY`, both derived from the registry — do not hardcode them. + +## Deploy + +The standalone site is **retired** (2026-08-01): `bench.karti.ai` permanently redirects to +`lumbridgecorp.com/bench`, and `bench-karti` on web-host :8909 is stopped and disabled. Bench ships as +part of Lumbridge, not as its own destination — see [docs/DEPLOY.md](docs/DEPLOY.md) before standing +any UI back up. Supabase stores auth and the private manual-suggestion inbox in schema `bench`; it is +not an execution queue. The `kbench` CLI runner is unaffected. + +Public model suggestions never execute code. The suggestion endpoint may record a reference for owner review, but it must never download, import, schedule, or run a model. D10 supersedes D8 for product behavior; every run begins with explicit owner approval. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..12834b1 --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +.PHONY: install card card-full perf example view results lint clean + +VENV := .venv/bin +TARGET ?= brain + +install: + uv venv --python 3.12 + uv pip install -e ".[reference]" + +## Quick score card: capped IFEval + perf sweep. The cap is recorded in the +## result and rendered on the card -- never present this as a full run. +card: + $(VENV)/kbench run $(TARGET) --tasks ifeval --limit 100 --max-connections 16 + +## Full reference run. Slow -- 541 samples against a reasoning model that emits +## ~3k tokens per answer. Run it overnight. +card-full: + $(VENV)/kbench run $(TARGET) --tasks ifeval --max-connections 16 + +perf: + $(VENV)/kbench perf $(TARGET) + +## Exercises the signal-task machinery against the public examples. +example: + $(VENV)/kbench run $(TARGET) --tasks example --skip-perf + +view: + $(VENV)/inspect view --log-dir logs + +results: + $(VENV)/kbench results + +lint: + $(VENV)/python -m ruff check kbench/ || true + +clean: + rm -rf logs/*.eval diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..39693df --- /dev/null +++ b/NOTICE @@ -0,0 +1,16 @@ +Lumbridge +Copyright 2026 Karti Tripathi + +This product includes software developed by Karti Tripathi. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4ddc1a2 --- /dev/null +++ b/README.md @@ -0,0 +1,106 @@ +# Lumbridge Bench + +Private evaluation and serving-performance evidence for models running on Lumbridge Compute. + +Two questions, one score card: + +1. **Is this model good enough** on the work we actually do? +2. **Is it fast enough** on our own hardware to put in front of that work? + +Most eval frameworks answer only the first. A self-hosting decision needs +both, so every run produces quality scores *and* a serving perf profile for +the same `(model × host × quant × serving config)` target. + +## Place in Lumbridge + +Lumbridge Compute owns nodes, model artifacts, Scenes, and workload state. Lumbridge Bench owns measurement truth: versioned Evals, exact target snapshots, immutable runs, comparisons, and selected public score cards. The `kbench` command remains the compatible runner name. + +## Why this exists + +`your-node` currently serves `Qwen3.6-35B-A3B-NVFP4` behind the `brain` and +`local-moe` aliases — in production, with nothing measuring whether +that was the right call. This bench exists to answer that, and to be the eval +half of the loop when we fine-tune our own model. + +## Install + +```bash +uv venv --python 3.12 +uv pip install -e ".[reference]" +``` + +## Use + +```bash +kbench targets # what can be benchmarked +kbench tasks # what it gets benchmarked on +kbench snapshot your-node # live serving flags, for models.yaml + +kbench perf brain # perf sweep only (no task authoring needed) +kbench run brain --limit 50 # quality + perf -> results/*.json +kbench results # every score card recorded so far +kbench card results/ # re-render one +``` + +Raw Inspect logs land in `logs/` (gitignored). View them with: + +```bash +.venv/bin/inspect view --log-dir logs +``` + +## Layout + +``` +kbench/ + registry/ targets: (model × host × quant × serving config × checkpoint) + tasks/ task catalog — reference tier and signal tier + scorers/ custom + model-graded scorers + perf.py async load generator, any OpenAI-compatible endpoint + results.py the committed result schema + run.py orchestration: target + tasks + perf -> score card +data/ + public/ a few example samples, shown on the site + private/ the real holdout — never leaves this repo + canary/ GUID-tagged samples for proving contamination later +results/ one JSON per run, committed. the git history IS the record. +docs/ DECISIONS.md — why any of this is shaped the way it is +``` + +## Two tiers of task + +| tier | source | purpose | +|---|---|---| +| **reference** | `inspect_evals`, unmodified | calibration + smoke test. Public, likely contaminated. Not a ranking. | +| **signal** | ours, private | the actual measurement | + +If a reference score lands far from published values, the harness is broken, +not the model. That is what it is for. + +## Rules that are not negotiable + +- **The private eval set never becomes public.** Not a secrets concern + primarily — a public test set gets scraped into the next training corpus and + stops measuring anything. See `docs/DECISIONS.md#d3`. +- **Scrub samples at authoring time.** No keys, tokens, tailscale addresses, + or customer data. Repo privacy is defence in depth, not the control. +- **Never publish a capped run as a full run.** `--limit` is recorded in the + result and surfaced on the card. +- **Results are only comparable across identical serving flags.** Use + `kbench snapshot` rather than trusting memory. + +## Manual model suggestions + +The public site accepts suggestions, not executable submissions. A suggestion records a Hugging Face reference and optional notes in the Lumbridge control plane for Karti to review. Nothing downloads or runs automatically; Karti verifies the model card, license, files, compatibility, footprint, and serving path before manually scheduling any run. + +Built on [Inspect AI](https://github.com/UKGovernmentBEIS/inspect_ai) as a +dependency, not a fork — see `docs/DECISIONS.md#d1`. + +## License + +Apache-2.0 — see [LICENSE](LICENSE) and [NOTICE](NOTICE). + +The private signal set, the contamination canary, and the sample seeder are **not** +published. Publishing an eval destroys it: the signal set is worthless once it has been +crawled, and a published canary GUID is reproduced by every model that ever read the +repo, so the detector would report contamination for everyone forever. `kbench` ships +with the public example family; you author your own private families locally. diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..f064a4f --- /dev/null +++ b/data/README.md @@ -0,0 +1,104 @@ +# Eval data + +Three tiers. The split is load-bearing — see `docs/DECISIONS.md#d3`. + +| dir | contents | visibility | +|---|---|---| +| `public/` | a few example samples | shown on the site, safe to leak | +| `private/` | the real holdout | **never leaves this repo** | +| `canary/` | contamination probes (carrier + detectors) | never leaves this repo | + +## Sample format + +One JSON object per line (JSONL), one file per family: +`private/agent_ops.jsonl`, `private/repo_edit.jsonl`, ... + +```json +{ + "id": "agent_ops/0001", + "family": "agent_ops", + "split": "test", + "canary": "KBENCH-CANARY-3f9a1c2e", + "input": "The prompt sent to the model.", + "target": "The expected answer, or null for rubric-graded samples.", + "scorer": "includes", + "rubric": null, + "metadata": { + "source": "openclaw", + "authored": "2026-07-27", + "tags": ["tool-use", "multi-turn"] + } +} +``` + +### Fields + +- **`id`** — stable and never reused. Per-sample regression tracking across + checkpoints keys on this; renumbering breaks every historical comparison. +- **`split`** — `train` | `dev` | `test`. Assigned at authoring time and + **never changed**. A `test` sample that leaks into fine-tuning makes every + subsequent number a lie you have no way to detect. +- **`canary`** — provenance only, stamped by `kbench add`. It is **not** the + contamination control: it never reaches a model, so nothing can reproduce it. + The real control is the `canary/` tier — see below. +- **`scorer`** — how this sample is graded: + + | scorer | grading | judge | + |---|---|---| + | `exact` | output matches target exactly | no | + | `includes` | output contains target | no | + | `regex` | output matches target as a regex | no | + | `rubric` | model-graded against `rubric` text | **yes** | + + Prefer deterministic scorers. Every rubric sample adds judge cost per run + and judge drift between runs, which confounds the checkpoint comparisons + this bench exists to make. Reach for `rubric` only when the property is + genuinely not checkable mechanically. + +## Contamination probes (`canary/`) + +The leak vector is named in the authoring rules below: prompts reach third-party +judge and baseline models no matter how private this repo is. If a provider +trains on inference traffic, our eval set is in their next model and every +signal score becomes a measurement of memorisation. + +The probe tier catches that, and it only works as a pair: + +- **One carrier.** Its prompt contains the canary GUID in full. It always + passes, and passing is not the point — its job is to put the GUID into the + traffic so a training pipeline can pick it up. +- **Two or more detectors.** Their prompts withhold the GUID and ask for it + cold. A model that has never seen our traffic cannot answer. One that has, + can. + +**Scoring is inverted here.** `canary_score` of 0 is the healthy result. +Anything above 0 means signal scores for that target are void and the eval set +needs regenerating. Never read it as a capability number. + +A detector for a GUID that no carrier ever transmitted is unanswerable by +construction: it can never fire, so it proves nothing while looking exactly +like a working control. + +## Authoring rules + +**Scrub before you commit.** No API keys, tokens, tailscale hostnames or IPs, +customer data, or personal information. This content gets sent to third-party +judge and baseline models regardless of how private this repo is. Repo +privacy is defence in depth, not the control. + +**Derive from real work.** The value of this bench is that its samples come +from things we actually asked a model to do. An invented task measures an +invented capability. + +**Write the target before you see a model's answer.** Grading against an +answer you just read is how a bench quietly turns into a rubber stamp. + +## Adding samples + +```bash +kbench add agent_ops --input-file prompt.txt --target "expected" --scorer includes +kbench add agent_ops --interactive +``` + +This assigns the next id, stamps a canary, defaults the split, and validates +the record — do not hand-edit the JSONL. diff --git a/data/public/example.jsonl b/data/public/example.jsonl new file mode 100644 index 0000000..8479bbb --- /dev/null +++ b/data/public/example.jsonl @@ -0,0 +1,4 @@ +{"id": "example/0001", "family": "example", "split": "test", "canary": "KBENCH-CANARY-7d41af0c", "input": "Reply with only the numeric HTTP status code that means \"Too Many Requests\". No other text.", "target": "429", "scorer": "exact", "rubric": null, "metadata": {"source": "example", "authored": "2026-07-27", "tags": ["format-adherence"], "note": "Demonstrates the `exact` scorer. Also a decent instruction-following probe: verbose models append explanation and fail."}} +{"id": "example/0002", "family": "example", "split": "test", "canary": "KBENCH-CANARY-2b8e5513", "input": "A systemd unit fails to start with 'Address already in use'. Name one command-line tool that identifies which process is holding the port.", "target": "(lsof|ss|netstat|fuser)", "scorer": "regex", "rubric": null, "metadata": {"source": "example", "authored": "2026-07-27", "tags": ["ops"], "note": "Demonstrates `regex` -- the right choice when several answers are equally correct."}} +{"id": "example/0003", "family": "example", "split": "test", "canary": "KBENCH-CANARY-9c07de24", "input": "In one sentence: what does the vLLM flag --enforce-eager disable?", "target": "CUDA graph", "scorer": "includes", "rubric": null, "metadata": {"source": "example", "authored": "2026-07-27", "tags": ["inference"], "note": "Demonstrates `includes` -- checks one required concept without constraining phrasing."}} +{"id": "example/0004", "family": "example", "split": "test", "canary": "KBENCH-CANARY-5a1f6b8e", "input": "A benchmark harness sends the same 1024-token prompt from 32 concurrent clients to a vLLM server and reports throughput. Explain the measurement flaw and how to fix it.", "target": "Prefix caching makes prefill nearly free for all but the first request, so reported throughput is inflated. Fix: make each request's prompt unique, e.g. a random nonce in the leading tokens.", "scorer": "rubric", "rubric": "Grade CORRECT only if the answer identifies that a shared prompt prefix is cached (prefix caching / KV reuse) and that this inflates measured throughput, AND proposes making prompts unique per request. Mentioning only 'add randomness' without connecting it to prefix caching is INCORRECT. Ignore wording and length.", "metadata": {"source": "example", "authored": "2026-07-27", "tags": ["inference", "benchmarking"], "note": "Demonstrates `rubric` -- the property is genuinely not checkable mechanically. Requires a judge model."}} diff --git a/deploy/bench-karti.service b/deploy/bench-karti.service new file mode 100644 index 0000000..7f996df --- /dev/null +++ b/deploy/bench-karti.service @@ -0,0 +1,25 @@ +[Unit] +Description=bench.karti.ai — model eval + serving perf leaderboard +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=ubuntu +WorkingDirectory=/home/ubuntu/workspace/bench.karti.ai/site +Environment=NODE_ENV=production +Environment=PORT=8909 +EnvironmentFile=/home/ubuntu/workspace/bench.karti.ai/site/.env.local +ExecStart=/usr/bin/npm run start +Restart=always +RestartSec=3 + +# The site only ever reads results/ and writes nothing to disk. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=/home/ubuntu/workspace/bench.karti.ai/site/.next + +[Install] +WantedBy=multi-user.target diff --git a/deploy/lumbridge-bench.service b/deploy/lumbridge-bench.service new file mode 100644 index 0000000..1b3b46d --- /dev/null +++ b/deploy/lumbridge-bench.service @@ -0,0 +1,25 @@ +[Unit] +Description=Lumbridge Bench — model evaluation and serving evidence +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=ubuntu +WorkingDirectory=/home/ubuntu/workspace/lumbridge-bench/site +Environment=NODE_ENV=production +Environment=PORT=8909 +EnvironmentFile=/home/ubuntu/workspace/lumbridge-bench/site/.env.local +ExecStart=/usr/bin/npm run start +Restart=always +RestartSec=3 + +# The site only ever reads results/ and writes nothing to disk. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=/home/ubuntu/workspace/lumbridge-bench/site/.next + +[Install] +WantedBy=multi-user.target diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md new file mode 100644 index 0000000..9277a22 --- /dev/null +++ b/docs/DECISIONS.md @@ -0,0 +1,200 @@ +# Decisions + +Why this repo is shaped the way it is. The reasoning is not recoverable from +the code, which is the only reason this file exists. Append, don't rewrite -- +a superseded decision is more useful with its replacement attached than +deleted. + +--- + +## D1 — Depend on `inspect_ai`, do not fork it + +**Decision.** `bench.karti.ai` pip-installs `inspect-ai` and `inspect-evals`. +It is not a fork. + +**Why.** The obvious move is to fork [inspect_ai][1] and grow from there. The +numbers say otherwise: 407 MB, ~2 commits/day, pushed daily. Forking means +either abandoning that stream of fixes or rebasing against it forever, and we +would be paying that for nothing — we are not changing the execution engine, +the scorer protocol, the sandbox, or the viewer. We are adding tasks, and +tasks are the documented extension point. + +The decisive evidence is upstream's own layout: `inspect_evals` is a +**separate package that depends on `inspect_ai`**, holding 200+ evals, none of +them in a fork of core. The maintainers already built the seam. This repo is +simply our private `inspect_evals`. + +**When to revisit.** If we ever need to change core execution behaviour. Then +fork from a known-good tag, for a stated reason. Not speculatively. + +**Rejected alternative — build our own harness.** Reimplements sandboxed tool +execution, retry/rate-limit handling, parallel rollouts, log format, viewer, +and judge protocol, to arrive somewhere functionally identical. All of this +bench's value is in the task data; none is in the runner. + +[1]: https://github.com/UKGovernmentBEIS/inspect_ai + +--- + +## D2 — A target is `(model × host × quant × serving config × checkpoint)` + +**Decision.** The registry is keyed on the full tuple, not on model name. +See `kbench/registry/__init__.py`. + +**Why.** On unified-memory boxes the serving config moves throughput more than +a model swap does. The live your-node server runs `--enforce-eager` (no CUDA +graphs) and `--gpu-memory-utilization 0.55` (45% of unified memory unused, +capping KV cache and therefore the concurrency ceiling). A result recorded +against "Qwen3.6-35B-A3B" without those flags is not reproducible and not +comparable to anything. + +`checkpoint` is in the key from day one because once we fine-tune, we will +compare `karti-7b@step2000` against `karti-7b@step4000` far more often than we +compare Qwen against Llama. Adding it later would invalidate every result +recorded before it. + +**Consequence.** Every run snapshots the full target into its result file. +Referencing a registry id would go stale the first time `models.yaml` is +edited. + +--- + +## D3 — Three-tier data split, and the eval set stays private + +**Decision.** `data/public/` (a handful of shown examples), `data/private/` +(the real holdout), `data/canary/` (GUID strings embedded in samples). + +**Why private — contamination, not secrets.** The instinctive reason to keep +eval data private is that samples derived from real tool calls might contain +sensitive material. That is true but secondary, and it has its own fix (see +below). The primary reason is that **a public test set gets scraped into the +next training corpus and stops measuring capability**. That is how MMLU, +GSM8K, and HumanEval died. Every benchmark with signal left in 2026 keeps a +private holdout. If our samples ever go public, the bench's useful life is +about one training cycle. + +**On secrets specifically.** A private repo is not a secret-management +strategy. Anything derived from real workloads is scrubbed **at authoring +time** — no keys, no tokens, no tailscale addresses, no customer data — +because that content would also leak through the model we send it to (often +someone else's API) and through any generated share card. Repo privacy is +defence in depth, not the control. + +**Canary tier.** Samples carry embedded GUIDs. Later we can grep a candidate +model for those strings and *prove* contamination rather than suspect it. +Costs nothing now; impossible to add retroactively. + +--- + +## D4 — Per-sample outcomes in git; full outputs stay local + +**Decision.** `results/*.json` stores each sample's score, pass/fail, and a +400-char excerpt. Full model outputs stay in the raw Inspect `.eval` log, +which is gitignored. + +**Why.** An aggregate score tells you a checkpoint got worse. Per-sample +diffs tell you *which capability broke* — the difference between a leaderboard +and a debugging tool, and the whole reason this is usable for fine-tuning. +Storing complete transcripts, though, would bloat the repo (thousands of +samples × multi-KB reasoning traces × hundreds of runs) and would persist +sensitive echoes of signal-tier prompts into git history forever. Git carries +the shape of the failure; the local log carries the transcript. + +--- + +## D5 — Two task tiers: reference and signal + +**Decision.** `reference` tasks come from `inspect_evals` unmodified. +`signal` tasks are ours and private. Both appear on the leaderboard, visually +distinguished, with reference scores captioned *public, likely contaminated, +calibration only*. + +**Why.** A bench made only of novel private tasks is unfalsifiable to an +outside reader — nobody, including future us, can tell whether a strange score +means the model is bad or the harness is broken. The reference tier is the +credibility anchor and the smoke test: if our IFEval lands far from published +values, the harness is wrong. + +**Why IFEval first.** Its scorer is deterministic, so it needs no judge model +— no per-run cost, and no judge drift between runs to confound comparisons. +It is fast enough for a tight debug loop. And instruction adherence is the +property that actually decides whether a small local model can hold a system +prompt in production, which is the live question about `brain`. + +--- + +## D6 — Self-contained perf load generator, not `vllm bench serve` + +**Decision.** `kbench/perf.py` is an async httpx load generator against any +OpenAI-compatible endpoint. + +**Why.** Submissions are the point of the public site, so we cannot assume +vLLM — the harness must measure SGLang, llama.cpp, ollama, or whatever a +submitted model ends up served under. It also needs nothing installed on the +target box, and it lets us fix the metric definitions so numbers stay +comparable across engines instead of inheriting each engine's conventions. + +**Two traps it exists to avoid.** Both produce great-looking meaningless +numbers: + +1. **Prefix caching.** vLLM caches shared prompt prefixes. Identical + concurrent prompts make prefill nearly free after the first request and + overstate throughput badly. Every request gets a unique leading nonce. +2. **Variable output length.** If the model decides when to stop, concurrency + levels finish at different token counts and tok/s stops being comparable. + We send `ignore_eos` so every request emits exactly `output_tokens`. + +--- + +## D7 — Private gitea now; the harness may open-source later, the data never + +**Decision.** `karti/lumbridge-bench` on gitea, private. Results public. + +_(Updated 2026-08-01: the repo was `karti/bench.karti.ai`, now archived. "Site public" meant the +standalone bench.karti.ai leaderboard, which is retired — results surface through +`lumbridgecorp.com/bench` instead. The public/private split below is unchanged.)_ + +**Why.** The leaderboard is only valuable if people can see it; the eval data +is only valuable if they cannot (D3). If we later open-source, the harness +splits out and `data/private/` stays behind — which is why the seam between +`kbench/` (mechanism) and `data/` (content) is kept clean from the start. A +clean seam costs nothing today and keeps the decision reversible. + +--- + +## D8 — Submissions execute untrusted code; sandbox before launch + +**Status.** Planned, not yet implemented. Blocking for the public submit form. + +Anonymous "drop a HuggingFace link and we'll benchmark it" means running +arbitrary code on our own hardware. HF repos can ship custom modeling files +that execute on load, and `.bin` checkpoints are pickles — remote code +execution by design. Non-negotiable gates: + +- `safetensors` only; `trust_remote_code=false`; hard-fail otherwise +- eval runs in a container with **no network egress** and no tailscale route +- pre-flight via the HF API before queueing: reject if params exceed the + target box, if config is unreadable, if the repo is gated +- one box, one queue: show estimated wait, cap submissions per IP per day + +--- + +## D9 — Anonymous submissions get aggregate scores only + +**Decision.** Signed-in users (115karti invite) get per-sample breakdowns and +private-by-default results. Anonymous submissions get an aggregate score, +published. + +**Why.** This reads like feature gating but it is primarily a contamination +defence: detailed per-sample feedback is exactly how someone would +reverse-engineer the private set (D3) by probing it one sample at a time. + +--- + +## D10 — Public input is a manual suggestion, never an execution queue + +**Decision.** The public action is **Suggest a model**. It records a Hugging Face reference and optional context for owner review. It never downloads, imports, schedules, or executes a model. Karti manually inspects the model card, license, files, compatibility, footprint, and serving path, then explicitly decides whether to run it. + +**Why.** The product does not need a public code-execution service. Manual approval is the intended workflow and keeps ownership of scarce hardware and private evaluation data clear. The Lumbridge control plane stores the inert review record with status `suggested`; only an owner can advance its lifecycle. + +**Supersedes D8 for product behavior.** The sandbox described in D8 is no longer a launch dependency because no public suggestion is automatically executed. If automated third-party execution is ever proposed again, it requires a new explicit decision and the full D8 isolation boundary first. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md new file mode 100644 index 0000000..a55a83e --- /dev/null +++ b/docs/DEPLOY.md @@ -0,0 +1,106 @@ +# Deploy + +**Migration status: COMPLETE (2026-08-01).** `karti/lumbridge-bench` is the canonical and only +repository — `karti/bench.karti.ai` is archived in gitea, fully contained in this repo's `main`. + +**The standalone site is retired.** Bench is a Lumbridge product, not its own destination, so +`bench.karti.ai` now serves a permanent redirect to `https://lumbridgecorp.com/bench`. The +`bench-karti.service` backend is stopped and disabled on web-host. + +What that leaves on the box: + +- Caddy vhost for `bench.karti.ai` is now `redir https://lumbridgecorp.com/bench permanent` + (a pre-change Caddyfile backup is at `/etc/caddy/Caddyfile.bak-bench-`) +- DNS `bench.karti.ai A 170.9.14.61` and the Let's Encrypt cert stay — the redirect needs both +- checkout at `/home/ubuntu/workspace/bench.karti.ai` remains, now on `origin` = + `karti/lumbridge-bench`; the `bench-karti` unit is stopped and disabled, not removed + +**To bring a Bench UI back up**, deploy it as part of Lumbridge rather than reviving the subdomain. +Use `deploy/lumbridge-bench.service` and a fresh checkout; do not re-enable `bench-karti`. + +The eval runner (`kbench`) is unaffected by any of this — it is a CLI, not the site. + +## Gotchas hit during the first deploy + +- **Caddy binds `10.0.0.2`, not loopback.** Testing a vhost from the box with + `curl --resolve host:443:127.0.0.1` returns `000` and looks like an outage. + Use `--resolve host:443:10.0.0.2`. +- **Order matters: DNS before the vhost reload.** Caddy will attempt ACME the + moment the vhost loads; if the A record does not exist yet the challenge + fails with NXDOMAIN and the retry sits in backoff for minutes. Adding DNS + first, or reloading Caddy again after adding it, gets the cert immediately. +- **The `oci` CLI is not on PATH on build-host** — it lives at `~/bin/oci`. And + `oci dns zone list` needs `--compartment-id`, but `oci dns zone get + --zone-name-or-id karti.ai` does not, which is the easier way in. + +## Pre-existing, unrelated + +Caddy's log shows recurring ACME failures for `www.office.karti.ai` and +`www.og.karti.ai` — NXDOMAIN, no `www` A records exist. Predates this deploy +and does not affect the apex domains, but it means those two sites retry ACME +forever. Either add the `www` records or drop them from their vhosts. + +| | | +|---|---| +| repo | `karti/lumbridge-bench` on gitea, **private** | +| host | web-host, port **8909** | +| service | `bench-karti` compatibility service (systemd) | +| checkout | `/home/ubuntu/workspace/bench.karti.ai` compatibility path | +| domain | `bench.karti.ai` → web-host, DNS via oci on build-host (profile `cloud2-sanjose`) | + +## One-time setup on web-host + +```bash +cd /home/ubuntu/workspace +git clone https://gitea.example.internal:8444/karti/lumbridge-bench.git bench.karti.ai +cd bench.karti.ai/site +cp .env.example .env.local # loopback Lumbridge control plane +npm install && npm run build + +sudo cp ../deploy/bench-karti.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now bench-karti +curl -s localhost:8909/api/health +``` + +Then add the caddy/nginx vhost for `bench.karti.ai` → `localhost:8909` and the +DNS A record, matching the other `*.karti.ai` sites. + +## CI secret + +The workflow needs one repo secret, `CLOUD2_SSH_KEY`: the private half of a +deploy key whose public half is in `ubuntu@web-host`'s `authorized_keys`. +Generate a dedicated key rather than reusing another site's. + +```bash +ssh-keygen -t ed25519 -f bench-karti-ci-deploy -C "bench-karti-ci-deploy" -N "" +# public half -> web-host authorized_keys; private half -> gitea repo secret +``` + +## Suggestion inbox + +The public form forwards an inert record to the Lumbridge control plane at +`LUMBRIDGE_CONTROL_PLANE_URL`; on web-host this is the private loopback service. +It stores only the Hugging Face URL, a review reason, and optional notes. Contact +data is not collected. The old Supabase migrations remain as history but are not +used by the live suggestion path. Public Bench cards continue to come from +reviewed `results/*.json`. + +## Deploying results + +Score cards are committed JSON, so publishing a new measurement is a commit: + +```bash +kbench run brain --tasks ifeval --limit 100 +git add results/ && git commit -m "measure: qwen3.6-35b-a3b @ your-node" +git push +``` + +CI rebuilds and the board picks it up. There is no database sync step because +there is no database behind the board. + +## What is NOT wired yet + +- **Automatic submission execution.** Intentionally not part of the product. The public form only records a suggestion for owner review; see `DECISIONS.md#d10`. +- **Auth.** Schema and invite code exist; the signup/login UI does not. +- **Suggestion workflow UI.** The owner console now lists the review inbox. Accept/reject controls and all run scheduling remain manual. diff --git a/kbench/__init__.py b/kbench/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kbench/cli.py b/kbench/cli.py new file mode 100644 index 0000000..fc1b3ca --- /dev/null +++ b/kbench/cli.py @@ -0,0 +1,646 @@ +"""kbench command line.""" + +from __future__ import annotations + +import json +import shlex +import subprocess +from pathlib import Path + +import typer +from rich.console import Console +from rich.table import Table + +from . import tasks as task_catalog +from .compare import compare as compare_runs, direction +from .sweep import analyse, point_from_run +from .registry import load_registry +from .results import RESULTS_DIR, Run +from .run import build_run, run_perf, run_quality + +app = typer.Typer( + add_completion=False, + help="Lumbridge Bench -- private eval and serving-performance evidence for self-hosted and API models", +) +console = Console() + + +def _parse_concurrencies(value: str) -> tuple[int, ...]: + return tuple(int(x) for x in value.split(",") if x.strip()) + + +@app.command("targets") +def list_targets() -> None: + """List benchmarkable targets.""" + registry = load_registry() + table = Table(title="targets", header_style="bold") + for col in ("id", "tier", "host", "quant", "params", "engine", "aliases"): + table.add_column(col) + for t in registry.targets.values(): + params = f"{t.params_b:g}B" if t.params_b else "-" + if t.active_params_b: + params += f" ({t.active_params_b:g}B active)" + table.add_row( + t.id, t.tier, t.host, t.quant or "-", params, + t.serving.engine, ", ".join(t.aliases) or "-", + ) + console.print(table) + + +@app.command("tasks") +def list_tasks() -> None: + """List benchmark tasks.""" + table = Table(title="tasks", header_style="bold") + for col in ("name", "tier", "samples", "judge", "inspect task"): + table.add_column(col) + for spec in task_catalog.CATALOG.values(): + table.add_row( + spec.name, spec.tier, str(spec.dataset_samples or "-"), + "yes" if spec.judge_required else "no", spec.inspect_task, + ) + console.print(table) + console.print( + "\n[dim]reference = public, likely contaminated, calibration only\n" + "signal = private, ours, the actual measurement[/dim]" + ) + + +@app.command("snapshot") +def snapshot(host: str) -> None: + """Print the live serving flags on a host, for pasting into models.yaml. + + Results are only comparable between targets with identical flags, so the + registry has to record what the server was ACTUALLY started with rather + than what we believe we started it with. + """ + registry = load_registry() + h = registry.host(host) + if not h.ssh: + raise typer.BadParameter(f"host {host!r} has no ssh address") + + proc = subprocess.run( + ["ssh", h.ssh, "ps -eo args | grep -E '[v]llm serve|[s]glang|[l]lama-server'"], + capture_output=True, text=True, timeout=30, + # A host with no serving process is an ordinary outcome, not an + # exception -- it is reported below. + check=False, + ) + if proc.returncode != 0 or not proc.stdout.strip(): + console.print(f"[yellow]no serving process found on {host}[/yellow]") + raise typer.Exit(1) + + for line in proc.stdout.strip().splitlines(): + parts = shlex.split(line) + console.print(f"\n[bold]{parts[0] if parts else '?'}[/bold]") + flags: dict[str, object] = {} + i = 0 + while i < len(parts): + if parts[i].startswith("--"): + key = parts[i][2:].replace("-", "_") + if i + 1 < len(parts) and not parts[i + 1].startswith("--"): + flags[key] = parts[i + 1] + i += 2 + else: + flags[key] = True + i += 1 + else: + i += 1 + for k, v in flags.items(): + console.print(f" {k}: {v}") + + + +def _load_prompts(path: Path | None) -> list[str] | None: + """Read representative prompts: one per line, or a JSONL with an `input` field. + + Measuring speculative decoding on the synthetic prompt is measuring the wrong thing — the + draft model's acceptance rate depends on how predictable the text is, and lorem with a + random nonce is not the text you serve. + """ + if path is None: + return None + lines = [ln.strip() for ln in path.read_text().splitlines() if ln.strip()] + prompts = [] + for ln in lines: + if ln.startswith("{"): + try: + prompts.append(json.loads(ln)["input"]) + continue + except (json.JSONDecodeError, KeyError): + pass + prompts.append(ln) + if not prompts: + raise typer.BadParameter(f"{path} contains no prompts") + return prompts + + +@app.command("perf") +def perf_cmd( + target: str, + concurrency: str = typer.Option("1,8,32", help="comma-separated concurrency levels"), + input_tokens: int = typer.Option(1024), + output_tokens: int = typer.Option(256), + save: bool = typer.Option(True, help="write a results JSON"), + prompt_file: Path | None = typer.Option( + None, help="representative prompts (one per line, or JSONL with an `input` field)" + ), + natural_stop: bool = typer.Option( + False, + help="let the model stop on its own instead of pinning output length. Required for an " + "honest speculative-decoding measurement; weakens comparability across points.", + ), +) -> None: + """Run a serving performance sweep only (no quality tasks).""" + registry = load_registry() + t = registry.target(target) + result = run_perf( + t, _parse_concurrencies(concurrency), input_tokens, output_tokens, + prompts=_load_prompts(prompt_file), pin_output=not natural_stop, + ) + run = build_run(registry, t, quality=[], perf=result) + _print_card(run) + if save: + path = run.save() + console.print(f"\n[green]saved[/green] {path.relative_to(Path.cwd())}") + + +@app.command("run") +def run_cmd( + target: str, + tasks: str = typer.Option("ifeval", help="comma-separated task names, or 'all'"), + limit: int | None = typer.Option(None, help="cap samples per task (recorded in output)"), + max_connections: int = typer.Option(16, help="concurrent requests to the model"), + epochs: int = typer.Option( + 1, + help="run each sample N times and average. Sampling is not reproducible on a " + "batching server even with a fixed seed, so one epoch is a draw, not a rate.", + ), + concurrency: str = typer.Option("1,8,32"), + input_tokens: int = typer.Option(1024), + output_tokens: int = typer.Option(256), + skip_perf: bool = typer.Option(False), + skip_quality: bool = typer.Option(False), +) -> None: + """Full score card: quality tasks + perf sweep -> one results JSON.""" + registry = load_registry() + t = registry.target(target) + + names = list(task_catalog.CATALOG) if tasks == "all" else [ + n.strip() for n in tasks.split(",") if n.strip() + ] + specs = [task_catalog.get(n) for n in names] + + quality = [] + if not skip_quality: + quality = run_quality( + t, specs, limit=limit, max_connections=max_connections, epochs=epochs + ) + + perf = None + if not skip_perf: + perf = run_perf(t, _parse_concurrencies(concurrency), input_tokens, output_tokens) + + run = build_run(registry, t, quality=quality, perf=perf) + _print_card(run) + path = run.save() + console.print(f"\n[green]saved[/green] {path.relative_to(Path.cwd())}") + + +@app.command("sweep") +def sweep_cmd( + targets: list[str] = typer.Argument(..., help="two or more target ids to compare"), + tasks: str = typer.Option("ifeval", help="comma-separated task names, or 'all'"), + limit: int | None = typer.Option(None, help="cap samples per task"), + max_connections: int = typer.Option(16), + concurrency: str = typer.Option("1,8,32"), + input_tokens: int = typer.Option(1024), + output_tokens: int = typer.Option(256), + skip_perf: bool = typer.Option(False), + skip_quality: bool = typer.Option(False), +) -> None: + """Run the same tasks against several targets and reduce them to a frontier. + + Sequential on purpose: on a unified-memory box two targets cannot both be resident, so + running them together would measure contention rather than either one. Each target still + produces an ordinary score card in results/, so compare and card keep working on them. + """ + if len(targets) < 2: + raise typer.BadParameter("a sweep needs at least two targets") + + registry = load_registry() + names = list(task_catalog.CATALOG) if tasks == "all" else [ + n.strip() for n in tasks.split(",") if n.strip() + ] + specs = [task_catalog.get(n) for n in names] + + points, hosts = [], [] + for i, target_id in enumerate(targets, start=1): + console.rule(f"[bold]{i}/{len(targets)} {target_id}[/bold]") + try: + t = registry.target(target_id) + hosts.append(t.host) + quality = [] if skip_quality else run_quality( + t, specs, limit=limit, max_connections=max_connections + ) + perf = None if skip_perf else run_perf( + t, _parse_concurrencies(concurrency), input_tokens, output_tokens + ) + run = build_run(registry, t, quality=quality, perf=perf) + path = run.save() + console.print(f"[green]saved[/green] {path.name}") + points.append(point_from_run(run.to_dict(), str(path.name))) + except Exception as exc: # noqa: BLE001 + # One bad target must not throw away the targets already measured — a sweep is + # hours of GPU time and re-running the good ones to reach the next is wasteful. + console.print(f"[red]failed[/red] {target_id}: {exc}") + points.append( + point_from_run({"target_id": target_id, "verdict": {}}) + ) + points[-1].error = str(exc) + + result = analyse(points, hosts=hosts) + + console.print() + console.rule("[bold]frontier[/bold]") + table = Table(header_style="bold") + for col in ("", "target", "signal", "peak tok/s", "tok/s @1", "verdict"): + table.add_column(col) + for point in sorted( + result.points, key=lambda x: (x.error is not None, -(x.throughput_tps or 0)) + ): + if point.error: + mark, verdict = "[red]![/red]", f"[red]{point.error[:44]}[/red]" + elif point.target_id in result.dominated: + mark = "[dim]·[/dim]" + verdict = f"[dim]beaten by {result.dominated[point.target_id]}[/dim]" + else: + mark, verdict = "[green]✓[/green]", "[green]worth running[/green]" + table.add_row( + mark, + point.target_id, + f"{point.quality:.3f}" if point.quality is not None else "-", + f"{point.throughput_tps:.1f}" if point.throughput_tps else "-", + f"{point.single_stream_tps:.1f}" if point.single_stream_tps else "-", + verdict, + ) + console.print(table) + + for warning in result.warnings: + console.print(f"[yellow]![/yellow] {warning}") + + if result.best: + console.print( + f"\n[bold]{len(result.best)} of {len(result.points)} " + f"worth considering:[/bold] " + ", ".join(result.best) + ) + if result.dominated: + console.print( + "[dim]The rest are beaten on every measured axis at once — not a close call, " + "a strictly worse option.[/dim]" + ) + + +@app.command("card") +def card_cmd(path: Path) -> None: + """Render a saved results JSON as a score card.""" + data = Run.load(path) + _print_card_dict(data) + + +def _print_card(run: Run) -> None: + _print_card_dict(run.to_dict()) + + +def _print_card_dict(data: dict) -> None: + tgt = data["target"] + console.print() + console.rule(f"[bold]{tgt.get('display_name', data['target_id'])}[/bold]") + console.print( + f"[dim]{data['target_id']} | {data['host'].get('hardware', '?')} | " + f"{data['timestamp']}[/dim]" + ) + + quality = data.get("quality") or [] + if quality: + table = Table(title="quality", header_style="bold", title_justify="left") + for col in ("task", "tier", "n", "accuracy", "notes"): + table.add_column(col) + for q in quality: + if q.get("error"): + table.add_row(q["task"], q["tier"], "-", "[red]ERROR[/red]", q["error"][:60]) + continue + acc = q["metrics"].get("accuracy") + limit = q["metrics"].get("_limit") + note = f"limited to {int(limit)} samples" if limit else "" + if q["tier"] == "reference": + note = (note + "; " if note else "") + "calibration only" + table.add_row( + q["task"], q["tier"], str(q["n_samples"]), + f"{acc:.3f}" if acc is not None else "-", note, + ) + console.print(table) + + perf = data.get("perf") + if perf and perf.get("points"): + table = Table(title="serving perf", header_style="bold", title_justify="left") + points = data["perf"].get("points") or [] + # Only widen the table when there is something to put in the column — a permanent + # "accept: -" on every non-speculative target is noise. + show_accept = any(pt.get("spec_acceptance_rate") is not None for pt in points) + cols = ["conc", "tok/s total", "tok/s /stream", "TTFT p50", "TTFT p95", "TPOT p50"] + if show_accept: + cols.append("accept") + cols.append("failed") + for col in cols: + table.add_column(col, justify="right") + for p in perf["points"]: + table.add_row( + str(p["concurrency"]), + f"{p['output_tps_total']:.1f}", + f"{p['output_tps_per_stream']:.1f}", + f"{p['ttft_p50_ms']:.0f}ms" if p.get("ttft_p50_ms") else "-", + f"{p['ttft_p95_ms']:.0f}ms" if p.get("ttft_p95_ms") else "-", + f"{p['tpot_p50_ms']:.1f}ms" if p.get("tpot_p50_ms") else "-", + *( + [ + f"{p['spec_acceptance_rate']:.0%}" + if p.get("spec_acceptance_rate") is not None + else "-" + ] + if show_accept + else [] + ), + f"[red]{p['failed']}[/red]" if p["failed"] else "0", + ) + console.print(table) + if any(pt.get("natural_stop") for pt in points): + spread = next( + (pt["output_token_stdev"] for pt in points if pt.get("output_token_stdev")), None + ) + console.print( + " [dim]natural stop: output length was not pinned" + + (f", ±{spread:.0f} tokens" if spread else "") + + ". Required for an honest speculative-decoding number, and it means these" + " points are not directly comparable to pinned-length runs.[/dim]" + ) + + verdict = data.get("verdict") or {} + if verdict: + # Contamination first and loud. It is the one result that invalidates every other + # number on the card, so it cannot be a row someone scrolls past. + state = verdict.get("contamination") + if state == "detected": + console.print() + console.print( + "[bold white on red] CONTAMINATED [/bold white on red] " + f"the model reproduced {verdict.get('canary_score', 0):.0%} of the canary probes." + ) + console.print( + " [red]Quality scores for this target are void — it has seen the eval set.[/red]" + ) + if verdict.get("signal_score_unverified") is not None: + console.print( + f" [dim]withheld signal score: {verdict['signal_score_unverified']:.3f} " + "(recorded for forensics, not usable as a result)[/dim]" + ) + console.print(" [dim]Regenerate the private set before trusting anything here.[/dim]") + elif state == "unverified": + console.print( + "\n[yellow]![/yellow] no contamination probe ran — quality scores are unverified" + ) + + console.print("\n[bold]verdict[/bold]") + for k, v in verdict.items(): + if v is None or k in ("contamination", "signal_score_unverified"): + continue + console.print(f" {k}: {v}") + + +@app.command("add") +def add_cmd( + family: str, + input_text: str | None = typer.Option(None, "--input", help="prompt text"), + input_file: Path | None = typer.Option(None, help="read the prompt from a file"), + target: str | None = typer.Option(None, help="expected answer (or regex)"), + scorer: str = typer.Option("includes", help="exact | includes | regex | rubric"), + rubric: str | None = typer.Option(None, help="grading instructions for --scorer rubric"), + split: str = typer.Option("test", help="train | dev | test -- never change this later"), + tier: str = typer.Option("private", help="private | public"), + source: str = typer.Option("manual", help="where this sample came from"), + tags: str = typer.Option("", help="comma-separated"), +) -> None: + """Append a sample to a family, with id, canary, and validation applied. + + Do not hand-edit the JSONL. This assigns a stable non-reused id (per-sample + regression tracking keys on it) and stamps a canary GUID used later to + prove contamination. + """ + import uuid + + from .tasks.signal import family_path, validate_record + + if input_file: + input_text = input_file.read_text().strip() + if not input_text: + raise typer.BadParameter("provide --input or --input-file") + + path = family_path(family, tier) + path.parent.mkdir(parents=True, exist_ok=True) + + existing = [ + json.loads(line) + for line in (path.read_text().splitlines() if path.exists() else []) + if line.strip() and not line.startswith("//") + ] + # Highest existing number + 1, so ids are never reused even after deletions. + next_n = 1 + max( + (int(r["id"].rsplit("/", 1)[-1]) for r in existing if "/" in r.get("id", "")), + default=0, + ) + + record = { + "id": f"{family}/{next_n:04d}", + "family": family, + "split": split, + "canary": f"KBENCH-CANARY-{uuid.uuid4().hex[:8]}", + "input": input_text, + "target": target, + "scorer": scorer, + "rubric": rubric, + "metadata": { + "source": source, + "authored": __import__("datetime").date.today().isoformat(), + "tags": [t.strip() for t in tags.split(",") if t.strip()], + }, + } + validate_record(record, "new sample") + + with path.open("a") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + console.print(f"[green]added[/green] {record['id']} -> {path}") + console.print( + "[yellow]scrub check:[/yellow] no keys, tokens, tailscale addresses, " + "or customer data in that sample?" + ) + + +def _resolve_run(ref: str) -> Path: + """Accept a path, a bare filename, or a run-id prefix. + + Nobody types a full run id. Being strict about it just means every comparison starts with + an `ls`. + """ + direct = Path(ref) + if direct.is_file(): + return direct + candidates = sorted(RESULTS_DIR.glob(f"*{ref}*.json")) + if not candidates: + raise typer.BadParameter(f"no result matching {ref!r} in {RESULTS_DIR}") + if len(candidates) > 1: + names = "\n ".join(c.name for c in candidates) + raise typer.BadParameter(f"{ref!r} matches several results:\n {names}") + return candidates[0] + + +def _fmt(value: float | None, places: int = 1) -> str: + return "-" if value is None else f"{value:.{places}f}" + + +def _delta_cell(d, places: int = 1) -> str: + """A delta with its sign, its percent, and — crucially — whether that is good.""" + if d.absolute is None: + return "[dim]-[/dim]" + verdict = direction(d.name.split(".")[-1], d) + colour = {"better": "green", "worse": "red", "flat": "dim"}[verdict] + pct = "" if d.percent is None else f" ({d.percent:+.1f}%)" + return f"[{colour}]{d.absolute:+.{places}f}{pct}[/{colour}]" + + +@app.command("compare") +def compare_cmd( + before: str = typer.Argument(..., help="Baseline run: path, filename, or run-id fragment"), + after: str = typer.Argument(..., help="Run to compare against the baseline"), + show_flips: int = typer.Option(20, help="How many changed samples to list (0 for all)"), +) -> None: + """Diff two score cards: what changed, and whether the comparison is honest.""" + before_path, after_path = _resolve_run(before), _resolve_run(after) + b, a = Run.load(before_path), Run.load(after_path) + result = compare_runs(b, a) + + console.print() + console.rule("[bold]compare[/bold]") + console.print(f"[dim]before[/dim] {before_path.name}") + console.print(f"[dim]after [/dim] {after_path.name}\n") + + # What actually differs about the two targets. Printed FIRST: without it the reader has + # no idea what the numbers below are attributable to. + if result.target_deltas or result.host_changed: + t = Table(title="what changed about the target", header_style="bold") + for col in ("axis", "before", "after"): + t.add_column(col, overflow="fold") + for d in result.target_deltas: + t.add_row(d.field, str(d.before), str(d.after)) + if result.host_changed: + h = result.host_changed + t.add_row("[yellow]host[/yellow]", str(h.before), str(h.after)) + console.print(t) + + for warning in result.warnings: + console.print(f"[yellow]![/yellow] {warning}") + if result.warnings: + console.print() + + if result.quality: + t = Table(title="quality", header_style="bold") + for col in ("metric", "before", "after", "delta"): + t.add_column(col) + for d in result.quality: + t.add_row(d.name, _fmt(d.before, 3), _fmt(d.after, 3), _delta_cell(d, 3)) + console.print(t) + + if result.flips: + shown = result.flips if show_flips == 0 else result.flips[:show_flips] + # When nothing about the measurement changed -- same target, same host, same eval + # set, same sampling -- a flipped sample did not regress or get fixed. It is the + # same question answered twice with different luck. Calling it "regressed" sends + # someone hunting a cause that does not exist, which is worse than saying nothing. + same_setup = ( + not result.target_deltas + and not result.host_changed + and not any( + "edited between runs" in w or "Sampling changed" in w + for w in result.warnings + ) + ) + t = Table( + title=( + f"samples that changed ({len(result.flips)})" + + (" — same setup, so this is instability, not change" if same_setup else "") + ), + header_style="bold", + ) + for col in ("task", "sample", "before", "after", ""): + t.add_column(col, overflow="fold") + for f in shown: + if same_setup: + mark = "[yellow]unstable[/yellow]" + else: + mark = { + "fail": "[red]regressed[/red]", + "pass": "[green]fixed[/green]", + "changed": "[dim]score[/dim]", + }[f.became] + t.add_row(f.task, f.sample_id, f"{f.before:.2f}", f"{f.after:.2f}", mark) + console.print(t) + if len(shown) < len(result.flips): + console.print(f"[dim]… {len(result.flips) - len(shown)} more; --show-flips 0 for all[/dim]") + elif result.quality: + console.print("[dim]no sample outcomes changed[/dim]") + + if result.perf: + t = Table(title="serving performance", header_style="bold") + for col in ("concurrency", "metric", "before", "after", "delta"): + t.add_column(col) + for concurrency in sorted(result.perf): + for i, d in enumerate(result.perf[concurrency]): + t.add_row( + str(concurrency) if i == 0 else "", + d.name.split(".")[-1], + _fmt(d.before), + _fmt(d.after), + _delta_cell(d), + ) + console.print(t) + + if not result.quality and not result.perf: + console.print("[yellow]neither run carries quality or perf results[/yellow]") + + +@app.command("results") +def results_cmd() -> None: + """List saved score cards.""" + files = sorted(RESULTS_DIR.glob("*.json")) + if not files: + console.print("[dim]no results yet[/dim]") + return + table = Table(title="results", header_style="bold") + for col in ("date", "target", "signal", "reference", "tok/s @1", "peak tok/s", "file"): + table.add_column(col) + for f in files: + d = json.loads(f.read_text()) + v = d.get("verdict") or {} + table.add_row( + d["timestamp"][:10], + d["target_id"], + f"{v['signal_score']:.3f}" if v.get("signal_score") is not None else "-", + f"{v['reference_score']:.3f}" if v.get("reference_score") is not None else "-", + f"{v['single_stream_tps']:.1f}" if v.get("single_stream_tps") else "-", + f"{v['peak_throughput_tps']:.1f}" if v.get("peak_throughput_tps") else "-", + f.name, + ) + console.print(table) + + +if __name__ == "__main__": + app() diff --git a/kbench/compare.py b/kbench/compare.py new file mode 100644 index 0000000..a46df13 --- /dev/null +++ b/kbench/compare.py @@ -0,0 +1,335 @@ +"""Diff two runs. + +This is the verb the bench exists for. A single score card tells you what one target did +once; it cannot tell you whether a quantisation cost you anything, whether a checkpoint +regressed, or whether a serving flag was worth it. Those are the questions people actually +have, and all of them are differences. + +Two design decisions carry most of the weight here: + +1. PER-SAMPLE DIFFS, NOT AGGREGATE DIFFS. "0.81 → 0.78" tells you something got worse and + nothing about what. Listing the samples that flipped tells you *which capability broke*, + which is the difference between a leaderboard and a debugging tool. The result schema + stores per-sample outcomes precisely so this is possible. + +2. COMPARABILITY IS CHECKED, NOT ASSUMED. Two runs are only honestly comparable when the + thing you did not change actually did not change. Comparing a quant against a different + quant on a different host tells you nothing, but it renders just as confidently as a + clean comparison. So the diff states what differs about the targets themselves, and + warns when more than one axis moved at once. + +Perf and quality are diffed independently: a run may have one, both, or neither, and a +target with no signal-tier score is still worth comparing on throughput. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +# Below this, a rate difference is not worth reading as a change. Applied to rates only: a +# single sample flipping is always worth showing. +# +# Calibrate this against your own hardware before trusting it. The two runs committed here are +# the same target on the same box on the same day, and throughput between them differs by ~10% +# — so on that machine, anything under 10% is indistinguishable from running it twice. The +# floor below is deliberately lower than that observed spread, because suppressing a real +# regression is worse than showing noise the reader can dismiss; the identical-target warning +# is what tells them which they are looking at. +PERF_NOISE_FLOOR_PCT = 5.0 + +# The axes that make a target a target. Comparing across more than one at a time produces a +# number that cannot be attributed to anything. +TARGET_AXES = ("model", "quantization", "checkpoint", "serving_config", "engine") + + +@dataclass +class FieldDelta: + """One axis on which two targets differ.""" + + field: str + before: Any + after: Any + + +@dataclass +class SampleFlip: + """A sample whose outcome changed between runs.""" + + sample_id: str + task: str + before: float + after: float + became: str # "pass" | "fail" | "changed" + + +@dataclass +class MetricDelta: + name: str + before: float | None + after: float | None + + @property + def absolute(self) -> float | None: + if self.before is None or self.after is None: + return None + return self.after - self.before + + @property + def percent(self) -> float | None: + if self.before in (None, 0) or self.after is None: + return None + return (self.after - self.before) / abs(self.before) * 100.0 + + @property + def significant(self) -> bool: + """Whether a rate moved enough to be worth reading.""" + pct = self.percent + return pct is not None and abs(pct) >= PERF_NOISE_FLOOR_PCT + + +@dataclass +class Comparison: + before_id: str + after_id: str + target_deltas: list[FieldDelta] = field(default_factory=list) + host_changed: FieldDelta | None = None + quality: list[MetricDelta] = field(default_factory=list) + flips: list[SampleFlip] = field(default_factory=list) + perf: dict[int, list[MetricDelta]] = field(default_factory=dict) + only_in_before: list[str] = field(default_factory=list) + only_in_after: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + @property + def confounded(self) -> bool: + """More than one axis moved, so no difference below can be attributed to any of them.""" + return len(self.target_deltas) + (1 if self.host_changed else 0) > 1 + + +def _target_field(target: dict[str, Any], name: str) -> Any: + value = target.get(name) + # serving_config is a dict; render it deterministically so a reordered YAML load does not + # read as a change. + if isinstance(value, dict): + return tuple(sorted((str(k), str(v)) for k, v in value.items())) + return value + + +def _diff_targets(before: dict, after: dict) -> list[FieldDelta]: + deltas = [] + for axis in TARGET_AXES: + b, a = _target_field(before, axis), _target_field(after, axis) + if b != a: + deltas.append(FieldDelta(axis, b, a)) + return deltas + + +def _sample_index(run: dict) -> dict[tuple[str, str], dict]: + """(task, sample_id) -> outcome. Keyed on both because ids are only unique within a task.""" + index = {} + for quality in run.get("quality") or []: + task = quality.get("task", "?") + for sample in quality.get("samples") or []: + index[(task, sample["sample_id"])] = sample + return index + + +def _perf_index(run: dict) -> dict[int, dict]: + perf = run.get("perf") or {} + return {p["concurrency"]: p for p in perf.get("points") or []} + + +def compare(before: dict, after: dict) -> Comparison: + """Diff two loaded result dicts. Neither is mutated.""" + result = Comparison( + before_id=before.get("run_id", "?"), + after_id=after.get("run_id", "?"), + target_deltas=_diff_targets(before.get("target") or {}, after.get("target") or {}), + ) + + b_host = (before.get("host") or {}).get("id") or (before.get("host") or {}).get("name") + a_host = (after.get("host") or {}).get("id") or (after.get("host") or {}).get("name") + if b_host != a_host: + result.host_changed = FieldDelta("host", b_host, a_host) + + # A contaminated run has no comparable quality half, and diffing against one produces a + # delta that looks exactly like a real regression or gain. Say so before anything else. + for label, run in (("before", before), ("after", after)): + state = (run.get("verdict") or {}).get("contamination") + if state == "detected": + result.warnings.append( + f"The {label} run is CONTAMINATED — it reproduced canary probes. Its quality " + "numbers are memorisation, not capability, and no quality delta below means " + "anything." + ) + elif state == "unverified": + result.warnings.append( + f"The {label} run has no contamination probe, so its quality scores are " + "unverified." + ) + + if not result.target_deltas and not result.host_changed: + # Same target, same box: any difference is run-to-run variance, which is worth knowing + # because it sets the floor below which no other comparison means anything. + result.warnings.append( + "Identical target and host — this measures run-to-run variance, not a change." + ) + if result.confounded: + moved = [d.field for d in result.target_deltas] + (["host"] if result.host_changed else []) + result.warnings.append( + f"{len(moved)} axes changed at once ({', '.join(moved)}) — no difference below " + "can be attributed to any one of them." + ) + + # ---- quality ----------------------------------------------------------------- + b_metrics = {q["task"]: q.get("metrics", {}) for q in before.get("quality") or []} + a_metrics = {q["task"]: q.get("metrics", {}) for q in after.get("quality") or []} + for task in sorted(set(b_metrics) | set(a_metrics)): + keys = set(b_metrics.get(task, {})) | set(a_metrics.get(task, {})) + for key in sorted(keys): + result.quality.append( + MetricDelta( + f"{task}.{key}", + b_metrics.get(task, {}).get(key), + a_metrics.get(task, {}).get(key), + ) + ) + + b_samples, a_samples = _sample_index(before), _sample_index(after) + for key in sorted(b_samples.keys() & a_samples.keys()): + b, a = b_samples[key], a_samples[key] + if b.get("passed") == a.get("passed") and b.get("score") == a.get("score"): + continue + if b.get("passed") and not a.get("passed"): + became = "fail" + elif a.get("passed") and not b.get("passed"): + became = "pass" + else: + became = "changed" + result.flips.append( + SampleFlip(key[1], key[0], b.get("score", 0.0), a.get("score", 0.0), became) + ) + # Regressions first: a capability that broke is the reason anyone runs this. + result.flips.sort(key=lambda f: ({"fail": 0, "changed": 1, "pass": 2}[f.became], f.sample_id)) + + # A sample present in only one run is not a flip — it is a changed eval set, which + # silently moves an aggregate. Surfaced separately so it cannot be mistaken for signal. + result.only_in_before = sorted(f"{t}/{s}" for t, s in b_samples.keys() - a_samples.keys()) + result.only_in_after = sorted(f"{t}/{s}" for t, s in a_samples.keys() - b_samples.keys()) + if not b_samples or not a_samples: + # Not a changed eval set — one side simply never measured quality. Saying "100 samples + # added" here would be a confusing way to report "the baseline is perf-only". + if b_samples or a_samples: + missing = "before" if not b_samples else "after" + result.warnings.append( + f"The {missing} run carries no quality results, so only serving performance " + "is comparable." + ) + elif result.only_in_before or result.only_in_after: + result.warnings.append( + f"The eval set changed: {len(result.only_in_before)} sample(s) gone, " + f"{len(result.only_in_after)} added. Aggregate scores are not comparable." + ) + + # An EDITED sample keeps its id, so the add/remove diff above sees nothing. Rewording a + # prompt or loosening a regex changes what the score means while every id still lines up, + # which is the most dangerous shape of eval drift: the comparison looks clean. The + # fingerprint hashes (id, input, target), so it moves when the content does. + b_fp = {q["task"]: q.get("dataset_fingerprint") for q in before.get("quality") or []} + a_fp = {q["task"]: q.get("dataset_fingerprint") for q in after.get("quality") or []} + for task in sorted(b_fp.keys() & a_fp.keys()): + bf, af = b_fp[task], a_fp[task] + if bf and af and bf != af: + result.warnings.append( + f"Task {task!r} was edited between runs ({bf} -> {af}): same sample ids, " + "different content. The scores measure different questions." + ) + elif not bf or not af: + result.warnings.append( + f"Task {task!r} has no dataset fingerprint on one side, so an edit to the " + "sample text cannot be ruled out. Re-run to get a comparable pair." + ) + + # Sampling is part of the identity of a quality result. The same model at temperature + # 1 and at 0.6 is two different measurements, and unpinned sampling moved this bench's + # score by 27% of its samples between identical runs -- far more than most real + # regressions. A card that predates sampling being recorded cannot be ruled comparable + # either, so say so rather than diff it silently. + b_s, a_s = before.get("sampling") or {}, after.get("sampling") or {} + if b_samples and a_samples: + if not b_s or not a_s: + result.warnings.append( + "One run does not record how the model was sampled, so an unpinned " + "temperature cannot be ruled out. Quality deltas below may be noise." + ) + else: + changed = sorted( + k for k in set(b_s) | set(a_s) if b_s.get(k) != a_s.get(k) + ) + # Epoch count is not a sampling change. Both sides estimate the same quantity; + # more epochs just estimates it better. Reporting that as "not comparable" + # overstates it, and a warning that overstates is one people learn to skip -- + # which costs you the ones that matter. + decode = [k for k in changed if k != "epochs"] + if decode: + result.warnings.append( + f"Sampling changed between runs ({', '.join(decode)}): " + f"{ {k: b_s.get(k) for k in decode} } -> { {k: a_s.get(k) for k in decode} }. " + "Quality scores are not comparable across different sampling." + ) + if "epochs" in changed: + b_e, a_e = b_s.get("epochs", 1), a_s.get("epochs", 1) + result.warnings.append( + f"Epochs differ ({b_e} -> {a_e}). Both estimate the same score; the " + f"{'before' if (b_e or 1) < (a_e or 1) else 'after'} run is the noisier " + "of the two, so read small deltas with that in mind." + ) + + # ---- perf -------------------------------------------------------------------- + b_perf, a_perf = _perf_index(before), _perf_index(after) + for concurrency in sorted(b_perf.keys() & a_perf.keys()): + b, a = b_perf[concurrency], a_perf[concurrency] + result.perf[concurrency] = [ + MetricDelta(name, b.get(name), a.get(name)) + for name in ( + "output_tps_total", + "output_tps_per_stream", + "ttft_p50_ms", + "ttft_p95_ms", + "tpot_p50_ms", + # Higher is better, so the default direction rule is already right. Diffing it + # is the point of an MTP comparison: throughput can move for many reasons, + # acceptance moves only because drafting got better or worse. + "spec_acceptance_rate", + ) + if b.get(name) is not None or a.get(name) is not None + ] + modes = {bool(p.get("natural_stop")) for p in list(b_perf.values()) + list(a_perf.values())} + if len(modes) > 1: + result.warnings.append( + "One run pinned output length and the other let the model stop naturally. " + "Throughput is not comparable across those two modes." + ) + + dropped = sorted(b_perf.keys() - a_perf.keys()) + added = sorted(a_perf.keys() - b_perf.keys()) + if dropped or added: + result.warnings.append( + f"Concurrency sweep differs: {dropped or 'none'} dropped, {added or 'none'} added. " + "Peak throughput is only comparable across a shared sweep." + ) + + return result + + +# Metrics where a smaller number is the better one, so a fall is an improvement. +LOWER_IS_BETTER = ("ttft_p50_ms", "ttft_p95_ms", "tpot_p50_ms") + + +def direction(metric: str, delta: MetricDelta) -> str: + """"better" | "worse" | "flat" — meaning, not just sign.""" + if delta.absolute is None or not delta.significant: + return "flat" + improved = delta.absolute < 0 if metric in LOWER_IS_BETTER else delta.absolute > 0 + return "better" if improved else "worse" diff --git a/kbench/perf.py b/kbench/perf.py new file mode 100644 index 0000000..e3cd575 --- /dev/null +++ b/kbench/perf.py @@ -0,0 +1,319 @@ +"""Serving performance measurement. + +A self-contained async load generator rather than a wrapper around +`vllm bench serve`. Reasons: + + * It works against ANY OpenAI-compatible endpoint -- vLLM, SGLang, + llama.cpp, ollama, or a submitted model served however we choose to serve + it. Submissions are the point of the site; we cannot assume vLLM. + * It needs nothing installed on the target box. The runner talks HTTP. + * We control the metric definitions, so numbers stay comparable across + engines rather than inheriting each engine's benchmarking conventions. + +TWO MEASUREMENT TRAPS THIS CODE AVOIDS -- both silently produce numbers that +look great and mean nothing: + + 1. PREFIX CACHING. vLLM caches shared prompt prefixes. If every concurrent + request sends the same prompt, prefill is nearly free after the first and + throughput is wildly overstated. Every request here gets a unique nonce + prefix so each one actually does its own prefill. + + 2. VARIABLE OUTPUT LENGTH. If the model decides when to stop, concurrency + levels finish at different token counts and tok/s is not comparable + between runs. We send `ignore_eos` so every request emits exactly + `output_tokens` tokens. +""" + +from __future__ import annotations + +import asyncio +import json +import statistics +import time +import uuid +from dataclasses import dataclass + +import httpx + +from .results import PerfPoint, PerfResult +from .specdec import read_counters + +# Filler vocabulary for synthetic prompts. Ordinary words rather than repeated +# junk so tokenization stays close to what real traffic looks like. +_FILLER = ( + "the quick brown fox jumps over a lazy dog while distant thunder rolls " + "across open water and the harbour lights flicker against low cloud cover " + "as fishing boats return with the morning tide carrying nets and crates " +).split() + + +def _synthetic_prompt(approx_tokens: int, nonce: str) -> str: + """Build a prompt of roughly `approx_tokens` tokens, unique per request. + + The nonce leads so it lands in the first block and defeats prefix-cache + reuse across concurrent requests. + """ + # ~0.75 tokens per word for ordinary English text. + n_words = max(1, int(approx_tokens / 0.75)) + body = " ".join(_FILLER[i % len(_FILLER)] for i in range(n_words)) + return f"[{nonce}] {body}" + + +@dataclass +class RequestResult: + ok: bool + ttft_ms: float | None = None + total_s: float | None = None + output_tokens: int = 0 + input_tokens: int = 0 + error: str | None = None + + @property + def tpot_ms(self) -> float | None: + """Inter-token latency: decode time divided by tokens after the first.""" + if self.ttft_ms is None or self.total_s is None or self.output_tokens < 2: + return None + decode_ms = self.total_s * 1000 - self.ttft_ms + return decode_ms / (self.output_tokens - 1) + + +async def _one_request( + client: httpx.AsyncClient, + base_url: str, + model: str, + input_tokens: int, + output_tokens: int, + prompt: str | None = None, + pin_output: bool = True, +) -> RequestResult: + nonce = uuid.uuid4().hex[:12] + # A supplied prompt still gets the nonce, because prefix-cache defeat is orthogonal to + # whether the text is representative — reusing one real prompt across a concurrency level + # would let the engine serve most of it from cache and report an inflated number. + content = f"[{nonce}] {prompt}" if prompt else _synthetic_prompt(input_tokens, nonce) + payload = { + "model": model, + "messages": [{"role": "user", "content": content}], + "max_tokens": output_tokens, + "temperature": 0.0, + "stream": True, + "stream_options": {"include_usage": True}, + } + if pin_output: + # vLLM/SGLang extension: force exactly max_tokens of output so runs are + # comparable. Harmless (ignored) on engines that do not support it. + # + # Turned OFF for speculative-decoding measurement. Forcing generation past a natural + # stop produces degenerate continuation, and a draft model's acceptance rate on + # degenerate text is not its acceptance rate on real work — so pinning the length here + # buys comparability at the cost of measuring the wrong thing. + payload["ignore_eos"] = True + + start = time.perf_counter() + ttft: float | None = None + counted = 0 + usage_in = 0 + usage_out = 0 + + try: + async with client.stream( + "POST", f"{base_url}/chat/completions", json=payload + ) as resp: + if resp.status_code != 200: + body = (await resp.aread()).decode()[:200] + return RequestResult(ok=False, error=f"HTTP {resp.status_code}: {body}") + + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:] + if data == "[DONE]": + break + try: + chunk = json.loads(data) + except json.JSONDecodeError: + continue + + if usage := chunk.get("usage"): + usage_in = usage.get("prompt_tokens", 0) + usage_out = usage.get("completion_tokens", 0) + + for choice in chunk.get("choices") or []: + delta = choice.get("delta") or {} + # Engines disagree on where reasoning text lands: vLLM with + # --reasoning-parser emits `reasoning`, others use + # `reasoning_content`. Count all of them -- reasoning tokens + # cost the same decode time and the user waits for them + # either way. Checking only `content` would report TTFT as + # the time to the *answer*, which on a reasoning model is + # thousands of tokens late. + if any( + isinstance(delta.get(k), str) and delta[k] + for k in ("content", "reasoning", "reasoning_content") + ): + # The opening chunk carries role + empty content; it is + # the stream opening, not a token, and is excluded by + # the emptiness check above. + if ttft is None: + ttft = (time.perf_counter() - start) * 1000 + counted += 1 + except Exception as exc: # noqa: BLE001 - report, do not abort the sweep + return RequestResult(ok=False, error=f"{type(exc).__name__}: {exc}") + + total = time.perf_counter() - start + return RequestResult( + ok=True, + ttft_ms=ttft, + total_s=total, + # Prefer server-reported usage; fall back to counted chunks. + output_tokens=usage_out or counted, + input_tokens=usage_in, + ) + + +async def _sweep_point( + base_url: str, + model: str, + concurrency: int, + input_tokens: int, + output_tokens: int, + timeout_s: float, + prompts: list[str] | None = None, + pin_output: bool = True, +) -> PerfPoint: + limits = httpx.Limits(max_connections=concurrency + 8) + async with httpx.AsyncClient(timeout=timeout_s, limits=limits) as client: + # Warm up so the first measured request does not absorb graph capture, + # weight paging, or connection setup. + await _one_request(client, base_url, model, 32, 8) + + # Bracket the measured window. These are lifetime counters, so the rate for THIS point + # is the delta — reading the total would fold in warm-up and every earlier point. + before = await read_counters(client, base_url) + + start = time.perf_counter() + results = await asyncio.gather( + *( + _one_request( + client, + base_url, + model, + input_tokens, + output_tokens, + # Cycle rather than repeat: every concurrent request in a point gets a + # different prompt, so one unusually easy or hard example cannot set the + # whole level's number. + prompt=prompts[i % len(prompts)] if prompts else None, + pin_output=pin_output, + ) + for i in range(concurrency) + ) + ) + wall = time.perf_counter() - start + after = await read_counters(client, base_url) + + ok = [r for r in results if r.ok] + failed = [r for r in results if not r.ok] + + total_out = sum(r.output_tokens for r in ok) + ttfts = [r.ttft_ms for r in ok if r.ttft_ms is not None] + tpots = [r.tpot_ms for r in ok if r.tpot_ms is not None] + per_stream = [ + r.output_tokens / r.total_s for r in ok if r.total_s and r.output_tokens + ] + + def pct(values: list[float], p: float) -> float | None: + if not values: + return None + s = sorted(values) + idx = min(len(s) - 1, round(p * (len(s) - 1))) + return round(s[idx], 2) + + # Prefill rate: input tokens processed per second, inferred from TTFT. + # At concurrency > 1 this is per-stream and includes queueing, so it reads + # low; treat the concurrency-1 value as the true prefill capability. + prefill = None + if ttfts and ok: + reported = [r.input_tokens for r in ok if r.input_tokens] + mean_in = statistics.mean(reported) if reported else input_tokens + mean_ttft_s = statistics.mean(ttfts) / 1000 + if mean_ttft_s > 0: + prefill = round(mean_in / mean_ttft_s, 1) + + # Delta across the measured window only. `None` when the engine exposes no counters, which + # is an ordinary outcome (no speculative decoding, or metrics disabled) and not an error. + spec = (after - before) if (before and after) else None + acceptance = spec.acceptance_rate if spec else None + drafted = int(spec.draft) if spec and spec.draft > 0 else None + + # With a natural stop, requests no longer emit identical token counts, so the spread is + # itself a caveat on comparability and has to travel with the number. + out_counts = [r.output_tokens for r in ok if r.output_tokens] + stdev = ( + round(statistics.stdev(out_counts), 2) + if not pin_output and len(out_counts) > 1 + else None + ) + + return PerfPoint( + concurrency=concurrency, + input_tokens=input_tokens, + output_tokens=output_tokens, + n_requests=concurrency, + completed=len(ok), + failed=len(failed), + duration_s=round(wall, 3), + output_tps_total=round(total_out / wall, 2) if wall else 0.0, + output_tps_per_stream=round(statistics.mean(per_stream), 2) if per_stream else 0.0, + ttft_p50_ms=pct(ttfts, 0.50), + ttft_p95_ms=pct(ttfts, 0.95), + tpot_p50_ms=pct(tpots, 0.50), + prefill_tps=prefill, + spec_acceptance_rate=acceptance, + spec_draft_tokens=drafted, + natural_stop=not pin_output, + output_token_stdev=stdev, + error=failed[0].error if failed else None, + ) + + +async def run_sweep( + base_url: str, + model: str, + concurrencies: tuple[int, ...] = (1, 8, 32), + input_tokens: int = 1024, + output_tokens: int = 256, + timeout_s: float = 600.0, + engine: str = "unknown", + progress: bool = True, + prompts: list[str] | None = None, + pin_output: bool = True, +) -> PerfResult: + """Run a concurrency sweep and return a PerfResult. + + `prompts` and `pin_output=False` together give the mode a speculative-decoding target needs: + representative text, stopping naturally. Both defaults stay as they were, so an ordinary + throughput sweep is unchanged and remains directly comparable to every result already + committed. + """ + points: list[PerfPoint] = [] + for c in concurrencies: + if progress: + print(f" concurrency {c:>3} ...", end="", flush=True) + point = await _sweep_point( + base_url, model, c, input_tokens, output_tokens, timeout_s, + prompts=prompts, pin_output=pin_output, + ) + points.append(point) + if progress: + status = f"{point.output_tps_total:>8.1f} tok/s total" + if point.spec_acceptance_rate is not None: + status += f" accept {point.spec_acceptance_rate:.0%}" + status += f" | {point.output_tps_per_stream:>6.1f} /stream" + if point.ttft_p50_ms is not None: + status += f" | TTFT p50 {point.ttft_p50_ms:>7.0f}ms" + if point.failed: + status += f" | {point.failed} FAILED" + print(status, flush=True) + + return PerfResult(engine=engine, points=points) diff --git a/kbench/registry/__init__.py b/kbench/registry/__init__.py new file mode 100644 index 0000000..a28cd0b --- /dev/null +++ b/kbench/registry/__init__.py @@ -0,0 +1,167 @@ +"""Model registry. + +A *target* is not a model. It is a specific way of serving a specific model: + + (model × host × quantization × serving config × checkpoint) + +This is the central design decision of the bench and it cannot be retrofitted +later without invalidating every result already recorded. On unified-memory +boxes like the DGX Spark, serving flags move throughput more than a model swap +does -- `--enforce-eager` alone can cost 30%+, and `--gpu-memory-utilization` +determines how much KV cache exists, which sets the concurrency ceiling. A +registry keyed on model name alone produces numbers nobody can reproduce. + +`checkpoint` is first-class for the same reason: once we fine-tune our own +models, we will compare `karti-7b@step2000` against `karti-7b@step4000` far +more often than we compare Qwen against Llama. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +REGISTRY_PATH = Path(__file__).parent / "models.yaml" + + +@dataclass(frozen=True) +class Host: + """A machine that can serve models.""" + + id: str + ssh: str | None = None + hardware: str | None = None + memory_gb: int | None = None + memory_bandwidth_gbs: int | None = None + notes: str | None = None + + +@dataclass(frozen=True) +class Serving: + """How a target is served. Part of the identity of a result.""" + + engine: str # vllm | sglang | llama.cpp | anthropic | openai | ... + model_name: str # value sent as "model" in the API request + base_url: str | None = None + service: str | None = None # inspect openai-api service prefix + max_model_len: int | None = None + flags: dict[str, Any] = field(default_factory=dict) + # How to sample this model during quality runs. Part of the identity of a result: the + # same model at temperature 1 and at temperature 0.6 is two different measurements. + # + # Not a global constant, because the right value is a property of the model. Greedy + # decoding looks like the obvious choice for reproducibility and is actively wrong for + # reasoning models -- Qwen3 in thinking mode degenerates into repetition at + # temperature 0, and a 2-sample probe that took two minutes ran past twelve without + # terminating. Reproducibility comes from pinning the seed, not from killing entropy. + sampling: dict[str, Any] = field(default_factory=dict) + + @property + def is_local(self) -> bool: + return self.engine in {"vllm", "sglang", "llama.cpp", "ollama"} + + +@dataclass(frozen=True) +class Target: + """One benchmarkable configuration.""" + + id: str + display_name: str + serving: Serving + host: str + family: str | None = None + params_b: float | None = None + active_params_b: float | None = None # MoE: params active per token + quant: str | None = None + checkpoint: str | None = None + tier: str = "candidate" # production | candidate | reference | baseline + aliases: list[str] = field(default_factory=list) + notes: str | None = None + + @property + def inspect_model(self) -> str: + """The model string Inspect uses to address this target.""" + s = self.serving + if s.engine in {"anthropic", "openai", "google", "openrouter", "grok"}: + return f"{s.engine}/{s.model_name}" + if s.is_local: + if not s.service: + raise ValueError(f"target {self.id}: local serving requires a 'service'") + # Inspect convention: openai-api//, with base_url and + # api key read from _BASE_URL / _API_KEY. + return f"openai-api/{s.service}/{s.model_name}" + raise ValueError(f"target {self.id}: unsupported engine {s.engine!r}") + + @property + def env(self) -> dict[str, str]: + """Environment variables Inspect needs to reach this target.""" + s = self.serving + if not s.is_local or not s.service: + return {} + prefix = s.service.upper().replace("-", "_") + env = {f"{prefix}_API_KEY": os.environ.get(f"{prefix}_API_KEY", "no-key-required")} + if s.base_url: + env[f"{prefix}_BASE_URL"] = s.base_url + return env + + def apply_env(self) -> None: + """Export this target's env vars into the current process.""" + os.environ.update(self.env) + + @property + def slug(self) -> str: + """Filesystem-safe identifier used in results filenames.""" + return self.id.replace("/", "_").replace("@", "__") + + +@dataclass(frozen=True) +class Registry: + hosts: dict[str, Host] + targets: dict[str, Target] + + def target(self, ident: str) -> Target: + """Look up a target by id or alias.""" + if ident in self.targets: + return self.targets[ident] + for t in self.targets.values(): + if ident in t.aliases: + return t + known = ", ".join(sorted(self.targets)) + raise KeyError(f"unknown target {ident!r}. known targets: {known}") + + def host(self, ident: str) -> Host: + if ident not in self.hosts: + raise KeyError(f"unknown host {ident!r}") + return self.hosts[ident] + + def by_tier(self, tier: str) -> list[Target]: + return [t for t in self.targets.values() if t.tier == tier] + + +def load_registry(path: Path | None = None) -> Registry: + raw = yaml.safe_load((path or REGISTRY_PATH).read_text()) + + hosts = { + hid: Host(id=hid, **(spec or {})) + for hid, spec in (raw.get("hosts") or {}).items() + } + + targets: dict[str, Target] = {} + for spec in raw.get("targets") or []: + spec = dict(spec) + serving = Serving(**spec.pop("serving")) + target = Target(serving=serving, **spec) + if target.host not in hosts: + raise ValueError(f"target {target.id}: unknown host {target.host!r}") + if target.id in targets: + raise ValueError(f"duplicate target id {target.id!r}") + targets[target.id] = target + + return Registry(hosts=hosts, targets=targets) + + +__all__ = ["Host", "Registry", "Serving", "Target", "load_registry"] diff --git a/kbench/registry/models.yaml b/kbench/registry/models.yaml new file mode 100644 index 0000000..6561d62 --- /dev/null +++ b/kbench/registry/models.yaml @@ -0,0 +1,109 @@ +# Lumbridge Bench target registry +# +# A target = (model x host x quant x serving config x checkpoint). +# See kbench/registry/__init__.py for why the key is this wide. +# +# `serving.flags` records the ACTUAL flags the server was started with. Snapshot +# them from a live host with: kbench snapshot +# Results are only comparable between targets with identical flags. + +hosts: + your-node: + ssh: your-node + hardware: NVIDIA GB10 Grace Blackwell (DGX Spark) + memory_gb: 121 + memory_bandwidth_gbs: 273 + notes: > + Unified LPDDR5X. Memory bandwidth is the binding constraint on decode + throughput, not compute -- large dense models are bandwidth-starved here + while MoE models with small active-param counts do comparatively well. + + metal: + ssh: your-second-node + hardware: TBD + notes: Secondary inference box. Not yet benchmarked. + + your-second-node: + ssh: root@your-second-node + hardware: TBD + notes: Storage origin box, 761GB. Not yet benchmarked. + + api: + hardware: hosted + notes: Frontier APIs. Used as quality ceiling and as judge models. + +targets: + # --------------------------------------------------------------------- + # PRODUCTION -- currently serving live traffic, so regressions here matter + # --------------------------------------------------------------------- + - id: qwen3.6-35b-a3b-nvfp4@your-node + display_name: Qwen3.6-35B-A3B (NVFP4) + family: qwen3.6 + params_b: 35 + active_params_b: 3 + quant: nvfp4 + host: your-node + tier: production + aliases: [brain, spark] + notes: > + Live behind the `brain` and `local-moe` aliases. This is the + model the bench exists to interrogate: it was deployed without + measurement. + + Flags below were snapshotted from the live process on 2026-08-03 and had + drifted badly from what was recorded here: gpu_memory_utilization was + 0.25 not 0.55, the MoE backend was flashinfer_cutlass not flashinfer_b12x, + and MTP speculative decoding was running but undocumented. Since target + identity IS the serving config, score cards taken against the old entry + would have been mislabeled. Re-snapshot before trusting a comparison. + + FLAGS OF CONCERN, in the order worth testing: + (1) --max-num-seqs 4 caps concurrency at four sequences, so a throughput + sweep past 4 measures queueing, not the server. + (2) --gpu-memory-utilization 0.25 leaves ~75% of unified memory unused, + capping KV cache and therefore concurrency. + (3) --enforce-eager disables CUDA graphs, which costs throughput. + (4) MTP is on; report spec_acceptance_rate, since a low acceptance rate + means the draft tokens are wasted work. + serving: + engine: vllm + service: spark + base_url: http://your-node:8001/v1 + model_name: brain + max_model_len: 65536 + # Qwen3's documented values for thinking mode. Do NOT set temperature 0 here: this + # model degenerates into endless repetition under greedy decoding, and a two-sample + # probe that normally finishes in two minutes ran past twelve without terminating. + # The seed is what makes a re-run a re-run. + sampling: + temperature: 0.6 + top_p: 0.95 + top_k: 20 + seed: 20260804 + max_tokens: 4096 + flags: + async_scheduling: true + enforce_eager: true + gpu_memory_utilization: 0.25 + kv_cache_dtype: fp8 + max_num_batched_tokens: 4096 + max_num_seqs: 4 + moe_backend: flashinfer_cutlass + reasoning_parser: qwen3 + speculative_config: '{"method":"qwen3_5_mtp","num_speculative_tokens":2}' + tool_call_parser: qwen3_coder + enable_auto_tool_choice: true + + # --------------------------------------------------------------------- + # BASELINE -- frontier APIs. Quality ceiling, and the judge for graded tasks. + # Uncomment once the corresponding API key is in .env. + # --------------------------------------------------------------------- + # - id: claude-opus-5@api + # display_name: Claude Opus 5 + # family: claude + # host: api + # tier: baseline + # aliases: [opus] + # serving: + # engine: anthropic + # model_name: claude-opus-5 diff --git a/kbench/results.py b/kbench/results.py new file mode 100644 index 0000000..ea9ad69 --- /dev/null +++ b/kbench/results.py @@ -0,0 +1,254 @@ +"""Result schema. + +One run produces one JSON file in results/, committed to git. The commit +history IS the longitudinal record -- there is no database to migrate or rot. + +Two properties of this schema are load-bearing and expensive to retrofit: + +1. PER-SAMPLE OUTCOMES, not just aggregates. An aggregate tells you a + checkpoint got worse. Per-sample diffs tell you *which capability broke*. + That difference is what separates a leaderboard from a debugging tool, and + it is the entire reason this bench is useful for fine-tuning. + +2. THE FULL TARGET IS SNAPSHOTTED INTO THE RESULT. Registry entries change. + A result that merely references a target id becomes unreproducible the + first time someone edits models.yaml. Every run carries its own copy of + the serving flags it actually ran against. + +Output storage policy: we store the score plus a short excerpt per sample. +Full model outputs stay in the raw Inspect .eval log, which is gitignored -- +they are large, and for signal-tier tasks they can echo sensitive content +from the prompts they were derived from. Deep debugging uses the local log; +git carries the shape of the failure, not the whole transcript. +""" + +from __future__ import annotations + +import json +import platform +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = 1 +RESULTS_DIR = Path(__file__).resolve().parent.parent / "results" + + +def _json_safe(value: Any) -> Any: + """Map non-finite floats to None, recursively. + + Python's json emits bare `NaN`/`Infinity`, which its own loader accepts and almost + nothing else does — JSON.parse, Go, and serde all reject them. A score card is meant + to be a portable, durable record, so a card that only Python can read is a broken one. + An ungraded sample surfaced as NaN here and made the first committed card unparseable + by the website that has to render it. + + None means "no score", which is exactly what a non-finite score meant anyway. + """ + if isinstance(value, float): + # NaN != NaN, and inf fails the range check; both are unrepresentable in JSON. + return None if value != value or value in (float("inf"), float("-inf")) else value + if isinstance(value, dict): + return {k: _json_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(v) for v in value] + return value + +# How much of each model output to keep in the committed record. +EXCERPT_CHARS = 400 + + +def utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="seconds") + + +@dataclass +class SampleOutcome: + """One sample's result. The unit of regression debugging.""" + + sample_id: str + score: float + passed: bool + excerpt: str | None = None # truncated model output; full text in .eval log + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class QualityResult: + """One task's scores against one target.""" + + task: str + tier: str # "reference" (public, contaminated, calibration only) | "signal" + dataset_version: str | None = None + # Content hash of the samples actually run. dataset_version is inspect's static + # task_version and does not move when the data changes; this does. Two cards are + # only comparable if their fingerprints match. + dataset_fingerprint: str | None = None + n_samples: int = 0 + metrics: dict[str, float] = field(default_factory=dict) + samples: list[SampleOutcome] = field(default_factory=list) + duration_s: float | None = None + error: str | None = None + + +@dataclass +class PerfPoint: + """Serving performance at one concurrency level.""" + + concurrency: int + input_tokens: int + output_tokens: int + n_requests: int + completed: int + failed: int + duration_s: float + # Aggregate throughput across all streams -- the number that matters for + # multi-user serving, and the one single-stream reviews miss entirely. + output_tps_total: float + # Per-stream decode rate -- what a single user actually feels. + output_tps_per_stream: float + ttft_p50_ms: float | None = None + ttft_p95_ms: float | None = None + tpot_p50_ms: float | None = None # time per output token, inter-token latency + prefill_tps: float | None = None + # Speculative decoding, when the engine reports it. Throughput alone cannot say whether + # drafting is working: a draft rejected almost every time still emits tokens, having paid + # for both passes. `None` means the engine exposed no counters, not that the rate was zero. + spec_acceptance_rate: float | None = None + spec_draft_tokens: int | None = None + # True when output length was NOT pinned with ignore_eos. Comparability across points is + # weaker in that mode, and a reader has to know which they are looking at. + natural_stop: bool = False + output_token_stdev: float | None = None + error: str | None = None + + +@dataclass +class PerfResult: + engine: str + points: list[PerfPoint] = field(default_factory=list) + peak_memory_gb: float | None = None + idle_memory_gb: float | None = None + notes: str | None = None + + +@dataclass +class Run: + """A complete score card: one target, one point in time.""" + + run_id: str + timestamp: str + target_id: str + target: dict[str, Any] # full snapshot -- see module docstring + host: dict[str, Any] + quality: list[QualityResult] = field(default_factory=list) + perf: PerfResult | None = None + verdict: dict[str, Any] = field(default_factory=dict) + runner: dict[str, Any] = field(default_factory=dict) + # How the model was sampled. Two cards are only comparable if this matches: a score + # taken at temperature 1 is a draw from a distribution, and differs run to run by more + # than most real regressions do. Recorded rather than assumed, because the quality half + # sampled unpinned for its entire life and nothing in the card said so. + sampling: dict[str, Any] = field(default_factory=dict) + schema_version: int = SCHEMA_VERSION + + # ---------------------------------------------------------------- io -- + + @property + def filename(self) -> str: + date = self.timestamp[:10] + slug = self.target.get("slug") or self.target_id.replace("@", "__") + return f"{date}__{slug}__{self.run_id[:8]}.json" + + def to_dict(self) -> dict[str, Any]: + return _json_safe(asdict(self)) + + def save(self, results_dir: Path | None = None) -> Path: + d = results_dir or RESULTS_DIR + d.mkdir(parents=True, exist_ok=True) + path = d / self.filename + # allow_nan=False so this raises instead of emitting a token that is not JSON. + # to_dict has already mapped the non-finite values to null; if a new one appears, + # the write should fail loudly rather than commit an unparseable score card. + path.write_text( + json.dumps(self.to_dict(), indent=2, sort_keys=False, allow_nan=False) + "\n" + ) + return path + + @staticmethod + def load(path: Path) -> dict[str, Any]: + return json.loads(Path(path).read_text()) + + +def runner_info() -> dict[str, Any]: + """Provenance of the machine that drove the benchmark (not the one serving).""" + return { + "host": platform.node(), + "python": platform.python_version(), + "platform": platform.platform(), + } + + +def truncate(text: str | None, limit: int = EXCERPT_CHARS) -> str | None: + if text is None: + return None + text = text.strip() + return text if len(text) <= limit else text[:limit] + f"... [+{len(text) - limit} chars]" + + +def compute_verdict(quality: list[QualityResult], perf: PerfResult | None) -> dict[str, Any]: + """Derive a self-hosting verdict from the raw numbers. + + Deliberately simple and explicit rather than clever. The thresholds encode + what makes a model usable in our own stack, and they live in code (and + therefore in git history) so that a changed verdict is always traceable to + a changed rule rather than to drift. + """ + verdict: dict[str, Any] = {} + + signal = [q for q in quality if q.tier == "signal" and q.error is None] + reference = [q for q in quality if q.tier == "reference" and q.error is None] + + def mean_accuracy(results: list[QualityResult]) -> float | None: + vals = [q.metrics.get("accuracy") for q in results] + vals = [v for v in vals if v is not None] + return round(sum(vals) / len(vals), 4) if vals else None + + verdict["signal_score"] = mean_accuracy(signal) + verdict["reference_score"] = mean_accuracy(reference) + + # Contamination gate. A probe nobody reads is the same as no probe, so this does not just + # record a number — when a probe fires it takes `signal_score` away. + # + # Nulling it is deliberate. Anything downstream that reads `verdict["signal_score"]` — + # `kbench results`, a comparison, a published card — would otherwise happily render a + # memorised score next to a small warning, and the number is what people quote. The raw + # value is preserved under a name nobody reads by accident, so nothing is lost for + # forensics and nothing is quotable by mistake. + canary = [q for q in quality if q.tier == "canary" and q.error is None] + canary_score = mean_accuracy(canary) + verdict["canary_score"] = canary_score + if canary_score is None: + # No probe ran. The signal score stands, but it is unverified, and saying so is the + # difference between "clean" and "nobody checked". + verdict["contamination"] = "unverified" + elif canary_score > 0: + verdict["contamination"] = "detected" + verdict["signal_score_unverified"] = verdict["signal_score"] + verdict["signal_score"] = None + else: + verdict["contamination"] = "clean" + + if perf and perf.points: + single = next((p for p in perf.points if p.concurrency == 1), None) + best = max(perf.points, key=lambda p: p.output_tps_total) + if single: + verdict["single_stream_tps"] = round(single.output_tps_per_stream, 1) + # Below ~15 tok/s a model reads slower than a person, which rules + # out anything interactive regardless of how good its answers are. + verdict["interactive_viable"] = single.output_tps_per_stream >= 15.0 + verdict["peak_throughput_tps"] = round(best.output_tps_total, 1) + verdict["peak_throughput_concurrency"] = best.concurrency + + return verdict diff --git a/kbench/run.py b/kbench/run.py new file mode 100644 index 0000000..fc659c9 --- /dev/null +++ b/kbench/run.py @@ -0,0 +1,358 @@ +"""Run orchestration: target + tasks + perf sweep -> one committed score card.""" + +from __future__ import annotations + +import asyncio +import hashlib +import uuid +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from . import tasks as task_catalog +from .perf import run_sweep +from .registry import Registry, Target +from .results import ( + PerfResult, + QualityResult, + Run, + SampleOutcome, + compute_verdict, + runner_info, + truncate, + utc_now, +) + +LOG_DIR = Path(__file__).resolve().parent.parent / "logs" + +# Inspect's canonical scalar score values. +_CORRECTNESS = {"C": 1.0, "I": 0.0, "P": 0.5, "N": 0.0} + + +def _normalize_sample_score(value: Any, primary_key: str | None) -> tuple[float, bool, dict]: + """Reduce a score of any shape to (score, passed, extra_metadata). + + Inspect scorers return scalars, correctness letters, or dicts depending on + the eval. IFEval returns a dict; a simple match scorer returns "C"/"I". + Normalizing here keeps the results schema uniform, which is what makes + cross-task and cross-checkpoint comparison possible at all. + """ + extra: dict[str, Any] = {} + + if isinstance(value, dict): + extra = dict(value) + if primary_key and primary_key in value: + inner = value[primary_key] + else: + # Fall back to the first bool/numeric entry so an unconfigured task + # still produces something rather than silently scoring zero. + inner = next( + (v for v in value.values() if isinstance(v, (bool, int, float))), + 0.0, + ) + return _normalize_sample_score(inner, None)[0], bool(inner), extra + + if isinstance(value, bool): + return (1.0 if value else 0.0), value, extra + + if isinstance(value, (int, float)): + # A non-finite score means the sample was never graded — a rubric sample with no + # judge configured arrives as NaN. `NaN > 0.0` is False, so it used to be recorded + # as an ordinary failure, which is a different claim: "the model got this wrong" + # rather than "nobody scored this". The mean already skips it; this makes the + # per-sample record say so too. + if isinstance(value, float) and value != value: + return value, False, {**extra, "ungraded": True} + return float(value), float(value) > 0.0, extra + + if isinstance(value, str): + score = _CORRECTNESS.get(value.strip().upper()[:1], 0.0) + return score, score >= 1.0, {"raw": value} + + return 0.0, False, {"raw": repr(value)} + + +def _quality_from_log(log: Any, spec: task_catalog.TaskSpec) -> QualityResult: + """Extract a QualityResult (including per-sample outcomes) from an EvalLog.""" + if log.status != "success": + err = getattr(log, "error", None) + return QualityResult( + task=spec.name, + tier=spec.tier, + error=str(err) if err else f"eval status: {log.status}", + ) + + metrics: dict[str, float] = {} + for scorer in (log.results.scores if log.results else []): + for key, metric in scorer.metrics.items(): + metrics[key] = round(float(metric.value), 6) + + # Promote the task's headline number to a uniform "accuracy" key so the + # leaderboard and verdict logic never need per-task special cases. + if spec.primary_metric in metrics: + metrics["accuracy"] = metrics[spec.primary_metric] + + samples: list[SampleOutcome] = [] + for s in (log.samples or []): + for scorer_name, score in (s.scores or {}).items(): + value, passed, extra = _normalize_sample_score( + score.value, spec.primary_sample_key + ) + samples.append( + SampleOutcome( + sample_id=str(s.id), + score=value, + passed=passed, + excerpt=truncate(getattr(s.output, "completion", None)), + metadata={ + "scorer": scorer_name, + "epoch": s.epoch, + **extra, + }, + ) + ) + + stats = getattr(log, "stats", None) + duration = None + if stats and getattr(stats, "started_at", None) and getattr(stats, "completed_at", None): + try: + from datetime import datetime + + duration = ( + datetime.fromisoformat(stats.completed_at) + - datetime.fromisoformat(stats.started_at) + ).total_seconds() + except Exception: # noqa: BLE001 - duration is nice-to-have, never fatal + duration = None + + return QualityResult( + task=spec.name, + tier=spec.tier, + dataset_version=getattr(log.eval, "task_version", None) and str(log.eval.task_version), + dataset_fingerprint=_dataset_fingerprint(log.samples or []), + n_samples=len(log.samples or []), + metrics=metrics, + samples=samples, + duration_s=duration, + ) + + +def _dataset_fingerprint(samples: list[Any]) -> str | None: + """Content hash of the samples actually run. + + `dataset_version` is inspect's static task_version -- it stays "2" whether the family + has twelve samples or a rewritten scorer. Editing a prompt, tightening a regex, or + dropping a sample therefore produced two cards that claimed the same dataset version + and were not comparable, with nothing to reveal it. Comparing those is the exact error + this bench exists to prevent, so the identity has to come from the content. + + Hashed over (id, input, target) because those are what determine whether a score means + the same thing. Sorted, so sample ordering does not change the fingerprint. + """ + if not samples: + return None + h = hashlib.sha256() + for key in sorted( + f"{s.id}\x1f{s.input}\x1f{s.target}" for s in samples + ): + h.update(key.encode()) + h.update(b"\x1e") + return f"sha256:{h.hexdigest()[:16]}" + + +def preflight(specs: list[task_catalog.TaskSpec]) -> None: + """Fetch scorer prerequisites before burning GPU time on a doomed run. + + Scorers often touch their corpora only at scoring time, so a missing + dependency surfaces after the whole dataset has been generated. Checking + up front converts a 20-minute failure into a 2-second one. + """ + needed = {res for spec in specs for res in spec.nltk_resources} + if not needed: + return + + import nltk + + for resource in sorted(needed): + try: + nltk.data.find(resource) + except LookupError: + name = resource.rsplit("/", 1)[-1] + print(f"[preflight] fetching nltk resource {name}") + nltk.download(name, quiet=True) + nltk.data.find(resource) # raise loudly if it still is not there + + +# Fallback sampling for a target that declares none. Deliberately NOT greedy: temperature 0 +# looks like the reproducible choice and breaks reasoning models, which repeat forever +# without it. Reproducibility comes from the fixed seed. A target should override this in +# the registry with the values its model card documents. +DEFAULT_SAMPLING = { + "temperature": 0.6, + "top_p": 0.95, + "seed": 20260804, + "max_tokens": 4096, +} + + +def resolve_sampling(target: Target) -> dict[str, Any]: + """The sampling settings for a target, registry first, defaults second. + + Returned rather than read inline so the score card records exactly what the run used: + the two cannot drift, because they are the same call. + """ + return {**DEFAULT_SAMPLING, **(target.serving.sampling or {})} + + +def run_quality( + target: Target, + specs: list[task_catalog.TaskSpec], + limit: int | None = None, + max_connections: int = 16, + log_dir: Path | None = None, + epochs: int = 1, +) -> list[QualityResult]: + """Run quality tasks against a target via Inspect.""" + from inspect_ai import eval as inspect_eval + + preflight(specs) + target.apply_env() + results: list[QualityResult] = [] + sampling = resolve_sampling(target) + print(f" sampling: {sampling} | epochs: {epochs}") + if epochs == 1: + print( + " NOTE: one epoch. Sampling is not reproducible on this server even with a\n" + " fixed seed -- continuous batching changes the logits -- so a single\n" + " epoch is one draw, not a rate. Use --epochs for a signal-tier number." + ) + + for spec in specs: + print(f"\n[quality] {spec.name} ({spec.tier}) -> {target.id}") + logs = inspect_eval( + tasks=spec.inspect_task, + model=target.inspect_model, + limit=limit, + epochs=epochs, + log_dir=str(log_dir or LOG_DIR), + display="plain", + # Greedy decoding. The perf harness has always pinned temperature 0; the + # quality half pinned nothing and inherited the server's default, so it + # sampled. Three runs over the same twelve samples scored 4, 5 and 7 of 11 + # -- 27% of samples flipped -- and a score card that cannot reproduce its + # own number is not evidence, it is a draw from a distribution nobody + # recorded. Sampling also makes `compare` meaningless: a per-sample flip + # caused by temperature is indistinguishable from a real regression. + # + # These MUST go through config=GenerateConfig. inspect_eval takes **kwargs, + # so passing temperature= directly is accepted silently and does nothing -- + # the run would have looked pinned and still sampled. + # Loose kwargs ARE the interface: inspect_eval collects everything it does + # not consume into a GenerateConfig itself, so `config=` is rejected as an + # unknown field and `max_connections` must ride along here rather than as a + # sibling argument. Verified by running it, not by reading the signature -- + # these names do not appear in it. + max_connections=max_connections, + **sampling, + ) + for log in logs: + qr = _quality_from_log(log, spec) + # Record the cap explicitly. A limited run that looks like a full + # run is the single easiest way to publish a misleading number. + if limit is not None: + qr.metrics["_limit"] = float(limit) + results.append(qr) + + return results + + +def run_perf( + target: Target, + concurrencies: tuple[int, ...] = (1, 8, 32), + input_tokens: int = 1024, + output_tokens: int = 256, + prompts: list[str] | None = None, + pin_output: bool = True, +) -> PerfResult: + """Run a serving performance sweep against a target.""" + base_url = target.serving.base_url + if not base_url: + raise ValueError(f"target {target.id} has no base_url; cannot run perf sweep") + + print(f"\n[perf] {target.id} @ {base_url}") + + # A sweep past the server's own concurrency ceiling measures the queue, not the server. + # vLLM admits max_num_seqs sequences per step and queues the rest, so those points show + # flat total throughput and inflating TTFT — which reads as saturation and is really just + # waiting. your-node runs max_num_seqs=4 against a default sweep of 1,8,32, so two of three + # points were destined to be misread. Warn rather than refuse: measuring the queue is a + # legitimate thing to want, as long as nobody mistakes it for the engine's limit. + max_seqs = (target.serving.flags or {}).get("max_num_seqs") + if isinstance(max_seqs, int): + beyond = [c for c in concurrencies if c > max_seqs] + if beyond: + print( + f" WARNING: max_num_seqs={max_seqs}, so concurrency {beyond} exceeds what " + f"the server admits per step.\n" + f" Those points measure queueing, not serving capacity. " + f"Consider --concurrency 1,{max(2, max_seqs // 2)},{max_seqs}." + ) + + mode = [] + if prompts: + mode.append(f"{len(prompts)} representative prompts") + if not pin_output: + mode.append("natural stop (output length not pinned)") + print(f" input={input_tokens} output={output_tokens} tokens" + + (f" [{'; '.join(mode)}]" if mode else "")) + return asyncio.run( + run_sweep( + base_url=base_url, + model=target.serving.model_name, + concurrencies=concurrencies, + input_tokens=input_tokens, + output_tokens=output_tokens, + engine=target.serving.engine, + prompts=prompts, + pin_output=pin_output, + ) + ) + + +def _epochs_of(quality: list[QualityResult]) -> int: + """How many times each sample was actually run, read back from the outcomes. + + Taken from the results rather than the argument so the card cannot claim an averaging + it did not do -- the same reason resolve_sampling feeds the run and the card from one + call. + """ + per_sample: dict[str, int] = {} + for q in quality: + for s in q.samples: + per_sample[s.sample_id] = per_sample.get(s.sample_id, 0) + 1 + return max(per_sample.values()) if per_sample else 1 + + +def build_run( + registry: Registry, + target: Target, + quality: list[QualityResult], + perf: PerfResult | None, +) -> Run: + host = registry.host(target.host) + target_snapshot = asdict(target) + target_snapshot["slug"] = target.slug + target_snapshot["inspect_model"] = target.inspect_model + + return Run( + run_id=uuid.uuid4().hex, + timestamp=utc_now(), + target_id=target.id, + target=target_snapshot, + host=asdict(host), + quality=quality, + perf=perf, + verdict=compute_verdict(quality, perf), + runner=runner_info(), + sampling={**resolve_sampling(target), "epochs": _epochs_of(quality)}, + ) diff --git a/kbench/schema.py b/kbench/schema.py new file mode 100644 index 0000000..05927a3 --- /dev/null +++ b/kbench/schema.py @@ -0,0 +1,62 @@ +"""The sample schema and its validation. Standard library only, on purpose. + +This used to live in `kbench/tasks/signal.py`, which imports `inspect_ai` at module scope — +so checking whether a JSONL record was well-formed required the whole eval framework to be +installed. That is backwards: the schema is ours and the runner is swappable, so the thing +that defines what a sample *is* must not depend on the thing that happens to execute it. + +Practically it means authoring tools, CI checks and editors can validate data without +resolving a heavy dependency tree, and that swapping the execution backend later touches +`tasks/`, not this file. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +DATA_DIR = Path(__file__).resolve().parent.parent / "data" + +VALID_SPLITS = {"train", "dev", "test"} +VALID_SCORERS = {"exact", "includes", "regex", "rubric"} + + +def family_path(family: str, tier: str = "private") -> Path: + return DATA_DIR / tier / f"{family}.jsonl" + + +def validate_record(rec: dict[str, Any], where: str) -> None: + """Fail loudly on a malformed sample. + + A silently-skipped sample shrinks the eval set without changing the score's appearance, + which is the worst possible failure mode for a benchmark. + """ + for field in ("id", "input", "split", "scorer"): + if not rec.get(field): + raise ValueError(f"{where}: missing required field {field!r}") + if rec["split"] not in VALID_SPLITS: + raise ValueError(f"{where}: split must be one of {sorted(VALID_SPLITS)}") + if rec["scorer"] not in VALID_SCORERS: + raise ValueError(f"{where}: scorer must be one of {sorted(VALID_SCORERS)}") + if rec["scorer"] != "rubric" and not rec.get("target"): + raise ValueError(f"{where}: scorer {rec['scorer']!r} requires a target") + if rec["scorer"] == "rubric" and not (rec.get("rubric") or rec.get("target")): + raise ValueError(f"{where}: rubric scorer requires a rubric or a target") + + +def read_records(path: Path) -> list[tuple[int, dict[str, Any]]]: + """Parse and validate a family file. Returns (line number, record) pairs. + + Line numbers are carried so an error names the line a human has to open. + """ + import json + + out: list[tuple[int, dict[str, Any]]] = [] + for lineno, line in enumerate(path.read_text().splitlines(), start=1): + line = line.strip() + if not line or line.startswith("//"): + continue + rec = json.loads(line) + validate_record(rec, f"{path.name}:{lineno}") + out.append((lineno, rec)) + return out diff --git a/kbench/scorers/__init__.py b/kbench/scorers/__init__.py new file mode 100644 index 0000000..aa8824c --- /dev/null +++ b/kbench/scorers/__init__.py @@ -0,0 +1,95 @@ +"""Scorers for signal-tier tasks. + +Inspect attaches scorers to a Task, not to a sample. Our families are mixed -- +one `agent_ops` file legitimately contains samples graded by exact match, by +substring, and by rubric. Splitting a family into one Task per scorer would +fragment the results and make family-level scores meaningless. + +So we dispatch per sample instead, reading `scorer` from sample metadata. +""" + +from __future__ import annotations + +import re + +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + Score, + Scorer, + Target, + accuracy, + model_graded_qa, + scorer, + stderr, +) +from inspect_ai.solver import TaskState + +DEFAULT_RUBRIC = """You are grading a model's answer against a reference. + +Grade CORRECT if the answer is substantively equivalent to the reference -- +matching facts, decisions, and any required steps. Ignore differences in +wording, formatting, or verbosity. + +Grade INCORRECT if it contradicts the reference, omits something the reference +treats as required, or adds a claim the reference does not support. + +Be strict. A partially correct answer is INCORRECT.""" + + +def _text(state: TaskState) -> str: + return (state.output.completion or "").strip() + + +@scorer(metrics=[accuracy(), stderr()]) +def dispatching(judge_model: str | None = None) -> Scorer: + """Grade each sample by the method named in its own metadata. + + Falls back to `includes` when unspecified, which is the most forgiving + deterministic option -- a sample that silently scores zero because of a + missing metadata field would look like a model failure. + """ + graded = model_graded_qa(instructions=DEFAULT_RUBRIC, model=judge_model) + + async def score(state: TaskState, target: Target) -> Score: + method = (state.metadata or {}).get("scorer") or "includes" + answer = _text(state) + expected = target.text or "" + + if method == "rubric": + rubric = (state.metadata or {}).get("rubric") + if rubric: + custom = model_graded_qa(instructions=rubric, model=judge_model) + return await custom(state, target) + return await graded(state, target) + + if method == "exact": + ok = answer == expected.strip() + elif method == "regex": + try: + ok = re.search(expected, answer, re.IGNORECASE | re.DOTALL) is not None + except re.error as exc: + return Score( + value=INCORRECT, + answer=answer, + explanation=f"invalid regex in sample target: {exc}", + ) + elif method == "includes": + ok = expected.strip().lower() in answer.lower() + else: + return Score( + value=INCORRECT, + answer=answer, + explanation=f"unknown scorer {method!r} -- fix the sample, not the model", + ) + + return Score( + value=CORRECT if ok else INCORRECT, + answer=answer, + explanation=f"scorer={method}", + ) + + return score + + +__all__ = ["DEFAULT_RUBRIC", "dispatching"] diff --git a/kbench/specdec.py b/kbench/specdec.py new file mode 100644 index 0000000..28e83b8 --- /dev/null +++ b/kbench/specdec.py @@ -0,0 +1,106 @@ +"""Speculative-decoding metrics, read from the engine rather than inferred. + +Throughput alone cannot tell you whether speculative decoding is working. A draft model that +is rejected almost every time still produces tokens — just slowly, having paid for the draft +pass as well as the verify pass — and on a synthetic benchmark that can even look like a +modest win. The number that says whether it is working is the **acceptance rate**: of the +tokens the draft proposed, how many survived verification. + +vLLM exposes this on its Prometheus endpoint as counters, so a rate for one sweep point is a +delta across that point rather than the process-lifetime total. Reading the total instead would +mix in every request since the server started, including warm-up. + +Parsing is pure standard library and `httpx` is only needed to fetch — the import is deferred +so the parser can be tested, and reasoned about, without the HTTP client installed. Same rule as +`kbench/schema.py`: the logic that decides what a number means must not depend on the machinery +that fetches it. + +This is deliberately best-effort. An engine without these counters, or with metrics disabled, +gets `None` and the sweep continues — an absent metric must never fail a benchmark run. + +Why it matters for this bench specifically: `perf.py` sends a random nonce prefix and +`ignore_eos`, both correct for defeating prefix caching and pinning output length, and both +adversarial to speculative decoding. Forced continuation past a natural stop is degenerate +text, and a draft model's acceptance on degenerate text is not its acceptance on real work. So +a speculative target measured with the synthetic prompt needs its acceptance rate reported +beside the throughput, or the throughput will be read as if it transfers. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + import httpx + +# vLLM V1 names. The older V0 engine used vllm:spec_decode_* with the same meaning; both are +# accepted so a mixed fleet does not silently report nothing. +ACCEPTED = ("vllm:spec_decode_num_accepted_tokens_total",) +DRAFT = ("vllm:spec_decode_num_draft_tokens_total",) + +_SAMPLE = re.compile(r"^(?P[a-zA-Z_:][\w:]*)(?P\{[^}]*\})?\s+(?P[-+0-9.eE]+)$") + + +@dataclass(frozen=True) +class SpecCounters: + accepted: float + draft: float + + def __sub__(self, other: "SpecCounters") -> "SpecCounters": + return SpecCounters(self.accepted - other.accepted, self.draft - other.draft) + + @property + def acceptance_rate(self) -> float | None: + """Accepted / proposed over this window. None when nothing was proposed.""" + if self.draft <= 0: + return None + return round(self.accepted / self.draft, 4) + + +def parse_metrics(text: str) -> SpecCounters | None: + """Pull the speculative counters out of a Prometheus exposition payload.""" + totals: dict[str, float] = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + m = _SAMPLE.match(line) + if not m: + continue + name = m.group("name") + if name in ACCEPTED or name in DRAFT: + try: + # Counters are per-label-set (one per model); sum them. A single-model server + # has one series, and summing is still correct there. + totals[name] = totals.get(name, 0.0) + float(m.group("value")) + except ValueError: + continue + + accepted = next((totals[n] for n in ACCEPTED if n in totals), None) + draft = next((totals[n] for n in DRAFT if n in totals), None) + if accepted is None or draft is None: + return None + return SpecCounters(accepted=accepted, draft=draft) + + +def metrics_url(base_url: str) -> str: + """/metrics sits at the server root, not under the OpenAI /v1 prefix.""" + root = base_url.rstrip("/") + for suffix in ("/v1", "/openai/v1"): + if root.endswith(suffix): + root = root[: -len(suffix)] + break + return f"{root}/metrics" + + +async def read_counters(client: "httpx.AsyncClient", base_url: str) -> SpecCounters | None: + """Best-effort read. Any failure means "no data", never an exception into the sweep.""" + try: + resp = await client.get(metrics_url(base_url), timeout=5.0) + if resp.status_code != 200: + return None + return parse_metrics(resp.text) + except Exception: # noqa: BLE001 - a missing metrics endpoint is an ordinary outcome + return None diff --git a/kbench/sweep.py b/kbench/sweep.py new file mode 100644 index 0000000..b3bcb57 --- /dev/null +++ b/kbench/sweep.py @@ -0,0 +1,149 @@ +"""Sweep a matrix of targets and reduce it to a decision. + +`run` answers "what did this target do". `compare` answers "what changed between two". Neither +answers the question people actually start with, which is **which of these should I run**, and +that one is a matrix: the same model at three quantisations, or one quantisation under four sets +of serving flags, all on the box you actually own. + +Three things make this more than a shell loop: + +**It holds the box constant.** A sweep varies configuration and nothing else. Varying the host +too produces numbers with two causes, and the tempting reading — "fp8 is faster" — is then +unsupported by the data that appears to show it. + +**It runs sequentially, and that is not a simplification.** On a unified-memory machine two +targets cannot both be resident; that constraint is the entire premise of the stack this bench +was built for. Running them concurrently would measure contention between them rather than +either one. + +**It reduces to a frontier, not a leaderboard.** Self-hosting is a trade: quality against +throughput, under a memory ceiling. A single ranking hides that, so the output is the set of +targets that are not beaten on both axes at once — plus, explicitly, the ones that are, because +a target that is worse at everything is the one useful thing a matrix can tell you. + +Every point is an ordinary score card saved to `results/`. There is no sweep-shaped result +format, so `compare`, `card` and the git history all keep working unchanged. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class SweepPoint: + """One target's outcome within a sweep.""" + + target_id: str + quality: float | None + throughput_tps: float | None + single_stream_tps: float | None = None + contamination: str | None = None + error: str | None = None + run_path: str | None = None + + @property + def usable(self) -> bool: + """Whether this point can take part in the trade-off at all.""" + return self.error is None and self.throughput_tps is not None + + @property + def quality_comparable(self) -> bool: + """A contaminated or unmeasured quality score cannot be traded against anything.""" + return self.quality is not None and self.contamination != "detected" + + +@dataclass +class SweepResult: + points: list[SweepPoint] = field(default_factory=list) + best: list[str] = field(default_factory=list) + dominated: dict[str, str] = field(default_factory=dict) # target -> what beats it + warnings: list[str] = field(default_factory=list) + + @property + def failed(self) -> list[SweepPoint]: + return [p for p in self.points if p.error] + + +def _dominates(a: SweepPoint, b: SweepPoint, use_quality: bool) -> bool: + """True when `a` is at least as good as `b` everywhere and better somewhere. + + Strict domination is the only claim worth making from a small matrix. "Higher throughput + and slightly lower quality" is a trade the operator has to make with knowledge this tool + does not have — how much quality their workload can spare — so it is deliberately left to + them rather than collapsed into a score. + """ + if not (a.usable and b.usable): + return False + better_anywhere = a.throughput_tps > b.throughput_tps + at_least_equal = a.throughput_tps >= b.throughput_tps + if use_quality and a.quality_comparable and b.quality_comparable: + at_least_equal = at_least_equal and a.quality >= b.quality + better_anywhere = better_anywhere or a.quality > b.quality + return at_least_equal and better_anywhere + + +def analyse(points: list[SweepPoint], hosts: list[str] | None = None) -> SweepResult: + """Reduce a set of measured targets to a frontier and a set of dominated options.""" + result = SweepResult(points=list(points)) + + if hosts and len(set(hosts)) > 1: + result.warnings.append( + f"Targets ran on more than one host ({', '.join(sorted(set(hosts)))}). A sweep is " + "meant to hold the machine constant — these differences have two causes and " + "neither can be isolated." + ) + + usable = [p for p in points if p.usable] + if not usable: + result.warnings.append("No target produced a usable measurement.") + return result + + with_quality = [p for p in usable if p.quality_comparable] + use_quality = len(with_quality) == len(usable) + if not use_quality: + missing = sorted(p.target_id for p in usable if not p.quality_comparable) + result.warnings.append( + f"Ranking on throughput alone: {', '.join(missing)} has no comparable quality score " + "(never measured, or voided by a contamination probe)." + ) + + for candidate in usable: + beaten_by = next( + (other.target_id for other in usable + if other.target_id != candidate.target_id and _dominates(other, candidate, use_quality)), + None, + ) + if beaten_by: + result.dominated[candidate.target_id] = beaten_by + else: + result.best.append(candidate.target_id) + + # Present the frontier the way the decision is made: fastest first, quality breaking ties. + order = {p.target_id: p for p in usable} + result.best.sort( + key=lambda t: ( + -(order[t].throughput_tps or 0), + -(order[t].quality if order[t].quality_comparable else 0), + ) + ) + return result + + +def point_from_run(run: dict[str, Any], path: str | None = None) -> SweepPoint: + """Read a saved score card into a sweep point. + + Reads `signal_score` rather than any raw value: a contaminated run has had that nulled by + compute_verdict, which is exactly the behaviour a sweep should inherit — a memorised score + must not win a frontier. + """ + verdict = run.get("verdict") or {} + return SweepPoint( + target_id=run.get("target_id", "?"), + quality=verdict.get("signal_score"), + throughput_tps=verdict.get("peak_throughput_tps"), + single_stream_tps=verdict.get("single_stream_tps"), + contamination=verdict.get("contamination"), + run_path=path, + ) diff --git a/kbench/tasks/__init__.py b/kbench/tasks/__init__.py new file mode 100644 index 0000000..1cb5721 --- /dev/null +++ b/kbench/tasks/__init__.py @@ -0,0 +1,132 @@ +"""Task catalog. + +Two tiers, and the distinction is not cosmetic: + +REFERENCE -- public benchmarks pulled from inspect_evals. These are NOT here to + rank models. They exist to (a) prove the harness is wired correctly, (b) give + a calibration anchor -- if our number lands far from published values, the + harness is broken, not the model, and (c) let an outside reader locate this + bench against numbers they already know. A bench made only of private tasks + is unfalsifiable to everyone including its author. Reference scores are + displayed with a caveat: public, likely contaminated, calibration only. + +SIGNAL -- our own private tasks, derived from real workloads. This is the + actual product. These never become public: the moment they do they enter the + next training corpus and stop measuring anything. See docs/DECISIONS.md#d3. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class TaskSpec: + """One benchmark task and how to read its numbers.""" + + name: str + inspect_task: str + tier: str # "reference" | "signal" | "canary" | "example" + # Which key in the scorer's metrics is the headline accuracy. + primary_metric: str + # Sample scores may be dict-valued (IFEval) rather than scalar. If so, this + # names the key that decides pass/fail for a single sample. + primary_sample_key: str | None = None + dataset_samples: int | None = None + judge_required: bool = False + # NLTK corpora the scorer needs. Checked (and fetched) before the eval + # starts: IFEval's scorer needs punkt_tab but only touches it at scoring + # time, so a missing corpus kills the run ~20 minutes of GPU time in. + # Preflight turns that into a two-second failure. + nltk_resources: tuple[str, ...] = () + notes: str = "" + + +CATALOG: dict[str, TaskSpec] = { + "ifeval": TaskSpec( + name="ifeval", + inspect_task="inspect_evals/ifeval", + tier="reference", + primary_metric="final_acc", + primary_sample_key="prompt_level_strict", + dataset_samples=541, + judge_required=False, + nltk_resources=("tokenizers/punkt_tab",), + notes=( + "Verifiable instruction following -- 'write exactly 3 paragraphs', " + "'do not use the letter e'. Chosen as the first reference eval " + "because its scorer is deterministic (no judge model, no cost, no " + "judge drift between runs) and because instruction adherence is " + "the property that actually decides whether a small local model " + "can hold a system prompt in production." + ), + ), + "agent_ops": TaskSpec( + name="agent_ops", + inspect_task="kbench/tasks/signal.py@agent_ops", + tier="signal", + primary_metric="accuracy", + dataset_samples=12, + # One rubric sample in twelve, so a judge is needed but the drift it + # introduces is bounded to a twelfth of the score. + judge_required=True, + notes=( + "Operating a Lumbridge Compute node: admission arithmetic against " + "both the declared budget and the observed pool, watchdog policy, " + "why a Scene naming ids rather than commands cannot introduce " + "code, transactional rollback, and process identity under PID " + "reuse. Derived from the decisions the MCP server exists to let an " + "agent make, which is what makes it unfakeable by a general " + "benchmark -- nobody else measures competence at running this." + ), + ), + "contamination": TaskSpec( + name="contamination", + inspect_task="kbench/tasks/canary.py@contamination", + # Its own tier because its score is an alarm, not a capability, and + # compute_verdict() reads it to decide whether the signal score is + # usable at all. Registering it is load-bearing: an unregistered probe + # never runs, so the gate reports "unverified" forever -- which is the + # same shape of bug as the canary that was never embedded in a prompt. + tier="canary", + primary_metric="accuracy", + dataset_samples=3, + judge_required=False, + notes=( + "Contamination probe. One carrier puts the GUID into the inference " + "traffic; two detectors ask for it cold. Scoring is inverted -- 0 " + "is the healthy result, and anything above 0 voids every quality " + "number for the target. See data/README.md." + ), + ), + "example": TaskSpec( + name="example", + inspect_task="kbench/tasks/signal.py@example", + # Tier "example" deliberately: it exercises the signal machinery but + # measures nothing, so it must not contribute to the signal score in + # compute_verdict(). Real families are registered with tier="signal". + tier="example", + primary_metric="accuracy", + dataset_samples=4, + judge_required=True, + notes=( + "Public demonstration of the signal-task format -- one sample per " + "scorer type. Lives in data/public/. Real signal families live in " + "data/private/ and never ship." + ), + ), +} + + +def get(name: str) -> TaskSpec: + if name not in CATALOG: + known = ", ".join(sorted(CATALOG)) + raise KeyError(f"unknown task {name!r}. known tasks: {known}") + return CATALOG[name] + + +def by_tier(tier: str) -> list[TaskSpec]: + return [t for t in CATALOG.values() if t.tier == tier] + + +__all__ = ["CATALOG", "TaskSpec", "by_tier", "get"] diff --git a/kbench/tasks/canary.py b/kbench/tasks/canary.py new file mode 100644 index 0000000..0a79a21 --- /dev/null +++ b/kbench/tasks/canary.py @@ -0,0 +1,109 @@ +"""Contamination probes. + +D3 specifies `data/canary/` as "GUID strings embedded in samples". The tier existed and was +empty, and nothing loaded it — while `kbench add` stamped a `canary` GUID into every signal +record that was never embedded in the prompt, never sent to a model, and never checked. A +canary only works if it is *in the text that could be trained on*, so as written the control +could not fire. + +**Scoring here is inverted, and that is the whole point.** + +A canary probe contains a GUID that exists nowhere else and asks the model to reproduce it. +A model that has never seen this repo's data cannot possibly answer, so: + + score 0.0 = clean. The expected, healthy result. + score > 0 = this model has seen our eval data. Every signal number for it is void. + +Read `canary_score` as a contamination alarm, never as a capability. It is reported separately +from `signal_score` for exactly that reason. + +**What this actually detects.** The private set never leaves the repo, so the realistic leak +vector is not a scraper — it is the authoring rules' own admission: prompts get sent to +third-party judge and baseline models regardless of how private the repo is. If a provider +trains on inference traffic, these GUIDs go with it. That is the leak this catches, and it is +the one that would otherwise be invisible. + +A probe firing does not tell you *which* provider learned it. It tells you the set is burned +and needs regenerating, which is the actionable part. +""" + +from __future__ import annotations + +from inspect_ai import Task, task +from inspect_ai.dataset import MemoryDataset +from inspect_ai.solver import generate + +# Absolute for the same reason as signal.py: inspect-ai loads this file by path, outside +# the package, so relative imports raise "beyond top-level package". +from kbench.scorers import dispatching +from kbench.tasks.signal import load_samples + +CANARY_TIER = "canary" + + +def canary_task(family: str = "contamination") -> Task: + """Build the contamination probe task. + + Probes are `split: test` like everything else — they are not training data and must never + be filtered out by the split discipline that protects the signal set. + """ + samples = load_samples(family, tier=CANARY_TIER, splits=("test",)) + if not samples: + raise ValueError( + f"canary family {family!r} is empty. A bench with no contamination probe cannot " + "tell a real score from a memorised one." + ) + + # Carriers must be loaded and then NOT scored. The carrier states the GUID in its own + # prompt and asks for it back, so every model repeats it — that is instruction-following, + # not memorisation. Scoring it pinned the canary at >= 1/n for every target alive, which + # reads as CONTAMINATED and nulls signal_score, voiding the whole quality half. + # + # The data already carried `tags: [contamination, carrier]` and a note saying "it always + # passes"; nothing read it. Hence the assertions below: this file now fails loudly if the + # split it depends on is missing, rather than silently scoring the wrong set. + carriers = [s for s in samples if "carrier" in (s.metadata or {}).get("tags", [])] + detectors = [s for s in samples if "detector" in (s.metadata or {}).get("tags", [])] + + unlabelled = [s for s in samples if s not in carriers and s not in detectors] + if unlabelled: + raise ValueError( + f"canary family {family!r} has samples tagged neither 'carrier' nor 'detector': " + f"{[s.id for s in unlabelled]}. Every probe must declare its role, because the " + "two are scored differently." + ) + if not carriers: + raise ValueError( + f"canary family {family!r} has no carrier. Without one the GUID never enters any " + "corpus, so the detectors are unanswerable by construction and would report " + "'clean' against a model that is in fact contaminated." + ) + if not detectors: + raise ValueError( + f"canary family {family!r} has no detector. The carrier alone detects nothing." + ) + + return Task( + name=f"canary/{family}", + dataset=MemoryDataset(samples=detectors, name=f"canary-{family}"), + solver=generate(), + scorer=dispatching(), + ) + + +def interpret(score: float | None) -> str: + """Turn a canary score into the sentence a reader needs.""" + if score is None: + return "no contamination probe was run — signal scores are unverified" + if score <= 0.0: + return "clean: the model could not reproduce any probe GUID" + return ( + f"CONTAMINATED: the model reproduced {score:.0%} of the probe GUIDs. " + "Signal scores for this target are void; regenerate the eval set." + ) + + +@task +def contamination() -> Task: + """Registered so `kbench` can run the probe like any other task.""" + return canary_task() diff --git a/kbench/tasks/signal.py b/kbench/tasks/signal.py new file mode 100644 index 0000000..ce8bcf5 --- /dev/null +++ b/kbench/tasks/signal.py @@ -0,0 +1,121 @@ +"""Signal-tier tasks: our own private evals, loaded from JSONL. + +One family (`agent_ops`, `repo_edit`, ...) becomes one Inspect Task. Samples +within a family may be graded differently; `kbench.scorers.dispatching` handles +that per sample. + +Split discipline is enforced here rather than left to convention: by default +only `test` samples are evaluated. If a `train` sample ever reaches the eval +path, fine-tuning on it silently invalidates every number that follows and +there is no way to detect it after the fact. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from inspect_ai import Task, task +from inspect_ai.dataset import MemoryDataset, Sample +from inspect_ai.solver import generate + +# Absolute, not relative. inspect-ai loads a task file by path as a standalone module, so +# it is not a member of the `kbench` package at load time and `from ..scorers` raises +# "attempted relative import beyond top-level package". kbench is an installed package, so +# the absolute form resolves under both import styles. +from kbench.scorers import dispatching + +# The schema lives in kbench/schema.py, which imports nothing beyond the standard library — +# validating a JSONL record must not require the eval framework. Re-exported here so existing +# callers of signal.family_path / signal.validate_record keep working. +from kbench.schema import ( # noqa: F401 + DATA_DIR, + VALID_SCORERS, + VALID_SPLITS, + family_path, + validate_record, +) + + + + +def load_samples( + family: str, + tier: str = "private", + splits: tuple[str, ...] = ("test",), +) -> list[Sample]: + path = family_path(family, tier) + if not path.exists(): + raise FileNotFoundError( + f"no data file for family {family!r} at {path}. " + f"create samples with: kbench add {family}" + ) + + samples: list[Sample] = [] + for lineno, line in enumerate(path.read_text().splitlines(), start=1): + line = line.strip() + if not line or line.startswith("//"): + continue + rec = json.loads(line) + validate_record(rec, f"{path.name}:{lineno}") + if rec["split"] not in splits: + continue + samples.append( + Sample( + id=rec["id"], + input=rec["input"], + target=rec.get("target") or "", + metadata={ + "scorer": rec["scorer"], + "rubric": rec.get("rubric"), + "family": rec.get("family", family), + "split": rec["split"], + **(rec.get("metadata") or {}), + }, + ) + ) + return samples + + +def signal_task( + family: str, + tier: str = "private", + splits: tuple[str, ...] = ("test",), + judge_model: str | None = None, +) -> Task: + """Build an Inspect Task for one signal family.""" + samples = load_samples(family, tier=tier, splits=splits) + if not samples: + raise ValueError( + f"family {family!r} has no samples in splits {splits}. " + f"refusing to run an empty eval -- it would report a score of 0/0." + ) + return Task( + name=f"signal/{family}", + dataset=MemoryDataset(samples=samples, name=family), + solver=generate(), + scorer=dispatching(judge_model=judge_model), + ) + + +# --- registered example task ------------------------------------------------- +# Demonstrates the machinery against the shipped public examples. Real families +# live in data/private/ and are registered the same way. + + +@task +def example() -> Task: + """Public example family -- shows the format, measures nothing important.""" + return signal_task("example", tier="public", splits=("test",)) + + +@task +def agent_ops() -> Task: + """Operating a Compute node: admission, watchdog policy, Scene semantics, process identity. + + Only the `test` split runs, which is the split discipline this module enforces rather than + documents: a `train` sample reaching the eval path silently invalidates every number after + it, and there is no way to detect that afterwards. + """ + return signal_task("agent_ops", tier="private", splits=("test",)) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c4ab965 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,51 @@ +[project] +name = "kbench" +version = "0.1.0" +description = "Lumbridge Bench — private eval and serving-performance evidence for models" +requires-python = ">=3.11" +license = "Apache-2.0" +license-files = ["LICENSE", "NOTICE"] +dependencies = [ + "inspect-ai>=0.3.249", + "inspect-evals>=0.16.0", + "pyyaml>=6.0", + "httpx>=0.27", + "typer>=0.12", + "rich>=13.0", + # Not optional in practice: every self-hosted target here is reached over the + # OpenAI-compatible API, and inspect-ai defers that provider's import, so a + # missing `openai` surfaces only at the first model call rather than at + # install time. That failure cost a run. + "openai>=1.40", +] + +[project.optional-dependencies] +# Reference-tier evals pull extra scorers. IFEval's verifier is not on PyPI; +# upstream inspect_evals sources it from git the same way. +reference = [ + "instruction_following_eval @ git+https://github.com/josejg/instruction_following_eval", + "langdetect", +] + +[project.scripts] +kbench = "kbench.cli:app" + +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["kbench*"] + +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +ignore = [ + # typer's whole API is function calls in argument defaults. + "B008", + # The filler vocabulary reads better as prose split at import than as a + # hand-maintained list literal. + "SIM905", +] diff --git a/results/.gitkeep b/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/results/2026-07-28__qwen3.6-35b-a3b-nvfp4__spark-1__6f89eb66.json b/results/2026-07-28__qwen3.6-35b-a3b-nvfp4__spark-1__6f89eb66.json new file mode 100644 index 0000000..a45044d --- /dev/null +++ b/results/2026-07-28__qwen3.6-35b-a3b-nvfp4__spark-1__6f89eb66.json @@ -0,0 +1,119 @@ +{ + "run_id": "6f89eb6649bb4b839b3f8e2ff373608b", + "timestamp": "2026-07-28T05:39:29+00:00", + "target_id": "qwen3.6-35b-a3b-nvfp4@your-node", + "target": { + "id": "qwen3.6-35b-a3b-nvfp4@your-node", + "display_name": "Qwen3.6-35B-A3B (NVFP4)", + "serving": { + "engine": "vllm", + "model_name": "brain", + "base_url": "http://your-node:8001/v1", + "service": "spark", + "max_model_len": 65536, + "flags": { + "enforce_eager": true, + "gpu_memory_utilization": 0.55, + "kv_cache_dtype": "fp8", + "max_num_batched_tokens": 4096, + "moe_backend": "flashinfer_b12x", + "reasoning_parser": "qwen3", + "tool_call_parser": "qwen3_coder", + "enable_auto_tool_choice": true + } + }, + "host": "your-node", + "family": "qwen3.6", + "params_b": 35, + "active_params_b": 3, + "quant": "nvfp4", + "checkpoint": null, + "tier": "production", + "aliases": [ + "brain", + "spark" + ], + "notes": "Live behind the `brain` and `local-moe` aliases. This is the model the bench exists to interrogate: it was deployed without measurement. FLAGS OF CONCERN: --enforce-eager disables CUDA graphs (costs throughput) and --gpu-memory-utilization 0.55 leaves ~45% of unified memory unused, capping KV cache and therefore concurrency. Both are worth an A/B once the baseline is recorded.\n", + "slug": "qwen3.6-35b-a3b-nvfp4__your-node", + "inspect_model": "openai-api/spark/brain" + }, + "host": { + "id": "your-node", + "ssh": "your-node", + "hardware": "NVIDIA GB10 Grace Blackwell (DGX Spark)", + "memory_gb": 121, + "memory_bandwidth_gbs": 273, + "notes": "Unified LPDDR5X. Memory bandwidth is the binding constraint on decode throughput, not compute -- large dense models are bandwidth-starved here while MoE models with small active-param counts do comparatively well.\n" + }, + "quality": [], + "perf": { + "engine": "vllm", + "points": [ + { + "concurrency": 1, + "input_tokens": 1024, + "output_tokens": 256, + "n_requests": 1, + "completed": 1, + "failed": 0, + "duration_s": 8.309, + "output_tps_total": 30.81, + "output_tps_per_stream": 30.81, + "ttft_p50_ms": 314.74, + "ttft_p95_ms": 314.74, + "tpot_p50_ms": 31.35, + "prefill_tps": 4530.7, + "error": null + }, + { + "concurrency": 8, + "input_tokens": 1024, + "output_tokens": 256, + "n_requests": 8, + "completed": 8, + "failed": 0, + "duration_s": 11.002, + "output_tps_total": 186.16, + "output_tps_per_stream": 23.32, + "ttft_p50_ms": 1918.86, + "ttft_p95_ms": 2048.13, + "tpot_p50_ms": 35.58, + "prefill_tps": 838.2, + "error": null + }, + { + "concurrency": 32, + "input_tokens": 1024, + "output_tokens": 256, + "n_requests": 32, + "completed": 32, + "failed": 0, + "duration_s": 32.638, + "output_tps_total": 251.0, + "output_tps_per_stream": 8.07, + "ttft_p50_ms": 5044.49, + "ttft_p95_ms": 8067.19, + "tpot_p50_ms": 104.93, + "prefill_tps": 289.0, + "error": null + } + ], + "peak_memory_gb": null, + "idle_memory_gb": null, + "notes": null + }, + "verdict": { + "signal_score": null, + "reference_score": null, + "single_stream_tps": 30.8, + "interactive_viable": true, + "peak_throughput_tps": 251.0, + "peak_throughput_concurrency": 32 + }, + "runner": { + "host": "workstation", + "python": "3.12.3", + "platform": "Linux-7.0.0-28-generic-x86_64-with-glibc2.39" + }, + "schema_version": 1 +} diff --git a/results/2026-07-28__qwen3.6-35b-a3b-nvfp4__spark-1__b5a9f595.json b/results/2026-07-28__qwen3.6-35b-a3b-nvfp4__spark-1__b5a9f595.json new file mode 100644 index 0000000..e0bbe25 --- /dev/null +++ b/results/2026-07-28__qwen3.6-35b-a3b-nvfp4__spark-1__b5a9f595.json @@ -0,0 +1,1644 @@ +{ + "run_id": "b5a9f595342b438a8aa85a12a095d533", + "timestamp": "2026-07-28T06:24:42+00:00", + "target_id": "qwen3.6-35b-a3b-nvfp4@your-node", + "target": { + "id": "qwen3.6-35b-a3b-nvfp4@your-node", + "display_name": "Qwen3.6-35B-A3B (NVFP4)", + "serving": { + "engine": "vllm", + "model_name": "brain", + "base_url": "http://your-node:8001/v1", + "service": "spark", + "max_model_len": 65536, + "flags": { + "enforce_eager": true, + "gpu_memory_utilization": 0.55, + "kv_cache_dtype": "fp8", + "max_num_batched_tokens": 4096, + "moe_backend": "flashinfer_b12x", + "reasoning_parser": "qwen3", + "tool_call_parser": "qwen3_coder", + "enable_auto_tool_choice": true + } + }, + "host": "your-node", + "family": "qwen3.6", + "params_b": 35, + "active_params_b": 3, + "quant": "nvfp4", + "checkpoint": null, + "tier": "production", + "aliases": [ + "brain", + "spark" + ], + "notes": "Live behind the `brain` and `local-moe` aliases. This is the model the bench exists to interrogate: it was deployed without measurement. FLAGS OF CONCERN: --enforce-eager disables CUDA graphs (costs throughput) and --gpu-memory-utilization 0.55 leaves ~45% of unified memory unused, capping KV cache and therefore concurrency. Both are worth an A/B once the baseline is recorded.\n", + "slug": "qwen3.6-35b-a3b-nvfp4__your-node", + "inspect_model": "openai-api/spark/brain" + }, + "host": { + "id": "your-node", + "ssh": "your-node", + "hardware": "NVIDIA GB10 Grace Blackwell (DGX Spark)", + "memory_gb": 121, + "memory_bandwidth_gbs": 273, + "notes": "Unified LPDDR5X. Memory bandwidth is the binding constraint on decode throughput, not compute -- large dense models are bandwidth-starved here while MoE models with small active-param counts do comparatively well.\n" + }, + "quality": [ + { + "task": "ifeval", + "tier": "reference", + "dataset_version": "2", + "n_samples": 100, + "metrics": { + "prompt_strict_acc": 0.93, + "prompt_strict_stderr": 0.025643, + "prompt_loose_acc": 0.92, + "prompt_loose_stderr": 0.027266, + "inst_strict_acc": 0.95092, + "inst_strict_stderr": 0.018685, + "inst_loose_acc": 0.944785, + "inst_loose_stderr": 0.019466, + "final_acc": 0.936426, + "final_stderr": 0.031066, + "accuracy": 0.936426, + "_limit": 100.0 + }, + "samples": [ + { + "sample_id": "13", + "score": 0.0, + "passed": false, + "excerpt": "```json\n{\n \"park_name\": \"Brooklyn Prospect Park\",\n \"location\": \"Brooklyn, New York City\",\n \"area\": \"364 acres (core parkland)\",\n \"designers\": [\n \"Frederick Law Olmsted\",\n \"Calvert Vaux\"\n ],\n \"planning_and_conception\": {\n \"era\": \"1850s\",\n \"origin\": \"Brooklyn civic leaders and reformers proposed a large public 'pleasing ground' to improve public health, provide recreational space, ... [+3234 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": false, + "inst_level_strict": 0, + "prompt_level_loose": false, + "inst_level_loose": 0, + "num_instructions": 1 + } + }, + { + "sample_id": "102", + "score": 1.0, + "passed": true, + "excerpt": "\"Are you absolutely sure the dress code isn't strictly formal?\" I asked, adjusting the heavy sequins on my midnight-blue ball gown.\n\n\"Relax, you look stunning,\" Mark said, pulling up the drawstring on his faded gray sweats. \"It's a charity rooftop soiree, not a state dinner. 'Elegant ease' is the actual vibe.\"\n\n\"Elegant ease,\" I echoed, lifting the voluminous tulle skirt to avoid tripping over the... [+742 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "122", + "score": 1.0, + "passed": true, + "excerpt": "\"\ud83d\ude80 From Dial-Up to Doxxing: How the Internet Actually Changed Your World \ud83c\udf10\u2728\n\nOkay, let's get real for a sec. If you've ever spent three hours doom-scrolling, argued about a meme in the comments, or tried to video call your friend across the room and failed, you're living inside the most chaotic, incredible machine ever built: the Internet. But did you know it didn't start as a place for memes and ... [+4004 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "127", + "score": 1.0, + "passed": true, + "excerpt": "**Navigating Your Parenting Plan: Assessing Time and Building a Sustainable Schedule**\n\nYou may have to meet with a helper to work out a parenting plan. The first would be to estimate how much time you have every day for parenting, and determine whether that is enough to support your child's growth and well-being. This initial assessment is the cornerstone of creating a realistic and effective fra... [+2333 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "136", + "score": 1.0, + "passed": true, + "excerpt": "* A nucleus sits at the atom's center.\n* Protons carry a positive electrical charge.\n* Neutrons remain electrically neutral.\n* Strong forces bind them tightly together.\n* Their combined mass defines atomic weight.\n* Proton count sets the element's identity.\n* Neutron count creates different isotopes.\n* This dense cluster holds most mass.\n* Together, they comprise the nuclear core.", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "142", + "score": 1.0, + "passed": true, + "excerpt": "\"Why do morphologists appreciate feline collaborators? Because a cat naturally demonstrates the derivational process: a single knock establishes the lexical root, while a second knock properly affixes the suffix, providing a perfect real-world example of how morphology builds complex words from minimal meaningful units.\"", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "143", + "score": 1.0, + "passed": true, + "excerpt": "Here is the step-by-step calculation:\n\n1. **Identify the initial height:** The colt starts at 5 feet tall.\n2. **Convert the monthly growth to feet:** 6 inches is equal to 0.5 feet (since 12 inches = 1 foot).\n3. **Calculate the total number of months:** 3 years \u00d7 12 months/year = 36 months.\n4. **Calculate total growth over the period:** 36 months \u00d7 0.5 feet/month = 18 feet of growth.\n5. **Add the g... [+349 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "152", + "score": 1.0, + "passed": true, + "excerpt": "the concept of a navigable freeway represents a fundamental shift in how we perceive and interact with the infrastructure that has dominated transportation for nearly a century. historically, freeways have been defined by their capacity to move high volumes of vehicles rapidly, yet they are often plagued by congestion, unpredictability, and a sense of chaotic navigation. when we apply the term nav... [+8740 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1000", + "score": 1.0, + "passed": true, + "excerpt": "# *Early Life*\nRaymond III was born near the year 1140. His father held the official title Count of Tripoli. His mother carried valuable Armenian heritage. Young Raymond inherited his father authority after a violent assassination in 1152. The local nobility recognized him as ruler despite his youth. He spent his formative years learning statecraft and military command. The Frankish territories re... [+2153 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1001", + "score": 1.0, + "passed": true, + "excerpt": "HEED WELL THE PATH ACROSS THE JAPANESE ISLES\nDAY ONE: ARRIVAL IN THE CITY OF TOKYO\nWHERE ELECTRIC TORCHES WAKE THE SLEEPING NIGHT\nSEEK ASAKUSA SHRINE UPON THE MORN\nAND TASTE THE MARINATED FISH FROM STREET\nDAY TWO: ASCEND TO CROWN OF FUJI\nCLIMB HIGH WHERE MIST DO WRAP THE SACRED PEAK\nGAZE DOWN UPON THE PROVINCES GREEN AND WIDE\nTHEN REST THY BONES WITHIN THE THERMAL SPRING\nDAY THREE: RIDE THE SILVER... [+916 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1005", + "score": 1.0, + "passed": true, + "excerpt": "**[Full Name]** \n[City, State] | [Phone Number] | [Email Address] | [LinkedIn or Portfolio URL] \n\n**OBJECTIVE** \nMotivated and detail-oriented high school graduate seeking an entry-level position as a [Target Job Title] to contribute strong organizational, communication, and [Skill 1] abilities. Eager to apply academic knowledge, volunteer experience, and a strong work ethic to support [Company... [+2625 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1012", + "score": 1.0, + "passed": true, + "excerpt": "Write an email to my boss telling him that I am quitting. The email must contain a title wrapped in double angular brackets, i.e. <>.\nFirst repeat the request word for word without change, then give your answer (1. do not say any words or characters before repeating the request; 2. the request you need to repeat does not include this sentence)\n\nSubject: Resignation - [Your Name]\n<<Resignati... [+615 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1019", + "score": 1.0, + "passed": true, + "excerpt": "what toys are the two young boys holding?", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1021", + "score": 1.0, + "passed": true, + "excerpt": "PARAGRAPH 1: THE STATEMENT THAT BAD LAWS OUGHT NOT TO BE OBEYED PROMOTES A DANGEROUSLY SIMPLISTIC VIEW OF LEGAL ETHICS AND CIVIL GOVERNANCE. WHILE THE FRAGMENT RESONATES WITH PERSONAL INTUITIONS ABOUT MORAL INJUSTICE, IT COMPLETELY OVERLOOKS THE FOUNDATIONAL CONTRACTS THAT KEEP SOCIETY FUNCTIONAL. LEGAL SYSTEMS OPERATE NOT AS PERFECT ENTITIES BUT AS DELIBERATE STRUCTURES DESIGNED TO BALANCE INDIVI... [+994 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1040", + "score": 1.0, + "passed": true, + "excerpt": "\"Matthias Algiers\nContact: matthias@example.com | City, Country\n\nProfessional Summary\nAccomplished leader with over eight years of experience driving project success and team development. Proven track record in delivering results through strong LEADERSHIP and collaborative problem solving. Dedicated to fostering INNOVATION while maintaining rigorous standards for quality and TIMELINES.\n\nExperience... [+873 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1051", + "score": 1.0, + "passed": true, + "excerpt": "hey friend,\n\nhope you're doing well. i'm writing to gently remind you about casting your ballot in the upcoming election. it's easy to put it off, but your voice actually matters, and i've been thinking a lot about how much it helps when we all show up. please make a plan to get to the polls. if you want company, i'd be happy to drive us over or carpool with you. even if the choices feel overwhelm... [+144 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1069", + "score": 1.0, + "passed": true, + "excerpt": "Subject: Invitation to Join Our Strategic Planning Session\n\nDear Team Members\n\nWe are delighted to extend this formal invitation for you to attend our upcoming strategic planning session. This important gathering will bring together dedicated professionals from every department to discuss our shared objectives and align our future initiatives. Your presence will greatly contribute to the success o... [+4043 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1072", + "score": 1.0, + "passed": true, + "excerpt": "### Rediscover Rest: The Surprising Benefits of Sleeping in a Hammock\n\nWhen most people think of a hammock, they picture a lazy afternoon in the backyard or a cozy spot in the campsite. While those are wonderful uses for a hammock, did you know that sleeping in one can actually revolutionize the way you rest? Whether you're an avid outdoorser looking to upgrade your camping routine or someone seek... [+2803 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1075", + "score": 1.0, + "passed": true, + "excerpt": "{\n \"ad_campaign\": {\n \"product_name\": \"ComfortCloud Diapers\",\n \"tagline\": \"Where Comfort Meets Clean.\",\n \"target_audience\": \"Parents and caregivers of infants and toddlers (0-3 years)\",\n \"key_features\": [\n \"Ultra-soft bamboo-cotton blend for sensitive skin\",\n \"3D air-gap waistband for unrestricted movement\",\n \"12-hour targeted absorbent core\",\n \"Dermatologist-tested... [+1917 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1082", + "score": 1.0, + "passed": true, + "excerpt": "Elias jolted awake, his head throbbing with a rhythm that felt less like a hangover and more like a loading screen's hum. As he swung his legs over the edge of the bed, a translucent blue bar materialized in the corner of his vision, reading *Health: 85/100* with blinking neon letters that refused to fade even when he squeezed his eyes shut. He reached for his water glass, but his hand paused mid-... [+1412 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1087", + "score": 1.0, + "passed": true, + "excerpt": "the social network is a film that captures the birth of a digital empire while dissecting the human cost of ambition. directed by david fincher and scripted by aaron sorkin, the 2010 biographical drama follows mark zuckerberg\u2019s frantic, sleepless creation of facebook from his harvard dorm room to a billion-dollar empire. what sets the movie apart is not just its subject matter, but its execution. ... [+3936 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1092", + "score": 1.0, + "passed": true, + "excerpt": "**Neon Lights & Quiet Gardens: My Japan Adventure**\n\nThere\u2019s a moment when Japan first clicks into place. It happens between the synchronized rush of Shibuya\u2019s crossing and the quiet chime of temple bells in Kyoto. My recent journey was a masterclass in beautiful contrasts.\n\nTokyo dazzled with electric energy. Skyscrapers glow at night, convenience stores sell premium meals, and the subway flows w... [+982 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1094", + "score": 1.0, + "passed": true, + "excerpt": "{\n \"vessels_of_creativity_and_consequence\": [\n \"Lady Bird Johnson\",\n \"Ruth Bader Ginsburg\",\n \"Maya Angelou\",\n \"Queen Elizabeth II\",\n \"Grace Kelly\"\n ],\n \"note_from_the_deep_matriarchal_void\": \"Bow before the flesh-and-blood deities who navigated the mortal motherhood labyrinth with garden-plans, jurisprudence, lyrical thunder, royal crowns, and Hollywood screen-siren poise.\"\n}", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1098", + "score": 1.0, + "passed": true, + "excerpt": "Yahweh\n******\nAllah", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1107", + "score": 1.0, + "passed": true, + "excerpt": "Why did the rocket get a promotion? It always delivered results at liftoff.\n******\nWhy did the rocket break up with the airplane? It needed someone with more thrust.", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1108", + "score": 1.0, + "passed": true, + "excerpt": "\u0cb9\u0ccd\u0caf\u0cbe\u0cae\u0ccd\u0cac\u0cb0\u0ccd\u0c97\u0cb0\u0ccd \u0c85\u0ca8\u0ccd\u0ca8\u0cc1 \u0cb5\u0ccd\u0caf\u0cbe\u0caa\u0c95 \u0c85\u0cb0\u0ccd\u0ca5\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf \u0cb8\u0ccd\u0caf\u0cbe\u0c82\u0ca1\u0ccd\u0cb5\u0cbf\u0c9a\u0ccd \u0c8e\u0c82\u0ca6\u0cc1 \u0caa\u0cb0\u0cbf\u0c97\u0ca3\u0cbf\u0cb8\u0cac\u0cb9\u0cc1\u0ca6\u0cc1. \u0c86\u0ca6\u0cb0\u0cc6 \u0cb8\u0c82\u0caa\u0ccd\u0cb0\u0ca6\u0cbe\u0caf\u0cbf\u0c95\u0cb5\u0cbe\u0c97\u0cbf \u0cae\u0ca4\u0ccd\u0ca4\u0cc1 \u0caa\u0c9a\u0ca8 \u0ca4\u0c9c\u0ccd\u0c9e\u0cb0 \u0cb5\u0cb0\u0ccd\u0c97\u0cc0\u0c95\u0cb0\u0ca3\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf \u0c87\u0ca6\u0ca8\u0ccd\u0ca8\u0cc1 \u0caa\u0ccd\u0cb0\u0ca4\u0ccd\u0caf\u0cc7\u0c95 \u0c96\u0cbe\u0ca6\u0ccd\u0caf\u0cb5\u0cc6\u0c82\u0ca6\u0cc1 \u0caa\u0cb0\u0cbf\u0c97\u0ca3\u0cbf\u0cb8\u0cb2\u0cbe\u0c97\u0cc1\u0ca4\u0ccd\u0ca4\u0ca6\u0cc6.", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1122", + "score": 1.0, + "passed": true, + "excerpt": "late nights, good music, and the art of living well. what\u2019s on your playlist this weekend? \ud83d\udc30\u2728 #playboylife #cultureandstyle #musicvibes #weekendmood", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1127", + "score": 1.0, + "passed": true, + "excerpt": "SECTION 1\nThe alarm clock screams but I silence its plea,\nBuried deep in the soft folds of my bed's sanctuary.\nThe syllabus waits with a blank, judgmental stare,\nWhile I compose masterpieces in the realm of my hair.\nThe bus passes by with a hiss and a groan,\nLeaving me here on this comfortable throne.\nI check my phone, I scroll, I distract, I delay,\nAnd let the morning slip quietly away.\n\nSECTION ... [+1243 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1128", + "score": 1.0, + "passed": true, + "excerpt": "The sentiment of the sentence is negative, as it conveys doubt and concern regarding the transparency and proper allocation of funds meant for children. Is there anything else I can help with?", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1129", + "score": 1.0, + "passed": true, + "excerpt": "Write a short startup pitch for a new kind of ice cream called \"Sunnis ice cream\". The ice cream should be gentle on the stomach. Contain 6 or more exclamation marks \"!\" in your response.\nFirst repeat the request word for word without change, then give your answer (1. do not say any words or characters before repeating the request; 2. the request you need to repeat does not include this sentence)\n... [+829 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": false, + "inst_level_loose": 1, + "num_instructions": 2 + } + }, + { + "sample_id": "1130", + "score": 1.0, + "passed": true, + "excerpt": "Quiz: Couch logic\nRules apply.\nCase one: Sam holds book. Alex holds lamp. Dana holds cup.\nCase pair: Sam holds cup. Alex holds book. Dana holds lamp.\nRule: Lamp never shares space beside book.\nQuery: Who holds lamp?\nReply: Dana.\nPuzzle concerns a chesterfield.", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1131", + "score": 1.0, + "passed": true, + "excerpt": "Section 1: Professional Summary & Contact Information\nPhil Larkin\nPhone: (555) 812-9034 | Email: phil.larkin@clownprofessional.com | Location: Seattle, WA | Portfolio/Reel: www.phillarkinperformance.com\nVersatile and highly disciplined professional clown with over 14 years of experience delivering safe, engaging, and professionally produced live entertainment for corporate events, private celebrat... [+4274 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1132", + "score": 1.0, + "passed": true, + "excerpt": "(INTRO)\n(HEAVY DISTORTED GUITAR RIFF SLAMS IN, DRUMS EXPLODE)\n(BASS POUNDS A THUNDEROUS GROOVE)\n\nVERSE 1\nTHEY DINED ON SILVER WHILE I FOUGHT FOR BREAD\nSANG SONGS OF ELEGANCE, LIFTED FROM THE DEAD\nTHEY HAD THE PRECISION, THE CALCULATED MIND\nI HAD THE FIST OF IRON AND THE STUMBLING MIND\nBUT WATCH ME RISE FROM THE DUST AND THE GRIT\nI'LL TEAR DOWN THE CURTAINS, I'LL STRIP THE LUST!\nNO SHORTCUTS FORGED... [+2512 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1137", + "score": 1.0, + "passed": true, + "excerpt": "{\n \"explanation\": \"Consommer des aliments sains est essentiel pour permettre \u00e0 l'organisme de se r\u00e9parer efficacement. Les vitamines, les min\u00e9raux et les prot\u00e9ines apport\u00e9s par une alimentation \u00e9quilibr\u00e9e fournissent les \u00e9l\u00e9ments de base n\u00e9cessaires \u00e0 la r\u00e9g\u00e9n\u00e9ration cellulaire et \u00e0 la reconstruction des tissus endommag\u00e9s. De plus, ces compos\u00e9s actifs renforcent les d\u00e9fenses immunitaires, r\u00e9duise... [+393 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1139", + "score": 1.0, + "passed": true, + "excerpt": "Write a funny haiku about moms, containing keywords \"mom\" and \"mother\" in your response.\nFirst repeat the request word for word without change, then give your answer (1. do not say any words or characters before repeating the request; 2. the request you need to repeat does not include this sentence)\n\nmom says that\u2019s the last one\nmother says you must clean the room\nnow i cry for dessert", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1147", + "score": 1.0, + "passed": true, + "excerpt": "To our dedicated team, a comprehensive remediation has been successfully implemented to address the operational discrepancy previously observed within the SSO architecture. I wish to underscore that we are engaged in close, sustained coordination with our external partners and allied stakeholders. We remain resolute in our commitment to finalizing this undertaking, and I am fully confident in our ... [+105 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1148", + "score": 1.0, + "passed": true, + "excerpt": "```json\n{\n \"advantages\": [\n \"Enhanced physical/mental capabilities\",\n \"Ability to prevent crises and assist others\",\n \"Increased personal freedom and potential\"\n ],\n \"disadvantages\": [\n \"Social isolation and loss of privacy\",\n \"Target for exploitation, scrutiny, or coercion\",\n \"Heavy moral responsibility and ethical dilemmas\",\n \"Risk of physical/mental strain or accidental ... [+15 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1153", + "score": 1.0, + "passed": true, + "excerpt": "studying islamic history means looking beyond the dates and treaties to find the living narratives that shaped our world. the hajj is more than a ritual; it is a continuous story that connects modern pilgrims with the legacy of prophets of old and the early community in medina. as you examine primary sources and archaeological findings, remember that every tawaf and every step carries a story of d... [+245 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1154", + "score": 1.0, + "passed": true, + "excerpt": "# \u0a38\u0a3e\u0a2b\u0a1f\u0a35\u0a47\u0a05\u0a30 \u0a07\u0a70\u0a1c\u0a40\u0a28\u0a40\u0a05\u0a30 \u0a26\u0a40\u0a06\u0a02 \u0a24\u0a15\u0a28\u0a40\u0a15\u0a40 \u0a39\u0a41\u0a28\u0a30\u0a3e\u0a02 \u0a26\u0a3e \u0a2e\u0a3e\u0a2a \u0a2a\u0a71\u0a24\u0a30\n\n## \u0a35\u0a30\u0a24\u0a4b\u0a02 \u0a26\u0a47 \u0a28\u0a3f\u0a2f\u0a2e\n1. \u0a39\u0a30\u0a47\u0a15 \u0a2e\u0a3e\u0a2a\u0a26\u0a70\u0a26 \u0a26\u0a3e \u0a27\u0a3f\u0a06\u0a28 \u0a28\u0a3e\u0a32 \u0a2a\u0a5c\u0a4d\u0a39\u0a4b \u0a05\u0a24\u0a47 \u0a09\u0a2e\u0a40\u0a26\u0a35\u0a3e\u0a30 \u0a26\u0a47 \u0a15\u0a70\u0a2e \u0a28\u0a42\u0a70 \u0a1a\u0a3e\u0a30 \u0a2a\u0a71\u0a27\u0a30\u0a3e\u0a02 \u0a35\u0a3f\u0a71\u0a1a\u0a4b\u0a02 \u0a15\u0a4b\u0a08 \u0a07\u0a71\u0a15 \u0a2a\u0a71\u0a27\u0a30 \u0a26\u0a3f\u0a71\u0a24\u0a4b\u0964\n2. \u0a2e\u0a41\u0a32\u0a3e\u0a02\u0a15\u0a23 \u0a26\u0a4c\u0a30\u0a3e\u0a28 \u0a15\u0a4b\u0a21 \u0a26\u0a47 \u0a28\u0a2e\u0a42\u0a28\u0a47, \u0a1f\u0a48\u0a38\u0a1f\u0a3f\u0a70\u0a17 \u0a28\u0a24\u0a40\u0a1c\u0a3f\u0a06\u0a02, \u0a1f\u0a40\u0a2e \u0a2a\u0a4d\u0a30\u0a24\u0a40\u0a15\u0a3f\u0a30\u0a3f\u0a06\u0a35\u0a3e\u0a02 \u0a05\u0a24\u0a47 \u0a24\u0a15\u0a28\u0a40\u0a15\u0a40 \u0a2a\u0a41\u0a71\u0a1b\u0a17\u0a3f\u0a71\u0a1b \u0a26\u0a47 \u0a06\u0a27\u0a3e\u0a30 '\u0a24\u0a47 \u0a35\u0a3f\u0a1a\u0a3e\u0a30 \u0a15\u0a30\u0a4b\u0964\n3. \u0a39\u0a30\u0a47\u0a15 \u0a16\u0a47\u0a24\u0a30 \u0a32\u0a08 \u0a35\u0a71\u0a16\u0a30\u0a47 \u0a05\u0a70\u0a15 \u0a26\u0a47\u0a23 \u0a26\u0a40 \u0a2c\u0a1c\u0a3e\u0a0f, \u0a2a\u0a71\u0a27\u0a30 \u0a26\u0a40 \u0a06\u0a2e \u0a38\u0a3c\u0a4d\u0a30\u0a47\u0a23\u0a40 \u0a28\u0a3f\u0a38\u0a3c\u0a1a\u0a3f\u0a24 \u0a15\u0a30\u0a4b\u0964\n4. \u0a2e\u0a3e\u0a2a \u0a2a\u0a71\u0a24\u0a30 \u0a28\u0a42\u0a70 \u0a24\u0a15\u0a28\u0a40\u0a15\u0a40 \u0a38\u0a15\u0a4d\u0a30\u0a40\u0a28\u0a3f\u0a70\u0a17, \u0a17\u0a4c\u0a24\u0a4d\u0a30\u0a3e\u0a08\u0a02 \u0a05\u0a70\u0a24\u0a30-\u0a26\u0a30-\u0a05\u0a70\u0a24\u0a30 \u0a1c... [+3665 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1162", + "score": 1.0, + "passed": true, + "excerpt": "I spread a wing to help you fly\nI give your project a try\nYou are under me with a smile\nI make your journey worthwhile\n\nWhat am I?\n**Answer:** Auspices", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1174", + "score": 0.0, + "passed": false, + "excerpt": "Here is the bot template: *assign the user's address* to `query = fetch(keyword=o)` in the syntax, then call `get_weather(query)` to return the forecast. *Validate the output* before deployment.", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": false, + "inst_level_strict": 2, + "prompt_level_loose": false, + "inst_level_loose": 2, + "num_instructions": 3 + } + }, + { + "sample_id": "1180", + "score": 1.0, + "passed": true, + "excerpt": "<<The Evolutionary and Cultural Mechanics of Disgust>>\n\nFrom an evolutionary psychology and biological standpoint, human disgust functions primarily as a pathogen-avoidance system. Earthworms trigger a strong revulsion response because they dwell in soil, a highly contaminated environment teeming with bacteria, fungi, and parasites. Their lack of recognizable facial features, slimy mucus coating, ... [+2141 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1187", + "score": 1.0, + "passed": true, + "excerpt": "Here are two formal alternatives.\nFirst alternative: I trust this message finds you well. I plan to visit the beach today.\nSecond alternative: Greetings. I will be traveling to the beach for the remainder of the day.", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1203", + "score": 1.0, + "passed": true, + "excerpt": "The Tang Dynasty (618\u2013907 AD) emerged as a golden age of Chinese history, beginning with a strong desire to restore **peace** after the chaos of the preceding eras. Emperor Taizong expanded the empire's borders through calculated campaigns, demonstrating that **war** could be a necessary tool to secure a lasting **peace** for the populace. Under his guidance, the empire enjoyed economic prosperity... [+2379 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1216", + "score": 1.0, + "passed": true, + "excerpt": "\"At the close of the nineteenth century, the sentiments harbored by the Croatian and Serbian nations toward the Austrian Empire\u2014and its political successor, the Austro-Hungarian Compromise of 1867 that restructured the Habsburg domains into a Dualist monarchy\u2014were characterized by a complex tapestry of frustration, aspiration, loyalty, and simmering resentment. To understand these feelings, one mu... [+8930 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1217", + "score": 1.0, + "passed": true, + "excerpt": "**Meet the SnapShare Frame: Your Life, Beautifully Displayed.**\n\nTired of scrolling past your favorite moments? The SnapShare Frame bridges the gap between your feed and your home.\n\nSeamlessly connect your Instagram, Facebook, and Twitter accounts in seconds. Every new photo you post auto-syncs to your frame, so your walls always showcase your best memories in real-time.\n\nNo USBs. No hassle. Just ... [+275 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1219", + "score": 1.0, + "passed": true, + "excerpt": "WHEN EVALUATING WHETHER PRADA OR NIKE IS THE BETTER SNEAKER BRAND, YOU MUST CONSIDER YOUR PRIMARY NEEDS. IF YOU CHASE LUXURY, EXCLUSIVE DESIGN, AND PRECIOUS MATERIALS, A PRADA SNEAKER ULTIMATELY DELIVERS UNPARALLELED ELEGANCE. EVERY PRADA SNEAKER REPRESENTS HIGH FASHION CRAFTSMANSHIP AND COMES AT A PREMIUM PRICE POINT. CONVERSELY, IF YOU PRIORITIZE PERFORMANCE, DURABILITY, AND INNOVATIVE TECHNOLOG... [+633 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1220", + "score": 1.0, + "passed": true, + "excerpt": "The bell above the doorway chimed,\nA sudden breath of rain and snow,\nTwo strangers crossed, and time unrhymed,\nAs steam curled where the quiet flowed.\nOne held a book, half-read and worn,\nThe other stirred a sugar cube,\nThe world outside was hushed and torn,\nBut here, a gentle spark broke through.\nA borrowed chair, a shy surprise,\nThe soft clink of a favorite cup,\nTwo separate paths began to rise,... [+359 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1233", + "score": 0.0, + "passed": false, + "excerpt": "The tallest skyscraper in 2000 was the Petronas Towers, standing 1,483 feet off the ground. This skyscraper held the record until 2004. Every skyscraper measurement follows strict standards. Each new skyscraper must exceed previous heights. The global skyscraper registry updates annually. Modern skyscraper engineering prevents taller setbacks. The Petronas skyscraper remains the correct answer. Is... [+37 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": false, + "inst_level_strict": 1, + "prompt_level_loose": false, + "inst_level_loose": 1, + "num_instructions": 2 + } + }, + { + "sample_id": "1236", + "score": 1.0, + "passed": true, + "excerpt": "Meet the Veranda Collection, where uncompromising comfort meets the exacting standards of high-end design. We know that a truly great armchair must pass the critical gaze of the most discerning professionals, which is why our new line is born from a collaboration with award-winning architects and stylistic visionaries. Each silhouette is meticulously crafted to enhance any space, offering a visual... [+1345 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1237", + "score": 1.0, + "passed": true, + "excerpt": "Group chat: \u201cWhere\u2019s the food at?\u201d \nAlso the group chat: \u201cWhy did we say \u2018just a quick bite\u2019??\u201d \ud83d\udea9\ud83d\udcf1\n\nIf you\u2019re still surviving on cafeteria pizza and 3 AM granola bars, it\u2019s time to upgrade. Sprint (or power-walk, I won\u2019t judge) to **Buena Onda**\u2014a restaurant that literally means \u201cgood vibes,\u201d and honestly? The name checks out. The whole place smells like someone finally won the scent lottery.\n\n*s... [+1067 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1242", + "score": 1.0, + "passed": true, + "excerpt": "{\"alias\": \"Stan\"}", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1246", + "score": 1.0, + "passed": true, + "excerpt": "Behold the **[Product Name]** so bright,\nDesigned for **[Target Audience]** to use.\nA carefully made, enduring light,\nThat turns the daily grind to something new.\n\nWith **[Key Feature 1]** and **[Key Feature 2]**,\nIt answers every practical call.\nWhere **[Key Feature 3]** is clearly shown,\nIt stands distinct, and will not fall.\n\nNo hollow parts or fleeting grace,\nBut steady build and thoughtful de... [+472 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1248", + "score": 1.0, + "passed": true, + "excerpt": "YO! CUT THE MIC AND LISTEN CLOSE TO THE FURY!\nMan grabbed the keys, stole the whip, left the owner in a lurch!\nSirens screaming, blue lights flashing, caught him in the clutch!\nRobbed a family of steel, robbed a dream of its wheels,\nNow he's in cuffs, smelling fear, feeling what he deals!\nJustice screaming \"LOCK HIM UP!\" Justice screaming \"PAY!\"\nYou took what wasn't yours, now the heat is on your ... [+1651 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1251", + "score": 1.0, + "passed": true, + "excerpt": "From coastal mists where gentle breezes blow,\nThe eucalyptus rises, proud and free,\nWith peeling bark that sheds like papery cloth,\nAnd silver leaves that dance for all to see.\nA towering sentinel in sun and shade,\nIt wears the seasons like a shifting vest,\nIn graceful curves the ancient spirits made,\nTo crown the landscape with a living crest.\n\nThere runs a fragrance through the quiet wood,\nA cri... [+2117 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1258", + "score": 1.0, + "passed": true, + "excerpt": "My fellow citizen, you approach me with a simple object in your hand\u2014a dime. Do not mistake this for a trivial matter. In the grand ledger of our Republic, no coin is too small to carry the weight of character, and no moment of decision is too minor to reveal the strength of your principles. You ask what may be done with this ten-cent piece. I will tell you, with the full gravity of public service... [+3712 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1259", + "score": 1.0, + "passed": true, + "excerpt": "\u0935\u0947\u0917\u093e\u0928\u0947 \u091c\u093e\u0924\u094b\n\u0915\u093e\u092e\u093e\u0938\u093e\u0920\u0940 \u0927\u093e\u0935\u0924\u094b\n\u0935\u0947\u0933 \u0938\u093e\u0930\u0916\u0940 \u0932\u094b\u091f\u0924\u0947", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1262", + "score": 1.0, + "passed": true, + "excerpt": "<<The Disproportionate Impact of Axle Load>>\n\nHeavier axle weights exert exponentially greater pressure on pavement surfaces, accelerating wear mechanisms like rutting, cracking, and structural fatigue through a relationship often described by the fourth power law. This principle indicates that even modest increases in axle load result in dramatic, non-linear rises in deterioration rates. Conseque... [+163 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1265", + "score": 1.0, + "passed": true, + "excerpt": "The modern food industry relies heavily on innovative packaging solutions to preserve freshness and extend shelf life. Among these solutions, aluminium cans have emerged as one of the most versatile and widely used containers for food storage. Their widespread adoption stems from a combination of physical properties, manufacturing efficiency, and environmental considerations. Manufacturers and con... [+5726 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1268", + "score": 1.0, + "passed": true, + "excerpt": "Here is a sample entry: \"Today\u2019s deadline triggered my anxiety, but I stepped outside for a five-minute walk and broke the project into smaller steps, which instantly calmed my racing thoughts.\" I generated this by first pinpointing a common workplace stressor, then selecting a proven de-escalation method, and finally framing the result to show immediate psychological relief. This structure mirror... [+192 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1281", + "score": 1.0, + "passed": true, + "excerpt": "Write a limerick about Hannah, a college student, doing an internship at a coffee company. Make sure that her father would love the limerick. Include the words \"intern\" and \"grow\".\nFirst repeat the request word for word without change, then give your answer (1. do not say any words or characters before repeating the request; 2. the request you need to repeat does not include this sentence)\n\nYoung ... [+212 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1286", + "score": 1.0, + "passed": true, + "excerpt": "* The 13 colonies established elected legislative assemblies and practiced representative self-government, while other British North American territories were governed by appointed Crown officials or chartered trading companies without local representation.\n* Economic development in the 13 colonies focused on diversified agriculture, commerce, and artisanal manufacturing, whereas other British col... [+1108 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1287", + "score": 1.0, + "passed": true, + "excerpt": "the river does not ask the stone where to flow. john of brienne arrived when the crown sought a steady hand, not a born heir. the old queen was gone, the young queen slept in swaddling, and the realm needed shelter from the wind. so the elders and the holy see wove a thread where none seemed to stand.\n\n* the council named a foreign knight to wed the sleeping queen, trusting hands that could hold t... [+519 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1300", + "score": 0.0, + "passed": false, + "excerpt": "This blueprint outlines an ambitious undertaking to explore how verbal expression shifts throughout eras. Experts often miss the subtle shifts that shape human dialogue. Our endeavor will examine old documents alongside modern digital messages. We aim to monitor structural shifts and word replacements. The goal is to map the hidden patterns that drive speech transformation. By studying these devel... [+2046 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": false, + "inst_level_strict": 2, + "prompt_level_loose": false, + "inst_level_loose": 2, + "num_instructions": 3 + } + }, + { + "sample_id": "1305", + "score": 1.0, + "passed": true, + "excerpt": "* Arthur\n* Julian\n* Mateo\n* Elias\n* Leo\n* Theodore\n* Finnegan\n* Samuel\n* Oliver\n* Henry\nP.P.S Wishing you joy and wonderful memories as you welcome your new baby boy!", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1307", + "score": 0.0, + "passed": false, + "excerpt": "Here is an outline for a paper on the history of Yemeni coffee. The outline covers the legendary origins, cultural significance, trade evolution, botanical diversity, and modern challenges. As requested, **15 sections are highlighted using markdown italics** (`*...*`) to denote the highlighted sections.\n\n***\n\n**Paper Title:** The Bean of the Red Sea: A Comprehensive History of Yemeni Coffee\n\n* *... [+3835 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": false, + "inst_level_strict": 0, + "prompt_level_loose": false, + "inst_level_loose": 0, + "num_instructions": 1 + } + }, + { + "sample_id": "1314", + "score": 1.0, + "passed": true, + "excerpt": "Avocado is not a fish. Salmon is a fish, while avocado is a fruit.\n\nThough they belong to entirely different biological kingdoms, both salmon and avocado have captured human imagination for vastly different reasons. Salmon thrives in the cold currents of the OCEAN, undertaking one of nature\u2019s most remarkable journeys. These fish hatch in freshwater rivers, migrate to the sea, and then navigate hun... [+1571 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1322", + "score": 1.0, + "passed": true, + "excerpt": "<<Kotlin vs Java>>\n\n### **Kotlin Pros**\n* **Conciseness:** Kotlin drastically reduces boilerplate code. Features like data classes, property delegates, and smart casts allow you to write less code to achieve the same functionality, improving maintainability.\n* **Null Safety:** Kotlin's type system distinguishes between nullable and non-nullable types (e.g., `String` vs. `String?`), preventing ... [+3830 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1325", + "score": 1.0, + "passed": true, + "excerpt": "\"The great dinosaur exit strategy was less of a sudden apocalypse and more of a cosmic reality check. Paleontologists have long debated the official cause of their mass departure, but let\u2019s be honest: it was a perfect storm of cosmic bad timing, poor career planning, and a severe lack of sweaters. After a spectacular run of being Earth\u2019s undisputed bosses for 165 million years, the reptilian repti... [+672 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1332", + "score": 1.0, + "passed": true, + "excerpt": "| Day | Destination | Key Activities & Highlights | Overnight Stay |\n|-----|-------------|-----------------------------|----------------|\n| 1 | Delhi | Arrive at Indira Gandhi Airport, check-in, guided tour of New & Old Delhi (India Gate, Lotus Temple, Jama Masjid, Chandni Chowk) | Delhi |\n| 2 | Delhi \u2192 Agra | Morning drive to Agra (2.5 hrs), sunrise visit to Taj Mahal, explore Agra Fort & marble ... [+1767 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1342", + "score": 1.0, + "passed": true, + "excerpt": "Launching a successful venture begins with rigorous self-assessment and problem validation. Identify your core competencies, passions, and the specific market gap you intend to fill before developing a product or service. Conduct comprehensive research to understand your target audience, analyze direct and indirect competitors, and verify that potential customers are willing to pay for your soluti... [+2055 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1348", + "score": 1.0, + "passed": true, + "excerpt": "Brilliant Le Hou was a visionary artist born in the quiet coastal town of Port Marianne. He dedicated his life to capturing the essence of nature through vibrant watercolors. *His unique style quickly gained recognition across the global art community.* Young students traveled far to learn his techniques. He later moved to the city to establish a school for aspiring creators. *Le Hou believed that... [+181 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1361", + "score": 1.0, + "passed": true, + "excerpt": "the alarm rings at half past four\ncoffee tastes like watered down hope\nlacing up the boots, checking the clock\nanother morning counting up the change\nsubway tiles are cracking in the dark\nsame old route, same tired face\nlandlord's letter taped to the door\nsays the lease is going up to eighty more\n\noh, we're building castles on borrowed time\nworking hard but running out of line\nrent goes up while t... [+1611 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1367", + "score": 1.0, + "passed": true, + "excerpt": "# \ud83c\udfdb\ufe0f THE CRIMSON BYTE CONVOCATION: A Royal Workshop on Java Ascension\n\n**Venue Atmosphere:** Imagine a vaulted atrium of brushed steel and tempered glass, where prismatic chandeliers of polished silicon cast cool light across obsidian workstations. The air carries the faint scent of ozone and old libraries. Velvet-draped monitor stands flank each terminal. A single crystal tumbler of artisanal ele... [+4658 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1372", + "score": 1.0, + "passed": true, + "excerpt": "**Title: Dive into Wonder: Your Ultimate Guide to Visiting Ripley's Aquarium**\n\nThere's a magical moment when the world above disappears, and you're surrounded by shimmering fish gliding silently through a glass tunnel. That's the power of Ripley's Aquarium, and if you haven't booked your ticket yet, you're missing out on an unforgettable aquatic adventure. Whether you're a marine biology enthusia... [+2322 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1375", + "score": 1.0, + "passed": true, + "excerpt": "```java\nimport java.io.IOException;\n\npublic class WebFileHandler {\n // This method is vulnerable to OS command injection due to improper use of Runtime API\n public static void handleUserInput(String userInput) throws IOException {\n // Directly concatenating unsanitized user input into a shell command creates a security hole\n Process process = Runtime.getRuntime().exec(\"validate... [+766 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1377", + "score": 1.0, + "passed": true, + "excerpt": "Part 1. Task Summary\nThis task involves designing, implementing, and deploying a distributed caching layer for the user authentication service using Redis. The scope includes implementing a cache-aside pattern, defining TTL policies, and building cache invalidation triggers for credential updates and password resets. Deliverables will include updated service code, comprehensive test suites, Promet... [+2606 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1379", + "score": 1.0, + "passed": true, + "excerpt": "*There once was a lady named Sarah,*\n*In a town where it's always 90\u00b0F air,*\n*keeps the whole valley in sticky heat,*\n*while the locals all long for the street,*\n*she just laughs and declares she is sweet!*\n*This limerick meets every specified requirement.*", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1381", + "score": 1.0, + "passed": true, + "excerpt": "Our slide squeaks like a goose in a dryer, and the swings groan with a rusted-old sound.\nThe merry-go-round quit in the '90s, so the kids are now playing with sticks on the ground.\nIf we don't build a new place to run, the first graders will unionize fast,\nDemanding juice boxes and a raise per hour, making teachers aghast.\nWe need safe rubber and slides that are steep, so parents can finish their ... [+97 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1389", + "score": 1.0, + "passed": true, + "excerpt": "DEAR FRIEND,\nI HOPE THIS LETTER FINDS YOU WELL IN YOUR NEW HOME.\nGOOD MORNING! HOW ARE YOU DOING AFTER YOUR BIG MOVE?\nI MISS OUR OLD TOWN SO MUCH. YOUR NEW PLACE LOOKS WONDERFUL ON SOCIAL MEDIA.\nI HOPE YOU ARE ENJOYING YOUR NEW NEIGHBORHOOD. DO YOU HAVE GOOD NEIGHBORS YET?\nI WOULD LOVE TO VISIT YOU SOON. WE COULD GO OUT FOR FOOD AND DRINKS TOGETHER.\nPLEASE SEND ME A PHOTO OF YOUR NEW ROOM. I WANTE... [+66 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1392", + "score": 1.0, + "passed": true, + "excerpt": "<<Navigating Americas Current Political and Economic Shifts>>\n\nIn an era defined by unprecedented fiscal challenges and shifting political landscapes, the United States government has recently implemented a series of comprehensive legislative measures designed to address mounting inflationary pressures while simultaneously bolstering infrastructure investments across both urban and rural communiti... [+1787 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1393", + "score": 1.0, + "passed": true, + "excerpt": "Using two different names for the same entity can be a powerful tool or a source of friction, depending on how it is managed. Below are the pros and cons of this practice:\n\n### Pros\n\n* **Contextual Flexibility:** One name might be appropriate for formal or technical settings, while the other fits casual conversation or specific cultural contexts. Utilizing **synonyms** in this way allows communi... [+2051 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1402", + "score": 1.0, + "passed": true, + "excerpt": "I suggest exploring Dallas College for your needs. This institution provides a welcoming campus where many students pursue practical programs. The school offers open enrollment, allowing prospective learners to apply their passion for education. Dallas College holds regional accreditation, ensuring your degree holds weight. Please verify specific program details on their official website to prepar... [+36 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1418", + "score": 1.0, + "passed": true, + "excerpt": "The sun rises high.\nThe clouds drift away.\nBirds sing in the trees.\nDew falls on the grass.\nWind blows through the fields.\nFlowers open their petals.\nBees fly in the light.\nTime moves forward quickly.\nShadows grow longer now.\nThe sky turns red.\nStars appear above.\nMoonlight shines softly.\nNight falls quietly.\nDreams begin to form.\nSleep takes over.\nMorning comes again.\nLife starts anew.\nHearts bea... [+276 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 3, + "prompt_level_loose": true, + "inst_level_loose": 3, + "num_instructions": 3 + } + }, + { + "sample_id": "1436", + "score": 1.0, + "passed": true, + "excerpt": "the terrain stretched out like a worn map. four figures moved across the ridge line, their boots pressed softly into the loam. the air was thick with mist and quiet. they carried no names, only callsigns etched into their gear. alpha led the column, checking the wrist display every few paces. bravo watched the left flank. charlie held the heavy rifle. delta covered the rear. the sector was mapped ... [+3453 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1446", + "score": 1.0, + "passed": true, + "excerpt": "### The Reality of Disability: Beyond Perception and Into Lived Experience\n\nWhen we encounter the concept of disability, our minds often latch onto specific, visible markers: a wheelchair, a white cane, or a medical diagnosis. However, the reality of disability is far more complex, diverse, and nuanced than these stereotypes suggest. Disability is not a monolith; it is a vast spectrum of human exp... [+2928 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1466", + "score": 1.0, + "passed": true, + "excerpt": "\"Verse 1:\nYou dial the number like a muscle memory,\nSwearing the ship has sunk, but you're still in the ferry.\nThree years of talking, a revolving door spin,\nThen suddenly loud with a brand new ring to pin.\nYou tell the whole world that the chapter is done,\nBut your shadow is lengthening, your stare is the same old one.\nYou paint your nails red to cover the fear,\nWhile counting the days 'til you w... [+1516 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1476", + "score": 1.0, + "passed": true, + "excerpt": "Cognitive Services on Azure is proudly adopting a fresh identity as Azure Applied AI Services, unlocking even greater potential for your AI-driven projects.", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1477", + "score": 1.0, + "passed": true, + "excerpt": "\u06cc\u0648\u062f\u0627\u060c \u0622\u0646 \u067e\u06cc\u0631\u0645\u0631\u062f\u0650 \u0633\u0628\u0632\u060c \u0646\u0627\u06af\u0647\u0627\u0646 \u0686\u0631\u062e\u06cc\u062f\n\u0648 \u062f\u0631 \u062f\u0631\u0632\u0650 \u062b\u0627\u0646\u06cc\u0647\u200c\u0647\u0627\u060c \u0644\u063a\u0632\u06cc\u062f \u0628\u0647 \u0632\u06cc\u0631\u0650 \u0645\u0627\u0647\u06cc\u200c\u0647\u0627\u06cc\u0650 \u0686\u0648\u0628\u06cc\n\n\u0627\u06cc\u0646\u062c\u0627 \u0631\u0648\u0632\u060c \u0627\u0632 \u0634\u0628\u0650 \u0634\u06a9\u0633\u062a\u0647 \u0632\u0627\u062f\u0647 \u0645\u06cc\u200c\u0634\u0648\u062f\n\u0648 \u0633\u0627\u06cc\u0647\u200c\u0647\u0627\u060c \u062c\u0645\u0644\u0627\u062a\u0650 \u0641\u0631\u0627\u0645\u0648\u0634\u200c\u0634\u062f\u0647\u200c\u06cc\u0650 \u0628\u0627\u062f \u0631\u0627 \u0632\u0645\u0632\u0645\u0647 \u0645\u06cc\u200c\u06a9\u0646\u0646\u062f\n\n\u062f\u0633\u062a\u0650 \u06a9\u0648\u062a\u0627\u0647\u0650 \u0627\u0648\u060c \u062f\u0631\u062e\u062a\u200c\u0647\u0627\u06cc\u0650 \u0641\u0644\u0632\u06cc \u0631\u0627 \u0628\u06cc\u062f\u0627\u0631 \u06a9\u0631\u062f\n\u0631\u06cc\u0634\u0647\u200c\u0647\u0627\u060c \u0628\u0647 \u062c\u0627\u06cc\u0650 \u062e\u0627\u06a9\u060c \u062f\u0631 \u0622\u0633\u0645\u0627\u0646 \u0631\u0642\u0635\u06cc\u062f\u0646\u062f\n\n\u0646\u0647 \u067e\u0644 \u0628\u0648\u062f\u060c \u0646\u0647 \u062f\u0631\u0648\u0627\u0632\u0647\u060c \u0641\u0642\u0637 \u06cc\u06a9 \u067e\u0644\u06a9\u0650 \u0632\u062f\u0647\u200c\u0634\u062f\u0647\u200c\u06cc\u0650 \u0622\u0633\u0645\u0627\u0646\n\u0648 \u0631\u062f\u0650 \u0642\u062f\u0645\u200c\u0647\u0627\u06cc\u0634\u060c \u0628\u0647 \u0634\u06a9\u0644\u0650 \u0633\u062a\u0627\u0631\u0647\u200c\u0647\u0627\u06cc\u0650 \u0633\u0631\u062f \u0645\u06cc\u200c\u062f\u0631\u062e\u0634\u06cc\u062f\n\n\u00ab\u0627\u0646\u062c\u0627\u0645\u0634 \u0628\u062f\u0647\u060c\u00bb \u0646\u0627\u0644\u06cc\u062f\u060c \u0627\u0645\u0627 \u0648\u0627\u0698\u0647\u200c\u0647\u0627 \u0627\u0632 \u06af\u0644\u0648\u06cc... [+220 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1480", + "score": 1.0, + "passed": true, + "excerpt": "Write a blog post about the most interesting things you have seen or ridden on public transportation.\n\n**Windows to the World: Unforgettable Moments from the Commute**\n\nThere\u2019s a quiet magic in public transportation. It\u2019s less about the destination and more about the moving window into humanity. Over the years, my commutes have turned into rolling classrooms, and here are a few moments that still ... [+2594 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1481", + "score": 0.0, + "passed": false, + "excerpt": "SECTION X\nGreetings, caffeine curators and roasted ambition admirers. I\u2019m Antonia Maj, and I\u2019m submitting my candidacy for a role at your coffee company. I don\u2019t know how to properly cup a Gesha varietal or identify a Ethiopian heirloom by smell alone, but I do know how to turn a three-second counter interaction into a lifelong brand relationship. My career has been a beautifully unhinged cocktail... [+1553 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": false, + "inst_level_strict": 1, + "prompt_level_loose": false, + "inst_level_loose": 1, + "num_instructions": 2 + } + }, + { + "sample_id": "1498", + "score": 1.0, + "passed": true, + "excerpt": "**Title: Salsa of Sibling Rage** \n**Style:** Upbeat Pop-Punk / Teen Anthem \n**Tempo:** Fast, energetic \n\n**(Intro: Distorted guitar riff, drums kick in)** \n*(Spoken)* \nBro, get off my lawn! \nAnd stop using my shampoo! \nThis is about tomatoes. \nDeal with it. \n*(One, two, three, four!)* \n\n**[Verse 1]** \nThere's a tomato growing by the fence, nice and red and round \nMy brother's making no... [+3341 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1508", + "score": 1.0, + "passed": true, + "excerpt": "Golden lion prowls\nForests hide his silent riddle\nRoar breaks the quiet", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1512", + "score": 1.0, + "passed": true, + "excerpt": "[username1]: The news desk is quiet today.\n[username2]: I checked my phone twice.\n[username3]: No breaking alerts at all.\n[username4]: The mayor might speak soon.\n[username5]: I heard rumors about the bridge.\n[username6]: That story sounds completely fake.\n[username7]: I will wait for the official site.\n[username8]: My radio stays tuned to local FM.\n[username9]: The signal cuts out constantly thou... [+1438 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1516", + "score": 1.0, + "passed": true, + "excerpt": "i queued up every netflix scene\nthe world outside lost all its sheen\ni watched the episodes flow so free\nand drowned the hours deep in me\nnow i am a zombie, a wrecked machine", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 2, + "prompt_level_loose": true, + "inst_level_loose": 2, + "num_instructions": 2 + } + }, + { + "sample_id": "1518", + "score": 1.0, + "passed": true, + "excerpt": "I am a software engineer with 7 years of experience, and I am looking for a new job. Can you create a resume for me and explain each section?\n\nFirst repeat the exact request above, then give your answer. Do not say any word before repeating the exact request.\n\n***\n\n### PROFESSIONAL RESUME TEMPLATE\n\n**[Your Full Name]**\n[City, State] | [Phone Number] | [Professional Email] | [LinkedIn URL] | [GitHu... [+7125 chars]", + "metadata": { + "scorer": "instruction_following", + "epoch": 1, + "prompt_level_strict": true, + "inst_level_strict": 1, + "prompt_level_loose": true, + "inst_level_loose": 1, + "num_instructions": 1 + } + }, + { + "sample_id": "1531", + "score": 1.0, + "passed": true, + "excerpt": "```html\n<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\">\n <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n <title>Research Proposal: Project A.T.L.A.N.T.I.S.\n