Lumbridge Bench

Makefile

@@ -0,0 +1,37 @@
+.PHONY: install card card-full perf example view results lint clean
+
+VENV := .venv/bin
+TARGET ?= brain
+
+install:
+	uv venv --python 3.12
+	uv pip install -e ".[reference]"
+
+## Quick score card: capped IFEval + perf sweep. The cap is recorded in the
+## result and rendered on the card -- never present this as a full run.
+card:
+	$(VENV)/kbench run $(TARGET) --tasks ifeval --limit 100 --max-connections 16
+
+## Full reference run. Slow -- 541 samples against a reasoning model that emits
+## ~3k tokens per answer. Run it overnight.
+card-full:
+	$(VENV)/kbench run $(TARGET) --tasks ifeval --max-connections 16
+
+perf:
+	$(VENV)/kbench perf $(TARGET)
+
+## Exercises the signal-task machinery against the public examples.
+example:
+	$(VENV)/kbench run $(TARGET) --tasks example --skip-perf
+
+view:
+	$(VENV)/inspect view --log-dir logs
+
+results:
+	$(VENV)/kbench results
+
+lint:
+	$(VENV)/python -m ruff check kbench/ || true
+
+clean:
+	rm -rf logs/*.eval