Karti Tripathi
38 lines
938 B
Makefile
38 lines
938 B
Makefile
.PHONY: install card card-full perf example view results lint clean
|
|
|
|
VENV := .venv/bin
|
|
TARGET ?= brain
|
|
|
|
install:
|
|
uv venv --python 3.12
|
|
uv pip install -e ".[reference]"
|
|
|
|
## Quick score card: capped IFEval + perf sweep. The cap is recorded in the
|
|
## result and rendered on the card -- never present this as a full run.
|
|
card:
|
|
$(VENV)/kbench run $(TARGET) --tasks ifeval --limit 100 --max-connections 16
|
|
|
|
## Full reference run. Slow -- 541 samples against a reasoning model that emits
|
|
## ~3k tokens per answer. Run it overnight.
|
|
card-full:
|
|
$(VENV)/kbench run $(TARGET) --tasks ifeval --max-connections 16
|
|
|
|
perf:
|
|
$(VENV)/kbench perf $(TARGET)
|
|
|
|
## Exercises the signal-task machinery against the public examples.
|
|
example:
|
|
$(VENV)/kbench run $(TARGET) --tasks example --skip-perf
|
|
|
|
view:
|
|
$(VENV)/inspect view --log-dir logs
|
|
|
|
results:
|
|
$(VENV)/kbench results
|
|
|
|
lint:
|
|
$(VENV)/python -m ruff check kbench/ || true
|
|
|
|
clean:
|
|
rm -rf logs/*.eval
|