Lumbridge Bench

2026-08-04 00:44:07 -07:00
commit 006feee0f7
65 changed files with 13516 additions and 0 deletions
@@ -0,0 +1,51 @@
+[project]
+name = "kbench"
+version = "0.1.0"
+description = "Lumbridge Bench — private eval and serving-performance evidence for models"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+license-files = ["LICENSE", "NOTICE"]
+dependencies = [
+    "inspect-ai>=0.3.249",
+    "inspect-evals>=0.16.0",
+    "pyyaml>=6.0",
+    "httpx>=0.27",
+    "typer>=0.12",
+    "rich>=13.0",
+    # Not optional in practice: every self-hosted target here is reached over the
+    # OpenAI-compatible API, and inspect-ai defers that provider's import, so a
+    # missing `openai` surfaces only at the first model call rather than at
+    # install time. That failure cost a run.
+    "openai>=1.40",
+]
+
+[project.optional-dependencies]
+# Reference-tier evals pull extra scorers. IFEval's verifier is not on PyPI;
+# upstream inspect_evals sources it from git the same way.
+reference = [
+    "instruction_following_eval @ git+https://github.com/josejg/instruction_following_eval",
+    "langdetect",
+]
+
+[project.scripts]
+kbench = "kbench.cli:app"
+
+[build-system]
+requires = ["setuptools>=64"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+include = ["kbench*"]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+
+[tool.ruff.lint]
+ignore = [
+    # typer's whole API is function calls in argument defaults.
+    "B008",
+    # The filler vocabulary reads better as prose split at import than as a
+    # hand-maintained list literal.
+    "SIM905",
+]