pin verifiers 0.3.1.dev59, and make every config say subprocess

The 600-second per-model-call ceiling that capped the thinking-on run at n=6 is
gone: DEFAULT_TIMEOUT is read=None in all nine venvs.

There is NO verifiers 0.4.0. The `version = "0.4.0"` in the vendored tree belongs
to a [[tool.uv.dependency-metadata]] block for nemo-gym, and reading it as
verifiers' own sends you hunting a release that does not exist and concluding the
bump is blocked. The vendored checkout is v0.3.0-59-g4bcb48e5, published to PyPI
as 0.3.1.dev59, and the installed wheel diffs clean against the whole vendored
v1 tree. Pinned as an ordinary index dependency: a path source is reproducible
only on this box and a git URL pins a commit no wheel matches.

⚠️ dev59 changes the default agent runtime from subprocess to `prime`, a REMOTE
sandbox. Without an explicit runtime every eval here exits "not authenticated
with prime" before a single rollout — having already created its output directory
and a zero-line traces.jsonl, so it reads as a failed eval rather than an
unauthenticated one. All nine configs now say subprocess. Nothing here needs a
sandbox: the reward is a pure function of the trace and the harness is null.

Output layout is flat now — <env>--<model>--<harness>--<hex8>/ with configs/,
traces.jsonl and a logs/eval.log that 0.3.0 never wrote. `-o` is the output_dir,
not the run dir; `--run.dir` pins the leaf.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-08-21 23:14:05 -07:00
co-authored by Claude Opus 5
parent 8048a61a1b
commit 122a858b61
19 changed files with 388 additions and 34 deletions
+1 -1
View File
@@ -3,7 +3,7 @@ name = "bot-detection"
version = "0.1.0"
description = "bot-detection — find the scripts in an activity log without banning the players who grind like one."
requires-python = ">=3.11"
dependencies = ["verifiers"]
dependencies = ["verifiers==0.3.1.dev59"]
[build-system]
requires = ["hatchling"]
+1 -1
View File
@@ -3,7 +3,7 @@ name = "canary-trap"
version = "0.1.0"
description = "canary-trap — write contamination probes that survive a paraphrase, without accusing a clean model."
requires-python = ">=3.11"
dependencies = ["verifiers"]
dependencies = ["verifiers==0.3.1.dev59"]
[build-system]
requires = ["hatchling"]
@@ -3,7 +3,7 @@ name = "drop-table-inference"
version = "0.1.0"
description = "drop-table-inference — estimate a drop table from a kill log, graded on the kills you never saw."
requires-python = ">=3.11"
dependencies = ["verifiers"]
dependencies = ["verifiers==0.3.1.dev59"]
[build-system]
requires = ["hatchling"]
@@ -3,7 +3,7 @@ name = "fault-localisation"
version = "0.1.0"
description = "fault-localisation — name the service that broke, not the ones complaining about it."
requires-python = ">=3.11"
dependencies = ["verifiers"]
dependencies = ["verifiers==0.3.1.dev59"]
[build-system]
requires = ["hatchling"]
+1 -1
View File
@@ -3,7 +3,7 @@ name = "grand-exchange"
version = "0.1.0"
description = "grand-exchange — place orders executed against a market window you never saw."
requires-python = ">=3.11"
dependencies = ["verifiers"]
dependencies = ["verifiers==0.3.1.dev59"]
[build-system]
requires = ["hatchling"]
@@ -3,7 +3,7 @@ name = "grand-exchange-live"
version = "0.1.0"
description = "grand-exchange-live — quote, watch the tape, re-quote, over a turn budget."
requires-python = ">=3.11"
dependencies = ["verifiers"]
dependencies = ["verifiers==0.3.1.dev59"]
[build-system]
requires = ["hatchling"]
@@ -6,7 +6,7 @@ requires-python = ">=3.11"
dependencies = [
"pydantic==2.13.4",
"regex==2026.7.19",
"verifiers==0.3.0",
"verifiers==0.3.1.dev59",
]
[build-system]
+1 -1
View File
@@ -3,7 +3,7 @@ name = "schema-migration"
version = "0.1.0"
description = "schema-migration — split a text column into a number and a unit, graded on rows you never saw."
requires-python = ">=3.11"
dependencies = ["verifiers"]
dependencies = ["verifiers==0.3.1.dev59"]
[build-system]
requires = ["hatchling"]
+1 -1
View File
@@ -3,7 +3,7 @@ name = "tera-spatial"
version = "0.1.0"
description = "tera-spatial — Arena's bridge to Tera's four renderer-independent spatial environments, executed in TypeScript and graded in Python, and the tasksets that sit on it."
requires-python = ">=3.11"
dependencies = ["verifiers"]
dependencies = ["verifiers==0.3.1.dev59"]
# One directory, several tasksets: the environments share a vendored simulator and
# a single worker, so splitting them into four wheels would ship the same 283 KB