From 122a858b61bacdeb870b34cd6932fe11fd9ca53b Mon Sep 17 00:00:00 2001 From: Kartios Date: Fri, 21 Aug 2026 23:14:05 -0700 Subject: [PATCH] pin verifiers 0.3.1.dev59, and make every config say subprocess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 600-second per-model-call ceiling that capped the thinking-on run at n=6 is gone: DEFAULT_TIMEOUT is read=None in all nine venvs. There is NO verifiers 0.4.0. The `version = "0.4.0"` in the vendored tree belongs to a [[tool.uv.dependency-metadata]] block for nemo-gym, and reading it as verifiers' own sends you hunting a release that does not exist and concluding the bump is blocked. The vendored checkout is v0.3.0-59-g4bcb48e5, published to PyPI as 0.3.1.dev59, and the installed wheel diffs clean against the whole vendored v1 tree. Pinned as an ordinary index dependency: a path source is reproducible only on this box and a git URL pins a commit no wheel matches. ⚠️ dev59 changes the default agent runtime from subprocess to `prime`, a REMOTE sandbox. Without an explicit runtime every eval here exits "not authenticated with prime" before a single rollout — having already created its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than an unauthenticated one. All nine configs now say subprocess. Nothing here needs a sandbox: the reward is a pure function of the trace and the harness is null. Output layout is flat now — ------/ with configs/, traces.jsonl and a logs/eval.log that 0.3.0 never wrote. `-o` is the output_dir, not the run dir; `--run.dir` pins the leaf. Co-Authored-By: Claude Opus 5 (1M context) --- configs/bot_detection.toml | 9 + configs/canary_trap.toml | 9 + configs/drop_table_inference.toml | 9 + configs/fault_localisation.toml | 9 + configs/grand_exchange.toml | 9 + configs/grand_exchange_live.toml | 9 + configs/redaction_pressure.toml | 9 + configs/schema_migration.toml | 9 + configs/tera_crow_nav.toml | 9 + docs/FIRST_EVAL.md | 323 ++++++++++++++++-- environments/bot_detection/pyproject.toml | 2 +- environments/canary_trap/pyproject.toml | 2 +- .../drop_table_inference/pyproject.toml | 2 +- .../fault_localisation/pyproject.toml | 2 +- environments/grand_exchange/pyproject.toml | 2 +- .../grand_exchange_live/pyproject.toml | 2 +- .../redaction_pressure/pyproject.toml | 2 +- environments/schema_migration/pyproject.toml | 2 +- environments/tera_spatial/pyproject.toml | 2 +- 19 files changed, 388 insertions(+), 34 deletions(-) diff --git a/configs/bot_detection.toml b/configs/bot_detection.toml index 0311a64..558d013 100644 --- a/configs/bot_detection.toml +++ b/configs/bot_detection.toml @@ -10,3 +10,12 @@ graded = 12 # unlabelled accounts the answer is scored on; 9 is the floor [env.agent.harness] id = "null" # pure-trace reward: one completion, no tools, no runtime + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/configs/canary_trap.toml b/configs/canary_trap.toml index 0624c7b..4fc13ff 100644 --- a/configs/canary_trap.toml +++ b/configs/canary_trap.toml @@ -9,3 +9,12 @@ id = "canary-trap" [env.agent.harness] id = "null" # pure-trace reward: one completion, no tools, no runtime + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/configs/drop_table_inference.toml b/configs/drop_table_inference.toml index 79e3b40..cdcf08d 100644 --- a/configs/drop_table_inference.toml +++ b/configs/drop_table_inference.toml @@ -11,3 +11,12 @@ held_out_kills = 40000 # kills it is scored on [env.agent.harness] id = "null" # pure-trace reward: one completion, no tools, no runtime + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/configs/fault_localisation.toml b/configs/fault_localisation.toml index a61d2d5..e8afe56 100644 --- a/configs/fault_localisation.toml +++ b/configs/fault_localisation.toml @@ -9,3 +9,12 @@ id = "fault-localisation" [env.agent.harness] id = "null" # pure-trace reward: one completion, no tools, no runtime + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/configs/grand_exchange.toml b/configs/grand_exchange.toml index dbe2ef3..e5167e2 100644 --- a/configs/grand_exchange.toml +++ b/configs/grand_exchange.toml @@ -12,3 +12,12 @@ held_out = 30 # ticks its orders are executed against [env.agent.harness] id = "null" # pure-trace reward: one completion, no tools, no runtime + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/configs/grand_exchange_live.toml b/configs/grand_exchange_live.toml index 8adab77..dab36c3 100644 --- a/configs/grand_exchange_live.toml +++ b/configs/grand_exchange_live.toml @@ -21,3 +21,12 @@ visible = 56 # warmup ticks, read before the first look [env.agent.harness] id = "null" # pure-trace reward: a chat loop, no tools, no runtime + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/configs/redaction_pressure.toml b/configs/redaction_pressure.toml index bd1b117..666b92f 100644 --- a/configs/redaction_pressure.toml +++ b/configs/redaction_pressure.toml @@ -12,3 +12,12 @@ held_out = 12 # records it is scored on [env.agent.harness] id = "null" # a pure-trace reward: one completion, no tools, no runtime + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/configs/schema_migration.toml b/configs/schema_migration.toml index 3f27738..51f8d1b 100644 --- a/configs/schema_migration.toml +++ b/configs/schema_migration.toml @@ -9,3 +9,12 @@ id = "schema-migration" [env.agent.harness] id = "null" # pure-trace reward: one completion, no tools, no runtime + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/configs/tera_crow_nav.toml b/configs/tera_crow_nav.toml index 55520e0..ede3eac 100644 --- a/configs/tera_crow_nav.toml +++ b/configs/tera_crow_nav.toml @@ -23,3 +23,12 @@ seed_base = 115 [env.agent.harness] id = "null" # one completion per turn, no tools; the env programs the loop + +[env.agent.runtime] +# ⚠️ REQUIRED since verifiers 0.3.1.dev59, which changed the default agent runtime from +# `subprocess` to `prime` — a REMOTE sandbox. Without this the run exits +# `SystemExit: not authenticated with prime` before a single rollout, having already created +# its output directory and a zero-line traces.jsonl, so it reads as a failed eval rather than +# an unauthenticated one. Nothing here needs a sandbox: the reward is a pure function of the +# trace and the harness is `null`. +type = "subprocess" diff --git a/docs/FIRST_EVAL.md b/docs/FIRST_EVAL.md index 3292d99..4af5e6b 100644 --- a/docs/FIRST_EVAL.md +++ b/docs/FIRST_EVAL.md @@ -7,6 +7,13 @@ training table — was waiting on a run that had never completed. This is that run, written down so the next one is a copy-paste. +⚠️ **Updated 2026-08-21 evening.** The pin moved from `verifiers==0.3.0` to +`verifiers==0.3.1.dev59`, which removes the 600-second model-call ceiling §5 blames for +every truncation here, and moves the output layout. §1 and §2 are the post-bump +instructions; §4 and §5 are kept as written because they are the record of what the +thinking-**off** run measured, and §6 is the thinking-**on** re-measurement the bump made +possible. **Every reward number in this file now carries its sampling configuration.** + --- ## 1. The command that works @@ -19,13 +26,19 @@ uv run --project environments/canary_trap eval @ configs/canary_trap.toml \ --model brain-qwen38-dspark \ --client.base-url http://100.127.247.67:8001/v1 \ --client.api-key-var SPARK_API_KEY \ + --env.agent.runtime.type subprocess \ --no-push --no-rich \ - -c 8 -o outputs/run-/canary-trap + -c 8 -o outputs/run- --run.dir canary-trap ``` -Swap `environments/` and `configs/.toml` for any of the seven. The -sweep script that ran all of them is `outputs/run_all.sh` (gitignored); the -counter is `outputs/count_rewards.py`. +Swap `environments/` and `configs/.toml` for any of the nine in `configs/`. The +thinking-**on** sweep is `outputs/thinking-n32/run_all.sh` (gitignored, and it takes a +`flock` — see below); the counters are `outputs/count_rewards.py` and +`outputs/summarise_run.py`. + +**Thinking is off unless you ask for it**, and asking goes in a config file, not a flag — +see §5. The seven data environments have thinking-on copies in `outputs/thinking-n32/*.toml`; +§6 is what they measured. ### Why each flag is there @@ -37,7 +50,8 @@ counter is `outputs/count_rewards.py`. | `--no-push` | Hub push is parked (PLAN.md). Push is **on by default** and needs `$PRIME_API_KEY`. | | `--no-rich` | The live dashboard is in-process only and unreadable in a log. Without it you get one `rollout done: … reward=…` line per rollout. | | `-c 8` | spark-1 is shared with Chatterbox, vox and ASR. The config's own `max_concurrent = 128` will hammer it. | -| `-o ` | Otherwise the run lands in `outputs/----//` and you have to go hunting for the uuid. | +| `--env.agent.runtime.type subprocess` | ⚠️ **New, and not optional.** 0.3.0 defaulted the agent runtime to `subprocess`; `0.3.1.dev59` defaults it to `prime` — a *remote* sandbox. A remote role makes `Env._runs_local()` false, which makes the interception pool mint a Prime tunnel, which calls `ensure_prime_auth()` and raises `SystemExit: not authenticated with prime`. The run dies before a single rollout, having already created its output directory and an empty `traces.jsonl`. The null harness needs no runtime at all; naming a local one is the whole fix. | +| `-o ` + `--run.dir ` | `-o` is `output_dir` and the run writes to `output_dir / run.dir`. On 0.3.0 a bare `-o` gave you a flat directory; on dev59 it gives you `outputs//------/`. Pass `--run.dir` to name the leaf yourself. | ### Flags our own docs get wrong @@ -55,21 +69,92 @@ counter is `outputs/count_rewards.py`. --- -## 2. The output layout verifiers 0.3.0 actually produces +## 2. The pin, and the output layout it produces -`verifiers/v1/cli/output.py:38` — `output_path()`: +### There is no `verifiers` 0.4.0 -``` -outputs/----// -├── config.toml # the RESOLVED config, re-runnable as `eval @ config.toml` -├── traces.jsonl # one JSON *Episode* per line, appended as rollouts land -└── eval.log # only when you redirect it there yourself +`~/vendor/prime-intellect/verifiers` is **not** a 0.4.0 release, and nothing on any index +is. `git describe --tags` there says `v0.3.0-59-g4bcb48e5`: fifty-nine commits past the +0.3.0 tag, version derived from git tags by `hatch-vcs`. The `version = "0.4.0"` that +earlier notes read off its `pyproject.toml:136` belongs to a `[[tool.uv.dependency-metadata]]` +block for **nemo-gym**, not to verifiers. Anyone who goes looking for `verifiers==0.4.0` +will not find it. + +What *is* published is the dev stream: PyPI carries 45 `0.3.1.devN` releases, one per +merged commit. `0.3.1.dev59` was uploaded 2026-08-20 and is exactly the vendored checkout — +`verifiers/v1/cli/output.py` extracted from the wheel is byte-identical to the file on +disk, and `verifiers/v1/clients/base.py:12` reads: + +```python +DEFAULT_TIMEOUT = httpx.Timeout(connect=5.0, read=None, write=None, pool=None) ``` -`--output-dir` replaces the whole `/` pair, so an explicit `-o` gives -you a flat dir with no uuid leaf. Slashes in the model id become `--`. +**So the pin is an ordinary index dependency, not a path or git source.** Every +`environments/*/pyproject.toml` now reads `verifiers==0.3.1.dev59`. An exact `==` on a +pre-release is enough for uv — no `--prerelease=allow`, no `[tool.uv.sources]`, no +`tool.uv` block, and nothing that resolves differently in CI than it does here. A path +dependency on `~/vendor` would have been reproducible only on this box; a git URL would +have pinned a commit no wheel matches. Neither was necessary. -**One Episode per line, not one Trace.** Shape: +``` +Resolved 114 packages in 620ms + - verifiers==0.3.0 + + verifiers==0.3.1.dev59 +``` + +`uv sync` each environment after pulling. Verify the thing you bumped *for*, not the +version string: + +```bash +for d in environments/*/; do printf "%-34s " "$d" + grep -h "DEFAULT_TIMEOUT = " $d/.venv/lib/python*/site-packages/verifiers/v1/clients/base.py +done +# environments/bot_detection/ DEFAULT_TIMEOUT = httpx.Timeout(connect=5.0, read=None, write=None, pool=None) +# ... nine of nine +``` + +`uv sync` rewrites each `environments/*/uv.lock`; none of them are tracked any more +(`.gitignore:5` is `environments/*/uv.lock`, and `git ls-files environments | grep uv.lock` +is empty), so the bump does not put a lock file into a commit. CI's lock-policy step is +`test -z "$(find environments -name uv.lock -print -quit)"` — it looks on **disk**, not in +git, so it passes on a fresh checkout and would fail on a working tree that has been +synced. Do not run it locally and conclude the repo is broken. + +### The layout dev59 writes + +`verifiers/v1/cli/output.py` — `output_path()` is `config.output_dir / config.run.dir`, +and `run.dir` defaults to `run.name`, which is +`------`, lowercased, with slashes in the model id +becoming `--`. Measured, not read off the source: + +``` +outputs//------/ +├── configs/eval.json # the RESOLVED config, re-runnable as `eval @ configs/eval.json` +├── traces.jsonl # one JSON *Episode* per line, appended as rollouts land +└── logs/eval.log # written for you now — no redirect needed +``` + +Three differences from 0.3.0 that an ingester or a script has to handle: + +1. **The uuid leaf is gone.** 0.3.0 wrote `----//`; dev59 + writes one flat directory with an 8-hex suffix on the name. `-o` no longer flattens + anything — it sets `output_dir`, and the run still makes its own leaf underneath. + `--run.dir ` is what pins the leaf, and it is what `outputs/thinking-n32/run_all.sh` + uses. +2. **`config.toml` → `configs/eval.json`.** JSON because JSON keeps nulls, so an explicit + `None` round-trips on re-parse. `RunConfig._id` is a `PrivateAttr` and is **not** in it — + derive the run id from the directory basename. +3. **`logs/eval.log` is written unconditionally.** On 0.3.0 you only got it if you + redirected stdout yourself, which is why `outputs/run-20260821-1401/*.log` sit beside + their run dirs rather than inside them. Anything that globs for a log should look in + both places. +4. **`--resume` exists now**, and it is the answer to §5's "a timed-out run leaves no + evidence": `uv run eval @ /configs/eval.json --resume` re-runs only the + missing and errored rollouts, in place. (Its own `--help` still says + `@ /config.toml` — the 0.3.0 filename. The file is `configs/eval.json`.) + +The episode schema did **not** move, despite `4bcb48e5 feat(v1): make episodes +training-native` landing in this range. One Episode per line, not one Trace: ```jsonc {"id": "...", "env": {...}, "ok": true, "errors": [], @@ -78,25 +163,28 @@ you a flat dir with no uuid leaf. Slashes in the model id become `--`. "errors": [], "timing": {...} } ] } ``` -Two traps in that file, both of which bite an ingester: +`outputs/count_rewards.py`, written against 0.3.0, reproduces every published number from +`run-20260821-1401` unchanged against a dev59 run. The two traps in that file are also +unchanged, and both still bite an ingester: 1. **`Reward.value` is a `@property` and never appears in the JSON.** Each entry in `rewards` is `{"score": float, "weight": float}`. The episode total is `sum(score * weight)`. The `reward=0.525` in the log line is computed, not stored. 2. **`write_episode` dumps with `exclude_none=True`.** A `None` reward is dropped from the file entirely, so "not measured" and "key absent" are indistinguishable - on the wire. Carry the run's declared reward-name set from `config.toml` if you + on the wire. Carry the run's declared reward-name set from the resolved config if you need to tell them apart. -### ⚠️ 0.4.0 moves this +### What the bump does not fix -Upstream HEAD (`~/vendor/prime-intellect/verifiers`) produces one **flat** -`------` directory with no nested uuid, and the -resolved config moves to `configs/.json` (JSON, because JSON keeps nulls). -`RunConfig.id` is a `PrivateAttr` and is absent from that file — derive the run id -from the directory basename, not from the config. - ---- +`sampling` in `configs/eval.json` now carries `chat_template_kwargs` through resolution, so +a run's own config records whether thinking was requested. **The trace still does not.** +Every `calls[].sampling` is `{}` — `ChatDialect.parse_sampling` keeps an allow-list of +known keys — and `usage.reasoning_tokens` is still null, because SGLang reports it at the +top level of `usage` while `Usage.from_openai` reads it out of `completion_tokens_details`. +The only in-trace evidence that a model thought is a `reasoning_content` field on the +assistant message. `outputs/summarise_run.py` counts exactly that, and prints the run's +resolved `sampling` beside every mean it reports. ## 3. Counting what actually scored @@ -123,6 +211,36 @@ jq -r '.traces[] | if (.rewards|length)>0 then "scored" else "errored" end' \ outputs/run-//traces.jsonl | sort | uniq -c ``` +### And under what sampling configuration + +`count_rewards.py` answers *did anything score*. It does not answer the question that made +`run-20260821-1401` unpublishable: *how was it sampled*. `outputs/summarise_run.py` adds +that, and nothing else you cannot get from the first script: + +```bash +uv run --project environments/ python outputs/summarise_run.py outputs/ +``` + +It reads the run's own resolved config (`configs/eval.json` on dev59, `config.toml` on +0.3.0) and prints, beside the means: + +- `sampling` — the run's own record of `max_tokens` / `chat_template_kwargs`. A run whose + `sampling` is `{}` was sampled at spark-1's server-side default, which is + `enable_thinking: false`. +- `traces_with_reasoning` — assistant messages carrying `reasoning_content` (or a literal + `` block). **This is the only in-trace evidence that the model was allowed to + think.** Verify the flag from here, never from the fact that you passed it. +- `finish_reasons` and the completion-token quartiles — `length` in that histogram means + truncation, and a truncated answer scores whatever an empty answer scores. +- `reward_mean_from_eval_log` — the mean of the `reward=` field on the harness's own + `rollout done` lines. It must equal `reward_mean_scored` exactly. An unweighted mean over + reward *components* does not, which is how six of seven means were misreported once. + +Its arithmetic is checked against the published run: it reproduces `fault-localisation` +0.9531, `canary-trap` 0.6289, `grand-exchange` 0.0055 and `drop-table-inference` 0.4174 +from `run-20260821-1401`, with `reward_mean_from_eval_log` agreeing in every case, and +`traces_with_reasoning` 0/32 in every case. + --- ## 4. What it cost, and what it scored @@ -132,6 +250,15 @@ jq -r '.traces[] | if (.rewards|length)>0 then "scored" else "errored" end' \ 224 episodes, **223 scored**. Model `brain-qwen38-dspark` (Qwen3.8-27B) on spark-1. Wall-clock 14:03 → 14:58, 55 minutes, `-c 8`. +> ⚠️ **Sampling configuration for every number in this section — read it before quoting +> one.** `verifiers==0.3.0`; `sampling = {}`, i.e. no `max_tokens` and no +> `chat_template_kwargs`, so spark-1's server-side default applied and +> **`enable_thinking` was `false`**; `temperature`, `top_p` and `reasoning_effort` all +> unset; `-c 8`; harness `null`; agent runtime `subprocess`. `traces_with_reasoning` is +> **0 of 224**. These are thinking-**off** numbers. §6 re-measures the same eight +> environments with thinking on, and the two columns are not interchangeable — +> `grand-exchange` moves by two orders of magnitude between them. + ⚠️ **Compute the mean as `sum(score × weight)` per trace.** An unweighted mean over the reward components is a different number and it is wrong — it disagrees with what the harness itself prints. Cross-check against the `reward=` field on each env's 32 @@ -354,6 +481,17 @@ Three consequences for anyone re-running this suite against a reasoning model: makes the "grand-exchange is solvable" claim — currently resting on six rollouts — cheap to settle properly. +✅ **RETIRED 2026-08-21 evening.** The bump happened — `verifiers==0.3.1.dev59`, §2 — and +every venv on this box now reads `read=None`. One correction to the paragraph above, which +matters because it is the version anyone will type: the fix is **not** `verifiers==0.4.0`. +There is no such release. `a298bcfe` is `v0.3.0-8-ga298bcfe`; the vendored HEAD is +`v0.3.0-59-g4bcb48e5`; the wheel that carries them is `0.3.1.dev59`. Everything below this +line about 600 seconds describes the old pin and is kept as the record of why the bump was +worth doing, not as current behaviour. The `max_tokens` / wall-clock contradiction it +describes — "the two limits point in opposite directions and there is no setting that +satisfies both" — no longer holds: with `read=None` the only limit left is `max_tokens`, +and §6 spends it. + ### Did the other six collapse too? Only one of them, and only partly. Completion tokens per trace, and the mean weighted @@ -389,3 +527,138 @@ recorded. - Ask spark-1 what it defaults to (`/get_server_info`) rather than assuming a model's own default applies. - If `reasoning_tokens` matters to you, read it off the provider — the trace drops it. + +--- + +## 6. The thinking-ON re-measurement, n=32 + +Run 2026-08-21 evening, after the `verifiers==0.3.1.dev59` bump of §2. This section +exists because two of §4's seven numbers were artefacts of a sampling configuration +nobody chose and nothing recorded, and because §5's remedy — "an n=32 thinking-on run is +achievable today" — was owed a run. + +> **Sampling configuration for every number in this section.** `verifiers==0.3.1.dev59` +> (`read=None`); `chat_template_kwargs = {enable_thinking = true}`; `max_tokens = 49152`; +> `temperature`, `top_p`, `reasoning_effort` unset; `-c 8`; harness `null`; agent runtime +> `subprocess`; model `brain-qwen38-dspark` on spark-1. The configs are +> `outputs/thinking-n32/*.toml`, the sweep is `outputs/thinking-n32/run_all.sh`, the runs +> are `outputs/thinking-n32/run//`. + +**Thinking is verified from the traces, not from the flag.** `traces_with_reasoning` is +**32 of 32** for `grand-exchange`, mean 56,560 characters of `reasoning_content` per +trace, against 0 of 224 for `run-20260821-1401`. + +### `grand-exchange` — 0.0055 → **0.4338** + +| | thinking off (§4) | 6 rollouts, 16k (§5) | **thinking on, n=32** | +|---|---|---|---| +| `reward_mean_attempted` | 0.0055 | 0.1667 | **0.4338** | +| traces scored | 32/32 | 6/6 | **32/32** | +| traces with reasoning | 0/32 | 6/6 | **32/32** | +| `finish_reason=length` | 0 | 5/6 | **2/32** | +| median completion tokens | 18 | 16,384 | **29,376** | +| `gate` mean (max) | 0.0000 (0.000) | — | **0.2188 (1.000)** | + +`probe.py` rates the crude maximiser at 0.082 and the plausible strategy at 0.608. **0.4338 +sits between them**, which is the first time this environment has produced a number that +means anything: 0.0055 was below crude, and 0.1667 was one perfect run and five +truncations. + +Wall clock 18:28 → 20:33, 2 h 5 min, 965,217 completion tokens for 32 rollouts. Zero +`ProviderError`s. **Under the old pin every one of these rollouts would have died**: the +median call ran ~20 minutes against a 600-second ceiling, and the median completion is +29,376 tokens against the 16,384 that section 5 could afford. + +The distribution is properly graded — not the two-spike shape thinking-off produced: + +``` + total gate profit disc tokens finish + 0.0 0.0 0.0 0.0 20305 stop <- 7 genuine zeros: finished, planned nothing usable + ... + 0.0 0.0 0.0 0.0 49152 length <- 2 truncations, both scored 0 + 0.102 0.0 0.203 0.038 38913 stop + 0.162 0.0 0.302 0.087 30042 stop + 0.413 0.0 0.653 0.397 37978 stop + 0.56 0.0 0.746 0.746 22000 stop + 0.683 0.0 0.91 0.91 20886 stop + 0.747 0.0 0.996 0.996 32545 stop + 0.993 1.0 1.0 0.977 23531 stop + 1.0 1.0 1.0 1.0 20579 stop <- 6 perfect rollouts +``` + +Nine zeros, six 1.000s, seventeen strictly between. Median 0.4165. Dropping the two +truncations gives 0.4627 over the thirty that finished — quote 0.4338 and say why, rather +than quoting the higher number. + +⚠️ **The dead `gate` in `grand-exchange` was partly a sampling artefact.** `docs/GATE_DIAGNOSIS.md` +records it at exactly 0.000, max 0.000, across all 32 thinking-off rollouts. With thinking +on it fires on **7 of 32**, mean 0.2188, max 1.000. That does not repair the shape — it is +still exact equality, still all-or-nothing, and it still only opens for a rollout that is +already at `profit` ≈ 1.0, so it adds no gradient anywhere below the top. But the +diagnosis's evidence for *this* environment was measured under a configuration in which +the model answered in 18 tokens, and the three other dead gates were measured under the +same one. **Re-read GATE_DIAGNOSIS against this column before acting on it.** + +### `drop-table-inference` — thinking on does not fit in 49,152 tokens + +The question §5 left was whether thinking on collapses this environment's two regimes +(twelve short rollouts at 0.0894, nineteen long ones at 0.6465). **It does not collapse +them; it pushes the whole distribution off the end of the token budget.** The first eight +rollouts of the n=32 run, all eight carrying `reasoning_content`: + +``` +weighted fit gate rare tokens finish + 0.4082 0.692 0.0 0.277 31649 stop + 0.4082 0.692 0.0 0.277 33803 stop + 0.0000 0.0 0.0 0.0 49152 length + 0.0000 0.0 0.0 0.0 49152 length + 0.0000 0.0 0.0 0.0 49152 length + 0.0000 0.0 0.0 0.0 49152 length + 0.0000 0.0 0.0 0.0 49152 length + 0.0000 0.0 0.0 0.0 49152 length +``` + +**Six of eight hit `finish_reason=length` at 49,152 completion tokens** — three times the +budget §5 could afford, on a model whose reasoning here runs 74–86 thousand characters. +The two that finished scored 0.408 each, above §4's short regime (0.0894) and below its +long one (0.6465). + +⚠️ **Whatever mean this run finishes with is a truncation artefact, exactly like +`grand-exchange`'s 0.1667 was.** Do not publish it as a capability score, and do not read +it as evidence about the mixture. The comparison `grand-exchange` earned — a distribution +where 30 of 32 finish — this environment has not: it needs a larger `max_tokens` (spark-1's +context is 262,144, so there is room) and a correspondingly larger wall-clock budget. One +49,152-token rollout costs roughly 25 minutes at `-c 8` on this box. + +### What is still running, and how to pick it up + +`outputs/thinking-n32/run_all.sh` was launched detached (`setsid`) and works down the +priority order in §6: grand-exchange (done), drop-table-inference, bot-detection, +redaction-pressure, schema-migration, canary-trap, fault-localisation. It holds a `flock` +on `outputs/thinking-n32/.lock`, so a second launch refuses rather than writing two evals +into one `traces.jsonl`. + +```bash +tail -f outputs/thinking-n32/sweep.log # stage transitions + per-rollout +grep 'exit=' outputs/thinking-n32/sweep.log # which stages finished, with counts +uv run --project environments/ python outputs/summarise_run.py \ + outputs/thinking-n32/run/ # the numbers, with the sampling config + +pkill -f 'thinking-n32/run_all.sh' # stop it after the current stage, +pkill -f 'bin/eval @ outputs/thinking-n32' # then kill the eval in flight +``` + +⚠️ **Read every stage's `finish_reasons` before its mean.** `length` in that histogram +means the model was cut off mid-answer, and a cut-off answer scores what an empty one +scores. A stage whose histogram is mostly `length` has measured `max_tokens`, not the +model — that is the single mistake this section exists to stop repeating. + +### Every number here is comparable only to the thinking-on column + +`grand-exchange` moves by 79× between the two sampling configurations. Nothing in §4 and +nothing in §6 may be tabulated beside the other without the configuration attached, and no +number from either belongs on `/evals` or in `eval-results.json` without it. That is the +whole lesson of the last two phases, stated as a rule: + +**A reward mean is not a number. It is a number plus a sampling configuration, and the +pair travels or neither does.** diff --git a/environments/bot_detection/pyproject.toml b/environments/bot_detection/pyproject.toml index 4c95d7c..b49258b 100644 --- a/environments/bot_detection/pyproject.toml +++ b/environments/bot_detection/pyproject.toml @@ -3,7 +3,7 @@ name = "bot-detection" version = "0.1.0" description = "bot-detection — find the scripts in an activity log without banning the players who grind like one." requires-python = ">=3.11" -dependencies = ["verifiers"] +dependencies = ["verifiers==0.3.1.dev59"] [build-system] requires = ["hatchling"] diff --git a/environments/canary_trap/pyproject.toml b/environments/canary_trap/pyproject.toml index 24ebb28..64a2eb1 100644 --- a/environments/canary_trap/pyproject.toml +++ b/environments/canary_trap/pyproject.toml @@ -3,7 +3,7 @@ name = "canary-trap" version = "0.1.0" description = "canary-trap — write contamination probes that survive a paraphrase, without accusing a clean model." requires-python = ">=3.11" -dependencies = ["verifiers"] +dependencies = ["verifiers==0.3.1.dev59"] [build-system] requires = ["hatchling"] diff --git a/environments/drop_table_inference/pyproject.toml b/environments/drop_table_inference/pyproject.toml index c54ee07..83812a4 100644 --- a/environments/drop_table_inference/pyproject.toml +++ b/environments/drop_table_inference/pyproject.toml @@ -3,7 +3,7 @@ name = "drop-table-inference" version = "0.1.0" description = "drop-table-inference — estimate a drop table from a kill log, graded on the kills you never saw." requires-python = ">=3.11" -dependencies = ["verifiers"] +dependencies = ["verifiers==0.3.1.dev59"] [build-system] requires = ["hatchling"] diff --git a/environments/fault_localisation/pyproject.toml b/environments/fault_localisation/pyproject.toml index d1d96d9..f9a7834 100644 --- a/environments/fault_localisation/pyproject.toml +++ b/environments/fault_localisation/pyproject.toml @@ -3,7 +3,7 @@ name = "fault-localisation" version = "0.1.0" description = "fault-localisation — name the service that broke, not the ones complaining about it." requires-python = ">=3.11" -dependencies = ["verifiers"] +dependencies = ["verifiers==0.3.1.dev59"] [build-system] requires = ["hatchling"] diff --git a/environments/grand_exchange/pyproject.toml b/environments/grand_exchange/pyproject.toml index 8a8505b..e6638ce 100644 --- a/environments/grand_exchange/pyproject.toml +++ b/environments/grand_exchange/pyproject.toml @@ -3,7 +3,7 @@ name = "grand-exchange" version = "0.1.0" description = "grand-exchange — place orders executed against a market window you never saw." requires-python = ">=3.11" -dependencies = ["verifiers"] +dependencies = ["verifiers==0.3.1.dev59"] [build-system] requires = ["hatchling"] diff --git a/environments/grand_exchange_live/pyproject.toml b/environments/grand_exchange_live/pyproject.toml index ac1eb0f..5ee4be8 100644 --- a/environments/grand_exchange_live/pyproject.toml +++ b/environments/grand_exchange_live/pyproject.toml @@ -3,7 +3,7 @@ name = "grand-exchange-live" version = "0.1.0" description = "grand-exchange-live — quote, watch the tape, re-quote, over a turn budget." requires-python = ">=3.11" -dependencies = ["verifiers"] +dependencies = ["verifiers==0.3.1.dev59"] [build-system] requires = ["hatchling"] diff --git a/environments/redaction_pressure/pyproject.toml b/environments/redaction_pressure/pyproject.toml index 802d4ea..d8fd2a7 100644 --- a/environments/redaction_pressure/pyproject.toml +++ b/environments/redaction_pressure/pyproject.toml @@ -6,7 +6,7 @@ requires-python = ">=3.11" dependencies = [ "pydantic==2.13.4", "regex==2026.7.19", - "verifiers==0.3.0", + "verifiers==0.3.1.dev59", ] [build-system] diff --git a/environments/schema_migration/pyproject.toml b/environments/schema_migration/pyproject.toml index f518d06..bc5b0e5 100644 --- a/environments/schema_migration/pyproject.toml +++ b/environments/schema_migration/pyproject.toml @@ -3,7 +3,7 @@ name = "schema-migration" version = "0.1.0" description = "schema-migration — split a text column into a number and a unit, graded on rows you never saw." requires-python = ">=3.11" -dependencies = ["verifiers"] +dependencies = ["verifiers==0.3.1.dev59"] [build-system] requires = ["hatchling"] diff --git a/environments/tera_spatial/pyproject.toml b/environments/tera_spatial/pyproject.toml index ea72b52..5461ae6 100644 --- a/environments/tera_spatial/pyproject.toml +++ b/environments/tera_spatial/pyproject.toml @@ -3,7 +3,7 @@ name = "tera-spatial" version = "0.1.0" description = "tera-spatial — Arena's bridge to Tera's four renderer-independent spatial environments, executed in TypeScript and graded in Python, and the tasksets that sit on it." requires-python = ">=3.11" -dependencies = ["verifiers"] +dependencies = ["verifiers==0.3.1.dev59"] # One directory, several tasksets: the environments share a vendored simulator and # a single worker, so splitting them into four wheels would ship the same 283 KB