From 3025c4cf85f7e6c764d573f307c168f32070210f Mon Sep 17 00:00:00 2001 From: Kartios Date: Tue, 25 Aug 2026 14:59:00 -0700 Subject: [PATCH] Document the current public Arena boundary --- EXECUTION.md | 2 +- README.md | 16 ++++++++++------ docs/FIRST_EVAL.md | 4 ++-- probe.py | 7 +++---- tests/test_probe.py | 14 +++++--------- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/EXECUTION.md b/EXECUTION.md index ebe59f8..f8329d6 100644 --- a/EXECUTION.md +++ b/EXECUTION.md @@ -87,7 +87,7 @@ Corrected, blocking: 6. **`portal_host_samples` and `portal_container_samples` have no node identity** — no `node_id`, no host column, no PK. Per-node charts need `ALTER TABLE … ADD COLUMN IF NOT EXISTS node_id text` plus an index first. 7. **`scored = ok AND rewards non-empty` is wrong.** `write_episode` dumps with `exclude_none=True`, so a `None` reward is dropped from the file entirely — "not measured" and "key absent" are indistinguishable on the wire. Require at least one non-null `Reward`, and carry the run's declared reward-name set from the config. 8. **`Reward.value` is a `@property`** and never appears in dumped JSON. Compute `score × weight` in the ingester. -9. ⚠️ **CORRECTED 2026-08-25: there is NO verifiers 0.4.0.** The `version = "0.4.0"` at `~/vendor/prime-intellect/verifiers/pyproject.toml:136` belongs to a `[[tool.uv.dependency-metadata]]` block for **nemo-gym**, not to verifiers. Stable **`verifiers==0.3.1`** was published 2026-08-24 and is what all nine manifests pin. The layout warning itself is correct: 0.3.1 produces one flat `------` directory with no nested UUID, and `RunConfig.id` is a `PrivateAttr` absent from `configs/eval.json`. Derive the run id from the directory basename. +9. ⚠️ **CORRECTED 2026-08-25: there is NO verifiers 0.4.0.** The `version = "0.4.0"` at `~/vendor/prime-intellect/verifiers/pyproject.toml:136` belongs to a `[[tool.uv.dependency-metadata]]` block for **nemo-gym**, not to verifiers. Stable **`verifiers==0.3.1`** was published 2026-08-24 and is what all eight current public manifests pin; the Tera spatial package moved to private source. The layout warning itself is correct: 0.3.1 produces one flat `------` directory with no nested UUID, and `RunConfig.id` is a `PrivateAttr` absent from `configs/eval.json`. Derive the run id from the directory basename. 10. **Name the private console `/nodes`**, not `/compute` — `/compute` is already a public marketing section with its own nav. 11. Put `isAppHost` in `packages/shared` — two independent string checks that must agree, with a silent asymmetric failure mode (dashboard rendered under a 404 status). 12. `sendShell(url, reply)` takes no request; the signature and both call sites change. `spa-fallback.test.ts` needs a case. `@radix-ui/react-select` is **not** installed. diff --git a/README.md b/README.md index 2625112..cf13210 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Arena RL environments from Lumbridge. Apache-2.0, built to the [`verifiers`](https://github.com/PrimeIntellect-ai/verifiers) -v1 spec so the same wheel installs from the Prime Intellect Environments Hub and runs here -unchanged. +v1 spec so the same wheel can run locally or, after a separate owner-approved release, +through the Prime Intellect Environments Hub unchanged. An environment is an eval you take the gradient of. That is not a rename — it changes what the artifact has to carry. A benchmark's score is its output, so a benchmark can be a number @@ -167,12 +167,16 @@ uv run --with regex python probe.py Arena contains independently published environment libraries, so per-environment `uv.lock` files are intentionally not committed. Redaction v0.2 pins its runtime contract -in `pyproject.toml`; CI resolves it on Python 3.11 and 3.12, builds every environment, and -runs the shared four-environment probe. +in `pyproject.toml`; CI resolves it on Python 3.11 and 3.12, builds all eight public +environments, and runs the auto-discovered probe for every one of them. -## Publishing +## Publishing (deferred) -The layout mirrors `verifiers`' own, so an environment goes to the Hub without a fork: +The layout mirrors `verifiers`' own, so an environment can eventually go to the Hub without +a fork. No current environment is published there from this repository. Hub publication is +an explicit owner-approved outward action; ordinary development and CI must never run it. + +After that separate approval, the upstream command is: ```bash prime env push redaction-pressure -v PUBLIC diff --git a/docs/FIRST_EVAL.md b/docs/FIRST_EVAL.md index 58c6cc9..a5e421b 100644 --- a/docs/FIRST_EVAL.md +++ b/docs/FIRST_EVAL.md @@ -31,7 +31,7 @@ uv run --project environments/canary_trap eval @ configs/canary_trap.toml \ -c 8 -o outputs/run- --run.dir canary-trap ``` -Swap `environments/` and `configs/.toml` for any of the nine in `configs/`. The +Swap `environments/` and `configs/.toml` for any of the eight in `configs/`. The thinking-**on** sweep is `outputs/thinking-n32/run_all.sh` (gitignored, and it takes a `flock` — see below); the counters are `outputs/count_rewards.py` and `outputs/summarise_run.py`. @@ -109,7 +109,7 @@ for d in environments/*/; do printf "%-34s " "$d" grep -h "DEFAULT_TIMEOUT = " $d/.venv/lib/python*/site-packages/verifiers/v1/clients/base.py done # environments/bot_detection/ DEFAULT_TIMEOUT = httpx.Timeout(connect=5.0, read=None, write=None, pool=None) -# ... nine of nine +# ... eight of eight ``` `uv sync` rewrites each `environments/*/uv.lock`; none of them are tracked any more diff --git a/probe.py b/probe.py index 47a680d..37f8f5e 100644 --- a/probe.py +++ b/probe.py @@ -115,11 +115,10 @@ ENVS = Path(__file__).parent / "environments" def discover() -> dict[str, list[Path]]: """Taskset id -> the package directories that ship it, read off the manifests. - The seven names this file used to carry as a literal tuple are `environments/*/pyproject.toml` + The names this file once carried as a literal tuple are `environments/*/pyproject.toml` now: an environment is gated by existing, not by being remembered here. One directory may - declare more than one taskset — the Tera bridge ships four spatial tasksets out of a single - package — through an optional `[tool.arena] tasksets = [...]`. Absent that key the taskset id - is the project name, which is what all seven of the originals do. + declare more than one taskset through an optional `[tool.arena] tasksets = [...]`. Absent + that key the taskset id is the project name, which is what all eight public packages do. Other sessions create directories under `environments/` while this runs, so a manifest that disappears between the glob and the read is skipped rather than fatal. A manifest that is diff --git a/tests/test_probe.py b/tests/test_probe.py index c0411c8..64b77d9 100644 --- a/tests/test_probe.py +++ b/tests/test_probe.py @@ -41,21 +41,17 @@ STEP_AT_ORACLE = frozenset({ ("fault-localisation", "gate"), }) -# The environments that are allowed to carry no probe yet, because their taskset layer is not -# written. Everything discovered and NOT in here MUST be gated. +# The environments that are allowed to carry no probe yet. Everything discovered and NOT in +# here MUST be gated. # # This list is the floor, and it is why it exists rather than a bare count: without it, both # loops below are vacuous when nothing is gated, and CI goes green while gating zero # environments. Commenting out a single `@probes(...)` decorator used to pass — the degated # name simply moved into the warning line, which already legitimately carries five names. # -# It only ever shrinks. Delete a name here in the same commit that lands its probe. -PENDING_PROBES = frozenset({ - "tera-crow-nav", - "tera-drive-101", - "tera-office-nav", - "tera-california-flight", -}) +# It only ever shrinks. The former Tera entries moved to the private Tera repository with their +# simulator source, so every environment remaining in public Arena is gated today. +PENDING_PROBES: frozenset[str] = frozenset() def discovered_tasksets() -> set[str]: