probe: discover environments from manifests, not a hardcoded list
The environment list was written out four times — probe.py's shim tuple and its results dict, tests/test_probe.py's ENVIRONMENTS, and ci.yml's wheel loop — so adding an environment meant four edits and five workstreams collided on three files. Discovery is now a tomllib scan of environments/*/pyproject.toml plus a @probes(name) registry, with an optional [tool.arena] tasksets key for one package that ships several (tera_spatial declares four). An environment discovered without a registered probe WARNS rather than exits 1, because interactive environments land their engine before their probe. That is temporary; it becomes a hard failure with the last registration. The warning is not a licence to stop being gated. test_probe.py pins the ungated set to an explicit PENDING_PROBES allowlist that only ever shrinks — without it, removing a @probes decorator moved the name from the table into a warning line that already carries five, and every remaining assertion stayed vacuously true while the environment silently stopped being gated. Also untracks three environments/*/uv.lock files, which had been failing the lock-policy step on main. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+73
-21
@@ -1,24 +1,47 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tomllib
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
# Every environment probe.py gates. A new environment that is not added here still has to
|
||||
# print "ok", because the count assertion below counts them — so this list going stale fails
|
||||
# the build rather than quietly narrowing what the test covers.
|
||||
ENVIRONMENTS = (
|
||||
"redaction-pressure",
|
||||
"canary-trap",
|
||||
"fault-localisation",
|
||||
"schema-migration",
|
||||
"bot-detection",
|
||||
"grand-exchange",
|
||||
"drop-table-inference",
|
||||
)
|
||||
ROW = re.compile(r"^(?P<name>\S(?:.*\S)?)\s{2,}(?:-?\d+\.\d+\s*){4}\s+(?P<verdict>.+?)$")
|
||||
|
||||
# The environments that are allowed to carry no probe yet, because their taskset layer is not
|
||||
# written. Everything discovered and NOT in here MUST be gated.
|
||||
#
|
||||
# This list is the floor, and it is why it exists rather than a bare count: without it, both
|
||||
# loops below are vacuous when nothing is gated, and CI goes green while gating zero
|
||||
# environments. Commenting out a single `@probes(...)` decorator used to pass — the degated
|
||||
# name simply moved into the warning line, which already legitimately carries five names.
|
||||
#
|
||||
# It only ever shrinks. Delete a name here in the same commit that lands its probe.
|
||||
PENDING_PROBES = frozenset({
|
||||
"grand-exchange-live",
|
||||
"tera-crow-nav",
|
||||
"tera-drive-101",
|
||||
"tera-office-nav",
|
||||
"tera-california-flight",
|
||||
})
|
||||
|
||||
|
||||
def discovered_tasksets() -> set[str]:
|
||||
"""The same denominator `probe.py:discover()` uses, derived independently of it.
|
||||
|
||||
Deriving it a second time here rather than importing `probe` is the point: if discovery
|
||||
and the manifests ever disagree, this test is what says so. A directory with no
|
||||
`pyproject.toml` is not an environment to either of us.
|
||||
"""
|
||||
names: set[str] = set()
|
||||
for manifest in sorted((ROOT / "environments").glob("*/pyproject.toml")):
|
||||
data = tomllib.loads(manifest.read_text(encoding="utf-8"))
|
||||
declared = data.get("tool", {}).get("arena", {}).get("tasksets")
|
||||
names.update(declared or [data["project"]["name"]])
|
||||
return names
|
||||
|
||||
|
||||
class RootProbeIntegrationTests(unittest.TestCase):
|
||||
@@ -28,18 +51,47 @@ class RootProbeIntegrationTests(unittest.TestCase):
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
# The probe sweeps constants and runs multi-block ladders, and the interactive
|
||||
# environments will make it dearer still. A tight timeout here only ever fails a
|
||||
# slow machine, never a wrong reward.
|
||||
timeout=180,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
completed.returncode,
|
||||
0,
|
||||
msg=f"root probe failed:\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}",
|
||||
detail = f"\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
|
||||
self.assertEqual(completed.returncode, 0, msg=f"root probe failed:{detail}")
|
||||
|
||||
discovered = discovered_tasksets()
|
||||
self.assertTrue(discovered, msg="no environment manifests found — the scan is empty")
|
||||
|
||||
rows = {}
|
||||
for line in completed.stdout.splitlines()[1:]:
|
||||
match = ROW.match(line.rstrip())
|
||||
if match:
|
||||
rows[match["name"]] = match["verdict"].strip()
|
||||
|
||||
for name, verdict in rows.items():
|
||||
with self.subTest(environment=name):
|
||||
self.assertIn(name, discovered, msg=f"probe gated {name}, no manifest declares it")
|
||||
self.assertEqual(verdict, "ok", msg=f"{name} is not ok:{detail}")
|
||||
|
||||
# Anything discovered and not gated has to be named in the warning. That warning is
|
||||
# temporary — it becomes a hard exit 1 in probe.py once the last unregistered
|
||||
# environment lands its probe — and this assertion holds either way.
|
||||
ungated = discovered - set(rows)
|
||||
for name in sorted(ungated):
|
||||
with self.subTest(environment=name):
|
||||
self.assertIn(name, completed.stderr,
|
||||
msg=f"{name} is neither gated nor reported as ungated:{detail}")
|
||||
|
||||
# The floor. Being named in the warning is not a licence to stop being gated: an
|
||||
# environment may only be ungated if it is on the pending list. Without this, removing
|
||||
# a `@probes(...)` decorator moves the name from the table into the warning and every
|
||||
# assertion above stays vacuously true.
|
||||
self.assertLessEqual(
|
||||
ungated, PENDING_PROBES,
|
||||
msg=f"these were gated and no longer are: {sorted(ungated - PENDING_PROBES)}{detail}",
|
||||
)
|
||||
for environment in ENVIRONMENTS:
|
||||
with self.subTest(environment=environment):
|
||||
self.assertIn(environment, completed.stdout)
|
||||
self.assertEqual(completed.stdout.count(" ok"), len(ENVIRONMENTS))
|
||||
self.assertTrue(rows, msg=f"the probe gated nothing at all:{detail}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user