Pin Arena environments to stable Verifiers
This commit is contained in:
@@ -74,6 +74,30 @@ def discovered_tasksets() -> set[str]:
|
||||
|
||||
|
||||
class RootProbeIntegrationTests(unittest.TestCase):
|
||||
def test_prime_verifiers_boundary_is_pinned_and_upstream_owned(self) -> None:
|
||||
manifests = sorted((ROOT / "environments").glob("*/pyproject.toml"))
|
||||
configs = sorted((ROOT / "configs").glob("*.toml"))
|
||||
self.assertTrue(manifests)
|
||||
self.assertTrue(configs)
|
||||
|
||||
for manifest in manifests:
|
||||
with self.subTest(manifest=manifest.parent.name):
|
||||
data = tomllib.loads(manifest.read_text(encoding="utf-8"))
|
||||
self.assertIn("verifiers==0.3.1", data["project"]["dependencies"])
|
||||
|
||||
for config in configs:
|
||||
with self.subTest(config=config.stem):
|
||||
data = tomllib.loads(config.read_text(encoding="utf-8"))
|
||||
agent = data["env"]["agent"]
|
||||
self.assertEqual(agent["harness"]["id"], "null")
|
||||
self.assertEqual(agent["runtime"]["type"], "subprocess")
|
||||
|
||||
self.assertEqual(
|
||||
list((ROOT / "environments").glob("**/harness.py")),
|
||||
[],
|
||||
msg="Arena environments must use Verifiers harnesses, not ship a parallel one",
|
||||
)
|
||||
|
||||
def test_public_root_probe_gates_every_environment(self) -> None:
|
||||
completed = subprocess.run(
|
||||
[sys.executable, "probe.py"],
|
||||
|
||||
Reference in New Issue
Block a user