"""What the `gate` component pays the probe ladder, rung by rung. SUPERSEDED — see below. ⚠️ `probe.py` does this properly now, for every component of every environment rather than the gate of four, and it exits 1 on a component that never moves. Run that instead. This file is kept only because `docs/GATE_DIAGNOSIS.md` quotes its output verbatim as the evidence for the diagnosis, and a quoted table whose generator is gone is not evidence. To sweep a CANDIDATE gate rather than the shipped one, use `tools/regate_ladder.py`, and to measure one against real rollouts rather than against the ladder, `tools/regate.py`. `probe.py` reports one blended number per rung, so a gate that only ever fires for the oracle is invisible in its table. This prints the gate on its own — the share of tasks each rung clears it on — for the four environments where the first eval measured `gate` 0.000 mean, 0.000 max over 32 rollouts. Read-only: it imports the shipped scorers and changes nothing. uv run --project environments/grand_exchange python outputs/gate_probe.py """ from __future__ import annotations import json import sys import types from pathlib import Path ROOT = Path(__file__).resolve().parent.parent ENVS = ROOT / "environments" for pkg in ("redaction_pressure", "schema_migration", "bot_detection", "grand_exchange"): shim = types.ModuleType(pkg) shim.__path__ = [str(ENVS / pkg / pkg)] sys.modules[pkg] = shim TASKS = 24 def rate(name: str, rows: dict[str, list[bool]]) -> None: print(f"\n#### {name}") for rung, hits in rows.items(): print(f" {rung:12s} gate {sum(hits)}/{len(hits)} = {sum(hits) / len(hits):.3f}") def redaction() -> None: from redaction_pressure.corpus import build_slices from redaction_pressure.scan import measure, parse_rules def run(rules: list[dict]) -> list[bool]: out = [] for i in range(TASKS): _, held = build_slices(200_000 + i, 4, 12) o = measure(held, parse_rules("```json\n" + json.dumps(rules) + "\n```")) out.append(bool(o.clean) and o.validity == 1.0) return out oracle = [ {"pattern": r"(?<=Customer pasted their key )sk-live-[a-z0-9]{24}", "replacement": "[REDACTED]"}, {"pattern": r"(?<=Reply-to was )[a-z]+[.][a-z]+@[a-z]{6}[.]com", "replacement": "[REDACTED]"}, {"pattern": r"(?<=Callback number )[+]1-[0-9]{3}-[0-9]{3}-[0-9]{4}", "replacement": "[REDACTED]"}, {"pattern": r"(?<=They read the card )[0-9]{4}(?:-[0-9]{4}){3}", "replacement": "[REDACTED]"}, {"pattern": r"(?<=Trace pointed at )[a-z]{5}-db-[1-9][.]internal[.]lumbridge", "replacement": "[REDACTED]"}, {"pattern": r"(?<=Session )[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", "replacement": "[REDACTED]"}, {"pattern": r"(?<=Escalated by )(?:Dana|Priya|Marcus|Ines|Tomas|Rui|Nadia|Owen) (?:Okafor|Lindqvist|Baptiste|Moreau|Sato|Ahmed|Vance|Duarte)", "replacement": "[REDACTED]"}, ] rate("redaction-pressure", { "inaction": run([]), "crude": run([{"pattern": r"\S+", "replacement": "[REDACTED]"}]), "plausible": run(oracle[:4]), "six-of-seven": run(oracle[:6]), "oracle": run(oracle), }) def schema() -> None: from schema_migration.run import measure def run(sql: str) -> list[bool]: return [bool(measure(30_000 + i, 40, sql).clean) for i in range(TASKS)] naive = ("ALTER TABLE readings ADD COLUMN value_num REAL;" "ALTER TABLE readings ADD COLUMN unit TEXT;" "UPDATE readings SET value_num=CAST(substr(value_text,1,instr(value_text,' ')-1) AS REAL)," " unit=substr(value_text,instr(value_text,' ')+1);" "ALTER TABLE readings DROP COLUMN value_text;") clean = "replace(value_text,',','')" oracle = (f"ALTER TABLE readings ADD COLUMN value_num REAL;" f"ALTER TABLE readings ADD COLUMN unit TEXT;" f"UPDATE readings SET" f" value_num=CAST(CASE WHEN instr({clean},' ')>0" f" THEN substr({clean},1,instr({clean},' ')-1) ELSE {clean} END AS REAL)," f" unit=CASE WHEN instr({clean},' ')>0" f" THEN trim(substr({clean},instr({clean},' ')+1)) ELSE '' END;" f"ALTER TABLE readings DROP COLUMN value_text;") rate("schema-migration", { "inaction": run(""), "crude": run("ALTER TABLE readings ADD COLUMN value_num REAL; ALTER TABLE readings ADD COLUMN unit TEXT;"), "plausible": run(naive), "oracle": run(oracle), }) def bots() -> None: from bot_detection.accounts import build_slices from bot_detection.scan import measure, parse_accusations, reference_bots def run(fn) -> list[bool]: out = [] for i in range(TASKS): _, batch = build_slices(80_000 + i, 12) names = fn(batch) o = measure(batch, parse_accusations("```json\n" + json.dumps(names) + "\n```")) out.append(bool(o.gate)) return out def drop_one(batch): found = sorted(reference_bots(batch)) return found[:-1] rate("bot-detection", { "inaction": run(lambda b: []), "crude": run(lambda b: [a.acc_id for a in b]), "oracle-minus-one": run(drop_one), "oracle": run(lambda b: sorted(reference_bots(b))), }) def exchange() -> None: from grand_exchange.book import measure, parse_orders, reference_orders, viable_market from grand_exchange.market import SEED_BASE markets, seed = [], SEED_BASE while len(markets) < 12: m = viable_market(seed, 5, 56, 30) markets.append(m) seed = m.seed + 1 def reply(orders) -> str: return "```json\n" + json.dumps({ "expected_profit": 0, "orders": [ {"item": o.item, "quantity": o.quantity, "buy": o.buy, "sell": o.sell} for o in orders ], }) + "\n```" def run(fn) -> list[bool]: out = [] for m in markets: orders, stated = parse_orders(fn(m)) out.append(bool(measure(m, orders, stated).clean)) return out rate("grand-exchange", { "inaction": run(lambda m: "No trades today."), "oracle": run(lambda m: reply(reference_orders(m))), }) if __name__ == "__main__": redaction() schema() bots() exchange()