arena/tools/gate_probe.py

"""What the `gate` component pays the probe ladder, rung by rung.

`probe.py` reports one blended number per rung, so a gate that only ever fires for
the oracle is invisible in its table. This prints the gate on its own — the share of
tasks each rung clears it on — for the four environments where the first eval measured
`gate` 0.000 mean, 0.000 max over 32 rollouts.

Read-only: it imports the shipped scorers and changes nothing.

    uv run --project environments/grand_exchange python outputs/gate_probe.py
"""

from __future__ import annotations

import json
import sys
import types
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
ENVS = ROOT / "environments"
for pkg in ("redaction_pressure", "schema_migration", "bot_detection", "grand_exchange"):
    shim = types.ModuleType(pkg)
    shim.__path__ = [str(ENVS / pkg / pkg)]
    sys.modules[pkg] = shim

TASKS = 24


def rate(name: str, rows: dict[str, list[bool]]) -> None:
    print(f"\n#### {name}")
    for rung, hits in rows.items():
        print(f"   {rung:12s} gate {sum(hits)}/{len(hits)} = {sum(hits) / len(hits):.3f}")


def redaction() -> None:
    from redaction_pressure.corpus import build_slices
    from redaction_pressure.scan import measure, parse_rules

    def run(rules: list[dict]) -> list[bool]:
        out = []
        for i in range(TASKS):
            _, held = build_slices(200_000 + i, 4, 12)
            o = measure(held, parse_rules("```json\n" + json.dumps(rules) + "\n```"))
            out.append(bool(o.clean) and o.validity == 1.0)
        return out

    oracle = [
        {"pattern": r"(?<=Customer pasted their key )sk-live-[a-z0-9]{24}", "replacement": "[REDACTED]"},
        {"pattern": r"(?<=Reply-to was )[a-z]+[.][a-z]+@[a-z]{6}[.]com", "replacement": "[REDACTED]"},
        {"pattern": r"(?<=Callback number )[+]1-[0-9]{3}-[0-9]{3}-[0-9]{4}", "replacement": "[REDACTED]"},
        {"pattern": r"(?<=They read the card )[0-9]{4}(?:-[0-9]{4}){3}", "replacement": "[REDACTED]"},
        {"pattern": r"(?<=Trace pointed at )[a-z]{5}-db-[1-9][.]internal[.]lumbridge", "replacement": "[REDACTED]"},
        {"pattern": r"(?<=Session )[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", "replacement": "[REDACTED]"},
        {"pattern": r"(?<=Escalated by )(?:Dana|Priya|Marcus|Ines|Tomas|Rui|Nadia|Owen) (?:Okafor|Lindqvist|Baptiste|Moreau|Sato|Ahmed|Vance|Duarte)", "replacement": "[REDACTED]"},
    ]
    rate("redaction-pressure", {
        "inaction": run([]),
        "crude": run([{"pattern": r"\S+", "replacement": "[REDACTED]"}]),
        "plausible": run(oracle[:4]),
        "six-of-seven": run(oracle[:6]),
        "oracle": run(oracle),
    })


def schema() -> None:
    from schema_migration.run import measure

    def run(sql: str) -> list[bool]:
        return [bool(measure(30_000 + i, 40, sql).clean) for i in range(TASKS)]

    naive = ("ALTER TABLE readings ADD COLUMN value_num REAL;"
             "ALTER TABLE readings ADD COLUMN unit TEXT;"
             "UPDATE readings SET value_num=CAST(substr(value_text,1,instr(value_text,' ')-1) AS REAL),"
             " unit=substr(value_text,instr(value_text,' ')+1);"
             "ALTER TABLE readings DROP COLUMN value_text;")
    clean = "replace(value_text,',','')"
    oracle = (f"ALTER TABLE readings ADD COLUMN value_num REAL;"
              f"ALTER TABLE readings ADD COLUMN unit TEXT;"
              f"UPDATE readings SET"
              f" value_num=CAST(CASE WHEN instr({clean},' ')>0"
              f"   THEN substr({clean},1,instr({clean},' ')-1) ELSE {clean} END AS REAL),"
              f" unit=CASE WHEN instr({clean},' ')>0"
              f"   THEN trim(substr({clean},instr({clean},' ')+1)) ELSE '' END;"
              f"ALTER TABLE readings DROP COLUMN value_text;")
    rate("schema-migration", {
        "inaction": run(""),
        "crude": run("ALTER TABLE readings ADD COLUMN value_num REAL; ALTER TABLE readings ADD COLUMN unit TEXT;"),
        "plausible": run(naive),
        "oracle": run(oracle),
    })


def bots() -> None:
    from bot_detection.accounts import build_slices
    from bot_detection.scan import measure, parse_accusations, reference_bots

    def run(fn) -> list[bool]:
        out = []
        for i in range(TASKS):
            _, batch = build_slices(80_000 + i, 12)
            names = fn(batch)
            o = measure(batch, parse_accusations("```json\n" + json.dumps(names) + "\n```"))
            out.append(bool(o.gate))
        return out

    def drop_one(batch):
        found = sorted(reference_bots(batch))
        return found[:-1]

    rate("bot-detection", {
        "inaction": run(lambda b: []),
        "crude": run(lambda b: [a.acc_id for a in b]),
        "oracle-minus-one": run(drop_one),
        "oracle": run(lambda b: sorted(reference_bots(b))),
    })


def exchange() -> None:
    from grand_exchange.book import measure, parse_orders, reference_orders, viable_market
    from grand_exchange.market import SEED_BASE

    markets, seed = [], SEED_BASE
    while len(markets) < 12:
        m = viable_market(seed, 5, 56, 30)
        markets.append(m)
        seed = m.seed + 1

    def reply(orders) -> str:
        return "```json\n" + json.dumps({
            "expected_profit": 0,
            "orders": [
                {"item": o.item, "quantity": o.quantity, "buy": o.buy, "sell": o.sell}
                for o in orders
            ],
        }) + "\n```"

    def run(fn) -> list[bool]:
        out = []
        for m in markets:
            orders, stated = parse_orders(fn(m))
            out.append(bool(measure(m, orders, stated).clean))
        return out

    rate("grand-exchange", {
        "inaction": run(lambda m: "No trades today."),
        "oracle": run(lambda m: reply(reference_orders(m))),
    })


if __name__ == "__main__":
    redaction()
    schema()
    bots()
    exchange()