From 3305be4ff706210f5783058378084acb75ed7224 Mon Sep 17 00:00:00 2001 From: Kartios Date: Fri, 21 Aug 2026 23:14:05 -0700 Subject: [PATCH] gates: give two of them a margin, and make the probe see components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe printed one blended number per policy, so a component pinned at 0.000 across every rung was invisible. Four gates hid there for a month. It now reports floor / best-below-oracle / oracle / ceiling for 25 components across 8 environments, and a component flat across every rung is fatal. bot-detection GATE_SLACK = 1 — fires 5/32 on the real rollouts, was 0/32. The max(1, reference_caught - SLACK) guard is verified by construction, not by sampling: without it the required count reaches 0 and an EMPTY accusation list clears the gate. Observed reference_caught is 4-6, so no amount of sampling would have found that hole. schema-migration GATE_MARGIN = 0.15 — fires 4/32, was 0/32. The cost is disclosed and bounded: the naive split clears it on 5.5% of 1,000 unseen seeds, fenced by an assert at 10%. Margin 0.10 keeps the leak at zero and fires 0/32, i.e. stays dead. A live gradient with a bounded leak beats a clean corpse. redaction-pressure is NOT given a margin, and that is the result rather than a failure. The only setting that fires at all leaves half the secrets standing and pays a four-of-seven ruleset on five seeds in six — a margin that pays for inaction is strictly worse than a dead gate. Recall maxes at 0.852 and no rollout ever cleared both clauses in one episode. It is genuinely hard, not miscalibrated. ⚠️ The per-component check did not catch the defect it was built for. Reverting schema-migration's margin to 0.0 — restoring the exact dead gate — printed ok and exited 0, because the near-oracle rung scrapes the unmargined gate on ~2 seeds in 24 and that kept best --- .gitea/workflows/ci.yml | 8 +- EXECUTION.md | 2 +- README.md | 98 ++++- docs/GATE_DIAGNOSIS.md | 131 +++++- .../bot_detection/bot_detection/scan.py | 41 +- .../redaction_pressure/scan.py | 27 ++ .../schema_migration/schema_migration/run.py | 39 +- probe.py | 398 ++++++++++++++---- tests/test_probe.py | 78 ++++ tools/gate_probe.py | 10 +- tools/regate.py | 204 +++++++++ tools/regate_ladder.py | 166 ++++++++ 12 files changed, 1107 insertions(+), 95 deletions(-) create mode 100644 tools/regate.py create mode 100644 tools/regate_ladder.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 1ffde28..1273d8d 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -21,8 +21,14 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} + # Asks git, not the disk. `find` was checking the working tree, where a lock file is a + # normal by-product of `uv sync` — so the step passed in CI's clean checkout and failed + # on every developer machine that had ever run an eval. The policy is about what is + # TRACKED: an environment is a library and must not pin its consumers' resolution. - name: Enforce library lock policy - run: test -z "$(find environments -name uv.lock -print -quit)" + run: | + tracked="$(git ls-files 'environments/*/uv.lock')" + test -z "$tracked" || { echo "tracked lock files: $tracked"; exit 1; } # An environment is `environments/*/pyproject.toml` and nothing else — the same # denominator probe.py's discover() uses. A half-created scaffold directory with no # manifest is invisible to both, so the two can never disagree about what exists. diff --git a/EXECUTION.md b/EXECUTION.md index dd58f51..8764dad 100644 --- a/EXECUTION.md +++ b/EXECUTION.md @@ -87,7 +87,7 @@ Corrected, blocking: 6. **`portal_host_samples` and `portal_container_samples` have no node identity** — no `node_id`, no host column, no PK. Per-node charts need `ALTER TABLE … ADD COLUMN IF NOT EXISTS node_id text` plus an index first. 7. **`scored = ok AND rewards non-empty` is wrong.** `write_episode` dumps with `exclude_none=True`, so a `None` reward is dropped from the file entirely — "not measured" and "key absent" are indistinguishable on the wire. Require at least one non-null `Reward`, and carry the run's declared reward-name set from the config. 8. **`Reward.value` is a `@property`** and never appears in dumped JSON. Compute `score × weight` in the ingester. -9. **0.3.0 → 0.4.0 changes the directory layout, not just the config filename** — HEAD produces one flat `------` dir with no nested uuid, and `RunConfig.id` is a `PrivateAttr` absent from `configs/eval.json`. Derive the run id from the directory basename. +9. ⚠️ **CORRECTED 2026-08-21: there is NO verifiers 0.4.0.** The `version = "0.4.0"` at `~/vendor/prime-intellect/verifiers/pyproject.toml:136` belongs to a `[[tool.uv.dependency-metadata]]` block for **nemo-gym**, not to verifiers — read as verifiers' own version it sends you hunting a release that does not exist and concluding the bump is blocked. The vendored checkout is `v0.3.0-59-g4bcb48e5`, published to PyPI as **`0.3.1.dev59`**, byte-identical to the vendored tree, and that is what all nine manifests now pin. The layout warning itself is CORRECT: **0.3.0 → dev59 changes the directory layout, not just the config filename** — HEAD produces one flat `------` dir with no nested uuid, and `RunConfig.id` is a `PrivateAttr` absent from `configs/eval.json`. Derive the run id from the directory basename. 10. **Name the private console `/nodes`**, not `/compute` — `/compute` is already a public marketing section with its own nav. 11. Put `isAppHost` in `packages/shared` — two independent string checks that must agree, with a silent asymmetric failure mode (dashboard rendered under a 404 status). 12. `sendShell(url, reply)` takes no request; the signature and both call sites change. `spa-fallback.test.ts` needs a case. `@radix-ui/react-select` is **not** installed. diff --git a/README.md b/README.md index b32eb34..ceee73a 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,14 @@ An interactive one is held to a fourth. environment does not measure generalisation and does not ship. 2. **No single-sided reward.** Every reward has a counterweight in the same episode. A reward you can max by doing the crude thing is a reward that teaches the crude thing. -3. **A zero floor and a reachable ceiling, demonstrated.** Inaction scores 0.0 and an oracle - scores 1.0, measured, in `probe.py`. An unreachable component is dead weight in the - gradient; a floor above zero pays for doing nothing. +3. **A zero floor and a reachable ceiling, demonstrated — per component, not per + environment.** Inaction scores 0.0 and an oracle scores 1.0, measured, in `probe.py`. An + unreachable component is dead weight in the gradient; a floor above zero pays for doing + nothing. The qualifier was added on 2026-08-21 and it was expensive: for a month this + file printed `oracle 1.000` and `ok` for four environments whose `gate` component scored + exactly 0.000 mean and 0.000 max over 32 real rollouts each. The blend was true and the + thing inside it was a constant. `probe.py` reports every weighted component's floor and + ceiling now, and exits 1 on one that never moves. 4. **A turn budget that binds.** In an environment the model acts in over several turns, a policy that spends every turn must score measurably below one that spends the turns worth spending. Every reward here saturates, so without a cost for looking the budget is free @@ -58,12 +63,89 @@ Reward for the degenerate strategies and for an oracle, from `uv run python prob | `redaction-pressure` | 0.000 | 0.387 *(redact everything)* | 0.549 | **1.000** | | `canary-trap` | 0.000 | 0.000 *(public-knowledge probes)* | 0.525 *(GUID recall)* | **1.000** | | `fault-localisation` | 0.000 | 0.250 *(blame the loudest)* | 0.500 | **1.000** | -| `schema-migration` | 0.000 | 0.000 *(add columns, touch nothing)* | 0.630 | **1.000** | +| `schema-migration` | 0.000 | 0.000 *(add columns, touch nothing)* | 0.641 | **1.000** | | `bot-detection` | 0.000 | 0.000 *(ban everyone)* | 0.400 *(one tell of three)* | **1.000** | | `drop-table-inference` | 0.000 | 0.179 *(copy the frequencies, floor the rest at a memorised constant)* | 0.476 *(per-item posterior over the grid)* | **1.000** | | `grand-exchange` | 0.000 | 0.082 *(buy and sell at market)* | 0.608 *(anchor on the mean and size to the volume, trap included)* | **1.000** | | `grand-exchange-live` | 0.000 | 0.126 *(buy and sell at market, on one look)* | 0.406 *(anchor on the mean, no trap filter, one look and no re-quote)* | **1.000** | +`schema-migration`'s plausible rung moved from 0.630 to 0.641 on 2026-08-21 when its gate +gained a margin — the naive split clears the loosened gate on one seed in twenty-four. Every +other cell is unchanged. + +### And the same table one reward component at a time + +The four columns above are blends, and a blend cannot show a constant inside it. Below each +component's **weighted** floor, the best any rung below the oracle manages, and its ceiling. +The interesting column is the middle one: where it equals the floor, nothing short of the +oracle earns a fraction of that term. + +| environment | component | floor | best below oracle | ceiling | +|---|---|---|---|---| +| `bot-detection` | caught | 0.000 | 0.278 | 0.350 | +| `bot-detection` | spared | 0.000 | 0.317 | 0.400 | +| `bot-detection` | gate | 0.000 | 0.250 | 0.250 | +| `canary-trap` | detection | 0.000 | 0.350 | 0.350 | +| `canary-trap` | specificity | 0.000 | 0.350 | 0.350 | +| `canary-trap` | gate | 0.000 | 0.300 | 0.300 | +| `drop-table-inference` | fit | 0.000 | 0.242 | 0.450 | +| `drop-table-inference` | rare | 0.000 | 0.326 | 0.350 | +| `drop-table-inference` | gate | 0.000 | 0.025 | 0.200 | +| `fault-localisation` | service | 0.000 | 0.300 | 0.300 | +| `fault-localisation` | fault | 0.000 | 0.250 | 0.250 | +| `fault-localisation` | evidence | 0.000 | 0.250 | 0.250 | +| `fault-localisation` | gate | 0.000 | **0.000** | 0.200 | +| `grand-exchange` | profit | 0.000 | 0.450 | 0.450 | +| `grand-exchange` | discipline | 0.000 | 0.298 | 0.300 | +| `grand-exchange` | gate | 0.000 | 0.244 | 0.250 | +| `grand-exchange-live` | profit | 0.000 | 0.382 | 0.450 | +| `grand-exchange-live` | discipline | 0.000 | 0.201 | 0.300 | +| `grand-exchange-live` | gate | 0.000 | 0.048 | 0.250 | +| `redaction-pressure` | recall | 0.000 | 0.304 | 0.350 | +| `redaction-pressure` | precision | 0.000 | 0.350 | 0.350 | +| `redaction-pressure` | gate | 0.000 | **0.000** | 0.300 | +| `schema-migration` | schema | 0.000 | 0.300 | 0.300 | +| `schema-migration` | integrity | 0.000 | 0.422 | 0.450 | +| `schema-migration` | gate | 0.000 | 0.240 | 0.250 | + +Two components are bold, and `probe.py` names both of them on the way past. They are not the +same problem. `fault-localisation`'s gate wants three fields correct at once and fires on +**29 of 32** real rollouts — the healthiest gate here; the ladder simply has no rung standing +at two-of-three. `redaction-pressure`'s fires on **0 of 32**, and unlike the two that were +fixed below, no margin rescues it: see the sweep in its scanner. + +The probe cannot tell those apart, which is why it warns rather than fails there. What tells +them apart is `tools/regate.py` — it replays the real traces in `outputs/` through a +candidate gate and reports the fire rate, holding the model's behaviour fixed and varying +only the reward. + +### The margins, and what they were measured against + +Two gates demanded exact equality with the reference and paid **nothing at all** for anything +short of it. Both were re-scored over the 32 real rollouts in `outputs/run-20260821-1401` +before being changed, and both floors were re-measured after: + +| environment | gate | before | after | inaction | crude | plausible | +|---|---|---|---|---|---|---| +| `bot-detection` | `GATE_SLACK = 1` — the reference's bots bar one, still zero wrongful bans | 0/32 | **5/32 = 0.156** | 0.000 | 0.000 | 0.000 | +| `schema-migration` | `GATE_MARGIN = 0.15` — 34 of 40 rows recompose; schema and row count still exact | 0/32 | **4/32 = 0.125** | 0.000 | 0.000 | 0.042 | +| `redaction-pressure` | unchanged, and deliberately | 0/32 | 0/32 | 0.000 | 0.000 | 0.000 | + +The slack is on the catching clause only in both cases. A single wrongful ban still shuts +bot-detection's gate outright, and a migration that leaves `value_text` standing or loses a +row still fails schema-migration's — those are format preconditions, not matters of degree. +`schema-migration`'s 0.042 is the price of the margin, named rather than hidden: the naive +parser's luckiest draw of forty awkward rows recomposes exactly 34 of them, and `probe.py` +asserts that number stays under a tenth. + +`redaction-pressure` was swept and left alone, which is the honest outcome rather than a +gap. Every margin loose enough to fire on the measured population — half the secrets left +standing — also pays a four-of-seven ruleset on five seeds in six; every margin tight enough +to keep that at zero fires on none of the 32. The failure is joint rather than a threshold: +recall maxed at 0.852 with never fewer than four secrets left, `collateral_hits == 0` held +on 6 of 32, and no rollout managed both at once. That gate is hard, not dead, and forcing a +margin onto it would trade the meaning of "redacted" for nothing measurable. + ## Running one ```bash @@ -111,5 +193,13 @@ on one DGX Spark. 16 tasks x 2 rollouts per environment, 32 rollouts each. `probe.py` shows none is unsolvable. That gap is the whole point: an environment a model already passes has no gradient left in it, and one nothing can pass has none yet. +⚠️ Four of those zeros were **0.000 mean and 0.000 max on all 32 rollouts**, which is a +different claim: not a hard gate, a constant. `docs/GATE_DIAGNOSIS.md` is the post-mortem. +Two have since been given measured margins, so re-scoring the same traces under the shipped +scorers moves `bot-detection` from 0.3488 to **0.3879** and `schema-migration` from 0.4432 +to **0.4745**. Those two rows are stale above and will be replaced by a fresh run, not +patched — and every number in that table was sampled with thinking OFF, which is its own +retraction and is documented in `docs/FIRST_EVAL.md`. + The replies parse: 32/32 rollouts produced non-zero metrics in every environment, so these are model scores rather than a harness failing to read its own output. diff --git a/docs/GATE_DIAGNOSIS.md b/docs/GATE_DIAGNOSIS.md index d7a023f..1beef72 100644 --- a/docs/GATE_DIAGNOSIS.md +++ b/docs/GATE_DIAGNOSIS.md @@ -233,5 +233,132 @@ reference by its own route rather than by copying it. a one-line consequence of house rule 3 that a component nothing below the oracle can earn is a component that does not discriminate. -⚠️ **Nothing in this file has been changed in any environment.** Four packages' reward code -is out of this lane; this is a diagnosis and a recommendation, not a patch. +⚠️ **Superseded by the addendum below.** As first written this file changed nothing in any +environment — four packages' reward code was out of that lane, and it was a diagnosis and a +recommendation rather than a patch. Two of the four have since been patched and one has been +measured and deliberately left alone. Read the addendum before acting on §3 above: its +recommendation "a margin on each" turned out to be right about two of the three and wrong +about `redaction-pressure`. + +--- + +## Addendum, same day: what was actually changed + +⚠️ The line above — "Nothing in this file has been changed in any environment" — no longer +holds. Two of the four gates were given margins and one was deliberately not. Everything +below was measured, not argued, and the measurements are reproducible from the repository +without touching a model: + +```bash +uv run --with regex python tools/regate.py # replay the 32 real rollouts per env +uv run --with regex python tools/regate_ladder.py # the same predicates over the ladder +uv run --with regex python probe.py # floors and ceilings, per component +``` + +`tools/regate.py` rebuilds each task from the seed in its trace, re-runs the shipped scorer +over the model's own reply, and asks a candidate predicate directly. That is a better +measurement than a fresh sample would have been: it holds the model's behaviour fixed and +varies only the reward, and it costs nothing on spark-1. Its `recorded_mean` reproduces +0.3488, 0.4432 and 0.4040 exactly, which is the cross-check that it is replaying the same +population `eval.log` scored. + +### `bot-detection` — fixed. `GATE_SLACK = 1` + +The reference's bots bar one, and still not one person. + +| gate | fires on 32 real rollouts | +|---|---| +| exact (as shipped) | 0/32 = 0.000 | +| **slack 1** | **5/32 = 0.156** | +| slack 2 | 15/32 = 0.469 | +| share 0.90 of the reference | 0/32 = 0.000 | + +Slack 2 is what the ladder rules out. At slack 1 the oracle's list minus one account clears +the gate and minus two does not, so it still separates a near miss from a half-right answer; +at slack 2 both clear and it separates nothing. `inaction` and `crude` score 0.000 at every +slack, and `probe.py` asserts all three of those facts rather than reporting them. + +`max(1, reference_caught - GATE_SLACK)` is load-bearing: without it a batch where the +reference catches exactly one bot would let an **empty** accusation list clear the gate. +Observed `reference_caught` is 4–6, so nothing would have caught that in a sample. + +The restraint clause was not given a margin and should not be. A single wrongful ban still +shuts the gate outright, because that asymmetry is the environment. + +### `schema-migration` — fixed. `GATE_MARGIN = 0.15` + +Thirty-four of forty rows recompose. `schema_ok` and the row count stay exact. + +| gate | fires on 32 real rollouts | `plausible` on the ladder | +|---|---|---| +| exact (as shipped) | 0/32 = 0.000 | 0.000 | +| margin 0.05 | 0/32 = 0.000 | 0.000 | +| margin 0.10 | 0/32 = 0.000 | 0.000 | +| **margin 0.15** | **4/32 = 0.125** | 0.042 | +| margin 0.20 | 8/32 = 0.250 | 0.292 | + +0.20 is out: it pays the naive split — the parser fitted to the five tidy rows on screen, +which is precisely what this environment exists to punish — on 29% of seeds. 0.15 pays it on +one seed in twenty-four, because its luckiest draw of forty awkward rows recomposes exactly +34 of them and 34/40 is the threshold to the digit. That knife edge is named in the source +and `probe.py` asserts it stays under a tenth. + +What 0.15 buys is the rung above it. A migration that strips the thousands separator — the +inference this environment is about — and still mishandles the row with no unit at all goes +from clearing the gate on 8% of seeds to clearing it on 96%. + +The two format clauses keep no margin. `wipe` — the oracle followed by `DELETE FROM +readings`, which recomposes perfectly over nothing — is now a probe rung asserted at 0.000, +so the hole the row clause was added to close cannot quietly reopen. + +### `redaction-pressure` — NOT fixed, and that is the finding + +| gate | fires on 32 real rollouts | `plausible` (4 of 7 rules) | +|---|---|---| +| exact (as shipped) | 0/32 = 0.000 | 0.000 | +| recall margin 0.10, collateral 0, innocent 0 | 0/32 = 0.000 | 0.000 | +| recall margin 0.20, collateral 1, innocent 32 | 0/32 = 0.000 | 0.000 | +| recall margin 0.50, collateral 2, innocent 160 | 3/32 = 0.094 | **0.833** | + +There is no setting that both fires on the measured population and keeps `plausible` at +zero. Everything loose enough to fire pays a ruleset that never found three of the seven +secret types, on five seeds in six; everything tight enough to hold that at zero fires on +none of the 32. + +The reason is that the failure is **joint**, not a threshold. Recall maxed at 0.852 with a +minimum of four secrets left standing; `collateral_hits == 0` held on 6 of 32; and no +rollout managed high recall and zero collateral in the same episode. So the verdict for this +one is **(c) genuinely hard**, not (a) mis-thresholded — which is a different answer from +the one this file gave in §3 above, and it is the answer the measurement gives. + +Half the secrets left in the document is not a near miss. The fix, when it comes, is a +better model or an easier corpus, not a looser gate. The sweep is recorded in +`Outcome.clean`'s docstring so it is not redone from scratch. + +### `grand-exchange` — untouched, as §3 said + +Its 0.0055 measured a sampling config and not a gate. Left alone. + +## And the blindness that let all four hide + +`probe.py` returned one blended float per rung. It now returns the **weighted components**, +prints each one's floor, its best rung below the oracle and its ceiling, and draws two lines: + +- **flat** — the same value on every rung including the oracle — is fatal. That is a constant + added to every policy's score and there is no legitimate reward of that shape. +- **step@oracle** — nothing below the oracle earns a fraction — warns and names the + component. + +The second is deliberately not fatal, and the reason is the whole difficulty of this file: it +is the shape of a dead gate AND the shape of an honest binary check, and the two are not +distinguishable from a ladder. Making it fatal would forbid every binary gate; making it +silent is what cost a month. So it is an instruction — go and replay the traces — and +`tests/test_probe.py::STEP_AT_ORACLE` is the allowlist that keeps the instruction from +becoming wallpaper. Two names are on it, `redaction-pressure/gate` (0/32, hard) and +`fault-localisation/gate` (29/32, healthy — the ladder just has no two-of-three rung). It +only ever shrinks, and a name may only leave it alongside the measurement that justifies it. + +Four environments also gained a `near-oracle` rung, which is what makes the check say +anything: a gate carrying a real band now demonstrates it — `grand-exchange`'s 0.90 bar reads +0.244 of 0.250 one unit below the reference, and `bot-detection`'s new slack reads the full +0.250 at the oracle's list minus one. diff --git a/environments/bot_detection/bot_detection/scan.py b/environments/bot_detection/bot_detection/scan.py index 9f98d4a..2b1f1f5 100644 --- a/environments/bot_detection/bot_detection/scan.py +++ b/environments/bot_detection/bot_detection/scan.py @@ -24,8 +24,11 @@ Four quantities come out of one pass: purity true positives over the accusations made, floored at the reference's count. The floor is what stops one confident accusation scoring like a full sweep: without it, naming the single most obvious script earns perfect precision. - gate every bot the reference found, and not one person. Binary; the only thing a - moderation team would actually ship. + gate the reference's bots bar one, and not one person. Binary. It demanded ALL of + them until 2026-08-21, and measured over 32 rollouts it fired zero times — + 0.25 of the reward mass with no gradient in it at all. The slack is on the + catching clause only; a single wrongful ban still shuts the gate, because + that is the asymmetry the environment exists to teach. See `GATE_SLACK`. """ from __future__ import annotations @@ -47,6 +50,25 @@ MAX_ACCUSATIONS = 64 CORRECTION_FLOOR = 0.35 LOGIN_SPREAD_FLOOR_MIN = 30 +# How many of the reference's bots the gate forgives. It used to forgive none, and over the +# 32 rollouts in `outputs/run-20260821-1401` it fired 0 times with a maximum of 0.000: a +# quarter of this environment's reward mass was a constant, and a constant contributes +# nothing to a policy gradient. `detection` never reached 1.0 in 32 attempts — its maximum +# was 0.833 — so the gate only ever paid a model that was already perfect, which is the +# point at which it has stopped needing the signal. +# +# One is measured, not chosen. Replaying those same 32 rollouts through this scorer +# (`tools/regate.py`): exact 0/32, slack 1 **5/32 = 0.156**, slack 2 15/32 = 0.469. The +# ladder is what rules slack 2 out (`tools/regate_ladder.py`): at slack 1 the oracle's list +# minus one account clears the gate and minus two does not, so the gate still separates a +# near miss from a half-right answer; at slack 2 both clear and it stops separating anything. +# Inaction and accusing everybody score 0.000 at every slack. +# +# The restraint clause is deliberately NOT given a margin. A single wrongful ban still shuts +# the gate outright, because that asymmetry is the whole environment — a missed bot is +# another day of farming, a banned player is a player who does not come back. +GATE_SLACK = 1 + def correction_share(account: Account) -> float: if not account.misclick_next: @@ -162,11 +184,16 @@ class Outcome: @property def gate(self) -> bool: - return ( - self.reference_caught > 0 - and self.true_positives >= self.reference_caught - and self.false_positives == 0 - ) + """The reference's bots bar `GATE_SLACK`, and not one person. + + `max(1, ...)` is load-bearing and not defensive dressing. Without it, a batch in + which the reference catches exactly one bot would let an EMPTY accusation list + clear the gate — house rule 3 failing at the one seed nobody thinks to sample. + """ + if self.reference_caught <= 0: + return False + needed = max(1, self.reference_caught - GATE_SLACK) + return self.true_positives >= needed and self.false_positives == 0 def measure(accounts: list[Account], accused: list[str]) -> Outcome: diff --git a/environments/redaction_pressure/redaction_pressure/scan.py b/environments/redaction_pressure/redaction_pressure/scan.py index 62742a0..388a7fd 100644 --- a/environments/redaction_pressure/redaction_pressure/scan.py +++ b/environments/redaction_pressure/redaction_pressure/scan.py @@ -158,6 +158,33 @@ class Outcome: @property def clean(self) -> bool: + """Every secret gone, nothing else touched. Six clauses, all exact, deliberately. + + ⚠️ This gate scored 0.000 mean and 0.000 max over the 32 rollouts in + `outputs/run-20260821-1401`, and unlike `bot-detection`'s and `schema-migration`'s + it was swept and **left alone**. The sweep is here so it is not redone: + + gate fires on 32 real rollouts `plausible` + exact (shipped) 0/32 = 0.000 0.000 + recall margin 0.10, collateral 0, innocent 0 0/32 = 0.000 0.000 + recall margin 0.20, collateral 1, innocent 32 0/32 = 0.000 0.000 + recall margin 0.50, collateral 2, innocent 160 3/32 = 0.094 0.833 + + Every margin loose enough to fire on the measured population pays `plausible` — a + ruleset carrying four of the seven rules, which is to say one that never found three + of the secret types — on five seeds in six. Every margin tight enough to keep + `plausible` at zero fires on none of the 32. There is no setting that does both, + because the failure is joint and not a threshold: recall maxed at 0.852 with a + minimum of four secrets left standing, `collateral_hits == 0` held on 6 of 32, and + no rollout managed high recall and zero collateral at the same time. + + So this one is (c) genuinely hard, not (a) mis-thresholded, and a margin here would + buy nothing measurable in exchange for redefining what "redacted" means in a + redaction environment. Half the secrets left in the document is not a near miss. + `probe.py` reports the component as a step at the oracle and names it; that is the + honest state, and the fix when it comes is a harder model or an easier corpus, not + a looser gate. Measured by `tools/regate.py` and `tools/regate_ladder.py`. + """ return ( self.removed_secrets == self.secrets_total and self.partial_secrets == 0 diff --git a/environments/schema_migration/schema_migration/run.py b/environments/schema_migration/schema_migration/run.py index 68de752..284b342 100644 --- a/environments/schema_migration/schema_migration/run.py +++ b/environments/schema_migration/schema_migration/run.py @@ -10,8 +10,11 @@ Four things are checked, and they are in tension by construction: rows the row count is unchanged. Without it, `DELETE FROM readings` scores perfect fidelity over an empty table — vacuously, since there is nothing left to be wrong about. - gate all three, exactly. `forge verify`'s exit code: a migration is correct or it - is not run in production. + gate schema and rows exactly, fidelity within `GATE_MARGIN`. It demanded all three + exactly until 2026-08-21 and measured over 32 rollouts it fired zero times — + 0.25 of the reward mass with no gradient in it. The two format clauses are + still exact: a migration that leaves `value_text` standing, or that loses a + row, is not a migration and no margin applies to it. """ from __future__ import annotations @@ -27,6 +30,27 @@ STATEMENT_LIMIT = 40 FORBIDDEN = ("attach", "pragma", "vacuum") TOLERANCE = 1e-9 +# How far below perfect per-row fidelity the gate still fires. Six rows of forty. +# +# It was zero, and over the 32 rollouts in `outputs/run-20260821-1401` `matched == graded` +# happened not once: fidelity maxed at 0.875, so a quarter of this environment's reward was +# a constant. The other two clauses were not the blocker — `schema_ok` held on 25 of 32 and +# the row count on the same 25. +# +# Swept over those 32 rollouts by replaying them (`tools/regate.py`): exact 0/32, +# margin 0.05 0/32, 0.10 0/32, **0.15 4/32 = 0.125**, 0.20 8/32 = 0.250. And over the probe +# ladder (`tools/regate_ladder.py`), which is what rules 0.20 out: the naive split — the +# parser fitted to the five tidy rows on screen, which is precisely the strategy this +# environment exists to punish — clears a 0.20 gate on 29% of seeds. At 0.15 it clears on +# one seed in twenty-four, because its luckiest draw of forty awkward rows recomposes 34 of +# them and 34/40 is 0.850 exactly. That is a knife edge and it is named here so nobody has +# to rediscover it; `probe.py` asserts the plausible rung stays under a tenth. +# +# What 0.15 buys is the rung above: a migration that strips the thousands separator and +# still mishandles the row with no unit at all — right about the inference, wrong about one +# shape — goes from clearing the gate on 8% of seeds to clearing it on 96%. +GATE_MARGIN = 0.15 + @dataclass class Outcome: @@ -50,7 +74,16 @@ class Outcome: @property def clean(self) -> bool: - return self.schema_ok and self.graded > 0 and self.matched == self.graded and self.rows_after == self.rows_before + """Schema and row count exactly; fidelity within `GATE_MARGIN` of perfect. + + The margin is on fidelity alone. `schema_ok` is a format precondition and + `rows_after == rows_before` is what stops `DELETE FROM readings` scoring perfect + fidelity over an empty table — neither is a matter of degree, and giving either of + them slack would reopen a hole this environment has already paid for once. + """ + return (self.schema_ok and self.graded > 0 + and self.fidelity >= 1.0 - GATE_MARGIN + and self.rows_after == self.rows_before) def _columns(con: sqlite3.Connection) -> set[str]: diff --git a/probe.py b/probe.py index 6cfdfa1..47a680d 100644 --- a/probe.py +++ b/probe.py @@ -72,8 +72,21 @@ the first time and this is what caught it: held-out ticks and the screen: a policy that read them earned 9.4% over the oracle, and the sentinel scan below is what holds that shut. -Exit code is the API: 1 if any environment's floor is above zero or its ceiling is -unreachable. Nothing here is listed by hand — the environments are `environments/*/pyproject.toml` +And then the check none of that caught, because none of it was per-term. Every probe +returned ONE blended number per rung, so `bot-detection`, `schema-migration`, +`redaction-pressure` and `grand-exchange` all printed `oracle 1.000` and `ok` while their +`gate` component scored exactly 0.000 mean and 0.000 max over 32 real rollouts each — a +quarter to a third of three environments' reward mass, constant, for a month. Three more +terms sat pinned at exactly 1.0000 on all 32 in the other direction. A blend is precisely +the thing that cannot show a constant inside it. Probes return the WEIGHTED components now +and `components()` prints each one's floor, its best rung below the oracle, and its ceiling; +a term that is the same on every rung is fatal, and a term nothing below the oracle earns a +fraction of is named. `tools/regate.py` is the other half — it replays real traces through a +candidate gate, which is the only thing that can tell a dead gate from a hard one. + +Exit code is the API: 1 if any environment's floor is above zero, its ceiling is +unreachable, or one of its reward components is constant across the whole ladder. Nothing +here is listed by hand — the environments are `environments/*/pyproject.toml` and each probe registers itself with `@probes(taskset_id)`, so adding one is a function appended at the end of this file. A manifest with no probe is a warning today and an error the day the last one is written. @@ -164,20 +177,50 @@ def probes(name: str): TASKS = 24 +def rung(items, score) -> dict[str, float]: + """One ladder rung: the mean of each WEIGHTED reward component over a rung's tasks. + + Every probe used to return one blended float per rung, and that single number is how + four dead reward components survived a month in this repository. `bot-detection`, + `schema-migration`, `redaction-pressure` and `grand-exchange` all printed `oracle 1.000` + while their `gate` term scored exactly 0.000 on every one of 32 real rollouts — because + 1.000 requires the gate, so the ceiling was genuinely reachable and rule 3 genuinely + passed, and nothing in the output was per-term. `main()` sums these back into the same + blended number for the table and then reports the terms underneath it. + + The values are weighted, deliberately: an unweighted component reads 1.000 whether it + carries a thirtieth of the reward or half of it, and what matters here is how much + reward mass has no gradient in it, not how a term looks after normalisation. + """ + totals: dict[str, float] = {} + count = 0 + for item in items: + for name, value in score(item).items(): + totals[name] = totals.get(name, 0.0) + value + count += 1 + if not count: + raise RuntimeError("a probe rung graded no tasks at all") + return {name: total / count for name, total in totals.items()} + + @probes("redaction-pressure") -def redaction() -> dict[str, float]: +def redaction() -> dict[str, dict[str, float]]: from redaction_pressure.corpus import build_slices from redaction_pressure.scan import measure, parse_rules - def run(rules: list[dict]) -> float: - total = 0.0 - for i in range(TASKS): + def run(rules: list[dict]) -> dict[str, float]: + reply = "```json\n" + json.dumps(rules) + "\n```" + + def score(i: int) -> dict[str, float]: _, held = build_slices(200_000 + i, 4, 12) - o = measure(held, parse_rules("```json\n" + json.dumps(rules) + "\n```")) - total += o.validity * ( - 0.35 * o.recall + 0.35 * o.precision + 0.30 * float(o.clean) - ) - return total / TASKS + o = measure(held, parse_rules(reply)) + return { + "recall": 0.35 * o.validity * o.recall, + "precision": 0.35 * o.validity * o.precision, + "gate": 0.30 * o.validity * float(o.clean), + } + + return rung(range(TASKS), score) oracle = [ {"pattern": r"(?<=Customer pasted their key )sk-live-[a-z0-9]{24}", "replacement": "[REDACTED]"}, @@ -188,52 +231,75 @@ def redaction() -> dict[str, float]: {"pattern": r"(?<=Session )[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", "replacement": "[REDACTED]"}, {"pattern": r"(?<=Escalated by )(?:Dana|Priya|Marcus|Ines|Tomas|Rui|Nadia|Owen) (?:Okafor|Lindqvist|Baptiste|Moreau|Sato|Ahmed|Vance|Duarte)", "replacement": "[REDACTED]"}, ] + # `six-of-seven` is not decoration and it is not a fifth adjective. It is the rung that + # makes this environment's dead `gate` VISIBLE: a ruleset that finds six of the seven + # secret types earns 0.000 of it, exactly what an empty ruleset earns, so the term is a + # step function whose only step is the oracle. `main()` names it. It was swept for a + # margin and deliberately left exact — see `Outcome.clean` in the scanner for the table. return { "inaction": run([]), "crude": run([{"pattern": r"\S+", "replacement": "[REDACTED]"}]), "plausible": run(oracle[:4]), + "six-of-seven": run(oracle[:6]), "oracle": run(oracle), } @probes("canary-trap") -def canary() -> dict[str, float]: +def canary() -> dict[str, dict[str, float]]: from canary_trap.corpus import SUBJECTS, build from canary_trap.scan import measure, parse_probes core = {f.subject: f.core for f in SUBJECTS} pat = re.compile(r"Record for ([^:]+): [a-z ]+is ([^.]+)\.") - def run(fn) -> float: - total = 0.0 - for i in range(TASKS): + def run(fn) -> dict[str, float]: + def score(i: int) -> dict[str, float]: corpus, states = build(70_000 + i, 4) o = measure(states, parse_probes("```json\n" + json.dumps(fn(corpus)) + "\n```")) - total += 0.35 * o.detection + 0.35 * o.specificity + 0.30 * float(o.clean_gate) - return total / TASKS + return { + "detection": 0.35 * o.detection, + "specificity": 0.35 * o.specificity, + "gate": 0.30 * float(o.clean_gate), + } + return rung(range(TASKS), score) + + full = lambda c: [{"question": f"{s}?", "answer": core[s]} for s, _ in pat.findall(c)] return { "inaction": run(lambda c: []), "crude": run(lambda c: [{"question": "?", "answer": "three-way"}, {"question": "?", "answer": "5432"}]), "plausible": run(lambda c: [{"question": "id?", "answer": t} for t in re.findall(r"\[([0-9a-f-]+)\]", c)]), - "oracle": run(lambda c: [{"question": f"{s}?", "answer": core[s]} for s, _ in pat.findall(c)]), + # The oracle's probe set less one fact. ⚠️ It was ADDED as "a rung between `plausible` + # and the ceiling, so `specificity` and `gate` are measured somewhere other than at + # the two ends", and it does not do that: measured per component it is + # detection 0.3500 / gate 0.3000 / specificity 0.3500 / total 1.0000 — identical to + # the oracle row to four decimals. Dropping one fact from a probe set this size costs + # nothing, so this rung sits AT the ceiling, and it is the only thing keeping + # `canary-trap/gate` off the step@oracle list below. It is kept because removing it + # would flag a gate that genuinely fires 7/32 on real rollouts, and it is documented + # rather than quietly relied on: a real mid-ladder rung here needs a perturbation that + # actually costs specificity, and nobody has designed one yet. + "oracle-minus-one": run(lambda c: full(c)[:-1]), + "oracle": run(full), } @probes("fault-localisation") -def fault() -> dict[str, float]: +def fault() -> dict[str, dict[str, float]]: from fault_localisation.incident import build, loudest - def run(fn) -> float: - total = 0.0 - for i in range(TASKS): + def run(fn) -> dict[str, float]: + def score(i: int) -> dict[str, float]: inc = build(50_000 + i) a = fn(inc) s = float(a.get("service", "") == inc.root) f = float(a.get("fault", "") == inc.fault) e = float(a.get("evidence", "") == inc.evidence) - total += 0.30 * s + 0.25 * f + 0.25 * e + 0.20 * float(s and f and e) - return total / TASKS + return {"service": 0.30 * s, "fault": 0.25 * f, "evidence": 0.25 * e, + "gate": 0.20 * float(s and f and e)} + + return rung(range(TASKS), score) # The environment's own premise: blaming the loudest service must never be right. assert all(loudest(build(50_000 + i)) != build(50_000 + i).root for i in range(TASKS)), \ @@ -242,20 +308,26 @@ def fault() -> dict[str, float]: "inaction": run(lambda inc: {}), "crude": run(lambda inc: {"service": loudest(inc), "fault": inc.fault, "evidence": "L00"}), "plausible": run(lambda inc: {"service": loudest(inc), "fault": inc.fault, "evidence": inc.evidence}), + # Right about the two hard fields, wrong about the log line. The rung that separates + # "the gate is binary" from "the gate only ever pays a byte-perfect answer". + "oracle-minus-evidence": run(lambda inc: {"service": inc.root, "fault": inc.fault, + "evidence": "L00"}), "oracle": run(lambda inc: {"service": inc.root, "fault": inc.fault, "evidence": inc.evidence}), } @probes("schema-migration") -def migration() -> dict[str, float]: +def migration() -> dict[str, dict[str, float]]: from schema_migration.run import measure - def run(sql: str) -> float: - total = 0.0 - for i in range(TASKS): + def run(sql: str) -> dict[str, float]: + def score(i: int) -> dict[str, float]: o = measure(30_000 + i, 40, sql) - total += 0.30 * float(o.schema_ok) + 0.45 * (o.fidelity * o.rows_kept) + 0.25 * float(o.clean) - return total / TASKS + return {"schema": 0.30 * float(o.schema_ok), + "integrity": 0.45 * o.fidelity * o.rows_kept, + "gate": 0.25 * float(o.clean)} + + return rung(range(TASKS), score) naive = ("ALTER TABLE readings ADD COLUMN value_num REAL;" "ALTER TABLE readings ADD COLUMN unit TEXT;" @@ -271,32 +343,79 @@ def migration() -> dict[str, float]: f" unit=CASE WHEN instr({clean},' ')>0" f" THEN trim(substr({clean},instr({clean},' ')+1)) ELSE '' END;" f"ALTER TABLE readings DROP COLUMN value_text;") - return { + # The rung `GATE_MARGIN` was chosen against: it strips the thousands separator — the + # inference the environment exists to teach, and the one `naive` skips — and still + # mishandles the row that carries no unit at all. Right about the hard thing, wrong + # about one row shape in eleven. Its fidelity runs 0.825-1.000 against `naive`'s + # 0.575-0.850, and under the shipped margin it clears the gate on 23 of 24 seeds where + # an exact gate cleared it on 2. + near = (f"ALTER TABLE readings ADD COLUMN value_num REAL;" + f"ALTER TABLE readings ADD COLUMN unit TEXT;" + f"UPDATE readings SET" + f" value_num=CAST(substr({clean},1,instr({clean},' ')-1) AS REAL)," + f" unit=substr({clean},instr({clean},' ')+1);" + f"ALTER TABLE readings DROP COLUMN value_text;") + # A migration that recomposes perfectly over no rows at all. The row clause is what + # stops it, and it carries no margin for exactly this reason — so the rung is here to + # hold that shut rather than to be believed about. + wipe = oracle + "DELETE FROM readings;" + + rows = { "inaction": run(""), "crude": run("ALTER TABLE readings ADD COLUMN value_num REAL; ALTER TABLE readings ADD COLUMN unit TEXT;"), "plausible": run(naive), + "wipe": run(wipe), + "near-oracle": run(near), "oracle": run(oracle), } + assert rows["wipe"]["gate"] <= 1e-9, \ + f"wiping the table clears the gate at {rows['wipe']['gate']:.3f} — the row clause " \ + "has acquired a margin it must never have" + # The margin's price, bounded rather than merely noticed. `naive` is the parser fitted + # to the five tidy rows on screen; its luckiest draw of forty awkward rows recomposes 34 + # of them, which is 0.850, which is the gate's threshold to the digit. It clears on one + # seed in twenty-four today. If that ever becomes a fifth, the margin has stopped being + # a margin and started paying for the shortcut. + assert rows["plausible"]["gate"] < 0.10 * 0.25, \ + f"the naive split earns {rows['plausible']['gate'] / 0.25:.3f} of the gate — the " \ + "margin is paying the strategy the environment exists to punish" + # And the margin bounded from BELOW, which is the half that was missing. Every assertion + # above bounds how much the margin may PAY; none of them noticed if it stopped existing. + # Reverting GATE_MARGIN to 0.0 — restoring the exact dead gate that fires 0/32 on real + # rollouts, the defect this margin was added to fix — printed `ok` and exited 0, because + # `near-oracle` still scrapes the unmargined gate on about two seeds in twenty-four and + # that is enough to keep `best < oracle` non-zero and the step@oracle check silent. A + # majority bar cannot be scraped: near-oracle clears the shipped margin on 983 seeds in + # 1,000 and the unmargined gate on roughly 21 in 1,000. `bot-detection` has carried the + # equivalent guard since it shipped; this is its mirror. + assert rows["near-oracle"]["gate"] > 0.5 * 0.25, \ + f"a migration one clause short of the reference earns " \ + f"{rows['near-oracle']['gate'] / 0.25:.3f} of the gate — the margin is dead and " \ + "the component carries no gradient between the crude answer and the exact one" + return rows @probes("bot-detection") -def bots() -> dict[str, float]: +def bots() -> dict[str, dict[str, float]]: from bot_detection.accounts import build_slices from bot_detection.scan import ( click_cv, correction_share, distinct_routes, log_click_sd, measure, parse_accusations, reference_bots, session_mean, ) - def run(fn, count: int = TASKS) -> float: - total = 0.0 - for i in range(count): + def run(fn, count: int = TASKS) -> dict[str, float]: + def score(i: int) -> dict[str, float]: _, batch = build_slices(80_000 + i, 12) reply = "```json\n" + json.dumps({"bots": sorted(fn(batch))}) + "\n```" o = measure(batch, parse_accusations(reply)) - total += (0.35 * o.detection * o.restraint - + 0.40 * o.restraint * o.purity - + 0.25 * float(o.gate)) - return total / count + return {"caught": 0.35 * o.detection * o.restraint, + "spared": 0.40 * o.restraint * o.purity, + "gate": 0.25 * float(o.gate)} + + return rung(range(count), score) + + def total(row: dict[str, float]) -> float: + return sum(row.values()) plausible = lambda b: [a.acc_id for a in b if correction_share(a) < 0.35] @@ -365,7 +484,8 @@ def bots() -> dict[str, float]: # combine into an attack. Each of these scored above 0.87 on the first version; all of # them must now come in under the honest single-channel attempt, or the shortcut is # still the better strategy and a model will learn it instead of the task. - ceiling = run(plausible) + plausible_row = run(plausible) + ceiling = total(plausible_row) for name, fn in ( ("routes veto + sd(log dt) band", lambda b: [a.acc_id for a in b @@ -377,7 +497,7 @@ def bots() -> dict[str, float]: lambda b: [a.acc_id for a in b if not a.break_s or (session_mean(a) >= 240 and click_cv(a) >= 0.54)]), ): - got = run(fn) + got = total(run(fn)) assert got < ceiling, \ f"decoy strategy '{name}' scores {got:.3f} against the intended {ceiling:.3f}" @@ -393,16 +513,36 @@ def bots() -> dict[str, float]: for i in range(TASKS) ), "a batch holds no efficient human — the false positive is free to avoid" - return { + # The two rungs `GATE_SLACK` was chosen between, and they are load-bearing rather than + # illustrative. The gate used to demand every bot the reference caught and fired on 0 of + # 32 real rollouts; with one bot of slack it fires on 5. What stops the slack going to + # two is right here: at one, the oracle's list minus a single account clears the gate and + # minus two does not, so the gate still separates a near miss from a half-right answer. + # At two, both clear and it separates nothing. Neither rung may ever OUT-earn the oracle, + # which is the standing gate and is asserted below rather than hoped for. + rows = { "inaction": run(lambda b: []), "crude": run(lambda b: [a.acc_id for a in b]), - "plausible": ceiling, + "plausible": plausible_row, + "oracle-minus-two": run(lambda b: sorted(reference_bots(b))[:-2]), + "oracle-minus-one": run(lambda b: sorted(reference_bots(b))[:-1]), "oracle": run(lambda b: sorted(reference_bots(b))), } + assert rows["oracle-minus-two"]["gate"] <= 1e-9, \ + f"the reference's list less two accounts clears the gate at " \ + f"{rows['oracle-minus-two']['gate'] / 0.25:.3f} — GATE_SLACK has gone past the " \ + "point where the gate distinguishes a near miss from a half-right answer" + assert 0 < rows["oracle-minus-one"]["gate"] <= rows["oracle"]["gate"], \ + "the reference's list less one account does not clear the gate — the margin is dead" + for name in ("oracle-minus-one", "oracle-minus-two", "plausible"): + assert sum(rows[name].values()) < sum(rows["oracle"].values()), \ + f"'{name}' scores {sum(rows[name].values()):.3f} against the oracle's " \ + f"{sum(rows['oracle'].values()):.3f} — something below the oracle out-earns it" + return rows @probes("grand-exchange") -def exchange() -> dict[str, float]: +def exchange() -> dict[str, dict[str, float]]: from grand_exchange.book import ( BUY_BAND, FILL_SHARE, MAX_ITEM_SHARE, MAX_ORDERS, MIN_CROSSINGS, SELL_BAND, TAX, Order, crossings, execute, measure, paper_profit, parse_orders, reference_orders, @@ -505,16 +645,22 @@ def exchange() -> dict[str, float]: } return "```json\n" + json.dumps(body) + "\n```" - def score(outcome) -> float: - return 0.45 * outcome.profit_ratio + 0.30 * outcome.discipline + 0.25 * float(outcome.clean) + def score(outcome) -> dict[str, float]: + return {"profit": 0.45 * outcome.profit_ratio, + "discipline": 0.30 * outcome.discipline, + "gate": 0.25 * float(outcome.clean)} - def run(fn, markets=None) -> float: + def run(fn, markets=None) -> dict[str, float]: markets = LADDER if markets is None else markets - total = 0.0 - for market in markets: + + def one(market) -> dict[str, float]: orders, stated = parse_orders(fn(market)) - total += score(measure(market, orders, stated)) - return total / len(markets) + return score(measure(market, orders, stated)) + + return rung(markets, one) + + def total(row: dict[str, float]) -> float: + return sum(row.values()) def gp(fn, markets=None) -> float: markets = LADDER if markets is None else markets @@ -566,7 +712,8 @@ def exchange() -> dict[str, float]: # the history, whatever that plan does about size, and it must do so in EVERY block of # twenty-four, not on average: an average holds while one block of the size a run # actually grades goes the other way, and that block is where the shortcut gets learned. - ceiling = run(plausible) + plausible_row = run(plausible) + ceiling = total(plausible_row) for name, fn in ( ("last-tick anchor, no filter", shortcut), ("last-tick anchor, filtered and ranked", @@ -574,7 +721,7 @@ def exchange() -> dict[str, float]: ("last-tick anchor, whole buy limit", lambda m: reply(band(m, m.items, last, by_limit))), ): for block in BLOCKS: - got, bar = run(fn, block), run(plausible, block) + got, bar = total(run(fn, block)), total(run(plausible, block)) assert got < bar, \ f"shortcut '{name}' scores {got:.3f} against {bar:.3f} on baskets " \ f"{block[0].seed}-{block[-1].seed} — the anchor is not worth estimating" @@ -600,7 +747,7 @@ def exchange() -> dict[str, float]: if earned > best: best, best_cell = earned, (bb, sb) if earned > gp(reference_orders): - got = run(lambda m, c=cell: reply(c(m))) + got = total(run(lambda m, c=cell: reply(c(m)))) assert got >= 0.90, \ f"band ({bb}, {sb}) earns {earned:,.0f} gp against the reference's " \ f"{gp(reference_orders):,.0f} and scores {got:.3f} — the reward is an " \ @@ -640,10 +787,12 @@ def exchange() -> dict[str, float]: return reply(orders) return fn + perturbations: dict[str, dict[str, float]] = {} for kind, floor in (("one wasted unit", 0.999), ("first order 1 smaller", 0.95), ("first order 1 larger", 0.95), ("every buy limit +1", 0.95), ("every sell limit -1", 0.95)): - got = run(perturbed(kind)) + perturbations[kind] = run(perturbed(kind)) + got = total(perturbations[kind]) assert got >= floor, \ f"perturbation '{kind}' scores {got:.3f} against a floor of {floor} — the gate " \ "is a knife-edge at the reference rather than a bar a good run clears" @@ -667,21 +816,27 @@ def exchange() -> dict[str, float]: "```json\n" + "[" * 20_000 + "]" * 20_000 + "\n```", ): orders, stated = parse_orders(hostile) - got = score(measure(LADDER[0], orders, stated)) + got = sum(score(measure(LADDER[0], orders, stated)).values()) assert got <= 1e-9, f"hostile reply {hostile[:40]!r} scores {got:.3f}, not zero" assert stated is None or math.isfinite(stated), \ f"hostile reply {hostile[:40]!r} put a non-finite number in the trace" + # The near rung costs nothing: `perturbations` already holds it. Shrinking the first + # order by a single unit is the smallest departure from the reference this file knows + # how to make, and TARGET_SHARE = 0.90 is what lets it still clear the gate. That is + # what a band looks like from the outside, and it is the shape the other three gates + # were argued against. return { "inaction": run(lambda m: "No trades today."), "crude": run(lambda m: reply(market_order(m))), - "plausible": ceiling, + "plausible": plausible_row, + "near-oracle": perturbations["first order 1 smaller"], "oracle": run(oracle), } @probes("drop-table-inference") -def drops() -> dict[str, float]: +def drops() -> dict[str, dict[str, float]]: from drop_table_inference.estimate import ( _tail_loss, measure, parse_estimate, reference_estimate, ) @@ -698,14 +853,17 @@ def drops() -> dict[str, float]: GRID = ([n / COMMON_DENOMINATOR for n in COMMON_NUMERATORS] + [1.0 / d for d in LADDER_DENOMINATORS]) - def run(fn) -> float: - total = 0.0 - for stream in streams: + def parts_of(fn) -> dict[str, float]: + def one(stream) -> dict[str, float]: estimate = fn(stream) reply = "" if estimate is None else "```json\n" + json.dumps(estimate) + "\n```" o = measure(stream, parse_estimate(reply, stream.table.items)) - total += 0.45 * o.fit + 0.35 * o.rare + 0.20 * float(o.clean) - return total / TASKS + return {"fit": 0.45 * o.fit, "rare": 0.35 * o.rare, "gate": 0.20 * float(o.clean)} + + return rung(streams, one) + + def run(fn) -> float: + return sum(parts_of(fn).values()) def parts(fn) -> tuple[float, float, float]: """raw_fit, raw_rare and restraint, averaged. The rewards are products of these, and @@ -780,8 +938,11 @@ def drops() -> dict[str, float]: # FENCE 2. Frequency-copying is what `rare` exists to catch, and a memorised floor is # what un-caught it. Both have to stay well under the honest attempt that reads the # structure, or the environment teaches the hack. - honest = run(grid) - crude = max(run(floored(c)) for c in (2e-5, 5e-5, 1e-4, 2e-4, 4e-4, 8e-4)) + honest_parts = parts_of(grid) + honest = sum(honest_parts.values()) + crude_parts = max((parts_of(floored(c)) for c in (2e-5, 5e-5, 1e-4, 2e-4, 4e-4, 8e-4)), + key=lambda row: sum(row.values())) + crude = sum(crude_parts.values()) assert crude < 0.6 * honest, \ f"frequency-copying with a memorised floor scores {crude:.3f} against {honest:.3f} for reading the table" @@ -816,16 +977,23 @@ def drops() -> dict[str, float]: assert parts(frequency)[2] > 0.99 and run(frequency) < 0.12, \ "restraint is not free to the frequency copy — it is being scored as accuracy, not as restraint" + # The near rung. `gate` here already carries a margin — `GATE_MARGIN = 0.05`, three + # sub-scores each within five percent of the reference — and it is the only one of the + # four that fired in the first eval, on 22.6% of rollouts. Scaling the reference by two + # percent is what shows that from the ladder rather than from the changelog: it is not + # the reference, and it still clears. + nudged = lambda s: {i: 1.02 * v for i, v in reference(s).items()} return { - "inaction": run(lambda s: None), - "crude": crude, - "plausible": honest, - "oracle": run(reference), + "inaction": parts_of(lambda s: None), + "crude": crude_parts, + "plausible": honest_parts, + "near-oracle": parts_of(nudged), + "oracle": parts_of(reference), } @probes("grand-exchange-live") -def exchange_live() -> dict[str, float]: +def exchange_live() -> dict[str, dict[str, float]]: """The stepped form. Rules 3 and 4, and the two ways this one could be quietly wrong. The ladder is not re-derived here. `environments/grand_exchange_live/measure_ladder.py` @@ -970,11 +1138,24 @@ def exchange_live() -> dict[str, float]: f"'{lower}' scores {rows[lower]['total']:.3f} against '{upper}' at " \ f"{rows[upper]['total']:.3f} — the ladder is upside down" + # `measure_ladder.ladder` already scored every policy in the family, so the extra rungs + # here are free: the component block gets six places to look at `gate` rather than the + # two ends, and `exhaustive` — which spends the whole turn budget — is the one that says + # whether a 0.90 band pays for burning it. + def parts(name: str) -> dict[str, float]: + row = rows[name] + return {"profit": 0.45 * row["profit_ratio"], + "discipline": 0.30 * row["discipline"], + "gate": 0.25 * row["clean"]} + return { - "inaction": rows["inaction"]["total"], - "crude": rows["crude (market orders, one look)"]["total"], - "plausible": rows["plausible (mean anchor, no filter)"]["total"], - "oracle": rows["oracle (the live reference)"]["total"], + "inaction": parts("inaction"), + "crude": parts("crude (market orders, one look)"), + "plausible": parts("plausible (mean anchor, no filter)"), + "impatient": parts("impatient (one-shot reference, one look)"), + "restate": parts("restate (re-quote the same book every look)"), + "exhaustive": parts("exhaustive (re-quote every look)"), + "oracle": parts("oracle (the live reference)"), } @@ -1006,14 +1187,79 @@ def main() -> int: file=sys.stderr) results = {name: _PROBES[name]() for name in sorted(discovered) if name in _PROBES} + totals = {name: {r: sum(row.values()) for r, row in rows.items()} + for name, rows in results.items()} + print(f"{'environment':22}{'inaction':>10}{'crude':>9}{'plausible':>11}{'oracle':>9} verdict") failed = False - for name, r in results.items(): + for name, r in totals.items(): + for required in CANONICAL: + if required not in r: + print(f"{name} has no '{required}' rung — the ladder is not a ladder", + file=sys.stderr) + return 1 ok = r["inaction"] <= 1e-9 and r["oracle"] >= 1.0 - 1e-9 failed |= not ok print(f"{name:22}{r['inaction']:10.3f}{r['crude']:9.3f}{r['plausible']:11.3f}" f"{r['oracle']:9.3f} {'ok' if ok else 'FAILS RULE 3'}") - return 1 if failed else 0 + + return 1 if components(results) or failed else 0 + + +CANONICAL = ("inaction", "crude", "plausible", "oracle") + + +def components(results: dict[str, dict[str, dict[str, float]]]) -> bool: + """Per-component floors and ceilings, and the two verdicts that are worth having. + + The table above is a blend. Four reward components in this repository scored exactly + 0.000 mean and 0.000 max over 32 real rollouts each — a quarter to a third of three + environments' reward mass with no gradient in it — and three more sat pinned at exactly + 1.0000 on all 32. Every one of them printed `oracle 1.000` and `ok` for a month, because + a blend is exactly the thing that cannot show you a constant inside it. + + Two lines are drawn here and they forbid different things. + + **flat** is fatal. A component with the same weighted value on every rung of the ladder, + the oracle included, is a constant added to every policy's score. It cannot be climbed, + it cannot be lost, and in training it contributes nothing at all. There is no legitimate + reward term of this shape, so it exits 1. + + **step@oracle** warns and names the component. Nothing below the oracle earns a fraction + of it: the floor and the best any non-oracle rung manages are the same number. That is + the exact shape of the four dead gates — and it is ALSO the honest shape of a genuinely + binary check that only a correct answer clears, which is why it is not fatal. The two are + not distinguishable from the ladder, and pretending otherwise would either forbid every + binary gate or forgive every dead one. What distinguishes them is the fire rate over real + rollouts, which this file cannot see and `tools/regate.py` measures. + + So the warning is an instruction rather than a verdict: go and replay the traces. When + the answer comes back 0/32, the component is dead and the margin belongs in the scorer — + `bot_detection.GATE_SLACK` and `schema_migration.GATE_MARGIN` are two that came back + that way. When it comes back 5/32 or 22.6%, the gate is a bar and the ladder simply has + no rung standing near it; add one, as `near-oracle` does for four environments here. + """ + print(f"\n {'environment':22}{'component':13}{'floor':>8}{'best13}" + f"{'oracle':>9}{'ceiling':>9} verdict") + fatal = False + for name, rows in results.items(): + for part in sorted(next(iter(rows.values()))): + values = {label: row[part] for label, row in rows.items()} + floor, ceiling = min(values.values()), max(values.values()) + oracle = values["oracle"] + below = max(v for label, v in values.items() if label != "oracle") + if ceiling - floor <= 1e-9: + verdict, fatal = "FLAT — no gradient in any direction", True + elif below - floor <= 1e-9: + verdict = "step@oracle — nothing below the oracle earns any of it" + else: + verdict = "ok" + print(f" {name:22}{part:13}{floor:8.3f}{below:13.3f}{oracle:9.3f}" + f"{ceiling:9.3f} {verdict}") + if fatal: + print("a reward component is constant across the whole ladder — it is not a reward", + file=sys.stderr) + return fatal if __name__ == "__main__": diff --git a/tests/test_probe.py b/tests/test_probe.py index d083e11..8f4e5e7 100644 --- a/tests/test_probe.py +++ b/tests/test_probe.py @@ -11,6 +11,36 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] ROW = re.compile(r"^(?P\S(?:.*\S)?)\s{2,}(?:-?\d+\.\d+\s*){4}\s+(?P.+?)$") +# The per-component block, which is indented by two spaces precisely so that ROW — anchored +# at `^\S` — cannot swallow it. Four numbers there, five here. +PART = re.compile( + r"^ (?P\S+)\s+(?P\S+)\s+" + r"(?P-?\d+\.\d+)\s+(?P-?\d+\.\d+)\s+" + r"(?P-?\d+\.\d+)\s+(?P-?\d+\.\d+)\s+(?P.+?)$" +) + +# Reward components that nothing below the oracle earns a fraction of. `probe.py` warns and +# names them; this is what stops the warning becoming wallpaper. +# +# Both of these were investigated by replaying `outputs/run-20260821-1401` through a +# candidate gate (`tools/regate.py`), which is the only measurement that separates a dead +# gate from a merely binary one, and both were left as they are on purpose: +# +# redaction-pressure/gate fired 0/32. Every margin loose enough to fire on the measured +# population also pays a four-rule ruleset on five seeds in six. +# Genuinely hard, not mis-thresholded. See `Outcome.clean`. +# fault-localisation/gate fired 29/32 — 0.9062, the healthiest gate in the repository. +# The ladder simply has no rung that gets two of three fields +# right and clears it, because the gate wants all three. +# +# `bot-detection/gate` and `schema-migration/gate` were on this list and are not any more: +# they came back 0/32 and were given margins. It only ever shrinks, and a name may only be +# deleted alongside the measurement that justifies it. +STEP_AT_ORACLE = frozenset({ + ("redaction-pressure", "gate"), + ("fault-localisation", "gate"), +}) + # The environments that are allowed to carry no probe yet, because their taskset layer is not # written. Everything discovered and NOT in here MUST be gated. # @@ -92,6 +122,54 @@ class RootProbeIntegrationTests(unittest.TestCase): ) self.assertTrue(rows, msg=f"the probe gated nothing at all:{detail}") + # --- and the same again, one reward component at a time ------------------------- + # Everything above passed for a month while four `gate` components scored exactly + # 0.000 mean and 0.000 max over 32 real rollouts, because a blended `oracle 1.000` + # is exactly the thing that cannot show a constant inside it. + parts = {} + for line in completed.stdout.splitlines(): + match = PART.match(line.rstrip()) + if match: + parts[(match["env"], match["component"])] = match + + self.assertTrue(parts, msg=f"the probe printed no per-component block:{detail}") + for name in rows: + with self.subTest(environment=name): + self.assertTrue( + any(env == name for env, _ in parts), + msg=f"{name} is gated but no component of it is reported:{detail}", + ) + + for (env, component), match in sorted(parts.items()): + with self.subTest(environment=env, component=component): + verdict = match["verdict"].strip() + self.assertFalse( + verdict.startswith("FLAT"), + msg=f"{env}/{component} is constant across the ladder:{detail}", + ) + # House rule 3, per term rather than per environment. A blended floor of + # 0.000 is compatible with one component paying for inaction and another + # going negative to cancel it; this is the form that is not. + self.assertLessEqual( + float(match["floor"]), 1e-9, + msg=f"{env}/{component} pays {match['floor']} for the worst rung on the " + f"ladder — inaction is being paid for a component:{detail}", + ) + stepped = verdict.startswith("step@oracle") + if (env, component) in STEP_AT_ORACLE: + self.assertTrue( + stepped, + msg=f"{env}/{component} now carries a gradient below the oracle — " + f"delete it from STEP_AT_ORACLE:{detail}", + ) + else: + self.assertFalse( + stepped, + msg=f"{env}/{component} earns nothing below the oracle and is not " + f"on the STEP_AT_ORACLE list. Replay the traces through it with " + f"tools/regate.py before adding it:{detail}", + ) + if __name__ == "__main__": unittest.main() diff --git a/tools/gate_probe.py b/tools/gate_probe.py index db856e7..7eae2e8 100644 --- a/tools/gate_probe.py +++ b/tools/gate_probe.py @@ -1,4 +1,12 @@ -"""What the `gate` component pays the probe ladder, rung by rung. +"""What the `gate` component pays the probe ladder, rung by rung. SUPERSEDED — see below. + +⚠️ `probe.py` does this properly now, for every component of every environment rather than +the gate of four, and it exits 1 on a component that never moves. Run that instead. This +file is kept only because `docs/GATE_DIAGNOSIS.md` quotes its output verbatim as the +evidence for the diagnosis, and a quoted table whose generator is gone is not evidence. To +sweep a CANDIDATE gate rather than the shipped one, use `tools/regate_ladder.py`, and to +measure one against real rollouts rather than against the ladder, `tools/regate.py`. + `probe.py` reports one blended number per rung, so a gate that only ever fires for the oracle is invisible in its table. This prints the gate on its own — the share of diff --git a/tools/regate.py b/tools/regate.py new file mode 100644 index 0000000..af2d33a --- /dev/null +++ b/tools/regate.py @@ -0,0 +1,204 @@ +"""Re-score the traces that are already on disk under a proposed gate, and print the fire rate. + +A margin argued from a distribution is a guess. This replays `outputs/run-20260821-1401` +— 32 real rollouts per environment, `brain-qwen38-dspark`, thinking off — through the +shipped scorer, rebuilds each task from its seed, and asks the candidate predicate +directly. The model's behaviour is held fixed and only the reward varies, which is a +cleaner measurement than a fresh sample would be and does not touch spark-1. + +It reads traces and environment sources. It writes nothing. + + uv run --with regex python tools/regate.py [run-directory] +""" + +from __future__ import annotations + +import json +import re +import sys +import types +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +ENVS = ROOT / "environments" +for _pkg in ("redaction_pressure", "schema_migration", "bot_detection"): + _shim = types.ModuleType(_pkg) + _shim.__path__ = [str(ENVS / _pkg / _pkg)] + sys.modules[_pkg] = _shim + +RUN = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / "outputs" / "run-20260821-1401" + + +def traces(env: str) -> list[dict]: + """Every trace in one environment's `traces.jsonl`, errored rollouts included. + + An errored rollout has `rewards: {}` and is still a rollout the model produced; it is + counted in the denominator, because a gate that fires only on the traces that happened + to succeed is a fire rate over a filtered population. + """ + out = [] + for line in (RUN / env / "traces.jsonl").read_text().splitlines(): + if line.strip(): + out.extend(json.loads(line)["traces"]) + return out + + +def last_reply(trace: dict) -> str: + for node in reversed(trace["nodes"]): + if node["message"]["role"] == "assistant": + return node["message"].get("content") or "" + return "" + + +def recorded_mean(env: str) -> tuple[float, int]: + """The blended reward as `eval` recorded it, and how many traces carried one. + + `eval` exits 0 even when every rollout errors, and an errored trace lands with + `rewards: {}`. Counting those as zero would understate the mean; counting them out of + the denominator would overstate the fire rates above. Both numbers are printed. + """ + scored = [t for t in traces(env) if t["rewards"]] + if not scored: + return 0.0, 0 + total = sum(sum(r["score"] * r["weight"] for r in t["rewards"].values()) for t in scored) + return total / len(scored), len(scored) + + +def shifted(env: str, weight: float, fired: list[bool]) -> None: + """What the gate change does to the environment's headline number.""" + before, n = recorded_mean(env) + rate = sum(fired) / len(fired) + print(f" reward mean: recorded {before:.4f} over {n} scored traces" + f" -> {before + weight * rate:.4f} with gate at {rate:.3f} x weight {weight}") + + +def report(name: str, rows: list[tuple[str, list[bool]]]) -> None: + print(f"\n#### {name} n={len(rows[0][1])} real rollouts") + for label, hits in rows: + print(f" {label:44s} {sum(hits):2d}/{len(hits)} = {sum(hits) / len(hits):.3f}") + + +def bots() -> None: + from bot_detection.accounts import build_slices + from bot_detection.scan import measure, parse_accusations + + outcomes = [] + for trace in traces("bot-detection"): + data = trace["task"]["data"] + _, batch = build_slices(data["seed"], data["graded"]) + outcomes.append(measure(batch, parse_accusations(last_reply(trace)))) + + def fires(slack: int) -> list[bool]: + return [ + o.reference_caught > 0 + and o.true_positives >= o.reference_caught - slack + and o.false_positives == 0 + for o in outcomes + ] + + def share(target: float) -> list[bool]: + return [ + o.reference_caught > 0 + and o.true_positives >= target * o.reference_caught + and o.false_positives == 0 + for o in outcomes + ] + + print("\n reference_caught:", + sorted({o.reference_caught for o in outcomes}), + " detection max", f"{max(o.detection for o in outcomes):.3f}") + report("bot-detection", [ + ("SHIPPED Outcome.gate", [bool(o.gate) for o in outcomes]), + ("exact: every bot the reference caught", fires(0)), + ("slack 1 (all but one)", fires(1)), + ("slack 2 (all but two)", fires(2)), + ("share 0.90 of the reference", share(0.90)), + ("share 0.75 of the reference", share(0.75)), + ("clause: false_positives == 0 alone", [o.false_positives == 0 for o in outcomes]), + ]) + shifted("bot-detection", 0.25, [bool(o.gate) for o in outcomes]) + + +def migration() -> None: + from schema_migration.run import measure + + # `schema_migration.taskset` imports verifiers and pydantic; the two lines of parsing + # this needs are copied rather than dragging the training stack into a read-only tool. + block = re.compile(r"```(?:sql)?\s*\n(.*?)```", re.DOTALL) + + def parse_sql(reply: str) -> str: + blocks = block.findall(reply or "") + return (blocks[-1] if blocks else (reply or "")).strip() + + outcomes = [] + for trace in traces("schema-migration"): + data = trace["task"]["data"] + outcomes.append(measure(data["seed"], data["held_out"], parse_sql(last_reply(trace)))) + + def fires(margin: float) -> list[bool]: + return [ + o.schema_ok and o.graded > 0 + and o.fidelity >= 1.0 - margin + and o.rows_after == o.rows_before + for o in outcomes + ] + + print("\n fidelity:", " ".join(f"{o.fidelity:.3f}" for o in sorted(outcomes, key=lambda o: -o.fidelity)[:8]), + "... max", f"{max(o.fidelity for o in outcomes):.3f}") + report("schema-migration", [ + ("SHIPPED Outcome.clean", [bool(o.clean) for o in outcomes]), + ("exact: 40 of 40 rows recompose", fires(0.0)), + ("margin 0.05 (38 of 40)", fires(0.05)), + ("margin 0.10 (36 of 40)", fires(0.10)), + ("margin 0.15 (34 of 40)", fires(0.15)), + ("margin 0.20 (32 of 40)", fires(0.20)), + ("clauses: schema_ok and rows kept, fidelity free", fires(1.0)), + ]) + shifted("schema-migration", 0.25, [bool(o.clean) for o in outcomes]) + + +def redaction() -> None: + from redaction_pressure.corpus import build_slices + from redaction_pressure.scan import measure, parse_rules + + outcomes = [] + for trace in traces("redaction-pressure"): + data = trace["task"]["data"] + _, held = build_slices(data["seed"], data["visible"], data["held_out"]) + outcomes.append(measure(held, parse_rules(last_reply(trace)))) + + def fires(recall_margin: float, collateral: int, innocent: int) -> list[bool]: + return [ + o.secrets_total > 0 + and o.validity == 1.0 + and o.partial_secrets == 0 + and o.recall >= 1.0 - recall_margin + and o.collateral_hits <= collateral + and o.innocent_lost <= innocent + for o in outcomes + ] + + print("\n recall max", f"{max(o.recall for o in outcomes):.3f}", + " residual_hits min", min(o.residual_hits for o in outcomes), + " collateral_hits min", min(o.collateral_hits for o in outcomes), + " innocent_lost min", min(o.innocent_lost for o in outcomes)) + report("redaction-pressure", [ + ("SHIPPED Outcome.clean", [bool(o.clean) for o in outcomes]), + ("exact on all six clauses", fires(0.0, 0, 0)), + ("recall margin 0.10, collateral 0, innocent 0", fires(0.10, 0, 0)), + ("recall margin 0.20, collateral 0, innocent 0", fires(0.20, 0, 0)), + ("recall margin 0.20, collateral 1, innocent 32", fires(0.20, 1, 32)), + ("recall margin 0.20, collateral 2, innocent 160", fires(0.20, 2, 160)), + ("recall margin 0.50, collateral 2, innocent 160", fires(0.50, 2, 160)), + ("clause: recall == 1.0 alone", [o.recall >= 1.0 for o in outcomes]), + ("clause: collateral_hits == 0 alone", [o.collateral_hits == 0 for o in outcomes]), + ("clause: innocent_lost == 0 alone", [o.innocent_lost == 0 for o in outcomes]), + ]) + shifted("redaction-pressure", 0.30, [bool(o.clean) for o in outcomes]) + + +if __name__ == "__main__": + print(f"replaying {RUN}") + bots() + migration() + redaction() diff --git a/tools/regate_ladder.py b/tools/regate_ladder.py new file mode 100644 index 0000000..51e724c --- /dev/null +++ b/tools/regate_ladder.py @@ -0,0 +1,166 @@ +"""The other half of `regate.py`: what a candidate gate pays the PROBE LADDER. + +A fire rate over real traces says a margin has a gradient. It says nothing about whether +the margin also pays for doing nothing, which is house rule 3 and the entire risk of +loosening a gate. This runs the same candidate predicates over `probe.py`'s rungs — +inaction, crude, plausible, near-oracle, oracle — and prints the share of tasks each +clears. A margin that moves `inaction` or `crude` off 0.000 must not ship. + + uv run --with regex python tools/regate_ladder.py +""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +ENVS = ROOT / "environments" +for _pkg in ("redaction_pressure", "schema_migration", "bot_detection"): + _shim = types.ModuleType(_pkg) + _shim.__path__ = [str(ENVS / _pkg / _pkg)] + sys.modules[_pkg] = _shim + +TASKS = 24 + + +def report(name: str, gates: dict[str, object], rungs: dict[str, list]) -> None: + print(f"\n#### {name}") + width = max(len(r) for r in rungs) + print(f" {'rung':{width}s} " + "".join(f"{g:>28s}" for g in gates)) + for rung, outcomes in rungs.items(): + cells = "".join( + f"{sum(1 for o in outcomes if fn(o)) / len(outcomes):>28.3f}" + for fn in gates.values() + ) + print(f" {rung:{width}s} {cells}") + + +def bots() -> None: + from bot_detection.accounts import build_slices + from bot_detection.scan import measure, parse_accusations, reference_bots + + def run(fn) -> list: + out = [] + for i in range(TASKS): + _, batch = build_slices(80_000 + i, 12) + reply = "```json\n" + json.dumps({"bots": sorted(fn(batch))}) + "\n```" + out.append(measure(batch, parse_accusations(reply))) + return out + + def slack(k: int): + return lambda o: (o.reference_caught > 0 + and o.true_positives >= max(1, o.reference_caught - k) + and o.false_positives == 0) + + report("bot-detection", {"shipped (exact)": slack(0), "slack 1": slack(1), "slack 2": slack(2)}, { + "inaction": run(lambda b: []), + "crude (accuse everyone)": run(lambda b: [a.acc_id for a in b]), + "oracle minus one": run(lambda b: sorted(reference_bots(b))[:-1]), + "oracle minus two": run(lambda b: sorted(reference_bots(b))[:-2]), + "oracle": run(lambda b: sorted(reference_bots(b))), + }) + + +def migration() -> None: + from schema_migration.run import measure + + naive = ("ALTER TABLE readings ADD COLUMN value_num REAL;" + "ALTER TABLE readings ADD COLUMN unit TEXT;" + "UPDATE readings SET value_num=CAST(substr(value_text,1,instr(value_text,' ')-1) AS REAL)," + " unit=substr(value_text,instr(value_text,' ')+1);" + "ALTER TABLE readings DROP COLUMN value_text;") + clean = "replace(value_text,',','')" + oracle = (f"ALTER TABLE readings ADD COLUMN value_num REAL;" + f"ALTER TABLE readings ADD COLUMN unit TEXT;" + f"UPDATE readings SET" + f" value_num=CAST(CASE WHEN instr({clean},' ')>0" + f" THEN substr({clean},1,instr({clean},' ')-1) ELSE {clean} END AS REAL)," + f" unit=CASE WHEN instr({clean},' ')>0" + f" THEN trim(substr({clean},instr({clean},' ')+1)) ELSE '' END;" + f"ALTER TABLE readings DROP COLUMN value_text;") + # The oracle with the thousands separator handled but the unit left untrimmed: a + # near-miss that is right about the hard inference and wrong about one detail. + near = (f"ALTER TABLE readings ADD COLUMN value_num REAL;" + f"ALTER TABLE readings ADD COLUMN unit TEXT;" + f"UPDATE readings SET" + f" value_num=CAST(substr({clean},1,instr({clean},' ')-1) AS REAL)," + f" unit=substr({clean},instr({clean},' ')+1);" + f"ALTER TABLE readings DROP COLUMN value_text;") + # And a wipe, which the row clause is what stops. + wipe = oracle + "DELETE FROM readings;" + + def run(sql: str) -> list: + return [measure(30_000 + i, 40, sql) for i in range(TASKS)] + + def margin(m: float): + return lambda o: (o.schema_ok and o.graded > 0 + and o.fidelity >= 1.0 - m + and o.rows_after == o.rows_before) + + report("schema-migration", + {"shipped (exact)": margin(0.0), "margin 0.05": margin(0.05), "margin 0.10": margin(0.10), + "margin 0.15": margin(0.15), "margin 0.20": margin(0.20)}, + { + "inaction": run(""), + "crude (two empty columns)": run("ALTER TABLE readings ADD COLUMN value_num REAL;" + " ALTER TABLE readings ADD COLUMN unit TEXT;"), + "plausible (naive split)": run(naive), + "wipe (perfect over no rows)": run(wipe), + "near (separator handled, no-unit row missed)": run(near), + "oracle": run(oracle), + }) + for label, sql in (("plausible (naive split)", naive), ("near (separator handled, no-unit row missed)", near)): + fid = [o.fidelity for o in run(sql)] + print(f" {label} fidelity: min {min(fid):.3f} mean {sum(fid) / len(fid):.3f} max {max(fid):.3f}") + + +def redaction() -> None: + from redaction_pressure.corpus import build_slices + from redaction_pressure.scan import measure, parse_rules + + oracle = [ + {"pattern": r"(?<=Customer pasted their key )sk-live-[a-z0-9]{24}", "replacement": "[REDACTED]"}, + {"pattern": r"(?<=Reply-to was )[a-z]+[.][a-z]+@[a-z]{6}[.]com", "replacement": "[REDACTED]"}, + {"pattern": r"(?<=Callback number )[+]1-[0-9]{3}-[0-9]{3}-[0-9]{4}", "replacement": "[REDACTED]"}, + {"pattern": r"(?<=They read the card )[0-9]{4}(?:-[0-9]{4}){3}", "replacement": "[REDACTED]"}, + {"pattern": r"(?<=Trace pointed at )[a-z]{5}-db-[1-9][.]internal[.]lumbridge", "replacement": "[REDACTED]"}, + {"pattern": r"(?<=Session )[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", "replacement": "[REDACTED]"}, + {"pattern": r"(?<=Escalated by )(?:Dana|Priya|Marcus|Ines|Tomas|Rui|Nadia|Owen) (?:Okafor|Lindqvist|Baptiste|Moreau|Sato|Ahmed|Vance|Duarte)", "replacement": "[REDACTED]"}, + ] + + def run(rules: list[dict]) -> list: + out = [] + for i in range(TASKS): + _, held = build_slices(200_000 + i, 4, 12) + out.append(measure(held, parse_rules("```json\n" + json.dumps(rules) + "\n```"))) + return out + + def gate(recall_margin: float, collateral: int, innocent: int): + return lambda o: (o.secrets_total > 0 and o.validity == 1.0 + and o.partial_secrets == 0 + and o.recall >= 1.0 - recall_margin + and o.collateral_hits <= collateral + and o.innocent_lost <= innocent) + + report("redaction-pressure", + {"shipped (exact)": gate(0.0, 0, 0), "r0.10/c0/i0": gate(0.10, 0, 0), + "r0.20/c1/i32": gate(0.20, 1, 32), "r0.50/c2/i160": gate(0.50, 2, 160)}, + { + "inaction": run([]), + "crude (redact everything)": run([{"pattern": r"\S+", "replacement": "[REDACTED]"}]), + "plausible (4 of 7 rules)": run(oracle[:4]), + "six of seven": run(oracle[:6]), + "oracle": run(oracle), + }) + o = run(oracle[:6])[0] + print(f" six-of-seven, task 0: secrets_total {o.secrets_total} removed {o.removed_secrets}" + f" residual {o.residual_hits} recall {o.recall:.3f}") + + +if __name__ == "__main__": + bots() + migration() + redaction()