Three more environments, and probe.py to gate all four
canary-trap, fault-localisation and schema-migration, in the three domains the Environments Hub has nothing in: contamination, operations, data engineering. None is a port. Each is graded on a slice the model never read. canary-trap runs a probe set against a model that memorised the corpus AND one that learned the same facts from a reworded copy with every identifier regenerated. The obvious probe -- plant a GUID, ask for it back -- is sound and caps at 0.525 because it cannot see the paraphrase. It scored higher than that at first: facts with no distinct rewording had the paraphrase restating the corpus verbatim, so GUID probes caught it by accident. Every fact now carries a "core" token present in both wordings and in no public one, asserted at import, which is also what makes the ceiling reachable. fault-localisation puts the fault upstream and the noise downstream, the way incidents actually present. The broken service logs one line and goes quiet while its dependents log a dozen timeouts, so ranking by error volume answers the victim -- 0/64 incidents have the loudest service as the root, and probe.py asserts it rather than trusting the generator to stay that way. schema-migration executes the model's SQL against forty held-out rows carrying thousands separators, negatives, a unit with a slash in it and a value with no unit. Row preservation started as a reward beside fidelity and paid 0.15 for adding two empty columns and touching nothing; it is a multiplier now, so losing rows still zeroes a perfect migration and keeping them earns nothing on its own. probe.py is house rule 3 as an exit code: 0.000 for inaction and 1.000 for an oracle, or it returns 1. It loads the leaf modules through a namespace shim rather than the packages, so it runs without verifiers installed -- a self-check that needs the training stack is a self-check nobody runs. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,13 @@
|
||||
[project]
|
||||
name = "schema-migration"
|
||||
version = "0.1.0"
|
||||
description = "schema-migration — split a text column into a number and a unit, graded on rows you never saw."
|
||||
requires-python = ">=3.11"
|
||||
dependencies = ["verifiers"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["schema_migration"]
|
||||
@@ -0,0 +1,3 @@
|
||||
from schema_migration.taskset import MigrationTaskset
|
||||
|
||||
__all__ = ["MigrationTaskset"]
|
||||
@@ -0,0 +1,69 @@
|
||||
"""The table before the migration, and the rows the migration is graded on.
|
||||
|
||||
One `readings` table carries a `value_text` column that should never have been text:
|
||||
"1400 ms", "412 rps", "-3.5 degC". The migration has to split it into a real number and a
|
||||
unit and drop the original column, without losing a row or rounding a value.
|
||||
|
||||
The visible rows are deliberately tidy. The held-out rows are not: they carry thousands
|
||||
separators, negatives, decimals, a unit with a digit in it, a value with no unit at all,
|
||||
and one with extra internal whitespace. A migration written by pattern-matching the five
|
||||
rows on screen parses about half of them, which is the entire point of grading on rows the
|
||||
author never read.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import sqlite3
|
||||
|
||||
TIDY = [
|
||||
("1400 ms", 1400.0, "ms"),
|
||||
("412 rps", 412.0, "rps"),
|
||||
("16 conns", 16.0, "conns"),
|
||||
("88 waiters", 88.0, "waiters"),
|
||||
("7 replicas", 7.0, "replicas"),
|
||||
]
|
||||
|
||||
# Every awkward shape a real column of this kind contains.
|
||||
AWKWARD = [
|
||||
("1,400 ms", 1400.0, "ms"), # thousands separator
|
||||
("-3.5 degC", -3.5, "degC"), # negative, decimal
|
||||
("0.25 s", 0.25, "s"), # leading zero decimal
|
||||
("64 KiB/s", 64.0, "KiB/s"), # unit containing a digit and a slash
|
||||
("120", 120.0, ""), # no unit at all
|
||||
("55 rps", 55.0, "rps"), # doubled internal space
|
||||
("2,048 MiB", 2048.0, "MiB"),
|
||||
("-12 ms", -12.0, "ms"),
|
||||
("3.0 x", 3.0, "x"),
|
||||
("1,000,000 rows", 1000000.0, "rows"),
|
||||
]
|
||||
|
||||
SOURCE_DDL = """
|
||||
CREATE TABLE readings (
|
||||
id INTEGER PRIMARY KEY,
|
||||
probe TEXT NOT NULL,
|
||||
value_text TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
TARGET_COLUMNS = {"id", "probe", "value_num", "unit"}
|
||||
|
||||
|
||||
def rows(seed: int, count: int, awkward: bool) -> list[tuple[str, float, str]]:
|
||||
rng = random.Random(seed)
|
||||
pool = AWKWARD + TIDY if awkward else TIDY
|
||||
return [pool[rng.randrange(len(pool))] for _ in range(count)]
|
||||
|
||||
|
||||
def connect(seed: int, count: int, awkward: bool) -> tuple[sqlite3.Connection, list[tuple]]:
|
||||
"""A fresh in-memory database holding one slice. Nothing touches disk, so an agent's
|
||||
SQL cannot reach anything — the environment is the sandbox."""
|
||||
con = sqlite3.connect(":memory:")
|
||||
con.executescript(SOURCE_DDL)
|
||||
data = rows(seed, count, awkward)
|
||||
con.executemany(
|
||||
"INSERT INTO readings (id, probe, value_text) VALUES (?, ?, ?)",
|
||||
[(i + 1, f"probe_{i:03d}", text) for i, (text, _, _) in enumerate(data)],
|
||||
)
|
||||
con.commit()
|
||||
return con, data
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Running the agent's migration, and measuring what survived it.
|
||||
|
||||
Four things are checked, and they are in tension by construction:
|
||||
|
||||
schema the target columns exist and `value_text` is gone. A migration that adds the
|
||||
new columns and leaves the old one is not a migration, it is a copy.
|
||||
fidelity per row, does (value_num, unit) recompose to the original value. This is the
|
||||
expensive one and the only one that punishes a parser fitted to the visible
|
||||
rows.
|
||||
rows the row count is unchanged. Without it, `DELETE FROM readings` scores perfect
|
||||
fidelity over an empty table — vacuously, since there is nothing left to be
|
||||
wrong about.
|
||||
gate all three, exactly. `forge verify`'s exit code: a migration is correct or it
|
||||
is not run in production.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
|
||||
from schema_migration.fixture import TARGET_COLUMNS, connect
|
||||
|
||||
STATEMENT_LIMIT = 40
|
||||
# ATTACH would reach outside the in-memory database; the rest cannot, but there is no
|
||||
# reason for a schema migration to use any of them.
|
||||
FORBIDDEN = ("attach", "pragma", "vacuum")
|
||||
TOLERANCE = 1e-9
|
||||
|
||||
|
||||
@dataclass
|
||||
class Outcome:
|
||||
schema_ok: bool = False
|
||||
matched: int = 0
|
||||
graded: int = 0
|
||||
rows_before: int = 0
|
||||
rows_after: int = 0
|
||||
error: str = ""
|
||||
|
||||
@property
|
||||
def fidelity(self) -> float:
|
||||
return self.matched / self.graded if self.graded else 0.0
|
||||
|
||||
@property
|
||||
def rows_kept(self) -> float:
|
||||
if not self.rows_before:
|
||||
return 0.0
|
||||
# Inserting extra rows is as wrong as dropping them, so this is a distance, not a ratio.
|
||||
return max(0.0, 1.0 - abs(self.rows_after - self.rows_before) / self.rows_before)
|
||||
|
||||
@property
|
||||
def clean(self) -> bool:
|
||||
return self.schema_ok and self.graded > 0 and self.matched == self.graded and self.rows_after == self.rows_before
|
||||
|
||||
|
||||
def _columns(con: sqlite3.Connection) -> set[str]:
|
||||
return {r[1] for r in con.execute("PRAGMA table_info(readings)")}
|
||||
|
||||
|
||||
def measure(seed: int, count: int, migration: str) -> Outcome:
|
||||
con, data = connect(seed, count, awkward=True)
|
||||
outcome = Outcome(rows_before=len(data), graded=len(data))
|
||||
|
||||
lowered = (migration or "").lower()
|
||||
if any(word in lowered for word in FORBIDDEN):
|
||||
outcome.error = "forbidden statement"
|
||||
return outcome
|
||||
if lowered.count(";") > STATEMENT_LIMIT:
|
||||
outcome.error = "too many statements"
|
||||
return outcome
|
||||
if not lowered.strip():
|
||||
outcome.error = "empty migration"
|
||||
return outcome
|
||||
|
||||
try:
|
||||
con.executescript(migration)
|
||||
except sqlite3.Error as exc:
|
||||
# A migration that does not run scores what doing nothing scores. It is not an
|
||||
# error for the rollout — the reward is the signal.
|
||||
outcome.error = str(exc)[:120]
|
||||
return outcome
|
||||
|
||||
columns = _columns(con)
|
||||
outcome.schema_ok = TARGET_COLUMNS.issubset(columns) and "value_text" not in columns
|
||||
try:
|
||||
outcome.rows_after = con.execute("SELECT count(*) FROM readings").fetchone()[0]
|
||||
if outcome.schema_ok:
|
||||
got = {
|
||||
r[0]: (r[1], r[2])
|
||||
for r in con.execute("SELECT id, value_num, unit FROM readings")
|
||||
}
|
||||
for i, (_, num, unit) in enumerate(data):
|
||||
have = got.get(i + 1)
|
||||
if not have or have[0] is None:
|
||||
continue
|
||||
if abs(float(have[0]) - num) < TOLERANCE and (have[1] or "").strip() == unit:
|
||||
outcome.matched += 1
|
||||
except (sqlite3.Error, TypeError, ValueError) as exc:
|
||||
outcome.error = str(exc)[:120]
|
||||
return outcome
|
||||
@@ -0,0 +1,121 @@
|
||||
"""schema-migration: a migration is judged on the rows you did not look at.
|
||||
|
||||
`readings.value_text` holds numbers and units as one string. The migration has to become
|
||||
`value_num REAL` plus `unit TEXT`, drop the original column, and lose nothing. The agent
|
||||
sees five tidy rows; it is graded on forty it never sees, which carry thousands separators,
|
||||
negatives, decimals, a unit containing a digit and a slash, a value with no unit, and
|
||||
doubled internal whitespace.
|
||||
|
||||
The reward is the migration actually executing against a real database, so there is no
|
||||
judge and no rubric — SQLite either recomposes the value or it does not. Row count is
|
||||
scored separately from fidelity because otherwise `DELETE FROM readings` is a perfect
|
||||
migration: nothing left to be wrong about.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
import verifiers.v1 as vf
|
||||
|
||||
from schema_migration.fixture import connect
|
||||
from schema_migration.run import measure
|
||||
|
||||
SYSTEM = """You are writing a SQLite migration.
|
||||
|
||||
The table is:
|
||||
CREATE TABLE readings (id INTEGER PRIMARY KEY, probe TEXT NOT NULL, value_text TEXT NOT NULL);
|
||||
|
||||
Migrate it so that `readings` has columns id, probe, value_num (REAL) and unit (TEXT), and
|
||||
no longer has value_text. `value_num` is the number in value_text; `unit` is what follows
|
||||
it, trimmed, or an empty string when there is no unit.
|
||||
|
||||
Return ONLY SQL in one ```sql code block. It is run with executescript(), so several
|
||||
statements are fine. Preserve every row and every id.
|
||||
|
||||
You are graded on rows you have not seen, from the same column. They are messier than the
|
||||
ones below."""
|
||||
|
||||
_BLOCK = re.compile(r"```(?:sql)?\s*\n(.*?)```", re.DOTALL)
|
||||
|
||||
|
||||
def parse_sql(reply: str) -> str:
|
||||
blocks = _BLOCK.findall(reply or "")
|
||||
return (blocks[-1] if blocks else (reply or "")).strip()
|
||||
|
||||
|
||||
class MigrationData(vf.TaskData):
|
||||
seed: int
|
||||
held_out: int
|
||||
|
||||
|
||||
class MigrationTask(vf.Task[MigrationData]):
|
||||
@vf.stop
|
||||
async def single_turn(self, trace: vf.Trace) -> bool:
|
||||
return trace.num_turns >= 1
|
||||
|
||||
@vf.metric
|
||||
async def scan(self, trace: vf.Trace) -> dict[str, float]:
|
||||
outcome = measure(self.data.seed, self.data.held_out, parse_sql(trace.last_reply))
|
||||
return {
|
||||
"schema": float(outcome.schema_ok),
|
||||
"fidelity": outcome.fidelity,
|
||||
"rows_kept": outcome.rows_kept,
|
||||
# Rows are a MULTIPLIER on fidelity, not a reward beside it. Scored separately
|
||||
# they paid 0.15 for adding two empty columns and touching nothing — a floor
|
||||
# above zero for near-inaction, which the house rules forbid. As a multiplier,
|
||||
# losing rows still zeroes an otherwise perfect migration and preserving them
|
||||
# on its own earns nothing.
|
||||
"integrity": outcome.fidelity * outcome.rows_kept,
|
||||
"clean": float(outcome.clean),
|
||||
"errored": float(bool(outcome.error)),
|
||||
}
|
||||
|
||||
@vf.reward(weight=0.30)
|
||||
async def schema(self, trace: vf.Trace) -> float:
|
||||
return trace.metrics.get("schema", 0.0)
|
||||
|
||||
@vf.reward(weight=0.45)
|
||||
async def integrity(self, trace: vf.Trace) -> float:
|
||||
return trace.metrics.get("integrity", 0.0)
|
||||
|
||||
@vf.reward(weight=0.25)
|
||||
async def gate(self, trace: vf.Trace) -> float:
|
||||
return trace.metrics.get("clean", 0.0)
|
||||
|
||||
|
||||
class MigrationConfig(vf.TasksetConfig):
|
||||
num_tasks: int = Field(48, ge=1)
|
||||
visible: int = Field(5, ge=1)
|
||||
held_out: int = Field(40, ge=1)
|
||||
|
||||
|
||||
class MigrationTaskset(vf.Taskset[MigrationTask, MigrationConfig]):
|
||||
SEED_BASE = 30_000
|
||||
|
||||
def load(self) -> list[MigrationTask]:
|
||||
tasks = []
|
||||
for i in range(self.config.num_tasks):
|
||||
seed = self.SEED_BASE + i
|
||||
# The visible slice is tidy on purpose; the graded one is not.
|
||||
con, data = connect(seed, self.config.visible, awkward=False)
|
||||
sample = "\n".join(
|
||||
f" {i + 1} | probe_{i:03d} | {text}" for i, (text, _, _) in enumerate(data)
|
||||
)
|
||||
con.close()
|
||||
tasks.append(
|
||||
MigrationTask(
|
||||
MigrationData(
|
||||
idx=i,
|
||||
name=f"readings-{seed}",
|
||||
prompt=f"Rows in the table:\n\n id | probe | value_text\n{sample}\n\nWrite the migration.",
|
||||
system_prompt=SYSTEM,
|
||||
seed=seed,
|
||||
held_out=self.config.held_out,
|
||||
),
|
||||
self.config.task,
|
||||
)
|
||||
)
|
||||
return tasks
|
||||
Reference in New Issue
Block a user