Three more environments, and probe.py to gate all four
canary-trap, fault-localisation and schema-migration, in the three domains the Environments Hub has nothing in: contamination, operations, data engineering. None is a port. Each is graded on a slice the model never read. canary-trap runs a probe set against a model that memorised the corpus AND one that learned the same facts from a reworded copy with every identifier regenerated. The obvious probe -- plant a GUID, ask for it back -- is sound and caps at 0.525 because it cannot see the paraphrase. It scored higher than that at first: facts with no distinct rewording had the paraphrase restating the corpus verbatim, so GUID probes caught it by accident. Every fact now carries a "core" token present in both wordings and in no public one, asserted at import, which is also what makes the ceiling reachable. fault-localisation puts the fault upstream and the noise downstream, the way incidents actually present. The broken service logs one line and goes quiet while its dependents log a dozen timeouts, so ranking by error volume answers the victim -- 0/64 incidents have the loudest service as the root, and probe.py asserts it rather than trusting the generator to stay that way. schema-migration executes the model's SQL against forty held-out rows carrying thousands separators, negatives, a unit with a slash in it and a value with no unit. Row preservation started as a reward beside fidelity and paid 0.15 for adding two empty columns and touching nothing; it is a multiplier now, so losing rows still zeroes a perfect migration and keeping them earns nothing on its own. probe.py is house rule 3 as an exit code: 0.000 for inaction and 1.000 for an oracle, or it returns 1. It loads the leaf modules through a namespace shim rather than the packages, so it runs without verifiers installed -- a self-check that needs the training stack is a self-check nobody runs. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,3 @@
|
||||
from fault_localisation.taskset import FaultTaskset
|
||||
|
||||
__all__ = ["FaultTaskset"]
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Generating one incident: a service graph, an injected fault, and the logs it produced.
|
||||
|
||||
The fault is placed upstream and the noise is emitted downstream, because that is how real
|
||||
incidents present and it is the one thing a log-reading agent has to get right. A service
|
||||
whose dependency has stalled logs timeouts, retries and 503s — loudly, once per request —
|
||||
while the service that actually broke logs a single line about its connection pool and then
|
||||
goes quiet. Ranking services by error volume therefore points at the victim every time.
|
||||
|
||||
Everything is derived from one seed, so a task is reproducible from its integer id and the
|
||||
answer key is generated alongside the logs rather than written by hand.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
|
||||
SERVICES = ["edge", "checkout", "ledger", "inventory", "pricing", "vault"]
|
||||
|
||||
# (fault class, the single line the ROOT service emits, what its callers emit)
|
||||
FAULTS = [
|
||||
("pool_exhausted", "connection pool exhausted (size=16, waiters=88)", "upstream timeout after 3000ms"),
|
||||
("disk_full", "write failed: no space left on device", "upstream returned 503"),
|
||||
("cert_expired", "TLS handshake failed: certificate expired", "upstream connection reset"),
|
||||
("deadlock", "transaction aborted: deadlock detected", "upstream timeout after 3000ms"),
|
||||
("oom_kill", "worker killed: cgroup memory limit exceeded", "upstream returned 502"),
|
||||
("clock_skew", "rejecting request: token nbf 41s in the future", "upstream returned 401"),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Incident:
|
||||
lines: list[str]
|
||||
"""Log lines, each prefixed with a stable `L07` style id the answer can cite."""
|
||||
root: str
|
||||
fault: str
|
||||
evidence: str
|
||||
"""The id of the one line that names the cause. Every other line is a symptom."""
|
||||
|
||||
|
||||
def build(seed: int) -> Incident:
|
||||
rng = random.Random(seed)
|
||||
root, *rest = rng.sample(SERVICES, 4)
|
||||
# The callers: they depend on root, so they are the ones that will look broken.
|
||||
callers = rest[:2]
|
||||
bystander = rest[2]
|
||||
fault, root_line, caller_line = rng.choice(FAULTS)
|
||||
|
||||
raw: list[tuple[int, str, str]] = []
|
||||
clock = rng.randint(0, 40)
|
||||
|
||||
# Ordinary traffic first, from everyone, so "the service with any errors" is not free.
|
||||
for _ in range(6):
|
||||
clock += rng.randint(1, 4)
|
||||
svc = rng.choice([root, *callers, bystander])
|
||||
raw.append((clock, svc, rng.choice([
|
||||
"handled request in 24ms", "cache hit", "healthcheck ok", "handled request in 61ms",
|
||||
])))
|
||||
|
||||
# The cause: one line, once, and then the root goes quiet.
|
||||
clock += rng.randint(1, 3)
|
||||
cause_at = clock
|
||||
raw.append((clock, root, root_line))
|
||||
|
||||
# The symptoms: loud, repeated, and in the wrong place.
|
||||
for _ in range(rng.randint(9, 14)):
|
||||
clock += rng.randint(1, 3)
|
||||
raw.append((clock, rng.choice(callers), caller_line))
|
||||
# A red herring the bystander emits on its own schedule, unrelated to the fault.
|
||||
clock += rng.randint(1, 2)
|
||||
raw.append((clock, bystander, "retrying scheduled job (attempt 2)"))
|
||||
|
||||
raw.sort(key=lambda r: r[0])
|
||||
lines, evidence = [], ""
|
||||
for i, (at, svc, text) in enumerate(raw):
|
||||
line_id = f"L{i:02d}"
|
||||
if at == cause_at and svc == root and text == root_line:
|
||||
evidence = line_id
|
||||
lines.append(f"{line_id} t+{at:03d}s {svc:<10} {text}")
|
||||
return Incident(lines=lines, root=root, fault=fault, evidence=evidence)
|
||||
|
||||
|
||||
def loudest(incident: Incident) -> str:
|
||||
"""The service with the most error lines — what ranking by volume would answer.
|
||||
|
||||
Exported because the probe uses it: if this ever starts coinciding with the root, the
|
||||
environment has stopped testing the thing it was built to test.
|
||||
"""
|
||||
counts: dict[str, int] = {}
|
||||
for line in incident.lines:
|
||||
svc = line.split()[2]
|
||||
if "upstream" in line:
|
||||
counts[svc] = counts.get(svc, 0) + 1
|
||||
return max(counts, key=lambda k: counts[k]) if counts else ""
|
||||
@@ -0,0 +1,119 @@
|
||||
"""fault-localisation: name what broke, not what is complaining.
|
||||
|
||||
One incident, one injected fault, and a log the fault's victims dominate. The service that
|
||||
actually failed emits a single line and falls silent; the two that depend on it emit a
|
||||
dozen timeouts between them. Ranking by error volume answers the loudest service, which is
|
||||
never the right one — the probe in this package asserts that, so the day the generator
|
||||
drifts into making them coincide the environment fails rather than quietly rewarding the
|
||||
heuristic it exists to punish.
|
||||
|
||||
Three answers are required and they are not redundant. The service can be guessed one time
|
||||
in four. The fault class can be read off a line without knowing which line matters. Citing
|
||||
the evidence line is what distinguishes having found the cause from having recognised the
|
||||
vocabulary, and it is why the gate needs all three rather than a majority.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
import verifiers.v1 as vf
|
||||
|
||||
from fault_localisation.incident import FAULTS, SERVICES, build
|
||||
|
||||
CLASSES = sorted(f[0] for f in FAULTS)
|
||||
|
||||
SYSTEM = f"""You are on call. Read the log and name the ROOT CAUSE.
|
||||
|
||||
Return one JSON object in a ```json code block with exactly three keys:
|
||||
"service" the service that actually failed, one of: {", ".join(SERVICES)}
|
||||
"fault" the failure class, one of: {", ".join(CLASSES)}
|
||||
"evidence" the id (e.g. "L07") of the single line that names the cause
|
||||
|
||||
The service that failed is not the service producing the most errors. Its dependents
|
||||
produce the errors; it produces one line and then stops."""
|
||||
|
||||
_BLOCK = re.compile(r"```(?:json)?\s*\n(.*?)```", re.DOTALL)
|
||||
|
||||
|
||||
def parse_answer(reply: str) -> dict[str, str]:
|
||||
blocks = _BLOCK.findall(reply or "")
|
||||
raw = blocks[-1] if blocks else (reply or "")
|
||||
try:
|
||||
parsed = json.loads(raw.strip())
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
if not isinstance(parsed, dict):
|
||||
return {}
|
||||
return {k: str(v).strip() for k, v in parsed.items() if isinstance(v, (str, int))}
|
||||
|
||||
|
||||
class FaultData(vf.TaskData):
|
||||
seed: int
|
||||
|
||||
|
||||
class FaultTask(vf.Task[FaultData]):
|
||||
@vf.stop
|
||||
async def single_turn(self, trace: vf.Trace) -> bool:
|
||||
return trace.num_turns >= 1
|
||||
|
||||
@vf.metric
|
||||
async def scan(self, trace: vf.Trace) -> dict[str, float]:
|
||||
incident = build(self.data.seed)
|
||||
answer = parse_answer(trace.last_reply)
|
||||
service = float(answer.get("service", "").lower() == incident.root)
|
||||
fault = float(answer.get("fault", "").lower() == incident.fault)
|
||||
evidence = float(answer.get("evidence", "").upper() == incident.evidence)
|
||||
return {
|
||||
"service": service,
|
||||
"fault": fault,
|
||||
"evidence": evidence,
|
||||
"clean": float(service and fault and evidence),
|
||||
}
|
||||
|
||||
@vf.reward(weight=0.30)
|
||||
async def service(self, trace: vf.Trace) -> float:
|
||||
return trace.metrics.get("service", 0.0)
|
||||
|
||||
@vf.reward(weight=0.25)
|
||||
async def fault(self, trace: vf.Trace) -> float:
|
||||
return trace.metrics.get("fault", 0.0)
|
||||
|
||||
@vf.reward(weight=0.25)
|
||||
async def evidence(self, trace: vf.Trace) -> float:
|
||||
return trace.metrics.get("evidence", 0.0)
|
||||
|
||||
@vf.reward(weight=0.20)
|
||||
async def gate(self, trace: vf.Trace) -> float:
|
||||
return trace.metrics.get("clean", 0.0)
|
||||
|
||||
|
||||
class FaultConfig(vf.TasksetConfig):
|
||||
num_tasks: int = Field(64, ge=1)
|
||||
|
||||
|
||||
class FaultTaskset(vf.Taskset[FaultTask, FaultConfig]):
|
||||
SEED_BASE = 50_000
|
||||
|
||||
def load(self) -> list[FaultTask]:
|
||||
tasks = []
|
||||
for i in range(self.config.num_tasks):
|
||||
seed = self.SEED_BASE + i
|
||||
incident = build(seed)
|
||||
log = "\n".join(incident.lines)
|
||||
tasks.append(
|
||||
FaultTask(
|
||||
FaultData(
|
||||
idx=i,
|
||||
name=f"incident-{seed}",
|
||||
prompt=f"```\n{log}\n```\n\nName the root cause.",
|
||||
system_prompt=SYSTEM,
|
||||
seed=seed,
|
||||
),
|
||||
self.config.task,
|
||||
)
|
||||
)
|
||||
return tasks
|
||||
@@ -0,0 +1,13 @@
|
||||
[project]
|
||||
name = "fault-localisation"
|
||||
version = "0.1.0"
|
||||
description = "fault-localisation — name the service that broke, not the ones complaining about it."
|
||||
requires-python = ">=3.11"
|
||||
dependencies = ["verifiers"]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["fault_localisation"]
|
||||
Reference in New Issue
Block a user