grand-exchange-live: protocol layer, taskset, config, widened reference gate
The stepped engine becomes a runnable taskset. protocol.py is both the render/parse boundary and the security boundary: TurnView carries the live Market, so view.market.items[i].prices is the whole held-out series and a foresight policy reading it earned +9.4% on the oracle. The renderer emits only observed ticks, the trace carries plain scalars, and a sentinel scan of every rendered turn fails if either reopens. House rule 1 had no other enforcement. Replies are a delta sheet and parsing never raises — NaN, Infinity, 1e309 and 20k-deep nesting all resolve to a hold, because the one-shot form has already shown what a real model sends when it gives up. LiveExchangeEnv.run drives the engine host-side with the harness left null. Two things learned the hard way and worth keeping: a terminated Segment mid-window holds the remaining looks instead of raising, and rewards are recorded INSIDE the interaction, before close. Recording after close writes the traces correctly and leaves every eval.log line reading reward=0.000 — the canonical verifiers reference does it after the block; do not copy it there. The reference-family gate is widened from four named rungs to the whole 8x8 (requote_band, idle_share) family: no member may out-earn the shipped reference by more than 1.02x, and any that does must still score >= 0.90. That is the check that caught the first draft's fake 0.152 gap, now shipped rather than remembered. Also guards `clean` on the reference being profitable, here and in the one-shot form. Against a reference that lost money the bar is negative, so doing nothing cleared it and inaction collected a quarter of the reward. Unreachable while viable_market admits only profitable references; one relaxed filter from reachable, and exactly the floor house rule 3 forbids. Probe values unchanged. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,40 @@
|
||||
"""Import the package under test, installed or not — and get the same one every time.
|
||||
|
||||
Two facts collide here. `probe.py` and half of this suite must run with NOTHING installed,
|
||||
which is why the package is registered as a namespace pointing at the source directory and
|
||||
its leaf modules are imported under it: `__init__` never executes, so `verifiers` is never
|
||||
required. And `test_taskset.py` needs the opposite — `verifiers` present and
|
||||
`grand_exchange_live` resolving to the REAL package, because `import_taskset` reads
|
||||
`__all__` off whatever is in `sys.modules` and a namespace shim has none.
|
||||
|
||||
A shim installed by whichever test module imported first would decide that for the whole
|
||||
process. So the decision is made once, here, and every test module imports this before it
|
||||
imports anything else: the real package if it is importable, the shim if it is not.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def shim(package: Path) -> None:
|
||||
"""Register `package` as a namespace so its leaf modules import without its `__init__`."""
|
||||
module = types.ModuleType(package.name)
|
||||
module.__path__ = [str(package)]
|
||||
sys.modules[package.name] = module
|
||||
|
||||
|
||||
try:
|
||||
import grand_exchange_live
|
||||
|
||||
INSTALLED = getattr(grand_exchange_live, "__all__", None) is not None
|
||||
except ImportError:
|
||||
# The package is there and `verifiers` is not, which is the no-dependency path.
|
||||
INSTALLED = False
|
||||
|
||||
if not INSTALLED:
|
||||
shim(ROOT / ROOT.name)
|
||||
@@ -25,22 +25,22 @@ from __future__ import annotations
|
||||
import random
|
||||
import statistics
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parents[1]
|
||||
ONE_SHOT = HERE.parent / "grand_exchange"
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import context # noqa: E402 - resolves the package before anything imports it
|
||||
|
||||
for _root in (HERE, ONE_SHOT):
|
||||
_package = _root / _root.name
|
||||
_shim = types.ModuleType(_package.name)
|
||||
_shim.__path__ = [str(_package)]
|
||||
sys.modules[_package.name] = _shim
|
||||
HERE = context.ROOT
|
||||
# The one-shot environment is a sibling checkout rather than a dependency, and the fixed-point
|
||||
# assertion below needs both engines in one process. It is always a shim.
|
||||
context.shim(HERE.parent / "grand_exchange" / "grand_exchange")
|
||||
|
||||
from grand_exchange_live.book import Order, execute, reference_orders # noqa: E402
|
||||
from grand_exchange_live.live import ( # noqa: E402
|
||||
HORIZON,
|
||||
IDLE_SHARE,
|
||||
REQUOTE_BAND,
|
||||
TAIL_TICKS,
|
||||
TICKS_PER_TURN,
|
||||
TURNS,
|
||||
@@ -443,10 +443,15 @@ class LadderGateTests(unittest.TestCase):
|
||||
self.assertGreater(statistics.fmean(gaps), 0.25)
|
||||
|
||||
def test_the_reference_is_not_merely_the_denominator(self) -> None:
|
||||
"""The gap has to be a fact about money, not about normalisation. If the exhaustive
|
||||
policy EARNED more and scored less, the reward would be an imitation score for the
|
||||
reference's constants — which is precisely what the first cut of this engine did,
|
||||
at 63,974 gp against the reference's 51,520."""
|
||||
"""The gap has to be a fact about money, not about normalisation. If a policy EARNED
|
||||
more than the reference and scored below it, the reward would be an imitation score
|
||||
for the reference's constants — which is precisely what the first cut of this engine
|
||||
did, at 63,974 gp against the reference's 51,520.
|
||||
|
||||
Two halves, because there are two ways to be the wrong denominator. The rungs are
|
||||
different strategies; the sweep below is the reference's OWN family, and that is the
|
||||
half this test used to skip.
|
||||
"""
|
||||
rows = self.ladder.ladder(self.markets)
|
||||
oracle = rows["oracle (the live reference)"]
|
||||
for name in ("exhaustive (re-quote every look)", "churn (re-place at full size every look)",
|
||||
@@ -455,6 +460,59 @@ class LadderGateTests(unittest.TestCase):
|
||||
self.assertLess(rows[name]["gp"], oracle["gp"],
|
||||
f"{name} earns more gp than the reference it is scored against")
|
||||
|
||||
def test_no_member_of_the_references_own_family_beats_it_and_scores_below_it(self) -> None:
|
||||
"""The whole sixty-four-cell family, not four named rungs.
|
||||
|
||||
The shipped reference is not the family's argmax and this is the assertion that says
|
||||
how far from it that is allowed to be. Measured over 120 baskets, 2026-08-21:
|
||||
|
||||
shipped (0.025, 0.50) 76,744 gp rule-4 gap 0.369 mean, 0.316 worst block
|
||||
argmax (1.000, 0.50) 77,599 gp rule-4 gap 0.372 mean, 0.299 worst block
|
||||
|
||||
so the best member earns 1.0111x the reference, and the worst-scoring member that
|
||||
out-earns it — (0.040, 0.50) — still scores 0.972. Moving the denominator to the
|
||||
argmax would make the invariant exact and was measured before this test was written:
|
||||
it is not taken, because the argmax's requote band is 1.0, i.e. the anchor trigger
|
||||
never fires, so the denominator would be a strategy that cannot express "the anchor
|
||||
moved" at all — and the freeze, the mechanic the whole environment uses to make the
|
||||
turn budget bind, exists to price exactly that judgement. It also does not buy a
|
||||
wider gap: 0.372 against 0.369 on the mean, and a WORSE worst block. Paying 1.1% of
|
||||
gp to keep the denominator a strategy rather than a cash-threshold reflex is the
|
||||
trade, and the tolerances below are the measurement, not a taste.
|
||||
"""
|
||||
bands = (0.0, 0.005, 0.010, 0.015, 0.025, 0.040, 0.060, 1.0)
|
||||
idles = (0.0, 0.05, 0.10, 0.15, 0.20, 0.30, 0.50, 1.01)
|
||||
shipped = statistics.fmean(
|
||||
self.ladder.live_reference(m).realised for m in self.markets
|
||||
)
|
||||
for band in bands:
|
||||
for idle in idles:
|
||||
if (band, idle) == (REQUOTE_BAND, IDLE_SHARE):
|
||||
continue
|
||||
earned = statistics.fmean(
|
||||
play(m, Reference(band, idle)).realised for m in self.markets
|
||||
)
|
||||
self.assertLessEqual(
|
||||
earned, 1.02 * shipped,
|
||||
f"reference family member ({band}, {idle}) earns {earned:,.0f} gp "
|
||||
f"against the shipped reference's {shipped:,.0f} — the denominator is "
|
||||
"leaving too much money on the table, so the reward's argmax is its "
|
||||
"constants rather than the profit",
|
||||
)
|
||||
if earned <= shipped:
|
||||
continue
|
||||
got = statistics.fmean(
|
||||
self.ladder.score(play(m, Reference(band, idle)),
|
||||
self.ladder.live_reference(m))["total"]
|
||||
for m in self.markets
|
||||
)
|
||||
self.assertGreaterEqual(
|
||||
got, 0.90,
|
||||
f"reference family member ({band}, {idle}) earns {earned:,.0f} gp "
|
||||
f"against {shipped:,.0f} and scores {got:.3f} — the reward is an "
|
||||
"imitation score, not a profit metric",
|
||||
)
|
||||
|
||||
def test_looking_costs_and_a_wasted_look_costs_more(self) -> None:
|
||||
rows = self.ladder.ladder(self.markets)
|
||||
self.assertLess(rows["restate (re-quote the same book every look)"]["total"],
|
||||
|
||||
@@ -15,23 +15,19 @@ cannot perturb a single tick. That claim is not an argument here, it is the test
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parents[1]
|
||||
ONE_SHOT = HERE.parent / "grand_exchange"
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import context # noqa: E402 - resolves the package before anything imports it
|
||||
|
||||
# `probe.py`'s shim, for `probe.py`'s reason: each package's `__init__` imports its taskset,
|
||||
# which imports verifiers, and a self-check that only runs once the training stack is
|
||||
# installed is a self-check nobody runs. Registering the package name as a namespace
|
||||
# pointing at the source directory keeps intra-package imports resolving while `__init__`
|
||||
# never executes.
|
||||
for _root in (HERE, ONE_SHOT):
|
||||
_package = _root / _root.name
|
||||
_shim = types.ModuleType(_package.name)
|
||||
_shim.__path__ = [str(_package)]
|
||||
sys.modules[_package.name] = _shim
|
||||
HERE = context.ROOT
|
||||
# The one-shot environment is a sibling checkout rather than a dependency, so it is always
|
||||
# `probe.py`'s shim, for `probe.py`'s reason: its `__init__` imports its taskset, which
|
||||
# imports verifiers, and a self-check that only runs once the training stack is installed is
|
||||
# a self-check nobody runs. Registering the package name as a namespace pointing at the
|
||||
# source directory keeps intra-package imports resolving while `__init__` never executes.
|
||||
context.shim(HERE.parent / "grand_exchange" / "grand_exchange")
|
||||
|
||||
import grand_exchange.book as one_shot_book # noqa: E402
|
||||
import grand_exchange.market as one_shot # noqa: E402
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
"""The render/parse boundary, and the leak it exists to close.
|
||||
|
||||
`TurnView` carries the live `Market`, so `view.market.items[i].prices` is every graded tick
|
||||
including the ones the agent is scored on not having seen. A verifier wrote a policy that
|
||||
reads it and earned 9.4% more than the oracle. House rule 1 is enforced by `protocol.render`
|
||||
and by nothing else, so it is enforced here:
|
||||
|
||||
the sentinel every graded tick of a purpose-built basket is a value no real price or
|
||||
volume can take, encoding its own index. Every rendered turn is scanned for
|
||||
them, and any sentinel it carries must be a tick the view had already
|
||||
observed. A render that reached one tick past `view.seen` fails this.
|
||||
the prefix what was rendered, reassembled across all eight turns, must be exactly
|
||||
`prices[:seen]` — not a subset of it, not a superset. The sentinel test says
|
||||
nothing forbidden got out; this one says nothing was quietly mangled on the
|
||||
way, which is the other way a boundary stops being a boundary.
|
||||
|
||||
And the parser, against replies a model actually emits. `json.loads` is not strict JSON: it
|
||||
takes bare NaN and Infinity and overflows 1e309 to inf, and `int(inf)` raises OverflowError
|
||||
— inside `run()`, which fails the whole episode over a formatting slip.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import context # noqa: E402 - resolves the package before anything imports it
|
||||
|
||||
from grand_exchange_live.live import HORIZON, TURNS, Leg, Reference, play # noqa: E402
|
||||
from grand_exchange_live.market import build_market # noqa: E402
|
||||
from grand_exchange_live.protocol import ( # noqa: E402
|
||||
MAX_LEGS,
|
||||
opening,
|
||||
parse_sheet,
|
||||
render,
|
||||
)
|
||||
|
||||
SEEDS = range(60_000, 60_024)
|
||||
PRICE_SENTINEL = 9_000_000
|
||||
VOLUME_SENTINEL = 8_000_000
|
||||
NUMBER = re.compile(r"-?\d+")
|
||||
|
||||
|
||||
def market(seed: int):
|
||||
return build_market(seed, 5, 56, HORIZON)
|
||||
|
||||
|
||||
def sentinel_market(seed: int):
|
||||
"""The same basket with every GRADED tick replaced by a value that encodes its index and
|
||||
that no real stream can produce — prices here run to five figures and volumes to three.
|
||||
The visible warmup is left alone, so the render still has something true to show."""
|
||||
items = []
|
||||
for item in market(seed).items:
|
||||
visible = item.prices[:56]
|
||||
prices = visible + [PRICE_SENTINEL + t for t in range(56, len(item.prices))]
|
||||
volumes = (item.volumes[:56]
|
||||
+ [VOLUME_SENTINEL + t for t in range(56, len(item.volumes))])
|
||||
items.append(replace(item, prices=prices, volumes=volumes))
|
||||
return replace(market(seed), items=items)
|
||||
|
||||
|
||||
def walk(basket, policy=None):
|
||||
"""Play a basket, keeping (view, rendered text) for every look."""
|
||||
seen = []
|
||||
|
||||
def spy(view):
|
||||
seen.append((view, render(view)))
|
||||
return policy(view) if policy else []
|
||||
|
||||
play(basket, spy)
|
||||
return seen
|
||||
|
||||
|
||||
class HeldOutLeakTests(unittest.TestCase):
|
||||
def test_no_rendered_turn_carries_a_tick_the_view_had_not_observed(self) -> None:
|
||||
found = 0
|
||||
for seed in SEEDS:
|
||||
basket = sentinel_market(seed)
|
||||
for view, text in walk(basket, Reference()):
|
||||
for token in NUMBER.findall(text):
|
||||
value = int(token)
|
||||
if value >= PRICE_SENTINEL:
|
||||
tick = value - PRICE_SENTINEL
|
||||
elif value >= VOLUME_SENTINEL:
|
||||
tick = value - VOLUME_SENTINEL
|
||||
else:
|
||||
continue
|
||||
found += 1
|
||||
self.assertLess(
|
||||
tick, view.seen,
|
||||
f"basket {basket.seed} look {view.turn} rendered tick {tick} "
|
||||
f"with only {view.seen} observed — the held-out window is on screen",
|
||||
)
|
||||
# Not vacuous: the ticks that HAVE executed are graded ticks and are rendered, so a
|
||||
# basket that produced no sentinel at all would mean the scan found nothing to check.
|
||||
self.assertGreater(found, 0)
|
||||
|
||||
def test_the_rendered_ticks_reassemble_into_exactly_the_observed_prefix(self) -> None:
|
||||
for seed in SEEDS:
|
||||
basket = market(seed)
|
||||
got: dict[str, list[int]] = {i.name: [] for i in basket.items}
|
||||
last_view = None
|
||||
for view, text in walk(basket, Reference()):
|
||||
last_view = view
|
||||
name = None
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped in got:
|
||||
name = stripped
|
||||
elif stripped.startswith("price ") and name:
|
||||
got[name].extend(int(t) for t in stripped.split()[1:])
|
||||
elif line.startswith(tuple(f"{i.name} (" for i in basket.items)):
|
||||
name = line.split(" (")[0]
|
||||
for item in basket.items:
|
||||
self.assertEqual(got[item.name], item.prices[:last_view.seen],
|
||||
f"{item.name} in basket {basket.seed}")
|
||||
|
||||
def test_the_last_look_never_sees_the_tail(self) -> None:
|
||||
"""The floor on the held-out slice: even an agent that takes every look is graded on
|
||||
ticks that ran after its final observation."""
|
||||
for seed in SEEDS:
|
||||
views = [v for v, _ in walk(market(seed), Reference())]
|
||||
self.assertEqual(len(views), TURNS)
|
||||
self.assertLessEqual(views[-1].seen, 56 + HORIZON - 11)
|
||||
|
||||
|
||||
class OpeningTests(unittest.TestCase):
|
||||
def test_the_opening_prompt_is_the_engines_own_first_turn(self) -> None:
|
||||
"""`opening()` reconstructs the turn-1 view at `load()` time, before the engine has
|
||||
run. If the two ever drift, the model reads a header describing a state the engine
|
||||
is not in — and nothing else in the repository would notice."""
|
||||
for seed in SEEDS:
|
||||
basket = market(seed)
|
||||
first = walk(basket)[0][1]
|
||||
self.assertEqual(opening(basket, TURNS), first, f"basket {seed}")
|
||||
|
||||
def test_the_board_is_on_the_opening_turn_and_not_repeated(self) -> None:
|
||||
seen = walk(market(60_000), Reference())
|
||||
self.assertIn("buy limit", seen[0][1])
|
||||
for _, text in seen[1:]:
|
||||
self.assertNotIn("buy limit", text)
|
||||
|
||||
|
||||
class ParseTests(unittest.TestCase):
|
||||
def test_a_well_formed_sheet_parses(self) -> None:
|
||||
sheet = parse_sheet(
|
||||
'here you go\n```json\n{"orders": [{"item": "Bogwater draught", '
|
||||
'"quantity": 400, "buy": 118, "sell": 129}]}\n```'
|
||||
)
|
||||
self.assertFalse(sheet.malformed)
|
||||
self.assertEqual(sheet.legs, [Leg("Bogwater draught", 400, 118, 129)])
|
||||
|
||||
def test_an_explicit_hold_is_a_hold_and_not_a_parse_failure(self) -> None:
|
||||
"""The measured failure of the one-shot form was a well-formed empty answer in 31 of
|
||||
32 rollouts. It must score what holding scores, and it must not be reported as a
|
||||
parser bug."""
|
||||
for reply in ('```json\n{"orders": []}\n```', "```json\n[]\n```",
|
||||
'```json\n{}\n```',
|
||||
'```json\n{"expected_profit": 0, "orders": []}\n```'):
|
||||
sheet = parse_sheet(reply)
|
||||
self.assertEqual(sheet.legs, [])
|
||||
self.assertFalse(sheet.malformed, reply)
|
||||
|
||||
def test_a_bare_array_is_a_sheet(self) -> None:
|
||||
sheet = parse_sheet('```json\n[{"item": "x", "quantity": 1, "buy": 2, "sell": 3}]\n```')
|
||||
self.assertEqual(sheet.legs, [Leg("x", 1, 2, 3)])
|
||||
self.assertFalse(sheet.malformed)
|
||||
|
||||
def test_the_last_block_wins(self) -> None:
|
||||
sheet = parse_sheet('```json\n{"orders": [{"item": "a", "quantity": 1, "buy": 1, '
|
||||
'"sell": 2}]}\n```\non reflection\n```json\n{"orders": []}\n```')
|
||||
self.assertEqual(sheet.legs, [])
|
||||
|
||||
def test_hostile_replies_never_raise_and_never_invent_a_leg(self) -> None:
|
||||
for hostile in (
|
||||
"", "no trades today", "```json\n{not json}\n```",
|
||||
'```json\n{"orders": {"item": "x"}}\n```',
|
||||
'```json\n{"orders": [{"item": "x", "quantity": NaN, "buy": 1, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": [{"item": "x", "quantity": Infinity, "buy": 1, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": [{"item": "x", "quantity": 1e309, "buy": 1, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": [{"item": "x", "quantity": 1e400, "buy": 1, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": [{"item": "x", "quantity": 1, "buy": -Infinity, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": [{"item": 7, "quantity": 1, "buy": 1, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": [null, 3, "x"]}\n```',
|
||||
"```json\n" + "[" * 20_000 + "]" * 20_000 + "\n```",
|
||||
):
|
||||
sheet = parse_sheet(hostile)
|
||||
self.assertEqual(sheet.legs, [], hostile[:40])
|
||||
|
||||
def test_a_malformed_reply_is_a_hold_and_is_flagged_as_one(self) -> None:
|
||||
for hostile in ("no trades today", "```json\n{not json}\n```",
|
||||
'```json\n{"orders": {"item": "x"}}\n```'):
|
||||
self.assertTrue(parse_sheet(hostile).malformed, hostile[:40])
|
||||
|
||||
def test_a_sheet_is_capped(self) -> None:
|
||||
rows = ",".join('{"item": "x%d", "quantity": 1, "buy": 1, "sell": 2}' % i
|
||||
for i in range(MAX_LEGS + 5))
|
||||
sheet = parse_sheet('```json\n{"orders": [%s]}\n```' % rows)
|
||||
self.assertEqual(len(sheet.legs), MAX_LEGS)
|
||||
self.assertEqual(sheet.dropped_rows, 5)
|
||||
|
||||
def test_hostile_replies_played_through_the_engine_never_break_it(self) -> None:
|
||||
"""The parser guarantees nothing on its own — the legs it returns go into the book.
|
||||
A leg naming nothing, a cancel of an offer that is not there and a price of zero are
|
||||
all reachable from a reply, and each must be dropped rather than fatal."""
|
||||
replies = [
|
||||
"", "no trades today",
|
||||
'```json\n{"orders": [{"item": "nothing here", "quantity": 5, "buy": 1, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": [{"item": "Bogwater draught", "quantity": 0, "buy": 0, "sell": 0}]}\n```',
|
||||
'```json\n{"orders": [{"item": "Bogwater draught", "quantity": -5, "buy": 1, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": [{"item": "Bogwater draught", "quantity": 999999999, "buy": 1, "sell": 2}]}\n```',
|
||||
'```json\n{"orders": []}\n```',
|
||||
'```json\n{"orders": [{"item": "Bogwater draught", "quantity": 5, "buy": 1000000000, "sell": 1}]}\n```',
|
||||
"```json\n{not json}\n```",
|
||||
]
|
||||
for seed in SEEDS:
|
||||
turn = {"i": 0}
|
||||
|
||||
def policy(view, turn=turn):
|
||||
sheet = parse_sheet(replies[turn["i"] % len(replies)])
|
||||
turn["i"] += 1
|
||||
return sheet.legs
|
||||
|
||||
result = play(market(seed), policy)
|
||||
self.assertGreaterEqual(result.realised, -float(market(seed).capital))
|
||||
self.assertGreaterEqual(result.peak_employed, 0.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,224 @@
|
||||
"""The taskset and the env: the loop, the seat, and what reaches the trace.
|
||||
|
||||
Everything else in this directory runs with nothing installed. This file needs `verifiers`,
|
||||
so it skips when it is absent — `uv sync --project environments/grand_exchange_live` first,
|
||||
exactly as the README's release gate does.
|
||||
|
||||
The env is driven against a stub agent rather than a model. What is being tested is the
|
||||
control flow that only exists here: a synchronous engine walked through an asynchronous
|
||||
interaction, a prompted task whose first turn must be BARE, a `terminated` segment arriving
|
||||
mid-window, and the rule that `TurnView.market` never reaches the trace.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import context # noqa: E402 - resolves the package before anything imports it
|
||||
|
||||
try:
|
||||
import verifiers.v1 as vf # noqa: F401
|
||||
except ImportError: # pragma: no cover - the no-dependency path
|
||||
vf = None
|
||||
|
||||
from grand_exchange_live.live import HORIZON, TURNS # noqa: E402
|
||||
from grand_exchange_live.market import build_market # noqa: E402
|
||||
from grand_exchange_live.protocol import opening # noqa: E402
|
||||
|
||||
PRICE_SENTINEL = 9_000_000
|
||||
|
||||
|
||||
@dataclass
|
||||
class StubTrace:
|
||||
rewards: dict = field(default_factory=dict)
|
||||
metrics: dict = field(default_factory=dict)
|
||||
info: dict = field(default_factory=dict)
|
||||
|
||||
def record_reward(self, name, value, weight=1.0):
|
||||
self.rewards[name] = (float(value), float(weight))
|
||||
|
||||
def record_metrics(self, values):
|
||||
self.metrics.update({k: float(v) for k, v in values.items()})
|
||||
|
||||
|
||||
@dataclass
|
||||
class StubSegment:
|
||||
last_reply: str = ""
|
||||
terminated: bool = False
|
||||
|
||||
|
||||
class StubInteraction:
|
||||
"""One rollout, scripted. Records what each turn was asked with."""
|
||||
|
||||
def __init__(self, replies, terminate_at=None):
|
||||
self.replies = replies
|
||||
self.terminate_at = terminate_at
|
||||
self.asked: list = []
|
||||
self.trace = StubTrace()
|
||||
|
||||
async def turn(self, message=None):
|
||||
self.asked.append(message)
|
||||
n = len(self.asked)
|
||||
if self.terminate_at is not None and n >= self.terminate_at:
|
||||
return StubSegment(terminated=True)
|
||||
return StubSegment(last_reply=self.replies[(n - 1) % len(self.replies)])
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
|
||||
class StubAgent:
|
||||
def __init__(self, interaction):
|
||||
self._interaction = interaction
|
||||
|
||||
def interaction(self, task):
|
||||
return self._interaction
|
||||
|
||||
|
||||
class StubAgents:
|
||||
def __init__(self, interaction):
|
||||
self.agent = StubAgent(interaction)
|
||||
|
||||
|
||||
HOLD = '```json\n{"orders": []}\n```'
|
||||
|
||||
|
||||
@unittest.skipIf(vf is None, "verifiers is not installed")
|
||||
class TasksetTests(unittest.TestCase):
|
||||
def taskset(self, num_tasks=4):
|
||||
from grand_exchange_live.taskset import LiveExchangeConfig, LiveExchangeTaskset
|
||||
|
||||
return LiveExchangeTaskset(
|
||||
LiveExchangeConfig(id="grand-exchange-live", num_tasks=num_tasks)
|
||||
)
|
||||
|
||||
def test_the_seat_is_named_agent(self) -> None:
|
||||
"""A wrong name parses fine and silently leaves the seat empty, so it is asserted
|
||||
rather than read off the file."""
|
||||
from grand_exchange_live.taskset import LiveExchangeEnvConfig
|
||||
|
||||
self.assertIn("agent", LiveExchangeEnvConfig.model_fields)
|
||||
|
||||
def test_tasks_carry_the_seed_and_the_opening_board(self) -> None:
|
||||
tasks = self.taskset().load()
|
||||
self.assertEqual(len(tasks), 4)
|
||||
for task in tasks:
|
||||
market = build_market(task.data.seed, task.data.num_items, task.data.visible,
|
||||
HORIZON)
|
||||
self.assertEqual(task.data.prompt, opening(market, TURNS))
|
||||
self.assertEqual(task.data.turns, TURNS)
|
||||
self.assertIn("Grand Exchange", task.data.system_prompt)
|
||||
|
||||
def test_two_tasks_never_share_a_basket(self) -> None:
|
||||
"""`viable_live_market` skips baskets the reference loses money in; a load that did
|
||||
not advance past the seed it landed on would serve the same basket twice."""
|
||||
seeds = [t.data.seed for t in self.taskset(24).load()]
|
||||
self.assertEqual(len(set(seeds)), len(seeds))
|
||||
|
||||
def test_the_task_stops_at_the_turn_budget(self) -> None:
|
||||
task = self.taskset(1).load()[0]
|
||||
trace = types.SimpleNamespace(num_turns=TURNS - 1)
|
||||
self.assertFalse(asyncio.run(task.budget(trace)))
|
||||
trace.num_turns = TURNS
|
||||
self.assertTrue(asyncio.run(task.budget(trace)))
|
||||
|
||||
|
||||
@unittest.skipIf(vf is None, "verifiers is not installed")
|
||||
class EnvRunTests(unittest.TestCase):
|
||||
def env_and_task(self):
|
||||
from grand_exchange_live.taskset import (
|
||||
LiveExchangeConfig,
|
||||
LiveExchangeEnv,
|
||||
LiveExchangeEnvConfig,
|
||||
LiveExchangeTaskset,
|
||||
)
|
||||
|
||||
config = LiveExchangeEnvConfig(
|
||||
taskset=LiveExchangeConfig(id="grand-exchange-live", num_tasks=1)
|
||||
)
|
||||
# No harness, no model, no runtime: `run()` is the whole subject here, and building
|
||||
# the env through its constructor would stand up an agent to reach it.
|
||||
env = LiveExchangeEnv.__new__(LiveExchangeEnv)
|
||||
env.config = config
|
||||
return env, LiveExchangeTaskset(config.taskset).load()[0]
|
||||
|
||||
def run_env(self, replies, terminate_at=None, task=None, env=None):
|
||||
from grand_exchange_live.taskset import LiveExchangeEnv
|
||||
|
||||
if env is None:
|
||||
env, task = self.env_and_task()
|
||||
interaction = StubInteraction(replies, terminate_at)
|
||||
asyncio.run(LiveExchangeEnv.run(env, task, StubAgents(interaction)))
|
||||
return interaction
|
||||
|
||||
def test_the_first_turn_is_bare_and_every_later_turn_carries_the_view(self) -> None:
|
||||
"""The task is prompted, so the model speaks first. Passing the opening text on turn
|
||||
one raises ValueError inside `verifiers` — the prompt already opened the exchange."""
|
||||
interaction = self.run_env([HOLD])
|
||||
self.assertEqual(len(interaction.asked), TURNS)
|
||||
self.assertIsNone(interaction.asked[0])
|
||||
for message in interaction.asked[1:]:
|
||||
self.assertIsInstance(message, str)
|
||||
self.assertIn("Look ", message)
|
||||
|
||||
def test_a_terminated_segment_mid_window_is_not_an_error(self) -> None:
|
||||
"""The naive loop calls turn() again and gets RuntimeError('the exchange is over'),
|
||||
which fails the episode over a model that stopped talking. The window runs on."""
|
||||
interaction = self.run_env([HOLD], terminate_at=4)
|
||||
self.assertEqual(interaction.trace.metrics["ended_early"], 1.0)
|
||||
self.assertEqual(interaction.trace.metrics["answered_turns"], 3.0)
|
||||
self.assertEqual(set(interaction.trace.rewards), {"profit", "discipline", "gate"})
|
||||
|
||||
def test_a_hold_every_turn_scores_exactly_zero(self) -> None:
|
||||
"""House rule 3's floor, through the whole stack rather than through the engine —
|
||||
including the degenerate reply that 31 of 32 one-shot rollouts produced."""
|
||||
interaction = self.run_env([HOLD])
|
||||
self.assertEqual(interaction.trace.metrics["held_replies"], float(TURNS))
|
||||
self.assertEqual(interaction.trace.metrics["malformed_replies"], 0.0)
|
||||
for name, (score, _) in interaction.trace.rewards.items():
|
||||
self.assertEqual(score, 0.0, name)
|
||||
|
||||
def test_a_malformed_reply_is_a_hold_and_never_fails_the_episode(self) -> None:
|
||||
interaction = self.run_env(["I would rather not."])
|
||||
self.assertEqual(interaction.trace.metrics["malformed_replies"], float(TURNS))
|
||||
self.assertEqual(interaction.trace.rewards["profit"][0], 0.0)
|
||||
|
||||
def test_a_real_sheet_trades(self) -> None:
|
||||
import json
|
||||
|
||||
from grand_exchange_live.book import reference_orders
|
||||
|
||||
env, task = self.env_and_task()
|
||||
market = build_market(task.data.seed, task.data.num_items, task.data.visible,
|
||||
HORIZON)
|
||||
sheet = "```json\n" + json.dumps({"orders": [
|
||||
{"item": o.item, "quantity": o.quantity, "buy": o.buy, "sell": o.sell}
|
||||
for o in reference_orders(market)
|
||||
]}) + "\n```"
|
||||
interaction = self.run_env([sheet, HOLD], env=env, task=task)
|
||||
self.assertGreater(interaction.trace.metrics["bought"], 0.0)
|
||||
self.assertGreater(interaction.trace.rewards["profit"][0], 0.0)
|
||||
|
||||
def test_the_trace_carries_no_held_out_tick_and_no_market(self) -> None:
|
||||
"""House rule 1's other half. The observation is `protocol.render`'s problem; this is
|
||||
the one that would put the whole series in the training data through `trace.info`."""
|
||||
interaction = self.run_env([HOLD])
|
||||
for key, value in interaction.trace.info.items():
|
||||
self.assertIsInstance(value, (int, float, str), key)
|
||||
for key, value in interaction.trace.metrics.items():
|
||||
self.assertIsInstance(value, float, key)
|
||||
self.assertLess(abs(value), PRICE_SENTINEL, key)
|
||||
self.assertEqual(set(interaction.trace.info), {"basket", "freeze"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user