karti-ai
a56f097f28
The Python is the source of truth; src/demos/wordle/engine.ts will be a port of it, and CI gates the two against a SHA-256 over all 21.2M (guess, answer) pattern pairs rather than a hand-picked vector file — a vector file only ever catches the cases somebody thought of. The reward is three weighted components, and the third one is the reason this demo is worth building. `solved` and `economy` pull toward winning. `consistency` pulls against them, because a player maximising information deliberately guesses words that cannot win — a word that splits the remaining candidates evenly teaches more than a word that might happen to be right. That is good play, and it costs consistency. The probe ladder proves the tension is real rather than asserted: inaction 0.0000 crude 0.0111 plausible 0.1224 candidate_only 0.8925 exhaustive 0.9031 oracle 0.9458 The two good policies are 0.05 apart and neither dominates — the entropy oracle takes 1.00 economy and 0.73 consistency, the candidate-only player takes 0.75 and 1.00. Which one wins is a decision about what you want, which is the whole argument the site exists to make. probe.py fails CI if either starts dominating. Two traps found by building it. `consistency` is scored over turns SPENT, not guesses accepted: counting only legal guesses hands a free 1.0 to a policy that plays one word and then jams the parser five times — one guess, no contradictions, perfect score. And `economy`'s denominator is the depth the SHIPPED solver reaches, not a depth-optimal search: entropy-greedy is not depth-optimal, so grading it against an exact optimum would make the oracle rung fail its own assertion on some seeds. The word lists are built from Wordnik (MIT) intersected with SCOWL, never from the original game's 2,315 answers. 4,603 answers makes this materially harder than the original, so the published SALET/3.4212 results are cited as belonging to that list and our own reference player's TARES/3.72 is measured here. verifiers is an optional extra. The engine, reward, solver and probe all run — and gate — without an RL stack resolvable. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
143 lines
5.1 KiB
Python
143 lines
5.1 KiB
Python
"""The duplicate-letter cases first, because they are the ones that get shipped wrong."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from wordle_five.engine import (
|
|
Game,
|
|
answer_for_seed,
|
|
answers,
|
|
conformance_digest,
|
|
guesses,
|
|
hard_mode_violation,
|
|
is_consistent,
|
|
score_guess,
|
|
)
|
|
|
|
# Every one of these was verified against an independent implementation and
|
|
# against TextArena's own scorer before being written down. They are the
|
|
# regression surface for the two-pass rule.
|
|
VECTORS = [
|
|
("alloy", "llama", "YGYXX"), # greens claim first, yellows take the rest
|
|
("speed", "erase", "YXYYX"), # both E's yellow: exactly two are available
|
|
("array", "radar", "YYYGX"), # double-double with one green
|
|
("sassy", "basis", "YGGXX"), # 3 S guessed, 2 in answer: one Y, one X
|
|
("eerie", "rebel", "YGYXX"), # the same shape on E
|
|
("level", "eagle", "YYXYX"), # two repeated letters, no greens at all
|
|
("geese", "these", "XXGGG"), # greens exhaust the pool, leading E greys
|
|
("abbey", "abbot", "GGGXX"),
|
|
("crane", "plane", "XXGGG"),
|
|
("alloy", "balmy", "YXGXG"),
|
|
("tares", "tares", "GGGGG"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("guess,answer,expected", VECTORS)
|
|
def test_scoring_vectors(guess: str, answer: str, expected: str) -> None:
|
|
assert score_guess(guess, answer) == expected
|
|
|
|
|
|
def test_score_is_case_insensitive() -> None:
|
|
assert score_guess("ALLOY", "LLAMA") == score_guess("alloy", "llama")
|
|
|
|
|
|
def test_a_letter_is_never_marked_more_often_than_it_occurs() -> None:
|
|
"""The invariant the two passes exist to preserve, checked over the list."""
|
|
pool = answers()[:200]
|
|
for answer in pool:
|
|
for guess in pool[:50]:
|
|
pattern = score_guess(guess, answer)
|
|
for letter in set(guess):
|
|
marked = sum(
|
|
1 for i, c in enumerate(guess) if c == letter and pattern[i] != "X"
|
|
)
|
|
assert marked <= answer.count(letter)
|
|
|
|
|
|
def test_consistency_is_symmetric_with_scoring() -> None:
|
|
"""`is_consistent` must agree with `score_guess`, since the reward uses it."""
|
|
for answer in answers()[:100]:
|
|
guess = "tares"
|
|
pattern = score_guess(guess, answer)
|
|
assert is_consistent(answer, guess, pattern)
|
|
|
|
|
|
def test_every_answer_is_guessable() -> None:
|
|
assert set(answers()) <= guesses()
|
|
|
|
|
|
def test_seeded_answer_is_deterministic() -> None:
|
|
assert answer_for_seed(42) == answer_for_seed(42)
|
|
assert answer_for_seed(1) != answer_for_seed(2)
|
|
|
|
|
|
def test_rejections_do_not_consume_a_row() -> None:
|
|
game = Game(seed=3, answer="tares")
|
|
assert game.play("zzzzz")[1] is not None # not a word
|
|
assert game.play("four")[1] is not None # wrong length
|
|
assert len(game.history) == 0
|
|
assert game.rejected == 2
|
|
assert game.play("crane")[0] is not None
|
|
assert len(game.history) == 1
|
|
|
|
|
|
def test_a_repeat_is_rejected() -> None:
|
|
game = Game(seed=3, answer="tares")
|
|
game.play("crane")
|
|
assert "already been guessed" in (game.play("crane")[1] or "")
|
|
|
|
|
|
def test_game_never_raises_on_junk() -> None:
|
|
game = Game(seed=3, answer="tares")
|
|
for junk in ["", " ", "!!!!!", "a", "abcdefghij", "12345"]:
|
|
pattern, reason = game.play(junk)
|
|
assert pattern is None and reason is not None
|
|
|
|
|
|
def test_solving_ends_the_game() -> None:
|
|
game = Game(seed=3, answer="tares")
|
|
game.play("tares")
|
|
assert game.solved and game.over
|
|
|
|
|
|
# ------------------------------------------------------------- hard mode --
|
|
|
|
|
|
def test_hard_mode_locks_greens() -> None:
|
|
assert hard_mode_violation("crown", "crane", "GGXXX") is None
|
|
assert hard_mode_violation("blown", "crane", "GGXXX") is not None
|
|
|
|
|
|
def test_hard_mode_counts_yellows_rather_than_testing_membership() -> None:
|
|
"""Two revealed E's require two E's, not merely one."""
|
|
pattern = score_guess("speed", "erase") # YXYYX — two E's revealed
|
|
assert hard_mode_violation("ester", "speed", pattern) is None # two E's
|
|
assert hard_mode_violation("crest", "speed", pattern) is not None # only one E
|
|
|
|
|
|
def test_hard_mode_does_not_ban_grey_letters() -> None:
|
|
"""The rule most implementations add and the real game does not have."""
|
|
pattern = score_guess("crane", "tares") # C grey
|
|
assert pattern[0] == "X", "C should be grey against TARES"
|
|
# A guess reusing that grey C is legal, so long as it still satisfies every
|
|
# green and yellow that was revealed.
|
|
assert hard_mode_violation("stare", "crane", pattern) is None
|
|
|
|
|
|
# ------------------------------------------------------ cross-language gate --
|
|
|
|
|
|
def test_conformance_digest_matches_the_committed_value() -> None:
|
|
"""The gate that keeps the TypeScript port honest.
|
|
|
|
This hashes every (guess, answer) pair in the answer list. If it changes,
|
|
either the scorer changed or the word list did, and `src/demos/wordle/
|
|
engine.ts` must be re-verified against it — `pnpm conformance` does the
|
|
other half.
|
|
"""
|
|
from pathlib import Path
|
|
|
|
committed = (Path(__file__).parent.parent / "CONFORMANCE.txt").read_text().split()[0]
|
|
assert conformance_digest() == committed
|