Karti Tripathi
96 lines
3.1 KiB
Python
96 lines
3.1 KiB
Python
"""Scorers for signal-tier tasks.
|
|
|
|
Inspect attaches scorers to a Task, not to a sample. Our families are mixed --
|
|
one `agent_ops` file legitimately contains samples graded by exact match, by
|
|
substring, and by rubric. Splitting a family into one Task per scorer would
|
|
fragment the results and make family-level scores meaningless.
|
|
|
|
So we dispatch per sample instead, reading `scorer` from sample metadata.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
from inspect_ai.scorer import (
|
|
CORRECT,
|
|
INCORRECT,
|
|
Score,
|
|
Scorer,
|
|
Target,
|
|
accuracy,
|
|
model_graded_qa,
|
|
scorer,
|
|
stderr,
|
|
)
|
|
from inspect_ai.solver import TaskState
|
|
|
|
DEFAULT_RUBRIC = """You are grading a model's answer against a reference.
|
|
|
|
Grade CORRECT if the answer is substantively equivalent to the reference --
|
|
matching facts, decisions, and any required steps. Ignore differences in
|
|
wording, formatting, or verbosity.
|
|
|
|
Grade INCORRECT if it contradicts the reference, omits something the reference
|
|
treats as required, or adds a claim the reference does not support.
|
|
|
|
Be strict. A partially correct answer is INCORRECT."""
|
|
|
|
|
|
def _text(state: TaskState) -> str:
|
|
return (state.output.completion or "").strip()
|
|
|
|
|
|
@scorer(metrics=[accuracy(), stderr()])
|
|
def dispatching(judge_model: str | None = None) -> Scorer:
|
|
"""Grade each sample by the method named in its own metadata.
|
|
|
|
Falls back to `includes` when unspecified, which is the most forgiving
|
|
deterministic option -- a sample that silently scores zero because of a
|
|
missing metadata field would look like a model failure.
|
|
"""
|
|
graded = model_graded_qa(instructions=DEFAULT_RUBRIC, model=judge_model)
|
|
|
|
async def score(state: TaskState, target: Target) -> Score:
|
|
method = (state.metadata or {}).get("scorer") or "includes"
|
|
answer = _text(state)
|
|
expected = target.text or ""
|
|
|
|
if method == "rubric":
|
|
rubric = (state.metadata or {}).get("rubric")
|
|
if rubric:
|
|
custom = model_graded_qa(instructions=rubric, model=judge_model)
|
|
return await custom(state, target)
|
|
return await graded(state, target)
|
|
|
|
if method == "exact":
|
|
ok = answer == expected.strip()
|
|
elif method == "regex":
|
|
try:
|
|
ok = re.search(expected, answer, re.IGNORECASE | re.DOTALL) is not None
|
|
except re.error as exc:
|
|
return Score(
|
|
value=INCORRECT,
|
|
answer=answer,
|
|
explanation=f"invalid regex in sample target: {exc}",
|
|
)
|
|
elif method == "includes":
|
|
ok = expected.strip().lower() in answer.lower()
|
|
else:
|
|
return Score(
|
|
value=INCORRECT,
|
|
answer=answer,
|
|
explanation=f"unknown scorer {method!r} -- fix the sample, not the model",
|
|
)
|
|
|
|
return Score(
|
|
value=CORRECT if ok else INCORRECT,
|
|
answer=answer,
|
|
explanation=f"scorer={method}",
|
|
)
|
|
|
|
return score
|
|
|
|
|
|
__all__ = ["DEFAULT_RUBRIC", "dispatching"]
|