"""Signal-tier tasks: our own private evals, loaded from JSONL.

One family (`agent_ops`, `repo_edit`, ...) becomes one Inspect Task. Samples
within a family may be graded differently; `kbench.scorers.dispatching` handles
that per sample.

Split discipline is enforced here rather than left to convention: by default
only `test` samples are evaluated. If a `train` sample ever reaches the eval
path, fine-tuning on it silently invalidates every number that follows and
there is no way to detect it after the fact.
"""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any

from inspect_ai import Task, task
from inspect_ai.dataset import MemoryDataset, Sample
from inspect_ai.solver import generate

# Absolute, not relative. inspect-ai loads a task file by path as a standalone module, so
# it is not a member of the `kbench` package at load time and `from ..scorers` raises
# "attempted relative import beyond top-level package". kbench is an installed package, so
# the absolute form resolves under both import styles.
from kbench.scorers import dispatching

# The schema lives in kbench/schema.py, which imports nothing beyond the standard library —
# validating a JSONL record must not require the eval framework. Re-exported here so existing
# callers of signal.family_path / signal.validate_record keep working.
from kbench.schema import (  # noqa: F401
    DATA_DIR,
    VALID_SCORERS,
    VALID_SPLITS,
    family_path,
    validate_record,
)




def load_samples(
    family: str,
    tier: str = "private",
    splits: tuple[str, ...] = ("test",),
) -> list[Sample]:
    path = family_path(family, tier)
    if not path.exists():
        raise FileNotFoundError(
            f"no data file for family {family!r} at {path}. "
            f"create samples with: kbench add {family}"
        )

    samples: list[Sample] = []
    for lineno, line in enumerate(path.read_text().splitlines(), start=1):
        line = line.strip()
        if not line or line.startswith("//"):
            continue
        rec = json.loads(line)
        validate_record(rec, f"{path.name}:{lineno}")
        if rec["split"] not in splits:
            continue
        samples.append(
            Sample(
                id=rec["id"],
                input=rec["input"],
                target=rec.get("target") or "",
                metadata={
                    "scorer": rec["scorer"],
                    "rubric": rec.get("rubric"),
                    "family": rec.get("family", family),
                    "split": rec["split"],
                    **(rec.get("metadata") or {}),
                },
            )
        )
    return samples


def signal_task(
    family: str,
    tier: str = "private",
    splits: tuple[str, ...] = ("test",),
    judge_model: str | None = None,
) -> Task:
    """Build an Inspect Task for one signal family."""
    samples = load_samples(family, tier=tier, splits=splits)
    if not samples:
        raise ValueError(
            f"family {family!r} has no samples in splits {splits}. "
            f"refusing to run an empty eval -- it would report a score of 0/0."
        )
    return Task(
        name=f"signal/{family}",
        dataset=MemoryDataset(samples=samples, name=family),
        solver=generate(),
        scorer=dispatching(judge_model=judge_model),
    )


# --- registered example task -------------------------------------------------
# Demonstrates the machinery against the shipped public examples. Real families
# live in data/private/ and are registered the same way.


@task
def example() -> Task:
    """Public example family -- shows the format, measures nothing important."""
    return signal_task("example", tier="public", splits=("test",))


@task
def agent_ops() -> Task:
    """Operating a Compute node: admission, watchdog policy, Scene semantics, process identity.

    Only the `test` split runs, which is the split discipline this module enforces rather than
    documents: a `train` sample reaching the eval path silently invalidates every number after
    it, and there is no way to detect that afterwards.
    """
    return signal_task("agent_ops", tier="private", splits=("test",))