"""
CLI runner for the failure-analyzer evaluation.
Runs the AI failure analyzer (Qwen3.8 27B via Groq) live on each case in the
golden dataset, then judges every produced diagnosis with the GPT-OSS judge on
four dimensions (correctness, groundedness, completeness, actionability). Prints
a per-case table plus aggregate metrics and writes a JSON report.
Both the analyzer under test and the judge call Groq, so only ``GROQ_API_KEY``
is required.
Usage::
poetry run python -m evals
poetry run python -m evals --judge-model openai/gpt-oss-120b
"""
import argparse
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from dotenv import load_dotenv
from loguru import logger
from evals.dataset import load_dataset
from evals.judge import DEFAULT_JUDGE_MODEL, judge_case, sanitize_diagnosis
DEFAULT_DATASET = "evals/golden_dataset.yaml"
DEFAULT_OUTPUT = "evals/results/eval_results.json"
DIMENSIONS = ("correctness", "groundedness", "completeness", "actionability")
[docs]
def _make_scout():
"""
Build a fresh, force-enabled analyzer instance.
Mirrors how ``tests/conftest.py`` force-enables analysis under
``--failure-analysis`` (sets ``enabled`` and re-reads the key), but on a new
instance rather than the module singleton — so the eval never appends to the
singleton's ``_results`` or writes ``ai_analysis/``, and works regardless of
the ``AI_ANALYSIS_ENABLED`` env var.
:returns: A ready-to-use ``FailureAnalyzer``.
"""
from tests.helpers.failure_analyzer import FailureAnalyzer
scout = FailureAnalyzer() # Here self.enabled is whatever AI_ANALYSIS_ENABLED is; likely false during an eval run
scout.enabled = True
scout.api_key = os.getenv("GROQ_API_KEY")
return scout
[docs]
def _build_payload(case: dict, diagnosis: dict) -> dict:
"""
Assemble the single JSON object sent to the judge for one case.
The payload bundles the three things the judge needs: the ``failure_context``
the analyzer saw, the ``diagnosis`` it produced (already cleaned), and the
``expected`` reference answer the dataset author wrote.
The ``failure_context`` is fed to Scout to produce the *whole* diagnosis, and
that one diagnosis is then judged on all four dimensions — ``failure_context``
is not tied to any single dimension. What sets correctness apart is the judge,
not the input: correctness compares Scout's ``diagnosis`` (its ``category`` and
``root_cause``) against ``expected`` — the known-true answer a human authored
for this case — so it needs that reference to grade against. Groundedness,
completeness, and actionability need no reference: they judge the diagnosis
against the ``failure_context`` evidence itself, asking whether it is
internally sound, not whether it is *right*.
This is why correctness is an offline-only dimension. ``expected`` exists only
in the curated dataset; it is hand-written by someone who already knew the true
cause, and is *not* derivable from ``failure_context`` (if it were, we would not
need Scout). A live failure has no such answer key — so ``judge_live`` drops
correctness and scores only the other three. This is the exact shape
``judge.judge_case`` documents and ``JUDGE_SYSTEM_PROMPT`` describes.
How it differs from the live path
┌──────────────────────┬───────────────────────────────────────────────┬─────────────────────────────┐
│ │ _build_payload (offline) │ judge_live's inline payload │
├──────────────────────┼───────────────────────────────────────────────┼─────────────────────────────┤
│ keys │ case_id, failure_context, diagnosis, expected │ failure_context, diagnosis │
├──────────────────────┼───────────────────────────────────────────────┼─────────────────────────────┤
│ expected? │ yes (enables correctness) │ no (no reference) │
├──────────────────────┼───────────────────────────────────────────────┼─────────────────────────────┤
│ sanitizes diagnosis? │ no — caller already did │ yes — inline │
└──────────────────────┴───────────────────────────────────────────────┴─────────────────────────────┘
:param case: A dataset case (``id``, ``failure_context``, ``expected``) from :func:`evals.dataset.load_dataset`
:param diagnosis: The cleaned diagnosis for that case.
:returns: The judge input object.
"""
return {
"case_id": case["id"],
"failure_context": case["failure_context"],
"diagnosis": diagnosis,
"expected": case["expected"],
}
[docs]
def _diagnose_cases(cases: list[dict]) -> list[dict]:
"""
Run the analyzer once per dataset case and collect the diagnoses.
Uses a single fresh analyzer instance (see :func:`_make_scout`) for the
whole pass. For each case it calls ``analyze(failure_context)`` and records
the produced diagnosis. ``analyze`` returns ``None`` when the analyzer's own
LLM call fails; that is captured as ``scout_failure`` so the case is carried
through the report (and excluded from quality metrics) rather than crashing
the run.
Exercise the analyzer (Qwen3.8 27B) on known inputs. Each dataset case carries a
realistic ``failure_context``; this function feeds that context to a fresh
``FailureAnalyzer`` and captures what the analyzer model says. This is the only
place in the eval where the system-under-test actually runs.
Purpose: Take the golden dataset and produce one diagnosis per case from
the live analyzer — the raw material the judge will later grade.
:param cases: The validated dataset cases.
:returns: One record per case, each with ``case_id``, ``expected_category``
(lifted out for the table/metrics), the cleaned ``diagnosis`` (or
``None``), and a ``scout_failure`` flag.
"""
scout = _make_scout()
records = []
for case in cases:
logger.info(f"Diagnosing {case['id']}")
diagnosis = scout.analyze(case["failure_context"])
records.append(
{
"case_id": case["id"],
"expected_category": case["expected"]["expected_category"],
"diagnosis": sanitize_diagnosis(diagnosis) if diagnosis else None,
"scout_failure": diagnosis is None,
}
)
return records
[docs]
def _judge_records(records: list[dict], cases_by_id: dict, model: str, api_key: str) -> None:
"""
Score every diagnosed record with the judge, mutating each record in place.
For each record it: (1) skips analyzer failures, marking them as not passing;
(2) computes ``category_match`` directly in code (exact string compare of the
diagnosed category vs. the expected one) — this is deterministic and does not
rely on the judge; (3) calls :func:`judge.judge_case` and stores the raw judge
output under ``judge``; (4) recomputes ``overall_pass`` here as "all four
dimensions passed" rather than trusting the judge's own ``overall_verdict``,
so the headline pass/fail is always consistent with the per-dimension verdicts.
A judge that errors or returns malformed JSON is recorded as ``judge_error``.
:param records: Records from :func:`_diagnose_cases`.
:param cases_by_id: The dataset keyed by case id, to rebuild each judge payload.
:param model: Judge model id.
:param api_key: Groq API key for the judge.
"""
for record in records:
if record["scout_failure"]:
record.update(category_match=False, judge=None, judge_error=False, overall_pass=False)
continue
case = cases_by_id[record["case_id"]]
record["category_match"] = (
record["diagnosis"].get("category") == record["expected_category"]
)
logger.info(f"Judging {record['case_id']}")
judgement = judge_case(_build_payload(case, record["diagnosis"]), model=model, api_key=api_key)
if judgement is None:
record.update(judge=None, judge_error=True, overall_pass=False)
continue
# Recompute overall in code — never trust the judge's own overall_verdict for aggregates.
record["judge"] = judgement
record["judge_error"] = False
record["overall_pass"] = all(
judgement[dim]["verdict"] == "pass" for dim in DIMENSIONS
)
[docs]
def _summarize(records: list[dict]) -> dict:
"""
Roll the per-case records up into the report's aggregate ``summary`` block.
Every number here is just a count or a count/total fraction:
- ``scout_failures`` / ``judge_errors`` — how many cases the analyzer or the
judge could not produce output for. These are surfaced separately and
excluded from the quality fractions below, so an infrastructure hiccup
doesn't masquerade as a quality regression.
- ``category_accuracy`` — of the cases the analyzer actually diagnosed, the
fraction whose category matched the expected one. Computed in code, not by
the judge.
- ``pass_rates`` — of the judged cases, the fraction the judge passed on each
dimension (correctness/groundedness/completeness/actionability) plus
``overall`` (passed all four). This is the headline quality score: one number per dimension
that you can compare across runs to see if a prompt/model change made the
analyzer better or worse.
:param records: The fully judged records.
:returns: The ``summary`` dict embedded in the report and printed to console.
"""
# Denominators: only cases that got a diagnosis / a judgement count toward
# the corresponding quality fractions.
diagnosed = [r for r in records if not r["scout_failure"]]
judged = [r for r in records if r.get("judge")]
def fraction(numerator: int, pool: list) -> float:
"""Safe count/total ratio rounded to 3 dp; 0.0 when the pool is empty."""
return round(numerator / len(pool), 3) if pool else 0.0
# How many judged cases the judge passed on each dimension
pass_rates = {
dim: fraction(sum(1 for r in judged if r["judge"][dim]["verdict"] == "pass"), judged)
for dim in DIMENSIONS
}
pass_rates["overall"] = fraction(sum(1 for r in judged if r["overall_pass"]), judged)
return {
"total_cases": len(records), # Every record, including failures.
"judged_cases": len(judged), # Size of the judged pool
"scout_failures": sum(1 for r in records if r["scout_failure"]), # Count over all records where the analyzer produced nothing
"judge_errors": sum(1 for r in records if r.get("judge_error")), # Count where the judge failed
"category_accuracy": fraction(sum(1 for r in diagnosed if r["category_match"]), diagnosed), # Fraction of diagnosed cases whose category matched
"pass_rates": pass_rates, # The per-dimension + overall dict built above
}
[docs]
def _print_table(records: list[dict], summary: dict) -> None:
"""
Print the per-case table and the aggregate summary to the console.
One row per case showing expected→diagnosed category (with a ✓/✗ match
mark) and the four per-dimension verdicts, followed by the counts and
pass-rate percentages from :func:`_summarize`.
:param records: The fully judged records.
:param summary: The summary dict from :func:`_summarize`.
"""
def verdict(record, dim):
"""Cell text for one dimension: the judge verdict, or an error marker if there's no judgement."""
if record["scout_failure"]:
return "SCOUT-ERR"
if record.get("judge_error") or not record.get("judge"):
return "JUDGE-ERR"
return record["judge"][dim]["verdict"]
header = f"{'case_id':<28} {'category (expected→got)':<34} {'corr':<5} {'grnd':<5} {'cmpl':<5} {'actn':<5} {'overall'}"
print("\n" + header)
print("-" * len(header))
for r in records:
got = "—" if r["scout_failure"] else (r["diagnosis"].get("category") or "?")
cat = f"{r['expected_category']}→{got}"
mark = "✓" if r.get("category_match") else "✗"
overall = "PASS" if r.get("overall_pass") else "FAIL"
print(
f"{r['case_id']:<28} {cat:<32}{mark:>2} "
f"{verdict(r, 'correctness'):<5} {verdict(r, 'groundedness'):<5} "
f"{verdict(r, 'completeness'):<5} {verdict(r, 'actionability'):<5} {overall}"
)
pr = summary["pass_rates"]
print(
f"\nCases: {summary['total_cases']} judged: {summary['judged_cases']} "
f"scout_failures: {summary['scout_failures']} judge_errors: {summary['judge_errors']}"
)
print(f"Category accuracy: {summary['category_accuracy']:.0%}")
print(
f"Pass rates — correctness: {pr['correctness']:.0%} "
f"groundedness: {pr['groundedness']:.0%} completeness: {pr['completeness']:.0%} "
f"actionability: {pr['actionability']:.0%} overall: {pr['overall']:.0%}\n"
)
[docs]
def main(argv=None) -> int:
"""
Entry point: diagnose every dataset case, judge each diagnosis, report.
The orchestrator that Loads the dataset, runs the analyzer over it, judges the results, writes the
JSON report, and prints the summary table.
The shape of it: It does no analysis or grading itself, it just sequences the named stages and handles the I/O
boundary (args in, env check, file out, exit code out). Three things stand out as intentional: the early fail-fast
on the missing key (exit 2 before any work), the single shared records list threaded through
produce→grade→aggregate, and the three distinct exit codes that let a caller distinguish can't-run / failed / passed
:param argv: Optional argument list (defaults to ``sys.argv``).
:returns: Process exit code — ``0`` if every judged case passed all four
dimensions, ``1`` if any failed, ``2`` if ``GROQ_API_KEY`` is missing.
"""
load_dotenv()
parser = argparse.ArgumentParser(prog="python -m evals", description=__doc__)
parser.add_argument("--dataset", default=DEFAULT_DATASET, help="golden dataset YAML")
parser.add_argument("--output", default=DEFAULT_OUTPUT, help="results JSON path")
parser.add_argument("--judge-model", default=DEFAULT_JUDGE_MODEL, help="Groq model id for the judge")
args = parser.parse_args(argv)
api_key = os.getenv("GROQ_API_KEY")
if not api_key:
print("ERROR: GROQ_API_KEY is not set (needed for both the analyzer and the judge).", file=sys.stderr)
return 2
cases = load_dataset(args.dataset)
cases_by_id = {c["id"]: c for c in cases}
# The heart of the run
records = _diagnose_cases(cases) # Produce
_judge_records(records, cases_by_id, model=args.judge_model, api_key=api_key) # Grade
summary = _summarize(records) # Aggregate
scout_model = _make_scout().model
# Assemble report
report = {
"run_at": datetime.now(timezone.utc).isoformat(),
"dataset": args.dataset,
"scout_model": scout_model,
"judge_model": args.judge_model,
"summary": summary,
"cases": records,
}
# Build Report
out_path = Path(args.output)
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(json.dumps(report, indent=2))
_print_table(records, summary)
print(f"Report written to {out_path}")
return 0 if summary["pass_rates"]["overall"] >= 1.0 else 1
if __name__ == "__main__":
sys.exit(main())