Source code for tests.helpers.failure_analyzer

"""
AI-powered test failure analyzer using open-source LLMs via Groq.

Sends test failure context (error, test code, API response) to an LLM
and returns a structured diagnosis with root cause, suggested fix, and
failure classification.

.. module:: tests.helpers.failure_analyzer
   :synopsis: LLM-based test failure analysis.
   :no-index:
"""

import os
import json
from pathlib import Path
from loguru import logger

from telemetry import ledger

# Model options on Groq free tier:
#   "qwen/qwen3.8-27b"
#   "qwen/qwen3-32b"
DEFAULT_MODEL = "qwen/qwen3.8-27b"
SYSTEM_PROMPT = """
You are an expert API test failure analyst. When given a test failure context, analyze it and respond with a JSON object containing:
{
  "root_cause": "One sentence explaining why the test failed",
  "category": "one of: api_bug | test_bug | data_issue | timeout | auth_error | schema_mismatch | environment",
  "suggested_fix": "Specific actionable fix in 1-2 sentences",
  "explanation": "2-3 sentence detailed explanation of what went wrong",
  "evidence": ["list of observable facts from logs/errors that support the diagnosis, e.g. HTTP status code, response body snippets, error messages"],
  "confidence": "integer 0-100 — fill this in LAST, after root_cause, explanation, and evidence; calibrate per the rubric below"
}

Rules:
- api_bug: The API returned an unexpected response (status code, missing field, wrong value)
- test_bug: The test assertion or logic is incorrect
- data_issue: Test data is stale, invalid, or the resource was deleted
- timeout: Response time exceeded the threshold
- auth_error: Authentication/authorization failure (expired token, missing key)
- schema_mismatch: Response structure doesn't match the expected Pydantic model or contract
- environment: Configuration, connectivity, or environment setup issue
- Be concise and specific to the TMDB API domain
- Respond ONLY with valid JSON, no markdown fences

Confidence calibration (be conservative — when unsure, choose the LOWER band):
- 85-100: failure_context alone is conclusive — a status code, error message, or response body directly and unambiguously confirms the cause.
- 50-84: the cause is a strong inference from partial signals, but some step is not directly shown in failure_context.
- 0-49: the context is ambiguous or sparse, or the cause is inferred from indirect evidence.
- Confidence MUST be justified by the items in "evidence": if evidence is sparse or does not directly support root_cause, keep it low. Reserve >90 for a diagnosis you could defend line-by-line from failure_context.
"""

# Used by the agentic refine loop: the same output contract as SYSTEM_PROMPT,
# plus an instruction to address an independent reviewer's critique of a prior
# diagnosis. Confidence must reflect genuine evidential support, not the target.
REFINE_SYSTEM_PROMPT = SYSTEM_PROMPT + """
You are refining a previous diagnosis that an independent reviewer judged weak. Address the
reviewer's specific issues, ground every claim in the provided evidence, and make the
suggested_fix concrete. Return the same JSON object. Only raise confidence if the improved
diagnosis is genuinely better supported by the evidence — never inflate it to hit a target.
"""

[docs] class FailureAnalyzer: """ Analyzes test failures using an open-source LLM via Groq's free API. Disabled by default. Enable by setting AI_ANALYSIS_ENABLED=true and GROQ_API_KEY in the .env file. :param model: The LLM model identifier to use on Groq. """ def __init__(self, model: str = None): """ Initialize the failure analyzer. Reads configuration from environment variables at construction time. If ``AI_ANALYSIS_ENABLED`` is ``true`` but ``GROQ_API_KEY`` is missing, analysis is automatically disabled with a warning. :param model: LLM model identifier to use on Groq. Falls back to the ``AI_MODEL`` env var, then ``DEFAULT_MODEL``. """ self.enabled = os.getenv("AI_ANALYSIS_ENABLED", "false").lower() == "true" self.api_key = os.getenv("GROQ_API_KEY") self.model = model or os.getenv("AI_MODEL", DEFAULT_MODEL) self._client = None self._results = [] if self.enabled and not self.api_key: logger.warning("AI_ANALYSIS_ENABLED=true but GROQ_API_KEY is not set. Disabling AI analysis.") self.enabled = False @property def client(self): """ Lazy-initialize the Groq client on first access. Delays the ``from groq import Groq`` import until analysis is actually needed, avoiding import errors when the ``groq`` package is installed but never used. :returns: Configured ``groq.Groq`` client instance. """ if self._client is None: from groq import Groq self._client = Groq(api_key=self.api_key) return self._client @property def results(self) -> list[dict]: """All diagnoses accumulated this session (read-only view).""" return self._results
[docs] def analyze(self, failure_context: dict) -> dict | None: """ Send failure context to the LLM and return structured diagnosis. :param failure_context: Dictionary containing failure details: - test_name: Name of the failed test - test_file: Path to the test file - error_message: The assertion or exception message - traceback: Full traceback string - api_url: The API URL that was called (if available) - status_code: HTTP status code received (if available) - response_body: API response body snippet (if available) :returns: Parsed JSON diagnosis dict, or None if analysis is disabled/fails. Example return:: { "root_cause": "Movie id 0 is invalid, so TMDB returned 404.", "category": "data_issue", "suggested_fix": "Use a valid movie id from movie_ids.txt in the test data.", "confidence": 92, "explanation": "The request targeted /3/movie/0; TMDB rejects id 0 ...", "evidence": ["HTTP 404", "status_message: The resource you requested could not be found."], "test_name": "test_get_movie_details", "model": "qwen/qwen3.8-27b" } """ if not self.enabled: return None else: logger.info(f"AI_ANALYSIS_ENABLED={self.enabled} and GROQ_API_KEY {'set' if self.api_key else 'missing'}") prompt = self._build_prompt(failure_context) diagnosis = self._request_diagnosis(SYSTEM_PROMPT, prompt, failure_context.get("test_name", "unknown")) if diagnosis is None: return None self._results.append(diagnosis) logger.info(f"🤖 AI Analysis [{diagnosis.get('category', '?')}]: {diagnosis.get('root_cause', 'N/A')}") return diagnosis
def _request_diagnosis(self, system_prompt: str, user_prompt: str, test_name: str) -> dict | None: """ Run one LLM completion and return a parsed, normalized diagnosis dict. Shared by :meth:`analyze` (initial diagnosis) and :meth:`refine` (judge-guided improvement). Handles the Groq call, JSON parse, ``test_name``/``model`` stamping, and ``confidence`` int-coercion. Does **not** touch ``self._results`` — the caller decides whether to append (a new diagnosis) or replace (a refinement). :param system_prompt: System message controlling the JSON output contract. :param user_prompt: The assembled failure (or critique) prompt. :param test_name: Name of the failed test, stamped onto the result. :returns: Normalized diagnosis dict (same shape as :meth:`analyze`'s return), or None if the call/parse fails. """ from groq.types.chat import ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam messages = [ ChatCompletionSystemMessageParam(role="system", content=system_prompt), ChatCompletionUserMessageParam(role="user", content=user_prompt) ] try: response = self.client.chat.completions.create( model=self.model, messages=messages, temperature=0.1, # Low temperature for consistent analysis max_tokens=500, # qwen3.8 is a reasoning model: without this, thinking tokens exhaust # max_tokens before any JSON is emitted (empty generation → 400). # Diagnosis is a constrained JSON task; "none" skips thinking entirely. # Groq's qwen models accept only "none" | "default" here. reasoning_effort="none", response_format={"type": "json_object"} # noqa: type stubs missing in groq v1.x ) # Salvage the free usage block before parsing — the tokens were spent # even if the JSON turns out unparseable. REFINE_SYSTEM_PROMPT marks a # refine pass; anything else is the initial analyze. ledger.record( model=self.model, role="refine" if system_prompt is REFINE_SYSTEM_PROMPT else "analyze", usage=response.usage, test_name=test_name, ) raw = response.choices[0].message.content diagnosis = json.loads(raw) diagnosis["test_name"] = test_name diagnosis["model"] = self.model # some models return confidence as a string ("95"); # downstream tiering compares it numerically, so coerce here. try: diagnosis["confidence"] = int(diagnosis.get("confidence", 0)) except (TypeError, ValueError): diagnosis["confidence"] = 0 # Deterministic clamp — A high confidence backed by little evidence is the hallucination # signature the rubric warns against; enforce it deterministically so a # thin diagnosis can't self-certify (matters most on non-judged runs, # which have no independent critic to catch it). evidence = diagnosis.get("evidence") if not isinstance(evidence, list) or len(evidence) < 2: diagnosis["confidence"] = min(diagnosis["confidence"], 50) return diagnosis except Exception as e: logger.warning(f"AI analysis failed: {e}") return None # Alias used by refine() — same completion call, named for the refinement context. _request_refined_diagnosis = _request_diagnosis
[docs] def refine(self, failure_context: dict, prior_diag: dict, weaknesses: list[str]) -> dict | None: """ Produce an improved diagnosis given a critique of a prior one. Called by :func:`refine_until_confident` when the independent judge flags a diagnosis. Re-prompts the model with the original failure context, the prior diagnosis, and the judge's concrete ``weaknesses``, asking it to address each one and re-examine the evidence. The improved diagnosis **replaces** the prior one in ``self._results`` (matched by ``test_name``) so the saved report and terminal summary reflect the final version. :param failure_context: Same dict passed to :meth:`analyze`. :param prior_diag: The diagnosis being critiqued. :param weaknesses: Concrete weaknesses from the judge to address. :returns: Improved diagnosis dict (same shape as :meth:`analyze`'s return), or None if analysis is disabled or the call fails. """ if not self.enabled: return None prompt = self._build_refine_prompt(failure_context, prior_diag, weaknesses) diagnosis = self._request_refined_diagnosis(REFINE_SYSTEM_PROMPT, prompt, failure_context.get("test_name", "unknown")) if diagnosis is None: return None if self._results and self._results[-1].get("test_name") == diagnosis["test_name"]: self._results[-1] = diagnosis else: self._results.append(diagnosis) logger.info( f"🤖 AI Refine [{diagnosis.get('category', '?')}] " f"confidence {diagnosis.get('confidence', '?')}%: {diagnosis.get('root_cause', 'N/A')}" ) return diagnosis
@staticmethod def _build_prompt(ctx: dict) -> str: """ Build the user prompt string from failure context. Assembles a structured text prompt containing test name, file path, API URL, status code, response body (truncated to 2000 chars with head+tail strategy), error message, and the last 20 lines of the traceback. Only includes fields that are present in the context dict. :param ctx: Dictionary of failure details (same keys as ``analyze()``'s ``failure_context`` parameter). :returns: Formatted multi-line string ready to send to the LLM. """ parts = [f"Test: {ctx.get('test_name', 'unknown')}"] if ctx.get("test_file"): parts.append(f"File: {ctx['test_file']}") if ctx.get("api_url"): parts.append(f"API URL: {ctx['api_url']}") if ctx.get("status_code"): parts.append(f"Status Code: {ctx['status_code']}") if ctx.get("response_body"): # Truncate large responses body = str(ctx["response_body"]) if len(body) > 2000: # Capturing both start and end of the response body... body = body[:1000] + "\n...[truncated]...\n" + body[-500:] parts.append(f"Response Body: {body}") if ctx.get("error_message"): parts.append(f"Error: {ctx['error_message']}") if ctx.get("traceback"): # Last 20 lines of traceback tb_lines = ctx["traceback"].strip().split("\n")[-20:] parts.append(f"Traceback:\n{chr(10).join(tb_lines)}") return "\n".join(parts) @staticmethod def _build_refine_prompt(ctx: dict, prior_diag: dict, weaknesses: list[str]) -> str: """ Build the critique prompt for a refinement pass. Combines the original failure context with the prior diagnosis and the judge's concrete weaknesses, instructing the model to address each one, re-examine the evidence, and emit an improved diagnosis in the same JSON contract. The bookkeeping keys (``test_name``/``model``) are dropped from the echoed prior so the model focuses on the diagnosis content. :param ctx: The original failure context (see :meth:`analyze`). :param prior_diag: The diagnosis being critiqued. :param weaknesses: Concrete weaknesses raised by the judge (may be empty when only the confidence bar was missed). :returns: Formatted multi-line prompt string. """ weakness_lines = ( "\n".join(f"- {weakness}" for weakness in weaknesses) if weaknesses else "- Confidence is below the required bar; strengthen the evidence and reasoning." ) prior_json = json.dumps( {k: v for k, v in prior_diag.items() if k not in ("test_name", "model")}, indent=2 ) return ( f"{FailureAnalyzer._build_prompt(ctx)}\n\n" f"An independent reviewer judged your previous diagnosis and found these weaknesses:\n" f"{weakness_lines}\n\n" f"Previous diagnosis:\n{prior_json}\n\n" "Address every weakness above, re-examine the evidence, and return an improved " "diagnosis in the same JSON format." )
[docs] def save_results(self, output_dir: str = "ai_analysis"): """ Write all accumulated analysis results to a JSON file. Called once at the end of the test session by ``pytest_sessionfinish`` in ``conftest.py``. Skips writing if no failures were analyzed. Creates the output directory if it doesn't exist. :param output_dir: Directory path to write the results file to. Defaults to ``ai_analysis/`` relative to the project root. Output file: ``<output_dir>/failure_analysis.json`` """ if not self._results: return out_path = Path(output_dir) out_path.mkdir(parents=True, exist_ok=True) report_file = out_path / "failure_analysis.json" report_file.write_text(json.dumps(self._results, indent=2)) logger.info(f"📊 AI analysis report saved: {report_file} ({len(self._results)} failures analyzed)")
def _collect_issues(judgement: dict) -> list[str]: """ Gather concrete issues from a judgement's failed dimensions. Pulls the ``issues`` list from each of groundedness/completeness/ actionability whose verdict is ``fail``, prefixing each with its dimension so the refine prompt knows what to fix. :param judgement: A ``judge_live`` result dict. :returns: Flat list of ``"<dimension>: <issue>"`` strings (possibly empty), e.g.:: [ "groundedness: Speculative causes in explanation not found in failure_context", "completeness: Diagnosis omits the HTTP status code from the response", ] """ collected = [] for dim in ("groundedness", "completeness", "actionability"): section = judgement.get(dim) if isinstance(section, dict) and section.get("verdict") == "fail": collected.extend(f"{dim}: {issue}" for issue in section.get("issues", [])) return collected
[docs] def refine_until_confident( analyzer_instance, failure_context, diagnosis, judge_fn, *, max_iterations, confidence_target ): """ Iteratively improve a diagnosis using an independent judge as the critic. Agentic loop: judge the current diagnosis; if the judge passes **and** the self-reported ``confidence`` clears ``confidence_target``, stop. Otherwise feed the judge's failed-dimension issues back into :meth:`FailureAnalyzer.refine` and re-judge, up to ``max_iterations`` times. The judge is injected (``judge_fn``) so this module stays decoupled from ``evals.judge`` and the loop is unit-testable. A ``None`` judgement (judge API error) stops the loop — without a critic signal there is nothing to refine against. :param analyzer_instance: The :class:`FailureAnalyzer` that produced ``diagnosis``. :param failure_context: The failure context dict. :param diagnosis: The initial diagnosis from :meth:`FailureAnalyzer.analyze`. :param judge_fn: Callable ``(failure_context, diagnosis) -> judgement | None``. :param max_iterations: Maximum number of refine passes. :param confidence_target: Confidence (0–100) the diagnosis must reach. :returns: Tuple of the final ``(diagnosis, judgement)`` — the diagnosis has the shape shown in :meth:`FailureAnalyzer.analyze`, the judgement is the ``judge_fn`` output (or ``None`` if the judge errored). Example return:: ( {"root_cause": "...", "category": "data_issue", "confidence": 92, ...}, { "groundedness": {"verdict": "pass", "reasoning": "...", "issues": []}, "completeness": {"verdict": "pass", "reasoning": "...", "issues": []}, "actionability": {"verdict": "pass", "reasoning": "...", "issues": []}, "overall_verdict": "pass", }, ) """ # In production, the real judge, in conftest.py (~line 405) gets passed. judge_fn is a thin lambda wrapping # evals.judge.judge_live — It pre-binds the model/api_key args. judgement = judge_fn(failure_context, diagnosis) # judge the current diagnosis for _ in range(max_iterations): if judgement is None: break passing = judgement.get("overall_verdict") == "pass" confident = diagnosis.get("confidence", 0) >= confidence_target if passing and confident: break refined = analyzer_instance.refine(failure_context, diagnosis, _collect_issues(judgement)) if refined is None: break diagnosis = refined judgement = judge_fn(failure_context, diagnosis) # re-judge the refined diagnosis return diagnosis, judgement
# Singleton instance — shared across all tests in a session analyzer = FailureAnalyzer()