Source code for failure_mcp.tools.analyze_test_failure

"""
MCP tool definitions for AI-powered test failure analysis.

Exposes three tools:

- ``analyze_failure`` — send a failure context to the LLM and get a structured diagnosis.
- ``get_results``     — retrieve accumulated diagnosis results (with optional confidence filter).
- ``save_results``    — flush all results to ``<output_dir>/failure_analysis.json``.

This module is imported by ``failure_mcp/server.py``, which registers the handlers
on the :class:`mcp.server.Server` instance.

Tool Schemas
------------
Each entry in :data:`TOOLS` is a :class:`mcp.types.Tool` that carries:

- ``name``        — identifier used by the MCP client when calling the tool.
- ``description`` — human-readable summary shown in the client UI.
- ``inputSchema`` — JSON Schema object describing required and optional arguments.

.. module:: failure_mcp.tools.analyze_test_failure
   :synopsis: MCP tool definitions wrapping FailureAnalyzer.
   :no-index:
"""

import json
from typing import Any

from mcp import types

# Import the singleton — failure_analyzer.py is NOT modified. The whole point is to expose the object's methods without
# modifying it.
from tests.helpers.failure_analyzer import analyzer


# ---------------------------------------------------------------------------
# Tool schemas
# ---------------------------------------------------------------------------

#: List of :class:`mcp.types.Tool` objects advertised to MCP clients.
#:
#: Contains three tools:
#:
#: ``analyze_failure``
#:     Required args: ``test_name``, ``error_message``.
#:     Optional args: ``test_file``, ``traceback``, ``api_url``, ``status_code``,
#:     ``response_body``.
#:
#: ``get_results``
#:     Optional arg: ``min_confidence`` (int 0–100, default 0).
#:
#: ``save_results``
#:     Optional arg: ``output_dir`` (str, default ``"ai_analysis"``).
TOOLS: list[types.Tool] = [
    types.Tool(
        name="analyze_failure",
        description=(
            "Send a test failure context to the LLM and receive a structured "
            "diagnosis with root_cause, category, suggested_fix, confidence (0-100 %), "
            "explanation, evidence, and a human-readable confidence_tier."
        ),
        inputSchema={
            "type": "object",
            "properties": {
                "test_name": {
                    "type": "string",
                    "description": "Name of the failed test function."
                },
                "test_file": {
                    "type": "string",
                    "description": "Relative path to the test file."
                },
                "error_message": {
                    "type": "string",
                    "description": "The assertion or exception message."
                },
                "traceback": {
                    "type": "string",
                    "description": "Full traceback string."
                },
                "api_url": {
                    "type": "string",
                    "description": "The API URL that was called (optional)."
                },
                "status_code": {
                    "type": "integer",
                    "description": "HTTP status code received (optional)."
                },
                "response_body": {
                    "type": "string",
                    "description": "API response body or snippet (optional)."
                }
            },
            "required": ["test_name", "error_message"]
        }
    ),
    types.Tool(
        name="get_results",
        description=(
            "Return all diagnosis results accumulated during this MCP server "
            "session as a JSON array. Optionally filter by minimum confidence."
        ),
        inputSchema={
            "type": "object",
            "properties": {
                "min_confidence": {
                    "type": "integer",
                    "description": "Only return results with confidence >= this value (0–100). Defaults to 0.",
                    "default": 0
                }
            },
            "required": []
        }
    ),
    types.Tool(
        name="save_results",
        description=(
            "Flush all accumulated diagnosis results to "
            "<output_dir>/failure_analysis.json. "
            "Returns the number of results written and the output path."
        ),
        inputSchema={
            "type": "object",
            "properties": {
                "output_dir": {
                    "type": "string",
                    "description": "Directory to write failure_analysis.json. Defaults to 'ai_analysis'.",
                    "default": "ai_analysis"
                }
            },
            "required": []
        }
    )
]


# ---------------------------------------------------------------------------
# Call dispatcher
# ---------------------------------------------------------------------------

[docs] def _confidence_tier(score: int) -> str: """A pure helper def: Map a 0–100 confidence score the LLM returns to a human-readable tier label. Used to attach a ``confidence_tier`` field alongside the raw numeric ``confidence`` value in the ``analyze_failure`` response, so MCP clients can display or filter results without implementing their own thresholds. This will enrich the diagnosis before returning it to the client without modifying :py:class:`tests.helpers.failure_analyzer.FailureAnalyzer`. Thresholds: +----------+-----------+ | Range | Tier | +==========+===========+ | >= 80 | ``high`` | +----------+-----------+ | 50 – 79 | ``medium``| +----------+-----------+ | < 50 | ``low`` | +----------+-----------+ :param score: Integer confidence score (0–100) returned by the LLM. :returns: One of ``"high"``, ``"medium"``, or ``"low"``. Example:: >>> _confidence_tier(92) 'high' >>> _confidence_tier(65) 'medium' >>> _confidence_tier(30) 'low' """ if score >= 80: return "high" if score >= 50: return "medium" return "low"
[docs] async def handle_call(name: str, arguments: dict[str, Any]) -> list[types.TextContent]: """Dispatch an MCP tool call to the appropriate :class:`FailureAnalyzer` method. Called by ``failure_mcp/server.py``'s ``call_tool`` hook for every tool invocation arriving from the MCP client. Delegates to the :data:`~tests.helpers.failure_analyzer.analyzer` singleton without modifying ``failure_analyzer.py``. **Tool behavior:** ``analyze_failure`` Calls :meth:`~tests.helpers.failure_analyzer.FailureAnalyzer.analyze` with the supplied ``arguments`` dict as the ``failure_context``. On success, enriches the diagnosis with a ``confidence_tier`` field. On failure or when analysis is disabled, returns an ``error`` and ``hint`` dict instead. ``get_results`` Reads :attr:`~tests.helpers.failure_analyzer.FailureAnalyzer._results` and filters entries whose ``confidence`` is >= ``min_confidence`` (defaults to ``0``, i.e. return everything). ``save_results`` Calls :meth:`~tests.helpers.failure_analyzer.FailureAnalyzer.save_results` with the given ``output_dir`` (defaults to ``"ai_analysis"``), then returns the count of results written and the output file path. :param name: Tool name requested by the MCP client. Must be one of ``"analyze_failure"``, ``"get_results"``, or ``"save_results"``. :param arguments: Key/value arguments supplied by the MCP client, validated against the JSON Schema in :data:`TOOLS`. :returns: A single-element list of :class:`mcp.types.TextContent` whose ``text`` field contains the JSON-serialized result (2-space indent). :raises ValueError: If ``name`` does not match any registered tool. Example response for ``analyze_failure``:: { "root_cause": "...", "category": "auth_error", "suggested_fix": "...", "confidence": 92, "explanation": "...", "evidence": ["HTTP 401", "Response: Invalid API key"], "test_name": "test_get_movie_details", "model": "qwen/qwen3.8-27b", "confidence_tier": "high" } When analysis is disabled or fails, ``analyze_failure`` instead returns:: {"error": "Analysis disabled or failed.", "hint": "Set AI_ANALYSIS_ENABLED=true and GROQ_API_KEY in the environment."} ``get_results`` returns a JSON array of the stored diagnoses (without the ``confidence_tier`` field, which only ``analyze_failure`` adds):: [ { "root_cause": "...", "category": "auth_error", "suggested_fix": "...", "confidence": 92, "explanation": "...", "evidence": ["HTTP 401"], "test_name": "test_get_movie_details", "model": "qwen/qwen3.8-27b" } ] ``save_results`` returns the count written and the output path:: {"saved": 3, "path": "ai_analysis/failure_analysis.json"} """ if name == "analyze_failure": diagnosis = analyzer.analyze(failure_context=arguments) # When `self.enabled == False`, return a friendly error if diagnosis is None: result = { "error": "Analysis disabled or failed.", "hint": "Set AI_ANALYSIS_ENABLED=true and GROQ_API_KEY in the environment." } else: confidence = diagnosis.get("confidence", 0) result = {**diagnosis, "confidence_tier": _confidence_tier(confidence)} elif name == "get_results": min_confidence = int(arguments.get("min_confidence", 0)) result = [ r for r in analyzer._results if r.get("confidence", 0) >= min_confidence ] elif name == "save_results": output_dir = arguments.get("output_dir", "ai_analysis") count = len(analyzer._results) analyzer.save_results(output_dir=output_dir) result = {"saved": count, "path": f"{output_dir}/failure_analysis.json"} else: raise ValueError(f"Unknown tool: {name}") # A human-readable output in the client UI. return [types.TextContent(type="text", text=json.dumps(result, indent=2))]