"""
Pytest configuration and shared fixtures for the TMDB API test suite.
This module is automatically loaded by pytest before test collection. It provides:
- **CLI options**: ``--loguru-log-level``, ``--log-to-file``, ``--failure-analysis``, ``--judge-diagnosis``
- **Hooks**: Logging setup, AI failure analysis, end-of-run AI diagnosis console summary, Allure/HTML report customization
- **Fixtures**: API client factory, Pydantic schema loader, test data loader
.. module:: tests.conftest
:synopsis: Pytest fixtures, hooks, and CLI options.
:no-index:
"""
import json
import os
import pytest
from pathlib import Path
from loguru import logger
from datetime import datetime
from zoneinfo import ZoneInfo
from tests.helpers.failure_analyzer import analyzer, refine_until_confident
# Registering logger configuration options and setup before any test runs
[docs]
def pytest_addoption(parser):
"""
Register custom command-line options for pytest.
Adds --loguru-log-level option to control application logging verbosity
during test runs. This hook runs during pytest's startup phase before
any tests are collected.
:param parser: Pytest's argument parser instance.
Usage:
poetry run pytest tests/* --loguru-log-level=DEBUG
poetry run pytest tests/* --loguru-log-level=WARNING
poetry run pytest tests/* --log-to-file
Available log levels:
DEBUG, INFO (default), WARNING, ERROR, CRITICAL
"""
parser.addoption(
"--loguru-log-level",
default="INFO",
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
help="Set log level for tests"
)
parser.addoption(
"--log-to-file",
action="store_true",
default=False,
help="Enable logging to file (default: False)"
)
parser.addoption(
"--failure-analysis",
action="store_true",
default=False,
help="Enable AI-powered test failure analysis (requires GROQ_API_KEY)"
)
parser.addoption(
"--judge-diagnosis",
action="store_true",
default=False,
help="Also judge each diagnosis's quality (groundedness + completeness + actionability) "
"with the LLM judge. Requires --failure-analysis and GROQ_API_KEY; "
"adds one judge LLM call per failed test. CLI equivalent of AI_JUDGE_ENABLED=true."
)
# Set LOG_LEVEL to environment variable before any test modules are imported, ensuring loguru is configured correctly
from tests.data.data_loader import load_test_data
from api.movies_api import MoviesAPI
from api.people_api import PeopleAPI
from api.lists_api import ListsAPI
from api.search_api import SearchAPI
from api.discover_api import DiscoverAPI
from api.networks_api import NetworksAPI
from api.trending_api import TrendingAPI
from api.reviews_api import ReviewsAPI
@pytest.fixture(autouse=True)
def _store_test_name(request):
"""
Auto-use fixture that injects the current test name into class-based test instances.
Runs automatically before every test. For class-based tests, it sets
``_test_name`` on the test instance so helper methods (e.g., FieldAssertions)
can include the test name in assertion messages and log output.
Function-based tests are silently skipped (no instance to attach to).
:param request: Pytest's FixtureRequest object providing test context.
Example:
class TestMovies(FieldAssertions):
def test_get_movie(self, movies_api):
# self._test_name is automatically set to "test_get_movie"
...
"""
if request.instance is not None:
request.instance._test_name = request.node.name
[docs]
@pytest.fixture
def movies_api() -> MoviesAPI:
"""
Provide a fresh ``MoviesAPI`` client for a test.
Dedicated, type-annotated alternative to ``get_api_instance('movies_api')``
— the dependency is declared in the test signature, typo-checked at
collection time, and gives editor autocomplete for movie endpoints.
:return: A configured ``MoviesAPI`` instance.
"""
return MoviesAPI()
[docs]
@pytest.fixture
def lists_api() -> ListsAPI:
"""
Provide a fresh ``ListsAPI`` client for a test.
Dedicated, type-annotated alternative to ``get_api_instance('lists_api')``
— the dependency is declared in the test signature, typo-checked at
collection time, and gives editor autocomplete for list endpoints.
:return: A configured ``ListsAPI`` instance.
"""
return ListsAPI()
[docs]
@pytest.fixture
def people_api() -> PeopleAPI:
"""
Provide a fresh ``PeopleAPI`` client for a test.
Dedicated, type-annotated alternative to ``get_api_instance('people_api')``
— the dependency is declared in the test signature, typo-checked at
collection time, and gives editor autocomplete for people endpoints.
:return: A configured ``PeopleAPI`` instance.
"""
return PeopleAPI()
[docs]
@pytest.fixture
def search_api() -> SearchAPI:
"""
Provide a fresh ``SearchAPI`` client for a test.
Dedicated, type-annotated alternative to ``get_api_instance('search_api')``
— the dependency is declared in the test signature, typo-checked at
collection time, and gives editor autocomplete for search endpoints.
:return: A configured ``SearchAPI`` instance.
"""
return SearchAPI()
[docs]
@pytest.fixture
def discover_api() -> DiscoverAPI:
"""
Provide a fresh ``DiscoverAPI`` client for a test.
Dedicated, type-annotated alternative to ``get_api_instance('discover_api')``
— the dependency is declared in the test signature, typo-checked at
collection time, and gives editor autocomplete for discover endpoints.
:return: A configured ``DiscoverAPI`` instance.
"""
return DiscoverAPI()
[docs]
@pytest.fixture
def networks_api() -> NetworksAPI:
"""
Provide a fresh ``NetworksAPI`` client for a test.
Dedicated, type-annotated alternative to ``get_api_instance('networks_api')``
— the dependency is declared in the test signature, typo-checked at
collection time, and gives editor autocomplete for network endpoints.
:return: A configured ``NetworksAPI`` instance.
"""
return NetworksAPI()
[docs]
@pytest.fixture
def trending_api() -> TrendingAPI:
"""
Provide a fresh ``TrendingAPI`` client for a test.
Dedicated, type-annotated alternative to ``get_api_instance('trending_api')``
— the dependency is declared in the test signature, typo-checked at
collection time, and gives editor autocomplete for trending endpoints.
:return: A configured ``TrendingAPI`` instance.
"""
return TrendingAPI()
[docs]
@pytest.fixture
def reviews_api() -> ReviewsAPI:
"""
Provide a fresh ``ReviewsAPI`` client for a test.
Dedicated, type-annotated alternative to ``get_api_instance('reviews_api')``
— the dependency is declared in the test signature, typo-checked at
collection time, and gives editor autocomplete for review endpoints.
:return: A configured ``ReviewsAPI`` instance.
"""
return ReviewsAPI()
from tests.schemas.models import (
GenericResponse, RatingResponse, MovieDetails,
PopularMoviesResponse, PersonDetails, SearchMoviesResponse,
DiscoverMoviesResponse, NetworkDetails, TrendingMoviesResponse,
ReviewDetails, MovieReviewsResponse
)
[docs]
@pytest.fixture
def load_schema():
"""
Fixture that provides a Pydantic model loader for response validation.
:return: A callable ``_load(name)`` that maps a registered schema name to
its Pydantic model class, raising ``ValueError`` for an unknown name.
Example use::
model = load_schema("movie_schema") # -> <class 'MovieDetails'>
model.model_validate(response.data)
"""
schema_map = {
'generic_schema': GenericResponse,
'add_delete_rating_schema': RatingResponse,
'movie_schema': MovieDetails,
'popular_movies_schema': PopularMoviesResponse,
'person_details_schema': PersonDetails,
'search_movies_schema': SearchMoviesResponse,
'discover_movies_schema': DiscoverMoviesResponse,
'network_details_schema': NetworkDetails,
'trending_movies_schema': TrendingMoviesResponse,
'review_details_schema': ReviewDetails,
'movie_reviews_schema': MovieReviewsResponse,
}
def _load(name):
model = schema_map.get(name)
if model is None:
raise ValueError(f"Unknown schema: {name}")
return model
return _load
[docs]
@pytest.fixture(scope="session")
def movies_test_data():
"""
Session-scoped fixture that loads movies API test data from YAML.
Loads test data once per test session and shares it across all tests,
improving performance by avoiding repeated file reads.
Note: This fixture is currently unused. The test module uses a module-level
constant with @pytest.mark.parametrize instead, which is required because
parametrize decorators are evaluated at collection time before fixtures
are available.
:return: Dictionary containing test data with defaults applied.
"""
return load_test_data("test_data.yaml")
[docs]
def pytest_html_report_title(report):
"""
Customize the title shown in the pytest-html report header.
:param report: The pytest-html report object.
"""
report.title = "Movies API Tests Report"
[docs]
def pytest_html_results_summary(prefix, summary, postfix):
"""
Customize the results summary section of the pytest-html report.
Adds a branded header with run timestamp (Eastern Time), a description
of the test scope, and links to external documentation. Called once
by pytest-html after all tests have finished.
:param prefix: List of HTML strings rendered above the pass/fail counts.
:param summary: List of HTML strings rendered alongside the counts.
:param postfix: List of HTML strings rendered below the counts.
"""
# Get current Eastern time
et_time = datetime.now(ZoneInfo("America/New_York"))
# Quick status header
prefix.extend([
"<h2>🎬 TMDB API Automation Suite</h2>",
f"<p>Run Date: {et_time.strftime('%Y-%m-%d %H:%M:%S')}</p>"
])
# Env callout
# Note: Passed/Failed counts are automatically added here by pytest-html
summary.extend([
"<h3>Report Summary</h3>",
"<p>Validation of critical TMDB endpoints including Movies, TV, and Search services.</p>"
])
# Links to doc, logs
postfix.extend([
"<hr>",
"<div>",
" <p>For detailed API specs, visit the "
" <a href='https://developer.themoviedb.org'>TMDB Documentation</a>"
" </p>",
" <p style='color: #666;'>Failure logs are archived in the /logs workspace directory.</p>",
"</div>"
])
# Judge verdicts accumulated this session, so pytest_sessionfinish can roll up a
# run-level judge pass-rate alongside the token/cost totals. Populated only on
# judged runs (AI_JUDGE_ENABLED); empty otherwise.
_judgements: list[dict] = []
def _current_git_sha() -> str:
"""Short commit SHA for the trend row: CI env first, else local git, else ''."""
sha = os.environ.get("GITHUB_SHA")
if sha:
return sha[:12]
try:
import subprocess
return subprocess.check_output(
["git", "rev-parse", "--short", "HEAD"], text=True, stderr=subprocess.DEVNULL
).strip()
except Exception:
return ""
[docs]
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""
Capture test failure details and send to AI analyzer.
Runs after each test phase (setup/call/teardown). On failure during
the 'call' phase, extracts context from the test and API response,
then sends it to the LLM for diagnosis. When judging is enabled
(--judge-diagnosis or AI_JUDGE_ENABLED=true), the diagnosis is scored by an
LLM judge for groundedness, completeness, and actionability (correctness is
not scored live — a failure has no reference).
The diagnosis (and, when judged, the quality verdict) are attached to the
Allure report as JSON attachments.
"""
outcome = yield
report = outcome.get_result()
if report.when != "call" or not report.failed:
return
# Build failure context from available information
failure_context = {
"test_name": item.name,
"test_file": str(item.fspath),
"error_message": str(report.longrepr).split("\n")[-1] if report.longrepr else "",
"traceback": str(report.longrepr) if report.longrepr else "",
}
# Try to extract API response details from the test's local variables
# (pytest stores them in the call excinfo)
if call.excinfo:
# Check if the test had an APIResponse in its locals
tb = call.excinfo.traceback[-1]
locals_dict = tb.locals if hasattr(tb, 'locals') else {}
if 'response' in locals_dict:
resp = locals_dict['response']
if hasattr(resp, 'url'):
failure_context["api_url"] = str(resp.url)
if hasattr(resp, 'status_code'):
failure_context["status_code"] = resp.status_code
if hasattr(resp, 'data'):
failure_context["response_body"] = resp.data
diagnosis = analyzer.analyze(failure_context)
if not diagnosis:
return
# When judging is enabled (--judge-diagnosis sets AI_JUDGE_ENABLED=true in
# pytest_configure; Docker/K8s/CI set it via .env), run an agentic refine
# loop: an independent judge critiques the diagnosis and the analyzer
# improves it until the judge passes and confidence clears the target, or
# AI_REFINE_MAX_ITERS is hit. A normal triage run skips this entirely, so it
# pays for the diagnosis but not the extra judge/refine calls.
judgement = None
if os.getenv("AI_JUDGE_ENABLED", "false").lower() == "true":
from evals.judge import DEFAULT_JUDGE_MODEL, judge_live
judge_model = os.getenv("AI_JUDGE_MODEL", DEFAULT_JUDGE_MODEL)
api_key = os.getenv("GROQ_API_KEY")
diagnosis, judgement = refine_until_confident(
analyzer,
failure_context,
diagnosis,
judge_fn=lambda ctx, diag: judge_live(ctx, diag, model=judge_model, api_key=api_key),
max_iterations=int(os.getenv("AI_REFINE_MAX_ITERS", "2")),
confidence_target=int(os.getenv("AI_REFINE_CONFIDENCE_TARGET", "90")),
)
# Log + attach the FINAL diagnosis (after any refinement).
confidence = diagnosis.get("confidence", 0)
if confidence >= 80:
logger.info(f"🤖 High confidence [{confidence}%] - {diagnosis['root_cause']}")
elif confidence >= 50:
logger.warning(f"🤖 Medium confidence [{confidence}%] - {diagnosis['root_cause']}")
else:
logger.debug(f"🤖 Low confidence [{confidence}%] - treat with caution: {diagnosis['root_cause']}")
try:
import allure
allure.attach(
json.dumps(diagnosis, indent=2),
name="🤖 AI Failure Analysis",
attachment_type=allure.attachment_type.JSON
)
except Exception:
pass # Allure not available
# Attach the final judge verdict (groundedness + completeness + actionability,
# everything except correctness — a live failure has no reference answer).
if judgement:
_judgements.append(judgement) # for the run-level judge pass-rate at sessionfinish
logger.info(
f"⚖️ Diagnosis quality [{judgement['overall_verdict']}] — "
f"groundedness: {judgement['groundedness']['verdict']}, "
f"completeness: {judgement['completeness']['verdict']}, "
f"actionability: {judgement['actionability']['verdict']}"
)
try:
import allure
allure.attach(
json.dumps(judgement, indent=2),
name="⚖️ Diagnosis Quality (LLM-as-a-judge)",
attachment_type=allure.attachment_type.JSON
)
except Exception:
pass # Allure not available
[docs]
def pytest_terminal_summary(terminalreporter, exitstatus, config):
"""
Print all accumulated AI diagnoses as a console section at the end of the run.
Renders after the built-in summary sections so a terminal-only run gets the
same triage information as the reports. Keyed off the analyzer's accumulated
results rather than the --failure-analysis flag, so it also covers analysis
enabled via AI_ANALYSIS_ENABLED=true in .env; no failures analyzed = no
section. This is a triage summary (root cause + fix) — the full diagnosis
(explanation, evidence) stays in the Allure attachment and
ai_analysis/failure_analysis.json.
:param terminalreporter: Pytest's ``TerminalReporter`` to write the section to.
:param exitstatus: Integer exit code of the test session (unused).
:param config: Pytest's ``Config`` object (unused).
"""
if not analyzer.results:
return
terminalreporter.section(f"🤖 AI Failure Analysis ({len(analyzer.results)} failures)")
for diagnosis in analyzer.results:
confidence = diagnosis.get("confidence", 0)
markup = {"green": True} if confidence >= 80 else {"yellow": True} if confidence >= 50 else {"red": True}
terminalreporter.write_line(
f"{diagnosis.get('test_name', 'unknown')} "
f"[{diagnosis.get('category', '?')}, confidence {confidence}%]",
bold=True, **markup,
)
terminalreporter.write_line(f" root cause: {diagnosis.get('root_cause', 'N/A')}")
terminalreporter.write_line(f" suggested fix: {diagnosis.get('suggested_fix', 'N/A')}")
terminalreporter.write_line("")
# One-line token/cost recap; the full run-level row is appended to the JSONL
# in pytest_sessionfinish (see below).
from telemetry import ledger
if ledger.records:
s = ledger.summary()
terminalreporter.write_line(
f"💸 LLM usage: {s['calls']} calls, {s['total_tokens']:,} tokens "
f"(~${s['est_cost_usd']:.4f}) → ai_analysis/token_usage.jsonl"
)
[docs]
def pytest_sessionfinish(session, exitstatus):
"""
Perform cleanup and metadata writing after all tests complete.
Fires once at the end of the test session. Handles two tasks:
- **Allure metadata** (if ``--alluredir`` was passed): copies
``tests/allure/categories.json`` into the allure results dir so Allure
can classify failures, and writes ``environment.properties`` with
runtime config values (base URL, API version, timeout, log level)
for the Allure Environment widget, plus GitHub run context
(event, branch, commit, actor, run number) when running in CI.
- **AI analysis report**: calls ``analyzer.save_results()`` to write all
accumulated LLM diagnoses to ``ai_analysis/failure_analysis.json``.
:param session: The pytest ``Session`` object.
:param exitstatus: Integer exit code (0 = all passed, 1 = failures, etc.).
"""
allure_dir = session.config.getoption("--alluredir", default=None)
if allure_dir:
# Copy categories.json
import shutil
allure_dir = Path(allure_dir)
categories_file = Path(__file__).parent / "allure" / "categories.json"
if categories_file.exists():
shutil.copy(str(categories_file), str(allure_dir / "categories.json"))
# Write environment.properties with actual runtime config values
from config.config import Config
properties = (
f"Base.URL={Config.BASE_URL}\n"
f"API.Version={Config.API_VERSION}\n"
f"Timeout={Config.TIMEOUT}\n"
f"Log.Level={os.environ.get('LOG_LEVEL', 'INFO')}\n"
f"Framework=pytest\n"
)
# GitHub run context, passed into the container via the CI-generated
# .env (see .github/workflows/tmdb_test.yml). Absent on local runs.
if os.environ.get("GITHUB_EVENT_NAME"):
properties += (
f"CI.Event={os.environ['GITHUB_EVENT_NAME']}\n"
f"CI.Branch={os.environ.get('GITHUB_REF_NAME', '')}\n"
f"CI.Commit={os.environ.get('GITHUB_SHA', '')[:12]}\n"
f"CI.Actor={os.environ.get('GITHUB_ACTOR', '')}\n"
f"CI.Run.Number={os.environ.get('GITHUB_RUN_NUMBER', '')}\n"
)
(allure_dir / "environment.properties").write_text(properties)
analyzer.save_results()
# Token/cost telemetry: append ONE run-level row to ai_analysis/token_usage.jsonl
# (gitignored) so cost and quality can be trended across runs together.
# Local-only for now: CI runners are ephemeral, so persisting this history in CI is a separate follow-up.
from telemetry import ledger
if ledger.records:
from evals.judge import DEFAULT_JUDGE_MODEL
judged = os.getenv("AI_JUDGE_ENABLED", "false").lower() == "true"
confidences = [d["confidence"] for d in analyzer.results if isinstance(d.get("confidence"), int)]
passes = sum(1 for j in _judgements if j.get("overall_verdict") == "pass")
row = {
"ts": datetime.now(ZoneInfo("UTC")).isoformat(),
"git_sha": _current_git_sha(),
"analyzer_model": analyzer.model,
"judge_model": os.getenv("AI_JUDGE_MODEL", DEFAULT_JUDGE_MODEL) if judged else None,
"judged": judged,
"n_failures": len(analyzer.results),
**ledger.summary(),
"judge_pass_rate": round(passes / len(_judgements), 3) if _judgements else None,
"mean_confidence": round(sum(confidences) / len(confidences)) if confidences else None,
}
out_dir = Path("ai_analysis")
out_dir.mkdir(parents=True, exist_ok=True)
with (out_dir / "token_usage.jsonl").open("a") as f:
f.write(json.dumps(row) + "\n")
logger.info(f"💸 Token/cost trend row appended: {out_dir / 'token_usage.jsonl'}")