Source code for telemetry

"""
Process-wide token/cost ledger for the AI failure-analysis pipeline.

Every Groq response carries a ``usage`` block (prompt/completion/total tokens)
that the analyzer and judge would otherwise discard. Both call sites record it
here via :meth:`TokenLedger.record`; ``conftest.py`` drains the ledger at
session end, rolls it up into one run-level row (tokens + estimated cost +
quality), and appends it to ``ai_analysis/token_usage.jsonl`` for cross-run
trending.

Tokens are the durable, provider-independent signal; the dollar figure is a
derived estimate from a small, dated price table and is best-effort only — a
model missing from the table contributes 0 to the cost estimate but its tokens
are still counted. Recording is defensive: a malformed ``usage`` object can
never raise into a test run.

.. module:: telemetry
   :synopsis: Token/cost ledger for the LLM failure-analysis pipeline.
   :no-index:
"""

from dataclasses import dataclass

# USD per 1,000,000 tokens, keyed by Groq model id, as ``(input, output)``.
# Dated table (2026-09, Groq published pricing) — prices drift, so this is the
# one place to update. A model absent here is still token-counted; it just
# contributes 0 to est_cost_usd.
_PRICE_PER_MILLION: dict[str, tuple[float, float]] = {
    "qwen/qwen3.8-27b": (0.80, 4.00),
    "qwen/qwen3.6-27b": (0.60, 3.00),  # deprecated on Groq; kept for pinned AI_MODEL runs
    "llama-3.3-70b-versatile": (0.59, 0.79),  # deprecated on Groq; kept for pinned AI_MODEL runs
    "openai/gpt-oss-120b": (0.15, 0.75),
}


@dataclass
class _Record:
    """One recorded LLM call's token usage."""

    model: str
    role: str  # analyze | refine | judge
    prompt_tokens: int
    completion_tokens: int
    test_name: str


[docs] class TokenLedger: """ Accumulates per-call token usage for one pytest session. A single process-wide instance (:data:`ledger`) is shared by the analyzer and the judge; ``conftest.py`` reads it once at session end. Not thread-safe — the pipeline runs calls serially per test. """ def __init__(self): self._records: list[_Record] = []
[docs] def record(self, *, model: str, role: str, usage, test_name: str = "unknown") -> None: """ Record one LLM call's token usage from a Groq ``response.usage`` object. Defensive by design: a ``None`` or malformed ``usage`` counts as zero tokens rather than raising, so telemetry can never break a test run. :param model: The Groq model id that served the call. :param role: Which pipeline step spent the tokens (``analyze`` | ``refine`` | ``judge``). :param usage: The Groq ``CompletionUsage`` object (``response.usage``); ``prompt_tokens`` / ``completion_tokens`` are read off it. :param test_name: The failed test this call was diagnosing. """ try: prompt = int(getattr(usage, "prompt_tokens", 0) or 0) completion = int(getattr(usage, "completion_tokens", 0) or 0) except (TypeError, ValueError): prompt = completion = 0 self._records.append(_Record(model, role, prompt, completion, test_name))
@property def records(self) -> list[_Record]: """All calls recorded this session (read-only view).""" return self._records
[docs] def reset(self) -> None: """Drop all recorded calls (used by tests to isolate runs).""" self._records.clear()
[docs] def estimate_cost_usd(self) -> float: """ Best-effort dollar estimate for all recorded calls. ``cost = Σ (prompt_tokens × in_price + completion_tokens × out_price)`` from :data:`_PRICE_PER_MILLION`. Calls whose model is not in the price table contribute 0. :returns: Estimated cost in USD, rounded to 6 decimals. Example return for 1000 prompt + 200 completion tokens on qwen/qwen3.8-27b (``1000/1e6 * 0.80 + 200/1e6 * 4.00``):: 0.0016 """ total = 0.0 for r in self._records: price = _PRICE_PER_MILLION.get(r.model) if not price: continue in_price, out_price = price total += r.prompt_tokens / 1e6 * in_price + r.completion_tokens / 1e6 * out_price return round(total, 6)
[docs] def summary(self) -> dict: """ Roll the recorded calls up into a run-level totals dict. :returns: ``{calls, prompt_tokens, completion_tokens, total_tokens, est_cost_usd}`` — the token/cost half of the JSONL trend row. Example:: {"calls": 5, "prompt_tokens": 8421, "completion_tokens": 1290, "total_tokens": 9711, "est_cost_usd": 0.0021} """ prompt = sum(r.prompt_tokens for r in self._records) completion = sum(r.completion_tokens for r in self._records) return { "calls": len(self._records), "prompt_tokens": prompt, "completion_tokens": completion, "total_tokens": prompt + completion, "est_cost_usd": self.estimate_cost_usd(), }
# Singleton instance — shared across the analyzer and judge for one session. ledger = TokenLedger()