presenter.py

Go to the documentation of this file.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# oc_chat/benchmarks/presenter.py
 
"""
BenchmarkPresenter — the measurement tap over the presenter contract.
 
Purpose
-------
A standalone :class:`~oc_ui.presenter_base.Presenter` (not a delegating
wrapper) that records, per turn, exactly what the measurement surface
exposes — and nothing else. It is the single lowest-blast-radius way to
observe a turn with zero production changes (plan 136 ADR-136-1):
 
- **TTFT** = monotonic at ``show_waiting`` (fired immediately before
  ``client.stream_chat``) → the first ``show_reasoning`` / ``show_content``
  chunk of that iteration. Iterations that emit only tool calls have no
  first-chunk token, so their TTFT is ``None`` — never fabricated.
- **Raw usage** — ``show_usage`` receives the raw provider dict (the
  only place ``reasoning_tokens`` and the DeepSeek cache split survive).
- **Main-loop tool names** — ``show_tool_call`` fires for main-loop
  calls only; sub-loop internal tools never reach the presenter, so the
  count is derived and the *names* are marked unavailable (ADR-136-5).
- **Turn summary** — ``iterations`` and ``tool_calls`` drive the
  sub-loop-count derivation (``tool_calls - main_loop_names``).
 
Every other presenter hook is a no-op (the benchmark has no UI, and
the real startup composition path calls ``show_info`` etc.).
 
Responsibilities
----------------
- :class:`BenchmarkPresenter` — implement the full ``Presenter`` ABC;
  capture per-turn buffers; :meth:`reset_turn`; :meth:`build_turn`.
 
Diagnostics
-----------
Domain: OC_CHAT.MAIN
Levels:
    L2 — lifecycle events
    L3 — semantic details
    L4 — deep tracing
 
Contracts
---------
- A *fresh* presenter (or one used only for a discarded warmup) leaks
  nothing into a measured run; the runner uses a separate throwaway
  presenter for warmup (plan §3.2 step 6).
- :meth:`build_turn` is pure given the captured buffers; it does not
  mutate them.
 
Dependencies
------------
- oc_diagnostics (structured debug logging via _dbg)
- oc_chat.benchmarks.metrics (CallMetrics, TurnMetrics, extract_call_metrics)
- oc_ui.presenter_base (Presenter ABC)
- oc_chat.core.session (lazy; ChatSession - type hint for the no-op show_welcome)
"""
from __future__ import annotations
 
import time
from contextlib import contextmanager
from typing import TYPE_CHECKING, Iterator
 
from oc_diagnostics import _dbg
 
from oc_chat.benchmarks.metrics import TurnMetrics, extract_call_metrics
from oc_ui.presenter_base import Presenter
 
if TYPE_CHECKING:
    from oc_chat.core.session import ChatSession
    from oc_ui.presenter_base import IterationStat
 
_dbg("OC_CHAT.MAIN", "module", "presenter loaded", 4)
 
 
class BenchmarkPresenter(Presenter):
    """Standalone presenter that records per-turn measurements only."""
 
    def __init__(self, clock=time.monotonic) -> None:
        self._clock = clock
        self.reset_turn()
 
    # ── Per-turn capture state ───────────────────────────────────────────────
 
    def reset_turn(self) -> None:
        """Clear all per-turn buffers. Called between measured turns."""
        self._t_wait: float | None = None
        self._await_first_chunk: bool = False
        self.ttft_ms: float | None = None       # first iteration's TTFT
        self.iteration_ttfts: list[float] = []
        self.usages: list[dict] = []
        self.tool_names: list[str] = []
        # From show_turn_summary:
        self.summary_iterations: int = 0
        self.summary_tool_calls: int = 0
        self.summary_model: str | None = None
 
    # ── Measurement hooks ────────────────────────────────────────────────────
 
    def show_waiting(self) -> None:
        # Fired immediately before client.stream_chat (react_loop.py:565):
        # the TTFT anchor for this iteration.
        self._t_wait = self._clock()
        self._await_first_chunk = True
 
    def _first_chunk(self) -> None:
        if self._await_first_chunk and self._t_wait is not None:
            ttft = (self._clock() - self._t_wait) * 1000.0
            self.iteration_ttfts.append(ttft)
            if self.ttft_ms is None:           # first iteration only
                self.ttft_ms = ttft
            self._await_first_chunk = False
 
    def show_reasoning(self, text: str) -> None:
        self._first_chunk()
 
    def show_content(self, text: str) -> None:
        self._first_chunk()
 
    def show_tool_call(self, name: str, arguments: str) -> None:
        # Main-loop tool call only — sub-loop tools never reach the presenter.
        self.tool_names.append(name)
 
    def show_usage(self, usage: dict) -> None:
        # The RAW provider usage dict (react_loop.py:651).
        if isinstance(usage, dict):
            self.usages.append(dict(usage))
 
    def show_turn_summary(
        self,
        prompt: int,
        completion: int,
        total: int,
        iterations: int,
        tool_calls: int,
        prompt_cached: int | None = None,
        *,
        cache_hit_pct: float | None = None,
        cache_saved_tokens: int | None = None,   # ignored for v1
        performance_stats: object = None,         # ignored for v1
        model: str | None = None,
        context_used_pct: float | None = None,
        duration_ms: int | None = None,           # ignored for v1
        turn_cost: object = None,                 # plan 188 T4 — the suite
        # prices its own runs from oc_chat.benchmarks.prices with the full
        # reasoning split, so the render-lane estimate is not captured here.
    ) -> None:
        self.summary_iterations = iterations
        self.summary_tool_calls = tool_calls
        self.summary_model = model
 
    # ── Build a TurnMetrics from the captured buffers ────────────────────────
 
    def build_turn(self, index: int, cold_label: str) -> TurnMetrics:
        """Materialise the captured buffers into a :class:`TurnMetrics`.
 
        Sub-loop tool *names* are not observable via the presenter, so
        ``sub_loop_count`` is derived as ``summary.tool_calls −
        len(main_loop_tool_names)`` (acc.tool_calls counts main-loop
        dispatches + sub-loop offloads — react_loop.py:545/721) and the
        names are marked unavailable.
        """
        calls = [extract_call_metrics(u) for u in self.usages]
        sub_loop_count = max(0, self.summary_tool_calls - len(self.tool_names))
        return TurnMetrics(
            index=index,
            cold_label=cold_label,
            calls=calls,
            ttft_ms=self.ttft_ms,
            iterations=self.summary_iterations,
            main_loop_tool_names=list(self.tool_names),
            sub_loop_count=sub_loop_count,
            sub_loop_tool_names_available=False,
        )
 
    # ── No-op UI hooks (the benchmark has no UI) ─────────────────────────────
 
    @contextmanager
    def turn(self) -> Iterator[None]:
        yield
 
    def start_turn(self) -> None: ...
    def end_turn(self) -> None: ...
    def show_tool_progress(self, name: str) -> None: ...
    def show_tool_result(self, name: str, result: object) -> None: ...
    def show_iteration_stat(self, stat: "IterationStat") -> None: ...
    def show_welcome(self, session: "ChatSession") -> None: ...
    def show_available_models(self, models: dict) -> None: ...
    def show_session_list(self, sessions: list, current_name: str | None = None) -> None: ...
    def show_help(self, command_help: dict, tools: list[str] | None = None) -> None: ...
    def show_compressed(self) -> None: ...
    def show_error(self, msg: str) -> None: ...
    def show_info(self, msg: str) -> None: ...
 
 
__all__ = ["BenchmarkPresenter"]