scenarios.py

Go to the documentation of this file.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# oc_chat/benchmarks/scenarios.py
 
"""
Fixed, version-stamped benchmark scenarios.
 
Purpose
-------
A scenario is a declarative bundle of an ordered list of user prompts
plus the mode it must run in. Scenarios are **fixed and version-
stamped** so before/after runs are comparable; changing a scenario's
prompts means a new id (or a bumped ``version``), never a silent
in-place edit (keep receipts). All v1 prompts are **read-only** (no
``write_file`` / no destructive ``run_command``) and reference stable
worktree files, so runs are comparable and safe.
 
All three v1 scenarios declare ``mode = "option-c"`` — the mode whose
tool-dump prompt Plan 135 slims; ``standard`` mode exposes no tools or
project tree and would not exercise the surface under test.
 
Responsibilities
----------------
- :class:`Scenario` — name / version / mode / turns / description.
- :data:`SCENARIOS` — the registry; :func:`get` / :func:`names`.
 
Diagnostics
-----------
Domain: OC_CHAT.MAIN
Levels:
    L2 — lifecycle events
    L3 — semantic details
    L4 — deep tracing
 
Contracts
---------
- :func:`get` raises ``KeyError`` with the valid names on an unknown id
  (the CLI turns that into a clean exit-1, never a traceback).
- Prompts are read-only and reference stable repository paths.
 
Dependencies
------------
- oc_diagnostics (structured debug logging via _dbg)
"""
from __future__ import annotations
 
from dataclasses import dataclass, field
 
from oc_diagnostics import _dbg
 
 
@dataclass(frozen=True)
class Scenario:
    """A fixed, version-stamped scripted scenario."""
 
    name: str
    version: str
    mode: str
    turns: tuple[str, ...]
    description: str = ""
 
 
SCENARIOS: dict[str, Scenario] = {
    "qa_tool_light": Scenario(
        name="qa_tool_light",
        version="1",
        mode="option-c",
        description="Plain Q&A, little/no tool use — isolates prompt-size / cache / TTFT.",
        turns=(
            "In two sentences, what is the ReAct loop in an agentic chat assistant?",
            "What is the difference between a model's context window and its prefix cache?",
            "Summarise your two previous answers as three bullet points.",
        ),
    ),
    "tool_heavy": Scenario(
        name="tool_heavy",
        version="1",
        mode="option-c",
        description="Read/list/search tools + a likely sub-loop offload.",
        turns=(
            "List the Python files in oc_chat/core/ and tell me how many there are.",
            # OCT-LINT: disable=no_hardcoded_secrets reason="benchmark scenario prompt text naming a repository file path; no credential material (operator-ruled 2026-09-28)"
            "Read oc_chat/core/sub_loop.py and explain in two sentences what "
            "_DEFAULT_SUB_SYSTEM is for.",
            "Search the codebase for where stream_chat is defined and list the files "
            "that define it.",
        ),
    ),
    "reasoning_heavy": Scenario(
        name="reasoning_heavy",
        version="1",
        mode="option-c",
        description="Multi-step reasoning to surface reasoning_tokens on a thinking model.",
        turns=(
            "Walk through, step by step, how a single user turn flows from run_turn to "
            "the provider and back, including when and why a sub-loop is used.",
            "Now identify the two places in that flow where token cost is recorded but "
            "context-window saturation is not, and explain why they are kept separate.",
        ),
    ),
}
 
 
def names() -> list[str]:
    """Sorted list of scenario ids."""
    return sorted(SCENARIOS)
 
 
def get(name: str) -> Scenario:
    """Return the scenario for *name*, or raise ``KeyError`` listing valid ids."""
    func = "get"
    _dbg("OC_CHAT.MAIN", func, f"scenario lookup name={name!r}", 3)
    try:
        return SCENARIOS[name]
    except KeyError:
        valid = ", ".join(names())
        raise KeyError(f"unknown scenario {name!r}; valid: {valid}") from None
 
 
def all_scenarios() -> list[Scenario]:
    """Every scenario, in sorted-name order."""
    return [SCENARIOS[n] for n in names()]
 
 
__all__ = ["Scenario", "SCENARIOS", "names", "get", "all_scenarios"]