oct_docs.py

Go to the documentation of this file.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# oct/docs/oct_docs.py
 
"""
Purpose
-------
Provide documentation generation pipelines for Option C projects:
``oct docs --sphinx``, ``oct docs --doxygen``, ``oct docs --all``, and ``oct docs --clean``.
 
Responsibilities
----------------
- Set up Sphinx configuration in the project's ``docs/`` directory (if not already present).
- Discover markdown documentation files already in ``docs/``.
- Generate API reference pages from Python module docstrings via autodoc.
- Generate dependency graph pages via ``oct deps``.
- Populate ``index.rst`` with all discovered content.
- Run ``sphinx-build`` to generate HTML output.
- Check for required Sphinx dependencies and emit a clear error if missing.
- Run Doxygen with a patched Doxyfile (correcting stale INPUT paths).
- Clean generated documentation artifacts while preserving user-authored files.
 
Diagnostics
-----------
Domain: OCT-DOCS
Levels:
    L2 - lifecycle
    L3 - semantic details
    L4 - deep tracing
 
Contracts
---------
- Must not overwrite existing ``oct/docs/_sphinx/conf.py`` if already present.
- ``oct/docs/_sphinx/index.rst`` is regenerated on every build (auto-generated content).
- Must exit 1 with a helpful message if Sphinx dependencies are not installed.
- Template files are bundled in ``oct/templates/sphinx/``.
- Doxygen runs on a temporary copy of the Doxyfile; the original is never modified.
- ``--clean`` preserves ``_sphinx/``, ``_static/``, ``_templates/``, ``doxygen_work_dir/``, and user-authored markdown files.
 
Dependencies
------------
- oct.core.exclusions (directory exclusion lists)
- oct.core.diagnostics (_dbg structured logger)
- oct.tools.fs_remove (remove_tree_or_warn - removes generated output, warning on a tree that refuses to go)
- oct.core.option_c_dir (lazy; resolve_octrc - finds the octrc that holds the docs settings)
- oct.deps.import_graph (lazy; build_import_graph and its renderers - the dependencies page)
- oct.docs.doxyfile_init (lazy; DOXYFILE_REL - where a project's Doxyfile lives)
- oct.docs.typedoc (lazy; has_typedoc, run_docs_typedoc - the TypeDoc step of --all)
- oct.docs.workspace_docs (lazy; is_workspace_root, run_workspace_docs - the workspace fan-out)
- sphinx.ext.apidoc (lazy; main - writes the per-module API pages)
- tomli (lazy; reads pyproject.toml on Python 3.10, where tomllib is absent)
"""
 
import datetime
import json
import re
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path
 
from oct.core.exclusions import LINTER_EXCLUDE_DIRS, WILDCARD_EXCLUDE_DIRS
from oct.core.diagnostics import _dbg
from oct.tools.fs_remove import remove_tree_or_warn
 
 
_TEMPLATES_DIR = Path(__file__).parent.parent / "templates" / "sphinx"
 
#: OI-518 - markers that identify auto-generated files. ``oct docs --sphinx``
#: must not silently overwrite files a user has authored themselves. Any file
#: whose first line contains the appropriate marker is considered generated
#: and is safe to regenerate; all other existing files are preserved.
_AUTO_MARKER_RST: str = ".. Auto-generated by oct docs --sphinx"
_AUTO_MARKER_MD: str = "<!-- Auto-generated by oct docs --sphinx -->"
 
 
def _is_auto_generated(path: Path, marker: str) -> bool:
    """Return True if *path* is absent or was written by ``oct docs``.
 
    OI-518: used as a write-safety guard around regeneration of
    ``docs/README.md``, ``docs/index.rst``, ``docs/dependencies.rst``,
    and ``docs/_api/modules.rst``. A user-authored file (no marker) is
    left untouched and a warning is emitted instead.
    """
    if not path.exists():
        return True
    try:
        head = path.read_text(encoding="utf-8", errors="ignore")
    except OSError:
        return False
    first = head.splitlines()[0] if head else ""
    return marker in first
 
_REQUIRED_PACKAGES = [
    "sphinx",
    "sphinx_book_theme",
    "myst_parser",
    "sphinxcontrib.mermaid",
    "sphinx_copybutton",
]
 
# Subdirectories within docs/ to exclude from markdown discovery
_DOCS_EXCLUDE_DIRS = {"old", "code_reviews", "sphinx_docs", "_build",
                      "_static", "_templates", "_autosummary", "_sphinx",
                      "_api", "html", "doctrees", "doxygen_output",
                      "doxygen_work_dir"}
 
# Preferred ordering for markdown documents in the toctree.
# Documents matching these stems appear first (in this order);
# all remaining discovered docs follow alphabetically.
_PRIORITY_DOCS = [
    "README",
    "00_Master_Option_C_Specs_Intro_",
    "01_Master_Option_C_Specs_Core_",
    "02_Master_Option_C_Specs_Tooling_",
    "03_Master_Option_C_Specs_Diagnostics_",
    "04_Master_Option_C_Specs_AI_Protocol_",
    "05_Master_Option_C_Specs_Changelog_",
    "ARCHITECTURE",
    "REFERENCE",
    "ROADMAP",
]
 
 
# ---- Dependency checking ----
 
def _check_sphinx_deps() -> list[str]:
    """Check which Sphinx dependencies are missing."""
    missing = []
    for pkg in _REQUIRED_PACKAGES:
        try:
            __import__(pkg.replace("-", "_").replace(".", "_"))
        except ImportError:
            # Try the actual import name
            try:
                __import__(pkg)
            except ImportError:
                missing.append(pkg)
    return missing
 
 
# ---- Project info ----
 
def _load_project_info(root: Path) -> dict:
    """Load project name, version, and author from pyproject.toml or defaults."""
    info = {
        "name": root.resolve().name,
        "version": "0.1.0",
        "author": "Option C Dev Team",
    }
 
    pyproject = root / "pyproject.toml"
    if not pyproject.exists():
        return info
 
    # Parsed as TOML rather than scanned line by line. The previous version
    # matched any line starting "name"/"version" anywhere in the file and let
    # the LAST match win, so a `name =` inside `[tool.*]` or a dependency table
    # silently became the documented project name. It happened to be correct for
    # every pyproject in this workspace, which is exactly why it survived.
    try:
        try:
            import tomllib
        except ImportError:  # pragma: no cover - Python 3.10, where tomllib is absent
            import tomli as tomllib  # type: ignore[no-redef]
        # TOMLDecodeError subclasses ValueError, so the handler below covers a
        # malformed file without naming the module's own exception type.
        data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
    except (OSError, ImportError, ValueError):
        return info
 
    project = data.get("project")
    if not isinstance(project, dict):
        return info
    for key in ("name", "version"):
        value = project.get(key)
        if isinstance(value, str) and value:
            info[key] = value
 
    return info
 
 
# ---- .octrc.json docs configuration ----
 
def _load_docs_config(root: Path) -> dict:
    """Load docs configuration from the project's octrc with defaults (Appendix H).
 
    Resolved through :func:`oct.core.option_c_dir.resolve_octrc`, which prefers
    ``.option_c/octrc.json`` and falls back to the legacy ``<root>/.octrc.json``.
    This function used to hardcode the legacy path, so on a MIGRATED project --
    which has no root ``.octrc.json`` -- the whole ``docs`` block was silently
    ignored and the defaults below were always used. ``oct`` itself is migrated,
    so its own configured ``autodoc_modules`` was being dropped: discovery fell
    back to walking the tree, which picked up the stale ``build/lib/oct`` copy
    alongside the real package and ran ``sphinx-apidoc`` over both.
    """
    # ``sphinx_theme`` and ``sphinx_output`` were removed on 2026-09-25: both
    # were read into this dict and used nowhere. The theme comes from the
    # generated ``docs/_sphinx/conf.py``, and the output path is fixed by
    # ``run_sphinx_build``. Setting either in an octrc never had any effect, and
    # while they sat here they looked configurable. An octrc that still carries
    # them is unaffected: the filter below adopts only keys present here.
    defaults = {
        "include_dependency_graphs": True,
        "include_mermaid_diagrams": True,
        "autodoc_modules": [],
    }
 
    from oct.core.option_c_dir import resolve_octrc
 
    octrc = resolve_octrc(root)
    if octrc.exists():
        try:
            data = json.loads(octrc.read_text(encoding="utf-8"))
            docs_cfg = data.get("docs", {})
            if isinstance(docs_cfg, dict):
                defaults.update({k: v for k, v in docs_cfg.items()
                                 if k in defaults})
        except (json.JSONDecodeError, OSError):
            pass
 
    return defaults
 
 
# ---- Content discovery ----
 
def _discover_markdown(docs_dir: Path) -> list[str]:
    """Discover markdown files in the docs directory, recursing into
    subdirectories that are not in :data:`_DOCS_EXCLUDE_DIRS`.
 
    Returns a sorted list of file identifiers relative to ``docs_dir`` and
    stripped of the ``.md`` extension. For files in the docs/ root these
    identifiers are bare stems (e.g. ``"ARCHITECTURE"``); for files in
    subdirectories they are POSIX-form relative paths
    (e.g. ``"option_c_specs/00_Master_Option_C_Specs_Intro_V3_5"``).
    Sphinx-MyST consumes both forms directly in the toctree.
    """
    found: list[str] = []
    for md_file in docs_dir.rglob("*.md"):
        if not md_file.is_file():
            continue
        rel = md_file.relative_to(docs_dir)
        # Skip files inside any excluded subdirectory.
        if any(part in _DOCS_EXCLUDE_DIRS for part in rel.parts[:-1]):
            continue
        ident = rel.with_suffix("").as_posix()
        found.append(ident)
    return sorted(found)
 
 
def _order_markdown(stems: list[str]) -> list[str]:
    """Order markdown identifiers with priority docs first, then alphabetical.
 
    Identifiers may be bare stems (root-level docs) or
    ``subdir/.../stem`` paths (subdirectory docs). Priority matching
    compares against the basename so ``_PRIORITY_DOCS`` entries can stay
    leaf-name-based even as files move into subdirectories.
    """
    ordered = []
    used = set()
 
    def _basename(ident: str) -> str:
        return ident.rsplit("/", 1)[-1]
 
    for entry in _PRIORITY_DOCS:
        if entry.endswith("_") or entry.endswith("-"):  # prefix marker
            match = next(
                (s for s in stems if _basename(s).startswith(entry) and s not in used),
                None,
            )
            if match:
                ordered.append(match)
                used.add(match)
        else:  # exact match (original behaviour)
            for s in stems:
                if _basename(s) == entry and s not in used:
                    ordered.append(s)
                    used.add(s)
                    break
    remaining = sorted(s for s in stems if s not in used)
    return ordered + remaining
 
 
def _should_skip_dir(path: Path, root: Path) -> bool:
    """Check if a directory should be excluded from module discovery."""
    rel_parts = path.relative_to(root).parts
    for part in rel_parts:
        if part in LINTER_EXCLUDE_DIRS or part.startswith("."):
            return True
        for wc in WILDCARD_EXCLUDE_DIRS:
            if part.startswith(wc.rstrip("*")):
                return True
    # Also skip docs, test, and scaffold directories
    if rel_parts and rel_parts[0] in {"docs", "old", "tests",
                                       "scaffold_test_project"}:
        return True
    return False
 
 
#: OI-426 - safety cap on recursive package discovery. Bounds the walk
#: so a pathological directory tree (symlink loop, runaway nesting) cannot
#: hang ``oct docs``.
_MAX_PACKAGE_DEPTH: int = 10
 
 
def _find_packages(root: Path, config: dict) -> list[tuple[str, Path]]:
    """Discover Python packages to document.
 
    Walks the project tree recursively, stopping at each package (a
    directory containing ``__init__.py``) rather than descending into
    its subpackages - Sphinx autodoc handles subpackages automatically.
    Respects ``LINTER_EXCLUDE_DIRS`` and ``WILDCARD_EXCLUDE_DIRS`` via
    :func:`_should_skip_dir`.
 
    Bounded by :data:`_MAX_PACKAGE_DEPTH` to prevent pathological walks
    on misconfigured projects (OI-426).
 
    Returns list of ``(package_name, package_path)`` tuples.
    """
    autodoc_modules = config.get("autodoc_modules", [])
 
    if autodoc_modules:
        explicit: list[tuple[str, Path]] = []
        for mod_dir in autodoc_modules:
            mod_path = root / mod_dir
            if (mod_path / "__init__.py").exists():
                explicit.append((mod_dir.replace("/", "."), mod_path))
        return explicit
 
    packages: list[tuple[str, Path]] = []
 
    def _walk(directory: Path, depth: int) -> None:
        if depth > _MAX_PACKAGE_DEPTH:
            return
        try:
            children = sorted(directory.iterdir())
        except (OSError, PermissionError):
            return
        for child in children:
            if not child.is_dir() or _should_skip_dir(child, root):
                continue
            if (child / "__init__.py").exists():
                packages.append((child.name, child))
                # Do not descend - autodoc handles subpackages.
            else:
                _walk(child, depth + 1)
 
    _walk(root, 0)
    return packages
 
 
def _generate_api_rst(root: Path, docs_dir: Path, config: dict) -> bool:
    """Generate per-module API documentation using sphinx-apidoc.
 
    Uses sphinx-apidoc to create comprehensive .rst files for every
    module, class, and function - similar to Doxygen's EXTRACT_ALL output.
 
    Returns True if at least one package was documented.
    """
    packages = _find_packages(root, config)
    if not packages:
        return False
 
    api_dir = docs_dir / "_api"
    api_dir.mkdir(exist_ok=True)
 
    try:
        from sphinx.ext.apidoc import main as apidoc_main
    except ImportError:
        print("Warning: sphinx.ext.apidoc not available", file=sys.stderr)
        return False
 
    for pkg_name, pkg_path in packages:
        apidoc_main([
            '-f',           # force overwrite existing files
            '-e',           # separate page per module
            '-M',           # module-first ordering (module docs before submodules)
            '-o', str(api_dir),
            str(pkg_path),
            '--no-toc',     # we generate our own toc
        ])
        print(f"  sphinx-apidoc: {pkg_name} ({pkg_path})")
 
    # Check if apidoc generated any files
    rst_files = list(api_dir.glob("*.rst"))
    if not rst_files:
        return False
 
    # Overwrite modules.rst to list only top-level packages (not all
    # submodules) - each package .rst has its own toctree for submodules,
    # so listing everything flat here causes "multiple toctrees" warnings.
    pkg_stems = sorted(name for name, _ in packages)
    lines = [
        f"{_AUTO_MARKER_RST}. Edits will be overwritten on next build.",
        "",
        "API Reference",
        "=============",
        "",
        ".. toctree::",
        "   :maxdepth: 4",
        "",
    ]
    for stem in pkg_stems:
        lines.append(f"   {stem}")
    lines.append("")
    modules_path = api_dir / "modules.rst"
    if _is_auto_generated(modules_path, _AUTO_MARKER_RST):
        modules_path.write_text("\n".join(lines), encoding="utf-8")
    else:
        print(
            f"OI-518: preserving user-authored {modules_path.relative_to(docs_dir)}",
            file=sys.stderr,
        )
 
    return True
 
 
#: Where the dependency page's generated assets go, relative to docs/.
#: Under _build/ because that is gitignored: a generated SVG must never
#: land in the tracked tree (the same rule as the root-README include).
_DEPS_ASSET_REL: Path = Path("_build") / "deps"
 
 
def _render_dot_svg(dot_src: str, target: Path) -> bool:
    """Render *dot_src* to *target* with Graphviz; False when dot is absent or fails."""
    func = "_render_dot_svg"
    dot = shutil.which("dot")
    if dot is None:
        _dbg("OCT-DOCS", func, "dot not on PATH", 3)
        return False
    target.parent.mkdir(parents=True, exist_ok=True)
    try:
        result = subprocess.run(
            [dot, "-Tsvg", "-o", str(target)], input=dot_src, text=True,
            capture_output=True, timeout=300,
        )
    except (OSError, subprocess.TimeoutExpired) as exc:
        _dbg("OCT-DOCS", func, f"dot failed: {exc}", 2)
        return False
    if result.returncode != 0:
        _dbg("OCT-DOCS", func, f"dot rc={result.returncode}: {result.stderr[:200]}", 2)
        return False
    return target.is_file()
 
 
def _generate_deps_rst(root: Path, docs_dir: Path) -> bool:
    """Generate the ``dependencies`` page: the subproject's dependency overview.
 
    Two levels on one page (plan A, docs/DOCS_OVERVIEW_CHARTS_PLAN_2026-09-27.md):
    a package-level Mermaid chart a person can read in one look, and the
    full module-level graph rendered by Graphviz to a zoomable SVG. Both are
    derived from the ``import`` statements, not the ``Dependencies``
    docstring declarations that ``oct deps`` reports, because a structural
    picture that can drift from the code is worse than none.
 
    Returns True when a page was written (the graph has at least one edge),
    so the caller can add it to the toctree.
    """
    func = "_generate_deps_rst"
    try:
        from oct.deps.import_graph import (
            build_import_graph, collapse_to_packages, to_dot, to_mermaid_flowchart,
        )
    except ImportError:
        return False
 
    graph = build_import_graph(root)
    if graph.edge_count == 0:
        _dbg("OCT-DOCS", func, "no internal import edges; page skipped", 3)
        return False
 
    packages = collapse_to_packages(graph.edges)
    n_packages = len({n for edge in packages for n in edge})
    svg_path = docs_dir / _DEPS_ASSET_REL / "modules.svg"
    have_svg = _render_dot_svg(to_dot(graph.edges, title=root.name), svg_path)
 
    lines = [
        f"{_AUTO_MARKER_RST} from the import statements.",
        "",
        "Dependency overview",
        "===================",
        "",
        f"{len(graph.edges)} modules, {graph.edge_count} internal import edges, "
        f"{n_packages} packages. Tests are excluded. Derived from the ``import`` "
        f"statements on every build, so this is what the code does, not what the "
        f"docstrings declare (``oct deps`` reports the declarations).",
        "",
        "Packages",
        "--------",
        "",
        "One node per package; an edge where any module in one imports any module "
        "in the other, labelled with how many module-level imports it stands for. "
        "This is the picture to read first.",
        "",
        ".. mermaid::",
        "",
    ]
    lines.extend(f"   {line}" for line in to_mermaid_flowchart(packages).splitlines())
    lines.extend(["", "Modules", "-------", ""])
    if have_svg:
        rel = _DEPS_ASSET_REL.as_posix() + "/modules.svg"
        lines.extend([
            "Every module, clustered by package, cross-package edges drawn. Rendered "
            "by Graphviz; open the image on its own to zoom.",
            "",
            f".. image:: {rel}",
            "   :width: 100%",
            f"   :alt: module-level import graph of {root.name}",
            "",
        ])
    else:
        lines.extend([
            "The module-level graph is rendered with Graphviz, which was not found "
            "on this machine (``dot`` is not on PATH), so only the package level is "
            "shown. Install Graphviz and rebuild to get it.",
            "",
        ])
    if graph.external:
        top = ", ".join(f"``{name}`` ({count})" for name, count in graph.external.most_common(8))
        lines.extend([
            "External imports",
            "----------------",
            "",
            f"Not drawn; the chart is about this project's own structure. The most "
            f"imported external names, with the number of importing modules: {top}.",
            "",
        ])
 
    deps_path = docs_dir / "dependencies.rst"
    if _is_auto_generated(deps_path, _AUTO_MARKER_RST):
        deps_path.write_text("\n".join(lines), encoding="utf-8")
        return True
 
    print(
        "OI-518: preserving user-authored docs/dependencies.rst",
        file=sys.stderr,
    )
    return False
 
 
# ---- Index generation ----
 
def _write_index_rst(docs_dir: Path, project_name: str,
                     md_stems: list[str], has_api: bool,
                     has_deps: bool, *,
                     include_root_readme: bool = False) -> None:
    """Write a populated index.rst with discovered content.
 
    With *include_root_readme* the project's root ``README.md`` is rendered
    in place at the top of the index through docutils ``include`` with the
    MyST parser, so the landing page opens with the project's own
    introduction and no copy of the README is written into ``docs/``.
    """
    title = f"{project_name} Documentation"
    underline = "=" * len(title)
 
    lines = [
        f"{_AUTO_MARKER_RST}. Edits will be overwritten on next build.",
        "",
        title,
        underline,
        "",
    ]
 
    if include_root_readme:
        lines.extend([
            ".. include:: ../README.md",
            "   :parser: myst_parser.sphinx_",
            "",
        ])
 
    # Main content toctree (markdown files, priority-ordered)
    ordered_stems = _order_markdown(md_stems)
    if ordered_stems:
        lines.extend([
            ".. toctree::",
            "   :maxdepth: 2",
            "   :caption: Contents:",
            "",
        ])
        for stem in ordered_stems:
            lines.append(f"   {stem}")
        lines.append("")
 
    # API reference toctree
    if has_api:
        lines.extend([
            ".. toctree::",
            "   :maxdepth: 3",
            "   :caption: API Reference:",
            "",
            "   _api/modules",
            "",
        ])
 
    # Dependencies toctree
    if has_deps:
        lines.extend([
            ".. toctree::",
            "   :maxdepth: 2",
            "   :caption: Analysis:",
            "",
            "   dependencies",
            "",
        ])
 
    index_path = docs_dir / "index.rst"
    if _is_auto_generated(index_path, _AUTO_MARKER_RST):
        index_path.write_text("\n".join(lines), encoding="utf-8")
    else:
        print(
            "OI-518: preserving user-authored docs/index.rst",
            file=sys.stderr,
        )
 
 
# ---- Sphinx setup ----
 
def setup_sphinx(root: Path) -> Path:
    """Set up Sphinx configuration in the project's docs/ directory.
 
    Uses docs/ as the Sphinx source directory directly - markdown files
    already present in docs/ are included without copying.
 
    Sphinx config files (conf.py, Makefile, make.bat) live in docs/_sphinx/
    to keep the docs/ root clean.  The ``-c`` flag in ``run_sphinx_build``
    points Sphinx at this configuration directory.
 
    Returns the path to the docs directory.
    """
    func = "setup_sphinx"
    _dbg("OCT-DOCS", func, f"scaffolding sphinx under {root}", 3)
    docs_dir = root / "docs"
    sphinx_dir = docs_dir / "_sphinx"
    static_dir = docs_dir / "_static"
 
    if not _TEMPLATES_DIR.exists():
        print(f"Error: Sphinx templates not found at {_TEMPLATES_DIR}",
              file=sys.stderr)
        raise SystemExit(1)
 
    # Create directories
    docs_dir.mkdir(parents=True, exist_ok=True)
    sphinx_dir.mkdir(exist_ok=True)
    static_dir.mkdir(exist_ok=True)
    (docs_dir / "_templates").mkdir(exist_ok=True)
 
    info = _load_project_info(root)
    year = datetime.datetime.now().year
 
    # Copy conf.py (from template, with substitutions) - only if absent
    conf_dest = sphinx_dir / "conf.py"
    if not conf_dest.exists():
        template = (_TEMPLATES_DIR / "conf.py.template").read_text(
            encoding="utf-8")
        conf_content = (
            template
            .replace("{{PROJECT_NAME}}", info["name"])
            .replace("{{YEAR}}", str(year))
            .replace("{{AUTHOR}}", info["author"])
            .replace("{{VERSION}}", info["version"])
        )
        conf_dest.write_text(conf_content, encoding="utf-8")
 
    # Copy static files (CSS, logo) - only if absent
    for static_file in ["custom.css", "Option_C_logo.svg"]:
        src = _TEMPLATES_DIR / "_static" / static_file
        dest = static_dir / static_file
        if src.exists() and not dest.exists():
            shutil.copy2(src, dest)
 
    # Copy Makefile and make.bat to _sphinx/ - only if absent
    for build_file in ["Makefile", "make.bat"]:
        src = _TEMPLATES_DIR / build_file
        dest = sphinx_dir / build_file
        if src.exists() and not dest.exists():
            shutil.copy2(src, dest)
 
    return docs_dir
 
 
# ---- Build ----
 
def run_sphinx_build(docs_dir: Path, config: dict) -> int:
    """Run sphinx-build to generate HTML output.
 
    Returns exit code: 0 on success, 1 on failure.
    """
    docs_abs = docs_dir.resolve()
    sphinx_dir = docs_abs / "_sphinx"
 
    # Clean previous output to avoid stale files. A tree that refuses to go
    # (ENV-03: the read-only attribute a Drive mirror sets on every synced
    # directory) is reported and the build carries on - Sphinx overwrites what
    # it rebuilds, so a surviving cache costs freshness, not the whole run.
    # A bare ``shutil.rmtree`` here ended ``oct docs --all`` in a traceback.
    for clean_dir, what in ((docs_abs / "html", "previous HTML output"),
                            (docs_abs / "doctrees", "Sphinx doctree cache")):
        remove_tree_or_warn(clean_dir, label=what)
 
    # Build into docs/ directly - produces docs/html/ and docs/doctrees/
    cmd = [
        sys.executable, "-m", "sphinx",
        "-M", "html",
        str(docs_abs),       # source dir
        str(docs_abs),       # build dir (html/ and doctrees/ created here)
        "-c", str(sphinx_dir),  # conf.py location
    ]
 
    print(f"Running: sphinx-build -M html {docs_abs} {docs_abs} "
          f"-c {sphinx_dir}")
    try:
        result = subprocess.run(cmd, cwd=str(docs_abs))
        if result.returncode == 0:
            html_index = docs_abs / "html" / "index.html"
            print(f"\nBuild complete. Open: {html_index}")
        return result.returncode
    except FileNotFoundError:
        print("Error: sphinx-build not found.", file=sys.stderr)
        return 1
 
 
# ---- Full pipeline ----
 
def run_docs_sphinx(root: Path, *, no_workspace: bool = False) -> int:
    """Full oct docs --sphinx pipeline.
 
    1. Check dependencies
    2. Set up Sphinx config (one-time)
    3. Load .octrc.json docs configuration
    4. Discover markdown files
    5. Generate API reference (autodoc)
    6. Generate dependency graph
    7. Write populated index.rst
    8. Run sphinx-build
 
    Returns exit code.
 
    Workspace mode (FS-WS-DOCS): when ``root`` is a workspace root
    (``.option_c_workspace.json`` present) and *no_workspace* is
    ``False``, dispatches to
    :func:`oct.docs.workspace_docs.run_workspace_docs` for fan-out
    + aggregation. Subproject invocations and ``--no-workspace``
    keep the existing single-project body.
    """
    func = "run_docs_sphinx"
    _dbg("OCT-DOCS", func, f"sphinx build under {root}", 3)
    if not no_workspace:
        from oct.docs.workspace_docs import is_workspace_root, run_workspace_docs
        if is_workspace_root(root):
            return run_workspace_docs(root, mode="sphinx")
 
    # Check dependencies first
    missing = _check_sphinx_deps()
    if missing:
        print(
            f"Error: Missing Sphinx dependencies: {', '.join(missing)}\n"
            f"Install them with: pip install oct-tools[docs]\n"
            f"Or individually: pip install {' '.join(missing)}",
            file=sys.stderr,
        )
        return 1
 
    # Set up Sphinx config (copies template if conf.py absent)
    docs_dir = setup_sphinx(root)
 
    # The root README is pulled into the index by an ``include`` directive
    # rather than copied into docs/. The copy (with an auto-generated marker
    # so later runs recognised it) was a generated file landing in the
    # tracked source tree: committing it meant every root README edit
    # dirtied the copy until the next build, and ignoring docs/README.md
    # would have hidden the hand-authored one that oct and oc_diagnostics
    # rely on. An include leaves nothing behind. A copy from before this
    # change is withdrawn here, since it would otherwise be discovered as a
    # second, stale README page; a hand-authored docs/README.md (no marker)
    # is untouched and is discovered like any other document.
    readme_dest = docs_dir / "README.md"
    if readme_dest.exists() and _is_auto_generated(readme_dest, _AUTO_MARKER_MD):
        readme_dest.unlink()
        # OCT-LINT: disable=no_hardcoded_secrets reason="Fixed user-facing message; no credential material (operator-ruled 2026-09-27, same waiver the replaced line carried)"
        print("Removed the auto-generated docs/README.md copy; the root "
              "README is now included in place (2026-09-27).")
    include_root_readme = (root / "README.md").is_file()
 
    # Load configuration from .octrc.json
    config = _load_docs_config(root)
    info = _load_project_info(root)
 
    # Discover markdown files already in docs/
    md_stems = _discover_markdown(docs_dir)
    print(f"Discovered {len(md_stems)} markdown file(s) in docs/")
 
    # Generate API reference
    has_api = _generate_api_rst(root, docs_dir, config)
    if has_api:
        print("Generated API reference (api.rst)")
 
    # Generate dependency graph
    has_deps = False
    if config.get("include_dependency_graphs", True):
        has_deps = _generate_deps_rst(root, docs_dir)
        if has_deps:
            print("Generated dependency graph (dependencies.rst)")
 
    # Write populated index.rst
    _write_index_rst(docs_dir, info["name"], md_stems, has_api, has_deps,
                     include_root_readme=include_root_readme)
    print("Updated index.rst with discovered content")
 
    # Build
    return run_sphinx_build(docs_dir, config)
 
 
# ---- Doxygen pipeline ----
 
def _check_doxygen_installed() -> bool:
    """Check if doxygen is on PATH via shutil.which()."""
    return shutil.which("doxygen") is not None
 
 
def _find_doxyfile(root: Path) -> Path | None:
    """Locate the project's Doxyfile if it exists.
 
    The path is :data:`oct.docs.doxyfile_init.DOXYFILE_REL`, shared with the
    workspace fan-out, the hub card filter and the scaffold so that "has a
    Doxyfile" means the same thing everywhere.
    """
    from oct.docs.doxyfile_init import DOXYFILE_REL
    doxyfile = root / DOXYFILE_REL
    if doxyfile.is_file():
        return doxyfile
    return None
 
 
#: Files Doxygen writes into every HTML output directory it owns. One of
#: these must be present before :func:`_clear_doxygen_pages` will
#: delete a directory, so a mis-set ``OUTPUT_DIRECTORY`` cannot turn a
#: docs build into a recursive delete of someone's source tree.
_DOXYGEN_HTML_MARKERS: tuple[str, ...] = (
    "index.html", "doxygen.css", "doxygen.svg", "doxygen_crawl.html",
)
 
 
def _doxygen_setting(content: str, key: str) -> str | None:
    """Return the value of Doxyfile tag *key*, or ``None`` when unset.
 
    Only single-line assignments are read, which is what both settings
    this module cares about (``OUTPUT_DIRECTORY``, ``HTML_OUTPUT``) are
    in every Doxyfile Doxygen itself generates. Surrounding quotes are
    stripped; a tag present but empty reads as ``None``.
    """
    match = re.search(
        rf'^{re.escape(key)}\s*=\s*(.*)$', content, flags=re.MULTILINE,
    )
    if match is None:
        return None
    value = match.group(1).strip().strip('"')
    return value or None
 
 
def _doxygen_html_output_dir(doxyfile_path: Path) -> Path:
    """Resolve the HTML output directory *doxyfile_path* writes to.
 
    Doxygen resolves a relative ``OUTPUT_DIRECTORY`` against its working
    directory, and this module runs it with ``cwd=doxyfile.parent``, so
    that is the base used here. ``HTML_OUTPUT`` is then resolved against
    ``OUTPUT_DIRECTORY``. Both default the way Doxygen defaults them.
    """
    content = doxyfile_path.read_text(encoding="utf-8", errors="replace")
    base = doxyfile_path.parent
    output_dir = _doxygen_setting(content, "OUTPUT_DIRECTORY")
    resolved = base if output_dir is None else (base / output_dir)
    html_output = _doxygen_setting(content, "HTML_OUTPUT") or "html"
    return (resolved / html_output).resolve()
 
 
#: Suffixes of Doxygen's dot cache. For every graph Doxygen writes the
#: rendered ``.svg``, an image ``.map``, and a ``.md5`` of the dot source;
#: on the next run it compares the md5 and skips ``dot`` when the graph is
#: unchanged. Kept across builds by :func:`_clear_doxygen_pages`.
_DOT_CACHE_SUFFIXES: frozenset[str] = frozenset({".svg", ".md5", ".map"})
 
 
def _clear_doxygen_pages(doxyfile_path: Path, root: Path) -> None:
    """Delete the previous run's pages but keep the graph cache.
 
    Doxygen regenerates every page it still has an input for, but it
    never deletes the page for an input it no longer sees. Excluding a
    file therefore leaves its generated page on disk indefinitely: the
    navigation stops linking it, yet it keeps being copied into the
    published site and keeps answering at its old URL. Clearing the pages
    first is what makes the output mean "what the current configuration
    produces".
 
    The graphs are deliberately NOT cleared. Doxygen keys every dot graph
    on an md5 of its source and skips ``dot`` when the rendered file
    already matches, and that cache is most of the build: oc_pricing
    rebuilds in 0.8 s with 0 dot invocations when the cache is present
    and in 79 s with 304 when it is not (measured 2026-09-27). The
    2026-09-26 version of this function removed the whole directory and
    so threw the cache away on every build - oct's 3,185 graphs were
    being re-rendered from scratch each time. Graphs whose page no
    longer exists are pruned after the run by
    :func:`_prune_unreferenced_graphs`, which is what keeps the original
    orphan problem solved.
 
    Two guards, because this deletes inside a directory whose location
    comes from a config file: the target must resolve inside *root*, and
    it must look like Doxygen's own output. Anything else is left alone
    with a warning - a stale page is a much smaller problem than a
    deleted source tree.
 
    Failure is never fatal. Whatever cannot be removed is reported, and
    the build still runs.
    """
    func = "_clear_doxygen_pages"
    try:
        html_dir = _doxygen_html_output_dir(doxyfile_path)
    except OSError as exc:
        _dbg("OCT-DOCS", func, f"could not read Doxyfile: {exc}", 3)
        return
 
    if not html_dir.is_dir():
        _dbg("OCT-DOCS", func, f"nothing to clear at {html_dir}", 4)
        return
 
    try:
        html_dir.relative_to(root.resolve())
    except ValueError:
        print(f"Warning: Doxygen HTML output {html_dir} is outside the "
              f"project; not clearing it. Pages for inputs you have since "
              f"excluded may survive there.")
        _dbg("OCT-DOCS", func, f"refused: {html_dir} outside {root}", 2)
        return
 
    if not any((html_dir / marker).exists() for marker in _DOXYGEN_HTML_MARKERS):
        print(f"Warning: {html_dir} does not look like Doxygen output "
              f"(no {_DOXYGEN_HTML_MARKERS[0]}); not clearing it.")
        _dbg("OCT-DOCS", func, f"refused: {html_dir} has no marker file", 2)
        return
 
    removed = 0
    failed: list[Path] = []
    for entry in html_dir.iterdir():
        if entry.is_dir():
            # search/ and any other subtree are page-side; no cache lives there.
            if remove_tree_or_warn(entry, label="previous Doxygen output"):
                removed += 1
            else:
                failed.append(entry)
            continue
        if entry.suffix in _DOT_CACHE_SUFFIXES:
            continue
        try:
            entry.unlink()
            removed += 1
        except OSError as exc:
            print(f"  could not remove {entry}: {exc.strerror or exc}")
            failed.append(entry)
    _dbg("OCT-DOCS", func,
         f"cleared {removed} page-side entries in {html_dir}, {len(failed)} failed", 3)
 
 
def _prune_unreferenced_graphs(html_dir: Path) -> int:
    """Remove graph files no page in *html_dir* embeds; return the count.
 
    The complement of :func:`_clear_doxygen_pages`: pages are always
    rebuilt fresh, so after a run the set of ``.svg`` files the pages
    reference is exactly the set of live graphs. Everything else in the
    dot cache belongs to a function or class Doxygen no longer sees and
    is removed - the rendered ``.svg``, the ``_org.svg`` original that
    ``INTERACTIVE_SVG`` keeps beside it, and the ``.map`` and ``.md5``
    that would otherwise keep it "cached" forever.
 
    Doxygen's own non-graph SVGs (``doxygen.svg``, the legend and the
    theme's icons) are referenced from the pages like any graph and so
    survive this on the same rule, with no special-casing.
    """
    func = "_prune_unreferenced_graphs"
    if not html_dir.is_dir():
        return 0
    referenced: set[str] = set()
    for page in html_dir.rglob("*.html"):
        try:
            text = page.read_text(encoding="utf-8", errors="ignore")
        except OSError:
            continue
        referenced.update(re.findall(r'([A-Za-z0-9_.-]+)\.svg', text))
    # An interactive graph X.svg loads X_org.svg itself; keep the pair.
    live = set(referenced) | {f"{name}_org" for name in referenced}
 
    pruned = 0
    for entry in html_dir.iterdir():
        if entry.suffix not in _DOT_CACHE_SUFFIXES or entry.stem in live:
            continue
        try:
            entry.unlink()
            pruned += 1
        except OSError as exc:
            print(f"  could not remove {entry}: {exc.strerror or exc}")
    _dbg("OCT-DOCS", func, f"{len(referenced)} graphs referenced, {pruned} files pruned", 3)
    return pruned
 
 
def _create_patched_doxyfile(doxyfile_path: Path, root: Path) -> Path:
    """Create a temporary copy of the Doxyfile with INPUT patched to actual root.
 
    The existing Doxyfiles have hardcoded stale paths.  This creates a temp
    copy with the correct INPUT path.  Does NOT modify the original.
 
    Returns the path to the temp Doxyfile.
    """
    content = doxyfile_path.read_text(encoding="utf-8", errors="replace")
 
    # Patch INPUT line - use forward slashes for Doxygen cross-platform compat
    root_forward = str(root.resolve()).replace("\\", "/")
    content = re.sub(
        r'^(INPUT\s*=\s*).*$',
        rf'\g<1>"{root_forward}"',
        content,
        flags=re.MULTILINE,
    )
 
    # OI-423: stable prefix lets the stale-file sweeper distinguish OCT's
    # temp Doxyfiles from unrelated files in the system temp directory.
    tmp = tempfile.NamedTemporaryFile(
        mode="w",
        prefix="oct-doxyfile-",
        suffix=".Doxyfile",
        delete=False,
        encoding="utf-8",
    )
    tmp.write(content)
    tmp.close()
    return Path(tmp.name)
 
 
#: OI-423 - maximum age (in seconds) a stale ``oct-doxyfile-*`` temp file
#: may reach before the sweeper considers it orphaned. 24 hours is long
#: enough to avoid racing a concurrent ``oct docs --doxygen`` run.
_STALE_DOXYFILE_MAX_AGE_SECONDS: int = 24 * 60 * 60
 
 
def _sweep_stale_doxyfiles(
    max_age_seconds: int = _STALE_DOXYFILE_MAX_AGE_SECONDS,
) -> int:
    """Remove orphaned ``oct-doxyfile-*.Doxyfile`` temp files.
 
    The ``finally`` block in :func:`run_docs_doxygen` cleans up on normal
    exit, exceptions, and ``FileNotFoundError`` - but it is skipped on
    ``SIGKILL``, power loss, or OOM-kill. Without this sweep the system
    temp directory would slowly accumulate orphaned Doxyfiles (OI-423).
 
    Best-effort: any OS error (per-file or directory-wide) is silently
    swallowed so that a failed sweep never blocks a legitimate doxygen
    build.
 
    Returns the number of files removed.
    """
    removed = 0
    tmp_dir = Path(tempfile.gettempdir())
    now = time.time()
    try:
        for stale in tmp_dir.glob("oct-doxyfile-*.Doxyfile"):
            try:
                if now - stale.stat().st_mtime > max_age_seconds:
                    stale.unlink()
                    removed += 1
            except OSError:
                continue
    except OSError:
        pass
    return removed
 
 
def run_docs_doxygen(root: Path, *, no_workspace: bool = False) -> int:
    """Full oct docs --doxygen pipeline.
 
    1. Check doxygen is installed (shutil.which)
    2. Find existing Doxyfile in docs/doxygen_work_dir/
    3. Create patched temp copy with correct INPUT
    4. Run doxygen on the temp Doxyfile
    5. Clean up temp Doxyfile
    6. Report output location (docs/doxygen_output/)
 
    Returns exit code: 0 on success, 1 on failure.
 
    Workspace mode (FS-WS-DOCS): when ``root`` is a workspace root and
    *no_workspace* is ``False``, dispatches to
    :func:`oct.docs.workspace_docs.run_workspace_docs`.
    """
    if not no_workspace:
        from oct.docs.workspace_docs import is_workspace_root, run_workspace_docs
        if is_workspace_root(root):
            return run_workspace_docs(root, mode="doxygen")
 
    # OI-423: best-effort cleanup of orphans from earlier runs killed
    # before the finally block could run. Must never block the build.
    _sweep_stale_doxyfiles()
 
    if not _check_doxygen_installed():
        print(
            # OCT-LINT: disable=no_hardcoded_secrets reason="Fixed user-facing message; no credential material"
            "Error: doxygen is not installed or not on PATH.\n"
            "Install it from https://www.doxygen.nl/download.html\n"
            "On Windows: winget install doxygen\n"
            "On macOS: brew install doxygen\n"
            "On Linux: apt install doxygen",
            file=sys.stderr,
        )
        return 1
 
    doxyfile = _find_doxyfile(root)
    if doxyfile is None:
        print(
            "Error: No Doxyfile found at docs/doxygen_work_dir/Doxyfile.\n"
            "Run 'oct scaffold' to generate one, or create it manually.",
            file=sys.stderr,
        )
        return 1
 
    print(f"Using Doxyfile: {doxyfile}")
    # Before the run, not after: Doxygen leaves the page for every input it
    # no longer sees, so without this an EXCLUDE_PATTERNS edit never actually
    # takes pages out of the published site. Graphs are kept (the dot cache
    # is most of the build time) and pruned to the live set afterwards.
    _clear_doxygen_pages(doxyfile, root)
    patched = _create_patched_doxyfile(doxyfile, root)
 
    try:
        print(f"Running doxygen (INPUT patched to {root.resolve()})...")
        result = subprocess.run(
            ["doxygen", str(patched)],
            cwd=str(doxyfile.parent),
        )
        if result.returncode == 0:
            html_dir = _doxygen_html_output_dir(doxyfile)
            pruned = _prune_unreferenced_graphs(html_dir)
            if pruned:
                print(f"Pruned {pruned} graph file(s) no page references.")
            html_index = html_dir / "index.html"
            if html_index.exists():
                print(f"\nDoxygen build complete. Open: {html_index}")
            else:
                print(f"\nDoxygen build complete. Output: {html_dir}")
        return result.returncode
    except FileNotFoundError:
        print("Error: doxygen binary not found.", file=sys.stderr)
        return 1
    finally:
        try:
            patched.unlink()
        except OSError:
            pass
 
 
# ---- Clean pipeline ----
 
# Directories removed by --clean (relative to docs/)
_DOCS_CLEAN_DIRS = ["html", "doctrees", "_api", "doxygen_output", "typedoc"]
 
# Generated files removed by --clean (relative to docs/)
_DOCS_CLEAN_FILES = ["index.rst", "dependencies.rst"]
 
 
def run_docs_clean(root: Path, *, dry_run: bool = False, no_workspace: bool = False) -> int:
    """Remove generated documentation artifacts from docs/.
 
    Cleans: html/, doctrees/, _api/, doxygen_output/, typedoc/,
            index.rst, dependencies.rst
 
    Preserves: _sphinx/ (user conf.py), _static/, _templates/,
               user-authored .md files, doxygen_work_dir/Doxyfile
 
    Returns 0 when the artifacts are gone (or when there were none, or under
    *dry_run*), and 1 when one or more could not be removed - the count it
    prints is of directories verified absent afterwards, never of removal
    calls made, so a refused delete cannot report as a clean.
 
    Workspace mode (FS-WS-DOCS): from a workspace root, dispatches to
    :func:`oct.docs.workspace_docs.run_workspace_docs` with
    ``mode="clean"`` (Phase F implementation). ``--no-workspace`` falls
    back to single-project clean.
    """
    func = "run_docs_clean"
    _dbg("OCT-DOCS", func, f"cleaning generated docs under {root}", 3)
    if not no_workspace:
        from oct.docs.workspace_docs import is_workspace_root, run_workspace_docs
        if is_workspace_root(root):
            return run_workspace_docs(root, mode="clean", dry_run=dry_run)
 
    docs_dir = root / "docs"
    if not docs_dir.is_dir():
        print("Nothing to clean: docs/ directory does not exist.")
        return 0
 
    removed_count = 0
    survived: list[Path] = []
    action = "Would remove" if dry_run else "Removed"
 
    for dirname in _DOCS_CLEAN_DIRS:
        target = docs_dir / dirname
        if target.is_dir():
            # Count what is really gone. Reporting a directory as removed while
            # it is still on disk is the 2026-09-07 `oct clean` defect (947 of
            # them), and a read-only tree under a Drive mirror reproduces it.
            if not dry_run and not remove_tree_or_warn(
                    target, label="generated docs directory"):
                survived.append(target)
                continue
            print(f"  {action}: {target.relative_to(root)}/")
            removed_count += 1
 
    for filename in _DOCS_CLEAN_FILES:
        target = docs_dir / filename
        if target.is_file():
            if not dry_run:
                target.unlink()
            print(f"  {action}: {target.relative_to(root)}")
            removed_count += 1
 
    # A docs/README.md is only ours when it carries the marker: builds
    # before 2026-09-27 copied the root README there, and a hand-authored
    # one (oct, oc_diagnostics) has no marker and must survive a clean.
    readme = docs_dir / "README.md"
    if readme.is_file() and _is_auto_generated(readme, _AUTO_MARKER_MD):
        if not dry_run:
            readme.unlink()
        print(f"  {action}: {readme.relative_to(root)} (auto-generated copy)")
        removed_count += 1
 
    if removed_count == 0 and not survived:
        print("Nothing to clean: no generated artifacts found in docs/.")
    elif removed_count:
        prefix = "[DRY RUN] " if dry_run else ""
        print(f"\n{prefix}Cleaned {removed_count} artifact(s) from docs/.")
 
    if survived:
        # A non-zero exit is the honest verdict: the caller asked for these to be
        # gone and they are not. Each one already printed its own warning.
        print(f"\n{len(survived)} artifact(s) could not be removed:", file=sys.stderr)
        for target in survived:
            print(f"  {target.relative_to(root)}/", file=sys.stderr)
        return 1
 
    return 0
 
 
# ---- Combined pipeline ----
 
def run_docs_all(root: Path, *, no_workspace: bool = False) -> int:
    """Run both Sphinx and Doxygen pipelines.
 
    Returns 0 if both succeed, 1 if either fails.
 
    Workspace mode (FS-WS-DOCS): from a workspace root, dispatches to
    :func:`oct.docs.workspace_docs.run_workspace_docs` with
    ``mode="all"``.
    """
    if not no_workspace:
        from oct.docs.workspace_docs import is_workspace_root, run_workspace_docs
        if is_workspace_root(root):
            return run_workspace_docs(root, mode="all")
 
    print("=" * 40)
    print("Running Sphinx pipeline...")
    print("=" * 40)
    sphinx_rc = run_docs_sphinx(root, no_workspace=True)
 
    print()
    print("=" * 40)
    print("Running Doxygen pipeline...")
    print("=" * 40)
    doxygen_rc = run_docs_doxygen(root, no_workspace=True)
 
    # TypeDoc only when declared: a Python project has no typedoc.json and
    # asking it to build one would be a false failure in every --all run.
    # Doxygen above is NOT gated the same way, deliberately - a Doxyfile is
    # the norm for a Python project here and its absence is worth failing.
    from oct.docs.typedoc import has_typedoc, run_docs_typedoc
    typedoc_rc = 0
    if has_typedoc(root):
        print()
        print("=" * 40)
        print("Running TypeDoc pipeline...")
        print("=" * 40)
        typedoc_rc = run_docs_typedoc(root, no_workspace=True)
 
    failures = [name for name, rc in (("Sphinx", sphinx_rc),
                                      ("Doxygen", doxygen_rc),
                                      ("TypeDoc", typedoc_rc)) if rc]
    if failures:
        print(f"\nFailed: {', '.join(failures)}")
    else:
        print("\nAll pipelines completed successfully.")
 
    return 1 if failures else 0