oct_scaffold.py

Go to the documentation of this file.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# oct/scaffold/oct_scaffold.py
 
"""
Purpose
-------
OI-512 / FS-518 - project scaffolding. Lifts the scaffold body out of
``oct/cli.py`` so the CLI becomes a thin dispatcher and so the venv
integration (FS-503 / FS-504) can layer on a single ``include_venv``
parameter rather than a second 135-line rewrite.
 
The public entry point is :func:`run_scaffold`, which returns a
:class:`ScaffoldResult` describing what was created, skipped, or warned
about. Callers are expected to render the result themselves - this
module never touches ``click.echo`` directly.
 
Responsibilities
----------------
- Materialise the standard Option C directory layout under a target path.
- Write bundled templates: ``.octrc.json``, Sphinx ``conf.py`` /
  ``index.rst``, the debug-config skeleton, stub READMEs.
- Optionally create a ``.venv/`` via :mod:`venv` (``include_venv``).
- Run in dry-run mode where no filesystem state changes.
 
Diagnostics
-----------
Domain: SCAFFOLD
Levels:
    L2 - lifecycle: start / end / target path
    L3 - semantic details: per-template decisions
    L4 - deep tracing: per-file / per-dir outcomes
 
Contracts
---------
- ``run_scaffold`` must be side-effect-free when ``dry_run=True``.
- Venv creation must never shell out - uses :class:`venv.EnvBuilder`.
- The Sphinx template directory is resolved at import time to keep
  ``run_scaffold`` self-contained.
 
Dependencies
------------
- oct.core.diagnostics (_dbg structured logger)
- oct.core.option_c_dir (OcStatus, write_oc_status)
- oct.docs.doxyfile_init (lazy; stamps the Doxyfile from the bundled template)
"""
 
from __future__ import annotations
 
import datetime
import json
import os
import platform
import shutil
import venv
from dataclasses import dataclass, field
from pathlib import Path
 
from oct.core.diagnostics import _dbg
from oct.core.option_c_dir import OcStatus, write_oc_status
 
 
# ---------------------------------------------------------------------
# Template bundle - resolved relative to the installed package.
# ---------------------------------------------------------------------
 
_SPHINX_TEMPLATES_DIR: Path = Path(__file__).resolve().parents[1] / "templates" / "sphinx"
 
 
# ---------------------------------------------------------------------
# Debug-config skeleton (v2 schema).
# ---------------------------------------------------------------------
 
_DEBUG_CONFIG_TEMPLATE: dict = {
    "_schema_version": "2.0",
    "DEBUG_LEVELS": {
        "GENERAL": {"level": 2, "color": "default"}
    },
    "global_settings": {
        "silent_mode": False,
        "enable_timestamps": True,
        "enable_colors": True,
        "include_filename": True,
        "output_mode": "both",
        "log_to_file": True,
        "log_file_path": None,
        "log_format": "text",
        "log_rotation": {
            "max_files": 10,
            "max_size_mb": 5.0,
        },
        "global_debug_level": None,
        "watch_config": False,
        "ai_trace_level": False,
        "redaction_patterns": [],
        "mode": "dev",
    },
    "instrumentation_enabled": False,
    "http_redact_headers": ["authorization", "cookie", "set-cookie", "x-api-key"],
}
 
 
# ---------------------------------------------------------------------
# Result dataclass
# ---------------------------------------------------------------------
 
 
@dataclass
class ScaffoldResult:
    """Structured output from :func:`run_scaffold`.
 
    Attributes
    ----------
    created
        Paths (directories + files) that were created.
    skipped
        Paths that already existed and were not overwritten.
    messages
        Informational strings the caller may render - warnings, tips,
        or dry-run descriptions.
    venv_path
        The ``.venv/`` that was created, or ``None`` when
        ``include_venv=False``.
    """
 
    created: list[Path] = field(default_factory=list)
    skipped: list[Path] = field(default_factory=list)
    messages: list[str] = field(default_factory=list)
    venv_path: Path | None = None
 
 
# ---------------------------------------------------------------------
# File-content helpers
# ---------------------------------------------------------------------
 
 
def _architecture_md(name: str) -> str:
    """Seed ``docs/ARCHITECTURE.md`` with the workspace's overview-chart slot.
 
    Every subproject's architecture document carries a section headed
    exactly ``Architecture overview``: one Mermaid flowchart of intent,
    then how to read it and what it leaves out (plan B of
    ``docs/DOCS_OVERVIEW_CHARTS_PLAN_2026-09-27.md``). Seeding the slot is
    what makes a new project arrive with the convention rather than
    acquire it later. The placeholder chart is valid Mermaid, so the
    first Sphinx build renders it instead of failing on it.
    """
    return (
        f"# {name} - Architecture\n\n"
        "## Architecture overview\n\n"
        f"What {name} is for, in one paragraph.\n\n"
        "```mermaid\n"
        "flowchart TB\n"
        '    user(["Who or what uses it"]) --> entry["Entry point"]\n'
        '    entry --> core["The layer everything else stands on"]\n'
        "```\n\n"
        "**How to read it.** The direction of the chart, what solid and "
        "dotted arrows mean, and the one property the picture exists to "
        "show.\n\n"
        "**What it leaves out.** Name the deliberate omissions. The "
        "module-level truth is the generated `dependencies` page, derived "
        "from the imports on every docs build.\n\n"
        "## Components\n\nList major components and their responsibilities.\n\n"
        "## Data Flow\n\nDescribe how data flows through the system.\n"
    )
 
 
def _run_tests_py(name: str) -> str:
    return (
        "#!/usr/bin/env python3\n"
        "# -*- coding: utf-8 -*-\n"
        f"# {name}/tests/run_tests.py\n\n"
        "import subprocess\nimport sys\n\n"
        "sys.exit(subprocess.call([sys.executable, '-m', 'pytest', '--tb=short', '-q']))\n"
    )
 
 
def _tests_readme(name: str) -> str:
    return (
        f"# {name} - Tests\n\n"
        "## Running Tests\n\n"
        "```bash\npython -m pytest tests/ -v\n```\n\n"
        "## Test Structure\n\nDescribe the test organization.\n"
    )
 
 
def _docs_readme(name: str) -> str:
    return (
        f"# {name} - Documentation\n\n"
        "This directory contains project documentation.\n\n"
        "## Building\n\n"
        "```bash\noct docs --sphinx\n```\n\n"
        "Output will be generated in ``docs/_build/html/``.\n"
    )
 
 
def _octrc_json() -> str:
    """OI-530: build the scaffolded ``.octrc.json``.
 
    Ships with a live ``linter.profile`` default plus an underscore-prefixed
    ``_git_example`` stanza. JSON has no comments, so we use the same
    ``_<name>`` convention that ``_schema_version`` and other sentinels
    already use elsewhere: :func:`oct.core.octrc.load_octrc_safe` ignores
    underscore-prefixed top-level keys, so the example is discoverable but
    inert until the operator renames it.
    """
    return json.dumps(
        {
            "linter": {"profile": "compact"},
            # sphinx_theme / sphinx_output are deliberately absent: nothing
            # reads them (removed from oct.docs.oct_docs on 2026-09-25). The
            # theme lives in the generated docs/_sphinx/conf.py, which is the
            # file to edit, and the output path is fixed by the build.
            "docs": {
                "include_dependency_graphs": True,
                "include_mermaid_diagrams": True,
                "autodoc_modules": [],
            },
            "_git_example": {
                "protected_branches": ["main", "master"],
                "branch_profile_map": {"main": "strict", "dev": "compact"},
                "require_signed_commits": False,
            },
        },
        indent=2,
    )
 
 
# FS-503 / FS-504: default ignore lines for scaffolded projects. ``.venv/``
# ships in both ``.gitignore`` and ``.octignore`` so a freshly-scaffolded
# project never commits its venv and the linter never walks into it.
# FS-539: logs and lint/linter caches live under ``.option_c/`` - those
# subdirs are ignored so only the canonical config files stay tracked.
_GITIGNORE_LINES: tuple[str, ...] = (
    "# Python",
    "__pycache__/",
    "*.py[cod]",
    "*.egg-info/",
    "",
    "# Virtual environments",
    ".venv/",
    "venv/",
    "env/",
    "",
    "# IDE / OS",
    ".vscode/",
    ".idea/",
    ".DS_Store",
    "",
    "# Build output",
    "build/",
    "dist/",
    "docs/_build/",
    "",
    "# Generated documentation - everything oct docs writes. Sphinx lands in",
    "# html/ and doctrees/ (source and output are the same directory), the",
    "# API stubs in _api/, Doxygen in doxygen_output/. docs/doxygen_work_dir/",
    "# holds the Doxyfile and IS tracked.",
    "docs/html/",
    "docs/doctrees/",
    "docs/_api/",
    "docs/_autosummary/",
    "docs/doxygen_output/",
    "",
    "# Option C runtime data (FS-539) - config files stay tracked",
    ".option_c/logs/",
    ".option_c/cache/",
    "",
    "# Logs (legacy location pre-v0.19.0; kept for projects mid-migration)",
    "logs/",
    "*.log",
    "",
    "# Secrets (mirror NEVER_EXPORT_FILE_PATTERNS)",
    ".env",
    ".env.*",
    "*.pem",
    "*.key",
    "credentials.json",
    "secrets.json",
    "secrets.yaml",
)
 
 
_OCTIGNORE_LINES: tuple[str, ...] = (
    "# oct-specific ignores - lint/format/test/deps all consult this file.",
    ".venv/",
    "venv/",
    "env/",
    "docs/_build/",
    "docs/html/",
    "docs/doxygen_output/",
    "build/",
    "dist/",
    ".formatter_archive/",
)
 
 
# ---------------------------------------------------------------------
# Public entry point
# ---------------------------------------------------------------------
 
 
def run_scaffold(
    name: str,
    base_dir: Path,
    *,
    dry_run: bool = False,
    include_venv: bool = True,
) -> ScaffoldResult:
    """Create a new Option C project under ``base_dir / name``.
 
    Parameters
    ----------
    name
        Project name (becomes the top-level directory).
    base_dir
        Parent directory - the project is created at ``base_dir / name``.
    dry_run
        When True, describe what would happen without touching the
        filesystem.
    include_venv
        When True, create a ``.venv/`` at the project root using the
        stdlib :mod:`venv` module.
 
    Returns
    -------
    ScaffoldResult
        Structured result. Always returns; never raises for existing
        files (they are recorded in ``skipped``).
    """
    func = "run_scaffold"
    project_dir = base_dir / name
    _dbg("SCAFFOLD", func, f"start name={name} dir={project_dir} dry_run={dry_run}", 2)
 
    result = ScaffoldResult()
    config_text = json.dumps(_DEBUG_CONFIG_TEMPLATE, indent=2)
 
    # FS-539: canonical project layout. ``.option_c/`` holds the OCT
    # runtime state (config, logs, cache, migration status). ``docs/``
    # and ``tests/`` stay at the project root because they're the
    # developer-facing surface. ``oc_diagnostics/`` is no longer a
    # scaffolded dir - its former contents (``debug_config.json``) live
    # under ``.option_c/`` now.
    dirs_to_create: list[Path] = [
        project_dir,
        project_dir / ".option_c",
        project_dir / ".option_c" / "logs",
        project_dir / ".option_c" / "cache",
        project_dir / "docs",
        project_dir / "docs" / "_static",
        project_dir / "docs" / "_build",
        project_dir / "docs" / "_templates",
        project_dir / "tests",
    ]
 
    files_to_write: list[tuple[Path, str]] = [
        (project_dir / "docs" / "ARCHITECTURE.md", _architecture_md(name)),
        (project_dir / "tests" / "run_tests.py", _run_tests_py(name)),
        (project_dir / "tests" / "Tests_README.md", _tests_readme(name)),
        # FS-539: debug_config.json moved from oc_diagnostics/ to .option_c/
        (project_dir / ".option_c" / "debug_config.json", config_text),
        (project_dir / "docs" / "README.md", _docs_readme(name)),
        # FS-539: .octrc.json moved from root to .option_c/octrc.json
        (project_dir / ".option_c" / "octrc.json", _octrc_json()),
        (project_dir / ".gitignore", "\n".join(_GITIGNORE_LINES) + "\n"),
        (project_dir / ".octignore", "\n".join(_OCTIGNORE_LINES) + "\n"),
    ]
 
    if dry_run:
        for d in dirs_to_create:
            result.messages.append(f"[DRY RUN] Would create directory: {d}")
        for path, _ in files_to_write:
            result.messages.append(f"[DRY RUN] Would write file: {path}")
        result.messages.append(
            f"[DRY RUN] Would write file: {project_dir / '.option_c' / 'oc_status.json'}"
        )
        result.messages.append("[DRY RUN] Would copy Sphinx templates to docs/")
        if include_venv:
            result.messages.append(
                f"[DRY RUN] Would create virtualenv: {project_dir / '.venv'}"
            )
        result.messages.append("[DRY RUN] Scaffold complete (no files written)")
        _dbg("SCAFFOLD", func, "dry-run end", 2)
        return result
 
    # Create directories (parents first).
    for d in dirs_to_create:
        if d.exists():
            result.skipped.append(d)
        else:
            d.mkdir(parents=True, exist_ok=True)
            result.created.append(d)
 
    # Write bundled files, never overwriting.
    for path, content in files_to_write:
        if path.exists():
            result.skipped.append(path)
            continue
        path.write_text(content, encoding="utf-8")
        result.created.append(path)
 
    # FS-539: oc_status.json declares this project compliant at birth.
    _write_fresh_oc_status(project_dir, result)
 
    # Executable bit on run_tests.py (POSIX only).
    if platform.system() != "Windows":
        run_tests_path = project_dir / "tests" / "run_tests.py"
        if run_tests_path.exists():
            os.chmod(str(run_tests_path), 0o755)
 
    # Sphinx template pack.
    _copy_sphinx_templates(project_dir, name, result)
 
    # Doxyfile: its presence is what declares the project for Doxygen
    # builds, so a scaffolded project is Doxygen-ready from birth without
    # anyone copying a 3,000-line file by hand. Never overwrites.
    _write_doxyfile(project_dir, name, result)
 
    # FS-503 / FS-504: optional venv.
    if include_venv:
        _create_venv(project_dir, result)
 
    _dbg(
        "SCAFFOLD", func,
        f"end created={len(result.created)} skipped={len(result.skipped)} "
        f"venv={'yes' if result.venv_path else 'no'}",
        2,
    )
    return result
 
 
# ---------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------
 
 
def _write_doxyfile(
    project_dir: Path, name: str, result: ScaffoldResult,
) -> None:
    """Stamp ``docs/doxygen_work_dir/Doxyfile`` via :mod:`oct.docs.doxyfile_init`.
 
    A missing bundled template is reported in ``messages`` rather than
    raised, matching how the Sphinx pack is handled: the scaffold as a
    whole must still succeed on a checkout that omits templates.
    """
    func = "_write_doxyfile"
    from oct.docs.doxyfile_init import DoxyfileTemplateMissing, write_doxyfile
    try:
        path, written = write_doxyfile(project_dir, name)
    except DoxyfileTemplateMissing as exc:
        result.messages.append(f"  Warning: {exc}; skipping Doxyfile.")
        _dbg("SCAFFOLD", func, "template missing; skipped", 3)
        return
    (result.created if written else result.skipped).append(path)
    _dbg("SCAFFOLD", func, f"{path} {'written' if written else 'kept'}", 3)
 
 
def _copy_sphinx_templates(
    project_dir: Path, name: str, result: ScaffoldResult,
) -> None:
    """Render and copy the bundled Sphinx template pack into ``docs/``.
 
    Silently skips (with a ``messages`` note) when the bundled template
    directory is missing - typical for source checkouts that omit the
    templates pack.
    """
    func = "_copy_sphinx_templates"
    if not _SPHINX_TEMPLATES_DIR.is_dir():
        result.messages.append(
            f"  Warning: Sphinx templates not found at {_SPHINX_TEMPLATES_DIR}; skipping."
        )
        _dbg("SCAFFOLD", func, "templates missing; skipped", 3)
        return
 
    docs_dir = project_dir / "docs"
    static_dir = docs_dir / "_static"
 
    # conf.py
    conf_template = (_SPHINX_TEMPLATES_DIR / "conf.py.template").read_text(
        encoding="utf-8",
    )
    year = datetime.datetime.now().year
    conf_content = (
        conf_template
        .replace("{{PROJECT_NAME}}", name)
        .replace("{{YEAR}}", str(year))
        .replace("{{AUTHOR}}", "Option C Dev Team")
        .replace("{{VERSION}}", "0.1.0")
    )
    (docs_dir / "conf.py").write_text(conf_content, encoding="utf-8")
    result.created.append(docs_dir / "conf.py")
 
    # index.rst
    idx_template = (_SPHINX_TEMPLATES_DIR / "index.rst.template").read_text(
        encoding="utf-8",
    )
    title = f"{name} Documentation"
    underline = "=" * len(title)
    idx_content = (
        idx_template
        .replace("{{PROJECT_NAME}}", name)
        .replace("{{TITLE_UNDERLINE}}", underline)
    )
    (docs_dir / "index.rst").write_text(idx_content, encoding="utf-8")
    result.created.append(docs_dir / "index.rst")
 
    # Static files.
    for static_file in ("custom.css", "Option_C_logo.svg"):
        src = _SPHINX_TEMPLATES_DIR / "_static" / static_file
        if src.exists():
            shutil.copy2(src, static_dir / static_file)
            result.created.append(static_dir / static_file)
 
    # Build files.
    for build_file in ("Makefile", "make.bat"):
        src = _SPHINX_TEMPLATES_DIR / build_file
        if src.exists():
            shutil.copy2(src, docs_dir / build_file)
            result.created.append(docs_dir / build_file)
 
    result.messages.append("  Copied Sphinx docs template -> docs/")
    _dbg("SCAFFOLD", func, "templates copied", 3)
 
 
def _write_fresh_oc_status(project_dir: Path, result: ScaffoldResult) -> None:
    """FS-539: emit ``oc_status.json`` for a newly-scaffolded project.
 
    A fresh scaffold is ``stage="compliant"`` with every pillar enabled -
    the scaffolder writes the full Option C layout, so every pillar is
    satisfied on day zero. ``migrated_from`` is an empty dict because the
    project was born in the ``.option_c/`` layout rather than migrated.
    """
    func = "_write_fresh_oc_status"
    status_path = project_dir / ".option_c" / "oc_status.json"
    if status_path.exists():
        result.skipped.append(status_path)
        return
 
    try:
        from importlib.metadata import PackageNotFoundError, version
        try:
            oct_version = version("oct")
        except PackageNotFoundError:
            oct_version = "unknown"
    except ImportError:
        oct_version = "unknown"
 
    today = datetime.date.today().isoformat()
    pillars = {
        name: {"enabled": True, "since": today}
        for name in ("linter", "diagnostics", "git", "docs", "tests", "scaffold")
    }
    status = OcStatus(
        stage="compliant",
        pillars=pillars,
        migrated_from={},
        oct_version_at_migration=oct_version,
    )
    write_oc_status(project_dir, status)
    result.created.append(status_path)
    _dbg("SCAFFOLD", func, f"oc_status written stage=compliant version={oct_version}", 3)
 
 
def _create_venv(project_dir: Path, result: ScaffoldResult) -> None:
    """Create a ``.venv/`` at the project root using :mod:`venv`.
 
    Any OS-level failure is downgraded to a ``messages`` warning so the
    caller can continue - a failed venv should not abort scaffolding.
    """
    func = "_create_venv"
    venv_dir = project_dir / ".venv"
    if venv_dir.exists():
        result.skipped.append(venv_dir)
        result.messages.append(f"  .venv/ already exists at {venv_dir}")
        _dbg("SCAFFOLD", func, "venv already exists; skipped", 3)
        return
    try:
        builder = venv.EnvBuilder(with_pip=True, clear=False, upgrade_deps=False)
        builder.create(str(venv_dir))
    except (OSError, RuntimeError) as exc:
        result.messages.append(f"  Warning: venv creation failed: {exc}")
        _dbg("SCAFFOLD", func, f"venv create failed: {exc}", 1)
        return
    result.created.append(venv_dir)
    result.venv_path = venv_dir
    result.messages.append(f"  Created virtualenv -> {venv_dir}")
    _dbg("SCAFFOLD", func, f"venv created at {venv_dir}", 3)
 
 
_CLAUDE_SKILLS_TEMPLATE_DIR: Path = (
    Path(__file__).resolve().parents[1] / "templates" / "claude_skills"
)
 
 
def _copy_claude_skills(
    project_dir: Path, result: ScaffoldResult, *, dry_run: bool = False,
) -> None:
    """FS-541: copy the bundled Claude Code skills into the project."""
    func = "_copy_claude_skills"
    skills_dir = project_dir / ".claude" / "skills"
 
    if not _CLAUDE_SKILLS_TEMPLATE_DIR.is_dir():
        result.messages.append(
            "  Warning: Claude skills templates not found; skipping --with-claude."
        )
        _dbg("SCAFFOLD", func, "skills templates missing", 3)
        return
 
    if dry_run:
        result.messages.append(
            f"[DRY RUN] Would copy Claude skills to {skills_dir}"
        )
        return
 
    skills_dir.mkdir(parents=True, exist_ok=True)
    for skill_subdir in _CLAUDE_SKILLS_TEMPLATE_DIR.iterdir():
        if not skill_subdir.is_dir():
            continue
        dest_skill = skills_dir / skill_subdir.name
        dest_skill.mkdir(exist_ok=True)
        for f in skill_subdir.iterdir():
            dest_file = dest_skill / f.name
            if dest_file.exists():
                result.skipped.append(dest_file)
                continue
            shutil.copy2(f, dest_file)
            result.created.append(dest_file)
 
    result.messages.append(f"  Copied Claude skills -> {skills_dir}")
    _dbg("SCAFFOLD", func, f"skills copied to {skills_dir}", 3)