skeleton_exporter.py

Go to the documentation of this file.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# oct/tools/skeleton_exporter.py
 
"""
Purpose
-------
Export structural skeletons of source files for AI inspection, code review,
or project overview. Emits headers, module docstrings, and function/class
signatures without implementation bodies.
 
Responsibilities
----------------
- Extract Python file skeletons via AST: headers, docstrings, signatures.
- Produce one-line placeholders for non-Python files.
- Reuse the source exporter's config, exclusion, and output infrastructure.
- Support ``--no-diagnostics`` to omit Diagnostics docstring sections.
 
Diagnostics
-----------
Domain: OCT-SKELETON
Levels:
    L2 - lifecycle
    L3 - semantic details
    L4 - deep tracing
 
Contracts
---------
- All public helpers from ``source_exporter`` are reused, not duplicated.
- Skeleton output never contains implementation bodies.
- Non-Python files produce a single-line size summary.
 
Dependencies
------------
- oct.core.parsing (safe_parse)
- oct.tools.source_exporter (shared walk, filters and cleaner)
- ast (stdlib parser for the skeleton)
- os (stdlib directory walk)
- re (stdlib pattern matching)
- shutil (stdlib tree removal)
- sys (stdlib streams)
- datetime (stdlib export timestamp)
- pathlib (stdlib paths)
- oct.core.diagnostics (_dbg structured logger)
"""
 
from __future__ import annotations
 
import ast
import os
import re
import shutil
import sys
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Tuple
 
from oct.core.parsing import safe_parse
from oct.tools.source_exporter import (
    load_exporter_config,
    should_skip_dirname,
    _is_never_export_file,
    collect_recursive_files,
    walk_included_dirs,
    get_file_stats,
    _make_safe_output_path,
)
from oct.core.diagnostics import _dbg
 
 
# ============================================================
# Output formatting constants
# ============================================================
 
_HEADER_TEXT = (
    "This file contains structural skeletons (headers, docstrings, signatures) "
    "of all source files in the directory {directory}.\n"
    "Implementation bodies are omitted - only the public API surface is shown.\n\n"
)
 
_SEPARATOR_FILE = "##### File name {num:02d}: {path}\n"
_SEPARATOR_SUBDIR = "####### Subdirectory name {num:02d}: {dirname}\n"
_ROOT_HEADER = "####### Root directory: .\n\n"
 
 
# ============================================================
# Docstring section stripping
# ============================================================
 
def _strip_docstring_section(docstring: str, section_name: str) -> str:
    """Remove a named RST-style section from a docstring.
 
    Sections are identified by the pattern::
 
        SectionName
        -----------
        (content until next section header or end of docstring)
    """
    # Match: section header + underline + everything until the next section
    # header (non-blank line followed by dashes) or end of string.
    # Uses \S (any non-whitespace start) instead of [A-Za-z] to correctly
    # handle headers with numbers, unicode characters, or multi-word titles
    # starting with any printable character (OI-415).
    pattern = (
        rf"(?m)^{re.escape(section_name)}[ \t]*\n"
        r"-{3,}[ \t]*\n"
        r"(?:(?!^\S.*\n-{3,})[\s\S])*?"
        r"(?=^\S.*\n-{3,}|\Z)"
    )
    result = re.sub(pattern, "", docstring, flags=re.MULTILINE)
    result = re.sub(r"\n{3,}", "\n\n", result)
    return result.rstrip() + "\n"
 
 
# ============================================================
# AST-based skeleton extraction
# ============================================================
 
def _format_decorators(node: ast.AST, indent: str = "") -> str:
    """Reconstruct decorator lines from an AST node's decorator_list."""
    lines = []
    for dec in getattr(node, "decorator_list", []):
        lines.append(f"{indent}@{ast.unparse(dec)}")
    return "\n".join(lines)
 
 
def _format_function(node: ast.FunctionDef | ast.AsyncFunctionDef,
                     indent: str = "") -> str:
    """Reconstruct a function/method signature with decorators and docstring."""
    parts = []
 
    # Decorators
    decs = _format_decorators(node, indent)
    if decs:
        parts.append(decs)
 
    # Signature
    prefix = "async def" if isinstance(node, ast.AsyncFunctionDef) else "def"
    args_str = ast.unparse(node.args)
    ret = f" -> {ast.unparse(node.returns)}" if node.returns else ""
    parts.append(f"{indent}{prefix} {node.name}({args_str}){ret}:")
 
    # Docstring
    docstring = ast.get_docstring(node, clean=False)
    if docstring:
        parts.append(f'{indent}    """{docstring}"""')
 
    # Ellipsis body placeholder
    parts.append(f"{indent}    ...")
 
    return "\n".join(parts)
 
 
def _format_class(node: ast.ClassDef, indent: str = "",
                  *, no_diagnostics: bool = False) -> str:
    """Reconstruct a class definition with methods and nested classes."""
    parts = []
 
    # Decorators
    decs = _format_decorators(node, indent)
    if decs:
        parts.append(decs)
 
    # Class signature
    bases = ", ".join(ast.unparse(b) for b in node.bases)
    keywords = ", ".join(ast.unparse(kw) for kw in node.keywords)
    all_args = ", ".join(filter(None, [bases, keywords]))
    sig = f"{indent}class {node.name}({all_args}):" if all_args else f"{indent}class {node.name}:"
    parts.append(sig)
 
    # Class docstring
    docstring = ast.get_docstring(node, clean=False)
    if docstring:
        if no_diagnostics:
            docstring = _strip_docstring_section(docstring, "Diagnostics")
        parts.append(f'{indent}    """{docstring}"""')
 
    inner_indent = indent + "    "
    has_body = False
 
    for child in node.body:
        # Skip the docstring node (already handled above)
        if (isinstance(child, ast.Expr) and isinstance(child.value, ast.Constant)
                and isinstance(child.value.value, str) and not has_body
                and ast.get_docstring(node, clean=False)):
            has_body = True
            continue
 
        if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
            parts.append("")
            parts.append(_format_function(child, inner_indent))
            has_body = True
        elif isinstance(child, ast.ClassDef):
            parts.append("")
            parts.append(_format_class(child, inner_indent,
                                       no_diagnostics=no_diagnostics))
            has_body = True
        elif isinstance(child, ast.AnnAssign) and child.target:
            parts.append(f"{inner_indent}{ast.unparse(child)}")
            has_body = True
        elif isinstance(child, ast.Assign):
            # Include module-level / class-level constants (ALL_CAPS or _ALL_CAPS)
            names = [t.id for t in child.targets
                     if isinstance(t, ast.Name) and t.id.lstrip("_").isupper()]
            if names:
                parts.append(f"{inner_indent}{' = '.join(names)} = ...")
                has_body = True
 
    if not has_body:
        parts.append(f"{inner_indent}...")
 
    return "\n".join(parts)
 
 
def extract_skeleton(text: str, filepath: str, *,
                     no_diagnostics: bool = False) -> str:
    """Extract structural skeleton from a Python file's source text.
 
    Returns a string containing headers, module docstring, and
    class/function signatures - no implementation bodies.
    """
    lines = text.splitlines()
    parts: list[str] = []
 
    # Header block (first 3 lines)
    header_lines = lines[:3] if len(lines) >= 3 else lines[:]
    parts.append("\n".join(header_lines))
 
    # Parse AST
    tree, _warnings = safe_parse(text)
    if tree is None:
        parts.append("")
        parts.append("# [syntax error - file could not be parsed]")
        return "\n".join(parts) + "\n"
 
    # Module docstring
    docstring = ast.get_docstring(tree, clean=False)
    if docstring:
        if no_diagnostics:
            docstring = _strip_docstring_section(docstring, "Diagnostics")
        parts.append("")
        parts.append(f'"""{docstring}"""')
 
    # Walk top-level body
    for node in tree.body:
        # Skip the module docstring node
        if (isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant)
                and isinstance(node.value.value, str) and docstring):
            continue
 
        if isinstance(node, ast.ClassDef):
            parts.append("")
            parts.append(_format_class(node, no_diagnostics=no_diagnostics))
        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
            parts.append("")
            parts.append(_format_function(node))
        elif isinstance(node, ast.AnnAssign) and node.target:
            target_name = getattr(node.target, "id", "")
            if target_name.lstrip("_").isupper():
                parts.append("")
                parts.append(ast.unparse(node))
        elif isinstance(node, ast.Assign):
            names = [t.id for t in node.targets
                     if isinstance(t, ast.Name) and t.id.lstrip("_").isupper()]
            if names:
                parts.append("")
                parts.append(f"{' = '.join(names)} = ...")
 
    return "\n".join(parts) + "\n"
 
 
def _skeleton_for_non_python(filepath: Path) -> str:
    """Return a one-line placeholder for a non-Python file."""
    lines, _words, bytes_ = get_file_stats(filepath)
    kb = bytes_ / 1024.0
    return f"# [{filepath.name}] {lines} lines, {kb:.1f} KB\n"
 
 
# ============================================================
# Directory-level skeleton writing
# ============================================================
 
def write_skeleton_file_for_dir(folder: Path, output_path: Path, root: Path,
                                cfg: dict, *,
                                no_diagnostics: bool = False) -> tuple:
    """Write a skeleton export file for all source files in a directory."""
    files = collect_recursive_files(folder, cfg)
 
    file_count = 0
    total_lines = 0
    total_words = 0
    total_bytes = 0
 
    try:
        fh = output_path.open("w", encoding="utf-8")
    except OSError as e:
        print(f"  Warning: cannot write '{output_path.name}': {e}")
        return 0, 0, 0, 0, files
 
    with fh as out:
        out.write(_HEADER_TEXT.format(directory=folder))
        out.write(_ROOT_HEADER)
 
        for idx, file_path in enumerate(files, start=1):
            rel_path = file_path.relative_to(folder)
            out.write(_SEPARATOR_FILE.format(num=idx, path=str(rel_path)))
 
            if file_path.suffix == ".py":
                try:
                    text = file_path.read_text(encoding="utf-8", errors="replace")
                except (OSError, UnicodeDecodeError):
                    print(f"Warning: Could not read {file_path}: skipping",
                          file=sys.stderr)
                    text = ""
                skeleton = extract_skeleton(text, str(rel_path),
                                            no_diagnostics=no_diagnostics)
                out.write(skeleton)
            else:
                out.write(_skeleton_for_non_python(file_path))
 
            out.write("\n\n")
 
            file_count += 1
            total_bytes += file_path.stat().st_size
            try:
                content = file_path.read_text(encoding="utf-8", errors="replace")
                total_lines += content.count("\n") + 1
                total_words += len(content.split())
            except (OSError, UnicodeDecodeError):
                pass
 
    return file_count, total_lines, total_words, total_bytes, files
 
 
# ============================================================
# Cleanup
# ============================================================
 
def clean_skeleton_dirs(root: Path, dry_run: bool = False) -> Tuple[int, List[Path]]:
    """Remove all ``_skeleton_code-*`` directories under *root*.
 
    Returns ``(removed, failed)`` exactly like
    :func:`oct.tools.source_exporter.clean_matching_dirs`, which does the
    work (read-only directories cleared and retried; failures named).
    """
    func = "clean_skeleton_dirs"
    _dbg("OCT-SKELETON", func, f"cleaning skeleton dirs under {root}", 3)
    from oct.tools.source_exporter import clean_matching_dirs
    return clean_matching_dirs(
        root, "_skeleton_code-*", lambda d: d.startswith("_skeleton_code-"), dry_run,
    )
 
 
# ============================================================
# Public entry point used by OCT CLI
# ============================================================
 
def run_skeleton_exporter(
    root: Path,
    args: list[str],
    json_mode: bool = False,
) -> None:
    """Execute the skeleton exporter with the given root directory and arguments.
 
    When *json_mode* is True, emits a machine-readable JSON summary to stdout
    instead of the plain-text summary:
    ``{"tool": "export-skeleton", "directories_processed": N,
       "files_exported": N, "output_dirs": [...], "exit_code": 0}``
    """
    func = "run_skeleton_exporter"
    _dbg("OCT-SKELETON", func, f"skeleton export starting at {root}", 3)
    do_clean = "--clean" in args
    verbose = "--verbose" in args
    single_dir_mode = "--single-dir" in args
    no_diagnostics = "--no-diagnostics" in args
 
    if do_clean:
        clean_skeleton_dirs(root)
        return
 
    cfg = load_exporter_config(root)
 
    timestamp = datetime.now().strftime("%Y%m%d-%H%M")
    root_name = root.name
 
    summary: Dict[str, Tuple[int, int, int, int]] = {}
    per_dir_files: Dict[str, List[Path]] = {}
    skipped: List[str] = []
 
    single_out_root: Path | None = None
    if single_dir_mode:
        single_out_root = root / f"_skeleton_code-{timestamp}"
        try:
            single_out_root.mkdir(parents=True, exist_ok=True)
        except OSError as e:
            print(f"  Error: cannot create single output dir: {e}")
            return
 
    # Per-directory exports
    for folder in walk_included_dirs(root, cfg):
        if should_skip_dirname(folder.name, cfg):
            continue
 
        if single_dir_mode:
            rel_to_root = folder.relative_to(root) if folder != root else Path(".")
            out_dir = single_out_root / rel_to_root
        else:
            out_dir = folder / f"_skeleton_code-{timestamp}"
 
        try:
            out_dir.mkdir(parents=True, exist_ok=True)
        except OSError as e:
            rel_err = "." if folder == root else str(folder.relative_to(root))
            print(f"  Warning: cannot create output dir for '{folder.name}': {e}")
            skipped.append(rel_err)
            continue
 
        rel = "." if folder == root else str(folder.relative_to(root))
        dotpath = root_name if folder == root else f"{root_name}.{rel.replace(os.sep, '.')}"
        out_path = _make_safe_output_path(out_dir, dotpath, cfg,
                                          prefix="_skeleton")
 
        try:
            file_count, total_lines, total_words, total_bytes, files = (
                write_skeleton_file_for_dir(folder, out_path, root, cfg,
                                            no_diagnostics=no_diagnostics)
            )
        except Exception as e:
            print(f"  Warning: export failed for '{rel}': {e}")
            skipped.append(rel)
            continue
 
        summary[rel] = (file_count, total_lines, total_words, total_bytes)
        per_dir_files[rel] = files
 
    # Full project skeleton
    if single_dir_mode:
        root_out_dir = single_out_root
    else:
        root_out_dir = root / f"_skeleton_code-{timestamp}"
    full_output_path = _make_safe_output_path(root_out_dir, root_name, cfg,
                                              prefix="_full_skeleton")
 
    all_files = collect_recursive_files(root, cfg)
    grouped: Dict[str, List[Path]] = {}
    for p in all_files:
        parent_rel = p.parent.relative_to(root)
        key = "." if str(parent_rel) == "." else str(parent_rel)
        grouped.setdefault(key, []).append(p)
 
    try:
        fh = full_output_path.open("w", encoding="utf-8")
    except OSError as e:
        print(f"  Warning: cannot write full-skeleton file: {e}")
        fh = None
 
    if fh is not None:
        with fh as out:
            out.write(_HEADER_TEXT.format(directory=root))
            out.write(_ROOT_HEADER)
 
            for idx_dir, dirname in enumerate(sorted(grouped.keys()), start=1):
                out.write(_SEPARATOR_SUBDIR.format(num=idx_dir, dirname=dirname))
                for idx_file, file_path in enumerate(grouped[dirname], start=1):
                    rel_path = file_path.relative_to(root)
                    out.write(_SEPARATOR_FILE.format(num=idx_file,
                                                     path=str(rel_path)))
 
                    if file_path.suffix == ".py":
                        try:
                            text = file_path.read_text(encoding="utf-8",
                                                       errors="replace")
                        except (OSError, UnicodeDecodeError):
                            text = ""
                        out.write(extract_skeleton(text, str(rel_path),
                                                   no_diagnostics=no_diagnostics))
                    else:
                        out.write(_skeleton_for_non_python(file_path))
 
                    out.write("\n\n")
 
    # Summary
    if "." in summary:
        total_files, total_lines, total_words, total_bytes = summary["."]
    else:
        total_files = total_lines = total_words = total_bytes = 0
        for p in all_files:
            flines, fwords, fbytes = get_file_stats(p)
            total_files += 1
            total_lines += flines
            total_words += fwords
            total_bytes += fbytes
 
    if json_mode:
        import json as _json
        output_dirs: List[str] = []
        if single_dir_mode and single_out_root is not None:
            output_dirs = [str(single_out_root)]
        else:
            output_dirs = [
                str(root / f"_skeleton_code-{timestamp}"),
            ]
        print(_json.dumps({
            "tool": "export-skeleton",
            "directories_processed": len(summary),
            "files_exported": total_files,
            "total_lines": total_lines,
            "total_words": total_words,
            "total_bytes": total_bytes,
            "output_dirs": output_dirs,
            "skipped": skipped,
            "exit_code": 0,
        }))
    else:
        print(f"\n========== SKELETON SUMMARY ==========")
        seen_files_v: set[Path] = set()
        for dirname in sorted(summary.keys()):
            files_count, lines, words, bytes_ = summary[dirname]
            kb = bytes_ / 1024.0
            print(f"Directory '{dirname}': {files_count} files, "
                  f"{lines} lines, {words} words, {kb:.1f} KB")
            if verbose:
                for p in per_dir_files.get(dirname, []):
                    if p not in seen_files_v:
                        seen_files_v.add(p)
                        flines, fwords, fbytes = get_file_stats(p)
                        fkb = fbytes / 1024.0
                        rel_path = p.relative_to(root)
                        print(f"    {rel_path} -- {flines} lines, "
                              f"{fwords} words, {fkb:.1f} KB")
        print(f"\nTotal directories processed: {len(summary)}")
        print(f"Total source files processed: {total_files}")
        print(f"Total lines (original): {total_lines}")
        print(f"Total words (original): {total_words}")
        print(f"Total size (original): {total_bytes / 1024.0:.1f} KB")
        if single_dir_mode:
            print(f"Output stored in single directory: {single_out_root}")
        else:
            print(f"Output stored under _skeleton_code-{timestamp} directories")
        if skipped:
            n = len(skipped)
            print(f"\nSkipped {n} director{'y' if n == 1 else 'ies'} due to errors:")
            for s in skipped:
                print(f"  - {s}")
        print("======================================\n")