oct_deps.py

Go to the documentation of this file.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# oct/deps/oct_deps.py
 
"""
Purpose
-------
Analyse inter-module dependencies within an Option C project by parsing
the ``Dependencies`` docstring section from all Python files.
 
Responsibilities
----------------
- Walk the project tree and extract ``Dependencies`` sections from module
  docstrings. The section parsing (:func:`dependency_items`,
  :func:`entry_marker`) is shared with the ``dependencies_match`` lint
  rule in :mod:`oct.deps.dependency_check`, so both read an entry alike.
- Build a directed dependency graph (adjacency list).
- Detect circular dependencies via depth-first search, over the EAGER
  entries only: an entry whose purpose starts with ``lazy`` or ``not
  imported`` (operator ruling 2026-09-28) is drawn but never part of a
  cycle, because lazy imports are how the code breaks import cycles.
- Output the graph in multiple formats: terminal summary, JSON, Mermaid,
  and DOT.
 
Diagnostics
-----------
Domain: OCT-DEPS
Levels:
    L2 - lifecycle
    L3 - semantic details
    L4 - deep tracing
 
Contracts
---------
- Must not import project modules; analysis is purely static.
- Must exit 1 if circular dependencies are detected.
- Must handle missing or malformed Dependencies sections gracefully.
 
Dependencies
------------
- oct.core.exclusions (directory exclusion lists)
- oct.core.diagnostics (_dbg structured logger)
- tomli (lazy; TOML parsing for audit_pins on Python before 3.11)
"""
 
import ast
import json
import re
import sys
from dataclasses import dataclass
from pathlib import Path
 
from oct.core.exclusions import LINTER_EXCLUDE_DIRS, WILDCARD_EXCLUDE_DIRS
from oct.core.diagnostics import _dbg
 
 
# ---- Dependency extraction ----
 
_DEP_SECTION_RE = re.compile(
    r'Dependencies\s*\n\s*[-]+\s*\n(.*?)(?:\n\s*\n|\Z)',
    re.DOTALL,
)
_BULLET_RE = re.compile(r'^[ \t]*[-*][ \t]+(.*)$')
# The description may follow the module name after "-", ":" or "(". The
# repo-wide convention is "- module (what it provides)"; until 2026-09-27
# the "(" form was not accepted, so this matched zero lines in 331 oct
# modules, oct deps reported "0 dependencies" for every project, and no
# Sphinx dependencies page was ever generated. A bare name with no
# description still matches. Applied to one physical line at a time: the
# earlier multi-line form let a bare-name bullet swallow the next entry.
_DEP_NAME_RE = re.compile(r'^`?(\S+?)`?[ \t]*(?:[-—:(].*)?$')
# A marker is a whole word at the start of the purpose: "lazy;" counts,
# "lazy-evaluation helpers" does not.
# OCT-LINT: disable=no_hardcoded_secrets reason="Regex for the lazy and not-imported entry markers; no credential material (operator-ruled 2026-09-28)"
_MARKER_RE = re.compile(r'^(lazy|not imported)(?![\w-])')
 
 
@dataclass(frozen=True)
class SectionItem:
    """One item of a Dependencies section.
 
    *first_line* is the item's first physical line with the bullet marker
    stripped; *text* is that line with its continuation lines joined by
    single spaces, so a wrapped purpose reads as one.
    """
 
    bulleted: bool
    first_line: str
    text: str
 
 
def dependency_items(docstring: str | None) -> list[SectionItem] | None:
    """The items of *docstring*'s Dependencies section, or ``None`` if it has none.
 
    A non-bullet line after a bullet continues it; a non-bullet line before
    any bullet is an item of its own (``bulleted`` false).
    """
    m = _DEP_SECTION_RE.search(docstring or "")
    if not m:
        return None
    items: list[list] = []
    for line in m.group(1).splitlines():
        b = _BULLET_RE.match(line)
        if b:
            items.append([True, b.group(1).strip(), b.group(1).strip()])
        elif line.strip() and items:
            items[-1][2] += " " + line.strip()
        elif line.strip():
            items.append([False, line.strip(), line.strip()])
    return [SectionItem(*item) for item in items]
 
 
def entry_marker(purpose: str) -> str | None:
    """``"lazy"``, ``"not imported"`` or ``None`` for an entry's purpose text."""
    m = _MARKER_RE.match(purpose.strip().lstrip("`*_").lower())
    return m.group(1) if m else None
 
 
def dependency_entries(docstring: str | None) -> list[tuple[str, bool]]:
    """``(name, eager)`` for every bulleted entry of *docstring*'s Dependencies section.
 
    The name comes from the entry's first line; the marker from the joined
    text, so ``oct deps`` and the ``dependencies_match`` rule read a
    wrapped entry the same way. An entry marked ``lazy`` or ``not
    imported`` is not eager.
    """
    out: list[tuple[str, bool]] = []
    for item in dependency_items(docstring) or []:
        if not item.bulleted:
            continue
        m = _DEP_NAME_RE.match(item.first_line)
        if not m:
            continue
        purpose = item.text[m.end(1):].lstrip(" \t`-—:(")
        out.append((m.group(1), entry_marker(purpose) is None))
    return out
 
 
def _extract_module_docstring(source: str) -> str | None:
    """Extract the module-level docstring from Python source."""
    try:
        tree = ast.parse(source)
    except SyntaxError:
        return None
    if (tree.body
            and isinstance(tree.body[0], ast.Expr)
            and isinstance(tree.body[0].value, ast.Constant)
            and isinstance(tree.body[0].value.value, str)):
        return tree.body[0].value.value
    return None
 
 
def extract_dependencies(source: str, *, eager_only: bool = False) -> list[str]:
    """Parse the Dependencies section from a module docstring.
 
    Returns a list of dependency module names. With *eager_only*, entries
    marked ``lazy`` or ``not imported`` are left out (cycle detection).
    """
    return [name for name, eager in dependency_entries(_extract_module_docstring(source))
            if eager or not eager_only]
 
 
# ---- Graph building ----
 
def _should_skip(path: Path, root: Path) -> bool:
    """Check if a path should be excluded.
 
    The exclusion lists name DIRECTORIES, and are matched against the
    directory components only, as the linter's ``is_excluded_dir`` does.
    Until 2026-09-27 the file name was matched too, so the ``cache``,
    ``logs`` and ``venv`` wildcards silently dropped any module whose name
    merely started with one of them: ``oct.testing.cache``,
    ``oc_chat.core.cache_usage`` and three others were missing from
    ``oct deps``, every dependency overview page and every call graph.
    """
    rel_parts = path.relative_to(root).parts[:-1]
    for part in rel_parts:
        if part in LINTER_EXCLUDE_DIRS:
            return True
        for wc in WILDCARD_EXCLUDE_DIRS:
            if part.startswith(wc.rstrip("*")):
                return True
    return False
 
 
def module_name_for(py_file: Path, root: Path) -> str:
    """Compute a dotted import-path key for *py_file* relative to *root*.
 
    OI-521: using ``py_file.stem`` collided whenever two packages contain
    modules of the same name (e.g. ``a/util.py`` and ``b/util.py`` both
    reduced to ``util``). Returning the dotted import path makes each key
    unique and aligns with the dotted names typically used in
    ``Dependencies`` docstring sections.
 
    ``__init__.py`` resolves to its parent package's dotted name; a file
    directly under *root* keeps its stem as the fallback.
    """
    try:
        rel = py_file.relative_to(root).with_suffix("")
    except ValueError:
        return py_file.stem
    parts = list(rel.parts)
    if parts and parts[-1] == "__init__":
        parts = parts[:-1]
    return ".".join(parts) if parts else py_file.stem
 
 
def build_dependency_graphs(root: Path) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
    """Walk the project once; return ``(every entry, eager entries only)`` graphs.
 
    Keys are dotted import paths (OI-521), so modules sharing a stem
    across packages no longer overwrite one another.
    """
    func = "build_dependency_graphs"
    _dbg("OCT-DEPS", func, "building the dependency graph", 3)
    graph: dict[str, list[str]] = {}
    eager: dict[str, list[str]] = {}
 
    for py_file in sorted(root.rglob("*.py")):
        if _should_skip(py_file, root):
            continue
        try:
            source = py_file.read_text(encoding="utf-8")
        except (OSError, UnicodeDecodeError):
            continue
 
        module_name = module_name_for(py_file, root)
        entries = dependency_entries(_extract_module_docstring(source))
        if entries or module_name not in graph:
            graph[module_name] = [name for name, _ in entries]
            eager[module_name] = [name for name, is_eager in entries if is_eager]
 
    return graph, eager
 
 
def build_dependency_graph(root: Path, *, eager_only: bool = False) -> dict[str, list[str]]:
    """One graph of :func:`build_dependency_graphs`: every entry, or the eager ones."""
    graph, eager = build_dependency_graphs(root)
    return eager if eager_only else graph
 
 
# ---- Cycle detection ----
 
def detect_cycles(graph: dict[str, list[str]]) -> list[list[str]]:
    """Detect circular dependencies using DFS. Returns list of cycle paths."""
    func = "detect_cycles"
    _dbg("OCT-DEPS", func, "scanning the graph for cycles", 3)
    cycles: list[list[str]] = []
    visited: set[str] = set()
    rec_stack: set[str] = set()
 
    def _dfs(node: str, path: list[str]) -> None:
        visited.add(node)
        rec_stack.add(node)
        path.append(node)
 
        for dep in graph.get(node, []):
            if dep in rec_stack:
                cycle_start = path.index(dep)
                cycles.append(path[cycle_start:] + [dep])
            elif dep not in visited and dep in graph:
                _dfs(dep, path)
 
        path.pop()
        rec_stack.discard(node)
 
    for node in graph:
        if node not in visited:
            _dfs(node, [])
 
    return cycles
 
 
# ---- Output formatters ----
 
def to_json(graph: dict[str, list[str]], cycles: list[list[str]]) -> str:
    """Format dependency graph as JSON."""
    return json.dumps({
        "graph": graph,
        "cycles": cycles,
        "has_cycles": len(cycles) > 0,
    }, indent=2)
 
 
def to_mermaid(graph: dict[str, list[str]]) -> str:
    """Format dependency graph as Mermaid diagram syntax."""
    lines = ["graph TD"]
    for module, deps in sorted(graph.items()):
        for dep in deps:
            lines.append(f"    {module} --> {dep}")
    return "\n".join(lines)
 
 
def to_dot(graph: dict[str, list[str]]) -> str:
    """Format dependency graph as DOT (Graphviz) syntax."""
    lines = ["digraph dependencies {"]
    lines.append("    rankdir=LR;")
    for module, deps in sorted(graph.items()):
        for dep in deps:
            lines.append(f'    "{module}" -> "{dep}";')
    lines.append("}")
    return "\n".join(lines)
 
 
# ---- Main entry point ----
 
def audit_pins(root: Path, json_output: bool = False) -> int:
    """Audit pyproject.toml dependencies for unpinned version specifiers.
 
    Reads ``[project.dependencies]`` and
    ``[project.optional-dependencies]`` from ``pyproject.toml``. Reports
    dependencies that use open ranges (``>=``, ``>``) with no exact or
    upper-bound constraint alongside those that are strictly pinned.
 
    Parameters
    ----------
    root
        Project root containing ``pyproject.toml``.
    json_output
        If ``True``, prints a JSON summary. Otherwise prints a human-
        readable table.
 
    Returns
    -------
    Exit code: ``0`` if all dependencies have upper-bound or exact pins,
    ``1`` if any open-ended ``>=``-only or ``>``-only specs are found,
    ``2`` if ``pyproject.toml`` could not be read.
    """
    import json as _json
    import re
 
    toml_path = root / "pyproject.toml"
    if not toml_path.exists():
        msg = f"pyproject.toml not found at {toml_path}"
        if json_output:
            print(_json.dumps({"error": msg, "pinned": [], "open": [], "total": 0}))
        else:
            print(f"oct deps --audit-pins: {msg}", flush=True)
        return 2
 
    try:
        try:
            import tomllib  # Python 3.11+
        except ImportError:
            try:
                import tomli as tomllib  # type: ignore[no-redef]
            except ImportError:
                # Fallback: minimal TOML parser for dependencies list only.
                tomllib = None  # type: ignore[assignment]
 
        if tomllib is not None:
            data = tomllib.loads(toml_path.read_text(encoding="utf-8"))
        else:
            # Minimal fallback: parse pyproject.toml manually for deps lines.
            data = _parse_toml_deps_fallback(toml_path)
 
        project = data.get("project", {})
        all_deps: list[str] = list(project.get("dependencies", []))
        opt_deps = project.get("optional-dependencies", {})
        for extra_deps in opt_deps.values():
            all_deps.extend(extra_deps)
 
    except Exception as exc:
        msg = f"Failed to read pyproject.toml: {exc}"
        if json_output:
            print(_json.dumps({"error": msg, "pinned": [], "open": [], "total": 0}))
        else:
            print(f"oct deps --audit-pins: {msg}", flush=True)
        return 2
 
    # Classify each dep specifier.
    # "Open" = has >= or > with NO upper bound (no < or == in the specifier).
    # "Pinned" = has ==, ~=, <=, <, or a [version] marker that constrains upward.
    pinned: list[str] = []
    open_: list[str] = []
 
    for dep in all_deps:
        spec = dep.strip()
        if not spec:
            continue
        # Strip environment markers (e.g., "; python_version >= '3.10'")
        spec_no_marker = spec.split(";")[0].strip()
        # Check for exact pin or upper-bound constraint.
        has_upper = bool(re.search(r"(==|~=|<=|<)", spec_no_marker))
        has_lower_only = bool(re.search(r"[>]=?", spec_no_marker)) and not has_upper
        if has_lower_only:
            open_.append(spec)
        else:
            pinned.append(spec)
 
    if json_output:
        print(_json.dumps({
            "pinned": pinned,
            "open": open_,
            "total": len(pinned) + len(open_),
        }, indent=2))
    else:
        print(f"Dependencies audited: {len(pinned) + len(open_)} total")
        if open_:
            print(f"\n  Open (no upper bound) - {len(open_)} found:")
            for dep in open_:
                print(f"    {dep}")
        if pinned:
            print(f"\n  Pinned - {len(pinned)} OK")
        if not open_:
            print("\nAll dependencies have upper-bound or exact pins. OK")
 
    return 1 if open_ else 0
 
 
def _parse_toml_deps_fallback(toml_path: Path) -> dict:
    """Minimal fallback TOML parser - extracts dependency lines only."""
    import re
    data: dict = {"project": {"dependencies": [], "optional-dependencies": {}}}
    text = toml_path.read_text(encoding="utf-8")
    in_deps = False
    for line in text.splitlines():
        stripped = line.strip()
        if stripped == "dependencies = [":
            in_deps = True
            continue
        if in_deps:
            if stripped == "]":
                in_deps = False
                continue
            m = re.match(r'^["\'](.+)["\'],?$', stripped)
            if m:
                data["project"]["dependencies"].append(m.group(1))
    return data
 
 
def run_deps(root: Path, argv: list[str] | None = None) -> int:
    """Run dependency analysis on the given project root.
 
    Returns exit code: 0 if no cycles, 1 if circular dependencies found.
    """
    import argparse
 
    parser = argparse.ArgumentParser(prog="oct deps", add_help=False)
    parser.add_argument("--json", action="store_true", help="JSON output")
    parser.add_argument("--mermaid", action="store_true", help="Mermaid diagram output")
    parser.add_argument("--dot", action="store_true", help="DOT/Graphviz output")
    parser.add_argument(
        "--audit-pins",
        action="store_true",
        help=(
            "Audit pyproject.toml dependencies for unpinned version specifiers. "
            "Exit 1 if any open >= / >-only ranges are found."
        ),
    )
    args = parser.parse_args(argv or [])
 
    if args.audit_pins:
        return audit_pins(root, json_output=args.json)
 
    graph, eager = build_dependency_graphs(root)
    # Cycles over eager entries only: a lazy import is a deliberate break.
    cycles = detect_cycles(eager)
 
    if args.json:
        print(to_json(graph, cycles))
    elif args.mermaid:
        print(to_mermaid(graph))
    elif args.dot:
        print(to_dot(graph))
    else:
        # Terminal summary
        total_modules = len(graph)
        total_deps = sum(len(deps) for deps in graph.values())
        print(f"Dependency Analysis: {total_modules} modules, {total_deps} dependencies")
        if cycles:
            print(f"\nCircular dependencies detected ({len(cycles)}):")
            for cycle in cycles:
                print(f"  {' -> '.join(cycle)}")
        else:
            print("No circular dependencies found.")
 
        # Show modules with dependencies
        has_deps = {m: d for m, d in graph.items() if d}
        if has_deps:
            print(f"\nModules with declared dependencies ({len(has_deps)}):")
            for module, deps in sorted(has_deps.items()):
                print(f"  {module}: {', '.join(deps)}")
 
    return 1 if cycles else 0