secret_scanner.py

Go to the documentation of this file.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# oct/tools/secret_scanner.py
 
"""
Purpose
-------
OI-514 - standalone hardcoded-secret detection. Extracted from
:mod:`oct.linter.oct_lint` so :mod:`oct.tools.source_exporter`,
:mod:`oct.git.quality_gate`, and any future AI review tools no longer
need to import the linter just to reach the scanner.
 
The linter continues to re-export the same names for backwards
compatibility - call sites that already import them from
``oct.linter.oct_lint`` keep working unchanged.
 
Responsibilities
----------------
- Host :data:`_SECRET_NAME_PATTERNS` - the canonical set of substrings
  that flag a variable/parameter/key name as "secret-shaped".
- Provide :func:`_name_matches_secret` for substring matching.
- Provide :func:`check_no_hardcoded_secrets` - the AST-based scanner.
- Provide :func:`_is_string_literal` / :func:`_is_string_like` helpers.
 
Diagnostics
-----------
Domain: SECRET-SCAN
Levels:
    L2 - scan lifecycle (start / end)
    L3 - pattern hits (per-line matches)
    L4 - AST walk details
 
Contracts
---------
- Must not import :mod:`oct.linter.oct_lint` (no circular dependency).
- Function signatures and return shapes are stable - the linter
  re-exports them, so any change here is a breaking change there.
- Name patterns are case-insensitive substring matches.
 
Dependencies
------------
- oct.core.parsing (safe_parse - parses the source when the caller passes no tree)
"""
 
# OCT-LINT: disable-file=dbg_import reason="Plan 263: the isolation table excludes the secret scanner from the sweep; operator-ruled waiver rather than an import edit (2026-09-14)"
# OCT-LINT: disable-file=dbg_usage reason="Plan 263: the isolation table excludes the secret scanner from the sweep; operator-ruled waiver rather than a _dbg edit (2026-09-14)"
 
from __future__ import annotations
 
import ast
import math
import re
from collections import Counter
 
from oct.core.parsing import safe_parse
 
 
# OI-529: word-segment mode splits names on delimiters + camelCase so
# false positives like ``tokenizer`` (contains ``token``) or
# ``pwdistance`` (contains ``pwd``) don't trigger. Pattern inserts an
# underscore between a lowercase/digit and a following uppercase
# (``apiKey`` → ``api_Key``) so compound patterns like ``api_key`` match
# via the shared regex below.
_CAMEL_SPLIT_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
 
 
# ---------------------------------------------------------------------
# Secret-indicating variable-name patterns (case-insensitive substring).
# ---------------------------------------------------------------------
 
_SECRET_NAME_PATTERNS = {
    "password", "passwd", "pwd",
    "secret", "secret_key", "private_key",
    "api_key", "apikey", "api_secret",
    "token", "auth_token", "access_token", "refresh_token",
    "connection_string", "conn_str", "dsn", "database_url", "db_url",
    "aws_secret", "aws_key", "aws_access",
    "credentials", "client_secret",
}
 
 
def _name_matches_secret(name: str, *, mode: str = "substring") -> bool:
    """Return True when ``name`` looks secret-shaped.
 
    OI-529: two matching strategies are supported.
 
    ``mode="substring"`` (default, back-compat) - any pattern that
    appears anywhere inside ``name.lower()`` flags the name. This is
    strict but noisy: ``tokenizer`` (contains ``token``), ``pwdistance``
    (contains ``pwd``), etc. all flag.
 
    ``mode="word"`` - camelCase boundaries are converted to underscores
    (``apiKey`` → ``api_Key``), the name is lowercased, and each pattern
    must appear delimited by start-of-string, end-of-string, ``_``, or
    ``-`` - so compound patterns like ``api_key`` still match when the
    original was ``apiKey`` or ``api-key``. Whole-segment hits still
    flag (``user_password``) but false positives like ``tokenizer`` and
    ``pwdistance`` pass clean.
    """
    lower = name.lower()
    if mode == "substring":
        return any(p in lower for p in _SECRET_NAME_PATTERNS)
    if mode == "word":
        normalised = _CAMEL_SPLIT_RE.sub("_", name).lower()
        for pattern in _SECRET_NAME_PATTERNS:
            regex = rf"(?:^|[_\-]){re.escape(pattern)}(?:$|[_\-])"
            if re.search(regex, normalised):
                return True
        return False
    raise ValueError(f"Unknown secret-match mode: {mode!r}")
 
 
def _is_string_literal(node: ast.expr) -> bool:
    """True when the AST node is a string constant (hardcoded literal)."""
    return isinstance(node, ast.Constant) and isinstance(node.value, str)
 
 
def _is_string_like(node: ast.expr) -> tuple[bool, str]:
    """Return ``(is_string_like, kind)`` for use by :func:`check_no_hardcoded_secrets`.
 
    ``kind`` is one of ``"literal"``, ``"f-string"``, or ``"concatenation"``
    (OI-408). Extended detection catches f-strings and ``"a" + "b"`` patterns
    in addition to plain string constants.
    """
    if isinstance(node, ast.Constant) and isinstance(node.value, str):
        return True, "literal"
    if isinstance(node, ast.JoinedStr):
        return True, "f-string"
    if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
        left_ok, _ = _is_string_like(node.left)
        right_ok, _ = _is_string_like(node.right)
        if left_ok or right_ok:
            return True, "concatenation"
    return False, ""
 
 
# ---------------------------------------------------------------------
# FS-507: Shannon entropy detection for high-entropy string literals
# ---------------------------------------------------------------------
 
_ENTROPY_DEFAULT_THRESHOLD: float = 4.5
_DOCSTRING_ENTROPY_DEFAULT: float = 5.0
_COMMENT_ENTROPY_DEFAULT: float = 5.0
_ENTROPY_MIN_LENGTH: int = 16
 
 
def _shannon_entropy(s: str) -> float:
    """Return the Shannon entropy (bits per character) of *s*.
 
    H = -sum(p * log2(p)) for each unique character's frequency p.
    Returns 0.0 for empty strings.
    """
    if not s:
        return 0.0
    length = len(s)
    counts = Counter(s)
    return -sum((c / length) * math.log2(c / length) for c in counts.values())
 
 
def _collect_docstring_node_ids(tree: ast.Module) -> set[int]:
    """Return ``id()`` values of ast.Constant nodes that are docstrings."""
    ids: set[int] = set()
    for node in ast.walk(tree):
        if isinstance(node, (ast.Module, ast.ClassDef,
                             ast.FunctionDef, ast.AsyncFunctionDef)):
            if (node.body
                    and isinstance(node.body[0], ast.Expr)
                    and isinstance(node.body[0].value, ast.Constant)
                    and isinstance(node.body[0].value.value, str)):
                ids.add(id(node.body[0].value))
    return ids
 
 
def _collect_keyword_excluded_node_ids(
    tree: ast.Module,
    field_patterns: list[str],
) -> set[int]:
    """Return ``id()`` values of ast.Constant nodes that are keyword
    arguments with names matching *field_patterns*.
 
    This identifies strings like ``correct_pattern="..."`` in
    ``RuleInfo(correct_pattern="...")`` calls - documentation/example
    fields in namedtuple/dataclass constructors.
    """
    if not field_patterns:
        return set()
    ids: set[int] = set()
    pattern_set = set(field_patterns)
    for node in ast.walk(tree):
        if isinstance(node, ast.Call):
            for kw in node.keywords:
                if (kw.arg in pattern_set
                        and isinstance(kw.value, ast.Constant)
                        and isinstance(kw.value.value, str)):
                    ids.add(id(kw.value))
    return ids
 
 
def _check_entropy_violations(
    tree: ast.Module,
    *,
    threshold: float = _ENTROPY_DEFAULT_THRESHOLD,
    docstring_threshold: float = _DOCSTRING_ENTROPY_DEFAULT,
    comment_threshold: float = _COMMENT_ENTROPY_DEFAULT,   # reserved
    min_length: int = _ENTROPY_MIN_LENGTH,
    excluded_node_ids: frozenset[int] = frozenset(),
) -> list[tuple[int, str, float]]:
    """Walk AST for string constants exceeding the entropy threshold.
 
    Returns a list of ``(lineno, value_preview, entropy_bits)`` tuples.
    Only strings longer than *min_length* are checked.  The preview is
    truncated to 24 characters for readability.
 
    Code strings use ``>=`` (catches at threshold).  Docstrings use
    strict ``>`` (must exceed threshold).  ``comment_threshold`` is
    accepted for forward compatibility but not yet consumed - comments
    do not appear in the AST.
 
    Nodes whose ``id()`` is in *excluded_node_ids* are skipped entirely
    (field-name exclusions from octrc ``entropy_exclusions``).
    """
    docstring_ids = _collect_docstring_node_ids(tree)
    violations: list[tuple[int, str, float]] = []
    for node in ast.walk(tree):
        if isinstance(node, ast.Constant) and isinstance(node.value, str):
            if id(node) in excluded_node_ids:
                continue
            value = node.value
            if len(value) < min_length:
                continue
            is_docstring = id(node) in docstring_ids
            effective_threshold = (
                docstring_threshold if is_docstring else threshold
            )
            entropy = _shannon_entropy(value)
            # Docstrings: strict > (must exceed threshold)
            # Code strings: >= (catches at threshold)
            if is_docstring:
                flagged = entropy > effective_threshold
            else:
                flagged = entropy >= effective_threshold
            if flagged:
                preview = value[:24] + ("..." if len(value) > 24 else "")
                violations.append((node.lineno, preview, entropy))
    return violations
 
 
def check_no_hardcoded_secrets(
    text: str,
    tree: ast.Module | None = None,
    *,
    mode: str = "substring",
    code_entropy_threshold: float = _ENTROPY_DEFAULT_THRESHOLD,
    docstring_entropy_threshold: float = _DOCSTRING_ENTROPY_DEFAULT,
    comment_entropy_threshold: float = _COMMENT_ENTROPY_DEFAULT,
    excluded_field_patterns: list[str] | None = None,
) -> tuple[bool, str]:
    """Detect string literals bound to secret-shaped names and high-entropy strings.
 
    Inspects three AST shapes for name-based detection:
 
    * Direct/annotated assignments - ``password = "literal"``.
    * Dict literals - ``{"api_key": "literal"}``.
    * Function default arguments - ``def f(token="literal"): ...``.
 
    Safe patterns are ignored - calls, subscripts, env lookups.
 
    FS-507: when *code_entropy_threshold* > 0, also flags any string literal
    whose Shannon entropy exceeds the threshold (default 4.5 bits/char).
    Code strings use ``>=``; docstrings use strict ``>``.
    Set to 0 to disable entropy checking.
 
    *docstring_entropy_threshold* (default 5.0) sets the bar for
    docstring constants.  *comment_entropy_threshold* (default 5.0) is
    accepted for forward compatibility but not yet consumed.
 
    OI-529: ``mode`` forwards to :func:`_name_matches_secret`. Default
    is ``"substring"`` (legacy behaviour). Pass ``mode="word"`` for
    segment-aware matching that avoids false positives like
    ``compass``/``tokenizer``.
 
    OI-540: *excluded_field_patterns* lists keyword-argument names
    (e.g. ``["correct_pattern", "example"]``) whose string values are
    exempt from entropy scanning - documentation fields in
    namedtuple/dataclass constructors.
 
    Returns ``(passed, detail_message)``; ``passed`` is False when at least
    one violation is found.
    """
    if tree is None:
        tree, _ = safe_parse(text)
    if tree is None:
        # Plan 176 (175 item 3, remedy A - fail-closed): an unparseable file
        # cannot be certified secret-free, so the scanner MUST NOT pass it.
        # This return becomes a normal no_hardcoded_secrets violation, flowing
        # into secrets_findings and tripping the always-on secrets gate
        # (exit 5, --force-proof). The message names the file as UNSCANNABLE,
        # not as containing a secret, so the operator is not misled into
        # hunting a nonexistent credential.
        return False, "file is unparseable; secret scan could not run (cannot certify secret-free)"
 
    violations: list[str] = []
 
    for node in ast.walk(tree):
        if isinstance(node, ast.Assign):
            for target in node.targets:
                if isinstance(target, ast.Name) and _name_matches_secret(target.id, mode=mode):
                    is_str, kind = _is_string_like(node.value)
                    if is_str:
                        violations.append(
                            f"L{node.lineno}: '{target.id}' assigned {kind}"
                            if kind != "literal"
                            else f"L{node.lineno}: '{target.id}' assigned string literal"
                        )
 
        if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
            if _name_matches_secret(node.target.id, mode=mode) and node.value is not None:
                is_str, kind = _is_string_like(node.value)
                if is_str:
                    violations.append(
                        f"L{node.lineno}: '{node.target.id}' assigned {kind}"
                        if kind != "literal"
                        else f"L{node.lineno}: '{node.target.id}' assigned string literal"
                    )
 
        if isinstance(node, ast.Dict):
            for key, value in zip(node.keys, node.values):
                if key is None:
                    continue
                if _is_string_literal(key) and _name_matches_secret(key.value, mode=mode):
                    is_str, kind = _is_string_like(value)
                    if is_str:
                        violations.append(
                            f"L{key.lineno}: dict key '{key.value}' mapped to {kind}"
                            if kind != "literal"
                            else f"L{key.lineno}: dict key '{key.value}' mapped to string literal"
                        )
 
        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
            args = node.args
            n_defaults = len(args.defaults)
            if n_defaults > 0:
                defaulted_args = args.args[-n_defaults:]
                for arg, default in zip(defaulted_args, args.defaults):
                    if _name_matches_secret(arg.arg, mode=mode):
                        is_str, kind = _is_string_like(default)
                        if is_str:
                            violations.append(
                                f"L{node.lineno}: parameter '{arg.arg}' has {kind} default"
                                if kind != "literal"
                                else f"L{node.lineno}: parameter '{arg.arg}' has string literal default"
                            )
            for arg, default in zip(args.kwonlyargs, args.kw_defaults):
                if default is not None and _name_matches_secret(arg.arg, mode=mode):
                    is_str, kind = _is_string_like(default)
                    if is_str:
                        violations.append(
                            f"L{node.lineno}: parameter '{arg.arg}' has {kind} default"
                            if kind != "literal"
                            else f"L{node.lineno}: parameter '{arg.arg}' has string literal default"
                        )
 
    # FS-507: entropy-based detection
    if code_entropy_threshold > 0 and tree is not None:
        excluded_ids = _collect_keyword_excluded_node_ids(
            tree, excluded_field_patterns or [],
        )
        for lineno, preview, entropy in _check_entropy_violations(
            tree,
            threshold=code_entropy_threshold,
            docstring_threshold=docstring_entropy_threshold,
            comment_threshold=comment_entropy_threshold,
            excluded_node_ids=frozenset(excluded_ids),
        ):
            violations.append(
                f"L{lineno}: high-entropy string ({entropy:.2f} bits/char): \"{preview}\""
            )
 
    if violations:
        return False, "; ".join(violations)
    return True, ""