oct
Loading...
Searching...
No Matches
source_exporter.py
Go to the documentation of this file.
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# oct/tools/source_exporter.py
4
5"""
6Purpose
7-------
8Provide a unified, Option C‑compliant source‑exporting tool that collects,
9combines, and consolidates all source‑code files in a project into structured
10inspection files. This enables fast review, debugging, auditing, and
11cross‑project comparison.
12
13Responsibilities
14----------------
15- Walk a project directory tree while respecting exclusion rules.
16- Collect all source files with supported extensions.
17- Generate per‑directory `_source.<path>.txt` files.
18- Generate a root‑level `_full_source.<root>.txt` file.
19- Support `--clean` to remove all `_source_code-*` directories.
20- Remove export and cache directories for real: clear the read-only
21 attribute on anything that refuses, count only what is gone, and name
22 what could not be removed (2026-09-07; `oct clean` used to report
23 "Removed N" while every read-only directory survived).
24- Support `--verbose` to print per‑file statistics.
25- Integrate cleanly with the OCT CLI as `oct export-source`.
26- Handle I/O errors (including Windows MAX_PATH) gracefully: warn and continue.
27
28Diagnostics
29-----------
30Domain: OCT-EXPORTER
31Levels:
32 L2 - lifecycle
33 L3 - semantic details
34 L4 - deep tracing
35
36Contracts
37---------
38- Must not modify project files except inside `_source_code-*` directories.
39- Must accept any directory as a valid export root.
40- Must remain deterministic and safe for large projects.
41- Must not require external dependencies beyond the standard library.
42- Must never crash on a recoverable I/O error; warn and continue instead.
43
44Dependencies
45------------
46- oct.core.exclusions (the shared exclusion rules)
47- fnmatch (stdlib glob matching for exclusion patterns)
48- hashlib (stdlib digest for the MAX_PATH-safe output names)
49- json (stdlib config parsing)
50- os (stdlib directory walk)
51- re (stdlib shallow secret guard)
52- sys (stdlib interpreter version and streams)
53- datetime (stdlib export timestamp)
54- pathlib (stdlib paths)
55- oct.core.diagnostics (_dbg structured logger)
56- oct.tools.fs_remove (remove_tree_force for read-only-safe directory removal; re-exported)
57- oct.core.git (lazy; git_diff_name_only for the --diff filter)
58- oct.tools.secret_scanner (lazy; the shared secret-name heuristics for the content guard)
59"""
60
61import fnmatch
62import hashlib
63import json
64import os
65import re
66import sys
67from datetime import datetime
68from pathlib import Path
69from typing import Callable, Dict, Iterator, List, Tuple
70
71
72# ============================================================
73# OI-416: content-level secret scanning
74# ============================================================
75
76# Compiled once; matches ``name = "value"`` / ``name: "value"`` style
77# assignments on a single line. Multi-line heredocs and concatenation are
78# intentionally out of scope - this is a "shallow guard" before upload.
79_SECRET_ASSIGN_RE = re.compile(
80 # OCT-LINT: disable=no_hardcoded_secrets reason="identifier-and-quoted-value regex pattern, not a credential (operator-ruled 2026-09-07)"
81 r'(?P<name>[A-Za-z_][A-Za-z0-9_]*)\s*[:=]\s*'
82 r'(?:"(?P<dq>[^"\n]{4,})"|\'(?P<sq>[^\'\n]{4,})\')'
83)
84
85
86def _scan_content_for_secrets(content: str, rel_path: str) -> list[str]:
87 """Scan ``content`` for likely hardcoded secrets. Returns warning strings.
88
89 Uses the same name-heuristics as the linter (``_SECRET_NAME_PATTERNS``).
90 Empty list means no warnings. The ``rel_path`` is embedded in each
91 warning for user attribution.
92 """
93 try:
94 # OI-514: scanner moved to a standalone module; source_exporter no
95 # longer pulls in the whole linter just to reach the heuristics.
96 from oct.tools.secret_scanner import _name_matches_secret
97 except Exception:
98 return []
99
100 warnings: list[str] = []
101 for lineno, line in enumerate(content.splitlines(), 1):
102 for m in _SECRET_ASSIGN_RE.finditer(line):
103 name = m.group("name")
104 if _name_matches_secret(name):
105 warnings.append(
106 f"{rel_path}:{lineno}: potential secret '{name}'"
107 )
108 break # one warning per line is enough
109 return warnings
110
111
112# Cache of per-file secret scan results, keyed by absolute path. Used to
113# dedup warnings between the per-directory and full-project passes while
114# still allowing both passes to make block decisions.
115_secret_scan_cache: dict[str, list[str]] = {}
116# Paths whose warnings have already been emitted to stderr this run.
117_scanned_secret_paths: set[str] = set()
118
119
120def _get_cached_secret_warnings(abs_path: str, content: str, rel_path: str) -> list[str]:
121 """Return (cached or computed) secret warnings for ``abs_path``."""
122 if abs_path not in _secret_scan_cache:
123 _secret_scan_cache[abs_path] = _scan_content_for_secrets(content, rel_path)
124 return _secret_scan_cache[abs_path]
125
126
127# ============================================================
128# Configuration
129# ============================================================
130
131_EXTENSION_PROFILES = {
132 "python": [".py"],
133 "web": [".js", ".ts", ".tsx", ".css", ".html"],
134 "jvm": [".java", ".kt"],
135 "systems": [".rs", ".go", ".cpp", ".c", ".h", ".hpp"],
136 "config": [".json", ".yaml", ".yml", ".toml", ".ini", ".xml"],
137 "docs": [".md"],
138 "scripts": [".sh", ".bat", ".ps1", ".ahk"],
139 "misc": [".npmignore", ".gitignore", ".sql", ".rb", ".swift"],
140}
141
142INCLUDE_EXTENSIONS = [ext for exts in _EXTENSION_PROFILES.values() for ext in exts]
143
144from oct.core.exclusions import EXPORTER_EXCLUDE_DIRS as EXCLUDE_DIRS
145from oct.core.exclusions import WILDCARD_EXCLUDE_DIRS
146from oct.core.exclusions import NEVER_EXPORT_FILE_PATTERNS
147from oct.core.diagnostics import _dbg
148# ``remove_tree_force`` moved to oct/tools/fs_remove.py when the docs pipeline
149# needed the same read-only-attribute survival (ENV-03). Imported by name, so
150# ``from oct.tools.source_exporter import remove_tree_force`` still resolves.
151from oct.tools.fs_remove import remove_tree_force
152
153HEADER_TEXT = (
154 # OCT-LINT: disable=no_hardcoded_secrets reason="prose header template with a format placeholder, not a credential (operator-ruled 2026-09-07)"
155 "This file contains all source-code files in the directory {directory} "
156 "(and its subdirectories) of this project.\n"
157 "Sections are separated as follows:\n"
158 "- '##### File name XX: <path>' marks individual files.\n"
159 "- '####### Subdirectory name XX: <dirname>' marks combined subdirectory sources.\n"
160 "- Root-level files use './<filename>' for clarity.\n\n"
161)
162
163SEPARATOR_FILE = "##### File name {num:02d}: {path}\n"
164SEPARATOR_SUBDIR = "####### Subdirectory name {num:02d}: {dirname}\n"
165ROOT_HEADER = "####### Root directory: .\n\n"
166
167# Windows MAX_PATH guard: keep well below the 260-character hard limit
168_MAX_SAFE_PATH_LEN = 240
169
170_CONFIG_FILENAME = "_oct_exporter_config.json"
171
172
173# ============================================================
174# Helpers
175# ============================================================
176
177def load_exporter_config(root: Path) -> dict:
178 """Load optional per-project config, merging with module-level defaults."""
179 func = "load_exporter_config"
180 _dbg("OCT-EXPORTER", func, f"loading exporter config for {root}", 4)
181 cfg = {
182 "include_extensions": list(INCLUDE_EXTENSIONS),
183 "exclude_dirs": set(EXCLUDE_DIRS),
184 "wildcard_exclude_dirs": list(WILDCARD_EXCLUDE_DIRS),
185 "header_text": HEADER_TEXT,
186 "separator_file": SEPARATOR_FILE,
187 "separator_subdir": SEPARATOR_SUBDIR,
188 "root_header": ROOT_HEADER,
189 "max_safe_path_len": _MAX_SAFE_PATH_LEN,
190 }
191
192 config_path = root / _CONFIG_FILENAME
193 if not config_path.is_file():
194 # Fall back to oct's own bundled config as the default.
195 _oct_root = Path(__file__).resolve().parent.parent.parent
196 fallback_path = _oct_root / _CONFIG_FILENAME
197 if fallback_path.is_file() and fallback_path != config_path:
198 print(f" Note: No {_CONFIG_FILENAME} in {root}; "
199 f"using oct default from {fallback_path}.",
200 file=sys.stderr)
201 config_path = fallback_path
202 else:
203 print(f" Warning: No {_CONFIG_FILENAME} found; using built-in defaults.",
204 file=sys.stderr)
205 return cfg
206
207 try:
208 raw = json.loads(config_path.read_text(encoding="utf-8"))
209 except json.JSONDecodeError as e:
210 print(f" Warning: {_CONFIG_FILENAME} has invalid JSON ({e}); using defaults.",
211 file=sys.stderr)
212 return cfg
213
214 if not isinstance(raw, dict):
215 print(f" Warning: {_CONFIG_FILENAME} root must be a JSON object; using defaults.",
216 file=sys.stderr)
217 return cfg
218
219 print(f"Using {_CONFIG_FILENAME}")
220
221 def _merge_collection(key, default, collection_type):
222 """Apply replace / add / remove semantics for a collection-type key."""
223 add_key = f"{key}_add"
224 remove_key = f"{key}_remove"
225
226 if key in raw:
227 result = collection_type(raw[key])
228 print(f" {key}: replaced -> {sorted(result)}")
229 else:
230 result = collection_type(default)
231
232 if add_key in raw:
233 added = raw[add_key]
234 if collection_type is set:
235 result = result | set(added)
236 else:
237 result = result + [v for v in added if v not in result]
238 print(f" {key}: added {added}")
239
240 if remove_key in raw:
241 removed = raw[remove_key]
242 removals = set(removed)
243 if collection_type is set:
244 result = result - removals
245 else:
246 result = [v for v in result if v not in removals]
247 print(f" {key}: removed {removed}")
248
249 return result
250
251 cfg["include_extensions"] = _merge_collection(
252 "include_extensions", INCLUDE_EXTENSIONS, list
253 )
254 cfg["exclude_dirs"] = _merge_collection(
255 "exclude_dirs", EXCLUDE_DIRS, set
256 )
257 cfg["wildcard_exclude_dirs"] = _merge_collection(
258 "wildcard_exclude_dirs", WILDCARD_EXCLUDE_DIRS, list
259 )
260
261 for scalar_key in ("header_text", "separator_file", "separator_subdir",
262 "root_header", "max_safe_path_len"):
263 if scalar_key in raw:
264 cfg[scalar_key] = raw[scalar_key]
265 print(f" {scalar_key}: overridden -> {raw[scalar_key]!r}")
266
267 return cfg
268
269
270def should_skip_dirname(dirname: str, cfg: dict) -> bool:
271 name = dirname.lower()
272 if name in cfg["exclude_dirs"]:
273 return True
274 return any(pattern in name for pattern in cfg["wildcard_exclude_dirs"])
275
276
277def _is_never_export_file(filename: str) -> bool:
278 """Check if a file matches NEVER_EXPORT_FILE_PATTERNS (§6b Secret Hygiene)."""
279 name_lower = filename.lower()
280 for pattern in NEVER_EXPORT_FILE_PATTERNS:
281 if fnmatch.fnmatch(name_lower, pattern.lower()):
282 return True
283 return False
284
285
286def collect_recursive_files(folder: Path, cfg: dict) -> List[Path]:
287 collected = []
288 for root, dirs, files in os.walk(folder):
289 root_path = Path(root)
290 if should_skip_dirname(root_path.name, cfg):
291 dirs[:] = []
292 continue
293 dirs[:] = [d for d in dirs if not should_skip_dirname(d, cfg)]
294 for f in files:
296 continue
297 if any(f.endswith(ext) for ext in cfg["include_extensions"]):
298 collected.append(root_path / f)
299 return sorted(collected)
300
301
302def get_file_stats(path: Path) -> Tuple[int, int, int]:
303 try:
304 content = path.read_text(encoding="utf-8", errors="replace")
305 except (OSError, UnicodeDecodeError):
306 print(f"Warning: Could not read {path}: skipping", file=sys.stderr)
307 return 0, 0, 0
308 lines = content.count("\n") + 1 if content else 0
309 words = len(content.split()) if content else 0
310 try:
311 bytes_ = path.stat().st_size
312 except OSError:
313 bytes_ = len(content.encode("utf-8"))
314 return lines, words, bytes_
315
316
317def walk_included_dirs(start: Path, cfg: dict) -> Iterator[Path]:
318 for root, dirs, _ in os.walk(start, topdown=True):
319 root_path = Path(root)
320 if should_skip_dirname(root_path.name, cfg):
321 dirs[:] = []
322 continue
323 dirs[:] = [d for d in dirs if not should_skip_dirname(d, cfg)]
324 yield root_path
325
326
328 root: Path, kind: str, matches: Callable[[str], bool], dry_run: bool = False,
329) -> Tuple[int, List[Path]]:
330 """Remove every directory under *root* whose name satisfies *matches*.
331
332 Returns ``(removed, failed)``: *removed* counts directories that are
333 gone (or would be, under *dry_run*), *failed* lists the ones still
334 present afterwards. Both are printed; a failure is never silent.
335 """
336 removed = 0
337 failed: List[Path] = []
338 for dirpath, dirs, _ in os.walk(root, topdown=True):
339 for d in list(dirs):
340 if not matches(d):
341 continue
342 target = Path(dirpath) / d
343 if dry_run:
344 print(f"[DRY RUN] Would remove: {target}")
345 removed += 1
346 elif remove_tree_force(target):
347 removed += 1
348 else:
349 failed.append(target)
350 dirs.remove(d)
351 if dry_run:
352 print(f"[DRY RUN] Would remove {removed} {kind} directories under {root}")
353 else:
354 print(f"Removed {removed} {kind} directories under {root}")
355 if failed:
356 print(f"Failed to remove {len(failed)} {kind} directories (still present):")
357 for target in failed:
358 print(f" {target}")
359 return removed, failed
360
361
362def clean_source_dirs(root: Path, dry_run: bool = False) -> Tuple[int, List[Path]]:
363 """Remove all ``_source_code-*`` directories under *root*; see :func:`clean_matching_dirs`."""
364 return clean_matching_dirs(
365 root, "_source_code-*", lambda d: d.startswith("_source_code-"), dry_run,
366 )
367
368
369def clean_pycache_dirs(root: Path, dry_run: bool = False) -> Tuple[int, List[Path]]:
370 """Remove all ``__pycache__`` directories under *root*; see :func:`clean_matching_dirs`."""
371 return clean_matching_dirs(root, "__pycache__", lambda d: d == "__pycache__", dry_run)
372
373
374def _make_safe_output_path(out_dir: Path, dotpath: str, cfg: dict,
375 prefix: str = "_source") -> Path:
376 """
377 Build the output file path, truncating the dotpath if the full path would
378 exceed the Windows MAX_PATH limit (~260 chars). A short MD5 hash suffix is
379 appended when truncation is applied to ensure uniqueness.
380 """
381 max_len = cfg["max_safe_path_len"]
382 filename = f"{prefix}.{dotpath}.txt"
383 candidate = out_dir / filename
384 if len(str(candidate)) <= max_len:
385 return candidate
386 h = hashlib.md5(dotpath.encode()).hexdigest()[:8]
387 overhead = len(str(out_dir)) + len(f"{prefix}....{h}.txt") + 2 # 2 for sep + slack
388 max_dotpath_len = max_len - overhead
389 truncated = dotpath[:max(0, max_dotpath_len)].rstrip(".")
390 return out_dir / f"{prefix}.{truncated}...{h}.txt"
391
392
393# ============================================================
394# Core exporter logic
395# ============================================================
396
397def write_source_file_for_dir(folder: Path, output_path: Path, root: Path,
398 cfg: dict, warn_secrets: bool = False,
399 block_secrets: bool = False,
400 diff_filter: set[Path] | None = None,
401 ) -> Tuple[int, int, int, int, list]:
402 files = collect_recursive_files(folder, cfg)
403 if diff_filter is not None:
404 files = [f for f in files if f.resolve() in diff_filter]
405 if not files:
406 return 0, 0, 0, 0, []
407
408 file_count = 0
409 total_lines = 0
410 total_words = 0
411 total_bytes = 0
412
413 try:
414 fh = output_path.open("w", encoding="utf-8")
415 except OSError as e:
416 print(f" Warning: cannot write '{output_path.name}': {e}")
417 return 0, 0, 0, 0, files
418
419 with fh as out:
420 out.write(cfg["header_text"].format(directory=folder))
421 out.write(cfg["root_header"])
422
423 for idx, file_path in enumerate(files, start=1):
424 rel_path = file_path.relative_to(folder)
425
426 try:
427 content = file_path.read_text(encoding="utf-8", errors="replace")
428 except (OSError, UnicodeDecodeError):
429 print(f"Warning: Could not read {file_path}: skipping", file=sys.stderr)
430 content = ""
431
432 # OI-416: content-level secret scanning
433 if warn_secrets and content:
434 abs_key = str(file_path.resolve())
435 rel_from_root = (
436 file_path.relative_to(root)
437 if file_path.is_relative_to(root) else rel_path
438 )
439 sec_warnings = _get_cached_secret_warnings(
440 abs_key, content, str(rel_from_root)
441 )
442 if abs_key not in _scanned_secret_paths:
443 _scanned_secret_paths.add(abs_key)
444 for w in sec_warnings:
445 print(f"Warning: {w}", file=sys.stderr)
446 if block_secrets and sec_warnings:
447 print(
448 f" Skipping {rel_from_root} due to potential secrets",
449 file=sys.stderr,
450 )
451 continue
452
453 out.write(cfg["separator_file"].format(num=idx, path=str(rel_path)))
454 out.write(content)
455 out.write("\n\n")
456
457 file_count += 1
458 total_bytes += file_path.stat().st_size
459 if content:
460 total_lines += content.count("\n") + 1
461 total_words += len(content.split())
462
463 return file_count, total_lines, total_words, total_bytes, files
464
465
466# ============================================================
467# Public entry point used by OCT CLI
468# ============================================================
469
470def run_exporter(root: Path, args: list[str]) -> None:
471 """
472 Execute the source exporter with the given root directory and arguments.
473 """
474 func = "run_exporter"
475 _dbg("OCT-EXPORTER", func, f"source export starting at {root}", 3)
476 do_clean = "--clean" in args
477 verbose = "--verbose" in args
478 single_dir_mode = "--single-dir" in args
479 warn_secrets = "--warn-secrets" in args or "--block-secrets" in args
480 block_secrets = "--block-secrets" in args
481
482 # Phase 4E - G-E6: --diff REF support.
483 diff_ref: str | None = None
484 for arg in args:
485 if arg.startswith("--diff="):
486 diff_ref = arg.split("=", 1)[1]
487 break
488
489 # OI-416: reset scan caches at the start of each run so repeated
490 # invocations (e.g. from tests) emit warnings independently.
491 _scanned_secret_paths.clear()
492 _secret_scan_cache.clear()
493
494 if do_clean:
496 return
497
498 # Pre-compute diff filter set when --diff is active.
499 diff_filter: set[Path] | None = None
500 if diff_ref:
501 try:
502 from oct.core.git import git_diff_name_only
503 diff_paths = git_diff_name_only(root, diff_ref)
504 diff_filter = set(diff_paths)
505 if not diff_filter:
506 print(f"No files changed since {diff_ref}. Nothing to export.", file=sys.stderr)
507 return
508 print(f"Exporting {len(diff_filter)} file(s) changed since {diff_ref}.", file=sys.stderr)
509 except Exception as exc:
510 print(f"Warning: --diff failed ({exc}). Exporting all files.", file=sys.stderr)
511
512 cfg = load_exporter_config(root)
513
514 timestamp = datetime.now().strftime("%Y%m%d-%H%M")
515 root_name = root.name
516
517 summary: Dict[str, Tuple[int, int, int, int]] = {}
518 per_dir_files: Dict[str, List[Path]] = {}
519 skipped: List[str] = []
520
521 # In single-dir mode, all output goes under one root-level directory
522 single_out_root: Path | None = None
523 if single_dir_mode:
524 single_out_root = root / f"_source_code-{timestamp}"
525 try:
526 single_out_root.mkdir(parents=True, exist_ok=True)
527 except OSError as e:
528 print(f" Error: cannot create single output dir: {e}")
529 return
530
531 # Per-directory exports
532 for folder in walk_included_dirs(root, cfg):
533 if should_skip_dirname(folder.name, cfg):
534 continue
535
536 if single_dir_mode:
537 rel_to_root = folder.relative_to(root) if folder != root else Path(".")
538 out_dir = single_out_root / rel_to_root
539 else:
540 out_dir = folder / f"_source_code-{timestamp}"
541
542 try:
543 out_dir.mkdir(parents=True, exist_ok=True)
544 except OSError as e:
545 rel_err = "." if folder == root else str(folder.relative_to(root))
546 print(f" Warning: cannot create output dir for '{folder.name}': {e}")
547 skipped.append(rel_err)
548 continue
549
550 rel = "." if folder == root else str(folder.relative_to(root))
551 dotpath = root_name if folder == root else f"{root_name}.{rel.replace(os.sep, '.')}"
552 out_path = _make_safe_output_path(out_dir, dotpath, cfg)
553
554 try:
555 file_count, total_lines, total_words, total_bytes, files = write_source_file_for_dir(
556 folder, out_path, root, cfg,
557 warn_secrets=warn_secrets, block_secrets=block_secrets,
558 diff_filter=diff_filter,
559 )
560 except Exception as e:
561 print(f" Warning: export failed for '{rel}': {e}")
562 skipped.append(rel)
563 continue
564
565 summary[rel] = (file_count, total_lines, total_words, total_bytes)
566 per_dir_files[rel] = files
567
568 # Full project export
569 if single_dir_mode:
570 root_out_dir = single_out_root
571 else:
572 root_out_dir = root / f"_source_code-{timestamp}"
573 full_output_path = _make_safe_output_path(root_out_dir, root_name, cfg,
574 prefix="_full_source")
575
576 all_files = collect_recursive_files(root, cfg)
577 if diff_filter is not None:
578 all_files = [f for f in all_files if f.resolve() in diff_filter]
579 grouped: Dict[str, List[Path]] = {}
580 for p in all_files:
581 parent_rel = p.parent.relative_to(root)
582 key = "." if str(parent_rel) == "." else str(parent_rel)
583 grouped.setdefault(key, []).append(p)
584
585 try:
586 fh = full_output_path.open("w", encoding="utf-8")
587 except OSError as e:
588 print(f" Warning: cannot write full-source file: {e}")
589 fh = None
590
591 if fh is not None:
592 with fh as out:
593 out.write(cfg["header_text"].format(directory=root))
594 out.write(cfg["root_header"])
595
596 for idx_dir, dirname in enumerate(sorted(grouped.keys()), start=1):
597 out.write(cfg["separator_subdir"].format(num=idx_dir, dirname=dirname))
598 for idx_file, file_path in enumerate(grouped[dirname], start=1):
599 rel_path = file_path.relative_to(root)
600 try:
601 content = file_path.read_text(encoding="utf-8", errors="replace")
602 except (OSError, UnicodeDecodeError):
603 print(f"Warning: Could not read {file_path}: skipping", file=sys.stderr)
604 content = ""
605
606 # OI-416: content-level secret scanning (dedup with per-dir pass).
607 if warn_secrets and content:
608 abs_key = str(file_path.resolve())
609 sec_warnings = _get_cached_secret_warnings(
610 abs_key, content, str(rel_path)
611 )
612 if abs_key not in _scanned_secret_paths:
613 _scanned_secret_paths.add(abs_key)
614 for w in sec_warnings:
615 print(f"Warning: {w}", file=sys.stderr)
616 if block_secrets and sec_warnings:
617 print(
618 f" Skipping {rel_path} due to potential secrets",
619 file=sys.stderr,
620 )
621 continue
622
623 out.write(cfg["separator_file"].format(num=idx_file, path=str(rel_path)))
624 out.write(content)
625 out.write("\n\n")
626
627 # Summary
628 print("\n========== SUMMARY ==========")
629
630 seen_files = set()
631
632 for dirname in sorted(summary.keys()):
633 files, lines, words, bytes_ = summary[dirname]
634 kb = bytes_ / 1024.0
635 print(f"Directory '{dirname}': {files} files, {lines} lines, {words} words, {kb:.1f} KB")
636
637 if verbose:
638 for p in per_dir_files.get(dirname, []):
639 if p not in seen_files:
640 seen_files.add(p)
641 flines, fwords, fbytes = get_file_stats(p)
642 fkb = fbytes / 1024.0
643 rel_path = p.relative_to(root)
644 print(f" {rel_path} - {flines} lines, {fwords} words, {fkb:.1f} KB")
645
646 # Grand total: each per-directory entry counts files recursively, so summing
647 # them all would multiply each file by the number of ancestor directories it
648 # has. Use the root entry instead - it already covers the full project.
649 # Fall back to counting unique files directly if root was skipped due to error.
650 if "." in summary:
651 total_files, total_lines, total_words, total_bytes = summary["."]
652 else:
653 total_files = total_lines = total_words = total_bytes = 0
654 for p in all_files:
655 flines, fwords, fbytes = get_file_stats(p)
656 total_files += 1
657 total_lines += flines
658 total_words += fwords
659 total_bytes += fbytes
660
661 print(f"\nTotal directories processed: {len(summary)}")
662 print(f"Total source files processed: {total_files}")
663 print(f"Total lines: {total_lines}")
664 print(f"Total words: {total_words}")
665 print(f"Total size: {total_bytes / 1024.0:.1f} KB")
666 if single_dir_mode:
667 print(f"Output stored in single directory: {single_out_root}")
668 else:
669 print(f"Output stored under _source_code-{timestamp} directories")
670
671 if skipped:
672 n = len(skipped)
673 print(f"\nSkipped {n} director{'y' if n == 1 else 'ies'} due to errors:")
674 for s in skipped:
675 print(f" - {s}")
676
677 print("=============================\n")
bool should_skip_dirname(str dirname, dict cfg)
list[str] _scan_content_for_secrets(str content, str rel_path)
List[Path] collect_recursive_files(Path folder, dict cfg)
Tuple[int, int, int] get_file_stats(Path path)
bool _is_never_export_file(str filename)
Tuple[int, int, int, int, list] write_source_file_for_dir(Path folder, Path output_path, Path root, dict cfg, bool warn_secrets=False, bool block_secrets=False, set[Path]|None diff_filter=None)
Iterator[Path] walk_included_dirs(Path start, dict cfg)
dict load_exporter_config(Path root)
Tuple[int, List[Path]] clean_pycache_dirs(Path root, bool dry_run=False)
Tuple[int, List[Path]] clean_source_dirs(Path root, bool dry_run=False)
list[str] _get_cached_secret_warnings(str abs_path, str content, str rel_path)
None run_exporter(Path root, list[str] args)
Path _make_safe_output_path(Path out_dir, str dotpath, dict cfg, str prefix="_source")
Tuple[int, List[Path]] clean_matching_dirs(Path root, str kind, Callable[[str], bool] matches, bool dry_run=False)