178 """Load optional per-project config, merging with module-level defaults."""
179 func =
"load_exporter_config"
180 _dbg(
"OCT-EXPORTER", func, f
"loading exporter config for {root}", 4)
182 "include_extensions": list(INCLUDE_EXTENSIONS),
183 "exclude_dirs": set(EXCLUDE_DIRS),
184 "wildcard_exclude_dirs": list(WILDCARD_EXCLUDE_DIRS),
185 "header_text": HEADER_TEXT,
186 "separator_file": SEPARATOR_FILE,
187 "separator_subdir": SEPARATOR_SUBDIR,
188 "root_header": ROOT_HEADER,
189 "max_safe_path_len": _MAX_SAFE_PATH_LEN,
192 config_path = root / _CONFIG_FILENAME
193 if not config_path.is_file():
195 _oct_root = Path(__file__).resolve().parent.parent.parent
196 fallback_path = _oct_root / _CONFIG_FILENAME
197 if fallback_path.is_file()
and fallback_path != config_path:
198 print(f
" Note: No {_CONFIG_FILENAME} in {root}; "
199 f
"using oct default from {fallback_path}.",
201 config_path = fallback_path
203 print(f
" Warning: No {_CONFIG_FILENAME} found; using built-in defaults.",
208 raw = json.loads(config_path.read_text(encoding=
"utf-8"))
209 except json.JSONDecodeError
as e:
210 print(f
" Warning: {_CONFIG_FILENAME} has invalid JSON ({e}); using defaults.",
214 if not isinstance(raw, dict):
215 print(f
" Warning: {_CONFIG_FILENAME} root must be a JSON object; using defaults.",
219 print(f
"Using {_CONFIG_FILENAME}")
221 def _merge_collection(key, default, collection_type):
222 """Apply replace / add / remove semantics for a collection-type key."""
223 add_key = f
"{key}_add"
224 remove_key = f
"{key}_remove"
227 result = collection_type(raw[key])
228 print(f
" {key}: replaced -> {sorted(result)}")
230 result = collection_type(default)
234 if collection_type
is set:
235 result = result | set(added)
237 result = result + [v
for v
in added
if v
not in result]
238 print(f
" {key}: added {added}")
240 if remove_key
in raw:
241 removed = raw[remove_key]
242 removals = set(removed)
243 if collection_type
is set:
244 result = result - removals
246 result = [v
for v
in result
if v
not in removals]
247 print(f
" {key}: removed {removed}")
251 cfg[
"include_extensions"] = _merge_collection(
252 "include_extensions", INCLUDE_EXTENSIONS, list
254 cfg[
"exclude_dirs"] = _merge_collection(
255 "exclude_dirs", EXCLUDE_DIRS, set
257 cfg[
"wildcard_exclude_dirs"] = _merge_collection(
258 "wildcard_exclude_dirs", WILDCARD_EXCLUDE_DIRS, list
261 for scalar_key
in (
"header_text",
"separator_file",
"separator_subdir",
262 "root_header",
"max_safe_path_len"):
263 if scalar_key
in raw:
264 cfg[scalar_key] = raw[scalar_key]
265 print(f
" {scalar_key}: overridden -> {raw[scalar_key]!r}")
375 prefix: str =
"_source") -> Path:
377 Build the output file path, truncating the dotpath if the full path would
378 exceed the Windows MAX_PATH limit (~260 chars). A short MD5 hash suffix is
379 appended when truncation is applied to ensure uniqueness.
381 max_len = cfg[
"max_safe_path_len"]
382 filename = f
"{prefix}.{dotpath}.txt"
383 candidate = out_dir / filename
384 if len(str(candidate)) <= max_len:
386 h = hashlib.md5(dotpath.encode()).hexdigest()[:8]
387 overhead = len(str(out_dir)) + len(f
"{prefix}....{h}.txt") + 2
388 max_dotpath_len = max_len - overhead
389 truncated = dotpath[:max(0, max_dotpath_len)].rstrip(
".")
390 return out_dir / f
"{prefix}.{truncated}...{h}.txt"
398 cfg: dict, warn_secrets: bool =
False,
399 block_secrets: bool =
False,
400 diff_filter: set[Path] |
None =
None,
401 ) -> Tuple[int, int, int, int, list]:
403 if diff_filter
is not None:
404 files = [f
for f
in files
if f.resolve()
in diff_filter]
406 return 0, 0, 0, 0, []
414 fh = output_path.open(
"w", encoding=
"utf-8")
416 print(f
" Warning: cannot write '{output_path.name}': {e}")
417 return 0, 0, 0, 0, files
420 out.write(cfg[
"header_text"].format(directory=folder))
421 out.write(cfg[
"root_header"])
423 for idx, file_path
in enumerate(files, start=1):
424 rel_path = file_path.relative_to(folder)
427 content = file_path.read_text(encoding=
"utf-8", errors=
"replace")
428 except (OSError, UnicodeDecodeError):
429 print(f
"Warning: Could not read {file_path}: skipping", file=sys.stderr)
433 if warn_secrets
and content:
434 abs_key = str(file_path.resolve())
436 file_path.relative_to(root)
437 if file_path.is_relative_to(root)
else rel_path
440 abs_key, content, str(rel_from_root)
442 if abs_key
not in _scanned_secret_paths:
443 _scanned_secret_paths.add(abs_key)
444 for w
in sec_warnings:
445 print(f
"Warning: {w}", file=sys.stderr)
446 if block_secrets
and sec_warnings:
448 f
" Skipping {rel_from_root} due to potential secrets",
453 out.write(cfg[
"separator_file"].format(num=idx, path=str(rel_path)))
458 total_bytes += file_path.stat().st_size
460 total_lines += content.count(
"\n") + 1
461 total_words += len(content.split())
463 return file_count, total_lines, total_words, total_bytes, files
472 Execute the source exporter with the given root directory and arguments.
474 func =
"run_exporter"
475 _dbg(
"OCT-EXPORTER", func, f
"source export starting at {root}", 3)
476 do_clean =
"--clean" in args
477 verbose =
"--verbose" in args
478 single_dir_mode =
"--single-dir" in args
479 warn_secrets =
"--warn-secrets" in args
or "--block-secrets" in args
480 block_secrets =
"--block-secrets" in args
483 diff_ref: str |
None =
None
485 if arg.startswith(
"--diff="):
486 diff_ref = arg.split(
"=", 1)[1]
491 _scanned_secret_paths.clear()
492 _secret_scan_cache.clear()
499 diff_filter: set[Path] |
None =
None
502 from oct.core.git
import git_diff_name_only
503 diff_paths = git_diff_name_only(root, diff_ref)
504 diff_filter = set(diff_paths)
506 print(f
"No files changed since {diff_ref}. Nothing to export.", file=sys.stderr)
508 print(f
"Exporting {len(diff_filter)} file(s) changed since {diff_ref}.", file=sys.stderr)
509 except Exception
as exc:
510 print(f
"Warning: --diff failed ({exc}). Exporting all files.", file=sys.stderr)
514 timestamp = datetime.now().strftime(
"%Y%m%d-%H%M")
515 root_name = root.name
517 summary: Dict[str, Tuple[int, int, int, int]] = {}
518 per_dir_files: Dict[str, List[Path]] = {}
519 skipped: List[str] = []
522 single_out_root: Path |
None =
None
524 single_out_root = root / f
"_source_code-{timestamp}"
526 single_out_root.mkdir(parents=
True, exist_ok=
True)
528 print(f
" Error: cannot create single output dir: {e}")
537 rel_to_root = folder.relative_to(root)
if folder != root
else Path(
".")
538 out_dir = single_out_root / rel_to_root
540 out_dir = folder / f
"_source_code-{timestamp}"
543 out_dir.mkdir(parents=
True, exist_ok=
True)
545 rel_err =
"." if folder == root
else str(folder.relative_to(root))
546 print(f
" Warning: cannot create output dir for '{folder.name}': {e}")
547 skipped.append(rel_err)
550 rel =
"." if folder == root
else str(folder.relative_to(root))
551 dotpath = root_name
if folder == root
else f
"{root_name}.{rel.replace(os.sep, '.')}"
556 folder, out_path, root, cfg,
557 warn_secrets=warn_secrets, block_secrets=block_secrets,
558 diff_filter=diff_filter,
560 except Exception
as e:
561 print(f
" Warning: export failed for '{rel}': {e}")
565 summary[rel] = (file_count, total_lines, total_words, total_bytes)
566 per_dir_files[rel] = files
570 root_out_dir = single_out_root
572 root_out_dir = root / f
"_source_code-{timestamp}"
574 prefix=
"_full_source")
577 if diff_filter
is not None:
578 all_files = [f
for f
in all_files
if f.resolve()
in diff_filter]
579 grouped: Dict[str, List[Path]] = {}
581 parent_rel = p.parent.relative_to(root)
582 key =
"." if str(parent_rel) ==
"." else str(parent_rel)
583 grouped.setdefault(key, []).append(p)
586 fh = full_output_path.open(
"w", encoding=
"utf-8")
588 print(f
" Warning: cannot write full-source file: {e}")
593 out.write(cfg[
"header_text"].format(directory=root))
594 out.write(cfg[
"root_header"])
596 for idx_dir, dirname
in enumerate(sorted(grouped.keys()), start=1):
597 out.write(cfg[
"separator_subdir"].format(num=idx_dir, dirname=dirname))
598 for idx_file, file_path
in enumerate(grouped[dirname], start=1):
599 rel_path = file_path.relative_to(root)
601 content = file_path.read_text(encoding=
"utf-8", errors=
"replace")
602 except (OSError, UnicodeDecodeError):
603 print(f
"Warning: Could not read {file_path}: skipping", file=sys.stderr)
607 if warn_secrets
and content:
608 abs_key = str(file_path.resolve())
610 abs_key, content, str(rel_path)
612 if abs_key
not in _scanned_secret_paths:
613 _scanned_secret_paths.add(abs_key)
614 for w
in sec_warnings:
615 print(f
"Warning: {w}", file=sys.stderr)
616 if block_secrets
and sec_warnings:
618 f
" Skipping {rel_path} due to potential secrets",
623 out.write(cfg[
"separator_file"].format(num=idx_file, path=str(rel_path)))
628 print(
"\n========== SUMMARY ==========")
632 for dirname
in sorted(summary.keys()):
633 files, lines, words, bytes_ = summary[dirname]
635 print(f
"Directory '{dirname}': {files} files, {lines} lines, {words} words, {kb:.1f} KB")
638 for p
in per_dir_files.get(dirname, []):
639 if p
not in seen_files:
642 fkb = fbytes / 1024.0
643 rel_path = p.relative_to(root)
644 print(f
" {rel_path} - {flines} lines, {fwords} words, {fkb:.1f} KB")
651 total_files, total_lines, total_words, total_bytes = summary[
"."]
653 total_files = total_lines = total_words = total_bytes = 0
657 total_lines += flines
658 total_words += fwords
659 total_bytes += fbytes
661 print(f
"\nTotal directories processed: {len(summary)}")
662 print(f
"Total source files processed: {total_files}")
663 print(f
"Total lines: {total_lines}")
664 print(f
"Total words: {total_words}")
665 print(f
"Total size: {total_bytes / 1024.0:.1f} KB")
667 print(f
"Output stored in single directory: {single_out_root}")
669 print(f
"Output stored under _source_code-{timestamp} directories")
673 print(f
"\nSkipped {n} director{'y' if n == 1 else 'ies'} due to errors:")
677 print(
"=============================\n")
Tuple[int, int, int, int, list] write_source_file_for_dir(Path folder, Path output_path, Path root, dict cfg, bool warn_secrets=False, bool block_secrets=False, set[Path]|None diff_filter=None)