oct
Loading...
Searching...
No Matches
parallel.py
Go to the documentation of this file.
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# oct/testing/parallel.py
4
5"""
6Purpose
7-------
8Resource-keyed test parallelism (Plan 239 / TEST-01). A pytest worker
9is a separate PROCESS, so in-process state is per-worker and safe;
10what collides across workers is shared EXTERNAL state (a registry
11database, a comms root, a tracked settings file, a bound port, writes
12outside ``tmp_path``). This module classifies every test file by the
13shared resources it touches, groups the conflicting files into the
14connected components of that conflict graph, and hands the grouping to
15pytest through xdist's ``--dist loadgroup``.
16
17Responsibilities
18----------------
19- Pin the CLOSED resource vocabulary (Plan 239 P2), its KEYED
20 refinement (Plan 247 P1: ``main_tree_write:<top>`` and
21 ``env_subprocess:<family>``) and the ``unknown:<detail>``
22 fail-closed escape hatch.
23- STATIC detector: an :mod:`ast` pass over each test file plus the
24 conftest fixtures it actually requests (:func:`analyse_source`,
25 :func:`classify_source`, :func:`classify_tree`).
26- DYNAMIC detector: the ``--oct-trace`` pytest plugin, which wraps
27 ``open`` / ``os.environ`` / ``socket.bind`` / ``subprocess`` /
28 ``os.chdir`` for the duration of each test and writes a per-process
29 trace (:func:`load_traces`).
30- Grouping and the committed manifest (:func:`build_groups`,
31 :func:`build_manifest`, :func:`load_manifest`,
32 :func:`resolve_groups`).
33- The pytest-side seams a root conftest calls
34 (:func:`resolve_worker_count`, :func:`configure_parallel_defaults`).
35- The regeneration CLI: ``python -m oct.testing.parallel classify
36 --root <subproject root>``.
37
38Diagnostics
39-----------
40Domain: OCT-TESTING
41Levels:
42 L2: lifecycle (classification start/end, tracer install/remove)
43 L3: semantic details (resolved cap, group counts)
44 L4: deep tracing (per-file resource sets)
45
46Contracts
47---------
48- A file's classification is the UNION of both detectors (P3):
49 neither alone is authoritative. The union is why BOTH detectors
50 must key what they can - one bare verdict re-widens a file the
51 other had already narrowed.
52- The vocabulary is CLOSED. A signature the detectors cannot map into
53 it is recorded as ``unknown:<detail>``, which conflicts with every
54 file that carries any resource at all (fail closed, P2/P4).
55- A KEY only ever NARROWS (Plan 247 P1). Two keyed tokens of one
56 family conflict when their keys are equal; a BARE family token
57 conflicts with every member of its family, so anything a detector
58 cannot attribute keeps precisely its pre-247 behaviour. Only
59 ``unknown:*`` is a GLOBAL wildcard; a bare family token is not.
60- The ENVIRONMENT is process-local for grouping (Plan 276 P1/P2). An
61 ``os.environ`` write is recorded as ``env_mutation`` and connects
62 the file to nothing: a worker is a separate PROCESS, so a write in
63 one can never reach a child another one spawns. ``env_subprocess``
64 is therefore recorded for a CHILD LAUNCH only, and a BARE launch is
65 the one family wildcard that holds just the other bare launches -
66 what a child does to genuinely shared state is what its
67 :data:`LAUNCH_FAMILIES` entry declares (``main_tree_write:*``), and
68 ``unknown:*`` still fails closed on everything else.
69- Manifest keys are ROOT-RELATIVE POSIX paths, identical to the path
70 part of a pytest nodeid. Every recorded value is machine
71 independent: the manifest is committed, so no absolute path, user
72 name or temp directory may ever reach it.
73- A conftest contributes its AUTOUSE fixtures' resources to every file
74 beneath it and its other fixtures' resources to the files that
75 request them by name. Module-level conftest code and conftest hook
76 functions are session-level uses that per-file grouping cannot
77 separate, so they are deliberately not attributed to files.
78- ``pytest`` is NEVER imported at module scope: it is an optional
79 (``[test]`` extra) dependency of oct, so the plugin hooks below are
80 plain functions that pytest discovers by name.
81- No ``sys.exit`` outside the ``__main__`` CLI block.
82
83Dependencies
84------------
85- oct.core.diagnostics (_dbg)
86- oct.testing.errors (the spec-10 §12.4 taxonomy as data)
87- oct.testing.manifest (the §9 workspace test blocks; ``max_workers``)
88- oct.tools.atomic_write (atomic_write_text for writing the committed manifest)
89- oct.core.workspace (lazy; find_workspace_root for the classify CLI)
90- oct (lazy; the oct version recorded in the manifest)
91- pytest (lazy; the xdist_group marker and UsageError in the plugin hooks)
92- xdist (not imported; the pytest-xdist plugin whose worker count and loadgroup scheduling this module configures)
93"""
94
95from __future__ import annotations
96
97import ast
98import builtins
99import functools
100import io
101import json
102import os
103import re
104import socket
105import subprocess
106import sys
107from dataclasses import dataclass
108from datetime import datetime, timezone
109from pathlib import Path, PurePosixPath
110from typing import Any, Callable, Iterable, Mapping, Sequence
111
112from oct.core.diagnostics import _dbg
113from oct.testing.errors import (
114 MANIFEST_TEST_BLOCK_INVALID,
115 WORKSPACE_FILE_NOT_FOUND,
116 OctTestError,
117)
118from oct.testing.manifest import load_test_manifest
119from oct.tools.atomic_write import atomic_write_text
120
121try: # pragma: no cover - exercised by the shadowed-install path only
122 from oct.testing.manifest import DEFAULT_MAX_WORKERS
123except ImportError: # pragma: no cover
124 # An INSTALLED oct predating the P1 key (a worktree running against
125 # an editable install rooted in another checkout). The literal is
126 # the same shipped default the parser applies.
127 DEFAULT_MAX_WORKERS = 24
128
129
130# ---------------------------------------------------------------------
131# P2 - the closed resource vocabulary
132# ---------------------------------------------------------------------
133
134#: The orchestration registry database (``agents.sqlite``).
135RES_REGISTRY_SQLITE: str = "registry_sqlite"
136
137#: The inter-agent comms root under ``.option_c/comms``.
138RES_COMMS_ROOT: str = "comms_root"
139
140#: The tracked runtime settings file.
141RES_SETTINGS_FILE: str = "settings_file"
142
143#: An ``octrc.json`` / ``.octrc.json`` tool-configuration file.
144RES_OCTRC: str = "octrc"
145
146#: A git worktree directory (creation, removal, or a write inside one).
147RES_WORKTREE_DIR: str = "worktree_dir"
148
149#: Any write outside ``tmp_path`` / ``tmp_path_factory`` and outside
150#: the test's own package temp. KEYED by the top-level directory the
151#: write lands in (P2): ``main_tree_write:docs`` and
152#: ``main_tree_write:.option_c`` cannot collide, so they do not group.
153RES_MAIN_TREE_WRITE: str = "main_tree_write"
154
155#: A CHILD LAUNCH, and nothing else (Plan 276 P1). A child inherits
156#: the environment, and what it does with it is invisible from here.
157#: A child is OPAQUE to both detectors unless its argv names a family
158#: the committed :data:`LAUNCH_FAMILIES` table vouches for, so a
159#: launch is KEYED by that family (P3) and BARE otherwise.
160RES_ENV_SUBPROCESS: str = "env_subprocess"
161
162#: A write to ``os.environ`` outside ``monkeypatch`` (Plan 276 P1).
163#: Recorded so the manifest keeps the full picture, but PROCESS-LOCAL:
164#: an xdist worker is a separate process, so a write in worker A can
165#: never reach a child worker B spawns. What CAN collide is what a
166#: child does to shared state, and the keyed launch families already
167#: declare that (``main_tree_write:*``) while ``unknown:*`` fails
168#: closed on the rest.
169RES_ENV_MUTATION: str = "env_mutation"
170
171#: Outbound network use.
172RES_NETWORK: str = "network"
173
174#: A process-global working-directory change (``os.chdir``).
175RES_CWD_CHANGE: str = "cwd_change"
176
177#: A call into a known process-wide reset seam (session trust, grants,
178#: runtime settings, the cooperative cancel flag, plan inventory).
179RES_GLOBAL_SINGLETON_RESET: str = "global_singleton_reset"
180
181#: Prefix of the parameterised port resource, ``port:<n>``.
182PORT_PREFIX: str = "port:"
183
184#: Prefix of the fail-closed escape hatch, ``unknown:<detail>``.
185UNKNOWN_PREFIX: str = "unknown:"
186
187#: Every non-parameterised member of the vocabulary.
188RESOURCE_VOCABULARY: tuple[str, ...] = (
189 RES_COMMS_ROOT,
190 RES_CWD_CHANGE,
191 RES_ENV_MUTATION,
192 RES_ENV_SUBPROCESS,
193 RES_GLOBAL_SINGLETON_RESET,
194 RES_MAIN_TREE_WRITE,
195 RES_NETWORK,
196 RES_OCTRC,
197 RES_REGISTRY_SQLITE,
198 RES_SETTINGS_FILE,
199 RES_WORKTREE_DIR,
200)
201
202#: Resources that are PROCESS-LOCAL: an xdist worker is a separate
203#: process, so two files that both touch one of these can never
204#: collide by running at the same time in different workers. They are
205#: recorded (the manifest keeps the full picture) but they do NOT
206#: connect files in the conflict graph.
207#:
208#: Grouping could not help them even if they did collide: a group
209#: pins its files to ONE worker, which prevents CONCURRENCY, not
210#: sequential adjacency - and adjacency is the only way a leaked
211#: cwd or singleton reaches another test. Measured on the oc_chat
212#: suite (2026-09-06): treating them as conflicting put 210 of 324
213#: files into a single group, i.e. no parallelism at all, because
214#: ``tests/unit/conftest.py`` resets the cooperative-cancel flag in
215#: an autouse fixture for all 184 unit files.
216#:
217#: :data:`RES_ENV_MUTATION` joined them under Plan 276 for the same
218#: reason: ``os.environ`` is per PROCESS, so a write in one worker is
219#: invisible to every other worker and to the children they spawn.
220#: Measured on the oc_chat suite (2026-09-15): treating an environment
221#: write as evidence of a cross-worker conflict was one half of a
222#: single 100-file group that held a quarter of the suite's tests.
223PROCESS_LOCAL_RESOURCES: frozenset[str] = frozenset({
224 RES_CWD_CHANGE,
225 RES_ENV_MUTATION,
226 RES_GLOBAL_SINGLETON_RESET,
227})
228
229
230#: Separator between a resource FAMILY and its key (P1).
231KEY_SEPARATOR: str = ":"
232
233#: The two families whose tokens may carry a key (Plan 247 P1). Both
234#: were single undifferentiated tokens through plan 239, and both are
235#: the reason the oc_chat conflict graph collapsed into one component
236#: of 92 files: fifty files that each write somewhere different, and
237#: fifty-eight that each launch something different, all conflicted.
238#:
239#: ``port:<n>`` and ``unknown:<detail>`` also carry a colon and are
240#: deliberately NOT here: reading them as family-plus-key would make
241#: ``unknown:a`` and ``unknown:b`` two non-conflicting resources, the
242#: exact opposite of the fail-closed rule they exist for.
243KEYED_FAMILIES: frozenset[str] = frozenset({
244 RES_MAIN_TREE_WRITE,
245 RES_ENV_SUBPROCESS,
246})
247
248#: Key of a traced write that landed ABOVE the package root and is not
249#: temporary (P2). It is a constant, never the real directory name,
250#: because the manifest that carries it is COMMITTED and no absolute
251#: path may reach it; the angle brackets are not legal in a path
252#: component, so it can never collide with a genuine top-level
253#: directory called ``outside``.
254OUTSIDE_KEY: str = "<outside>"
255
256
257def keyed(family: str, key: str) -> str:
258 """Build the keyed token ``<family>:<key>`` (P1).
259
260 An empty *key* yields the BARE family token: a caller that could
261 not attribute the use says so by producing the wildcard, never by
262 inventing a key.
263 """
264 if not key:
265 return family
266 return f"{family}{KEY_SEPARATOR}{key}"
267
268
269def resource_family(resource: str) -> str:
270 """The family of *resource*, or the token itself when not keyed."""
271 head, separator, _key = resource.partition(KEY_SEPARATOR)
272 if separator and head in KEYED_FAMILIES:
273 return head
274 return resource
275
276
277def resource_key(resource: str) -> str | None:
278 """The key *resource* carries, or None when it is BARE."""
279 head, separator, key = resource.partition(KEY_SEPARATOR)
280 if separator and head in KEYED_FAMILIES:
281 return key
282 return None
283
284
285def conflicts_with(left: str, right: str) -> bool:
286 """True when two resource tokens can collide (the P1 rule).
287
288 Equal keyed tokens conflict unless their launch family is known
289 to share nothing (Plan 251 P4). A BARE token still conflicts with
290 every member of its family, keyed or not - with ONE exception,
291 :data:`RES_ENV_SUBPROCESS` (Plan 276 P2): a bare launch conflicts
292 with other bare launches only.
293
294 That exception is not a relaxation of the fail-closed rule; it is
295 the rule applied to what an xdist worker actually is. A bare
296 ``main_tree_write`` means "a write to a directory this pass could
297 not read", and the directory it could not read may be any of the
298 keyed ones, so it must conflict with all of them. A bare
299 ``env_subprocess`` means "a child this pass could not name", and
300 the one thing a keyed sibling would share with it - the inherited
301 environment - is per PROCESS: a worker is a process, so that
302 inheritance never crosses one. What a child does to state that IS
303 shared is what its family declares through its own resources
304 (``main_tree_write:*``), and what ``unknown:*`` fails closed on.
305
306 Parameters
307 ----------
308 left, right
309 Resource tokens as the detectors record them.
310
311 Returns
312 -------
313 bool
314 Whether a file carrying *left* may run beside one carrying
315 *right*.
316 """
317 if left == right:
318 return is_conflicting(left)
319 family = resource_family(left)
320 if family != resource_family(right):
321 return False
322 if family == RES_ENV_SUBPROCESS:
323 return False
324 return resource_key(left) is None or resource_key(right) is None
325
326
327def is_conflicting(resource: str) -> bool:
328 """True when *resource* can collide ACROSS worker processes.
329
330 Every ``unknown:<detail>`` conflicts by construction. An empty
331 launch-family entry has no self-conflict (Plan 251 P4), but its
332 token remains visible to the GLOBAL ``unknown:*`` wildcard in
333 :func:`build_groups` - no longer to a bare launch, which since
334 Plan 276 P2 holds only the other bare launches. Other keyed
335 resources remain shared.
336 """
337 family = resource_family(resource)
338 if family == RES_ENV_SUBPROCESS and resource_key(resource) is not None:
339 return LAUNCH_FAMILIES.get(resource_key(resource)) != ()
340 return family not in PROCESS_LOCAL_RESOURCES
341
342
343def is_known_resource(resource: str) -> bool:
344 """True when *resource* is inside the closed P2 vocabulary.
345
346 Parameters
347 ----------
348 resource
349 A resource token as the detectors record it.
350
351 Returns
352 -------
353 bool
354 True for a vocabulary member, a keyed token of a
355 :data:`KEYED_FAMILIES` member, or a well-formed ``port:<n>``;
356 False for ``unknown:<detail>`` and anything unrecognised.
357 """
358 if resource in RESOURCE_VOCABULARY:
359 return True
360 if resource.startswith(PORT_PREFIX):
361 return resource[len(PORT_PREFIX):].isdigit()
362 return bool(resource_key(resource))
363
364
365def unknown(detail: str) -> str:
366 """Build the fail-closed ``unknown:<detail>`` token.
367
368 *detail* must be machine independent (no absolute paths, no user
369 names): the manifest that carries it is committed.
370 """
371 return f"{UNKNOWN_PREFIX}{detail}"
372
373
374# ---------------------------------------------------------------------
375# Manifest shape (P4)
376# ---------------------------------------------------------------------
377
378#: Schema version of ``parallel_manifest.json``.
379SCHEMA_VERSION: str = "1.0"
380
381#: Conventional basename of the committed manifest.
382MANIFEST_FILENAME: str = "parallel_manifest.json"
383
384#: Group of a file that touches no shared resource.
385GROUP_FREE: str = "free"
386
387#: Group applied at collection to a file with NO manifest entry.
388GROUP_UNCLASSIFIED: str = "unclassified-serial"
389
390#: Prefix of the per-test xdist group names a ``free`` file's items
391#: carry (Plan 286 P1). The manifest still records the group ``free``
392#: per FILE; only the name stamped on a collected ITEM is per test.
393FREE_GROUP_PREFIX: str = f"{GROUP_FREE}:"
394
395#: Marker that keeps ONE free file's tests in one group (Plan 286 P2).
396#: The opt-out for a file whose tests depend on each other's order or
397#: on module state; every use is named in the plan's report with the
398#: witness that justified it.
399WHOLE_FILE_MARKER: str = "whole_file"
400
401
402@dataclass(frozen=True)
404 """One classified file.
405
406 Attributes
407 ----------
408 path
409 Root-relative POSIX path (a pytest nodeid's path part).
410 resources
411 Sorted resource tokens, the UNION of both detectors.
412 group
413 The xdist group name the conftest applies to every item
414 collected from this file.
415 """
416
417 path: str
418 resources: tuple[str, ...]
419 group: str
420
421
422@dataclass(frozen=True)
424 """A parsed ``parallel_manifest.json``."""
425
426 schema_version: str
427 generated_at: str
428 tool_version: str
429 files: Mapping[str, FileEntry]
430
431
432# ---------------------------------------------------------------------
433# Path -> resource mapping (shared by both detectors)
434# ---------------------------------------------------------------------
435
436#: Ordered ``(substring, resource)`` markers matched against a
437#: lowercased, forward-slashed path hint. First match wins, so the
438#: specific shared files come before the generic directories.
439_PATH_MARKERS: tuple[tuple[str, str], ...] = (
440 ("agents.sqlite", RES_REGISTRY_SQLITE),
441 ("registry.sqlite", RES_REGISTRY_SQLITE),
442 (".option_c/orchestration", RES_REGISTRY_SQLITE),
443 (".option_c/comms", RES_COMMS_ROOT),
444 ("octrc.json", RES_OCTRC),
445 ("settings.json", RES_SETTINGS_FILE),
446 ("option_c_branches", RES_WORKTREE_DIR),
447 (".git/worktrees", RES_WORKTREE_DIR),
448)
449
450
451def resource_for_path_hint(hint: str) -> str:
452 """Map a path-ish *hint* to its shared resource.
453
454 Parameters
455 ----------
456 hint
457 Any path fragment: string constants lifted out of the source,
458 or a concrete path the dynamic tracer observed.
459
460 Returns
461 -------
462 str
463 The matching vocabulary member, or
464 :data:`RES_MAIN_TREE_WRITE` when the hint names no known
465 shared file - an unrecognised write is still a write, and
466 still shared.
467 """
468 needle = hint.replace("\\", "/").lower()
469 for marker, resource in _PATH_MARKERS:
470 if marker in needle:
471 return resource
472 return RES_MAIN_TREE_WRITE
473
474
475def path_key(hint: str) -> str | None:
476 """The top-level component of a ROOT-RELATIVE path *hint* (P2).
477
478 Parameters
479 ----------
480 hint
481 A path relative to the package root, in either slash style.
482
483 Returns
484 -------
485 str | None
486 The first component - the directory the write lands in - or
487 None for an absolute path, a drive-qualified path, or an
488 upward escape. None means "no key", which produces the bare
489 wildcard: a key that is not a real top inside the root would
490 separate two writes that really can collide.
491 """
492 text = hint.replace("\\", "/").strip()
493 if not text or text.startswith("/"):
494 return None
495 parts = [part for part in text.split("/") if part not in ("", ".")]
496 if not parts or parts[0] == ".." or KEY_SEPARATOR in parts[0]:
497 return None
498 return parts[0]
499
500
501def write_resource(hint: str, key: str | None) -> str:
502 """The resource of a write to *hint*, keyed where P2 allows.
503
504 Keying only ever refines the UNATTRIBUTED write: a hint the
505 vocabulary already names something specific - the comms root, the
506 registry database, an ``octrc.json`` - keeps that token, because
507 two writes to the same shared FILE collide whichever directory
508 they were reached through.
509
510 Parameters
511 ----------
512 hint
513 Any path fragment, as :func:`resource_for_path_hint` takes it.
514 key
515 The top-level directory the write landed in, or None when the
516 caller could not read one.
517
518 Returns
519 -------
520 str
521 A specific vocabulary member, ``main_tree_write:<key>``, or
522 the bare :data:`RES_MAIN_TREE_WRITE`.
523 """
524 resource = resource_for_path_hint(hint)
525 if resource != RES_MAIN_TREE_WRITE or not key:
526 return resource
527 return keyed(RES_MAIN_TREE_WRITE, key)
528
529
530# ---------------------------------------------------------------------
531# P3 - launch families (shared by both detectors)
532# ---------------------------------------------------------------------
533
534#: Stand-in for ``sys.executable`` in a STATICALLY read argv. It is
535#: the one non-constant argv head whose meaning is fixed, and the
536#: module name beside it is a literal; without it the interpreter
537#: launches would stay bare and re-widen every file the tracer had
538#: just keyed, because a classification is the UNION of both
539#: detectors.
540PYTHON_ARGV0: str = "<python>"
541
542#: Program stems that are a Python interpreter.
543_PYTHON_PROGRAM = re.compile(r"^python[0-9tw.]*$", re.IGNORECASE)
544
545#: Value-less interpreter flags that may sit between the interpreter
546#: and ``-m``. The oc_chat suite launches ``python -P -m oc_chat``
547#: (safe path) in five places; without skipping these the family reads
548#: as ``python`` and the launch stays bare for no reason.
549_PYTHON_FLAGS: frozenset[str] = frozenset({
550 "-b", "-bb", "-B", "-d", "-E", "-i", "-I", "-O", "-OO", "-P",
551 "-q", "-s", "-S", "-u", "-v", "-x",
552})
553
554#: What each launch FAMILY is KNOWN to touch, beyond its own keyed
555#: ``env_subprocess:<family>`` token (P3). This table is the ONLY
556#: evidence that narrows a child: a family absent from it contributes
557#: the BARE token and conflicts with every launcher, because a child
558#: process is otherwise opaque to an ast pass and an in-process
559#: tracer alike.
560#:
561#: Per-entry justification (Plan 247; widen an entry the moment a
562#: trace shows it touching more, never narrow one on a guess):
563#:
564#: - ``git`` - every git launch in the suite runs against a repository
565#: the test built under ``tmp_path``; git writes inside the
566#: repository it is pointed at and nowhere else.
567#: - ``oc_chat.testing`` - the §20 stand-in. Its sessions have lived
568#: under temp since plan 239. The existing empty resource entry is
569#: process-local under Plan 251 P4.
570#: - ``oct`` / ``oct.cli`` - the toolchain writes the timing ledger and
571#: its caches under ``.option_c``, so a launch joins the files that
572#: write there directly.
573LAUNCH_FAMILIES: Mapping[str, tuple[str, ...]] = {
574 "git": (),
575 "oc_chat.testing": (),
576 # Plan 251 serial child audit: 4 console/module and 9 replay launches
577 # completed; cross-PID diagnostic-log cleanup writes the shared logs.
578 # Registry stays opaque: its 67 native Git children were not probed.
579 "oc_chat": (f"{RES_MAIN_TREE_WRITE}{KEY_SEPARATOR}logs",),
580 "oc_chat.scripts.replay": (f"{RES_MAIN_TREE_WRITE}{KEY_SEPARATOR}logs",),
581 "oct": (f"{RES_MAIN_TREE_WRITE}{KEY_SEPARATOR}.option_c",),
582 "oct.cli": (f"{RES_MAIN_TREE_WRITE}{KEY_SEPARATOR}.option_c",),
583}
584
585
586def launch_family(argv: Any) -> str | None:
587 """The program family of a child launch, or None when opaque (P3).
588
589 ``python -m <module>`` names the MODULE, because that is what the
590 child actually IS - ``oc_chat.testing``, ``oct.cli`` and
591 ``oc_chat.registry`` are three families, not one interpreter. Any
592 other program names its own basename without extension. A bare
593 interpreter (``python -c``, ``python script.py``) names only
594 ``python``, which the table deliberately does not vouch for.
595
596 Parameters
597 ----------
598 argv
599 The argv as the caller passed it: a sequence whose unreadable
600 elements are None, a shell string, or anything else.
601
602 Returns
603 -------
604 str | None
605 The family name, or None when the argv has no readable head.
606 """
607 if not isinstance(argv, (list, tuple)) or not argv:
608 return None
609 head = argv[0]
610 if head is None:
611 return None
612 if head == PYTHON_ARGV0:
613 stem = "python"
614 else:
615 try:
616 text = os.fspath(head)
617 except TypeError:
618 return None
619 stem = PurePosixPath(str(text).replace("\\", "/")).stem
620 if not stem:
621 return None
622 if _PYTHON_PROGRAM.match(stem):
623 index = 1
624 while index < len(argv) and argv[index] in _PYTHON_FLAGS:
625 index += 1
626 if index + 1 < len(argv) and argv[index] == "-m" and argv[index + 1]:
627 return str(argv[index + 1])
628 return stem
629
630
631def launch_resources(argv: Any) -> frozenset[str]:
632 """The resources one child launch contributes (P3).
633
634 A family named in :data:`LAUNCH_FAMILIES` contributes its own
635 keyed token plus everything the table vouches for. A family the
636 table does not name - and an argv with no readable family at all -
637 contributes the BARE token, which conflicts with every launcher.
638 """
639 func = "launch_resources"
640 family = launch_family(argv)
641 if family is None or family not in LAUNCH_FAMILIES:
642 _dbg("OCT-TESTING", func, f"unattributed launch: {family}", 4)
643 return frozenset({RES_ENV_SUBPROCESS})
644 return frozenset({
645 keyed(RES_ENV_SUBPROCESS, family), *LAUNCH_FAMILIES[family],
646 })
647
648
649# ---------------------------------------------------------------------
650# P3a - the static detector
651# ---------------------------------------------------------------------
652
653#: Fixture/parameter names that seed the temp-path taint set.
654_TMP_SEEDS: frozenset[str] = frozenset({
655 "tmp_path", "tmp_path_factory", "tmpdir", "tmpdir_factory",
656})
657
658#: Calls whose result is a temporary path.
659_TMP_CALLS: frozenset[str] = frozenset({
660 "mkdtemp", "mkstemp", "mktemp", "gettempdir", "TemporaryDirectory",
661 "NamedTemporaryFile", "TemporaryFile",
662})
663
664#: ``pathlib.Path`` methods that write.
665_PATH_WRITES: frozenset[str] = frozenset({
666 "write_text", "write_bytes", "mkdir", "touch", "unlink", "rmdir",
667 "rename", "replace", "symlink_to", "hardlink_to", "chmod",
668})
669
670#: ``os`` functions that write.
671_OS_WRITES: frozenset[str] = frozenset({
672 "remove", "unlink", "mkdir", "makedirs", "rmdir", "removedirs",
673 "rename", "renames", "replace", "truncate", "symlink", "link",
674 "chmod", "utime", "mknod",
675})
676
677#: ``shutil`` functions that write, mapped to the argument index that
678#: names the DESTINATION (the path actually written).
679_SHUTIL_WRITES: Mapping[str, int] = {
680 "copy": 1, "copy2": 1, "copyfile": 1, "copytree": 1, "move": 1,
681 "rmtree": 0, "make_archive": 0, "unpack_archive": 1, "chown": 0,
682}
683
684#: Subprocess launch seams reached as ``<module>.<attr>``.
685_SUBPROCESS_ATTRS: frozenset[str] = frozenset({
686 "run", "Popen", "call", "check_call", "check_output",
687 "create_subprocess_exec", "create_subprocess_shell",
688})
689
690#: Modules whose ``_SUBPROCESS_ATTRS`` really do launch a child.
691_SUBPROCESS_MODULES: frozenset[str] = frozenset({
692 "subprocess", "asyncio",
693})
694
695#: Bare function names that launch a child process.
696_SUBPROCESS_NAMES: frozenset[str] = frozenset({
697 "run_cancellable", "getoutput", "getstatusoutput",
698})
699
700#: Network seams reached by attribute.
701_NETWORK_ATTRS: frozenset[str] = frozenset({
702 "urlopen", "urlretrieve", "create_connection",
703})
704
705#: Modules whose use is outbound network use.
706_NETWORK_MODULES: frozenset[str] = frozenset({
707 "requests", "httpx", "urllib",
708})
709
710#: Known process-wide reset seams (P2 ``global_singleton_reset``).
711_RESET_NAMES: frozenset[str] = frozenset({
712 "reset_tool_cancel", "signal_tool_cancel", "reset_session_trust",
713 "reset_inventory_for_tests", "reset_runtime_settings_for_tests",
714 "revoke_session_grants", "grant_work_dir_for_session",
715})
716
717#: Imported module basenames whose entry points reach a shared file
718#: unless the call is handed a temp-derived path.
719_MODULE_HINTS: Mapping[str, str] = {
720 "agent_instances": RES_REGISTRY_SQLITE,
721 "agent_registry": RES_REGISTRY_SQLITE,
722 "orchestration": RES_REGISTRY_SQLITE,
723}
724
725
726@dataclass(frozen=True)
728 """The static reading of one module (a test file or a conftest).
729
730 Attributes
731 ----------
732 resources
733 Union of every resource the module's own code touches.
734 fixtures
735 Fixture name -> the resources that fixture's body touches.
736 autouse
737 Names of the module's ``autouse=True`` fixtures.
738 tmp_fixtures
739 Fixtures whose value is derived from a temp path; requesting
740 one taints the requesting parameter.
741 requested
742 Fixture/parameter names every ``def`` in the module requests.
743 """
744
745 resources: frozenset[str]
746 fixtures: Mapping[str, frozenset[str]]
747 autouse: frozenset[str]
748 tmp_fixtures: frozenset[str]
749 requested: frozenset[str]
750
751
752def _target_names(node: ast.AST) -> set[str]:
753 """Every plain name bound by an assignment target *node*."""
754 return {
755 sub.id for sub in ast.walk(node) if isinstance(sub, ast.Name)
756 }
757
758
759def _is_tmp_expr(node: ast.AST | None, tainted: frozenset[str]) -> bool:
760 """True when *node* is derived from a temporary directory."""
761 if node is None:
762 return False
763 for sub in ast.walk(node):
764 if isinstance(sub, ast.Name) and sub.id in tainted:
765 return True
766 if isinstance(sub, ast.Call):
767 func = sub.func
768 name = getattr(func, "attr", None) or getattr(func, "id", None)
769 if name in _TMP_CALLS:
770 return True
771 return False
772
773
774def _fixture_decorated(node: ast.AST) -> tuple[bool, bool]:
775 """Return ``(is_fixture, is_autouse)`` for a function definition."""
776 is_fixture = False
777 autouse = False
778 for dec in getattr(node, "decorator_list", []):
779 call = dec if isinstance(dec, ast.Call) else None
780 target = call.func if call is not None else dec
781 name = getattr(target, "attr", None) or getattr(target, "id", None)
782 if name != "fixture":
783 continue
784 is_fixture = True
785 for keyword in (call.keywords if call is not None else []):
786 if keyword.arg == "autouse" and isinstance(
787 keyword.value, ast.Constant,
788 ):
789 autouse = bool(keyword.value.value)
790 return is_fixture, autouse
791
792
793def _returns_tmp(node: ast.AST, tainted: frozenset[str]) -> bool:
794 """True when every value a fixture body returns/yields is temp."""
795 seen = False
796 for sub in ast.walk(node):
797 if isinstance(sub, (ast.Return, ast.Yield)):
798 if sub.value is None:
799 continue
800 seen = True
801 if not _is_tmp_expr(sub.value, tainted):
802 return False
803 return seen
804
805
806def _collect_taint(tree: ast.AST, seeds: Iterable[str]) -> frozenset[str]:
807 """Fixpoint over the names bound to a temporary-path value.
808
809 Parameters
810 ----------
811 tree
812 The parsed module.
813 seeds
814 Extra names to seed with: the temp-returning fixtures of the
815 conftests above this module.
816
817 Returns
818 -------
819 frozenset[str]
820 Every name a write target may safely be derived from.
821 """
822 func = "_collect_taint"
823 tainted: set[str] = set(_TMP_SEEDS) | set(seeds)
824 for _ in range(4):
825 before = len(tainted)
826 frozen = frozenset(tainted)
827 for node in ast.walk(tree):
828 if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)):
829 if _is_tmp_expr(node.value, frozen):
830 targets = getattr(node, "targets", None) or [node.target]
831 for target in targets:
832 tainted |= _target_names(target)
833 elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
834 is_fixture, _autouse = _fixture_decorated(node)
835 if is_fixture and _returns_tmp(node, frozen):
836 tainted.add(node.name)
837 elif isinstance(node, ast.withitem):
838 if node.optional_vars is not None and _is_tmp_expr(
839 node.context_expr, frozen,
840 ):
841 tainted |= _target_names(node.optional_vars)
842 elif isinstance(node, ast.For):
843 if _is_tmp_expr(node.iter, frozen):
844 tainted |= _target_names(node.target)
845 if len(tainted) == before:
846 break
847 _dbg("OCT-TESTING", func, f"tainted names: {len(tainted)}", 4)
848 return frozenset(tainted)
849
850
851def _scope_nodes(tree: ast.AST) -> Iterable[ast.AST]:
852 """Walk one lexical body without importing names from nested bodies."""
853 pending = list(ast.iter_child_nodes(tree))
854 while pending:
855 node = pending.pop()
856 yield node
857 if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef,
858 ast.ClassDef, ast.Lambda, ast.ListComp,
859 ast.SetComp, ast.DictComp, ast.GeneratorExp)):
860 continue
861 pending.extend(ast.iter_child_nodes(node))
862
863
864def _proven_tmp_expr(node: ast.AST | None, tainted: frozenset[str]) -> bool:
865 """Read a helper actual without assuming an opaque call returns its input."""
866 if isinstance(node, ast.Name):
867 return node.id in tainted
868 if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div):
869 return _proven_tmp_expr(node.left, tainted)
870 if isinstance(node, ast.JoinedStr):
871 return bool(node.values) and isinstance(
872 node.values[0], ast.FormattedValue,
873 ) and _proven_tmp_expr(node.values[0].value, tainted)
874 if not isinstance(node, ast.Call):
875 return False
876 name = getattr(node.func, 'attr', None) or getattr(node.func, 'id', None)
877 if name in _TMP_CALLS:
878 return True
879 if name in _PATH_CONSTRUCTORS | {'str', 'fspath'}:
880 return bool(node.args) and _proven_tmp_expr(node.args[0], tainted)
881 if isinstance(node.func, ast.Attribute) and name in (
882 'joinpath', 'absolute', 'resolve', 'with_name', 'with_suffix',
883 ):
884 return _proven_tmp_expr(node.func.value, tainted)
885 return False
886
887
889 tree: ast.AST,
890 seeds: Iterable[str],
891 *,
892 parameters: Iterable[str] = (),
893) -> frozenset[str]:
894 """Prove local aliases only when every binding has a temporary origin."""
895 bindings: dict[str, list[ast.AST | None]] = {}
896 for node in _scope_nodes(tree):
897 pairs = []
898 if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
899 fixture = isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and (
900 isinstance(tree, ast.Module) and _fixture_decorated(node)[0]
901 and node.name in seeds
902 )
903 if not fixture:
904 bindings.setdefault(node.name, []).append(None)
905 elif isinstance(node, (ast.Import, ast.ImportFrom)):
906 for alias in node.names:
907 bindings.setdefault(alias.asname or alias.name.split(".")[0], []).append(None)
908 elif isinstance(node, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)):
909 if node.name:
910 bindings.setdefault(node.name, []).append(None)
911 elif isinstance(node, ast.MatchMapping) and node.rest:
912 bindings.setdefault(node.rest, []).append(None)
913 if isinstance(node, (ast.Assign, ast.AnnAssign, ast.NamedExpr)):
914 targets = getattr(node, 'targets', None) or [node.target]
915 pairs = [(target, node.value) for target in targets]
916 elif isinstance(node, ast.withitem) and node.optional_vars is not None:
917 pairs = [(node.optional_vars, node.context_expr)]
918 elif isinstance(node, (ast.For, ast.AsyncFor)):
919 pairs = [(node.target, node.iter)]
920 elif isinstance(node, (ast.AugAssign, ast.Delete)):
921 targets = getattr(node, 'targets', None) or [node.target]
922 pairs = [(target, None) for target in targets]
923 for target, value in pairs:
924 for bound in ast.walk(target):
925 if isinstance(bound, ast.Name) and isinstance(
926 bound.ctx, (ast.Store, ast.Del),
927 ):
928 bindings.setdefault(bound.id, []).append(value)
929 tainted = set(seeds) - set(bindings)
930 if isinstance(tree, (ast.FunctionDef, ast.AsyncFunctionDef)):
931 tainted -= _parameter_names(tree) - set(parameters)
932 while True:
933 before = set(tainted)
934 for name, values in bindings.items():
935 if all(_proven_tmp_expr(value, frozenset(tainted)) for value in values):
936 tainted.add(name)
937 if tainted == before:
938 return frozenset(tainted)
939
940
941def _parameter_names(node: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]:
942 """All names bound by one function's argument specification."""
943 spec = node.args
944 names = {arg.arg for arg in (*spec.posonlyargs, *spec.args, *spec.kwonlyargs)}
945 names.update(arg.arg for arg in (spec.vararg, spec.kwarg) if arg is not None)
946 return names
947
948
950 tree: ast.Module,
951 base: frozenset[str],
952 seeds: Iterable[str] = (),
953) -> dict[ast.AST, frozenset[str]]:
954 """Prove helper parameters from every visible call, without name leakage.
955
956 Only direct references to uniquely named module functions qualify.
957 An escaping reference, reflection, rebinding or starred call keeps the
958 helper opaque. Inferred names belong to that function's scope only.
959 """
960 functions = {node for node in ast.walk(tree) if isinstance(
961 node, (ast.FunctionDef, ast.AsyncFunctionDef),
962 )}
963 parents = {child: node for node in ast.walk(tree)
964 for child in ast.iter_child_nodes(node)}
965 boundaries = functions | {node for node in ast.walk(tree) if isinstance(
966 node, (ast.ClassDef, ast.Lambda, ast.ListComp, ast.SetComp,
967 ast.DictComp, ast.GeneratorExp),
968 )}
969 scopes: dict[ast.AST, ast.AST] = {}
970 for node in ast.walk(tree):
971 parent = parents.get(node)
972 while parent is not None and parent not in boundaries:
973 parent = parents.get(parent)
974 scopes[node] = parent or tree
975 fixture_seeds = set(_TMP_SEEDS) | set(seeds)
976 fixture_seeds.update(
977 node.name for node in functions
978 if _fixture_decorated(node)[0] and node.name in base
979 )
980 # A nested global/nonlocal declaration can change a visible origin
981 # without rebinding it in the caller body. Keep those names opaque.
982 mutable_origins = {name for node in ast.walk(tree)
983 if isinstance(node, (ast.Global, ast.Nonlocal))
984 for name in node.names}
985 module_proofs = _scope_proofs(tree, fixture_seeds) - mutable_origins
986 proofs = {
987 node: _scope_proofs(
988 node, module_proofs,
989 parameters=(fixture_seeds if node.name.startswith("test_")
990 or _fixture_decorated(node)[0] else ()),
991 ) for node in functions
992 }
993 proofs.update({node: frozenset() for node in boundaries - functions})
994 proofs[tree] = module_proofs
995 taints = {}
996 candidates = {}
997 for helper in tree.body:
998 if helper not in functions or helper.name.startswith("test_"):
999 continue
1000 if _fixture_decorated(helper)[0]:
1001 continue
1002 # Existing module-wide name taint cannot prove a helper's own
1003 # parameters or locals: both can shadow unrelated temporary names.
1004 local = _parameter_names(helper) | {
1005 node.id for node in _scope_nodes(helper)
1006 if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store)
1007 }
1008 taints[helper] = (base - local) | _scope_proofs(helper, module_proofs)
1009 calls = _closed_helper_calls(tree, helper, parents)
1010 if calls:
1011 candidates[helper] = calls
1012 while True:
1013 changed = False
1014 for helper, calls in candidates.items():
1015 inferred = set()
1016 parameters = (*helper.args.posonlyargs, *helper.args.args,
1017 *helper.args.kwonlyargs)
1018 for parameter in parameters:
1019 name = parameter.arg
1020 if all(_proven_tmp_expr(
1021 _helper_actual(helper, call, name), proofs[scopes[call]],
1022 ) for call in calls):
1023 inferred.add(name)
1024 # A write through a reassigned parameter is not proved by its
1025 # incoming value. Keep it opaque even if another call is safe.
1026 inferred -= _rebound_parameters(helper, inferred)
1027 proved = _scope_proofs(
1028 helper, module_proofs | inferred, parameters=inferred,
1029 )
1030 local = _parameter_names(helper) | {
1031 node.id for node in _scope_nodes(helper)
1032 if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store)
1033 }
1034 taints[helper] = (base - local) | proved
1035 if proved != proofs[helper]:
1036 proofs[helper] = proved
1037 changed = True
1038 if not changed:
1039 break
1040 return taints
1041
1042
1044 tree: ast.Module,
1045 helper: ast.FunctionDef | ast.AsyncFunctionDef,
1046 parents: Mapping[ast.AST, ast.AST],
1047) -> list[ast.Call]:
1048 """Return every direct call, or no proof when the helper can escape."""
1049 calls = []
1050 if helper.decorator_list and not _fixture_decorated(helper)[0]:
1051 return calls
1052 for node in ast.walk(tree):
1053 if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
1054 if node is not helper and node.name == helper.name:
1055 return []
1056 if isinstance(node, ast.arg) and node.arg == helper.name:
1057 return []
1058 if isinstance(node, ast.alias) and (
1059 node.asname or node.name.split('.')[0]
1060 ) == helper.name:
1061 return []
1062 if isinstance(node, ast.Attribute) and node.attr == helper.name:
1063 return []
1064 if isinstance(node, ast.Call) and (
1065 (getattr(node.func, 'id', None) or getattr(node.func, 'attr', None))
1066 in {'getattr', 'globals', 'locals', 'vars', 'eval', 'exec'}
1067 ):
1068 return []
1069 if not isinstance(node, ast.Name) or node.id != helper.name:
1070 continue
1071 parent = parents.get(node)
1072 if not (isinstance(node.ctx, ast.Load)
1073 and isinstance(parent, ast.Call) and parent.func is node):
1074 return []
1075 if any(isinstance(arg, ast.Starred) for arg in parent.args):
1076 return []
1077 if any(keyword.arg is None for keyword in parent.keywords):
1078 return []
1079 calls.append(parent)
1080 return calls
1081
1082
1084 helper: ast.FunctionDef | ast.AsyncFunctionDef,
1085 call: ast.Call,
1086 name: str,
1087) -> ast.AST | None:
1088 """Read a parameter's actual argument; omissions stay unresolved."""
1089 positional = [arg.arg for arg in (*helper.args.posonlyargs, *helper.args.args)]
1090 if len(call.args) > len(positional) and helper.args.vararg is None:
1091 return None
1092 if name in {arg.arg for arg in helper.args.posonlyargs} and any(
1093 keyword.arg == name for keyword in call.keywords
1094 ):
1095 return None
1096 values = [keyword.value for keyword in call.keywords if keyword.arg == name]
1097 if name in positional and positional.index(name) < len(call.args):
1098 values.append(call.args[positional.index(name)])
1099 return values[0] if len(values) == 1 else None
1100
1101
1102def _rebound_parameters(tree: ast.AST, names: set[str]) -> set[str]:
1103 """Reject new parameter proofs when the body assigns their names."""
1104 return {node.id for node in ast.walk(tree)
1105 if isinstance(node, ast.Name) and node.id in names
1106 and isinstance(node.ctx, (ast.Store, ast.Del))}
1107
1108
1109def _import_hints(tree: ast.AST) -> dict[str, str]:
1110 """Map names imported from a hinted module to their resource."""
1111 hints: dict[str, str] = {}
1112 for node in ast.walk(tree):
1113 if isinstance(node, ast.ImportFrom):
1114 resource = _hint_for_dotted(node.module or "")
1115 if resource is None:
1116 continue
1117 for alias in node.names:
1118 hints[alias.asname or alias.name] = resource
1119 elif isinstance(node, ast.Import):
1120 for alias in node.names:
1121 resource = _hint_for_dotted(alias.name)
1122 if resource is None:
1123 continue
1124 bound = alias.asname or alias.name.split(".")[0]
1125 hints[bound] = resource
1126 return hints
1127
1128
1129def _hint_for_dotted(dotted: str) -> str | None:
1130 """The hinted resource of a dotted module path, if any."""
1131 for part in dotted.split("."):
1132 if part in _MODULE_HINTS:
1133 return _MODULE_HINTS[part]
1134 return None
1135
1136
1137def _string_hint(node: ast.AST | None) -> str:
1138 """Join every string constant, attribute and name inside *node*."""
1139 if node is None:
1140 return ""
1141 parts: list[str] = []
1142 for sub in ast.walk(node):
1143 if isinstance(sub, ast.Constant) and isinstance(sub.value, str):
1144 parts.append(sub.value)
1145 elif isinstance(sub, ast.Attribute):
1146 parts.append(sub.attr)
1147 elif isinstance(sub, ast.Name):
1148 parts.append(sub.id)
1149 return "/".join(parts)
1150
1151
1152#: Constructors whose all-literal arguments still name a literal path.
1153_PATH_CONSTRUCTORS: frozenset[str] = frozenset({
1154 "Path", "PurePath", "PurePosixPath", "PureWindowsPath",
1155 "PosixPath", "WindowsPath",
1156})
1157
1158#: ``pathlib`` methods that return the SAME location they were called
1159#: on, so a resolved literal passes through them unchanged (Plan 284
1160#: P1). They are what stands between ``Path(__file__)`` and the
1161#: ``.parent`` beside it in the shape the suite actually writes.
1162_PATH_IDENTITY_CALLS: frozenset[str] = frozenset({"resolve", "absolute"})
1163
1164#: Stand-in for the pid inside a resolved f-string (Plan 284 P1). The
1165#: value is irrelevant - :func:`_literal_fstring` only admits a
1166#: placeholder that cannot reach the component :func:`path_key` reads -
1167#: but it must be a legal path character, because the joined result is
1168#: parsed as a path.
1169_PID_PLACEHOLDER: str = "0"
1170
1171#: Ceiling on ``.parents[N]``. A literal index this large has already
1172#: escaped every real tree, and the bound keeps a pathological constant
1173#: from walking a loop the classifier gains nothing from.
1174_MAX_PARENTS_INDEX: int = 64
1175
1176
1177def _is_absolute_literal(text: str) -> bool:
1178 """True when a literal path fragment is not root-relative.
1179
1180 The same three shapes :func:`path_key` refuses - a leading slash, a
1181 drive letter, and anything else carrying the key separator in its
1182 first component - because a fragment that is absolute swallows
1183 everything joined BEFORE it.
1184 """
1185 head = text.replace("\\", "/")
1186 return head.startswith("/") or KEY_SEPARATOR in head.split("/")[0]
1187
1188
1189def _join_literal(left: str, right: str) -> str:
1190 """Join two literal fragments the way ``pathlib`` joins them.
1191
1192 An absolute right operand DISCARDS the left one - ``Path('docs') /
1193 '/var/lib'`` is ``/var/lib`` - so reading the left as the key would
1194 invent a directory the write never touches. Returning the right
1195 operand whole hands :func:`path_key` an absolute path, which it
1196 already refuses.
1197 """
1198 if _is_absolute_literal(right):
1199 return right
1200 if not right:
1201 return left
1202 if not left:
1203 return right
1204 return f"{left.rstrip('/')}/{right}"
1205
1206
1207def _parent_literal(text: str) -> str | None:
1208 """The parent of a root-relative literal fragment, or None (P1).
1209
1210 Climbing past the package root yields a ``..`` prefix rather than
1211 an empty string, because the two mean different things and only the
1212 first is an escape: the root itself is a legitimate location with
1213 no top-level component, while ``..`` is outside the tree the
1214 manifest describes. :func:`path_key` refuses both, which is the
1215 fail-closed answer for each.
1216 """
1217 if _is_absolute_literal(text):
1218 return None
1219 parts = [part for part in text.replace("\\", "/").split("/")
1220 if part not in ("", ".")]
1221 if parts and parts[-1] != "..":
1222 parts.pop()
1223 else:
1224 parts.append("..")
1225 return "/".join(parts)
1226
1227
1228def _is_pid_expr(node: ast.AST | None) -> bool:
1229 """True when *node* is ``os.getpid()``, or a ``str()`` of one."""
1230 if not isinstance(node, ast.Call):
1231 return False
1232 name = getattr(node.func, "attr", None) or getattr(node.func, "id", None)
1233 if name == "getpid":
1234 return True
1235 return name == "str" and len(node.args) == 1 and _is_pid_expr(node.args[0])
1236
1237
1238def _literal_fstring(node: ast.JoinedStr) -> str | None:
1239 """An f-string read as a path when its placeholders cannot move the key.
1240
1241 The key is the FIRST component, so a placeholder is harmless
1242 exactly while every separator in the string comes BEFORE it: a pid
1243 stamped into the basename leaves the directory literal, and
1244 ``f'{os.getpid()}/run.txt'`` does not. Only the pid qualifies as a
1245 placeholder at all - a temp-derived one never reaches here, because
1246 :meth:`_StaticDetector._write` drops the write before it asks for a
1247 key, and anything else is a value this pass cannot read.
1248 """
1249 parts: list[str] = []
1250 placed = False
1251 for value in node.values:
1252 if isinstance(value, ast.Constant) and isinstance(value.value, str):
1253 if placed and ("/" in value.value or "\\" in value.value):
1254 return None
1255 parts.append(value.value)
1256 continue
1257 if not isinstance(value, ast.FormattedValue):
1258 return None
1259 if not _is_pid_expr(value.value):
1260 return None
1261 placed = True
1262 parts.append(_PID_PLACEHOLDER)
1263 return "".join(parts)
1264
1265
1266def _is_os_path_join(func: ast.AST) -> bool:
1267 """True for the fully spelled ``os.path.join`` attribute chain."""
1268 return (
1269 isinstance(func, ast.Attribute)
1270 and func.attr == "join"
1271 and isinstance(func.value, ast.Attribute)
1272 and func.value.attr == "path"
1273 and getattr(func.value.value, "id", None) == "os"
1274 )
1275
1276
1278 node: ast.Subscript,
1279 constants: Mapping[str, str],
1280) -> str | None:
1281 """``<literal>.parents[N]`` for a constant, non-negative literal N."""
1282 owner = node.value
1283 if not (isinstance(owner, ast.Attribute) and owner.attr == "parents"):
1284 return None
1285 index = node.slice
1286 if not isinstance(index, ast.Constant) or isinstance(index.value, bool):
1287 return None
1288 if not isinstance(index.value, int) or not (
1289 0 <= index.value <= _MAX_PARENTS_INDEX
1290 ):
1291 return None
1292 resolved = _literal_path(owner.value, constants)
1293 for _ in range(index.value + 1):
1294 if resolved is None:
1295 return None
1296 resolved = _parent_literal(resolved)
1297 return resolved
1298
1299
1301 node: ast.AST | None,
1302 constants: Mapping[str, str] | None = None,
1303) -> str | None:
1304 """The path *node* names when this pass can read it in full (P2).
1305
1306 ``_string_hint`` deliberately mashes names and attributes together
1307 so a MARKER can be spotted anywhere in an expression; that mash is
1308 useless as a key, because ``Path('docs/x')`` would key on ``Path``
1309 and land in a different group from ``open('docs/y', 'w')`` - two
1310 writes into the same directory, separated. So keying reads the
1311 expression strictly instead: a constant, a ``Path(...)`` of
1312 constants, or a ``/`` join of those.
1313
1314 Plan 284 P1 widens WHAT counts as readable without widening how
1315 loosely it is read. *constants* carries the module's own
1316 unambiguous names - ``__file__`` among them, bound to the module's
1317 ROOT-RELATIVE path - so ``Path(__file__).resolve().parent /
1318 'golden'`` names a real directory instead of an opaque one. Each
1319 added shape either resolves completely or yields None; none of them
1320 guesses at a component it could not read.
1321
1322 Parameters
1323 ----------
1324 node
1325 The expression naming a write target.
1326 constants
1327 Module-level names this pass proved to be literal paths, as
1328 :func:`_module_constants` builds them. Absent, the function
1329 reads exactly what it read before Plan 284.
1330
1331 Returns
1332 -------
1333 str | None
1334 A path in :func:`path_key`'s space - root-relative unless the
1335 expression itself was absolute or escaped upward - or None when
1336 any part of the expression is unreadable.
1337 """
1338 names: Mapping[str, str] = constants if constants is not None else {}
1339 if isinstance(node, ast.Constant):
1340 return node.value if isinstance(node.value, str) else None
1341 if isinstance(node, ast.Name):
1342 return names.get(node.id)
1343 if isinstance(node, ast.JoinedStr):
1344 return _literal_fstring(node)
1345 if isinstance(node, ast.Subscript):
1346 return _literal_parents_index(node, names)
1347 if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div):
1348 left = _literal_path(node.left, names)
1349 right = _literal_path(node.right, names)
1350 if left is None or right is None:
1351 return None
1352 return _join_literal(left, right)
1353 if isinstance(node, ast.Attribute):
1354 if node.attr != "parent":
1355 return None
1356 base = _literal_path(node.value, names)
1357 return None if base is None else _parent_literal(base)
1358 if not isinstance(node, ast.Call):
1359 return None
1360 name = getattr(node.func, "attr", None) or getattr(node.func, "id", None)
1361 if isinstance(node.func, ast.Attribute) and name in _PATH_IDENTITY_CALLS:
1362 return _literal_path(node.func.value, names) if not node.args else None
1363 if _is_os_path_join(node.func):
1364 return _literal_sequence(node, names, joiner=_join_literal)
1365 if name not in _PATH_CONSTRUCTORS or node.keywords or not node.args:
1366 return None
1367 return _literal_sequence(node, names, joiner=_join_literal)
1368
1369
1371 node: ast.Call,
1372 constants: Mapping[str, str],
1373 *,
1374 joiner: Callable[[str, str], str],
1375) -> str | None:
1376 """Fold a call's positional arguments through *joiner*, or None."""
1377 if not node.args or node.keywords:
1378 return None
1379 resolved: str | None = None
1380 for arg in node.args:
1381 part = _literal_path(arg, constants)
1382 if part is None:
1383 return None
1384 resolved = part if resolved is None else joiner(resolved, part)
1385 return resolved
1386
1387
1388def _binding_counts(tree: ast.AST) -> dict[str, int]:
1389 """How many times each name is BOUND anywhere in a module (P1).
1390
1391 A module constant is only readable while it is a constant, so the
1392 resolver admits a name bound exactly once - by the module-level
1393 assignment it read. Everything that can put a different value
1394 behind a name counts here: a second assignment, a parameter, a
1395 ``def``/``class``, an import alias, an ``except`` or ``match``
1396 capture. A ``global``/``nonlocal`` declaration counts twice, which
1397 disqualifies the name outright: it can rewrite the constant from a
1398 scope this walk cannot order.
1399 """
1400 counts: dict[str, int] = {}
1401
1402 def add(name: str, weight: int = 1) -> None:
1403 counts[name] = counts.get(name, 0) + weight
1404
1405 for node in ast.walk(tree):
1406 if isinstance(node, ast.Name) and isinstance(
1407 node.ctx, (ast.Store, ast.Del),
1408 ):
1409 add(node.id)
1410 elif isinstance(node, ast.arg):
1411 add(node.arg)
1412 elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef,
1413 ast.ClassDef)):
1414 add(node.name)
1415 elif isinstance(node, ast.alias):
1416 add(node.asname or node.name.split(".")[0])
1417 elif isinstance(node, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)):
1418 if node.name:
1419 add(node.name)
1420 elif isinstance(node, ast.MatchMapping) and node.rest:
1421 add(node.rest)
1422 elif isinstance(node, (ast.Global, ast.Nonlocal)):
1423 for name in node.names:
1424 add(name, 2)
1425 return counts
1426
1427
1428def _module_constants(tree: ast.Module, module_path: str | None) -> dict[str, str]:
1429 """The module-level names that unambiguously name a literal path (P1).
1430
1431 Read in SOURCE ORDER, so a constant may be built from one declared
1432 above it - the shape the suite writes as ``ROOT = ...`` then
1433 ``SEED = ROOT / 'seed'`` - and never from one declared below, which
1434 does not exist yet at that point in the module either.
1435
1436 Parameters
1437 ----------
1438 tree
1439 The parsed module.
1440 module_path
1441 The module's ROOT-RELATIVE POSIX path, or None when the caller
1442 analysed a source string with no place in the tree. It is what
1443 ``__file__`` resolves to; without it the anchor stays opaque
1444 and every shape built on it stays bare.
1445
1446 Returns
1447 -------
1448 dict[str, str]
1449 Name -> literal path, for names bound exactly once.
1450 """
1451 func = "_module_constants"
1452 bound = _binding_counts(tree)
1453 constants: dict[str, str] = {}
1454 if module_path is not None and not bound.get("__file__"):
1455 constants["__file__"] = module_path
1456 for node in tree.body:
1457 if isinstance(node, ast.Assign) and len(node.targets) == 1:
1458 target: ast.AST = node.targets[0]
1459 elif isinstance(node, ast.AnnAssign):
1460 target = node.target
1461 else:
1462 continue
1463 if not isinstance(target, ast.Name) or bound.get(target.id) != 1:
1464 continue
1465 resolved = _literal_path(getattr(node, "value", None), constants)
1466 if resolved is not None:
1467 constants[target.id] = resolved
1468 _dbg("OCT-TESTING", func, f"{len(constants)} literal constant(s)", 4)
1469 return constants
1470
1471
1472def _argv_element(node: ast.AST) -> str | None:
1473 """One argv element as a literal string, or None when unreadable."""
1474 if isinstance(node, ast.Constant) and isinstance(node.value, str):
1475 return node.value
1476 if (
1477 isinstance(node, ast.Attribute)
1478 and node.attr == "executable"
1479 and getattr(node.value, "id", None) == "sys"
1480 ):
1481 return PYTHON_ARGV0
1482 return None
1483
1484
1485def _literal_argv(node: ast.Call, attr: str) -> list[str | None] | None:
1486 """The argv of a launch call, as far as the literals allow (P3).
1487
1488 ``create_subprocess_exec`` takes the program and its arguments as
1489 separate positional parameters; every other seam takes one
1490 sequence.
1491
1492 A STARRED element hides the length and the position of everything
1493 AFTER it, so the argv is truncated there and only the readable
1494 PREFIX comes back (Plan 276 P3). The prefix decides the family and
1495 nothing else: the family is the PROGRAM - ``argv[0]``, plus the
1496 ``-m <module>`` pair when ``argv[0]`` is an interpreter - and a
1497 starred tail cannot change what the program is. A prefix too short
1498 to name a module still reads as the bare interpreter, which the
1499 table deliberately does not vouch for, and an argv whose FIRST
1500 element is starred has no readable program at all and stays bare.
1501 Measured on the oc_chat suite (2026-09-15): nine ``['git', *args]``
1502 helpers and one ``[sys.executable, '-m', 'oc_chat.scripts.replay',
1503 *args]`` helper were bare for this reason alone.
1504 """
1505 if not node.args:
1506 return None
1507 if attr == "create_subprocess_exec":
1508 return [_argv_element(arg) for arg in node.args]
1509 first = node.args[0]
1510 if not isinstance(first, (ast.List, ast.Tuple)):
1511 return None
1512 argv: list[str | None] = []
1513 for element in first.elts:
1514 if isinstance(element, ast.Starred):
1515 break
1516 argv.append(_argv_element(element))
1517 return argv or None
1518
1519
1520def _write_mode(node: ast.Call) -> bool:
1521 """True when an ``open``-like call requests a writing mode."""
1522 mode: str | None = None
1523 if len(node.args) > 1 and isinstance(node.args[1], ast.Constant):
1524 value = node.args[1].value
1525 mode = value if isinstance(value, str) else None
1526 for keyword in node.keywords:
1527 if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant):
1528 value = keyword.value.value
1529 mode = value if isinstance(value, str) else mode
1530 if mode is None:
1531 return False
1532 return any(flag in mode for flag in ("w", "a", "x", "+"))
1533
1534
1535def _is_path_replace(node: ast.Call) -> bool:
1536 """True only for the one-argument ``pathlib.Path.replace`` (P1).
1537
1538 ``replace`` is the one name in :data:`_PATH_WRITES` that several
1539 unrelated types also carry, and the ARITY separates them exactly.
1540 ``Path.replace(target)`` takes one argument, positionally or by
1541 that keyword name; ``str.replace(old, new)`` needs at least two;
1542 ``datetime.replace`` names a field, never ``target``; and
1543 ``dataclasses.replace(record, **changes)`` passes the record plus
1544 its changes. Measured on the oc_chat suite (Plan 284 P0): fifteen
1545 of the thirty-eight files carrying the BARE ``main_tree_write``
1546 carried it for a write that does not exist, and a bare token welds
1547 every keyed writer of its family into one group.
1548
1549 This narrows how the CALL is read, never the vocabulary. A real
1550 ``Path.replace`` still records its write, and ``os.replace`` /
1551 ``shutil`` are matched by their own earlier arms and never reach
1552 here.
1553 """
1554 if len(node.args) + len(node.keywords) != 1:
1555 return False
1556 if node.keywords:
1557 return node.keywords[0].arg == "target"
1558 return True
1559
1560
1561def _is_environ(node: ast.AST | None) -> bool:
1562 """True when *node* is ``os.environ`` or a subscript of it."""
1563 if isinstance(node, ast.Subscript):
1564 node = node.value
1565 return isinstance(node, ast.Attribute) and node.attr == "environ"
1566
1567
1568class _StaticDetector(ast.NodeVisitor):
1569 """Record the P2 resources one subtree touches."""
1570
1572 self,
1573 tainted: frozenset[str],
1574 hints: Mapping[str, str],
1575 scopes: Mapping[ast.AST, frozenset[str]] | None = None,
1576 constants: Mapping[str, str] | None = None,
1577 ) -> None:
1578 self.tainted = tainted
1579 self.base_tainted = tainted
1580 self.scopes = scopes or {}
1581 self.hints = hints
1582 self.constants = dict(constants or {})
1583 self.resources: set[str] = set()
1584 self.env_write = False
1585 self.spawns = False
1586
1587 def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
1588 """Apply newly inferred taint only inside the owning helper."""
1589 previous = self.tainted
1590 try:
1591 for child in ast.iter_child_nodes(node):
1592 self.tainted = (
1593 self.scopes.get(node, self.base_tainted)
1594 if child in node.body else previous
1595 )
1596 self.visit(child)
1597 finally:
1598 self.tainted = previous
1599
1600 visit_AsyncFunctionDef = visit_FunctionDef
1601
1602 def visit_ClassDef(self, node: ast.ClassDef) -> None:
1603 """Nested class bindings cannot inherit a helper parameter proof."""
1604 previous = self.tainted
1605 try:
1606 self.tainted = self.base_tainted
1607 self.generic_visit(node)
1608 finally:
1609 self.tainted = previous
1610
1611 def visit_Lambda(self, node: ast.Lambda) -> None:
1612 """A lambda has its own parameter bindings, which remain opaque."""
1613 previous = self.tainted
1614 try:
1615 self.tainted = self.base_tainted
1616 self.generic_visit(node)
1617 finally:
1618 self.tainted = previous
1619
1620 # -- helpers ----------------------------------------------------
1621 def _write(self, target: ast.AST | None) -> None:
1622 """Record a write to *target* unless it is temp-derived.
1623
1624 Keyed by the top-level directory when *target* is a literal
1625 path this pass can read in full (P2); bare otherwise.
1626 """
1627 if _is_tmp_expr(target, self.tainted):
1628 return
1629 literal = _literal_path(target, self.constants)
1630 key = path_key(literal) if literal is not None else None
1631 self.resources.add(write_resource(_string_hint(target), key))
1632
1633 def _launch(self, node: ast.Call, attr: str) -> None:
1634 """Record a child launch, keyed by family when readable (P3)."""
1635 self.resources |= set(launch_resources(_literal_argv(node, attr)))
1636
1637 def _bind(self, node: ast.Call) -> None:
1638 """Record a socket bind as ``port:<n>``, or fail closed."""
1639 port: int | None = None
1640 if node.args and isinstance(node.args[0], ast.Tuple):
1641 for element in node.args[0].elts:
1642 if isinstance(element, ast.Constant) and isinstance(
1643 element.value, int,
1644 ):
1645 port = element.value
1646 if port is None:
1647 self.resources.add(unknown("socket-bind-dynamic-port"))
1648 elif port > 0:
1649 self.resources.add(f"{PORT_PREFIX}{port}")
1650
1651 # -- visitors ---------------------------------------------------
1652 def visit_Assign(self, node: ast.Assign) -> None:
1653 """Catch ``os.environ[...] = ...``."""
1654 for target in node.targets:
1655 if _is_environ(target):
1656 self.env_write = True
1657 self.generic_visit(node)
1658
1659 def visit_Delete(self, node: ast.Delete) -> None:
1660 """Catch ``del os.environ[...]``."""
1661 for target in node.targets:
1662 if _is_environ(target):
1663 self.env_write = True
1664 self.generic_visit(node)
1665
1666 def visit_Call(self, node: ast.Call) -> None:
1667 """Dispatch one call expression through every signature."""
1668 func = node.func
1669 name = getattr(func, "id", None)
1670 attr = getattr(func, "attr", None)
1671 owner = getattr(func, "value", None)
1672 owner_name = getattr(owner, "id", None)
1673
1674 if name == "open" and _write_mode(node):
1675 self._write(node.args[0] if node.args else None)
1676 elif name in _RESET_NAMES or attr in _RESET_NAMES:
1677 self.resources.add(RES_GLOBAL_SINGLETON_RESET)
1678 elif name in _SUBPROCESS_NAMES:
1679 self.spawns = True
1680 elif attr is not None:
1681 self._visit_attr_call(node, attr, owner, owner_name)
1682
1683 hinted = self.hints.get(name or "") or self.hints.get(owner_name or "")
1684 if hinted is not None and not any(
1685 _is_tmp_expr(arg, self.tainted) for arg in node.args
1686 ):
1687 self.resources.add(hinted)
1688 if any(keyword.arg == "env" for keyword in node.keywords):
1689 self.env_write = True
1690 self.generic_visit(node)
1691
1693 self,
1694 node: ast.Call,
1695 attr: str,
1696 owner: ast.AST | None,
1697 owner_name: str | None,
1698 ) -> None:
1699 """The ``<something>.<attr>(...)`` half of :meth:`visit_Call`."""
1700 args = node.args
1701 if owner_name == "os" and attr == "chdir":
1702 self.resources.add(RES_CWD_CHANGE)
1703 elif owner_name == "os" and attr in _OS_WRITES:
1704 self._write(args[0] if args else None)
1705 elif owner_name == "os" and attr in ("system", "popen", "execv"):
1706 self.spawns = True
1707 elif owner_name == "shutil" and attr in _SHUTIL_WRITES:
1708 index = _SHUTIL_WRITES[attr]
1709 self._write(args[index] if len(args) > index else None)
1710 elif owner_name in _SUBPROCESS_MODULES and attr in _SUBPROCESS_ATTRS:
1711 self._launch(node, attr)
1712 elif owner_name == "sqlite3" and attr == "connect":
1713 target = args[0] if args else next(
1714 (kw.value for kw in node.keywords if kw.arg == "database"), None,
1715 )
1716 value = target.value if isinstance(target, ast.Constant) else None
1717 if not (isinstance(value, str) and (
1718 value == ":memory:" or value.startswith("file::memory:")
1719 )):
1720 self._write(target)
1721 elif attr == "bind":
1722 self._bind(node)
1723 elif attr in _NETWORK_ATTRS or owner_name in _NETWORK_MODULES:
1724 self.resources.add(RES_NETWORK)
1725 elif _is_environ(owner) and attr in (
1726 "update", "pop", "setdefault", "clear",
1727 ):
1728 self.env_write = True
1729 elif attr == "open" and _write_mode(node):
1730 self._write(owner)
1731 elif attr in _PATH_WRITES:
1732 if attr == "replace" and not _is_path_replace(node):
1733 return
1734 self._write(owner)
1735
1736 def result(self) -> frozenset[str]:
1737 """The recorded resources, with the launch and mutation joins.
1738
1739 A child launch alone is enough for :data:`RES_ENV_SUBPROCESS`:
1740 it inherits the environment, and what it does with it is
1741 invisible from here. ``self.spawns`` carries only the launches
1742 this pass could NOT attribute - a shell string, a computed
1743 argv, a bare ``os.system`` - and those contribute the bare
1744 wildcard (P3).
1745
1746 An ``os.environ`` write contributes :data:`RES_ENV_MUTATION`
1747 and nothing else (Plan 276 P1): it is recorded, so the manifest
1748 keeps the full picture, but it is process-local, so on its own
1749 it leaves the file free.
1750 """
1751 resources = set(self.resources)
1752 if self.spawns:
1753 resources.add(RES_ENV_SUBPROCESS)
1754 if self.env_write:
1755 resources.add(RES_ENV_MUTATION)
1756 return frozenset(resources)
1757
1758
1759def _requested_names(tree: ast.AST) -> frozenset[str]:
1760 """Every parameter name any function in *tree* requests."""
1761 names: set[str] = set()
1762 for node in ast.walk(tree):
1763 if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
1764 spec = node.args
1765 for arg in (*spec.posonlyargs, *spec.args, *spec.kwonlyargs):
1766 names.add(arg.arg)
1767 return frozenset(names)
1768
1769
1771 source: str,
1772 *,
1773 filename: str = "<test>",
1774 tmp_seeds: Iterable[str] = (),
1775) -> ModuleAnalysis:
1776 """Statically analyse one module.
1777
1778 Parameters
1779 ----------
1780 source
1781 Module source text.
1782 filename
1783 Reported in diagnostics only.
1784 tmp_seeds
1785 Fixture names from an enclosing conftest whose value is a
1786 temporary path; requesting one taints the parameter.
1787
1788 Returns
1789 -------
1790 :class:`ModuleAnalysis`
1791
1792 Notes
1793 -----
1794 A file that does not parse is not silently clean: it comes back
1795 carrying ``unknown:parse-error``, which conflicts with everything.
1796 """
1797 func = "analyse_source"
1798 try:
1799 tree = ast.parse(source, filename=filename)
1800 except SyntaxError as exc:
1801 _dbg("OCT-TESTING", func, f"parse error in {filename}: {exc}", 2)
1802 return ModuleAnalysis(
1803 resources=frozenset({unknown("parse-error")}),
1804 fixtures={},
1805 autouse=frozenset(),
1806 tmp_fixtures=frozenset(),
1807 requested=frozenset(),
1808 )
1809
1810 hints = _import_hints(tree)
1811 tainted = _collect_taint(tree, tmp_seeds)
1812 scopes = _helper_parameter_taints(tree, tainted, tmp_seeds)
1813 constants = _module_constants(
1814 tree, filename if filename.endswith(".py") else None,
1815 )
1816 module = _StaticDetector(tainted, hints, scopes, constants)
1817 module.visit(tree)
1818
1819 fixtures: dict[str, frozenset[str]] = {}
1820 autouse: set[str] = set()
1821 tmp_fixtures: set[str] = set()
1822 for node in ast.walk(tree):
1823 if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
1824 continue
1825 is_fixture, is_autouse = _fixture_decorated(node)
1826 if not is_fixture:
1827 continue
1828 detector = _StaticDetector(
1829 scopes.get(node, tainted), hints, scopes, constants,
1830 )
1831 for child in node.body:
1832 detector.visit(child)
1833 fixtures[node.name] = detector.result()
1834 if is_autouse:
1835 autouse.add(node.name)
1836 if node.name in tainted:
1837 tmp_fixtures.add(node.name)
1838
1839 return ModuleAnalysis(
1840 resources=module.result(),
1841 fixtures=fixtures,
1842 autouse=frozenset(autouse),
1843 tmp_fixtures=frozenset(tmp_fixtures),
1844 requested=_requested_names(tree),
1845 )
1846
1847
1849 source: str,
1850 *,
1851 filename: str = "<test>",
1852 conftests: Sequence[ModuleAnalysis] = (),
1853) -> frozenset[str]:
1854 """Classify one test module, folding in the conftests above it.
1855
1856 A file inherits the resources of every AUTOUSE conftest fixture on
1857 its path, plus those of the conftest fixtures it requests by name
1858 (P3a).
1859 """
1860 func = "classify_source"
1861 seeds: set[str] = set()
1862 for analysis in conftests:
1863 seeds |= set(analysis.tmp_fixtures)
1864 own = analyse_source(source, filename=filename, tmp_seeds=seeds)
1865
1866 resources = set(own.resources)
1867 for analysis in conftests:
1868 for name in analysis.autouse:
1869 resources |= analysis.fixtures.get(name, frozenset())
1870 for name in own.requested:
1871 resources |= analysis.fixtures.get(name, frozenset())
1872 _dbg("OCT-TESTING", func, f"{filename}: {sorted(resources)}", 4)
1873 return frozenset(resources)
1874
1875
1876def _read(path: Path) -> str:
1877 """Read *path* as UTF-8, tolerating a stray decoding error."""
1878 return path.read_text(encoding="utf-8", errors="replace")
1879
1880
1882 root: Path,
1883 path: Path,
1884 cache: dict[Path, ModuleAnalysis],
1885) -> list[ModuleAnalysis]:
1886 """Analyse every ``conftest.py`` from *root* down to *path*."""
1887 directories: list[Path] = []
1888 current = path.parent
1889 while True:
1890 directories.append(current)
1891 if current == root or current.parent == current:
1892 break
1893 current = current.parent
1894
1895 chain: list[ModuleAnalysis] = []
1896 for directory in reversed(directories):
1897 conftest = directory / "conftest.py"
1898 if not conftest.is_file():
1899 continue
1900 if conftest not in cache:
1901 cache[conftest] = analyse_source(
1902 _read(conftest),
1903 filename=conftest.relative_to(root).as_posix(),
1904 )
1905 chain.append(cache[conftest])
1906 return chain
1907
1908
1910 root: Path,
1911 test_roots: Sequence[str],
1912 *,
1913 pattern: str = "test_*.py",
1914) -> dict[str, frozenset[str]]:
1915 """Statically classify every test file under *test_roots*.
1916
1917 Parameters
1918 ----------
1919 root
1920 Subproject root; manifest keys are relative to it.
1921 test_roots
1922 Root-relative test directories (the workspace manifest's
1923 ``test.test_roots``).
1924 pattern
1925 Test-file glob, matching pytest's ``python_files``.
1926
1927 Returns
1928 -------
1929 dict[str, frozenset[str]]
1930 Root-relative POSIX path -> resource set.
1931 """
1932 func = "classify_tree"
1933 _dbg("OCT-TESTING", func, f"root={root} roots={list(test_roots)}", 2)
1934 cache: dict[Path, ModuleAnalysis] = {}
1935 results: dict[str, frozenset[str]] = {}
1936
1937 for test_root in test_roots:
1938 base = root / test_root
1939 if not base.is_dir():
1940 _dbg("OCT-TESTING", func, f"missing test root {test_root}", 2)
1941 continue
1942 for path in sorted(base.rglob(pattern)):
1943 if not path.is_file():
1944 continue
1945 rel = path.relative_to(root).as_posix()
1946 results[rel] = classify_source(
1947 _read(path),
1948 filename=rel,
1949 conftests=_conftest_chain(root, path, cache),
1950 )
1951 _dbg("OCT-TESTING", func, f"classified {len(results)} file(s)", 2)
1952 return results
1953
1954
1955# ---------------------------------------------------------------------
1956# P4 - grouping: connected components of the conflict graph
1957# ---------------------------------------------------------------------
1958
1959def _find(parent: dict[str, str], path: str) -> str:
1960 """Union-find root of *path*, with path compression."""
1961 while parent[path] != path:
1962 parent[path] = parent[parent[path]]
1963 path = parent[path]
1964 return path
1965
1966
1967def _union(parent: dict[str, str], left: str, right: str) -> None:
1968 """Merge the components of *left* and *right*."""
1969 root_left, root_right = _find(parent, left), _find(parent, right)
1970 if root_left != root_right:
1971 parent[root_right] = root_left
1972
1973
1974def _slug(resource: str) -> str:
1975 """Group-name form of a resource token."""
1976 return resource.replace(":", "-").replace("/", "-")
1977
1978
1980 components: Mapping[str, list[str]],
1981 sets: Mapping[str, frozenset[str]],
1982) -> list[tuple[str, list[str]]]:
1983 """Assign deterministic names to the conflict components."""
1984 ordered: list[tuple[tuple[str, ...], list[str]]] = []
1985 for members in components.values():
1986 resources: set[str] = set()
1987 for member in members:
1988 resources |= sets[member]
1989 ordered.append((tuple(sorted(resources)), sorted(members)))
1990 ordered.sort(key=lambda item: (item[0], item[1][0]))
1991
1992 named: list[tuple[str, list[str]]] = []
1993 used: dict[str, int] = {}
1994 for resources, members in ordered:
1995 stem = _slug(resources[0])
1996 used[stem] = used.get(stem, 0) + 1
1997 name = stem if used[stem] == 1 else f"{stem}-{used[stem]}"
1998 named.append((name, members))
1999 return named
2000
2001
2003 classified: Mapping[str, Iterable[str]],
2004 *,
2005 pins: Mapping[str, str] | None = None,
2006) -> dict[str, str]:
2007 """Group files into the connected components of the conflict graph.
2008
2009 Files sharing any CONFLICTING resource are connected, where
2010 "sharing" follows :func:`conflicts_with`: two KEYED tokens share
2011 only when their keys are equal, and a BARE family token shares
2012 with every keyed member of its own family (Plan 247 P1). A file
2013 carrying an ``unknown:*`` token connects to every file that
2014 carries any conflicting resource (fail closed, P2); a file left
2015 with none - no resources at all, or only
2016 :data:`PROCESS_LOCAL_RESOURCES` - is :data:`GROUP_FREE` and
2017 connects to nothing. Names derive from each component's own
2018 sorted resources, so the same input always produces the same
2019 names.
2020
2021 ``env_subprocess`` is the one family whose BARE token does not
2022 reach its keyed members (Plan 276 P2). A worker is a separate
2023 PROCESS, so the environment a child inherits never crosses one,
2024 and an unattributable launch therefore holds only the other
2025 unattributable launches. Measured on the oc_chat suite
2026 (2026-09-15): the removed weld alone put 100 files carrying a
2027 quarter of the suite's tests into a single group, of which 46
2028 held the bare token.
2029
2030 Parameters
2031 ----------
2032 classified
2033 Root-relative path -> resource set.
2034 pins
2035 Operator-pinned ``path -> group`` overrides (P7). A pinned
2036 file leaves the graph entirely and keeps its own group name.
2037
2038 Returns
2039 -------
2040 dict[str, str]
2041 Root-relative path -> group name.
2042 """
2043 func = "build_groups"
2044 pins = dict(pins or {})
2045 sets = {path: frozenset(res) for path, res in classified.items()}
2046 # A non-conflicting launch token must remain visible to the GLOBAL
2047 # wildcard, which fails closed on everything. A BARE launch no
2048 # longer makes it busy (Plan 276 P2).
2049 unpinned = [res for path, res in sets.items() if path not in pins]
2050 global_wildcard = any(r.startswith(UNKNOWN_PREFIX)
2051 for res in unpinned for r in res)
2052 busy = {
2053 path: frozenset(r for r in resources if is_conflicting(r) or (
2054 resource_family(r) == RES_ENV_SUBPROCESS and global_wildcard
2055 ))
2056 for path, resources in sets.items()
2057 if path not in pins
2058 }
2059 busy = {path: res for path, res in busy.items() if res}
2060
2061 parent: dict[str, str] = {path: path for path in busy}
2062 by_resource: dict[str, list[str]] = {}
2063 by_family: dict[str, list[str]] = {}
2064 family_wildcards: dict[str, list[str]] = {}
2065 wildcards: list[str] = []
2066 for path, resources in busy.items():
2067 for resource in resources:
2068 by_resource.setdefault(resource, []).append(path)
2069 family = resource_family(resource)
2070 if family in KEYED_FAMILIES:
2071 by_family.setdefault(family, []).append(path)
2072 if resource_key(resource) is None:
2073 family_wildcards.setdefault(family, []).append(path)
2074 if resource.startswith(UNKNOWN_PREFIX):
2075 wildcards.append(path)
2076 for resource, members in by_resource.items():
2077 if not is_conflicting(resource):
2078 continue
2079 for other in members[1:]:
2080 _union(parent, members[0], other)
2081 # A BARE family token conflicts with every keyed member of its own
2082 # family and with nothing else (P1); only ``unknown:*`` is global.
2083 # ``env_subprocess`` is exempt (Plan 276 P2): its bare token holds
2084 # only the other bare tokens, which the equal-token pass above has
2085 # already joined.
2086 for family, holders in family_wildcards.items():
2087 if family == RES_ENV_SUBPROCESS:
2088 continue
2089 for path in holders:
2090 for other in by_family.get(family, ()):
2091 _union(parent, path, other)
2092 for path in wildcards:
2093 for other in busy:
2094 _union(parent, path, other)
2095
2096 components: dict[str, list[str]] = {}
2097 for path in busy:
2098 components.setdefault(_find(parent, path), []).append(path)
2099
2100 groups: dict[str, str] = {}
2101 for name, members in _named_components(components, busy):
2102 for member in members:
2103 groups[member] = name
2104 for path in sets:
2105 if path in pins:
2106 groups[path] = pins[path]
2107 elif path not in groups:
2108 groups[path] = GROUP_FREE
2109 _dbg(
2110 "OCT-TESTING", func,
2111 f"{len(components)} conflict group(s) over {len(busy)} file(s)", 3,
2112 )
2113 return groups
2114
2115
2116# ---------------------------------------------------------------------
2117# Manifest read / write
2118# ---------------------------------------------------------------------
2119
2121 classified: Mapping[str, Iterable[str]],
2122 *,
2123 tool_version: str,
2124 pins: Mapping[str, str] | None = None,
2125 generated_at: str | None = None,
2126) -> dict[str, Any]:
2127 """Build the committed manifest document (the P4 shape)."""
2128 func = "build_manifest"
2129 groups = build_groups(classified, pins=pins)
2130 files = {
2131 path: {
2132 "resources": sorted(classified[path]),
2133 "group": groups[path],
2134 }
2135 for path in sorted(classified)
2136 }
2137 stamp = generated_at or datetime.now(timezone.utc).strftime(
2138 "%Y-%m-%dT%H:%M:%SZ",
2139 )
2140 _dbg("OCT-TESTING", func, f"manifest over {len(files)} file(s)", 2)
2141 return {
2142 "schema_version": SCHEMA_VERSION,
2143 "generated_at": stamp,
2144 "tool_version": tool_version,
2145 "files": files,
2146 }
2147
2148
2149def write_manifest(path: Path, document: Mapping[str, Any]) -> None:
2150 """Write *document* to *path* as stable, newline-terminated JSON."""
2151 func = "write_manifest"
2152 path.parent.mkdir(parents=True, exist_ok=True)
2153 text = json.dumps(document, indent=2) + "\n"
2154 atomic_write_text(path, text, encoding="utf-8", newline="\n")
2155 _dbg("OCT-TESTING", func, f"wrote {path}", 2)
2156
2157
2158def load_manifest(path: Path) -> ParallelManifest:
2159 """Load and validate a committed ``parallel_manifest.json``.
2160
2161 Raises
2162 ------
2163 OctTestError
2164 ``9102`` when the file is unreadable, is not JSON, or does not
2165 carry the P4 shape. A malformed manifest never degrades to an
2166 empty one: that would silently widen parallelism.
2167 """
2168 func = "load_manifest"
2169 try:
2170 raw = json.loads(Path(path).read_text(encoding="utf-8"))
2171 except (OSError, json.JSONDecodeError) as exc:
2172 raise OctTestError(
2173 MANIFEST_TEST_BLOCK_INVALID,
2174 f"parallel manifest {path} is unreadable: {exc}",
2175 ) from exc
2176 if not isinstance(raw, dict) or not isinstance(raw.get("files"), dict):
2177 raise OctTestError(
2178 MANIFEST_TEST_BLOCK_INVALID,
2179 f"parallel manifest {path} has no 'files' object",
2180 )
2181
2182 files: dict[str, FileEntry] = {}
2183 for key, value in raw["files"].items():
2184 group = value.get("group") if isinstance(value, dict) else None
2185 resources = value.get("resources") if isinstance(value, dict) else None
2186 if not isinstance(group, str) or not isinstance(resources, list):
2187 raise OctTestError(
2188 MANIFEST_TEST_BLOCK_INVALID,
2189 f"parallel manifest entry {key!r} is malformed",
2190 )
2191 files[key] = FileEntry(
2192 path=key, resources=tuple(resources), group=group,
2193 )
2194 _dbg("OCT-TESTING", func, f"loaded {len(files)} entry/entries", 3)
2195 return ParallelManifest(
2196 schema_version=str(raw.get("schema_version", "")),
2197 generated_at=str(raw.get("generated_at", "")),
2198 tool_version=str(raw.get("tool_version", "")),
2199 files=files,
2200 )
2201
2202
2204 manifest: ParallelManifest,
2205 relpaths: Iterable[str],
2206) -> tuple[dict[str, str], tuple[str, ...]]:
2207 """Map collected file paths to groups, failing closed on absence.
2208
2209 Returns
2210 -------
2211 tuple
2212 ``(path -> group, missing)``. A collected file with no
2213 manifest entry maps to :data:`GROUP_UNCLASSIFIED` and is
2214 listed in *missing*, so a stale manifest can never silently
2215 widen parallelism.
2216 """
2217 func = "resolve_groups"
2218 groups: dict[str, str] = {}
2219 missing: set[str] = set()
2220 for relpath in relpaths:
2221 entry = manifest.files.get(relpath)
2222 if entry is None:
2223 groups[relpath] = GROUP_UNCLASSIFIED
2224 missing.add(relpath)
2225 else:
2226 groups[relpath] = entry.group
2227 if missing:
2228 _dbg(
2229 "OCT-TESTING", func,
2230 f"{len(missing)} collected file(s) absent from the manifest", 2,
2231 )
2232 return groups, tuple(sorted(missing))
2233
2234
2235# ---------------------------------------------------------------------
2236# P1/P5 - the worker cap and the pytest option seams
2237# ---------------------------------------------------------------------
2238
2239def _raw_max_workers(workspace_root: Path, package: str) -> int:
2240 """Read ``max_workers`` straight from the manifest JSON.
2241
2242 The compatibility path for an INSTALLED ``oct.testing.manifest``
2243 that predates the P1 key; it applies the same validation the
2244 parser does, so an invalid value still refuses rather than
2245 degrading to the default.
2246 """
2247 func = "_raw_max_workers"
2248 path = workspace_root / ".option_c_workspace.json"
2249 try:
2250 raw = json.loads(path.read_text(encoding="utf-8"))
2251 except (OSError, json.JSONDecodeError) as exc:
2252 _dbg("OCT-TESTING", func, f"raw manifest read failed: {exc}", 2)
2253 return DEFAULT_MAX_WORKERS
2254 for entry in raw.get("subprojects", []):
2255 if not isinstance(entry, dict) or entry.get("name") != package:
2256 continue
2257 block = entry.get("test")
2258 if not isinstance(block, dict) or "max_workers" not in block:
2259 return DEFAULT_MAX_WORKERS
2260 value = block["max_workers"]
2261 if isinstance(value, bool) or not isinstance(value, int) or value < 1:
2262 raise OctTestError(
2263 MANIFEST_TEST_BLOCK_INVALID,
2264 f"subprojects[{package}].test.max_workers must be a "
2265 f"positive integer, got {value!r}",
2266 )
2267 return value
2268 return DEFAULT_MAX_WORKERS
2269
2270
2271def resolve_max_workers(workspace_root: Path, package: str) -> int:
2272 """Read ``subprojects[].test.max_workers`` for *package* (P1/P5).
2273
2274 REFUSES a manifest-level failure instead of degrading to the
2275 shipped default (Plan 247 P5): a workspace manifest that EXISTS
2276 and cannot be loaded, and a package that manifest does not
2277 declare, are both configuration errors, and the only caller is a
2278 root conftest deciding how many workers to inject - degrading
2279 there ran two dozen workers against a workspace nobody could
2280 parse, silently.
2281
2282 A manifest that is ABSENT ENTIRELY is not a misconfiguration and
2283 still resolves the shipped default (operator ruling 2026-09-08):
2284 there is no configuration to be wrong about, and the §20 stand-in
2285 is asked to run over standalone fixture trees that carry no
2286 workspace manifest by design - refusing there declined to
2287 parallelise fourteen witnesses that had nothing wrong with them.
2288 An ABSENT ``max_workers`` key on a DECLARED package likewise still
2289 defaults, as does an installed manifest parser that predates the
2290 key, through :func:`_raw_max_workers`.
2291
2292 Raises
2293 ------
2294 OctTestError
2295 Whatever :func:`load_test_manifest` raises for a workspace
2296 manifest that exists but is unusable, and ``9102`` when
2297 *package* is not declared in one that loaded.
2298 """
2299 func = "resolve_max_workers"
2300 try:
2301 manifest = load_test_manifest(Path(workspace_root))
2302 except OctTestError as exc:
2303 if exc.code != WORKSPACE_FILE_NOT_FOUND:
2304 raise
2305 _dbg("OCT-TESTING", func, f"no workspace at {workspace_root}", 2)
2306 return DEFAULT_MAX_WORKERS
2307 config = manifest.packages_by_name.get(package)
2308 if config is None:
2309 raise OctTestError(
2310 MANIFEST_TEST_BLOCK_INVALID,
2311 f"package {package!r} is not declared in the workspace "
2312 f"manifest at {workspace_root}, so its worker cap cannot "
2313 f"be resolved",
2314 )
2315 value = getattr(config, "max_workers", None)
2316 if value is None:
2317 value = _raw_max_workers(Path(workspace_root), package)
2318 _dbg("OCT-TESTING", func, f"{package} max_workers={value}", 3)
2319 return int(value)
2320
2321
2323 """The affinity-aware CPU count (ADR-T-11's v0.4 amendment).
2324
2325 The ONE normative resolution every worker-count formula reads::
2326
2327 os.process_cpu_count() # Python >= 3.13
2328 else len(os.sched_getaffinity(0)) # where the OS exposes it
2329 else (os.cpu_count() or 1)
2330
2331 A bare ``os.cpu_count()`` overcounts inside a container and under
2332 CPU affinity (OI-027) - exactly where over-spawning hurts most.
2333 """
2334 func = "available_cpu_count"
2335 getter = getattr(os, "process_cpu_count", None)
2336 if getter is not None:
2337 count = getter()
2338 if count:
2339 _dbg("OCT-TESTING", func, f"process_cpu_count={count}", 4)
2340 return int(count)
2341 affinity = getattr(os, "sched_getaffinity", None)
2342 if affinity is not None:
2343 try:
2344 count = len(affinity(0))
2345 except OSError as exc: # pragma: no cover - platform specific
2346 _dbg("OCT-TESTING", func, f"affinity unavailable: {exc}", 3)
2347 else:
2348 if count:
2349 _dbg("OCT-TESTING", func, f"affinity={count}", 4)
2350 return count
2351 return os.cpu_count() or 1
2352
2353
2355 workspace_root: Path,
2356 package: str,
2357 *,
2358 cpu_count: int | None = None,
2359) -> int:
2360 """Resolve ``auto`` to ``min(max_workers, available_cpu_count())``.
2361
2362 P1 for the cap, ADR-T-11's v0.4 amendment for the CPU count (P5).
2363 """
2364 func = "resolve_worker_count"
2365 cap = resolve_max_workers(workspace_root, package)
2366 cores = cpu_count if cpu_count is not None else available_cpu_count()
2367 count = max(1, min(cap, max(1, cores)))
2368 _dbg(
2369 "OCT-TESTING", func,
2370 f"cap={cap} cores={cores} -> {count} worker(s)", 3,
2371 )
2372 return count
2373
2374
2375def free_group_name(nodeid: str, *, whole_file: bool = False) -> str:
2376 """The per-test group name a ``free`` item carries (Plan 286 P1).
2377
2378 A free file shares no resource with anything, so its tests may run
2379 anywhere - including in different workers. The name is therefore
2380 the item's own nodeid, not the group ``free`` every free file used
2381 to share: ``--dist loadgroup`` keeps ONE group on ONE worker, so
2382 the shared name pinned the whole free set to a single worker
2383 (measured on the oc_chat suite from this worktree, BASE
2384 ``3297fae7``: of twenty workers, gw0 ran all 3877 tests of all 283
2385 free files and only three other workers ran anything at all).
2386
2387 *whole_file* is the P2 opt-out: the name is then the FILE's path,
2388 so that one file's tests stay together while every other free file
2389 still spreads.
2390
2391 Parameters
2392 ----------
2393 nodeid
2394 The collected item's nodeid, ``<relpath>::<test>``.
2395 whole_file
2396 True when the item carries :data:`WHOLE_FILE_MARKER`.
2397 """
2398 stem = nodeid.split("::")[0] if whole_file else nodeid
2399 return f"{FREE_GROUP_PREFIX}{stem}"
2400
2401
2403 config: Any,
2404 items: Iterable[Any],
2405 manifest_path: Path,
2406) -> tuple[tuple[str, ...], str | None]:
2407 """Stamp every collected item with its scheduling group (P4/P5).
2408
2409 Call from a root conftest's ``pytest_collection_modifyitems``
2410 under ``@pytest.hookimpl(tryfirst=True)``: xdist's worker-side
2411 hook is what turns the mark into the ``@<group>`` nodeid suffix
2412 its scheduler reads, and it runs FIRST otherwise, so a later mark
2413 is collected and then ignored.
2414
2415 A file in a real conflict group, and a file the manifest does not
2416 classify, are stamped with that group's ONE name, exactly as
2417 before: both mean "these tests must not run beside each other".
2418 A ``free`` file's items each carry their own name through
2419 :func:`free_group_name` (Plan 286 P1), because a free file shares
2420 nothing and one name is one worker.
2421
2422 Parameters
2423 ----------
2424 config
2425 The pytest ``Config``.
2426 items
2427 The collected items, modified in place.
2428 manifest_path
2429 The committed ``parallel_manifest.json``.
2430
2431 Returns
2432 -------
2433 tuple
2434 ``(missing, manifest_error)``: the collected files with no
2435 manifest entry, and the load failure when the manifest itself
2436 could not be read. Both are fail-closed states - every
2437 affected item is stamped :data:`GROUP_UNCLASSIFIED`, so it
2438 runs serially with respect to the other unclassified files
2439 and a completeness check can fail the run by name.
2440 """
2441 func = "apply_group_marks"
2442 import pytest # local: an optional dependency of oct
2443
2444 collected = list(items)
2445 groups: dict[str, str] = {}
2446 missing: tuple[str, ...] = ()
2447 error: str | None = None
2448 try:
2449 manifest = load_manifest(Path(manifest_path))
2450 except OctTestError as exc:
2451 error = f"{Path(manifest_path).name}: {exc}"
2452 _dbg("OCT-TESTING", func, f"manifest unusable: {error}", 2)
2453 else:
2454 groups, missing = resolve_groups(
2455 manifest,
2456 {item.nodeid.split("::")[0] for item in collected},
2457 )
2458 if config.pluginmanager.hasplugin("xdist"):
2459 for item in collected:
2460 group = groups.get(
2461 item.nodeid.split("::")[0], GROUP_UNCLASSIFIED,
2462 )
2463 if group == GROUP_FREE:
2464 group = free_group_name(
2465 item.nodeid,
2466 whole_file=item.get_closest_marker(
2467 WHOLE_FILE_MARKER,
2468 ) is not None,
2469 )
2470 item.add_marker(pytest.mark.xdist_group(name=group))
2471 return missing, error
2472
2473
2474def is_xdist_worker(config: Any) -> bool:
2475 """True when *config* belongs to an xdist worker process.
2476
2477 The guard that stops a worker re-injecting ``-n`` and spawning its
2478 own workers: xdist resets ``numprocesses`` to ``None`` in the
2479 worker's config, which otherwise looks exactly like "no ``-n`` was
2480 given" to :func:`configure_parallel_defaults`. The environment
2481 half also catches a NESTED pytest run launched from inside a
2482 worker, which inherits ``PYTEST_XDIST_WORKER`` and must stay
2483 serial.
2484 """
2485 return hasattr(config, "workerinput") or bool(
2486 os.environ.get("PYTEST_XDIST_WORKER"),
2487 )
2488
2489
2490def _adopt_group_dist_in_worker(config: Any, option: Any, dist: str) -> None:
2491 """Re-assert ``loadgroup`` inside an xdist worker process.
2492
2493 xdist ships the CONTROLLER's ORIGINAL command line to each worker
2494 (``workermanage.WorkerController.setup`` sends
2495 ``config.invocation_params.args``), so an in-memory ``--dist``
2496 chosen by :func:`configure_parallel_defaults` is invisible there.
2497 The worker's ``remote.setup_config`` derives its private
2498 ``option.loadgroup`` flag from the dist it can see and then forces
2499 ``dist='no'``; that flag is what makes the worker append the
2500 ``@<group>`` nodeid suffix the ``loadgroup`` scheduler reads, so
2501 without this the marks are collected and then ignored, and a group
2502 silently spreads across workers (measured, 2026-09-06).
2503
2504 An explicit ``--dist`` on the command line is left alone: there
2505 the worker already computed the flag from the operator's choice.
2506 """
2507 func = "_adopt_group_dist_in_worker"
2508 args = [str(arg) for arg in (config.invocation_params.args or ())]
2509 if any(arg.startswith("--dist") for arg in args):
2510 return
2511 option.loadgroup = dist == "loadgroup"
2512 _dbg("OCT-TESTING", func, f"worker loadgroup={option.loadgroup}", 3)
2513
2514
2516 config: Any,
2517 *,
2518 workspace_root: Path,
2519 package: str,
2520 dist: str = "loadgroup",
2521) -> int | None:
2522 """Turn parallelism on by default for a run (P5).
2523
2524 Call this from a root conftest's ``pytest_cmdline_main`` under
2525 ``@pytest.hookimpl(tryfirst=True)``: xdist's own ``tryfirst`` hook
2526 is what turns ``numprocesses`` into ``tx`` endpoints, so a later
2527 assignment engages nothing.
2528
2529 Returns
2530 -------
2531 int | None
2532 The resolved worker count, or ``None`` when the run was left
2533 alone: an xdist worker process, a nested run inside one,
2534 ``-p no:xdist``, or an explicit ``-n`` on the command line.
2535 """
2536 func = "configure_parallel_defaults"
2537 option = getattr(config, "option", None)
2538 if option is None or not hasattr(option, "numprocesses"):
2539 _dbg("OCT-TESTING", func, "xdist absent; serial run", 2)
2540 return None
2541 if hasattr(config, "workerinput"):
2542 _adopt_group_dist_in_worker(config, option, dist)
2543 return None
2544 if is_xdist_worker(config):
2545 _dbg("OCT-TESTING", func, "nested run inside a worker; serial", 2)
2546 return None
2547
2548 resolved: int | None = None
2549 if option.numprocesses is None:
2550 resolved = resolve_worker_count(workspace_root, package)
2551 option.numprocesses = resolved
2552 if option.numprocesses and getattr(option, "dist", "no") == "no":
2553 option.dist = dist
2554 _dbg(
2555 "OCT-TESTING", func,
2556 f"numprocesses={option.numprocesses} dist={option.dist}", 2,
2557 )
2558 return resolved
2559
2560
2561# ---------------------------------------------------------------------
2562# P3b - the dynamic detector (``--oct-trace``)
2563# ---------------------------------------------------------------------
2564
2565#: Command-line flag that switches the dynamic detector on.
2566TRACE_OPTION: str = "--oct-trace"
2567
2568#: Path fragments whose writes are never a shared-resource use.
2569_TRACE_IGNORED: tuple[str, ...] = (
2570 "__pycache__", "site-packages", ".pytest_cache", "/.git/",
2571 ".hypothesis", ".egg-info",
2572)
2573
2574
2575def _from_pytest_internals(depth: int = 4) -> bool:
2576 """True when the calling code is pytest's own machinery.
2577
2578 ``monkeypatch.setenv`` writes through ``os.environ`` like any
2579 other caller, but it records an undo, so it is not a leak and must
2580 not be recorded as one. Neither is pytest's own
2581 ``PYTEST_CURRENT_TEST`` bookkeeping - which reaches ``__delitem__``
2582 through ``MutableMapping.pop`` in ``<frozen _collections_abc>``,
2583 so the search skips FROZEN frames and judges the first real one.
2584 Without that skip every file in a suite reads as an environment
2585 writer (measured on all 324 oc_chat files, 2026-09-06).
2586 """
2587 frame: Any = sys._getframe(2)
2588 for _ in range(depth):
2589 if frame is None:
2590 return False
2591 filename = frame.f_code.co_filename.replace("\\", "/")
2592 if "/_pytest/" in filename:
2593 return True
2594 if not filename.startswith("<"):
2595 return False
2596 frame = frame.f_back
2597 return False
2598
2599
2600def _is_pid_stamped(basename: str, pid: str) -> bool:
2601 """True when *basename* carries *pid* as a whole token.
2602
2603 A pid-stamped artefact - oc_diagnostics writes
2604 ``logs/unified_debug_logs_<stamp>_p<pid>.txt`` - is per-process by
2605 construction, so it can never collide across workers, and it is
2606 written lazily by whichever test happens to initialise diagnostics
2607 first, so attributing it would make the classification depend on
2608 run order.
2609
2610 The match is on WHOLE tokens, never a substring. A bare
2611 ``str(pid) in basename`` silently swallowed a GENUINE
2612 ``main_tree_write``: with pid 648, the artefact filename
2613 ``..._1788651058142648000_6d28ee.txt`` (a nanosecond stamp) matched
2614 by coincidence, which dropped
2615 ``tests/unit/test_203_event_payload_completeness.py`` out of its
2616 conflict group and made the classification differ between two runs
2617 of the same tree (gate finding R1, measured 2026-09-06).
2618 """
2619 tokens = re.split(r"[^0-9A-Za-z]+", basename)
2620 return any(token in (pid, f"p{pid}") for token in tokens)
2621
2622
2624 """Wrap the process-global seams and attribute them to a file.
2625
2626 Only WRITE-shaped events are recorded - a read cannot collide.
2627 Every observation is attributed to the test file that was running
2628 when it happened, so the trace shares the static pass's key space.
2629
2630 Every wrapper is ``functools.wraps``-ed onto the seam it replaces,
2631 so a wrapped function still reports the original's ``__name__``,
2632 ``__qualname__``, ``__doc__`` and ``__wrapped__``: code under test
2633 that inspects the identity of a patched binding must see what it
2634 would see untraced (gate finding R2).
2635 """
2636
2637 def __init__(self, root: Path) -> None:
2638 self.root = Path(root).resolve()
2639 self.records: dict[str, set[str]] = {}
2640 self.env_writers: set[str] = set()
2641 self.spawners: set[str] = set()
2642 self.current: str | None = None
2643 self._installed: list[tuple[Any, str, Any, bool]] = []
2644 self._temp = os.path.realpath(
2645 os.environ.get("TEMP") or os.environ.get("TMPDIR") or "/tmp",
2646 ).replace("\\", "/").lower()
2647 self._pid = str(os.getpid())
2648
2649 # -- attribution ------------------------------------------------
2650 def record(self, resource: str) -> None:
2651 """Attribute *resource* to the file under test."""
2652 if self.current is not None:
2653 self.records.setdefault(self.current, set()).add(resource)
2654
2655 def record_path(self, target: Any) -> None:
2656 """Attribute a write to *target* to the file under test."""
2657 if self.current is None:
2658 return
2659 try:
2660 text = os.path.abspath(os.fspath(target))
2661 except (TypeError, ValueError):
2662 return
2663 normalised = text.replace("\\", "/")
2664 if any(part in normalised for part in _TRACE_IGNORED):
2665 return
2666 if _is_pid_stamped(normalised.rsplit("/", 1)[-1], self._pid):
2667 return
2668 try:
2669 # A write INSIDE the tree under test is a shared write by
2670 # definition, checked BEFORE the temp filters: a package
2671 # can legitimately be rooted inside a temp directory (the
2672 # W2 fixture tree is), and the tree under test always
2673 # wins over "it looks temporary".
2674 hint = Path(text).relative_to(self.root).as_posix()
2675 except ValueError:
2676 if normalised.lower().startswith(self._temp):
2677 return
2678 if "/pytest-of-" in normalised:
2679 return
2680 # Above the root: the real directory name is machine
2681 # specific and the manifest is committed, so the key is
2682 # the constant sentinel (P2).
2683 self.record(write_resource(normalised, OUTSIDE_KEY))
2684 return
2685 self.record(write_resource(hint, path_key(hint)))
2686
2687 def record_launch(self, argv: Any, *, cwd: Any = None) -> None:
2688 """Attribute a child launch to the file under test (P3).
2689
2690 The tracer sees the REAL argv, so it keys what the static pass
2691 can only guess at; an argv whose family the committed table
2692 does not vouch for still contributes the bare wildcard.
2693 """
2694 if self.current is None:
2695 return
2696 self.spawners.add(self.current)
2697 if launch_family(argv) == "git" and cwd is not None:
2698 if not self._git_is_temporary(argv, cwd):
2699 self.record(RES_ENV_SUBPROCESS)
2700 return
2701 for resource in launch_resources(argv):
2702 self.record(resource)
2703
2704 def _git_is_temporary(self, argv: Sequence[Any], cwd: Any) -> bool:
2705 """Vouch for Git only when cwd and every -C remain temporary.
2706
2707 Global configuration and explicit repository overrides can target
2708 locations outside cwd and therefore keep the launch unattributed.
2709 """
2710 try:
2711 target = Path(cwd).resolve()
2712 targets = [target]
2713 index = 1
2714 while index < len(argv):
2715 arg = os.fspath(argv[index])
2716 if arg in ("--global", "--system") or arg.startswith((
2717 "--git-dir", "--work-tree", "--exec-path",
2718 )):
2719 return False
2720 if arg == "-C":
2721 index += 1
2722 if index >= len(argv):
2723 return False
2724 target = (target / os.fspath(argv[index])).resolve()
2725 targets.append(target)
2726 elif arg.startswith("-C"):
2727 target = (target / arg[2:]).resolve()
2728 targets.append(target)
2729 index += 1
2730 temporary = Path(self._temp)
2731 return all(
2732 not path.is_relative_to(self.root) and (
2733 path.is_relative_to(temporary)
2734 or "/pytest-of-" in path.as_posix()
2735 ) for path in targets
2736 )
2737 except (TypeError, ValueError, OSError):
2738 return False
2739
2740 def resources(self) -> dict[str, set[str]]:
2741 """The trace as recorded.
2742
2743 The P2 env/subprocess join happens at the seam now:
2744 :meth:`record_launch` records the family's own tokens, or the
2745 bare one, at the moment of the launch, and the environment
2746 seam records :data:`RES_ENV_MUTATION` (Plan 276 P1) - which is
2747 process-local, so it never groups anything. :attr:`spawners`
2748 and :attr:`env_writers` stay as the diagnostic record of WHICH
2749 files launched a child and WHICH wrote the environment.
2750 """
2751 return {path: set(res) for path, res in self.records.items()}
2752
2753 # -- install / remove -------------------------------------------
2754 def _patch(self, owner: Any, name: str, factory: Callable) -> None:
2755 """Replace ``owner.name`` with ``factory(original)``."""
2756 original = getattr(owner, name)
2757 owned = name in getattr(owner, "__dict__", {})
2758 self._installed.append((owner, name, original, owned))
2759 setattr(owner, name, factory(original))
2760
2761 def install(self) -> None:
2762 """Wrap every traced seam."""
2763 func = "install"
2764 self._patch(builtins, "open", self._open_wrapper)
2765 self._patch(io, "open", self._open_wrapper)
2766 self._patch(os, "chdir", self._chdir_wrapper)
2767 self._patch(type(os.environ), "__setitem__", self._env_wrapper)
2768 self._patch(type(os.environ), "__delitem__", self._env_wrapper)
2769 self._patch(socket.socket, "bind", self._bind_wrapper)
2770 self._patch(subprocess.Popen, "__init__", self._popen_wrapper)
2771 _dbg("OCT-TESTING", func, f"{len(self._installed)} seams wrapped", 2)
2772
2773 def remove(self) -> None:
2774 """Restore every wrapped seam, newest first."""
2775 func = "remove"
2776 for owner, name, original, owned in reversed(self._installed):
2777 if owned:
2778 setattr(owner, name, original)
2779 else:
2780 delattr(owner, name)
2781 self._installed.clear()
2782 _dbg("OCT-TESTING", func, "seams restored", 2)
2783
2784 # -- wrappers ---------------------------------------------------
2785 def _open_wrapper(self, original: Callable) -> Callable:
2786 @functools.wraps(original)
2787 def wrapper(
2788 file: Any = None, mode: str = "r", *args: Any, **kwargs: Any,
2789 ) -> Any:
2790 if any(flag in mode for flag in ("w", "a", "x", "+")):
2791 self.record_path(file)
2792 return original(file, mode, *args, **kwargs)
2793 return wrapper
2794
2795 def _chdir_wrapper(self, original: Callable) -> Callable:
2796 @functools.wraps(original)
2797 def wrapper(*args: Any, **kwargs: Any) -> Any:
2798 self.record(RES_CWD_CHANGE)
2799 return original(*args, **kwargs)
2800 return wrapper
2801
2802 def _env_wrapper(self, original: Callable) -> Callable:
2803 @functools.wraps(original)
2804 def wrapper(mapping: Any, *args: Any, **kwargs: Any) -> Any:
2805 if self.current is not None and not _from_pytest_internals():
2806 self.env_writers.add(self.current)
2807 self.record(RES_ENV_MUTATION)
2808 return original(mapping, *args, **kwargs)
2809 return wrapper
2810
2811 def _bind_wrapper(self, original: Callable) -> Callable:
2812 @functools.wraps(original)
2813 def wrapper(
2814 sock: Any, address: Any, *args: Any, **kwargs: Any,
2815 ) -> Any:
2816 port = address[1] if isinstance(address, tuple) and len(
2817 address,
2818 ) > 1 else None
2819 if isinstance(port, int) and port > 0:
2820 self.record(f"{PORT_PREFIX}{port}")
2821 return original(sock, address, *args, **kwargs)
2822 return wrapper
2823
2824 def _popen_wrapper(self, original: Callable) -> Callable:
2825 @functools.wraps(original)
2826 def wrapper(popen: Any, *args: Any, **kwargs: Any) -> Any:
2827 if self.current is not None:
2828 self.record_launch(
2829 args[0] if args else kwargs.get("args"),
2830 cwd=(kwargs.get("cwd", args[9] if len(args) > 9 else None)
2831 or os.getcwd()),
2832 )
2833 return original(popen, *args, **kwargs)
2834 return wrapper
2835
2836
2837#: The active tracer, or None. Module state, because the pytest hooks
2838#: below are plain functions (no ``pytest`` import at module scope).
2839_TRACER: _Tracer | None = None
2840
2841#: Where the active tracer writes its per-process trace.
2842_TRACE_PATH: Path | None = None
2843
2844
2845#: Refusal text when ``--oct-trace`` meets a parallel run (R3).
2846TRACE_REQUIRES_SERIAL: str = (
2847 "--oct-trace requires a SERIAL run: re-run with -n 0. The tracer "
2848 "wraps process-global seams, so under xdist it would record only "
2849 "this process's activity and write a partial trace, and a partial "
2850 "trace silently misclassifies the files it never saw."
2851)
2852
2853
2854def pytest_addoption(parser: Any) -> None:
2855 """Register ``--oct-trace`` (pytest plugin hook, P3b)."""
2856 parser.addoption(
2857 TRACE_OPTION,
2858 action="store",
2859 default=None,
2860 metavar="PATH",
2861 help=(
2862 "record the shared resources each test file actually "
2863 "touches and write them to PATH; requires a serial run "
2864 "(-n 0), and refuses under xdist workers"
2865 ),
2866 )
2867
2868
2869def pytest_configure(config: Any) -> None:
2870 """Install the tracer when ``--oct-trace`` was given.
2871
2872 Refuses a PARALLEL traced run outright (R3). Measured on the
2873 oc_chat suite 2026-09-06: with workers, only the controller wrote
2874 a trace file, so the classification would have been rebuilt from
2875 collection-time activity alone. Refusing at configure time, before
2876 any worker starts, is the fail-closed choice; a partial trace that
2877 looks complete is the failure mode this whole mechanism exists to
2878 prevent.
2879 """
2880 func = "pytest_configure"
2881 global _TRACER, _TRACE_PATH
2882 target = config.getoption(TRACE_OPTION, default=None)
2883 if not target:
2884 return
2885 workers = getattr(config.option, "numprocesses", None) or 0
2886 if workers and not hasattr(config, "workerinput"):
2887 import pytest # local: an optional dependency of oct
2888
2889 _dbg("OCT-TESTING", func, f"refusing: {workers} worker(s)", 2)
2890 raise pytest.UsageError(TRACE_REQUIRES_SERIAL)
2891 _TRACE_PATH = Path(target)
2892 _TRACER = _Tracer(Path(str(config.rootpath)))
2893 _TRACER.install()
2894 _dbg("OCT-TESTING", func, f"tracing to {_TRACE_PATH}", 2)
2895
2896
2897def pytest_runtest_logstart(nodeid: str, location: Any) -> None:
2898 """Attribute the next observations to *nodeid*'s file."""
2899 if _TRACER is not None:
2900 _TRACER.current = nodeid.split("::")[0]
2901
2902
2903def pytest_runtest_logfinish(nodeid: str, location: Any) -> None:
2904 """Stop attributing observations to a test."""
2905 if _TRACER is not None:
2906 _TRACER.current = None
2907
2908
2909def pytest_sessionfinish(session: Any, exitstatus: Any) -> None:
2910 """Write this process's trace and restore the wrapped seams."""
2911 func = "pytest_sessionfinish"
2912 global _TRACER
2913 if _TRACER is None or _TRACE_PATH is None:
2914 return
2915 tracer, _TRACER = _TRACER, None
2916 tracer.remove()
2917 document = {
2918 "schema_version": SCHEMA_VERSION,
2919 "files": {
2920 path: sorted(res)
2921 for path, res in sorted(tracer.resources().items())
2922 },
2923 # Informational, not part of the classification: the files
2924 # that wrote to os.environ OUTSIDE monkeypatch, i.e. the P6
2925 # leak candidates. An env write is process-local and never a
2926 # cross-worker conflict on its own.
2927 "env_writers": sorted(tracer.env_writers),
2928 }
2929 path = _TRACE_PATH.with_name(
2930 f"{_TRACE_PATH.stem}.{os.getpid()}{_TRACE_PATH.suffix or '.json'}",
2931 )
2932 path.parent.mkdir(parents=True, exist_ok=True)
2933 path.write_text(
2934 json.dumps(document, indent=2) + "\n", encoding="utf-8", newline="\n",
2935 )
2936 _dbg("OCT-TESTING", func, f"trace written to {path}", 2)
2937
2938
2939def load_traces(sources: Sequence[Path]) -> dict[str, set[str]]:
2940 """Merge trace documents from files and/or directories."""
2941 func = "load_traces"
2942 merged: dict[str, set[str]] = {}
2943 for source in sources:
2944 paths = sorted(source.glob("*.json")) if source.is_dir() else [source]
2945 for path in paths:
2946 raw = json.loads(path.read_text(encoding="utf-8"))
2947 for key, values in raw.get("files", {}).items():
2948 merged.setdefault(key, set()).update(values)
2949 _dbg("OCT-TESTING", func, f"merged {len(merged)} traced file(s)", 2)
2950 return merged
2951
2952
2953# ---------------------------------------------------------------------
2954# CLI - ``python -m oct.testing.parallel classify --root <path>``
2955# ---------------------------------------------------------------------
2956
2957def _package_for_root(root: Path) -> tuple[Path, str, tuple[str, ...]]:
2958 """Resolve *root* to its workspace, package name and test roots."""
2959 from oct.core.workspace import find_workspace_root
2960
2961 workspace_root = find_workspace_root(root)
2962 if workspace_root is None:
2963 raise OctTestError(
2964 MANIFEST_TEST_BLOCK_INVALID,
2965 f"no workspace manifest above {root}",
2966 )
2967 manifest = load_test_manifest(workspace_root)
2968 for package in manifest.packages:
2969 if package.path.resolve() == root.resolve():
2970 return workspace_root, package.name, package.test_roots
2971 raise OctTestError(
2972 MANIFEST_TEST_BLOCK_INVALID,
2973 f"{root} is not a declared subproject of {workspace_root}",
2974 )
2975
2976
2977def default_manifest_path(root: Path, test_roots: Sequence[str]) -> Path:
2978 """The manifest path implied by *test_roots* (their common parent)."""
2979 parts = [Path(entry).parts for entry in test_roots]
2980 common: list[str] = []
2981 for index in range(min((len(part) for part in parts), default=0)):
2982 column = {part[index] for part in parts}
2983 if len(column) != 1:
2984 break
2985 common.append(column.pop())
2986 return root.joinpath(*common) / MANIFEST_FILENAME
2987
2988
2989def _parse_pins(values: Sequence[str]) -> dict[str, str]:
2990 """Parse repeated ``--pin <path>=<group>`` arguments."""
2991 pins: dict[str, str] = {}
2992 for value in values:
2993 path, separator, group = value.partition("=")
2994 if not separator or not path or not group:
2995 raise OctTestError(
2996 MANIFEST_TEST_BLOCK_INVALID,
2997 f"--pin expects <path>=<group>, got {value!r}",
2998 )
2999 pins[path] = group
3000 return pins
3001
3002
3003def _tool_version() -> str:
3004 """The oct version string recorded in the manifest."""
3005 try:
3006 from oct import __version__
3007 except ImportError: # pragma: no cover - oct is always importable
3008 return "oct unknown"
3009 return f"oct {__version__}"
3010
3011
3013 document: Mapping[str, Any],
3014 out: Path,
3015 workspace_root: Path,
3016 package: str,
3017) -> None:
3018 """Print an ASCII-only summary of a regeneration."""
3019 files = document["files"]
3020 counts: dict[str, int] = {}
3021 groups: dict[str, int] = {}
3022 for entry in files.values():
3023 groups[entry["group"]] = groups.get(entry["group"], 0) + 1
3024 for resource in entry["resources"]:
3025 counts[resource] = counts.get(resource, 0) + 1
3026 print(f"manifest: {out}")
3027 print(f"files: {len(files)} groups: {len(groups)}")
3028 print(f"workers: {resolve_worker_count(workspace_root, package)}")
3029 for resource, count in sorted(counts.items(), key=lambda kv: -kv[1]):
3030 print(f" resource {resource}: {count} file(s)")
3031 for name, count in sorted(groups.items(), key=lambda kv: -kv[1])[:8]:
3032 print(f" group {name}: {count} file(s)")
3033
3034
3035#: Exit code of ``classify --check`` when the committed manifest and a
3036#: fresh classification disagree (§12.7 configuration class).
3037EXIT_MANIFEST_DRIFT: int = 4
3038
3039
3041 committed: Mapping[str, Any],
3042 fresh: Mapping[str, Any],
3043) -> list[str]:
3044 """Compare two manifest documents by CONTENT, file by file.
3045
3046 Only ``files`` is compared: ``generated_at`` and ``tool_version``
3047 differ on every regeneration by design.
3048
3049 Returns
3050 -------
3051 list[str]
3052 One line per disagreeing file, each naming the file and the
3053 reason. Empty when the two documents classify identically.
3054 """
3055 func = "diff_manifests"
3056 left = committed.get("files", {}) or {}
3057 right = fresh.get("files", {}) or {}
3058 lines: list[str] = []
3059 for path in sorted(set(left) | set(right)):
3060 if path not in left:
3061 lines.append(f"{path}: absent from the committed manifest")
3062 continue
3063 if path not in right:
3064 lines.append(f"{path}: in the manifest but no longer collected")
3065 continue
3066 old, new = left[path], right[path]
3067 old_res = sorted(old.get("resources", []))
3068 new_res = sorted(new.get("resources", []))
3069 if old_res != new_res:
3070 lines.append(
3071 f"{path}: resources {old_res} -> {new_res}",
3072 )
3073 elif old.get("group") != new.get("group"):
3074 lines.append(
3075 f"{path}: group {old.get('group')!r} -> "
3076 f"{new.get('group')!r}",
3077 )
3078 _dbg("OCT-TESTING", func, f"{len(lines)} drifted file(s)", 2)
3079 return lines
3080
3081
3082def run_classify(argv: Sequence[str]) -> int:
3083 """Regenerate (or CHECK) a subproject's parallel manifest.
3084
3085 ``--check`` classifies in memory from the same inputs and diffs the
3086 result against the committed manifest instead of overwriting it:
3087 exit 0 on identity, :data:`EXIT_MANIFEST_DRIFT` listing every file
3088 whose resources or group differ. The completeness check inside the
3089 suite only proves a file is PRESENT; this is what proves the
3090 entries are still TRUE (gate finding R1).
3091
3092 A trace passed with ``--trace`` must come from a SERIAL traced run
3093 (``pytest -n 0 -p oct.testing.parallel --oct-trace=<path>``); the
3094 plugin refuses to trace a parallel run, because a partial trace
3095 would quietly reclassify every file it did not observe.
3096 """
3097 func = "run_classify"
3098 import argparse
3099
3100 parser = argparse.ArgumentParser(
3101 prog="python -m oct.testing.parallel classify",
3102 description="classify test files by shared resource and group them",
3103 )
3104 parser.add_argument("--root", required=True, type=Path)
3105 parser.add_argument("--out", type=Path, default=None)
3106 parser.add_argument("--trace", type=Path, action="append", default=[])
3107 parser.add_argument("--pin", action="append", default=[])
3108 parser.add_argument(
3109 "--check",
3110 action="store_true",
3111 help=(
3112 "do not write: diff a fresh classification against the "
3113 "committed manifest and exit non-zero on any difference"
3114 ),
3115 )
3116 args = parser.parse_args(list(argv))
3117
3118 root = args.root.resolve()
3119 workspace_root, package, test_roots = _package_for_root(root)
3120 _dbg("OCT-TESTING", func, f"classifying {package} at {root}", 2)
3121
3122 classified: dict[str, set[str]] = {
3123 path: set(resources)
3124 for path, resources in classify_tree(root, test_roots).items()
3125 }
3126 traced = load_traces(args.trace)
3127 for path, resources in traced.items():
3128 if path in classified:
3129 classified[path] |= resources
3130 document = build_manifest(
3131 classified,
3132 tool_version=_tool_version(),
3133 pins=_parse_pins(args.pin),
3134 )
3135 out = args.out or default_manifest_path(root, test_roots)
3136
3137 if args.check:
3138 return _run_check(out, document)
3139 write_manifest(out, document)
3140 _print_summary(document, out, workspace_root, package)
3141 return 0
3142
3143
3144def _run_check(out: Path, document: Mapping[str, Any]) -> int:
3145 """The ``--check`` half of :func:`run_classify`."""
3146 func = "_run_check"
3147 try:
3148 committed = json.loads(Path(out).read_text(encoding="utf-8"))
3149 except (OSError, json.JSONDecodeError) as exc:
3150 print(f"drift: {out} is unreadable: {exc}")
3151 return EXIT_MANIFEST_DRIFT
3152 lines = diff_manifests(committed, document)
3153 if not lines:
3154 print(f"manifest: {out}")
3155 print(f"check: OK, {len(document['files'])} file(s) identical")
3156 return 0
3157 print(f"manifest: {out}")
3158 print(f"check: DRIFT, {len(lines)} file(s) differ")
3159 for line in lines:
3160 print(f" {line}")
3161 print("remedy: regenerate with the same --trace and --pin arguments")
3162 _dbg("OCT-TESTING", func, f"drift on {len(lines)} file(s)", 2)
3163 return EXIT_MANIFEST_DRIFT
3164
3165
3166def main(argv: Sequence[str] | None = None) -> int:
3167 """CLI entry point; ``classify`` is the only subcommand."""
3168 args = list(sys.argv[1:] if argv is None else argv)
3169 if not args or args[0] != "classify":
3170 print("usage: python -m oct.testing.parallel classify --root PATH")
3171 return 4
3172 try:
3173 return run_classify(args[1:])
3174 except OctTestError as exc:
3175 print(f"error {exc.code}: {exc.message}")
3176 return exc.exit_code
3177
3178
3179__all__ = [
3180 "DEFAULT_MAX_WORKERS",
3181 "EXIT_MANIFEST_DRIFT",
3182 "FREE_GROUP_PREFIX",
3183 "FileEntry",
3184 "GROUP_FREE",
3185 "GROUP_UNCLASSIFIED",
3186 "KEYED_FAMILIES",
3187 "KEY_SEPARATOR",
3188 "LAUNCH_FAMILIES",
3189 "MANIFEST_FILENAME",
3190 "ModuleAnalysis",
3191 "OUTSIDE_KEY",
3192 "PORT_PREFIX",
3193 "PROCESS_LOCAL_RESOURCES",
3194 "PYTHON_ARGV0",
3195 "ParallelManifest",
3196 "RESOURCE_VOCABULARY",
3197 "RES_COMMS_ROOT",
3198 "RES_CWD_CHANGE",
3199 "RES_ENV_SUBPROCESS",
3200 "RES_GLOBAL_SINGLETON_RESET",
3201 "RES_MAIN_TREE_WRITE",
3202 "RES_NETWORK",
3203 "RES_OCTRC",
3204 "RES_REGISTRY_SQLITE",
3205 "RES_SETTINGS_FILE",
3206 "RES_WORKTREE_DIR",
3207 "SCHEMA_VERSION",
3208 "TRACE_OPTION",
3209 "TRACE_REQUIRES_SERIAL",
3210 "UNKNOWN_PREFIX",
3211 "WHOLE_FILE_MARKER",
3212 "analyse_source",
3213 "apply_group_marks",
3214 "available_cpu_count",
3215 "build_groups",
3216 "build_manifest",
3217 "classify_source",
3218 "classify_tree",
3219 "configure_parallel_defaults",
3220 "conflicts_with",
3221 "default_manifest_path",
3222 "diff_manifests",
3223 "free_group_name",
3224 "is_conflicting",
3225 "is_known_resource",
3226 "is_xdist_worker",
3227 "keyed",
3228 "launch_family",
3229 "launch_resources",
3230 "load_manifest",
3231 "load_traces",
3232 "main",
3233 "path_key",
3234 "resolve_groups",
3235 "resolve_max_workers",
3236 "resolve_worker_count",
3237 "resource_family",
3238 "resource_for_path_hint",
3239 "resource_key",
3240 "unknown",
3241 "write_manifest",
3242 "write_resource",
3243]
3244
3245
3246if __name__ == "__main__":
3247 sys.exit(main())
None _bind(self, ast.Call node)
Definition parallel.py:1637
None visit_Assign(self, ast.Assign node)
Definition parallel.py:1652
None _launch(self, ast.Call node, str attr)
Definition parallel.py:1633
None visit_Delete(self, ast.Delete node)
Definition parallel.py:1659
None __init__(self, frozenset[str] tainted, Mapping[str, str] hints, Mapping[ast.AST, frozenset[str]]|None scopes=None, Mapping[str, str]|None constants=None)
Definition parallel.py:1577
None visit_FunctionDef(self, ast.FunctionDef node)
Definition parallel.py:1587
None visit_ClassDef(self, ast.ClassDef node)
Definition parallel.py:1602
None _write(self, ast.AST|None target)
Definition parallel.py:1621
None visit_Lambda(self, ast.Lambda node)
Definition parallel.py:1611
None visit_Call(self, ast.Call node)
Definition parallel.py:1666
None _visit_attr_call(self, ast.Call node, str attr, ast.AST|None owner, str|None owner_name)
Definition parallel.py:1698
bool _git_is_temporary(self, Sequence[Any] argv, Any cwd)
Definition parallel.py:2704
None record_path(self, Any target)
Definition parallel.py:2655
None record_launch(self, Any argv, *, Any cwd=None)
Definition parallel.py:2687
None record(self, str resource)
Definition parallel.py:2650
dict[str, set[str]] resources(self)
Definition parallel.py:2740
None __init__(self, Path root)
Definition parallel.py:2637
None _patch(self, Any owner, str name, Callable factory)
Definition parallel.py:2754
Definition parallel.py:403
bool _from_pytest_internals(int depth=4)
Definition parallel.py:2575
dict[ast.AST, frozenset[str]] _helper_parameter_taints(ast.Module tree, frozenset[str] base, Iterable[str] seeds=())
Definition parallel.py:953
None pytest_runtest_logstart(str nodeid, Any location)
Definition parallel.py:2897
set[str] _parameter_names(ast.FunctionDef|ast.AsyncFunctionDef node)
Definition parallel.py:941
frozenset[str] _requested_names(ast.AST tree)
Definition parallel.py:1759
bool _returns_tmp(ast.AST node, frozenset[str] tainted)
Definition parallel.py:793
str|None path_key(str hint)
Definition parallel.py:475
ParallelManifest load_manifest(Path path)
Definition parallel.py:2158
bool _is_environ(ast.AST|None node)
Definition parallel.py:1561
str|None _literal_path(ast.AST|None node, Mapping[str, str]|None constants=None)
Definition parallel.py:1303
Iterable[ast.AST] _scope_nodes(ast.AST tree)
Definition parallel.py:851
bool _write_mode(ast.Call node)
Definition parallel.py:1520
str _slug(str resource)
Definition parallel.py:1974
int main(Sequence[str]|None argv=None)
Definition parallel.py:3166
str|None _literal_parents_index(ast.Subscript node, Mapping[str, str] constants)
Definition parallel.py:1280
str unknown(str detail)
Definition parallel.py:365
dict[str, str] _parse_pins(Sequence[str] values)
Definition parallel.py:2989
str|None resource_key(str resource)
Definition parallel.py:277
str|None _literal_fstring(ast.JoinedStr node)
Definition parallel.py:1238
dict[str, Any] build_manifest(Mapping[str, Iterable[str]] classified, *, str tool_version, Mapping[str, str]|None pins=None, str|None generated_at=None)
Definition parallel.py:2126
set[str] _target_names(ast.AST node)
Definition parallel.py:752
bool _is_tmp_expr(ast.AST|None node, frozenset[str] tainted)
Definition parallel.py:759
list[tuple[str, list[str]]] _named_components(Mapping[str, list[str]] components, Mapping[str, frozenset[str]] sets)
Definition parallel.py:1982
frozenset[str] _scope_proofs(ast.AST tree, Iterable[str] seeds, *, Iterable[str] parameters=())
Definition parallel.py:893
set[str] _rebound_parameters(ast.AST tree, set[str] names)
Definition parallel.py:1102
str|None _parent_literal(str text)
Definition parallel.py:1207
frozenset[str] launch_resources(Any argv)
Definition parallel.py:631
bool _is_pid_expr(ast.AST|None node)
Definition parallel.py:1228
None _adopt_group_dist_in_worker(Any config, Any option, str dist)
Definition parallel.py:2490
bool is_conflicting(str resource)
Definition parallel.py:327
dict[str, str] build_groups(Mapping[str, Iterable[str]] classified, *, Mapping[str, str]|None pins=None)
Definition parallel.py:2006
dict[str, set[str]] load_traces(Sequence[Path] sources)
Definition parallel.py:2939
list[str] diff_manifests(Mapping[str, Any] committed, Mapping[str, Any] fresh)
Definition parallel.py:3043
str|None _argv_element(ast.AST node)
Definition parallel.py:1472
None _union(dict[str, str] parent, str left, str right)
Definition parallel.py:1967
None pytest_addoption(Any parser)
Definition parallel.py:2854
dict[str, str] _import_hints(ast.AST tree)
Definition parallel.py:1109
str _join_literal(str left, str right)
Definition parallel.py:1189
ast.AST|None _helper_actual(ast.FunctionDef|ast.AsyncFunctionDef helper, ast.Call call, str name)
Definition parallel.py:1087
str _read(Path path)
Definition parallel.py:1876
None write_manifest(Path path, Mapping[str, Any] document)
Definition parallel.py:2149
list[str|None]|None _literal_argv(ast.Call node, str attr)
Definition parallel.py:1485
list[ast.Call] _closed_helper_calls(ast.Module tree, ast.FunctionDef|ast.AsyncFunctionDef helper, Mapping[ast.AST, ast.AST] parents)
Definition parallel.py:1047
tuple[dict[str, str], tuple[str,...]] resolve_groups(ParallelManifest manifest, Iterable[str] relpaths)
Definition parallel.py:2206
int _run_check(Path out, Mapping[str, Any] document)
Definition parallel.py:3144
bool _is_os_path_join(ast.AST func)
Definition parallel.py:1266
tuple[bool, bool] _fixture_decorated(ast.AST node)
Definition parallel.py:774
str _find(dict[str, str] parent, str path)
Definition parallel.py:1959
frozenset[str] classify_source(str source, *, str filename="<test>", Sequence[ModuleAnalysis] conftests=())
Definition parallel.py:1853
None pytest_runtest_logfinish(str nodeid, Any location)
Definition parallel.py:2903
str free_group_name(str nodeid, *, bool whole_file=False)
Definition parallel.py:2375
int _raw_max_workers(Path workspace_root, str package)
Definition parallel.py:2239
int resolve_max_workers(Path workspace_root, str package)
Definition parallel.py:2271
None pytest_sessionfinish(Any session, Any exitstatus)
Definition parallel.py:2909
bool _is_path_replace(ast.Call node)
Definition parallel.py:1535
str|None _literal_sequence(ast.Call node, Mapping[str, str] constants, *, Callable[[str, str], str] joiner)
Definition parallel.py:1375
int run_classify(Sequence[str] argv)
Definition parallel.py:3082
str keyed(str family, str key)
Definition parallel.py:257
bool is_xdist_worker(Any config)
Definition parallel.py:2474
str|None launch_family(Any argv)
Definition parallel.py:586
bool is_known_resource(str resource)
Definition parallel.py:343
int|None configure_parallel_defaults(Any config, *, Path workspace_root, str package, str dist="loadgroup")
Definition parallel.py:2521
str _string_hint(ast.AST|None node)
Definition parallel.py:1137
dict[str, int] _binding_counts(ast.AST tree)
Definition parallel.py:1388
str resource_family(str resource)
Definition parallel.py:269
Path default_manifest_path(Path root, Sequence[str] test_roots)
Definition parallel.py:2977
str|None _hint_for_dotted(str dotted)
Definition parallel.py:1129
None pytest_configure(Any config)
Definition parallel.py:2869
None _print_summary(Mapping[str, Any] document, Path out, Path workspace_root, str package)
Definition parallel.py:3017
bool _is_pid_stamped(str basename, str pid)
Definition parallel.py:2600
bool _is_absolute_literal(str text)
Definition parallel.py:1177
str resource_for_path_hint(str hint)
Definition parallel.py:451
dict[str, str] _module_constants(ast.Module tree, str|None module_path)
Definition parallel.py:1428
str write_resource(str hint, str|None key)
Definition parallel.py:501
list[ModuleAnalysis] _conftest_chain(Path root, Path path, dict[Path, ModuleAnalysis] cache)
Definition parallel.py:1885
bool conflicts_with(str left, str right)
Definition parallel.py:285
bool _proven_tmp_expr(ast.AST|None node, frozenset[str] tainted)
Definition parallel.py:864
tuple[tuple[str,...], str|None] apply_group_marks(Any config, Iterable[Any] items, Path manifest_path)
Definition parallel.py:2406
int resolve_worker_count(Path workspace_root, str package, *, int|None cpu_count=None)
Definition parallel.py:2359
dict[str, frozenset[str]] classify_tree(Path root, Sequence[str] test_roots, *, str pattern="test_*.py")
Definition parallel.py:1914
frozenset[str] _collect_taint(ast.AST tree, Iterable[str] seeds)
Definition parallel.py:806
ModuleAnalysis analyse_source(str source, *, str filename="<test>", Iterable[str] tmp_seeds=())
Definition parallel.py:1775
tuple[Path, str, tuple[str,...]] _package_for_root(Path root)
Definition parallel.py:2957