Always use spawn to start multiple process

- Remove `CONDUCTOR_MP_START_METHOD` env var

Author: Kowser

.github/workflows/agent-e2e.yml

@@ -28,7 +28,6 @@ jobs:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       AGENTSPAN_SERVER_URL: http://localhost:8080/api
       AGENTSPAN_CLI_PATH: ${{ github.workspace }}/agentspan
-      CONDUCTOR_MP_START_METHOD: fork
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

CHANGELOG.md

@@ -11,11 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Canonical metrics mode: opt-in harmonized metric surface via `WORKER_CANONICAL_METRICS=true` -- [details](METRICS.md#detailed-technical-notes--unreleased)
 - `MetricsSettings` gains `clean_directory` and `clean_dead_pids` for opt-in stale `.db` file cleanup (both default to `False`)
-- `CONDUCTOR_MP_START_METHOD` env var to control the worker pool's multiprocessing start method
 
 ### Changed
 
-- **Multiprocessing start method now defaults to `spawn` on all platforms** (was `fork` everywhere except Windows). `fork` caused silently-restarting `SIGSEGV` worker subprocesses on macOS (exitcode `-11`) and fork-with-held-lock deadlocks on POSIX. Opt back in with `CONDUCTOR_MP_START_METHOD=fork`. Entrypoint scripts must use the standard `if __name__ == "__main__":` guard, and workers must be defined at module level
+- **Multiprocessing start method now defaults to `spawn` on all platforms** (was `fork` everywhere except Windows). `fork` caused silently-restarting `SIGSEGV` worker subprocesses on macOS (exitcode `-11`) and fork-with-held-lock deadlocks on POSIX. 
 - Legacy metrics emit unchanged by default; no env var required
 - `metrics_collector.py` is now a compatibility shim; `from conductor.client.telemetry.metrics_collector import MetricsCollector` continues to work
 

METRICS.md

@@ -344,10 +344,6 @@ unreleased metrics harmonization work. For a summary, see the project
     `.db` files and `clean_dead_pids` (default `False`) to remove only `.db`
     files from PIDs that no longer exist. Both are executed by the factory
     against the resolved metrics directory.
-  - `CONDUCTOR_MP_START_METHOD` env var (`spawn` / `fork` / `forkserver`;
-    default `fork` on POSIX, `spawn` on Windows) to control the worker pool's
-    multiprocessing start method (motivated by a `prometheus_client` lock-fork
-    deadlock).
   - Harness manifest sets `WORKER_CANONICAL_METRICS=true`; `harness/main.py`
     logs which collector is active.
 

docs/WORKER.md

@@ -279,13 +279,6 @@ Requirements under `spawn` (standard Python multiprocessing rules):
 - `configuration`, `metrics_settings`, and `event_listeners` passed to
   `TaskHandler` must be picklable
 
-To opt back into the previous behavior (e.g., a Linux deployment relying on
-fork's copy-on-write memory sharing):
-
-```shell
-export CONDUCTOR_MP_START_METHOD=fork   # spawn (default) | fork | forkserver
-```
-
 If a worker process dies from a signal repeatedly at startup, set
 `PYTHONFAULTHANDLER=1` to capture the crashing stack trace from the child.
 

examples/spawn_safety_probe.py

@@ -19,10 +19,6 @@
 import os
 import sys
 
-# Force spawn BEFORE importing any conductor module: task_handler pins the
-# start method at import time from this env var.
-os.environ["CONDUCTOR_MP_START_METHOD"] = "spawn"
-
 import multiprocessing
 import pickle
 

src/conductor/ai/agents/runtime/_worker_entries.py

@@ -40,9 +40,7 @@
 _MAX_UNWRAP_DEPTH = 32
 
 _REMEDIES = (
-    "Define the callable at module level (importable by qualified name), "
-    "or run with CONDUCTOR_MP_START_METHOD=fork (compatibility mode; see the "
-    "start-method notes in conductor.client.automator.task_handler)."
+    "Define the callable at module level (importable by qualified name)."
 )
 
 

src/conductor/client/automator/task_handler.py

@@ -38,33 +38,7 @@
 _mp_fork_set = False
 if not _mp_fork_set:
     try:
-        # Default start method: "spawn" on every platform.
-        #
-        # fork() is fundamentally unsafe in a process that holds native
-        # framework state or non-main threads:
-        #   - macOS: Apple frameworks (CFNetwork, Security, libdispatch) may
-        #     SIGSEGV in forked children, producing silently-restarting worker
-        #     processes with exitcode=-11. CPython switched its own macOS
-        #     default to spawn in 3.8 for the same reason (bpo-33725).
-        #   - all POSIX: forking while another thread holds a lock (e.g. the
-        #     prometheus_client module-level lock, or the TaskHandler monitor
-        #     thread restarting a worker) leaves that lock permanently held in
-        #     the child -> silent deadlock.
-        # Workers (including the @worker_task decorator path) are pickle-safe,
-        # so spawn works everywhere. Deployments that rely on fork's
-        # copy-on-write inheritance can opt back in with
-        # CONDUCTOR_MP_START_METHOD=fork (re-exposing the hazards above).
-        # NOTE: spawn requires the standard `if __name__ == "__main__":` guard
-        # in the entrypoint script.
-        _default_method = "spawn"
-        _method = os.environ.get("CONDUCTOR_MP_START_METHOD", "").strip().lower() or _default_method
-        if _method not in _VALID_MP_START_METHODS:
-            logger.warning(
-                "Ignoring invalid CONDUCTOR_MP_START_METHOD=%r; falling back to %r",
-                _method, _default_method,
-            )
-            _method = _default_method
-        set_start_method(_method)
+        set_start_method("spawn")
         _mp_fork_set = True
     except Exception as e:
         logger.info("error when setting multiprocessing.set_start_method - maybe the context is set %s", e.args)
@@ -498,10 +472,7 @@ def __check_and_restart_processes(self) -> None:
                     logger.warning(
                         "Worker process %s was killed by signal %s. If this "
                         "repeats on every restart, the process is likely "
-                        "crashing at startup: use the 'spawn' start method "
-                        "(CONDUCTOR_MP_START_METHOD=spawn, the default) and "
-                        "set PYTHONFAULTHANDLER=1 to capture the crashing "
-                        "stack trace.",
+                        "crashing at startup. set PYTHONFAULTHANDLER=1 to capture the crashing stack trace.",
                         worker_name, -exitcode,
                     )
                 if not self.restart_on_failure:

tests/unit/ai/test_worker_entries.py

@@ -230,8 +230,7 @@ def test_transfer_entries_pickle(self):
 
 @pytest.fixture
 def force_spawn_probe(monkeypatch):
-    """Pin the probe's start-method check to 'spawn' so these tests don't
-    depend on the ambient CONDUCTOR_MP_START_METHOD (hermeticity)."""
+    """Pin the probe's start-method check to 'spawn'."""
     monkeypatch.setattr(
         we.multiprocessing, "get_start_method", lambda allow_none=True: "spawn"
     )

tests/unit/resources/spawn_worker_helpers.py

@@ -8,7 +8,7 @@
 Regression tests for GitHub issues #264 / #271:
 @worker_task workers were unpicklable ("cannot pickle '_thread.lock' object" /
 "it's not the same object as module.name"), so TaskHandler could not start
-worker subprocesses with CONDUCTOR_MP_START_METHOD=spawn (the only safe start
+worker subprocesses with set_start_method("spawn") (the only safe start
 method on macOS).
 """
 from conductor.client.http.models.task import Task

tests/unit/worker/test_worker_spawn_safety.py

@@ -173,7 +173,7 @@ class TestWorkerInRealSpawnChild(unittest.TestCase):
     """The definitive regression test: transfer a @worker_task Worker into a
     real 'spawn' child process (fresh interpreter, re-imports modules) and
     execute a task there. This is exactly what TaskHandler does when
-    CONDUCTOR_MP_START_METHOD=spawn (default), and exactly what failed in
+    set_start_method("spawn") (default), and exactly what failed in
     issues #264/#271."""
 
     def test_worker_executes_in_spawn_child_process(self):