From fcd42695cab8600d75ac277dccd3e5a9355390cb Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 11 Sep 2025 11:47:53 -0400 Subject: [PATCH] fix: add TASKGRAPH_SERIAL and set multiprocessing context for multiprocess kind generation This was supposed to be done in https://github.com/taskcluster/taskgraph/pull/765, but clearly I didn't push it before merging. --- src/taskgraph/generator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/taskgraph/generator.py b/src/taskgraph/generator.py index 94d0813aa..9b794ee48 100644 --- a/src/taskgraph/generator.py +++ b/src/taskgraph/generator.py @@ -4,6 +4,7 @@ import copy import logging +import multiprocessing import os import platform from concurrent.futures import ( @@ -302,7 +303,9 @@ def _load_tasks_parallel(self, kinds, kind_graph, parameters): futures = set() edges = set(kind_graph.edges) - with ProcessPoolExecutor() as executor: + with ProcessPoolExecutor( + mp_context=multiprocessing.get_context("fork") + ) as executor: def submit_ready_kinds(): """Create the next batch of tasks for kinds without dependencies.""" @@ -420,7 +423,7 @@ def _run(self): # redone in the new processes. Ideally this would be fixed, or we # would take another approach to parallel kind generation. In the # meantime, it's not supported outside of Linux. - if platform.system() != "Linux": + if platform.system() != "Linux" or os.environ.get("TASKGRAPH_SERIAL"): all_tasks = self._load_tasks_serial(kinds, kind_graph, parameters) else: all_tasks = self._load_tasks_parallel(kinds, kind_graph, parameters)