fix: use multiprocessing when Linux and fork context are both available

Also update the comment around this with more details, so I don't get confused next time...

Author: bhearsum

src/taskgraph/generator.py

@@ -434,14 +434,30 @@ def _run(self):
         yield "kind_graph", kind_graph
 
         logger.info("Generating full task set")
-        # Current parallel generation relies on multiprocessing, and forking.
-        # This causes problems on Windows and macOS due to how new processes
-        # are created there, and how doing so reinitializes global variables
-        # that are modified earlier in graph generation, that doesn't get
-        # redone in the new processes. Ideally this would be fixed, or we
-        # would take another approach to parallel kind generation. In the
-        # meantime, it's not supported outside of Linux.
-        if platform.system() != "Linux" or os.environ.get("TASKGRAPH_SERIAL"):
+        # The short version of the below is: we only support parallel kind
+        # processing on Linux when the `fork` method is available.
+        #
+        # Current parallel generation relies on multiprocessing, and more
+        # specifically: the "fork" multiprocessing method. This is not supported
+        # at all on Windows (it uses "spawn"). Forking is supported on macOS,
+        # but no longer works reliably in all cases, and our usage of it here
+        # causes crashes. See https://github.com/python/cpython/issues/77906
+        # and http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html
+        # for more details on that.
+        # Other methods of multiprocessing (both "spawn" and "forkserver")
+        # do not work for our use case, because they cause global variables
+        # to be reinitialized, which are sometimes modified earlier in graph
+        # generation. These issues can theoretically be worked around by
+        # eliminating all reliance on globals as part of task generation, but
+        # is far from a small amount of work in users like Gecko/Firefox.
+        # In the long term, the better path forward is likely to be switching
+        # to threading with a free-threaded python to achieve similar parallel
+        # processing.
+        if (
+            platform.system() != "Linux"
+            or "fork" not in multiprocessing.get_all_start_methods()
+            or os.environ.get("TASKGRAPH_SERIAL")
+        ):
             all_tasks = self._load_tasks_serial(kinds, kind_graph, parameters)
         else:
             all_tasks = self._load_tasks_parallel(kinds, kind_graph, parameters)

test/test_generator.py

@@ -3,6 +3,8 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import multiprocessing
+import platform
 from concurrent.futures import ProcessPoolExecutor
 
 import pytest
@@ -12,6 +14,12 @@
 from taskgraph.generator import Kind, load_tasks_for_kind, load_tasks_for_kinds
 from taskgraph.loader.default import loader as default_loader
 
+multiprocessonly = pytest.mark.skipif(
+    platform.system() != "Linux"
+    or "fork" not in multiprocessing.get_all_start_methods(),
+    reason="requires Linux and 'fork' multiprocessing support",
+)
+
 
 class FakePPE(ProcessPoolExecutor):
     loaded_kinds = []
@@ -21,6 +29,7 @@ def submit(self, kind_load_tasks, *args):
         return super().submit(kind_load_tasks, *args)
 
 
+@multiprocessonly
 def test_kind_ordering(mocker, maketgg):
     "When task kinds depend on each other, they are loaded in postorder"
     mocked_ppe = mocker.patch.object(generator, "ProcessPoolExecutor", new=FakePPE)
@@ -272,9 +281,9 @@ def _get_loader(self):
 )
 def test_default_loader(config, expected_transforms):
     loader = Kind("", "", config, {})._get_loader()
-    assert loader is default_loader, (
-        "Default Kind loader should be taskgraph.loader.default.loader"
-    )
+    assert (
+        loader is default_loader
+    ), "Default Kind loader should be taskgraph.loader.default.loader"
     loader("", "", config, {}, [], False)
 
     assert config["transforms"] == expected_transforms