From 36dc5323bd5736313fea9064f94eb9c3a9479423 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 7 Jan 2026 23:12:27 -0600 Subject: [PATCH 1/2] utils_test.popen: allow avoiding path modification, use sysconfig for portability --- distributed/tests/test_utils_test.py | 11 +++++++++++ distributed/utils_test.py | 19 ++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py index 7f676553a1..933ed76c5b 100755 --- a/distributed/tests/test_utils_test.py +++ b/distributed/tests/test_utils_test.py @@ -2,11 +2,13 @@ import asyncio import logging +import os import pathlib import signal import socket import subprocess import sys +import tempfile import textwrap import threading from contextlib import contextmanager @@ -1069,3 +1071,12 @@ def test_captured_context_meter(): ("a", "o", "u"): 6, } assert isinstance(metrics["foo", "s"], int) + + +def test_popen_file_not_found(): + tmp_fd, tmp_path = tempfile.mkstemp(prefix="tmp-python") + os.close(tmp_fd) + os.remove(tmp_path) + with pytest.raises(FileNotFoundError, match=rf"Could not find '{tmp_path}'"): + with popen([tmp_path]): + pass diff --git a/distributed/utils_test.py b/distributed/utils_test.py index 99928a4e15..cd1dc7ecaf 100644 --- a/distributed/utils_test.py +++ b/distributed/utils_test.py @@ -17,6 +17,7 @@ import ssl import subprocess import sys +import sysconfig import tempfile import threading import warnings @@ -1152,7 +1153,9 @@ def popen( Parameters ---------- args: list[str] - Command line arguments + Command line arguments. + The first argument is expected to be an executable. + If it is not an absolute path, this function assumes it is in ``sysconfig.get_path("scripts")``. capture_output: bool, default False Set to True if you need to read output from the subprocess. Stdout and stderr will both be piped to ``proc.stdout``. @@ -1194,10 +1197,16 @@ def popen( kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP args = list(args) - if sys.platform.startswith("win"): - args[0] = os.path.join(sys.prefix, "Scripts", args[0]) - else: - args[0] = os.path.join(sys.prefix, "bin", args[0]) + # avoid searching for executables... only accept absolute paths or look in this Python interpreter's default location for scripts + executable_path = args[0] + if not os.path.isabs(executable_path): + executable_path = os.path.join(sysconfig.get_path("scripts"), executable_path) + if not os.path.isfile(executable_path): + raise FileNotFoundError( + f"Could not find '{executable_path}'. To avoid this warning, provide an absolute path to an existing installation to popen()." + ) + + args[0] = executable_path with subprocess.Popen(args, **kwargs) as proc: try: yield proc From 67837d65855bc494adc922f98bc9d05f4532e9c7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 11 Jan 2026 11:35:01 -0600 Subject: [PATCH 2/2] fix Windows compatibility, stop passing a fule filepath for test regex --- distributed/tests/test_utils_test.py | 5 ++++- distributed/utils_test.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py index 933ed76c5b..62f0bf2f8d 100755 --- a/distributed/tests/test_utils_test.py +++ b/distributed/tests/test_utils_test.py @@ -1077,6 +1077,9 @@ def test_popen_file_not_found(): tmp_fd, tmp_path = tempfile.mkstemp(prefix="tmp-python") os.close(tmp_fd) os.remove(tmp_path) - with pytest.raises(FileNotFoundError, match=rf"Could not find '{tmp_path}'"): + with pytest.raises( + FileNotFoundError, + match=r"provide an absolute path to an existing installation to popen", + ): with popen([tmp_path]): pass diff --git a/distributed/utils_test.py b/distributed/utils_test.py index cd1dc7ecaf..932d98aee8 100644 --- a/distributed/utils_test.py +++ b/distributed/utils_test.py @@ -1201,7 +1201,21 @@ def popen( executable_path = args[0] if not os.path.isabs(executable_path): executable_path = os.path.join(sysconfig.get_path("scripts"), executable_path) - if not os.path.isfile(executable_path): + + # On Windows, it's valid to start a process using only '{program-name}' and Windows will + # automatically find and execute '{program-name}.exe'. + # + # That allows e.g. `popen(["dask-worker"])` to work despite the installed file being called 'dask-worker.exe'. + # + # docs: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw + # + if WINDOWS: + executable_exists = os.path.isfile(executable_path) or os.path.isfile( + f"{executable_path}.exe" + ) + else: + executable_exists = os.path.isfile(executable_path) + if not executable_exists: raise FileNotFoundError( f"Could not find '{executable_path}'. To avoid this warning, provide an absolute path to an existing installation to popen()." )