From 7e7ebafbf4749285a46465038a377586f34a6bb2 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 15:57:36 -0800 Subject: [PATCH 01/12] Initial commit for LLVM 21 --- PCbuild/get_externals.bat | 4 ++-- Tools/jit/README.md | 18 +++++++++--------- Tools/jit/_llvm.py | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 319024e0f50f46..82d2a740548516 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -82,7 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -92,7 +92,7 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - if "%%b"=="llvm-20.1.8.0" ( + if "%%b"=="llvm-21.1.4.0" ( %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b ) else ( %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b diff --git a/Tools/jit/README.md b/Tools/jit/README.md index d83b09aab59f8c..45875c5d165e69 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 20 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 20 on Ubuntu/Debian: +Install LLVM 21 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 20 +sudo ./llvm.sh 21 ``` -Install LLVM 20 on Fedora Linux 40 or newer: +Install LLVM 21 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' +sudo dnf install 'clang(major) = 21' 'llvm(major) = 21' ``` ### macOS -Install LLVM 20 with [Homebrew](https://brew.sh): +Install LLVM 21 with [Homebrew](https://brew.sh): ```sh -brew install llvm@20 +brew install llvm@21 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -48,13 +48,13 @@ Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub release Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=20.1.8 +choco install llvm --version=21.1.0 ``` ### Dev Containers If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no -need to install LLVM as the Fedora 42 base image includes LLVM 20 out of the box. +need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 54c2bf86a36ed6..517e07ffa8d94a 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -11,8 +11,8 @@ import _targets -_LLVM_VERSION = "20" -_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" +_LLVM_VERSION = "21" +_EXTERNALS_LLVM_TAG = "llvm-21.1.4.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") From ae4a01dccef194fd2e6e3dd76e88aed7792c1ba9 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 15:58:49 -0800 Subject: [PATCH 02/12] Update CI --- .github/workflows/jit.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69d900091a3bd1..93600b90f65e75 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 20 + - 21 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -138,7 +138,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -193,7 +193,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: From da9b795ea415c26991e7c7663ad8648b0946e581 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 15:59:33 -0800 Subject: [PATCH 03/12] Another CI fix --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 93600b90f65e75..62325250bd368e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -166,7 +166,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: From d079df91c16dff0d065614def5c6ec69b0c24238 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 18:49:32 -0800 Subject: [PATCH 04/12] Add debugging --- Tools/jit/_llvm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 517e07ffa8d94a..d33a5ba6861372 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -72,18 +72,25 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: + print(f"DEBUG: Searching for {tool} (LLVM {llvm_version})") # Unversioned executables: path = tool + print(f"DEBUG: Trying unversioned: {path}") if await _check_tool_version(path, llvm_version, echo=echo): + print(f"DEBUG: Found at: {path}") return path # Versioned executables: path = f"{tool}-{llvm_version}" + print(f"DEBUG: Trying versioned: {path}") if await _check_tool_version(path, llvm_version, echo=echo): + print(f"DEBUG: Found at: {path}") return path # PCbuild externals: externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) + print(f"DEBUG: Trying PCbuild externals: {path} (exists: {os.path.exists(path)})") if await _check_tool_version(path, llvm_version, echo=echo): + print(f"DEBUG: Found at: {path}") return path # Homebrew-installed executables: prefix = await _get_brew_llvm_prefix(llvm_version, echo=echo) From 322ea9b50751e2e284d8b9e8ce81b20d389a3d39 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 19:18:20 -0800 Subject: [PATCH 05/12] Add fix for exe --- .github/workflows/jit.yml | 258 +++++++++++++++++++------------------- Tools/jit/_llvm.py | 8 ++ 2 files changed, 137 insertions(+), 129 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 62325250bd368e..64a05447634ee1 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -33,24 +33,24 @@ env: FORCE_COLOR: 1 jobs: - interpreter: - name: Interpreter (Debug) - runs-on: ubuntu-24.04 - timeout-minutes: 90 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Build tier two interpreter - run: | - ./configure --enable-experimental-jit=interpreter --with-pydebug - make all --jobs 4 - - name: Test tier two interpreter - run: | - ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # interpreter: + # name: Interpreter (Debug) + # runs-on: ubuntu-24.04 + # timeout-minutes: 90 + # steps: + # - uses: actions/checkout@v4 + # with: + # persist-credentials: false + # - name: Build tier two interpreter + # run: | + # ./configure --enable-experimental-jit=interpreter --with-pydebug + # make all --jobs 4 + # - name: Test tier two interpreter + # run: | + # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) - needs: interpreter + # needs: interpreter runs-on: ${{ matrix.runner }} timeout-minutes: 90 strategy: @@ -60,10 +60,10 @@ jobs: - i686-pc-windows-msvc/msvc - x86_64-pc-windows-msvc/msvc - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc + # - x86_64-apple-darwin/clang + # - aarch64-apple-darwin/clang + # - x86_64-unknown-linux-gnu/gcc + # - aarch64-unknown-linux-gnu/gcc debug: - true - false @@ -79,18 +79,18 @@ jobs: - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-11-arm - - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-15-intel - - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm + # - target: x86_64-apple-darwin/clang + # architecture: x86_64 + # runner: macos-15-intel + # - target: aarch64-apple-darwin/clang + # architecture: aarch64 + # runner: macos-14 + # - target: x86_64-unknown-linux-gnu/gcc + # architecture: x86_64 + # runner: ubuntu-24.04 + # - target: aarch64-unknown-linux-gnu/gcc + # architecture: aarch64 + # runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 with: @@ -106,104 +106,104 @@ jobs: ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: macOS - if: runner.os == 'macOS' - run: | - brew update - brew install llvm@${{ matrix.llvm }} - export SDKROOT="$(xcrun --show-sdk-path)" - # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to - # make sure we don't break downstream distributors (like uv): - export CFLAGS_JIT='-Werror=unguarded-availability' - export MACOSX_DEPLOYMENT_TARGET=10.15 - ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} - make all --jobs 4 - ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # - name: macOS + # if: runner.os == 'macOS' + # run: | + # brew update + # brew install llvm@${{ matrix.llvm }} + # export SDKROOT="$(xcrun --show-sdk-path)" + # # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to + # # make sure we don't break downstream distributors (like uv): + # export CFLAGS_JIT='-Werror=unguarded-availability' + # export MACOSX_DEPLOYMENT_TARGET=10.15 + # ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} + # make all --jobs 4 + # ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Linux - if: runner.os == 'Linux' - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} - make all --jobs 4 - ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # - name: Linux + # if: runner.os == 'Linux' + # run: | + # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + # ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} + # make all --jobs 4 + # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - jit-with-disabled-gil: - name: Free-Threaded (Debug) - needs: interpreter - runs-on: ubuntu-24.04 - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - llvm: - - 21 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Build with JIT enabled and GIL disabled - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit --with-pydebug --disable-gil - make all --jobs 4 - - name: Run tests - run: | - ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - continue-on-error: true + # jit-with-disabled-gil: + # name: Free-Threaded (Debug) + # needs: interpreter + # runs-on: ubuntu-24.04 + # timeout-minutes: 90 + # strategy: + # fail-fast: false + # matrix: + # llvm: + # - 21 + # steps: + # - uses: actions/checkout@v4 + # with: + # persist-credentials: false + # - uses: actions/setup-python@v5 + # with: + # python-version: '3.11' + # - name: Build with JIT enabled and GIL disabled + # run: | + # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + # ./configure --enable-experimental-jit --with-pydebug --disable-gil + # make all --jobs 4 + # - name: Run tests + # run: | + # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # continue-on-error: true - no-opt-jit: - name: JIT without optimizations (Debug) - needs: interpreter - runs-on: ubuntu-24.04 - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - llvm: - - 21 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Build with JIT - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit --with-pydebug - make all --jobs 4 - - name: Run tests without optimizations - run: | - PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # no-opt-jit: + # name: JIT without optimizations (Debug) + # needs: interpreter + # runs-on: ubuntu-24.04 + # timeout-minutes: 90 + # strategy: + # fail-fast: false + # matrix: + # llvm: + # - 21 + # steps: + # - uses: actions/checkout@v4 + # with: + # persist-credentials: false + # - uses: actions/setup-python@v5 + # with: + # python-version: '3.11' + # - name: Build with JIT + # run: | + # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + # ./configure --enable-experimental-jit --with-pydebug + # make all --jobs 4 + # - name: Run tests without optimizations + # run: | + # PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - tail-call-jit: - name: JIT with tail calling interpreter - needs: interpreter - runs-on: ubuntu-24.04 - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - llvm: - - 21 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Build with JIT and tailcall - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug - make all --jobs 4 + # tail-call-jit: + # name: JIT with tail calling interpreter + # needs: interpreter + # runs-on: ubuntu-24.04 + # timeout-minutes: 90 + # strategy: + # fail-fast: false + # matrix: + # llvm: + # - 21 + # steps: + # - uses: actions/checkout@v4 + # with: + # persist-credentials: false + # - uses: actions/setup-python@v5 + # with: + # python-version: '3.11' + # - name: Build with JIT and tailcall + # run: | + # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + # CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug + # make all --jobs 4 diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index d33a5ba6861372..5256b3e7dfdcbf 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -89,6 +89,14 @@ async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) print(f"DEBUG: Trying PCbuild externals: {path} (exists: {os.path.exists(path)})") + # On Windows, executables need .exe extension + if os.name == "nt" and not path.endswith(".exe"): + path_with_exe = path + ".exe" + print( + f"DEBUG: Also trying with .exe: {path_with_exe} (exists: {os.path.exists(path_with_exe)})" + ) + if os.path.exists(path_with_exe): + path = path_with_exe if await _check_tool_version(path, llvm_version, echo=echo): print(f"DEBUG: Found at: {path}") return path From 27a91acafc58e3433de12a3c4b6a9d9fc2634082 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 19:29:48 -0800 Subject: [PATCH 06/12] Update tarball extraction --- PCbuild/get_external.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 07970624e8647e..a502430e72f13c 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -3,8 +3,8 @@ import argparse import os import pathlib -import shutil import sys +import tarfile import time import urllib.error import urllib.request @@ -55,9 +55,15 @@ def fetch_release(tag, tarball_dir, *, org='python', verbose=False): def extract_tarball(externals_dir, tarball_path, tag): - output_path = externals_dir / tag - shutil.unpack_archive(os.fspath(tarball_path), os.fspath(output_path)) - return output_path + # Extract to externals_dir and return the path to the first top-level directory + with tarfile.open(tarball_path) as tf: + # Get the first top-level directory name from the tarball + members = tf.getmembers() + if not members: + raise ValueError(f"Tarball {tarball_path} is empty") + top_level_dir = members[0].name.split('/')[0] + tf.extractall(os.fspath(externals_dir)) + return externals_dir / top_level_dir def extract_zip(externals_dir, zip_path): From 2ca86037c85d184cd4f9a1b9ec9c8aa2cb226ef4 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 19:41:37 -0800 Subject: [PATCH 07/12] Fix for windows --- PCbuild/get_external.py | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index a502430e72f13c..edf14ce578bce6 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -55,15 +55,10 @@ def fetch_release(tag, tarball_dir, *, org='python', verbose=False): def extract_tarball(externals_dir, tarball_path, tag): - # Extract to externals_dir and return the path to the first top-level directory + output_path = externals_dir / tag with tarfile.open(tarball_path) as tf: - # Get the first top-level directory name from the tarball - members = tf.getmembers() - if not members: - raise ValueError(f"Tarball {tarball_path} is empty") - top_level_dir = members[0].name.split('/')[0] tf.extractall(os.fspath(externals_dir)) - return externals_dir / top_level_dir + return output_path def extract_zip(externals_dir, zip_path): @@ -121,21 +116,23 @@ def main(): verbose=args.verbose, ) extracted = extract_zip(args.externals_dir, zip_path) - for wait in [1, 2, 3, 5, 8, 0]: - try: - extracted.replace(final_name) - break - except PermissionError as ex: - retry = f" Retrying in {wait}s..." if wait else "" - print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) - time.sleep(wait) - else: - print( - f"ERROR: Failed to extract {final_name}.", - "You may need to restart your build", - file=sys.stderr, - ) - sys.exit(1) + + if extracted != final_name: + for wait in [1, 2, 3, 5, 8, 0]: + try: + extracted.replace(final_name) + break + except PermissionError as ex: + retry = f" Retrying in {wait}s..." if wait else "" + print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) + time.sleep(wait) + else: + print( + f"ERROR: Failed to rename {extracted} to {final_name}.", + "You may need to restart your build", + file=sys.stderr, + ) + sys.exit(1) if __name__ == '__main__': From 39e1339c1561254516b9fbcd6fabf231dd623921 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 19:54:51 -0800 Subject: [PATCH 08/12] Clean up after debugging --- Tools/jit/_llvm.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 5256b3e7dfdcbf..0b9cb5192f1b75 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -72,33 +72,23 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: - print(f"DEBUG: Searching for {tool} (LLVM {llvm_version})") # Unversioned executables: path = tool - print(f"DEBUG: Trying unversioned: {path}") if await _check_tool_version(path, llvm_version, echo=echo): - print(f"DEBUG: Found at: {path}") return path # Versioned executables: path = f"{tool}-{llvm_version}" - print(f"DEBUG: Trying versioned: {path}") if await _check_tool_version(path, llvm_version, echo=echo): - print(f"DEBUG: Found at: {path}") return path # PCbuild externals: externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) - print(f"DEBUG: Trying PCbuild externals: {path} (exists: {os.path.exists(path)})") # On Windows, executables need .exe extension if os.name == "nt" and not path.endswith(".exe"): path_with_exe = path + ".exe" - print( - f"DEBUG: Also trying with .exe: {path_with_exe} (exists: {os.path.exists(path_with_exe)})" - ) if os.path.exists(path_with_exe): path = path_with_exe if await _check_tool_version(path, llvm_version, echo=echo): - print(f"DEBUG: Found at: {path}") return path # Homebrew-installed executables: prefix = await _get_brew_llvm_prefix(llvm_version, echo=echo) From 38105ce47c8ca28014468aaba4d3d71f01e85824 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 19:56:06 -0800 Subject: [PATCH 09/12] Uncomment CI --- .github/workflows/jit.yml | 258 +++++++++++++++++++------------------- 1 file changed, 129 insertions(+), 129 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 64a05447634ee1..62325250bd368e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -33,24 +33,24 @@ env: FORCE_COLOR: 1 jobs: - # interpreter: - # name: Interpreter (Debug) - # runs-on: ubuntu-24.04 - # timeout-minutes: 90 - # steps: - # - uses: actions/checkout@v4 - # with: - # persist-credentials: false - # - name: Build tier two interpreter - # run: | - # ./configure --enable-experimental-jit=interpreter --with-pydebug - # make all --jobs 4 - # - name: Test tier two interpreter - # run: | - # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + interpreter: + name: Interpreter (Debug) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build tier two interpreter + run: | + ./configure --enable-experimental-jit=interpreter --with-pydebug + make all --jobs 4 + - name: Test tier two interpreter + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) - # needs: interpreter + needs: interpreter runs-on: ${{ matrix.runner }} timeout-minutes: 90 strategy: @@ -60,10 +60,10 @@ jobs: - i686-pc-windows-msvc/msvc - x86_64-pc-windows-msvc/msvc - aarch64-pc-windows-msvc/msvc - # - x86_64-apple-darwin/clang - # - aarch64-apple-darwin/clang - # - x86_64-unknown-linux-gnu/gcc - # - aarch64-unknown-linux-gnu/gcc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc debug: - true - false @@ -79,18 +79,18 @@ jobs: - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-11-arm - # - target: x86_64-apple-darwin/clang - # architecture: x86_64 - # runner: macos-15-intel - # - target: aarch64-apple-darwin/clang - # architecture: aarch64 - # runner: macos-14 - # - target: x86_64-unknown-linux-gnu/gcc - # architecture: x86_64 - # runner: ubuntu-24.04 - # - target: aarch64-unknown-linux-gnu/gcc - # architecture: aarch64 - # runner: ubuntu-24.04-arm + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-15-intel + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 with: @@ -106,104 +106,104 @@ jobs: ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # - name: macOS - # if: runner.os == 'macOS' - # run: | - # brew update - # brew install llvm@${{ matrix.llvm }} - # export SDKROOT="$(xcrun --show-sdk-path)" - # # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to - # # make sure we don't break downstream distributors (like uv): - # export CFLAGS_JIT='-Werror=unguarded-availability' - # export MACOSX_DEPLOYMENT_TARGET=10.15 - # ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} - # make all --jobs 4 - # ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + - name: macOS + if: runner.os == 'macOS' + run: | + brew update + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to + # make sure we don't break downstream distributors (like uv): + export CFLAGS_JIT='-Werror=unguarded-availability' + export MACOSX_DEPLOYMENT_TARGET=10.15 + ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} + make all --jobs 4 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # - name: Linux - # if: runner.os == 'Linux' - # run: | - # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - # ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} - # make all --jobs 4 - # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + - name: Linux + if: runner.os == 'Linux' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # jit-with-disabled-gil: - # name: Free-Threaded (Debug) - # needs: interpreter - # runs-on: ubuntu-24.04 - # timeout-minutes: 90 - # strategy: - # fail-fast: false - # matrix: - # llvm: - # - 21 - # steps: - # - uses: actions/checkout@v4 - # with: - # persist-credentials: false - # - uses: actions/setup-python@v5 - # with: - # python-version: '3.11' - # - name: Build with JIT enabled and GIL disabled - # run: | - # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - # ./configure --enable-experimental-jit --with-pydebug --disable-gil - # make all --jobs 4 - # - name: Run tests - # run: | - # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # continue-on-error: true + jit-with-disabled-gil: + name: Free-Threaded (Debug) + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT enabled and GIL disabled + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit --with-pydebug --disable-gil + make all --jobs 4 + - name: Run tests + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + continue-on-error: true - # no-opt-jit: - # name: JIT without optimizations (Debug) - # needs: interpreter - # runs-on: ubuntu-24.04 - # timeout-minutes: 90 - # strategy: - # fail-fast: false - # matrix: - # llvm: - # - 21 - # steps: - # - uses: actions/checkout@v4 - # with: - # persist-credentials: false - # - uses: actions/setup-python@v5 - # with: - # python-version: '3.11' - # - name: Build with JIT - # run: | - # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - # ./configure --enable-experimental-jit --with-pydebug - # make all --jobs 4 - # - name: Run tests without optimizations - # run: | - # PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + no-opt-jit: + name: JIT without optimizations (Debug) + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit --with-pydebug + make all --jobs 4 + - name: Run tests without optimizations + run: | + PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # tail-call-jit: - # name: JIT with tail calling interpreter - # needs: interpreter - # runs-on: ubuntu-24.04 - # timeout-minutes: 90 - # strategy: - # fail-fast: false - # matrix: - # llvm: - # - 21 - # steps: - # - uses: actions/checkout@v4 - # with: - # persist-credentials: false - # - uses: actions/setup-python@v5 - # with: - # python-version: '3.11' - # - name: Build with JIT and tailcall - # run: | - # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - # CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug - # make all --jobs 4 + tail-call-jit: + name: JIT with tail calling interpreter + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT and tailcall + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug + make all --jobs 4 From e85c8f8572274b2c1aea2f12a903765203a834c4 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 4 Nov 2025 04:57:29 +0000 Subject: [PATCH 10/12] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst new file mode 100644 index 00000000000000..0a615ed131127f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 21 at build time. From 3ca97336c56a6c4599bcb920f9e97e417aed9a2f Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 3 Nov 2025 21:01:47 -0800 Subject: [PATCH 11/12] Fix README --- Tools/jit/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 45875c5d165e69..dd7deb7b256449 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -43,7 +43,7 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): From ef52c5adad58ac42608a689812fa469c9f103288 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 4 Nov 2025 08:12:44 -0800 Subject: [PATCH 12/12] Remove fixes for Windows release artifacts --- PCbuild/get_external.py | 37 +++++++++++++++++-------------------- Tools/jit/_llvm.py | 5 ----- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index edf14ce578bce6..07970624e8647e 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -3,8 +3,8 @@ import argparse import os import pathlib +import shutil import sys -import tarfile import time import urllib.error import urllib.request @@ -56,8 +56,7 @@ def fetch_release(tag, tarball_dir, *, org='python', verbose=False): def extract_tarball(externals_dir, tarball_path, tag): output_path = externals_dir / tag - with tarfile.open(tarball_path) as tf: - tf.extractall(os.fspath(externals_dir)) + shutil.unpack_archive(os.fspath(tarball_path), os.fspath(output_path)) return output_path @@ -116,23 +115,21 @@ def main(): verbose=args.verbose, ) extracted = extract_zip(args.externals_dir, zip_path) - - if extracted != final_name: - for wait in [1, 2, 3, 5, 8, 0]: - try: - extracted.replace(final_name) - break - except PermissionError as ex: - retry = f" Retrying in {wait}s..." if wait else "" - print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) - time.sleep(wait) - else: - print( - f"ERROR: Failed to rename {extracted} to {final_name}.", - "You may need to restart your build", - file=sys.stderr, - ) - sys.exit(1) + for wait in [1, 2, 3, 5, 8, 0]: + try: + extracted.replace(final_name) + break + except PermissionError as ex: + retry = f" Retrying in {wait}s..." if wait else "" + print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) + time.sleep(wait) + else: + print( + f"ERROR: Failed to extract {final_name}.", + "You may need to restart your build", + file=sys.stderr, + ) + sys.exit(1) if __name__ == '__main__': diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 0b9cb5192f1b75..517e07ffa8d94a 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -83,11 +83,6 @@ async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str # PCbuild externals: externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) - # On Windows, executables need .exe extension - if os.name == "nt" and not path.endswith(".exe"): - path_with_exe = path + ".exe" - if os.path.exists(path_with_exe): - path = path_with_exe if await _check_tool_version(path, llvm_version, echo=echo): return path # Homebrew-installed executables: