From d145747fa1c87004c65568014c3f81de24e7fd0a Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Thu, 18 Sep 2025 15:56:58 +0100 Subject: [PATCH 01/30] LLVM 20 bump --- .github/workflows/jit.yml | 8 ++++---- PCbuild/build.bat | 1 - PCbuild/get_externals.bat | 4 +--- Tools/jit/README.md | 18 +++++++++--------- Tools/jit/_llvm.py | 10 +--------- Tools/jit/_targets.py | 13 ++++--------- 6 files changed, 19 insertions(+), 35 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 52f7d0d2b3df95..2b0fcdc6b4964e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 19 + - 20 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -99,10 +99,10 @@ jobs: with: python-version: '3.11' - # PCbuild downloads LLVM automatically: - name: Windows if: runner.os == 'Windows' run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.8 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -143,7 +143,7 @@ jobs: fail-fast: false matrix: llvm: - - 19 + - 20 steps: - uses: actions/checkout@v4 with: @@ -171,7 +171,7 @@ jobs: # fail-fast: false # matrix: # llvm: - # - 19 + # - 20 # steps: # - uses: actions/checkout@v4 # with: diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 602357048867d6..6c8f1b31636c16 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -111,7 +111,6 @@ if "%IncludeExternals%"=="" set IncludeExternals=true if "%IncludeCTypes%"=="" set IncludeCTypes=true if "%IncludeSSL%"=="" set IncludeSSL=true if "%IncludeTkinter%"=="" set IncludeTkinter=true -if "%UseJIT%" NEQ "true" set IncludeLLVM=false if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index eff8d1ccd7f146..86cf98c0501f49 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -15,7 +15,6 @@ set IncludeSSLSrc=false if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts if "%~1"=="--no-libffi" (set IncludeLibffi=false) & shift & goto CheckOpts -if "%~1"=="--no-llvm" (set IncludeLLVM=false) & shift & goto CheckOpts if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts if "%~1"=="--libffi-src" (set IncludeLibffiSrc=true) & shift & goto CheckOpts @@ -82,7 +81,6 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.16.2 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-19.1.7.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -101,7 +99,7 @@ goto end :usage echo.Valid options: -c, --clean, --clean-only, --organization, --python, -echo.--no-tkinter, --no-openssl, --no-llvm +echo.--no-tkinter, --no-openssl echo. echo.Pull all sources and binaries necessary for compiling optional extension echo.modules that rely on external libraries. diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 8e817574b4d72b..5afbd9f096d1e0 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 19 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 20 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-20`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 19 on Ubuntu/Debian: +Install LLVM 20 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 19 +sudo ./llvm.sh 20 ``` -Install LLVM 19 on Fedora Linux 40 or newer: +Install LLVM 20 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 19' 'llvm(major) = 19' +sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' ``` ### macOS -Install LLVM 19 with [Homebrew](https://brew.sh): +Install LLVM 20 with [Homebrew](https://brew.sh): ```sh -brew install llvm@19 +brew install llvm@20 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,12 +43,12 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=19.1.0 +choco install llvm --version=20.1.8 ``` diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f09a8404871b24..7a18465fc5b1c0 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -8,11 +8,8 @@ import subprocess import typing -import _targets - -_LLVM_VERSION = 19 +_LLVM_VERSION = 20 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") -_EXTERNALS_LLVM_TAG = "llvm-19.1.7.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") @@ -75,11 +72,6 @@ async def _find_tool(tool: str, *, echo: bool = False) -> str | None: return path # Versioned executables: path = f"{tool}-{_LLVM_VERSION}" - if await _check_tool_version(path, echo=echo): - return path - # PCbuild externals: - externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) - path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) if await _check_tool_version(path, echo=echo): return path # Homebrew-installed executables: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 2185d8190a8935..c2912594dd7732 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,7 +25,6 @@ TOOLS_JIT = TOOLS_JIT_BUILD.parent TOOLS = TOOLS_JIT.parent CPYTHON = TOOLS.parent -EXTERNALS = CPYTHON / "externals" PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" @@ -161,10 +160,6 @@ async def _compile( "-fno-asynchronous-unwind-tables", # Don't call built-in functions that we can't find or patch: "-fno-builtin", - # Emit relaxable 64-bit calls/jumps, so we don't have to worry about - # about emitting in-range trampolines for out-of-range targets. - # We can probably remove this and emit trampolines in the future: - "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-std=c11", @@ -561,13 +556,13 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): # -mno-outline-atomics: Keep intrinsics from being emitted. - args = ["-fpic", "-mno-outline-atomics"] + args = ["-fpic", "-mno-outline-atomics", "-fno-plt"] condition = "defined(__aarch64__) && defined(__linux__)" optimizer = _optimizers.OptimizerAArch64 target = _ELF(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"i686-pc-windows-msvc", host): # -Wno-ignored-attributes: __attribute__((preserve_none)) is not supported here. - args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes"] + args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes", "-fno-plt"] optimizer = _optimizers.OptimizerX86 condition = "defined(_M_IX86)" target = _COFF32(host, condition, args=args, optimizer=optimizer) @@ -576,12 +571,12 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: optimizer = _optimizers.OptimizerX86 target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): - args = ["-fms-runtime-lib=dll"] + args = ["-fms-runtime-lib=dll", "-fno-plt"] condition = "defined(_M_X64)" optimizer = _optimizers.OptimizerX86 target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): - args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0"] + args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0", "-fno-plt"] condition = "defined(__x86_64__) && defined(__linux__)" optimizer = _optimizers.OptimizerX86 target = _ELF(host, condition, args=args, optimizer=optimizer) From 9324b14b590bcf30f09843bc16a95a2350616439 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Fri, 19 Sep 2025 12:56:57 +0100 Subject: [PATCH 02/30] Add flags for testing --- Tools/jit/_targets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index c2ff3b46e75748..533827ee47ae6c 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -573,6 +573,7 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-apple-darwin.*", host): host = "x86_64-apple-darwin" condition = "defined(__x86_64__) && defined(__APPLE__)" + args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0", "-fno-plt"] optimizer = _optimizers.OptimizerX86 target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): From e86764488fc29f4259e2bc9dbd877a02db3f5139 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sun, 5 Oct 2025 22:18:04 -0700 Subject: [PATCH 03/30] Fix windows --- PCbuild/build.bat | 1 + PCbuild/get_externals.bat | 6 ++++-- PCbuild/regen.targets | 2 +- Tools/jit/_llvm.py | 8 ++++++++ Tools/jit/_targets.py | 1 + 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 6c8f1b31636c16..602357048867d6 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -111,6 +111,7 @@ if "%IncludeExternals%"=="" set IncludeExternals=true if "%IncludeCTypes%"=="" set IncludeCTypes=true if "%IncludeSSL%"=="" set IncludeSSL=true if "%IncludeTkinter%"=="" set IncludeTkinter=true +if "%UseJIT%" NEQ "true" set IncludeLLVM=false if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index dcbfa7c7e32ce2..433e45a5e9f992 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -15,6 +15,7 @@ set IncludeSSLSrc=false if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts if "%~1"=="--no-libffi" (set IncludeLibffi=false) & shift & goto CheckOpts +if "%~1"=="--no-llvm" (set IncludeLLVM=false) & shift & goto CheckOpts if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts if "%~1"=="--libffi-src" (set IncludeLibffiSrc=true) & shift & goto CheckOpts @@ -81,6 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -99,7 +101,7 @@ goto end :usage echo.Valid options: -c, --clean, --clean-only, --organization, --python, -echo.--no-tkinter, --no-openssl +echo.--no-tkinter, --no-openssl, --no-llvm echo. echo.Pull all sources and binaries necessary for compiling optional extension echo.modules that rely on external libraries. @@ -115,4 +117,4 @@ echo.anything new. echo. exit /b -1 -:end +:end \ No newline at end of file diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 742597f5cb5ebd..e20627032ccd71 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -187,4 +187,4 @@ - + \ No newline at end of file diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 7a18465fc5b1c0..ac251698de1d98 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -8,8 +8,11 @@ import subprocess import typing +import _targets + _LLVM_VERSION = 20 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") +_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") @@ -72,6 +75,11 @@ async def _find_tool(tool: str, *, echo: bool = False) -> str | None: return path # Versioned executables: path = f"{tool}-{_LLVM_VERSION}" + if await _check_tool_version(path, echo=echo): + return path + # PCbuild externals: + externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) + path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) if await _check_tool_version(path, echo=echo): return path # Homebrew-installed executables: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 533827ee47ae6c..b8fdbea7425439 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,6 +25,7 @@ TOOLS_JIT = TOOLS_JIT_BUILD.parent TOOLS = TOOLS_JIT.parent CPYTHON = TOOLS.parent +EXTERNALS = CPYTHON / "externals" PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" From 0034f143eb888572ad4d3d21a0f7af7fcfd77dba Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Wed, 6 Aug 2025 16:06:46 -0700 Subject: [PATCH 04/30] Download binaries from GitHub releases With LLVM 20, individual files are greater than the 100MiB single file limit for items checked into git. Therefore, this PR pulls down binaries from GitHub releases, as `.tar.xz` files to additionally maximize compression ratio. Currently this is somewhat of a first draft, as there are things like hash checking needed to be done. --- PCbuild/get_external.py | 118 +++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 21 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index a78aa6a23041ad..26d0cd3ea385e8 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 import argparse +import contextlib +import io import os import pathlib +import shutil import sys import time import urllib.error @@ -10,28 +13,56 @@ import zipfile -def retrieve_with_retries(download_location, output_path, reporthook, - max_retries=7): - """Download a file with exponential backoff retry and save to disk.""" +# Mapping of binary dependency tag to GitHub release asset ID +TAG_TO_ASSET_ID = { + "libffi-3.4.4": 280027073, + "openssl-bin-3.0.16.2": 280041244, + "tcltk-8.6.15.0": 280042163, + "nasm-2.11.06": 280042740, + "llvm-19.1.7.0": 280052497, +} + + +def request_with_retry( + request_func, *args, max_retries=7, err_msg="Request failed.", **kwargs, +): + """Make a request using request_func with exponential backoff""" for attempt in range(max_retries + 1): try: - resp = urllib.request.urlretrieve( - download_location, - output_path, - reporthook=reporthook, - ) + resp = request_func(*args, **kwargs) except (urllib.error.URLError, ConnectionError) as ex: if attempt == max_retries: - msg = f"Download from {download_location} failed." - raise OSError(msg) from ex + raise OSError(err_msg) from ex time.sleep(2.25**attempt) else: return resp -def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): - repo = f'cpython-{"bin" if binary else "source"}-deps' - url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' +def retrieve_with_retries(download_location, output_path, reporthook): + """Download a file with retries.""" + return request_with_retry( + urllib.request.urlretrieve, + download_location, + output_path, + reporthook, + err_msg=f"Download from {download_location} failed.", + ) + + +def get_with_retries(url, headers): + req = urllib.request.Request( + url=url, + headers=headers, + method="GET", + ) + return request_with_retry( + urllib.request.urlopen, req, err_msg=f"Request to {url} failed.", + timeout=30, + ) + + +def fetch_zip(commit_hash, zip_dir, *, org='python', verbose): + url = f'https://github.com/{org}/cpython-source-deps/archive/{commit_hash}.zip' reporthook = None if verbose: reporthook = print @@ -44,6 +75,44 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): return filename +def fetch_release_asset(asset_id, output_path, org): + """Download a GitHub release asset. + + Release assets need the Content-Type header set to + application/octet-stream, so we can't use urlretrieve. Code here is + based on urlretrieve + """ + # TODO: digest/shasum checking + url = f"https://api.github.com/repos/{org}/cpython-bin-deps/releases/assets/{asset_id}" + with contextlib.closing( + get_with_retries(url, headers={"Accept": "application/octet-stream"}) + ) as resp: + headers = resp.info() + if resp.status != 200: + raise RuntimeError("Failed to download asset") + read = 0 + with open(output_path, 'wb') as fp: + while block := resp.read(io.DEFAULT_BUFFER_SIZE): + read += len(block) + fp.write(block) + + +def fetch_release(tag, tarball_dir, *, org='python'): + tarball_dir.mkdir(exist_ok=True) + asset_id = TAG_TO_ASSET_ID.get(tag) + if asset_id is None: + raise ValueError(f"Unknown tag for binary dependencies {tag}") + output_path = tarball_dir / f'{tag}.tar.xz' + fetch_release_asset(asset_id, output_path, org) + return output_path + + +def extract_tarball(externals_dir, tarball_path, tag): + output_path = externals_dir / tag + shutil.unpack_archive(os.fspath(tarball_path), os.fspath(output_path)) + return output_path + + def extract_zip(externals_dir, zip_path): with zipfile.ZipFile(os.fspath(zip_path)) as zf: zf.extractall(os.fspath(externals_dir)) @@ -67,15 +136,22 @@ def parse_args(): def main(): args = parse_args() - zip_path = fetch_zip( - args.tag, - args.externals_dir / 'zips', - org=args.organization, - binary=args.binary, - verbose=args.verbose, - ) + if args.binary: + tarball_path = fetch_release( + args.tag, + args.externals_dir / 'tarballs', + org=args.organization, + ) + extracted = extract_tarball(args.externals_dir, tarball_path, args.tag) + else: + zip_path = fetch_zip( + args.tag, + args.externals_dir / 'zips', + org=args.organization, + verbose=args.verbose, + ) + extracted = extract_zip(args.externals_dir, zip_path) final_name = args.externals_dir / args.tag - extracted = extract_zip(args.externals_dir, zip_path) for wait in [1, 2, 3, 5, 8, 0]: try: extracted.replace(final_name) From cc98d3002c81b74496851418db87c2f34a3e2425 Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Wed, 6 Aug 2025 20:20:37 -0700 Subject: [PATCH 05/30] Add hash checking --- PCbuild/get_external.py | 55 ++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 26d0cd3ea385e8..0061728c2bac1f 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -2,7 +2,9 @@ import argparse import contextlib +import hashlib import io +import json import os import pathlib import shutil @@ -15,17 +17,16 @@ # Mapping of binary dependency tag to GitHub release asset ID TAG_TO_ASSET_ID = { - "libffi-3.4.4": 280027073, - "openssl-bin-3.0.16.2": 280041244, - "tcltk-8.6.15.0": 280042163, - "nasm-2.11.06": 280042740, - "llvm-19.1.7.0": 280052497, + 'libffi-3.4.4': 280027073, + 'openssl-bin-3.0.16.2': 280041244, + 'tcltk-8.6.15.0': 280042163, + 'nasm-2.11.06': 280042740, + 'llvm-19.1.7.0': 280052497, } -def request_with_retry( - request_func, *args, max_retries=7, err_msg="Request failed.", **kwargs, -): +def request_with_retry(request_func, *args, max_retries=7, + err_msg='Request failed.', **kwargs): """Make a request using request_func with exponential backoff""" for attempt in range(max_retries + 1): try: @@ -45,19 +46,16 @@ def retrieve_with_retries(download_location, output_path, reporthook): download_location, output_path, reporthook, - err_msg=f"Download from {download_location} failed.", + err_msg=f'Download from {download_location} failed.', ) def get_with_retries(url, headers): - req = urllib.request.Request( - url=url, - headers=headers, - method="GET", - ) + req = urllib.request.Request(url=url, headers=headers, method='GET') return request_with_retry( - urllib.request.urlopen, req, err_msg=f"Request to {url} failed.", - timeout=30, + urllib.request.urlopen, + req, + err_msg=f'Request to {url} failed.' ) @@ -79,29 +77,36 @@ def fetch_release_asset(asset_id, output_path, org): """Download a GitHub release asset. Release assets need the Content-Type header set to - application/octet-stream, so we can't use urlretrieve. Code here is - based on urlretrieve + application/octet-stream to download the binary, so we can't use + urlretrieve. Code here is based on urlretrieve """ - # TODO: digest/shasum checking - url = f"https://api.github.com/repos/{org}/cpython-bin-deps/releases/assets/{asset_id}" + url = f'https://api.github.com/repos/{org}/cpython-bin-deps/releases/assets/{asset_id}' + rest = get_with_retries(url, + headers={'Accept': 'application/vnd.github+json'}) + json_data = json.loads(rest.read()) + hash_info = json_data['digest'] + algorithm, hashsum = hash_info.split(':') + if algorithm != 'sha256': + raise RuntimeError(f'Unknown hash algorithm {algorithm} for asset {asset_id}') with contextlib.closing( - get_with_retries(url, headers={"Accept": "application/octet-stream"}) + get_with_retries(url, headers={'Accept': 'application/octet-stream'}) ) as resp: - headers = resp.info() - if resp.status != 200: - raise RuntimeError("Failed to download asset") read = 0 + hasher = hashlib.sha256() with open(output_path, 'wb') as fp: while block := resp.read(io.DEFAULT_BUFFER_SIZE): + hasher.update(block) read += len(block) fp.write(block) + if hasher.hexdigest() != hashsum: + raise RuntimeError('Downloaded content hash did not match!') def fetch_release(tag, tarball_dir, *, org='python'): tarball_dir.mkdir(exist_ok=True) asset_id = TAG_TO_ASSET_ID.get(tag) if asset_id is None: - raise ValueError(f"Unknown tag for binary dependencies {tag}") + raise ValueError(f'Unknown tag for binary dependencies {tag}') output_path = tarball_dir / f'{tag}.tar.xz' fetch_release_asset(asset_id, output_path, org) return output_path From 76842ebe12d8fe71f65f48dabda92809eb90c00b Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 7 Oct 2025 21:33:05 -0700 Subject: [PATCH 06/30] Apply Emma's commits for grabbing binaries from release artifacts --- PCbuild/get_external.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 0061728c2bac1f..97bc88669ed147 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -17,11 +17,7 @@ # Mapping of binary dependency tag to GitHub release asset ID TAG_TO_ASSET_ID = { - 'libffi-3.4.4': 280027073, - 'openssl-bin-3.0.16.2': 280041244, - 'tcltk-8.6.15.0': 280042163, - 'nasm-2.11.06': 280042740, - 'llvm-19.1.7.0': 280052497, + 'llvm-20.1.8.0': 301710576, } From e8395ce88c03f28fba46f97f02f39e66cf2702d4 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 11 Oct 2025 15:57:21 -0700 Subject: [PATCH 07/30] Fix up LLVM via release artifacts --- .github/workflows/jit.yml | 1 - PCbuild/get_external.py | 51 +++++++++++++++++++++++++++------------ PCbuild/get_externals.bat | 2 +- PCbuild/regen.targets | 2 +- Tools/jit/_targets.py | 4 +-- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 3ed6ae7045ca67..8c3ffd6a3a610a 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -102,7 +102,6 @@ jobs: - name: Windows if: runner.os == 'Windows' run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.8 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 97bc88669ed147..38ed584d205fc0 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -55,8 +55,9 @@ def get_with_retries(url, headers): ) -def fetch_zip(commit_hash, zip_dir, *, org='python', verbose): - url = f'https://github.com/{org}/cpython-source-deps/archive/{commit_hash}.zip' +def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose=False): + repo = 'cpython-bin-deps' if binary else 'cpython-source-deps' + url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' reporthook = None if verbose: reporthook = print @@ -69,42 +70,48 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', verbose): return filename -def fetch_release_asset(asset_id, output_path, org): +def fetch_release_asset(asset_id, output_path, org, verbose=False): """Download a GitHub release asset. Release assets need the Content-Type header set to application/octet-stream to download the binary, so we can't use - urlretrieve. Code here is based on urlretrieve + urlretrieve. Code here is based on urlretrieve. """ url = f'https://api.github.com/repos/{org}/cpython-bin-deps/releases/assets/{asset_id}' - rest = get_with_retries(url, - headers={'Accept': 'application/vnd.github+json'}) - json_data = json.loads(rest.read()) - hash_info = json_data['digest'] + if verbose: + print(f'Fetching metadata for asset {asset_id}...') + metadata_resp = get_with_retries(url, + headers={'Accept': 'application/vnd.github+json'}) + json_data = json.loads(metadata_resp.read()) + hash_info = json_data.get('digest') + if not hash_info: + raise RuntimeError(f'Release asset {asset_id} missing digest field in metadata') algorithm, hashsum = hash_info.split(':') if algorithm != 'sha256': raise RuntimeError(f'Unknown hash algorithm {algorithm} for asset {asset_id}') + if verbose: + print(f'Downloading asset {asset_id}...') with contextlib.closing( get_with_retries(url, headers={'Accept': 'application/octet-stream'}) ) as resp: - read = 0 hasher = hashlib.sha256() with open(output_path, 'wb') as fp: while block := resp.read(io.DEFAULT_BUFFER_SIZE): hasher.update(block) - read += len(block) fp.write(block) if hasher.hexdigest() != hashsum: raise RuntimeError('Downloaded content hash did not match!') + if verbose: + print(f'Successfully downloaded and verified {output_path}') -def fetch_release(tag, tarball_dir, *, org='python'): - tarball_dir.mkdir(exist_ok=True) +def fetch_release(tag, tarball_dir, *, org='python', verbose=False): + tarball_dir.mkdir(parents=True, exist_ok=True) asset_id = TAG_TO_ASSET_ID.get(tag) if asset_id is None: raise ValueError(f'Unknown tag for binary dependencies {tag}') output_path = tarball_dir / f'{tag}.tar.xz' - fetch_release_asset(asset_id, output_path, org) + fetch_release_asset(asset_id, output_path, org, verbose) return output_path @@ -137,22 +144,36 @@ def parse_args(): def main(): args = parse_args() - if args.binary: + final_name = args.externals_dir / args.tag + + # Check if the dependency already exists in externals/ directory + # (either already downloaded/extracted, or checked into the git tree) + if final_name.exists(): + if args.verbose: + print(f'{args.tag} already exists at {final_name}, skipping download.') + return + + # Determine download method: release artifacts for large deps (like LLVM), + # otherwise zip download from GitHub branches + if args.tag in TAG_TO_ASSET_ID: tarball_path = fetch_release( args.tag, args.externals_dir / 'tarballs', org=args.organization, + verbose=args.verbose, ) extracted = extract_tarball(args.externals_dir, tarball_path, args.tag) else: + # Use zip download from GitHub branches + # (cpython-bin-deps if --binary, cpython-source-deps otherwise) zip_path = fetch_zip( args.tag, args.externals_dir / 'zips', org=args.organization, + binary=args.binary, verbose=args.verbose, ) extracted = extract_zip(args.externals_dir, zip_path) - final_name = args.externals_dir / args.tag for wait in [1, 2, 3, 5, 8, 0]: try: extracted.replace(final_name) diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 433e45a5e9f992..76dd59a36d672f 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -117,4 +117,4 @@ echo.anything new. echo. exit /b -1 -:end \ No newline at end of file +:end diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index e20627032ccd71..742597f5cb5ebd 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -187,4 +187,4 @@ - \ No newline at end of file + diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 5fede541d1f75f..bbb97d18029535 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -579,7 +579,7 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: host = "i686-pc-windows-msvc" condition = "defined(_M_IX86)" # -Wno-ignored-attributes: __attribute__((preserve_none)) is not supported here. - args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes", "-fno-plt"] + args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes"] optimizer = _optimizers.OptimizerX86 target = _COFF32(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-apple-darwin.*", host): @@ -591,7 +591,7 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-pc-windows-msvc", host): host = "x86_64-pc-windows-msvc" condition = "defined(_M_X64)" - args = ["-fms-runtime-lib=dll", "-fno-plt"] + args = ["-fms-runtime-lib=dll"] optimizer = _optimizers.OptimizerX86 target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): From 01aed67a6fe1a8d6b2fbf5c54886c8ae476fbfd9 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 11 Oct 2025 20:05:11 -0700 Subject: [PATCH 08/30] Remove model flags for x86_64 darwin causing GOT relocation issues --- Tools/jit/_targets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index bbb97d18029535..f6f48bad6219bb 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -565,7 +565,7 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"aarch64-pc-windows-msvc", host): host = "aarch64-pc-windows-msvc" condition = "defined(_M_ARM64)" - args = ["-fms-runtime-lib=dll", "-fplt"] + args = ["-fms-runtime-lib=dll"] optimizer = _optimizers.OptimizerAArch64 target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): @@ -585,7 +585,7 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-apple-darwin.*", host): host = "x86_64-apple-darwin" condition = "defined(__x86_64__) && defined(__APPLE__)" - args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0", "-fno-plt"] + args = ["-fno-plt"] optimizer = _optimizers.OptimizerX86 target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): From 94f1a8984c995ee9d50ee94f091e384e12c95a9f Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 11 Oct 2025 20:07:10 -0700 Subject: [PATCH 09/30] Clean up --- PCbuild/get_external.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 38ed584d205fc0..3caa388a7af8c9 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -70,7 +70,7 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose=False return filename -def fetch_release_asset(asset_id, output_path, org, verbose=False): +def fetch_release_asset(asset_id, output_path, org): """Download a GitHub release asset. Release assets need the Content-Type header set to @@ -78,8 +78,6 @@ def fetch_release_asset(asset_id, output_path, org, verbose=False): urlretrieve. Code here is based on urlretrieve. """ url = f'https://api.github.com/repos/{org}/cpython-bin-deps/releases/assets/{asset_id}' - if verbose: - print(f'Fetching metadata for asset {asset_id}...') metadata_resp = get_with_retries(url, headers={'Accept': 'application/vnd.github+json'}) json_data = json.loads(metadata_resp.read()) @@ -89,8 +87,6 @@ def fetch_release_asset(asset_id, output_path, org, verbose=False): algorithm, hashsum = hash_info.split(':') if algorithm != 'sha256': raise RuntimeError(f'Unknown hash algorithm {algorithm} for asset {asset_id}') - if verbose: - print(f'Downloading asset {asset_id}...') with contextlib.closing( get_with_retries(url, headers={'Accept': 'application/octet-stream'}) ) as resp: @@ -101,8 +97,6 @@ def fetch_release_asset(asset_id, output_path, org, verbose=False): fp.write(block) if hasher.hexdigest() != hashsum: raise RuntimeError('Downloaded content hash did not match!') - if verbose: - print(f'Successfully downloaded and verified {output_path}') def fetch_release(tag, tarball_dir, *, org='python', verbose=False): @@ -111,7 +105,7 @@ def fetch_release(tag, tarball_dir, *, org='python', verbose=False): if asset_id is None: raise ValueError(f'Unknown tag for binary dependencies {tag}') output_path = tarball_dir / f'{tag}.tar.xz' - fetch_release_asset(asset_id, output_path, org, verbose) + fetch_release_asset(asset_id, output_path, org) return output_path From e6450de9b49451eb55560018c819fa0e348f0009 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 13 Oct 2025 10:54:40 +0200 Subject: [PATCH 10/30] Only patch x86_64 GOT relocations when relaxation succeeds --- Python/jit.c | 7 +++---- Tools/jit/_targets.py | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index 01ec9c1fa6e8a9..5509a4ce51ff93 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -400,22 +400,21 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) if (loc8[-2] == 0x8B) { // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] loc8[-2] = 0x8D; - value = relaxed; + patch_32r(location, relaxed); } else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { // call qword ptr [rip + AAA] -> nop; call XXX loc8[-2] = 0x90; loc8[-1] = 0xE8; - value = relaxed; + patch_32r(location, relaxed); } else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { // jmp qword ptr [rip + AAA] -> nop; jmp XXX loc8[-2] = 0x90; loc8[-1] = 0xE9; - value = relaxed; + patch_32r(location, relaxed); } } - patch_32r(location, value); } void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index f6f48bad6219bb..51a3c6c63bd38e 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -585,7 +585,6 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-apple-darwin.*", host): host = "x86_64-apple-darwin" condition = "defined(__x86_64__) && defined(__APPLE__)" - args = ["-fno-plt"] optimizer = _optimizers.OptimizerX86 target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): From 1adf827821bc460d212f14ca8254508042db3dc3 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 13 Oct 2025 11:54:55 +0200 Subject: [PATCH 11/30] Revert "Only patch x86_64 GOT relocations when relaxation succeeds" This reverts commit e6450de9b49451eb55560018c819fa0e348f0009. --- Python/jit.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index 5509a4ce51ff93..01ec9c1fa6e8a9 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -400,21 +400,22 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) if (loc8[-2] == 0x8B) { // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] loc8[-2] = 0x8D; - patch_32r(location, relaxed); + value = relaxed; } else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { // call qword ptr [rip + AAA] -> nop; call XXX loc8[-2] = 0x90; loc8[-1] = 0xE8; - patch_32r(location, relaxed); + value = relaxed; } else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { // jmp qword ptr [rip + AAA] -> nop; jmp XXX loc8[-2] = 0x90; loc8[-1] = 0xE9; - patch_32r(location, relaxed); + value = relaxed; } } + patch_32r(location, value); } void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); From b9bfacf5c25a1c41b52a519a9bd150510ca5b463 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 14 Oct 2025 08:47:57 +0200 Subject: [PATCH 12/30] mcmodel=large --- Tools/jit/_targets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 51a3c6c63bd38e..c8e1e7aa4398f9 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -585,8 +585,9 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-apple-darwin.*", host): host = "x86_64-apple-darwin" condition = "defined(__x86_64__) && defined(__APPLE__)" + args = ["-mcmodel=large"] optimizer = _optimizers.OptimizerX86 - target = _MachO(host, condition, optimizer=optimizer) + target = _MachO(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): host = "x86_64-pc-windows-msvc" condition = "defined(_M_X64)" From 57c44ee4e82869552478c8d0339dda9400cb8492 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 16 Oct 2025 18:33:01 +0200 Subject: [PATCH 13/30] Add macro to handle debug --- Python/jit.c | 10 ++++++++++ Tools/jit/_targets.py | 3 +-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index 01ec9c1fa6e8a9..46ec52fedfd490 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -216,7 +216,17 @@ patch_32r(unsigned char *location, uint64_t value) value -= (uintptr_t)location; // Check that we're not out of range of 32 signed bits: assert((int64_t)value >= -(1LL << 31)); +#if defined(__APPLE__) && defined(Py_DEBUG) + // On macOS debug builds with LLVM 20, external symbols may be out of range. + // This is handled by GOT indirection, so we allow the truncation here. + // Release builds (-O3) don't have this issue due to better optimization. + if ((int64_t)value >= (1LL << 31)) { + // Truncate to 32-bit; the GOT entry will handle the indirection + value = (uint32_t)value; + } +#else assert((int64_t)value < (1LL << 31)); +#endif *loc32 = (uint32_t)value; } diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index c8e1e7aa4398f9..51a3c6c63bd38e 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -585,9 +585,8 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-apple-darwin.*", host): host = "x86_64-apple-darwin" condition = "defined(__x86_64__) && defined(__APPLE__)" - args = ["-mcmodel=large"] optimizer = _optimizers.OptimizerX86 - target = _MachO(host, condition, args=args, optimizer=optimizer) + target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): host = "x86_64-pc-windows-msvc" condition = "defined(_M_X64)" From 081ee86b355f2d71feeff4e5534294e632b56d4d Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 16 Oct 2025 20:26:04 +0200 Subject: [PATCH 14/30] fno-pic --- Python/jit.c | 6 ++---- Tools/jit/_targets.py | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index 46ec52fedfd490..fe3a832b339632 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -217,11 +217,9 @@ patch_32r(unsigned char *location, uint64_t value) // Check that we're not out of range of 32 signed bits: assert((int64_t)value >= -(1LL << 31)); #if defined(__APPLE__) && defined(Py_DEBUG) - // On macOS debug builds with LLVM 20, external symbols may be out of range. - // This is handled by GOT indirection, so we allow the truncation here. - // Release builds (-O3) don't have this issue due to better optimization. + // LLVM 20 on macOS debug builds: GOT entries may exceed ±2GB PC-relative + // range. Truncation is safe as the target is a GOT trampoline. if ((int64_t)value >= (1LL << 31)) { - // Truncate to 32-bit; the GOT entry will handle the indirection value = (uint32_t)value; } #else diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 51a3c6c63bd38e..5d4752b51d6022 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -585,8 +585,9 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-apple-darwin.*", host): host = "x86_64-apple-darwin" condition = "defined(__x86_64__) && defined(__APPLE__)" + args = ["-fno-pic"] optimizer = _optimizers.OptimizerX86 - target = _MachO(host, condition, optimizer=optimizer) + target = _MachO(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): host = "x86_64-pc-windows-msvc" condition = "defined(_M_X64)" From 0b773f9ba0687dbcaac20fa44b87495000898726 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 16 Oct 2025 20:29:07 +0200 Subject: [PATCH 15/30] remove fno-pic --- Tools/jit/_targets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 5d4752b51d6022..51a3c6c63bd38e 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -585,9 +585,8 @@ def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: elif re.fullmatch(r"x86_64-apple-darwin.*", host): host = "x86_64-apple-darwin" condition = "defined(__x86_64__) && defined(__APPLE__)" - args = ["-fno-pic"] optimizer = _optimizers.OptimizerX86 - target = _MachO(host, condition, args=args, optimizer=optimizer) + target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): host = "x86_64-pc-windows-msvc" condition = "defined(_M_X64)" From c78af6fbe81530da4582ee8732231203b9845f17 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 18 Oct 2025 12:55:26 +0200 Subject: [PATCH 16/30] remove hack --- Python/jit.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index fe3a832b339632..99ea41dcd34095 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -216,16 +216,13 @@ patch_32r(unsigned char *location, uint64_t value) value -= (uintptr_t)location; // Check that we're not out of range of 32 signed bits: assert((int64_t)value >= -(1LL << 31)); -#if defined(__APPLE__) && defined(Py_DEBUG) - // LLVM 20 on macOS debug builds: GOT entries may exceed ±2GB PC-relative - // range. Truncation is safe as the target is a GOT trampoline. + // assert((int64_t)value < (1LL << 31)); if ((int64_t)value >= (1LL << 31)) { - value = (uint32_t)value; + __builtin_debugtrap(); } -#else - assert((int64_t)value < (1LL << 31)); -#endif + *loc32 = (uint32_t)value; + } // 64-bit absolute address. From d715cf23743d20e14de021e1fb8ba9f889fa8160 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 18 Oct 2025 18:34:28 +0200 Subject: [PATCH 17/30] Trampoline attempt --- Python/jit.c | 52 ++++++++++++++++++++++++++++++++++++++---- Tools/jit/_stencils.py | 17 ++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index 99ea41dcd34095..8a7920b052c862 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -216,11 +216,7 @@ patch_32r(unsigned char *location, uint64_t value) value -= (uintptr_t)location; // Check that we're not out of range of 32 signed bits: assert((int64_t)value >= -(1LL << 31)); - // assert((int64_t)value < (1LL << 31)); - if ((int64_t)value >= (1LL << 31)) { - __builtin_debugtrap(); - } - + assert((int64_t)value < (1LL << 31)); *loc32 = (uint32_t)value; } @@ -424,12 +420,17 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) } void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); +void patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state); #include "jit_stencils.h" #if defined(__aarch64__) || defined(_M_ARM64) #define TRAMPOLINE_SIZE 16 #define DATA_ALIGN 8 +#elif defined(__x86_64__) && defined(__APPLE__) + // x86_64 trampolines: jmp *(%rip); .quad address (6 bytes + 8 bytes = 14 bytes) + #define TRAMPOLINE_SIZE 16 // Round up to 16 for alignment + #define DATA_ALIGN 16 #else #define TRAMPOLINE_SIZE 0 #define DATA_ALIGN 1 @@ -481,6 +482,47 @@ patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) patch_aarch64_26r(location, (uintptr_t)p); } +// Generate and patch x86_64 trampolines. +void +patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state) +{ + uint64_t value = (uintptr_t)symbols_map[ordinal]; + int64_t range = (int64_t)value - 4 - (int64_t)location; + + // If we are in range of 32 signed bits, patch directly + if (range >= -(1LL << 31) && range < (1LL << 31)) { + patch_32r(location, value - 4); + return; + } + + // Out of range - need a trampoline + const uint32_t symbol_mask = 1 << (ordinal % 32); + const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; + assert(symbol_mask & trampoline_mask); + + // Count the number of set bits in the trampoline mask lower than ordinal + int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); + for (int i = 0; i < ordinal / 32; i++) { + index += _Py_popcount32(state->trampolines.mask[i]); + } + + unsigned char *trampoline = state->trampolines.mem + index * TRAMPOLINE_SIZE; + assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + + /* Generate the trampoline (14 bytes, padded to 16): + 0: ff 25 00 00 00 00 jmp *(%rip) # Jump to address at offset 6 + 6: XX XX XX XX XX XX XX XX .quad value (64-bit address) + */ + trampoline[0] = 0xFF; // jmp opcode + trampoline[1] = 0x25; // ModRM byte for jmp *disp32(%rip) + // Offset 0: the address is right after this instruction (at offset 6) + *(uint32_t *)(trampoline + 2) = 0; + *(uint64_t *)(trampoline + 6) = value; + + // Patch the call site to call the trampoline instead + patch_32r(location, (uintptr_t)trampoline - 4); +} + static void combine_symbol_mask(const symbol_mask src, symbol_mask dest) { diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 14606b036db519..a52fd867e78769 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -239,6 +239,23 @@ def process_relocations(self, known_symbols: dict[str, int]) -> None: self._trampolines.add(ordinal) hole.addend = ordinal hole.symbol = None + # x86_64 Darwin trampolines for external symbols + elif ( + hole.kind == "X86_64_RELOC_BRANCH" + and hole.value is HoleValue.ZERO + and hole.symbol not in self.symbols + ): + hole.func = "patch_x86_64_trampoline" + hole.need_state = True + assert hole.symbol is not None + if hole.symbol in known_symbols: + ordinal = known_symbols[hole.symbol] + else: + ordinal = len(known_symbols) + known_symbols[hole.symbol] = ordinal + self._trampolines.add(ordinal) + hole.addend = ordinal + hole.symbol = None self.data.pad(8) for stencil in [self.code, self.data]: for hole in stencil.holes: From 38e11b9f779e631b1b646f2a4cfbe7a6a34bf168 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sun, 19 Oct 2025 11:25:36 +0200 Subject: [PATCH 18/30] Touch up and add better comments --- Python/jit.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index 8a7920b052c862..15268414be784b 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -428,8 +428,9 @@ void patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *st #define TRAMPOLINE_SIZE 16 #define DATA_ALIGN 8 #elif defined(__x86_64__) && defined(__APPLE__) - // x86_64 trampolines: jmp *(%rip); .quad address (6 bytes + 8 bytes = 14 bytes) - #define TRAMPOLINE_SIZE 16 // Round up to 16 for alignment + // LLVM 20 on macOS x86_64 debug builds: GOT entries may exceed ±2GB PC-relative + // range. Trampolines provide indirect jumps using 64-bit absolute addresses. + #define TRAMPOLINE_SIZE 16 // 14 bytes + 2 bytes padding for alignment #define DATA_ALIGN 16 #else #define TRAMPOLINE_SIZE 0 @@ -489,7 +490,7 @@ patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state) uint64_t value = (uintptr_t)symbols_map[ordinal]; int64_t range = (int64_t)value - 4 - (int64_t)location; - // If we are in range of 32 signed bits, patch directly + // If we are in range of 32 signed bits, we can patch directly if (range >= -(1LL << 31) && range < (1LL << 31)) { patch_32r(location, value - 4); return; @@ -510,12 +511,13 @@ patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state) assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); /* Generate the trampoline (14 bytes, padded to 16): - 0: ff 25 00 00 00 00 jmp *(%rip) # Jump to address at offset 6 - 6: XX XX XX XX XX XX XX XX .quad value (64-bit address) + 0: ff 25 00 00 00 00 jmp *(%rip) + 6: XX XX XX XX XX XX XX XX (64-bit target address) + + Reference: https://wiki.osdev.org/X86-64_Instruction_Encoding#FF (JMP r/m64) */ - trampoline[0] = 0xFF; // jmp opcode - trampoline[1] = 0x25; // ModRM byte for jmp *disp32(%rip) - // Offset 0: the address is right after this instruction (at offset 6) + trampoline[0] = 0xFF; + trampoline[1] = 0x25; *(uint32_t *)(trampoline + 2) = 0; *(uint64_t *)(trampoline + 6) = value; From ca956526f7b30dc02d3734df1810b66a1ee446e2 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sun, 19 Oct 2025 11:47:02 +0200 Subject: [PATCH 19/30] More clean up --- .github/workflows/jit.yml | 1 + Python/jit.c | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 8c3ffd6a3a610a..151b17e8442582 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -99,6 +99,7 @@ jobs: with: python-version: '3.11' + # PCbuild downloads LLVM automatically: - name: Windows if: runner.os == 'Windows' run: | diff --git a/Python/jit.c b/Python/jit.c index a4a274fead697c..c7df3ebbacc05d 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -226,7 +226,6 @@ patch_32r(unsigned char *location, uint64_t value) assert((int64_t)value < (1LL << 31)); uint32_t final_value = (uint32_t)value; memcpy(location, &final_value, sizeof(final_value)); - } // 64-bit absolute address. @@ -454,7 +453,7 @@ void patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *st #define DATA_ALIGN 8 #elif defined(__x86_64__) && defined(__APPLE__) // LLVM 20 on macOS x86_64 debug builds: GOT entries may exceed ±2GB PC-relative - // range. Trampolines provide indirect jumps using 64-bit absolute addresses. + // range. #define TRAMPOLINE_SIZE 16 // 14 bytes + 2 bytes padding for alignment #define DATA_ALIGN 16 #else From 47153a5d3ddac8e362d7f9244c520bf16de7cbea Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 19 Oct 2025 10:32:30 +0000 Subject: [PATCH 20/30] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst new file mode 100644 index 00000000000000..fffc264a8650e0 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 20 at build time. From 7b2df52f8edf285ad2455d04a3536517176221f3 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 20 Oct 2025 20:28:29 -0700 Subject: [PATCH 21/30] Redupe LLVM version reference from jit.yml --- .github/workflows/jit.yml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 151b17e8442582..6afd53f65f6f41 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -31,6 +31,7 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 20 jobs: interpreter: @@ -67,8 +68,6 @@ jobs: debug: - true - false - llvm: - - 20 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -110,7 +109,7 @@ jobs: if: runner.os == 'macOS' run: | brew update - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} export SDKROOT="$(xcrun --show-sdk-path)" # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to # make sure we don't break downstream distributors (like uv): @@ -123,8 +122,8 @@ jobs: - name: Linux if: runner.os == 'Linux' run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -134,11 +133,6 @@ jobs: needs: interpreter runs-on: ubuntu-24.04 timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - llvm: - - 20 steps: - uses: actions/checkout@v4 with: @@ -148,8 +142,8 @@ jobs: python-version: '3.11' - name: Build with JIT enabled and GIL disabled run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" ./configure --enable-experimental-jit --with-pydebug --disable-gil make all --jobs 4 - name: Run tests @@ -176,8 +170,8 @@ jobs: python-version: '3.11' - name: Build with JIT run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" ./configure --enable-experimental-jit --with-pydebug make all --jobs 4 - name: Run tests without optimizations From 450dd092876cc5482b74137988f465231211cebf Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 20 Oct 2025 20:48:29 -0700 Subject: [PATCH 22/30] Simplify fetching LLVM from bin-deps --- PCbuild/get_external.py | 59 ++++++--------------------------------- PCbuild/get_externals.bat | 4 ++- 2 files changed, 11 insertions(+), 52 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 3caa388a7af8c9..282f6a31fc0d90 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -1,10 +1,6 @@ #!/usr/bin/env python3 import argparse -import contextlib -import hashlib -import io -import json import os import pathlib import shutil @@ -15,10 +11,6 @@ import zipfile -# Mapping of binary dependency tag to GitHub release asset ID -TAG_TO_ASSET_ID = { - 'llvm-20.1.8.0': 301710576, -} def request_with_retry(request_func, *args, max_retries=7, @@ -46,15 +38,6 @@ def retrieve_with_retries(download_location, output_path, reporthook): ) -def get_with_retries(url, headers): - req = urllib.request.Request(url=url, headers=headers, method='GET') - return request_with_retry( - urllib.request.urlopen, - req, - err_msg=f'Request to {url} failed.' - ) - - def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose=False): repo = 'cpython-bin-deps' if binary else 'cpython-source-deps' url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' @@ -70,42 +53,14 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose=False return filename -def fetch_release_asset(asset_id, output_path, org): - """Download a GitHub release asset. - - Release assets need the Content-Type header set to - application/octet-stream to download the binary, so we can't use - urlretrieve. Code here is based on urlretrieve. - """ - url = f'https://api.github.com/repos/{org}/cpython-bin-deps/releases/assets/{asset_id}' - metadata_resp = get_with_retries(url, - headers={'Accept': 'application/vnd.github+json'}) - json_data = json.loads(metadata_resp.read()) - hash_info = json_data.get('digest') - if not hash_info: - raise RuntimeError(f'Release asset {asset_id} missing digest field in metadata') - algorithm, hashsum = hash_info.split(':') - if algorithm != 'sha256': - raise RuntimeError(f'Unknown hash algorithm {algorithm} for asset {asset_id}') - with contextlib.closing( - get_with_retries(url, headers={'Accept': 'application/octet-stream'}) - ) as resp: - hasher = hashlib.sha256() - with open(output_path, 'wb') as fp: - while block := resp.read(io.DEFAULT_BUFFER_SIZE): - hasher.update(block) - fp.write(block) - if hasher.hexdigest() != hashsum: - raise RuntimeError('Downloaded content hash did not match!') - - def fetch_release(tag, tarball_dir, *, org='python', verbose=False): + url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{tag}.tar.xz' + reporthook = None + if verbose: + reporthook = print tarball_dir.mkdir(parents=True, exist_ok=True) - asset_id = TAG_TO_ASSET_ID.get(tag) - if asset_id is None: - raise ValueError(f'Unknown tag for binary dependencies {tag}') output_path = tarball_dir / f'{tag}.tar.xz' - fetch_release_asset(asset_id, output_path, org) + retrieve_with_retries(url, output_path, reporthook) return output_path @@ -126,6 +81,8 @@ def parse_args(): p.add_argument('-v', '--verbose', action='store_true') p.add_argument('-b', '--binary', action='store_true', help='Is the dependency in the binary repo?') + p.add_argument('-r', '--release', action='store_true', + help='Download from GitHub release assets instead of branch') p.add_argument('-O', '--organization', help='Organization owning the deps repos', default='python') p.add_argument('-e', '--externals-dir', type=pathlib.Path, @@ -149,7 +106,7 @@ def main(): # Determine download method: release artifacts for large deps (like LLVM), # otherwise zip download from GitHub branches - if args.tag in TAG_TO_ASSET_ID: + if args.release: tarball_path = fetch_release( args.tag, args.externals_dir / 'tarballs', diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 76dd59a36d672f..807905f6085f8f 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -92,7 +92,9 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - %PYTHON% -E "%PCBUILD%\get_external.py" -b -O %ORG% -e "%EXTERNALS_DIR%" %%b + set "extra_flags=-b" + if "%%b"=="llvm-20.1.8.0" set "extra_flags=-r" + %PYTHON% -E "%PCBUILD%\get_external.py" %extra_flags% -O %ORG% -e "%EXTERNALS_DIR%" %%b ) ) From 5da33490e3bdc48e203852f32a6e51f9aed14879 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 20 Oct 2025 20:50:52 -0700 Subject: [PATCH 23/30] Remove duplicate 20 reference in jit.yml --- .github/workflows/jit.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 6afd53f65f6f41..0ef1dcbd7d0565 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -156,11 +156,6 @@ jobs: needs: interpreter runs-on: ubuntu-24.04 timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - llvm: - - 20 steps: - uses: actions/checkout@v4 with: From 620cd4f6173e0bab4637e876d1187dc1eb3424ec Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 20 Oct 2025 21:10:04 -0700 Subject: [PATCH 24/30] Fix flags --- PCbuild/get_external.py | 2 -- PCbuild/get_externals.bat | 8 +++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 282f6a31fc0d90..ea7287d0110ae9 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -11,8 +11,6 @@ import zipfile - - def request_with_retry(request_func, *args, max_retries=7, err_msg='Request failed.', **kwargs): """Make a request using request_func with exponential backoff""" diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 807905f6085f8f..c72b27f3b88dc3 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -92,9 +92,11 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - set "extra_flags=-b" - if "%%b"=="llvm-20.1.8.0" set "extra_flags=-r" - %PYTHON% -E "%PCBUILD%\get_external.py" %extra_flags% -O %ORG% -e "%EXTERNALS_DIR%" %%b + if "%%b"=="llvm-20.1.8.0" ( + %PYTHON% -E "%PCBUILD%\get_external.py" -r -O %ORG% -e "%EXTERNALS_DIR%" %%b + ) else ( + %PYTHON% -E "%PCBUILD%\get_external.py" -b -O %ORG% -e "%EXTERNALS_DIR%" %%b + ) ) ) From d1a68aba6f3dbcf91c3f4a2217686104e9c49a88 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 21 Oct 2025 06:30:05 -0700 Subject: [PATCH 25/30] Reduce DATA_ALIGN to 8 bytes --- Python/jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/jit.c b/Python/jit.c index c7df3ebbacc05d..f557904a582f98 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -455,7 +455,7 @@ void patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *st // LLVM 20 on macOS x86_64 debug builds: GOT entries may exceed ±2GB PC-relative // range. #define TRAMPOLINE_SIZE 16 // 14 bytes + 2 bytes padding for alignment - #define DATA_ALIGN 16 + #define DATA_ALIGN 8 #else #define TRAMPOLINE_SIZE 0 #define DATA_ALIGN 1 From 4b35c2f6af128525be94ffd44c477eaa49fbc029 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 22 Oct 2025 13:33:17 -0700 Subject: [PATCH 26/30] Update Devcontainer readme --- Tools/jit/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/jit/README.md b/Tools/jit/README.md index c4b29877967b8e..d83b09aab59f8c 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -54,7 +54,7 @@ choco install llvm --version=20.1.8 ### Dev Containers If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no -need to install LLVM as the Fedora 41 base image includes LLVM 19 out of the box. +need to install LLVM as the Fedora 42 base image includes LLVM 20 out of the box. ## Building From 618a4a6e3a2a3f84fb4083f9d81f5a539bc1dd2c Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 30 Oct 2025 21:15:10 -0700 Subject: [PATCH 27/30] Revert CI simplication and address comments for memcpy --- .github/workflows/jit.yml | 27 +++++++++++++++++++-------- Python/jit.c | 4 ++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 0ef1dcbd7d0565..151b17e8442582 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -31,7 +31,6 @@ concurrency: env: FORCE_COLOR: 1 - LLVM_VERSION: 20 jobs: interpreter: @@ -68,6 +67,8 @@ jobs: debug: - true - false + llvm: + - 20 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -109,7 +110,7 @@ jobs: if: runner.os == 'macOS' run: | brew update - brew install llvm@${{ env.LLVM_VERSION }} + brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to # make sure we don't break downstream distributors (like uv): @@ -122,8 +123,8 @@ jobs: - name: Linux if: runner.os == 'Linux' run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} - export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -133,6 +134,11 @@ jobs: needs: interpreter runs-on: ubuntu-24.04 timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 20 steps: - uses: actions/checkout@v4 with: @@ -142,8 +148,8 @@ jobs: python-version: '3.11' - name: Build with JIT enabled and GIL disabled run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} - export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit --with-pydebug --disable-gil make all --jobs 4 - name: Run tests @@ -156,6 +162,11 @@ jobs: needs: interpreter runs-on: ubuntu-24.04 timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 20 steps: - uses: actions/checkout@v4 with: @@ -165,8 +176,8 @@ jobs: python-version: '3.11' - name: Build with JIT run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} - export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit --with-pydebug make all --jobs 4 - name: Run tests without optimizations diff --git a/Python/jit.c b/Python/jit.c index f557904a582f98..8a40d415123c83 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -542,8 +542,8 @@ patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state) */ trampoline[0] = 0xFF; trampoline[1] = 0x25; - *(uint32_t *)(trampoline + 2) = 0; - *(uint64_t *)(trampoline + 6) = value; + memset(trampoline + 2, 0, 4); + memcpy(trampoline + 6, &value, 8); // Patch the call site to call the trampoline instead patch_32r(location, (uintptr_t)trampoline - 4); From 2e9ba9272e65f86c4bffd4ce14c2a345d41fe743 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 30 Oct 2025 21:30:58 -0700 Subject: [PATCH 28/30] Address comments about get_externals and trampoline helper --- PCbuild/get_external.py | 25 +++++++------------- PCbuild/get_externals.bat | 4 ++-- Python/jit.c | 48 ++++++++++++++++++--------------------- 3 files changed, 32 insertions(+), 45 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index ea7287d0110ae9..0d17b70667b563 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -11,31 +11,22 @@ import zipfile -def request_with_retry(request_func, *args, max_retries=7, - err_msg='Request failed.', **kwargs): - """Make a request using request_func with exponential backoff""" +def retrieve_with_retries(download_location, output_path, reporthook, max_retries=7): + """Download a file with retries.""" for attempt in range(max_retries + 1): try: - resp = request_func(*args, **kwargs) + resp = urllib.request.urlretrieve( + download_location, + output_path, + reporthook, + ) except (urllib.error.URLError, ConnectionError) as ex: if attempt == max_retries: - raise OSError(err_msg) from ex + raise OSError(f'Download from {download_location} failed.') from ex time.sleep(2.25**attempt) else: return resp - -def retrieve_with_retries(download_location, output_path, reporthook): - """Download a file with retries.""" - return request_with_retry( - urllib.request.urlretrieve, - download_location, - output_path, - reporthook, - err_msg=f'Download from {download_location} failed.', - ) - - def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose=False): repo = 'cpython-bin-deps' if binary else 'cpython-source-deps' url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index c72b27f3b88dc3..319024e0f50f46 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -93,9 +93,9 @@ for %%b in (%binaries%) do ( ) else ( echo.Fetching %%b... if "%%b"=="llvm-20.1.8.0" ( - %PYTHON% -E "%PCBUILD%\get_external.py" -r -O %ORG% -e "%EXTERNALS_DIR%" %%b + %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b ) else ( - %PYTHON% -E "%PCBUILD%\get_external.py" -b -O %ORG% -e "%EXTERNALS_DIR%" %%b + %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b ) ) ) diff --git a/Python/jit.c b/Python/jit.c index 8a40d415123c83..279e1ce6a0d2e5 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -461,6 +461,25 @@ void patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *st #define DATA_ALIGN 1 #endif +// Get the trampoline memory location for a given symbol ordinal. +static unsigned char * +get_trampoline_slot(int ordinal, jit_state *state) +{ + const uint32_t symbol_mask = 1 << (ordinal % 32); + const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; + assert(symbol_mask & trampoline_mask); + + // Count the number of set bits in the trampoline mask lower than ordinal + int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); + for (int i = 0; i < ordinal / 32; i++) { + index += _Py_popcount32(state->trampolines.mask[i]); + } + + unsigned char *trampoline = state->trampolines.mem + index * TRAMPOLINE_SIZE; + assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + return trampoline; +} + // Generate and patch AArch64 trampolines. The symbols to jump to are stored // in the jit_stencils.h in the symbols_map. void @@ -477,20 +496,8 @@ patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) return; } - // Masking is done modulo 32 as the mask is stored as an array of uint32_t - const uint32_t symbol_mask = 1 << (ordinal % 32); - const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; - assert(symbol_mask & trampoline_mask); - - // Count the number of set bits in the trampoline mask lower than ordinal, - // this gives the index into the array of trampolines. - int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); - for (int i = 0; i < ordinal / 32; i++) { - index += _Py_popcount32(state->trampolines.mask[i]); - } - - uint32_t *p = (uint32_t*)(state->trampolines.mem + index * TRAMPOLINE_SIZE); - assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + // Out of range - need a trampoline + uint32_t *p = (uint32_t *)get_trampoline_slot(ordinal, state); /* Generate the trampoline @@ -521,18 +528,7 @@ patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state) } // Out of range - need a trampoline - const uint32_t symbol_mask = 1 << (ordinal % 32); - const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; - assert(symbol_mask & trampoline_mask); - - // Count the number of set bits in the trampoline mask lower than ordinal - int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); - for (int i = 0; i < ordinal / 32; i++) { - index += _Py_popcount32(state->trampolines.mask[i]); - } - - unsigned char *trampoline = state->trampolines.mem + index * TRAMPOLINE_SIZE; - assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + unsigned char *trampoline = get_trampoline_slot(ordinal, state); /* Generate the trampoline (14 bytes, padded to 16): 0: ff 25 00 00 00 00 jmp *(%rip) From 795f466ffe146028e93cff4de24717386a14565a Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 30 Oct 2025 21:42:25 -0700 Subject: [PATCH 29/30] Restore get_externals --- PCbuild/get_external.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 0d17b70667b563..cf83eaa6b37687 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -11,14 +11,15 @@ import zipfile -def retrieve_with_retries(download_location, output_path, reporthook, max_retries=7): - """Download a file with retries.""" +def retrieve_with_retries(download_location, output_path, reporthook, + max_retries=7): + """Download a file with exponential backoff retry and save to disk.""" for attempt in range(max_retries + 1): try: resp = urllib.request.urlretrieve( download_location, output_path, - reporthook, + reporthook = reporthook, ) except (urllib.error.URLError, ConnectionError) as ex: if attempt == max_retries: @@ -27,7 +28,7 @@ def retrieve_with_retries(download_location, output_path, reporthook, max_retrie else: return resp -def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose=False): +def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): repo = 'cpython-bin-deps' if binary else 'cpython-source-deps' url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' reporthook = None From b880f26cd7f8f713423d447cc6df54c57bd7770f Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 30 Oct 2025 21:44:03 -0700 Subject: [PATCH 30/30] Fix spacing --- PCbuild/get_external.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index cf83eaa6b37687..07970624e8647e 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -19,7 +19,7 @@ def retrieve_with_retries(download_location, output_path, reporthook, resp = urllib.request.urlretrieve( download_location, output_path, - reporthook = reporthook, + reporthook=reporthook, ) except (urllib.error.URLError, ConnectionError) as ex: if attempt == max_retries: