diff --git a/Makefile.pre.in b/Makefile.pre.in index a27a9709330772..0db27b6c46fa68 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3373,7 +3373,7 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h # Module dependencies and platform-specific files cpython-sys: Modules/cpython-sys/Cargo.toml Modules/cpython-sys/build.rs Modules/cpython-sys/wrapper.h Modules/cpython-sys/parser.h - cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE) + CARGO_TARGET_DIR=$(abs_builddir)/target PYTHON_BUILD_DIR=$(abs_builddir) cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE) --manifest-path $(srcdir)/Cargo.toml # force rebuild when header file or module build flavor (static/shared) is changed MODULE_DEPS_STATIC=Modules/config.c diff --git a/Modules/cpython-sys/build.rs b/Modules/cpython-sys/build.rs index 71b73b477fd34d..680066c4fd5e9d 100644 --- a/Modules/cpython-sys/build.rs +++ b/Modules/cpython-sys/build.rs @@ -2,21 +2,55 @@ use std::env; use std::path::{Path, PathBuf}; fn main() { - let curdir = std::env::current_dir().unwrap(); - let srcdir = curdir.parent().and_then(Path::parent).unwrap(); + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let srcdir = manifest_dir + .parent() + .and_then(Path::parent) + .expect("expected Modules/cpython-sys to live under the source tree"); let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); - generate_c_api_bindings(srcdir, &out_path.as_path()); + let builddir = env::var("PYTHON_BUILD_DIR").ok(); + if gil_disabled(&srcdir, builddir.as_deref()) { + println!("cargo:rustc-cfg=py_gil_disabled"); + } + generate_c_api_bindings(srcdir, builddir.as_deref(), &out_path.as_path()); // TODO(emmatyping): generate bindings to the internal parser API // The parser includes things slightly differently, so we should generate // it's bindings independently //generate_parser_bindings(srcdir, &out_path.as_path()); } -fn generate_c_api_bindings(srcdir: &Path, out_path: &Path) { - let bindings = bindgen::Builder::default() - .header("wrapper.h") - .clang_arg(format!("-I{}", srcdir.as_os_str().to_str().unwrap())) - .clang_arg(format!("-I{}/Include", srcdir.as_os_str().to_str().unwrap())) +fn gil_disabled(srcdir: &Path, builddir: Option<&str>) -> bool { + let mut candidates = Vec::new(); + if let Some(build) = builddir { + candidates.push(PathBuf::from(build)); + } + candidates.push(srcdir.to_path_buf()); + for base in candidates { + let path = base.join("pyconfig.h"); + if let Ok(contents) = std::fs::read_to_string(&path) { + if contents.contains("Py_GIL_DISABLED 1") { + return true; + } + } + } + false +} + +fn generate_c_api_bindings(srcdir: &Path, builddir: Option<&str>, out_path: &Path) { + let mut builder = bindgen::Builder::default().header("wrapper.h"); + + // Always search the source dir and the public headers. + let mut include_dirs = vec![srcdir.to_path_buf(), srcdir.join("Include")]; + // Include the build directory if provided; out-of-tree builds place + // the generated pyconfig.h there. + if let Some(build) = builddir { + include_dirs.push(PathBuf::from(build)); + } + for dir in include_dirs { + builder = builder.clang_arg(format!("-I{}", dir.display())); + } + + let bindings = builder .allowlist_function("_?Py.*") .allowlist_type("_?Py.*") .allowlist_var("_?Py.*") diff --git a/Modules/cpython-sys/src/lib.rs b/Modules/cpython-sys/src/lib.rs index d3e21662a68472..ed1d68eedd600a 100644 --- a/Modules/cpython-sys/src/lib.rs +++ b/Modules/cpython-sys/src/lib.rs @@ -111,6 +111,14 @@ impl PyMethodDef { unsafe impl Sync for PyMethodDef {} unsafe impl Send for PyMethodDef {} +#[cfg(py_gil_disabled)] +pub const PyObject_HEAD_INIT: PyObject = { + let mut obj: PyObject = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; + obj.ob_flags = _Py_STATICALLY_ALLOCATED_FLAG as _; + obj +}; + +#[cfg(not(py_gil_disabled))] pub const PyObject_HEAD_INIT: PyObject = PyObject { __bindgen_anon_1: _object__bindgen_ty_1 { ob_refcnt_full: _Py_STATIC_IMMORTAL_INITIAL_REFCNT as i64, diff --git a/Modules/makesetup b/Modules/makesetup index b5fb40994a010b..0368f1d017ac87 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -287,7 +287,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | libs= # depends on the headers through cpython-sys rule="$objs: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS)" - rule="$rule; cargo build --lib --locked --package ${mods} --profile \$(CARGO_PROFILE)" + rule="$rule; CARGO_TARGET_DIR=\$(abs_builddir)/target PYTHON_BUILD_DIR=\$(abs_builddir) cargo build --lib --locked --package ${mods} --profile \$(CARGO_PROFILE) --manifest-path \$(srcdir)/Cargo.toml" echo "$rule" >>$rulesf for mod in $mods do diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 8937e666bbbdd5..54cbc7106069bc 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -10,6 +10,7 @@ static const char* _Py_stdlib_module_names[] = { "_ast", "_ast_unparse", "_asyncio", +"_base64", "_bisect", "_blake2", "_bz2", diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index bda72539640611..646f31b49761c1 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -3,6 +3,7 @@ from __future__ import annotations import _imp +import os import os.path import sys import sysconfig @@ -14,6 +15,7 @@ SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) STDLIB_PATH = os.path.join(SRC_DIR, 'Lib') +MODULES_PATH = os.path.join(SRC_DIR, 'Modules') IGNORE = { '__init__', @@ -84,6 +86,19 @@ def list_modules_setup_extensions(names: set[str]) -> None: names.update(checker.list_module_names(all=True)) +def list_rust_modules(names: set[str]) -> None: + if not os.path.isdir(MODULES_PATH): + return + for entry in os.scandir(MODULES_PATH): + if not entry.is_dir(): + continue + if entry.name == "cpython-sys": + continue + cargo_toml = os.path.join(entry.path, "Cargo.toml") + if os.path.isfile(cargo_toml): + names.add(entry.name) + + # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). # Use the "./Programs/_testembed list_frozen" command. def list_frozen(names: set[str]) -> None: @@ -109,6 +124,7 @@ def list_modules() -> set[str]: list_builtin_modules(names) list_modules_setup_extensions(names) + list_rust_modules(names) list_packages(names) list_python_modules(names) list_frozen(names)