Make it work on more platforms

Makefile.pre.in

@@ -3373,7 +3373,7 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
 # Module dependencies and platform-specific files
 
 cpython-sys: Modules/cpython-sys/Cargo.toml Modules/cpython-sys/build.rs Modules/cpython-sys/wrapper.h Modules/cpython-sys/parser.h
-	cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE)
+	CARGO_TARGET_DIR=$(abs_builddir)/target PYTHON_BUILD_DIR=$(abs_builddir) cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE) --manifest-path $(srcdir)/Cargo.toml
 
 # force rebuild when header file or module build flavor (static/shared) is changed
 MODULE_DEPS_STATIC=Modules/config.c

Modules/cpython-sys/build.rs

@@ -2,21 +2,55 @@ use std::env;
 use std::path::{Path, PathBuf};
 
 fn main() {
-    let curdir = std::env::current_dir().unwrap();
-    let srcdir = curdir.parent().and_then(Path::parent).unwrap();
+    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+    let srcdir = manifest_dir
+        .parent()
+        .and_then(Path::parent)
+        .expect("expected Modules/cpython-sys to live under the source tree");
     let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
-    generate_c_api_bindings(srcdir, &out_path.as_path());
+    let builddir = env::var("PYTHON_BUILD_DIR").ok();
+    if gil_disabled(&srcdir, builddir.as_deref()) {
+        println!("cargo:rustc-cfg=py_gil_disabled");
+    }
+    generate_c_api_bindings(srcdir, builddir.as_deref(), &out_path.as_path());
     // TODO(emmatyping): generate bindings to the internal parser API
     // The parser includes things slightly differently, so we should generate
     // it's bindings independently
     //generate_parser_bindings(srcdir, &out_path.as_path());
 }
 
-fn generate_c_api_bindings(srcdir: &Path, out_path: &Path) {
-    let bindings = bindgen::Builder::default()
-        .header("wrapper.h")
-        .clang_arg(format!("-I{}", srcdir.as_os_str().to_str().unwrap()))
-        .clang_arg(format!("-I{}/Include", srcdir.as_os_str().to_str().unwrap()))
+fn gil_disabled(srcdir: &Path, builddir: Option<&str>) -> bool {
+    let mut candidates = Vec::new();
+    if let Some(build) = builddir {
+        candidates.push(PathBuf::from(build));
+    }
+    candidates.push(srcdir.to_path_buf());
+    for base in candidates {
+        let path = base.join("pyconfig.h");
+        if let Ok(contents) = std::fs::read_to_string(&path) {
+            if contents.contains("Py_GIL_DISABLED 1") {
+                return true;
+            }
+        }
+    }
+    false
+}
+
+fn generate_c_api_bindings(srcdir: &Path, builddir: Option<&str>, out_path: &Path) {
+    let mut builder = bindgen::Builder::default().header("wrapper.h");
+
+    // Always search the source dir and the public headers.
+    let mut include_dirs = vec![srcdir.to_path_buf(), srcdir.join("Include")];
+    // Include the build directory if provided; out-of-tree builds place
+    // the generated pyconfig.h there.
+    if let Some(build) = builddir {
+        include_dirs.push(PathBuf::from(build));
+    }
+    for dir in include_dirs {
+        builder = builder.clang_arg(format!("-I{}", dir.display()));
+    }
+
+    let bindings = builder
         .allowlist_function("_?Py.*")
         .allowlist_type("_?Py.*")
         .allowlist_var("_?Py.*")

Modules/cpython-sys/src/lib.rs

@@ -111,6 +111,14 @@ impl PyMethodDef {
 unsafe impl Sync for PyMethodDef {}
 unsafe impl Send for PyMethodDef {}
 
+#[cfg(py_gil_disabled)]
+pub const PyObject_HEAD_INIT: PyObject = {
+    let mut obj: PyObject = unsafe { std::mem::MaybeUninit::zeroed().assume_init() };
+    obj.ob_flags = _Py_STATICALLY_ALLOCATED_FLAG as _;
+    obj
+};
+
+#[cfg(not(py_gil_disabled))]
 pub const PyObject_HEAD_INIT: PyObject = PyObject {
     __bindgen_anon_1: _object__bindgen_ty_1 {
         ob_refcnt_full: _Py_STATIC_IMMORTAL_INITIAL_REFCNT as i64,

Modules/makesetup

@@ -287,7 +287,7 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 			libs=
 			# depends on the headers through cpython-sys
 			rule="$objs: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS)"
-			rule="$rule; cargo build --lib --locked --package ${mods} --profile \$(CARGO_PROFILE)"
+			rule="$rule; CARGO_TARGET_DIR=\$(abs_builddir)/target PYTHON_BUILD_DIR=\$(abs_builddir) cargo build --lib --locked --package ${mods} --profile \$(CARGO_PROFILE) --manifest-path \$(srcdir)/Cargo.toml"
 			echo "$rule" >>$rulesf
 			for mod in $mods
 			do

Tools/build/generate_stdlib_module_names.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import _imp
+import os
 import os.path
 import sys
 import sysconfig
@@ -14,6 +15,7 @@
 
 SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
+MODULES_PATH = os.path.join(SRC_DIR, 'Modules')
 
 IGNORE = {
     '__init__',
@@ -84,6 +86,19 @@ def list_modules_setup_extensions(names: set[str]) -> None:
     names.update(checker.list_module_names(all=True))
 
 
+def list_rust_modules(names: set[str]) -> None:
+    if not os.path.isdir(MODULES_PATH):
+        return
+    for entry in os.scandir(MODULES_PATH):
+        if not entry.is_dir():
+            continue
+        if entry.name == "cpython-sys":
+            continue
+        cargo_toml = os.path.join(entry.path, "Cargo.toml")
+        if os.path.isfile(cargo_toml):
+            names.add(entry.name)
+
+
 # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
 # Use the "./Programs/_testembed list_frozen" command.
 def list_frozen(names: set[str]) -> None:
@@ -109,6 +124,7 @@ def list_modules() -> set[str]:
 
     list_builtin_modules(names)
     list_modules_setup_extensions(names)
+    list_rust_modules(names)
     list_packages(names)
     list_python_modules(names)
     list_frozen(names)