bpo-45548: Fix out-of-tree and Debian builds (GH-29263)

Add Modules subdirs to SRCDIRS to generate directories for out-of-tree
object files.

Debian wants ncurses lib. Works on Fedora, too.

Debian also needs pkg-config to detect correct flags.

Remove more outdated comments. Makefile now tracks header dependencies
-lintl is injected by configure when needed. Build _dbm with
gdbm-compat.

Group some modules by purpose. socket, select, and mmap work on Windows,
too.

Author: tiran

.github/workflows/posix-deps-apt.sh

@@ -3,6 +3,7 @@ apt-get update
 
 apt-get -yq install \
     build-essential \
+    pkg-config \
     ccache \
     gdb \
     lcov \

Modules/Setup

@@ -38,7 +38,9 @@
 #
 # <name> = <value>
 #
-# which defines a Make variable definition inserted into Makefile.in
+# which defines a Make variable definition inserted into Makefile.in.
+# You can also use any Make variable that is detected by configure and
+# defined in Makefile.pre.in, e.g. OpenSSL flags $(OPENSSL_INCLUDES).
 #
 # The build process works like this:
 #
@@ -65,6 +67,11 @@
 # platform should be listed below.  The distribution comes with all
 # modules enabled that are supported by most platforms and don't
 # require you to download sources from elsewhere.
+#
+# NOTE: Avoid editing this file directly. Local changes should go into
+# Modules/Setup.local file. To enable all modules for testing, run
+#
+#    sed -n -E 's/^#([a-z_\*].*)$/\1/p' Modules/Setup > Modules/Setup.local
 
 
 # Some special rules to define PYTHONPATH.
@@ -103,12 +110,12 @@ _abc _abc.c
 _codecs _codecsmodule.c
 _functools _functoolsmodule.c
 _io -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c
-_locale _localemodule.c  # -lintl
+_locale _localemodule.c
 _operator _operator.c
 _signal signalmodule.c
 _sre _sre.c
 _stat _stat.c
-_symtable symtablemodule.c  # setup.py can't track the .h file that _symtable depends on.
+_symtable symtablemodule.c
 _thread _threadmodule.c
 _tracemalloc _tracemalloc.c  # See bpo-35053 as to why this is built in.
 _weakref _weakref.c
@@ -129,47 +136,30 @@ time timemodule.c
 # Python binary, or need to specify some odd set of compiler switches,
 # you can uncomment the appropriate lines below.
 
-# To enable all modules for testing, run
-# sed -n -E 's/^#([a-z_\*].*)$/\1/p' Modules/Setup > Modules/Setup.local
-
 # Uncommenting the following line tells makesetup that all following
 # modules are to be built as shared libraries (see above for more
 # detail; also note that *static* or *disabled* cancels this effect):
 
 #*shared*
 
-# Modules that should always be present (non UNIX dependent):
+# Modules that should always be present (POSIX and Windows):
 
 #_asyncio _asynciomodule.c
 #_bisect _bisectmodule.c
-#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
-#_codecs_cn cjkcodecs/_codecs_cn.c
-#_codecs_hk cjkcodecs/_codecs_hk.c
-#_codecs_iso2022 cjkcodecs/_codecs_iso2022.c
-#_codecs_jp cjkcodecs/_codecs_jp.c
-#_codecs_kr cjkcodecs/_codecs_kr.c
-#_codecs_tw cjkcodecs/_codecs_tw.c
 #_contextvars _contextvarsmodule.c
 #_csv _csv.c
 #_datetime _datetimemodule.c
 # UNIVERSAL: let mpdecimal.h detect settings
 #_decimal -DUNIVERSAL -I$(srcdir)/Modules/_decimal/libmpdec _decimal/_decimal.c _decimal/libmpdec/basearith.c _decimal/libmpdec/constants.c _decimal/libmpdec/context.c _decimal/libmpdec/convolute.c _decimal/libmpdec/crt.c _decimal/libmpdec/difradix2.c _decimal/libmpdec/fnt.c _decimal/libmpdec/fourstep.c _decimal/libmpdec/io.c _decimal/libmpdec/mpalloc.c _decimal/libmpdec/mpdecimal.c _decimal/libmpdec/numbertheory.c _decimal/libmpdec/sixstep.c _decimal/libmpdec/transpose.c
-#_elementtree -I$(srcdir)/Modules/expat _elementtree.c
 #_heapq _heapqmodule.c
 #_json _json.c
 #_lsprof _lsprof.c rotatingtree.c
-#_md5 md5module.c
-#_multibytecodec cjkcodecs/multibytecodec.c
 #_multiprocessing -I$(srcdir)/Modules/_multiprocessing _multiprocessing/multiprocessing.c _multiprocessing/semaphore.c
 #_opcode _opcode.c
 #_pickle _pickle.c
-#_posixsubprocess _posixsubprocess.c
 #_queue _queuemodule.c
 #_random _randommodule.c
-#_sha1 sha1module.c
-#_sha256 sha256module.c
-#_sha512 sha512module.c
-#_sha3 _sha3/sha3module.c
+#_socket socketmodule.c
 #_statistics _statisticsmodule.c
 #_struct _struct.c
 #_typing _typingmodule.c
@@ -179,53 +169,65 @@ time timemodule.c
 #binascii binascii.c
 #cmath cmathmodule.c
 #math mathmodule.c
+#mmap mmapmodule.c
+#select selectmodule.c
+
+# XML
+#_elementtree -I$(srcdir)/Modules/expat _elementtree.c
 #pyexpat -I$(srcdir)/Modules/expat  expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c
-#unicodedata  unicodedata.c
 
-# Modules with some UNIX dependencies -- on by default:
-# (If you have a really backward UNIX, select and socket may not be
-# supported...)
+# hashing builtins
+#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
+#_md5 md5module.c
+#_sha1 sha1module.c
+#_sha256 sha256module.c
+#_sha512 sha512module.c
+#_sha3 _sha3/sha3module.c
 
+# text encodings and unicode
+#_codecs_cn cjkcodecs/_codecs_cn.c
+#_codecs_hk cjkcodecs/_codecs_hk.c
+#_codecs_iso2022 cjkcodecs/_codecs_iso2022.c
+#_codecs_jp cjkcodecs/_codecs_jp.c
+#_codecs_kr cjkcodecs/_codecs_kr.c
+#_codecs_tw cjkcodecs/_codecs_tw.c
+#_multibytecodec cjkcodecs/multibytecodec.c
+#unicodedata unicodedata.c
+
+# Modules with some UNIX dependencies
+
+#_posixsubprocess _posixsubprocess.c
 #_posixshmem -I$(srcdir)/Modules/_multiprocessing _multiprocessing/posixshmem.c -lrt
-#_socket socketmodule.c  # socket(2)
-#fcntl fcntlmodule.c  # fcntl(2) and ioctl(2)
-#grp grpmodule.c  # grp(3)
-#mmap mmapmodule.c  # Also works on win32.
+#fcntl fcntlmodule.c
+#grp grpmodule.c
 #ossaudiodev ossaudiodev.c
-#select selectmodule.c  # select(2); not on ancient System V
-#spwd spwdmodule.c  # spwd(3)
+#resource resource.c
+#spwd spwdmodule.c
 #syslog syslogmodule.c
+#termios termios.c
 
-# Some more UNIX dependent modules -- off by default, since these
-# are not supported by all UNIX systems:
+# Modules with UNIX dependencies that require external libraries
 
-#_crypt _cryptmodule.c -lcrypt	# crypt(3); breaks many builds.
-#nis nismodule.c -I/usr/include/tirpc -lnsl -ltirpc	# Sun yellow pages -- not everywhere
-#termios termios.c	# Steen Lumholt's termios module
-#resource resource.c	# Jeremy Hylton's rlimit interface
+#_crypt _cryptmodule.c -lcrypt
+#nis nismodule.c -I/usr/include/tirpc -lnsl -ltirpc
 
 # Modules that require external libraries.
 
 #_bz2 _bz2module.c -lbz2
 #_ctypes _ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/stgdict.c _ctypes/cfield.c -ldl -lffi -DHAVE_FFI_PREP_CIF_VAR -DHAVE_FFI_PREP_CLOSURE_LOC -DHAVE_FFI_CLOSURE_ALLOC
-# _dbm _dbmmodule.c 	# -lndbm # dbm(3)
+# The _dbm module supports NDBM, GDBM with compat module, and Berkeley DB.
+#_dbm _dbmmodule.c -lgdbm_compat -DHAVE_NDBM_H
 #_gdbm _gdbmmodule.c -lgdbm
 #_lzma _lzmamodule.c -llzma
 #_sqlite3 _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c -lsqlite3
 #_uuid _uuidmodule.c -luuid
 #zlib  zlibmodule.c -lz
 
-# GNU readline.  Unlike previous Python incarnations, GNU readline is
-# now incorporated in an optional module, configured in the Setup file
-# instead of by a configure script switch.  You may have to insert a
-# -L option pointing to the directory where libreadline.* lives,
-# and you may have to change -ltermcap to -ltermlib or perhaps remove
-# it, depending on your system -- see the GNU readline instructions.
-# It's okay for this to be a shared library, too.
-
+# The readline module also supports libeditline (-leditline).
+# Some systems may require -ltermcap or -ltermlib.
 #readline readline.c -lreadline -ltermcap
 
-# To dynamically link OpenSSL:
+# OpenSSL bindings
 #_ssl _ssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS)
 #_hashlib _hashopenssl.c $(OPENSSL_INCLUDES) $(OPENSSL_LDFLAGS) -lcrypto
 
@@ -277,13 +279,8 @@ time timemodule.c
 # *** Always uncomment this; X11 libraries to link with:
 #	-lX11
 
-# Curses support, requiring the System V version of curses, often
-# provided by the ncurses library.  e.g. on Linux, link with -lncurses
-# instead of -lcurses).
-
-#_curses -lcurses -lcursesw -ltermcap _cursesmodule.c
-
-# Wrapper for the panel library that's part of ncurses and SYSV curses.
+# Some system have -lcurses 
+#_curses -lncurses -lncursesw -ltermcap _cursesmodule.c
 #_curses_panel -lpanel -lncurses _curses_panel.c
 
 # macOS specific modules

configure

@@ -17288,7 +17288,23 @@ do
 done
 
 
-SRCDIRS="Parser Objects Python Modules Modules/_io Programs"
+SRCDIRS="\
+  Modules \
+  Modules/_blake2 \
+  Modules/_ctypes \
+  Modules/_decimal \
+  Modules/_decimal/libmpdec \
+  Modules/_io \
+  Modules/_multiprocessing \
+  Modules/_sha3 \
+  Modules/_sqlite \
+  Modules/_xxtestfuzz \
+  Modules/cjkcodecs \
+  Modules/expat \
+  Objects \
+  Parser \
+  Programs \
+  Python"
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for build directories" >&5
 $as_echo_n "checking for build directories... " >&6; }
 for dir in $SRCDIRS; do

configure.ac

@@ -5636,7 +5636,23 @@ do
 done
 
 AC_SUBST(SRCDIRS)
-SRCDIRS="Parser Objects Python Modules Modules/_io Programs"
+SRCDIRS="\
+  Modules \
+  Modules/_blake2 \
+  Modules/_ctypes \
+  Modules/_decimal \
+  Modules/_decimal/libmpdec \
+  Modules/_io \
+  Modules/_multiprocessing \
+  Modules/_sha3 \
+  Modules/_sqlite \
+  Modules/_xxtestfuzz \
+  Modules/cjkcodecs \
+  Modules/expat \
+  Objects \
+  Parser \
+  Programs \
+  Python"
 AC_MSG_CHECKING(for build directories)
 for dir in $SRCDIRS; do
     if test ! -d $dir; then