From de05ccc12ed5ada6566223593308bc05e9259ee1 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 07:36:01 -0700 Subject: [PATCH 01/11] Add free-threading support for Python 3.13+ Implements thread-safe concurrent database operations for Python's free-threading mode using native locks where available. - Uses native RW locks (pthread_rwlock_t on POSIX, SRWLOCK on Windows) to allow multiple concurrent readers - Uses GIL on platforms that don't support these native locks - Zero overhead when GIL is enabled (macros compile to no-ops) --- HISTORY.rst | 4 + extension/maxminddb.c | 283 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 279 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2429c5d..a88e49c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,10 @@ History * Databases can now be loaded from buffers. This can be done by passing in a buffer as the database and using mode ``MODE_FD``. Pull request by Emanuel Seemann. GitHub #234. +* The C extension now supports Python 3.13+ free-threading mode and is + thread-safe for concurrent reads on platforms with pthread support (such as + Linux and macOS) and Windows. On other platforms, the extension will use + GIL-based protection. 2.8.2 (2025-07-25) ++++++++++++++++++ diff --git a/extension/maxminddb.c b/extension/maxminddb.c index 96aca80..f1f2313 100644 --- a/extension/maxminddb.c +++ b/extension/maxminddb.c @@ -22,11 +22,57 @@ static PyTypeObject Metadata_Type; static PyObject *MaxMindDB_error; static PyObject *ipaddress_ip_network; -// clang-format off +// ============================================================================= +// Platform-specific lock type definition +// ============================================================================= + +// Determine which locking strategy to use +#if !defined(Py_GIL_DISABLED) +// GIL is enabled - no locks needed, GIL provides synchronization +#define MAXMINDDB_USE_GIL_ONLY +#elif defined(_WIN32) +// Free-threaded on Windows - use SRWLOCK +#define MAXMINDDB_USE_WINDOWS_LOCKS +#include +#elif (defined(__has_include) && __has_include()) || \ + (defined(_POSIX_THREADS) && (_POSIX_THREADS > 0)) || \ + defined(__unix__) || defined(__APPLE__) || defined(__linux__) +// Free-threaded with pthread support - use pthread_rwlock_t +// This covers POSIX systems, including Linux, macOS, BSDs, Android, Cygwin, +// etc. +#define MAXMINDDB_USE_PTHREAD_LOCKS +#include +#include +#include +#else +// Free-threaded on unknown platform - fall back to GIL protection +// The extension will work but won't benefit from fine-grained locking +#define MAXMINDDB_USE_GIL_ONLY +#endif + +// Define the lock type based on strategy +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS +typedef SRWLOCK reader_rwlock_t; +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) +typedef pthread_rwlock_t reader_rwlock_t; +#else +// Dummy lock type for GIL-only mode typedef struct { + // Dummy member to satisfy MSVC, which doesn't allow empty structs. + char dummy; +} reader_rwlock_t; +#endif + +// ============================================================================= +// Type definitions +// ============================================================================= + +// clang-format off +typedef struct Reader_obj_struct { PyObject_HEAD /* no semicolon */ MMDB_s *mmdb; PyObject *closed; + reader_rwlock_t rwlock; } Reader_obj; typedef struct record record; @@ -74,6 +120,169 @@ static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address); #define UNUSED(x) UNUSED_##x #endif +// ============================================================================= +// Lock function implementations +// ============================================================================= + +static int reader_lock_init(reader_rwlock_t *lock) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + InitializeSRWLock(lock); + return 0; + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_init(lock, NULL); + if (err != 0) { + PyErr_Format(PyExc_RuntimeError, + "Failed to initialize read-write lock: %s", + strerror(err)); + return -1; + } + return 0; + +#else + // GIL-only mode - no-op + (void)lock; + return 0; +#endif +} + +static void reader_lock_destroy(reader_rwlock_t *lock) { +#ifdef MAXMINDDB_USE_PTHREAD_LOCKS + int err = pthread_rwlock_destroy(lock); + if (err == EBUSY) { + PyErr_WarnEx(PyExc_RuntimeWarning, + "Destroying MaxMind Reader lock while still in use " + "(bug: lock not properly released)", + 1); + } else if (err != 0) { + PyErr_WarnFormat(PyExc_RuntimeWarning, + 1, + "pthread_rwlock_destroy failed: %s", + strerror(err)); + } + +#else + // Windows SRWLOCK and GIL-only mode - no cleanup needed + (void)lock; +#endif +} + +static int reader_acquire_read_lock(Reader_obj *reader) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + AcquireSRWLockShared(&(reader->rwlock)); + return 0; + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_rdlock(&(reader->rwlock)); + if (err != 0) { + const char *msg; + switch (err) { + case EAGAIN: + msg = "Maximum number of read locks exceeded"; + break; + case EDEADLK: + msg = "Deadlock detected"; + break; + case EINVAL: + msg = "Invalid lock state"; + break; + default: + msg = strerror(err); + break; + } + PyErr_Format( + PyExc_RuntimeError, "Failed to acquire read lock: %s", msg); + return -1; + } + return 0; + +#else + // GIL-only mode - no-op + (void)reader; + return 0; +#endif +} + +static void reader_release_read_lock(Reader_obj *reader) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + ReleaseSRWLockShared(&(reader->rwlock)); + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_unlock(&(reader->rwlock)); + if (err != 0) { + PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "pthread_rwlock_unlock failed: %s (unbalanced lock/unlock?)", + strerror(err)); + } + +#else + // GIL-only mode - no-op + (void)reader; +#endif +} + +static int reader_acquire_write_lock(Reader_obj *reader) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + AcquireSRWLockExclusive(&(reader->rwlock)); + return 0; + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_wrlock(&(reader->rwlock)); + if (err != 0) { + const char *msg; + switch (err) { + case EAGAIN: + msg = "Maximum number of write locks exceeded"; + break; + case EDEADLK: + msg = "Deadlock detected"; + break; + case EINVAL: + msg = "Invalid lock state"; + break; + default: + msg = strerror(err); + break; + } + PyErr_Format( + PyExc_RuntimeError, "Failed to acquire write lock: %s", msg); + return -1; + } + return 0; + +#else + // GIL-only mode - no-op + (void)reader; + return 0; +#endif +} + +static void reader_release_write_lock(Reader_obj *reader) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + ReleaseSRWLockExclusive(&(reader->rwlock)); + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_unlock(&(reader->rwlock)); + if (err != 0) { + PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "pthread_rwlock_unlock failed: %s (unbalanced lock/unlock?)", + strerror(err)); + } + +#else + // GIL-only mode - no-op + (void)reader; +#endif +} + +// ============================================================================= +// Reader implementation +// ============================================================================= + static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *filepath = NULL; int mode = 0; @@ -125,9 +334,16 @@ static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) { return -1; } + if (reader_lock_init(&mmdb_obj->rwlock) != 0) { + free(mmdb); + Py_XDECREF(filepath); + return -1; + } + int const status = MMDB_open(filename, MMDB_MODE_MMAP, mmdb); if (status != MMDB_SUCCESS) { + reader_lock_destroy(&mmdb_obj->rwlock); free(mmdb); PyErr_Format(MaxMindDB_error, "Error opening database file (%s). Is this a valid " @@ -166,13 +382,6 @@ static PyObject *Reader_get_with_prefix_len(PyObject *self, PyObject *args) { } static int get_record(PyObject *self, PyObject *args, PyObject **record) { - MMDB_s *mmdb = ((Reader_obj *)self)->mmdb; - if (mmdb == NULL) { - PyErr_SetString(PyExc_ValueError, - "Attempt to read from a closed MaxMind DB."); - return -1; - } - struct sockaddr_storage ip_address_ss = {0}; struct sockaddr *ip_address = (struct sockaddr *)&ip_address_ss; if (!PyArg_ParseTuple(args, "O&", ip_converter, &ip_address_ss)) { @@ -184,11 +393,26 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { return -1; } + Reader_obj *reader = (Reader_obj *)self; + + if (reader_acquire_read_lock(reader) != 0) { + return -1; + } + + MMDB_s *mmdb = reader->mmdb; + if (mmdb == NULL) { + reader_release_read_lock(reader); + PyErr_SetString(PyExc_ValueError, + "Attempt to read from a closed MaxMind DB."); + return -1; + } + int mmdb_error = MMDB_SUCCESS; MMDB_lookup_result_s result = MMDB_lookup_sockaddr(mmdb, ip_address, &mmdb_error); if (mmdb_error != MMDB_SUCCESS) { + reader_release_read_lock(reader); PyObject *exception; if (MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR == mmdb_error) { exception = PyExc_ValueError; @@ -213,6 +437,7 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { } if (!result.found_entry) { + reader_release_read_lock(reader); Py_INCREF(Py_None); *record = Py_None; return prefix_len; @@ -221,6 +446,7 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { MMDB_entry_data_list_s *entry_data_list = NULL; int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list); if (status != MMDB_SUCCESS) { + reader_release_read_lock(reader); char ipstr[INET6_ADDRSTRLEN] = {0}; if (format_sockaddr(ip_address, ipstr)) { PyErr_Format(MaxMindDB_error, @@ -236,6 +462,8 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { *record = from_entry_data_list(&entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); + reader_release_read_lock(reader); + // from_entry_data_list will return NULL on errors. if (*record == NULL) { return -1; @@ -344,7 +572,12 @@ static bool format_sockaddr(struct sockaddr *sa, char *dst) { static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; + if (reader_acquire_read_lock(mmdb_obj) != 0) { + return NULL; + } + if (mmdb_obj->mmdb == NULL) { + reader_release_read_lock(mmdb_obj); PyErr_SetString(PyExc_IOError, "Attempt to read from a closed MaxMind DB."); return NULL; @@ -354,6 +587,7 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { int status = MMDB_get_metadata_as_entry_data_list(mmdb_obj->mmdb, &entry_data_list); if (status != MMDB_SUCCESS) { + reader_release_read_lock(mmdb_obj); PyErr_Format(MaxMindDB_error, "Error decoding metadata. %s", MMDB_strerror(status)); @@ -364,10 +598,13 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { PyObject *metadata_dict = from_entry_data_list(&entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); if (metadata_dict == NULL || !PyDict_Check(metadata_dict)) { + reader_release_read_lock(mmdb_obj); PyErr_SetString(MaxMindDB_error, "Error decoding metadata."); return NULL; } + reader_release_read_lock(mmdb_obj); + PyObject *args = PyTuple_New(0); if (args == NULL) { Py_DECREF(metadata_dict); @@ -384,6 +621,10 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; + if (reader_acquire_write_lock(mmdb_obj) != 0) { + return NULL; + } + if (mmdb_obj->mmdb != NULL) { MMDB_close(mmdb_obj->mmdb); free(mmdb_obj->mmdb); @@ -392,6 +633,8 @@ static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) { mmdb_obj->closed = Py_True; + reader_release_write_lock(mmdb_obj); + Py_RETURN_NONE; } @@ -419,6 +662,8 @@ static void Reader_dealloc(PyObject *self) { Reader_close(self, NULL); } + reader_lock_destroy(&obj->rwlock); + PyObject_Del(self); } @@ -466,12 +711,17 @@ static PyObject *ReaderIter_next(PyObject *self) { return NULL; } + if (reader_acquire_read_lock(ri->reader) != 0) { + return NULL; + } + while (ri->next != NULL) { record *cur = ri->next; ri->next = cur->next; switch (cur->type) { case MMDB_RECORD_TYPE_INVALID: + reader_release_read_lock(ri->reader); PyErr_SetString(MaxMindDB_error, "Invalid record when reading node"); free(cur); @@ -487,6 +737,7 @@ static PyObject *ReaderIter_next(PyObject *self) { int status = MMDB_read_node( ri->reader->mmdb, (uint32_t)cur->record, &node); if (status != MMDB_SUCCESS) { + reader_release_read_lock(ri->reader); const char *error = MMDB_strerror(status); PyErr_Format( MaxMindDB_error, "Error reading node: %s", error); @@ -495,6 +746,7 @@ static PyObject *ReaderIter_next(PyObject *self) { } struct record *left = calloc(1, sizeof(record)); if (left == NULL) { + reader_release_read_lock(ri->reader); free(cur); PyErr_NoMemory(); return NULL; @@ -508,6 +760,7 @@ static PyObject *ReaderIter_next(PyObject *self) { struct record *right = left->next = calloc(1, sizeof(record)); if (right == NULL) { + reader_release_read_lock(ri->reader); free(cur); free(left); PyErr_NoMemory(); @@ -532,6 +785,7 @@ static PyObject *ReaderIter_next(PyObject *self) { int status = MMDB_get_entry_data_list(&cur->entry, &entry_data_list); if (status != MMDB_SUCCESS) { + reader_release_read_lock(ri->reader); PyErr_Format( MaxMindDB_error, "Error looking up data while iterating over tree: %s", @@ -546,6 +800,7 @@ static PyObject *ReaderIter_next(PyObject *self) { PyObject *record = from_entry_data_list(&entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); if (record == NULL) { + reader_release_read_lock(ri->reader); free(cur); return NULL; } @@ -567,6 +822,7 @@ static PyObject *ReaderIter_next(PyObject *self) { ip_length, cur->depth - ip_start * 8); if (network_tuple == NULL) { + reader_release_read_lock(ri->reader); Py_DECREF(record); free(cur); return NULL; @@ -574,6 +830,7 @@ static PyObject *ReaderIter_next(PyObject *self) { PyObject *args = PyTuple_Pack(1, network_tuple); Py_DECREF(network_tuple); if (args == NULL) { + reader_release_read_lock(ri->reader); Py_DECREF(record); free(cur); return NULL; @@ -582,6 +839,7 @@ static PyObject *ReaderIter_next(PyObject *self) { PyObject_CallObject(ipaddress_ip_network, args); Py_DECREF(args); if (network == NULL) { + reader_release_read_lock(ri->reader); Py_DECREF(record); free(cur); return NULL; @@ -591,10 +849,13 @@ static PyObject *ReaderIter_next(PyObject *self) { Py_DECREF(network); Py_DECREF(record); + reader_release_read_lock(ri->reader); + free(cur); return rv; } default: + reader_release_read_lock(ri->reader); PyErr_Format( MaxMindDB_error, "Unknown record type: %u", cur->type); free(cur); @@ -602,6 +863,7 @@ static PyObject *ReaderIter_next(PyObject *self) { } free(cur); } + reader_release_read_lock(ri->reader); return NULL; } @@ -979,6 +1241,11 @@ PyMODINIT_FUNC PyInit_extension(void) { return NULL; } +#ifndef MAXMINDDB_USE_GIL_ONLY + // Only declare module as GIL-free when we have proper locks available + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + Reader_Type.tp_new = PyType_GenericNew; if (PyType_Ready(&Reader_Type)) { return NULL; From 732c8ff9f2ee2a938b4331cbae9d3b6c13636506 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 08:06:25 -0700 Subject: [PATCH 02/11] Add some docs about thread safety --- README.rst | 10 ++++++++++ maxminddb/reader.py | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5d92acb..0ae81be 100644 --- a/README.rst +++ b/README.rst @@ -94,6 +94,16 @@ The module will return an ``InvalidDatabaseError`` if the database is corrupt or otherwise invalid. A ``ValueError`` will be thrown if you look up an invalid IP address or an IPv6 address in an IPv4 database. +Thread Safety +------------- + +Both the C extension and pure Python implementations are safe for concurrent +reads and support Python 3.13+ free-threading. The C extension provides +free-threading support on platforms with pthread support (such as Linux and +macOS) and Windows. On other platforms, the extension will use GIL-based +protection. Calling ``close()`` while reads are in progress may cause +exceptions in those threads. + Requirements ------------ diff --git a/maxminddb/reader.py b/maxminddb/reader.py index a1a1509..9997dff 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -288,7 +288,10 @@ def _load_buffer( return filename def close(self) -> None: - """Close the MaxMind DB file and returns the resources to the system.""" + """Close the MaxMind DB file and returns the resources to the system. + + Calling this method while reads are in progress may cause exceptions. + """ with contextlib.suppress(AttributeError): self._buffer.close() # type: ignore[union-attr] From 771321fac499b9313b225f076ef160e2f5590160 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 08:11:58 -0700 Subject: [PATCH 03/11] Fix wheel deprecation warning --- pyproject.toml | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f40efbf..4d7a74b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,6 @@ lint = [ requires = [ "setuptools>=77.0.3", "setuptools-scm", - "wheel", ] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 5b5a5d9..944426a 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ import sys from setuptools import Extension, setup +from setuptools.command.bdist_wheel import bdist_wheel from setuptools.command.build_ext import build_ext -from wheel.bdist_wheel import bdist_wheel # These were only added to setuptools in 59.0.1. try: From dd4827f757ee5403bec4a676a1ea85c357fd3228 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 08:13:08 -0700 Subject: [PATCH 04/11] Remove conditional logic for unsupported setuptools versions --- setup.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 944426a..d36cbce 100644 --- a/setup.py +++ b/setup.py @@ -5,20 +5,7 @@ from setuptools import Extension, setup from setuptools.command.bdist_wheel import bdist_wheel from setuptools.command.build_ext import build_ext - -# These were only added to setuptools in 59.0.1. -try: - from setuptools.errors import ( - CCompilerError, - DistutilsExecError, - DistutilsPlatformError, - ) -except ImportError: - from distutils.errors import ( - CCompilerError, - DistutilsExecError, - DistutilsPlatformError, - ) +from setuptools.errors import CCompilerError, ExecError, PlatformError cmdclass = {} PYPY = hasattr(sys, "pypy_version_info") @@ -74,7 +61,7 @@ # Cargo cult code for installing extension with pure Python fallback. # Taken from SQLAlchemy, but this same basic code exists in many modules. -ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError) +ext_errors = (CCompilerError, ExecError, PlatformError) class BuildFailed(Exception): @@ -88,7 +75,7 @@ class ve_build_ext(build_ext): def run(self) -> None: try: build_ext.run(self) - except DistutilsPlatformError: + except PlatformError: raise BuildFailed def build_extension(self, ext) -> None: From ea7a967d4036627b365225022b6da0c3ba004008 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 08:16:14 -0700 Subject: [PATCH 05/11] Require Python 3.10+ --- .github/workflows/test-libmaxminddb.yml | 2 +- .github/workflows/test.yml | 2 +- HISTORY.rst | 2 ++ README.rst | 2 +- pyproject.toml | 5 +---- uv.lock | 8 +------- 6 files changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-libmaxminddb.yml b/.github/workflows/test-libmaxminddb.yml index 0ca442b..51d2fee 100644 --- a/.github/workflows/test-libmaxminddb.yml +++ b/.github/workflows/test-libmaxminddb.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - env: [3.9, "3.10", 3.11, 3.12, 3.13, 3.14] + env: ["3.10", 3.11, 3.12, 3.13, 3.14] # We don't test on Windows currently due to issues # build libmaxminddb there. os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2539fef..b6035c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - env: [3.9, "3.10", 3.11, 3.12, 3.13, 3.14] + env: ["3.10", 3.11, 3.12, 3.13, 3.14] os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] steps: - uses: actions/checkout@v5 diff --git a/HISTORY.rst b/HISTORY.rst index a88e49c..697b8e2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,8 @@ History 2.9.0 ++++++++++++++++++ +* IMPORTANT: Python 3.10 or greater is required. If you are using an older + version, please use an earlier release. * Databases can now be loaded from buffers. This can be done by passing in a buffer as the database and using mode ``MODE_FD``. Pull request by Emanuel Seemann. GitHub #234. diff --git a/README.rst b/README.rst index 0ae81be..1f72e3e 100644 --- a/README.rst +++ b/README.rst @@ -107,7 +107,7 @@ exceptions in those threads. Requirements ------------ -This code requires Python 3.9+. Older versions are not supported. The C +This code requires Python 3.10+. Older versions are not supported. The C extension requires CPython. Versioning diff --git a/pyproject.toml b/pyproject.toml index 4d7a74b..7dca5a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Reader for the MaxMind DB format" authors = [ {name = "Gregory Oschwald", email = "goschwald@maxmind.com"}, ] -requires-python = ">=3.9" +requires-python = ">=3.10" readme = "README.rst" license = "Apache-2.0" classifiers = [ @@ -15,7 +15,6 @@ classifiers = [ "Intended Audience :: System Administrators", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -93,7 +92,6 @@ ignore = [ [tool.tox] env_list = [ - "3.9", "3.10", "3.11", "3.12", @@ -130,4 +128,3 @@ commands = [ "3.12" = ["3.12"] "3.11" = ["3.11"] "3.10" = ["3.10"] -"3.9" = ["3.9"] diff --git a/uv.lock b/uv.lock index 6026f7b..e2dca93 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.9" +requires-python = ">=3.10" [[package]] name = "colorama" @@ -97,12 +97,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/11/040983fad5132d85914c874a2836252bbc57832065548885b5bb5b0d4359/mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb", size = 13326683, upload-time = "2025-09-19T00:09:55.572Z" }, { url = "https://files.pythonhosted.org/packages/e9/ba/89b2901dd77414dd7a8c8729985832a5735053be15b744c18e4586e506ef/mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075", size = 13514749, upload-time = "2025-09-19T00:10:44.827Z" }, { url = "https://files.pythonhosted.org/packages/25/bc/cc98767cffd6b2928ba680f3e5bc969c4152bf7c2d83f92f5a504b92b0eb/mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf", size = 9982959, upload-time = "2025-09-19T00:10:37.344Z" }, - { url = "https://files.pythonhosted.org/packages/3f/a6/490ff491d8ecddf8ab91762d4f67635040202f76a44171420bcbe38ceee5/mypy-1.18.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:25a9c8fb67b00599f839cf472713f54249a62efd53a54b565eb61956a7e3296b", size = 12807230, upload-time = "2025-09-19T00:09:49.471Z" }, - { url = "https://files.pythonhosted.org/packages/eb/2e/60076fc829645d167ece9e80db9e8375648d210dab44cc98beb5b322a826/mypy-1.18.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2b9c7e284ee20e7598d6f42e13ca40b4928e6957ed6813d1ab6348aa3f47133", size = 11895666, upload-time = "2025-09-19T00:10:53.678Z" }, - { url = "https://files.pythonhosted.org/packages/97/4a/1e2880a2a5dda4dc8d9ecd1a7e7606bc0b0e14813637eeda40c38624e037/mypy-1.18.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6985ed057513e344e43a26cc1cd815c7a94602fb6a3130a34798625bc2f07b6", size = 12499608, upload-time = "2025-09-19T00:09:36.204Z" }, - { url = "https://files.pythonhosted.org/packages/00/81/a117f1b73a3015b076b20246b1f341c34a578ebd9662848c6b80ad5c4138/mypy-1.18.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22f27105f1525ec024b5c630c0b9f36d5c1cc4d447d61fe51ff4bd60633f47ac", size = 13244551, upload-time = "2025-09-19T00:10:17.531Z" }, - { url = "https://files.pythonhosted.org/packages/9b/61/b9f48e1714ce87c7bf0358eb93f60663740ebb08f9ea886ffc670cea7933/mypy-1.18.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:030c52d0ea8144e721e49b1f68391e39553d7451f0c3f8a7565b59e19fcb608b", size = 13491552, upload-time = "2025-09-19T00:10:13.753Z" }, - { url = "https://files.pythonhosted.org/packages/c9/66/b2c0af3b684fa80d1b27501a8bdd3d2daa467ea3992a8aa612f5ca17c2db/mypy-1.18.2-cp39-cp39-win_amd64.whl", hash = "sha256:aa5e07ac1a60a253445797e42b8b2963c9675563a94f11291ab40718b016a7a0", size = 9765635, upload-time = "2025-09-19T00:10:30.993Z" }, { url = "https://files.pythonhosted.org/packages/87/e3/be76d87158ebafa0309946c4a73831974d4d6ab4f4ef40c3b53a385a66fd/mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e", size = 2352367, upload-time = "2025-09-19T00:10:15.489Z" }, ] From a6dbd78f74aa500381a3cb64e185c164661ad4bf Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 08:26:05 -0700 Subject: [PATCH 06/11] Add threading test for C extension We already have some threading coverage in the general reader tests, but this provides more detailed implementation tests for the extension. --- tests/threading_test.py | 274 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 tests/threading_test.py diff --git a/tests/threading_test.py b/tests/threading_test.py new file mode 100644 index 0000000..d4afad4 --- /dev/null +++ b/tests/threading_test.py @@ -0,0 +1,274 @@ +"""Tests for thread-safety and free-threading support.""" + +from __future__ import annotations + +import threading +import time +import unittest +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from maxminddb.types import Record + +try: + import maxminddb.extension # noqa: F401 + + HAS_EXTENSION = True +except ImportError: + HAS_EXTENSION = False + +from maxminddb import open_database +from maxminddb.const import MODE_MMAP_EXT + + +@unittest.skipIf( + not HAS_EXTENSION, + "No C extension module found. Skipping threading tests", +) +class TestThreadSafety(unittest.TestCase): + """Test thread safety of the C extension.""" + + def test_concurrent_reads(self) -> None: + """Test multiple threads reading concurrently.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + results: list[Record | None] = [None] * 100 + errors: list[Exception] = [] + + def lookup(index: int, ip: str) -> None: + try: + results[index] = reader.get(ip) + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [] + for i in range(100): + ip = f"1.1.1.{(i % 32) + 1}" + t = threading.Thread(target=lookup, args=(i, ip)) + threads.append(t) + t.start() + + for t in threads: + t.join() + + reader.close() + + self.assertEqual(len(errors), 0, f"Errors during concurrent reads: {errors}") + # All lookups should have completed + self.assertNotIn(None, results) + + def test_read_during_close(self) -> None: + """Test that close is safe when reads are happening concurrently.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + errors: list[Exception] = [] + should_stop = threading.Event() + + def continuous_reader() -> None: + # Keep reading until signaled to stop or reader is closed + while not should_stop.is_set(): + try: + reader.get("1.1.1.1") + except ValueError as e: # noqa: PERF203 + # Expected once close() is called + if "closed MaxMind DB" not in str(e): + errors.append(e) + break + except Exception as e: # noqa: BLE001 + errors.append(e) + break + + # Start multiple readers + threads = [threading.Thread(target=continuous_reader) for _ in range(10)] + for t in threads: + t.start() + + # Let readers run for a bit + time.sleep(0.05) + + # Close while reads are happening + reader.close() + + # Signal threads to stop + should_stop.set() + + # Wait for all threads + for t in threads: + t.join(timeout=1.0) + + self.assertEqual(len(errors), 0, f"Errors during close test: {errors}") + + def test_read_after_close(self) -> None: + """Test that reads after close raise appropriate error.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + reader.close() + + with self.assertRaisesRegex( + ValueError, + "Attempt to read from a closed MaxMind DB", + ): + reader.get("1.1.1.1") + + def test_concurrent_reads_and_metadata(self) -> None: + """Test concurrent reads and metadata access.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + errors: list[Exception] = [] + results: list[bool] = [] + + def do_reads() -> None: + try: + for _ in range(50): + reader.get("1.1.1.1") + results.append(True) + except Exception as e: # noqa: BLE001 + errors.append(e) + + def do_metadata() -> None: + try: + for _ in range(50): + reader.metadata() + results.append(True) + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [] + for _ in range(5): + threads.append(threading.Thread(target=do_reads)) + threads.append(threading.Thread(target=do_metadata)) + + for t in threads: + t.start() + + for t in threads: + t.join() + + reader.close() + + self.assertEqual( + len(errors), 0, f"Errors during concurrent operations: {errors}" + ) + self.assertEqual(len(results), 10, "All threads should complete") + + def test_concurrent_iteration(self) -> None: + """Test that iteration is thread-safe.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + errors: list[Exception] = [] + counts: list[int] = [] + + def iterate() -> None: + try: + count = 0 + for _ in reader: + count += 1 + counts.append(count) + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [threading.Thread(target=iterate) for _ in range(10)] + + for t in threads: + t.start() + + for t in threads: + t.join() + + reader.close() + + self.assertEqual(len(errors), 0, f"Errors during iteration: {errors}") + # All threads should see the same number of entries + self.assertEqual(len(set(counts)), 1, "All threads should see same entry count") + + def test_stress_test(self) -> None: + """Stress test with many threads and operations.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + errors: list[Exception] = [] + operations_completed = threading.Event() + + def random_operations() -> None: + try: + for i in range(100): + # Mix different operations + if i % 3 == 0: + reader.get("1.1.1.1") + elif i % 3 == 1: + reader.metadata() + else: + reader.get_with_prefix_len("1.1.1.2") + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [threading.Thread(target=random_operations) for _ in range(20)] + + for t in threads: + t.start() + + for t in threads: + t.join() + + operations_completed.set() + reader.close() + + self.assertEqual(len(errors), 0, f"Errors during stress test: {errors}") + + def test_multiple_readers_different_databases(self) -> None: + """Test multiple readers on different databases in parallel.""" + errors: list[Exception] = [] + + def use_reader(filename: str) -> None: + try: + reader = open_database(filename, MODE_MMAP_EXT) + for _ in range(50): + reader.get("1.1.1.1") + reader.close() + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [ + threading.Thread( + target=use_reader, + args=("tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb",), + ) + for _ in range(5) + ] + threads.extend( + [ + threading.Thread( + target=use_reader, + args=("tests/data/test-data/MaxMind-DB-test-ipv6-24.mmdb",), + ) + for _ in range(5) + ] + ) + + for t in threads: + t.start() + + for t in threads: + t.join() + + self.assertEqual(len(errors), 0, f"Errors with multiple readers: {errors}") + + +if __name__ == "__main__": + unittest.main() From a8a403c4efd7b00483a86c0449bb6e90743606dc Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 08:43:45 -0700 Subject: [PATCH 07/11] Use modern union syntax throughout --- maxminddb/decoder.py | 12 ++++++++---- maxminddb/types.py | 14 +++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/maxminddb/decoder.py b/maxminddb/decoder.py index 069c3c8..fc03958 100644 --- a/maxminddb/decoder.py +++ b/maxminddb/decoder.py @@ -1,7 +1,9 @@ """Decoder for the MaxMind DB data section.""" +from __future__ import annotations + import struct -from typing import ClassVar, Union, cast +from typing import TYPE_CHECKING, ClassVar, cast try: import mmap @@ -10,8 +12,10 @@ from maxminddb.errors import InvalidDatabaseError -from maxminddb.file import FileBuffer -from maxminddb.types import Record + +if TYPE_CHECKING: + from maxminddb.file import FileBuffer + from maxminddb.types import Record class Decoder: @@ -19,7 +23,7 @@ class Decoder: def __init__( self, - database_buffer: Union[FileBuffer, "mmap.mmap", bytes], + database_buffer: FileBuffer | mmap.mmap | bytes, pointer_base: int = 0, pointer_test: bool = False, # noqa: FBT001, FBT002 ) -> None: diff --git a/maxminddb/types.py b/maxminddb/types.py index aefae65..4c0c478 100644 --- a/maxminddb/types.py +++ b/maxminddb/types.py @@ -1,14 +1,18 @@ """Types representing database records.""" -from typing import AnyStr, Union +from __future__ import annotations -Primitive = Union[AnyStr, bool, float, int] -Record = Union[Primitive, "RecordList", "RecordDict"] +from typing import AnyStr, TypeAlias +Primitive: TypeAlias = AnyStr | bool | float | int -class RecordList(list[Record]): + +class RecordList(list["Record"]): """RecordList is a type for lists in a database record.""" -class RecordDict(dict[str, Record]): +class RecordDict(dict[str, "Record"]): """RecordDict is a type for dicts in a database record.""" + + +Record: TypeAlias = Primitive | RecordList | RecordDict From 3a7a4359b2744ebf25b03406d260695387acc094 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 09:43:55 -0700 Subject: [PATCH 08/11] Pass through env variables to tox To ensure we are forcing correct checks in CI --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 7dca5a6..c8e2656 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,6 +105,13 @@ skip_missing_interpreters = false dependency_groups = [ "dev", ] +pass_env = [ + "CFLAGS", + "LDFLAGS", + "MAXMINDDB_REQUIRE_EXTENSION", + "MAXMINDDB_USE_SYSTEM_LIBMAXMINDDB", + "MM_FORCE_EXT_TESTS", +] commands = [ ["pytest", "tests"], ] From cc8e7609da1e2c69bc774e02e2bed8fc4e7c2aae Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 10:04:31 -0700 Subject: [PATCH 09/11] Use correct Python version for libmaxminddb build --- .github/workflows/test-libmaxminddb.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-libmaxminddb.yml b/.github/workflows/test-libmaxminddb.yml index 51d2fee..e380948 100644 --- a/.github/workflows/test-libmaxminddb.yml +++ b/.github/workflows/test-libmaxminddb.yml @@ -56,7 +56,7 @@ jobs: echo "LDFLAGS=-L/opt/homebrew/lib" >> "$GITHUB_ENV" - name: Build with Werror and Wall - run: uv build + run: uv build --python ${{ matrix.env }} env: CFLAGS: "${{ env.CFLAGS }} -Werror -Wall -Wextra" From 2f45722da367540aa534d0c0c92b9dc3fb13924a Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Fri, 10 Oct 2025 10:44:41 -0700 Subject: [PATCH 10/11] Fix C extension import in CI tests The C extension was being built but not imported during tests due to Python's sys.path ordering. The project root directory was appearing first in sys.path, causing Python to import from the source directory (which doesn't have the compiled .so file) instead of the installed package in site-packages. Changes: - Add pytest --import-mode=importlib to prevent pytest from adding the project root to sys.path, ensuring tests import from the installed package in the tox virtualenv - Add MAXMINDDB_REQUIRE_EXTENSION=1 to the "Setup test suite" step to ensure the extension is built during CI setup --- .github/workflows/test.yml | 1 + pyproject.toml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6035c6..849d2f7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,7 @@ jobs: - name: Setup test suite run: tox run -vv --notest --skip-missing-interpreters false env: + MAXMINDDB_REQUIRE_EXTENSION: 1 TOX_GH_MAJOR_MINOR: ${{ matrix.env }} - name: Run test suite run: tox run --skip-pkg-install diff --git a/pyproject.toml b/pyproject.toml index c8e2656..8eaed8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,13 @@ ignore = [ "setup.py" = ["ALL"] "tests/*" = ["ANN201", "D"] +[tool.pytest.ini_options] +# Use importlib mode to prevent pytest from adding the project root to sys.path. +# This ensures tests import from the tox virtualenv's installed package (which includes +# the compiled C extension) rather than directly from the source directory (which doesn't +# have the .so file). We still test the source code, just via the installed version. +addopts = "--import-mode=importlib" + [tool.tox] env_list = [ "3.10", From f833c8f03f0609ac72d854f2f1c5a91395e68ff5 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Tue, 14 Oct 2025 10:56:04 -0700 Subject: [PATCH 11/11] Take read lock before checking if closed This prevents a race in ReaderIter_next and protects against writes that are not atomic for the others. --- extension/maxminddb.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/extension/maxminddb.c b/extension/maxminddb.c index f1f2313..a883769 100644 --- a/extension/maxminddb.c +++ b/extension/maxminddb.c @@ -641,12 +641,19 @@ static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) { static PyObject *Reader__enter__(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; + if (reader_acquire_read_lock(mmdb_obj) != 0) { + return NULL; + } + if (mmdb_obj->closed == Py_True) { + reader_release_read_lock(mmdb_obj); PyErr_SetString(PyExc_ValueError, "Attempt to reopen a closed MaxMind DB."); return NULL; } + reader_release_read_lock(mmdb_obj); + Py_INCREF(self); return (PyObject *)self; } @@ -669,12 +676,20 @@ static void Reader_dealloc(PyObject *self) { static PyObject *Reader_iter(PyObject *obj) { Reader_obj *reader = (Reader_obj *)obj; + + if (reader_acquire_read_lock(reader) != 0) { + return NULL; + } + if (reader->closed == Py_True) { + reader_release_read_lock(reader); PyErr_SetString(PyExc_ValueError, "Attempt to iterate over a closed MaxMind DB."); return NULL; } + reader_release_read_lock(reader); + ReaderIter_obj *ri = PyObject_New(ReaderIter_obj, &ReaderIter_Type); if (ri == NULL) { return NULL; @@ -705,13 +720,15 @@ static bool is_ipv6(char ip[16]) { static PyObject *ReaderIter_next(PyObject *self) { ReaderIter_obj *ri = (ReaderIter_obj *)self; - if (ri->reader->closed == Py_True) { - PyErr_SetString(PyExc_ValueError, - "Attempt to iterate over a closed MaxMind DB."); + + if (reader_acquire_read_lock(ri->reader) != 0) { return NULL; } - if (reader_acquire_read_lock(ri->reader) != 0) { + if (ri->reader->closed == Py_True) { + reader_release_read_lock(ri->reader); + PyErr_SetString(PyExc_ValueError, + "Attempt to iterate over a closed MaxMind DB."); return NULL; }