diff --git a/.github/workflows/test-libmaxminddb.yml b/.github/workflows/test-libmaxminddb.yml index 0ca442b..e380948 100644 --- a/.github/workflows/test-libmaxminddb.yml +++ b/.github/workflows/test-libmaxminddb.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - env: [3.9, "3.10", 3.11, 3.12, 3.13, 3.14] + env: ["3.10", 3.11, 3.12, 3.13, 3.14] # We don't test on Windows currently due to issues # build libmaxminddb there. os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest] @@ -56,7 +56,7 @@ jobs: echo "LDFLAGS=-L/opt/homebrew/lib" >> "$GITHUB_ENV" - name: Build with Werror and Wall - run: uv build + run: uv build --python ${{ matrix.env }} env: CFLAGS: "${{ env.CFLAGS }} -Werror -Wall -Wextra" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2539fef..849d2f7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - env: [3.9, "3.10", 3.11, 3.12, 3.13, 3.14] + env: ["3.10", 3.11, 3.12, 3.13, 3.14] os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] steps: - uses: actions/checkout@v5 @@ -32,6 +32,7 @@ jobs: - name: Setup test suite run: tox run -vv --notest --skip-missing-interpreters false env: + MAXMINDDB_REQUIRE_EXTENSION: 1 TOX_GH_MAJOR_MINOR: ${{ matrix.env }} - name: Run test suite run: tox run --skip-pkg-install diff --git a/HISTORY.rst b/HISTORY.rst index 2429c5d..697b8e2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,9 +6,15 @@ History 2.9.0 ++++++++++++++++++ +* IMPORTANT: Python 3.10 or greater is required. If you are using an older + version, please use an earlier release. * Databases can now be loaded from buffers. This can be done by passing in a buffer as the database and using mode ``MODE_FD``. Pull request by Emanuel Seemann. GitHub #234. +* The C extension now supports Python 3.13+ free-threading mode and is + thread-safe for concurrent reads on platforms with pthread support (such as + Linux and macOS) and Windows. On other platforms, the extension will use + GIL-based protection. 2.8.2 (2025-07-25) ++++++++++++++++++ diff --git a/README.rst b/README.rst index 5d92acb..1f72e3e 100644 --- a/README.rst +++ b/README.rst @@ -94,10 +94,20 @@ The module will return an ``InvalidDatabaseError`` if the database is corrupt or otherwise invalid. A ``ValueError`` will be thrown if you look up an invalid IP address or an IPv6 address in an IPv4 database. +Thread Safety +------------- + +Both the C extension and pure Python implementations are safe for concurrent +reads and support Python 3.13+ free-threading. The C extension provides +free-threading support on platforms with pthread support (such as Linux and +macOS) and Windows. On other platforms, the extension will use GIL-based +protection. Calling ``close()`` while reads are in progress may cause +exceptions in those threads. + Requirements ------------ -This code requires Python 3.9+. Older versions are not supported. The C +This code requires Python 3.10+. Older versions are not supported. The C extension requires CPython. Versioning diff --git a/extension/maxminddb.c b/extension/maxminddb.c index 96aca80..a883769 100644 --- a/extension/maxminddb.c +++ b/extension/maxminddb.c @@ -22,11 +22,57 @@ static PyTypeObject Metadata_Type; static PyObject *MaxMindDB_error; static PyObject *ipaddress_ip_network; -// clang-format off +// ============================================================================= +// Platform-specific lock type definition +// ============================================================================= + +// Determine which locking strategy to use +#if !defined(Py_GIL_DISABLED) +// GIL is enabled - no locks needed, GIL provides synchronization +#define MAXMINDDB_USE_GIL_ONLY +#elif defined(_WIN32) +// Free-threaded on Windows - use SRWLOCK +#define MAXMINDDB_USE_WINDOWS_LOCKS +#include +#elif (defined(__has_include) && __has_include()) || \ + (defined(_POSIX_THREADS) && (_POSIX_THREADS > 0)) || \ + defined(__unix__) || defined(__APPLE__) || defined(__linux__) +// Free-threaded with pthread support - use pthread_rwlock_t +// This covers POSIX systems, including Linux, macOS, BSDs, Android, Cygwin, +// etc. +#define MAXMINDDB_USE_PTHREAD_LOCKS +#include +#include +#include +#else +// Free-threaded on unknown platform - fall back to GIL protection +// The extension will work but won't benefit from fine-grained locking +#define MAXMINDDB_USE_GIL_ONLY +#endif + +// Define the lock type based on strategy +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS +typedef SRWLOCK reader_rwlock_t; +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) +typedef pthread_rwlock_t reader_rwlock_t; +#else +// Dummy lock type for GIL-only mode typedef struct { + // Dummy member to satisfy MSVC, which doesn't allow empty structs. + char dummy; +} reader_rwlock_t; +#endif + +// ============================================================================= +// Type definitions +// ============================================================================= + +// clang-format off +typedef struct Reader_obj_struct { PyObject_HEAD /* no semicolon */ MMDB_s *mmdb; PyObject *closed; + reader_rwlock_t rwlock; } Reader_obj; typedef struct record record; @@ -74,6 +120,169 @@ static int ip_converter(PyObject *obj, struct sockaddr_storage *ip_address); #define UNUSED(x) UNUSED_##x #endif +// ============================================================================= +// Lock function implementations +// ============================================================================= + +static int reader_lock_init(reader_rwlock_t *lock) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + InitializeSRWLock(lock); + return 0; + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_init(lock, NULL); + if (err != 0) { + PyErr_Format(PyExc_RuntimeError, + "Failed to initialize read-write lock: %s", + strerror(err)); + return -1; + } + return 0; + +#else + // GIL-only mode - no-op + (void)lock; + return 0; +#endif +} + +static void reader_lock_destroy(reader_rwlock_t *lock) { +#ifdef MAXMINDDB_USE_PTHREAD_LOCKS + int err = pthread_rwlock_destroy(lock); + if (err == EBUSY) { + PyErr_WarnEx(PyExc_RuntimeWarning, + "Destroying MaxMind Reader lock while still in use " + "(bug: lock not properly released)", + 1); + } else if (err != 0) { + PyErr_WarnFormat(PyExc_RuntimeWarning, + 1, + "pthread_rwlock_destroy failed: %s", + strerror(err)); + } + +#else + // Windows SRWLOCK and GIL-only mode - no cleanup needed + (void)lock; +#endif +} + +static int reader_acquire_read_lock(Reader_obj *reader) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + AcquireSRWLockShared(&(reader->rwlock)); + return 0; + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_rdlock(&(reader->rwlock)); + if (err != 0) { + const char *msg; + switch (err) { + case EAGAIN: + msg = "Maximum number of read locks exceeded"; + break; + case EDEADLK: + msg = "Deadlock detected"; + break; + case EINVAL: + msg = "Invalid lock state"; + break; + default: + msg = strerror(err); + break; + } + PyErr_Format( + PyExc_RuntimeError, "Failed to acquire read lock: %s", msg); + return -1; + } + return 0; + +#else + // GIL-only mode - no-op + (void)reader; + return 0; +#endif +} + +static void reader_release_read_lock(Reader_obj *reader) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + ReleaseSRWLockShared(&(reader->rwlock)); + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_unlock(&(reader->rwlock)); + if (err != 0) { + PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "pthread_rwlock_unlock failed: %s (unbalanced lock/unlock?)", + strerror(err)); + } + +#else + // GIL-only mode - no-op + (void)reader; +#endif +} + +static int reader_acquire_write_lock(Reader_obj *reader) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + AcquireSRWLockExclusive(&(reader->rwlock)); + return 0; + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_wrlock(&(reader->rwlock)); + if (err != 0) { + const char *msg; + switch (err) { + case EAGAIN: + msg = "Maximum number of write locks exceeded"; + break; + case EDEADLK: + msg = "Deadlock detected"; + break; + case EINVAL: + msg = "Invalid lock state"; + break; + default: + msg = strerror(err); + break; + } + PyErr_Format( + PyExc_RuntimeError, "Failed to acquire write lock: %s", msg); + return -1; + } + return 0; + +#else + // GIL-only mode - no-op + (void)reader; + return 0; +#endif +} + +static void reader_release_write_lock(Reader_obj *reader) { +#ifdef MAXMINDDB_USE_WINDOWS_LOCKS + ReleaseSRWLockExclusive(&(reader->rwlock)); + +#elif defined(MAXMINDDB_USE_PTHREAD_LOCKS) + int err = pthread_rwlock_unlock(&(reader->rwlock)); + if (err != 0) { + PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "pthread_rwlock_unlock failed: %s (unbalanced lock/unlock?)", + strerror(err)); + } + +#else + // GIL-only mode - no-op + (void)reader; +#endif +} + +// ============================================================================= +// Reader implementation +// ============================================================================= + static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *filepath = NULL; int mode = 0; @@ -125,9 +334,16 @@ static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) { return -1; } + if (reader_lock_init(&mmdb_obj->rwlock) != 0) { + free(mmdb); + Py_XDECREF(filepath); + return -1; + } + int const status = MMDB_open(filename, MMDB_MODE_MMAP, mmdb); if (status != MMDB_SUCCESS) { + reader_lock_destroy(&mmdb_obj->rwlock); free(mmdb); PyErr_Format(MaxMindDB_error, "Error opening database file (%s). Is this a valid " @@ -166,13 +382,6 @@ static PyObject *Reader_get_with_prefix_len(PyObject *self, PyObject *args) { } static int get_record(PyObject *self, PyObject *args, PyObject **record) { - MMDB_s *mmdb = ((Reader_obj *)self)->mmdb; - if (mmdb == NULL) { - PyErr_SetString(PyExc_ValueError, - "Attempt to read from a closed MaxMind DB."); - return -1; - } - struct sockaddr_storage ip_address_ss = {0}; struct sockaddr *ip_address = (struct sockaddr *)&ip_address_ss; if (!PyArg_ParseTuple(args, "O&", ip_converter, &ip_address_ss)) { @@ -184,11 +393,26 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { return -1; } + Reader_obj *reader = (Reader_obj *)self; + + if (reader_acquire_read_lock(reader) != 0) { + return -1; + } + + MMDB_s *mmdb = reader->mmdb; + if (mmdb == NULL) { + reader_release_read_lock(reader); + PyErr_SetString(PyExc_ValueError, + "Attempt to read from a closed MaxMind DB."); + return -1; + } + int mmdb_error = MMDB_SUCCESS; MMDB_lookup_result_s result = MMDB_lookup_sockaddr(mmdb, ip_address, &mmdb_error); if (mmdb_error != MMDB_SUCCESS) { + reader_release_read_lock(reader); PyObject *exception; if (MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR == mmdb_error) { exception = PyExc_ValueError; @@ -213,6 +437,7 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { } if (!result.found_entry) { + reader_release_read_lock(reader); Py_INCREF(Py_None); *record = Py_None; return prefix_len; @@ -221,6 +446,7 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { MMDB_entry_data_list_s *entry_data_list = NULL; int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list); if (status != MMDB_SUCCESS) { + reader_release_read_lock(reader); char ipstr[INET6_ADDRSTRLEN] = {0}; if (format_sockaddr(ip_address, ipstr)) { PyErr_Format(MaxMindDB_error, @@ -236,6 +462,8 @@ static int get_record(PyObject *self, PyObject *args, PyObject **record) { *record = from_entry_data_list(&entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); + reader_release_read_lock(reader); + // from_entry_data_list will return NULL on errors. if (*record == NULL) { return -1; @@ -344,7 +572,12 @@ static bool format_sockaddr(struct sockaddr *sa, char *dst) { static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; + if (reader_acquire_read_lock(mmdb_obj) != 0) { + return NULL; + } + if (mmdb_obj->mmdb == NULL) { + reader_release_read_lock(mmdb_obj); PyErr_SetString(PyExc_IOError, "Attempt to read from a closed MaxMind DB."); return NULL; @@ -354,6 +587,7 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { int status = MMDB_get_metadata_as_entry_data_list(mmdb_obj->mmdb, &entry_data_list); if (status != MMDB_SUCCESS) { + reader_release_read_lock(mmdb_obj); PyErr_Format(MaxMindDB_error, "Error decoding metadata. %s", MMDB_strerror(status)); @@ -364,10 +598,13 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { PyObject *metadata_dict = from_entry_data_list(&entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); if (metadata_dict == NULL || !PyDict_Check(metadata_dict)) { + reader_release_read_lock(mmdb_obj); PyErr_SetString(MaxMindDB_error, "Error decoding metadata."); return NULL; } + reader_release_read_lock(mmdb_obj); + PyObject *args = PyTuple_New(0); if (args == NULL) { Py_DECREF(metadata_dict); @@ -384,6 +621,10 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) { static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; + if (reader_acquire_write_lock(mmdb_obj) != 0) { + return NULL; + } + if (mmdb_obj->mmdb != NULL) { MMDB_close(mmdb_obj->mmdb); free(mmdb_obj->mmdb); @@ -392,18 +633,27 @@ static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) { mmdb_obj->closed = Py_True; + reader_release_write_lock(mmdb_obj); + Py_RETURN_NONE; } static PyObject *Reader__enter__(PyObject *self, PyObject *UNUSED(args)) { Reader_obj *mmdb_obj = (Reader_obj *)self; + if (reader_acquire_read_lock(mmdb_obj) != 0) { + return NULL; + } + if (mmdb_obj->closed == Py_True) { + reader_release_read_lock(mmdb_obj); PyErr_SetString(PyExc_ValueError, "Attempt to reopen a closed MaxMind DB."); return NULL; } + reader_release_read_lock(mmdb_obj); + Py_INCREF(self); return (PyObject *)self; } @@ -419,17 +669,27 @@ static void Reader_dealloc(PyObject *self) { Reader_close(self, NULL); } + reader_lock_destroy(&obj->rwlock); + PyObject_Del(self); } static PyObject *Reader_iter(PyObject *obj) { Reader_obj *reader = (Reader_obj *)obj; + + if (reader_acquire_read_lock(reader) != 0) { + return NULL; + } + if (reader->closed == Py_True) { + reader_release_read_lock(reader); PyErr_SetString(PyExc_ValueError, "Attempt to iterate over a closed MaxMind DB."); return NULL; } + reader_release_read_lock(reader); + ReaderIter_obj *ri = PyObject_New(ReaderIter_obj, &ReaderIter_Type); if (ri == NULL) { return NULL; @@ -460,7 +720,13 @@ static bool is_ipv6(char ip[16]) { static PyObject *ReaderIter_next(PyObject *self) { ReaderIter_obj *ri = (ReaderIter_obj *)self; + + if (reader_acquire_read_lock(ri->reader) != 0) { + return NULL; + } + if (ri->reader->closed == Py_True) { + reader_release_read_lock(ri->reader); PyErr_SetString(PyExc_ValueError, "Attempt to iterate over a closed MaxMind DB."); return NULL; @@ -472,6 +738,7 @@ static PyObject *ReaderIter_next(PyObject *self) { switch (cur->type) { case MMDB_RECORD_TYPE_INVALID: + reader_release_read_lock(ri->reader); PyErr_SetString(MaxMindDB_error, "Invalid record when reading node"); free(cur); @@ -487,6 +754,7 @@ static PyObject *ReaderIter_next(PyObject *self) { int status = MMDB_read_node( ri->reader->mmdb, (uint32_t)cur->record, &node); if (status != MMDB_SUCCESS) { + reader_release_read_lock(ri->reader); const char *error = MMDB_strerror(status); PyErr_Format( MaxMindDB_error, "Error reading node: %s", error); @@ -495,6 +763,7 @@ static PyObject *ReaderIter_next(PyObject *self) { } struct record *left = calloc(1, sizeof(record)); if (left == NULL) { + reader_release_read_lock(ri->reader); free(cur); PyErr_NoMemory(); return NULL; @@ -508,6 +777,7 @@ static PyObject *ReaderIter_next(PyObject *self) { struct record *right = left->next = calloc(1, sizeof(record)); if (right == NULL) { + reader_release_read_lock(ri->reader); free(cur); free(left); PyErr_NoMemory(); @@ -532,6 +802,7 @@ static PyObject *ReaderIter_next(PyObject *self) { int status = MMDB_get_entry_data_list(&cur->entry, &entry_data_list); if (status != MMDB_SUCCESS) { + reader_release_read_lock(ri->reader); PyErr_Format( MaxMindDB_error, "Error looking up data while iterating over tree: %s", @@ -546,6 +817,7 @@ static PyObject *ReaderIter_next(PyObject *self) { PyObject *record = from_entry_data_list(&entry_data_list); MMDB_free_entry_data_list(original_entry_data_list); if (record == NULL) { + reader_release_read_lock(ri->reader); free(cur); return NULL; } @@ -567,6 +839,7 @@ static PyObject *ReaderIter_next(PyObject *self) { ip_length, cur->depth - ip_start * 8); if (network_tuple == NULL) { + reader_release_read_lock(ri->reader); Py_DECREF(record); free(cur); return NULL; @@ -574,6 +847,7 @@ static PyObject *ReaderIter_next(PyObject *self) { PyObject *args = PyTuple_Pack(1, network_tuple); Py_DECREF(network_tuple); if (args == NULL) { + reader_release_read_lock(ri->reader); Py_DECREF(record); free(cur); return NULL; @@ -582,6 +856,7 @@ static PyObject *ReaderIter_next(PyObject *self) { PyObject_CallObject(ipaddress_ip_network, args); Py_DECREF(args); if (network == NULL) { + reader_release_read_lock(ri->reader); Py_DECREF(record); free(cur); return NULL; @@ -591,10 +866,13 @@ static PyObject *ReaderIter_next(PyObject *self) { Py_DECREF(network); Py_DECREF(record); + reader_release_read_lock(ri->reader); + free(cur); return rv; } default: + reader_release_read_lock(ri->reader); PyErr_Format( MaxMindDB_error, "Unknown record type: %u", cur->type); free(cur); @@ -602,6 +880,7 @@ static PyObject *ReaderIter_next(PyObject *self) { } free(cur); } + reader_release_read_lock(ri->reader); return NULL; } @@ -979,6 +1258,11 @@ PyMODINIT_FUNC PyInit_extension(void) { return NULL; } +#ifndef MAXMINDDB_USE_GIL_ONLY + // Only declare module as GIL-free when we have proper locks available + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + Reader_Type.tp_new = PyType_GenericNew; if (PyType_Ready(&Reader_Type)) { return NULL; diff --git a/maxminddb/decoder.py b/maxminddb/decoder.py index 069c3c8..fc03958 100644 --- a/maxminddb/decoder.py +++ b/maxminddb/decoder.py @@ -1,7 +1,9 @@ """Decoder for the MaxMind DB data section.""" +from __future__ import annotations + import struct -from typing import ClassVar, Union, cast +from typing import TYPE_CHECKING, ClassVar, cast try: import mmap @@ -10,8 +12,10 @@ from maxminddb.errors import InvalidDatabaseError -from maxminddb.file import FileBuffer -from maxminddb.types import Record + +if TYPE_CHECKING: + from maxminddb.file import FileBuffer + from maxminddb.types import Record class Decoder: @@ -19,7 +23,7 @@ class Decoder: def __init__( self, - database_buffer: Union[FileBuffer, "mmap.mmap", bytes], + database_buffer: FileBuffer | mmap.mmap | bytes, pointer_base: int = 0, pointer_test: bool = False, # noqa: FBT001, FBT002 ) -> None: diff --git a/maxminddb/reader.py b/maxminddb/reader.py index a1a1509..9997dff 100644 --- a/maxminddb/reader.py +++ b/maxminddb/reader.py @@ -288,7 +288,10 @@ def _load_buffer( return filename def close(self) -> None: - """Close the MaxMind DB file and returns the resources to the system.""" + """Close the MaxMind DB file and returns the resources to the system. + + Calling this method while reads are in progress may cause exceptions. + """ with contextlib.suppress(AttributeError): self._buffer.close() # type: ignore[union-attr] diff --git a/maxminddb/types.py b/maxminddb/types.py index aefae65..4c0c478 100644 --- a/maxminddb/types.py +++ b/maxminddb/types.py @@ -1,14 +1,18 @@ """Types representing database records.""" -from typing import AnyStr, Union +from __future__ import annotations -Primitive = Union[AnyStr, bool, float, int] -Record = Union[Primitive, "RecordList", "RecordDict"] +from typing import AnyStr, TypeAlias +Primitive: TypeAlias = AnyStr | bool | float | int -class RecordList(list[Record]): + +class RecordList(list["Record"]): """RecordList is a type for lists in a database record.""" -class RecordDict(dict[str, Record]): +class RecordDict(dict[str, "Record"]): """RecordDict is a type for dicts in a database record.""" + + +Record: TypeAlias = Primitive | RecordList | RecordDict diff --git a/pyproject.toml b/pyproject.toml index f40efbf..8eaed8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Reader for the MaxMind DB format" authors = [ {name = "Gregory Oschwald", email = "goschwald@maxmind.com"}, ] -requires-python = ">=3.9" +requires-python = ">=3.10" readme = "README.rst" license = "Apache-2.0" classifiers = [ @@ -15,7 +15,6 @@ classifiers = [ "Intended Audience :: System Administrators", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -44,7 +43,6 @@ lint = [ requires = [ "setuptools>=77.0.3", "setuptools-scm", - "wheel", ] build-backend = "setuptools.build_meta" @@ -92,9 +90,15 @@ ignore = [ "setup.py" = ["ALL"] "tests/*" = ["ANN201", "D"] +[tool.pytest.ini_options] +# Use importlib mode to prevent pytest from adding the project root to sys.path. +# This ensures tests import from the tox virtualenv's installed package (which includes +# the compiled C extension) rather than directly from the source directory (which doesn't +# have the .so file). We still test the source code, just via the installed version. +addopts = "--import-mode=importlib" + [tool.tox] env_list = [ - "3.9", "3.10", "3.11", "3.12", @@ -108,6 +112,13 @@ skip_missing_interpreters = false dependency_groups = [ "dev", ] +pass_env = [ + "CFLAGS", + "LDFLAGS", + "MAXMINDDB_REQUIRE_EXTENSION", + "MAXMINDDB_USE_SYSTEM_LIBMAXMINDDB", + "MM_FORCE_EXT_TESTS", +] commands = [ ["pytest", "tests"], ] @@ -131,4 +142,3 @@ commands = [ "3.12" = ["3.12"] "3.11" = ["3.11"] "3.10" = ["3.10"] -"3.9" = ["3.9"] diff --git a/setup.py b/setup.py index 5b5a5d9..d36cbce 100644 --- a/setup.py +++ b/setup.py @@ -3,22 +3,9 @@ import sys from setuptools import Extension, setup +from setuptools.command.bdist_wheel import bdist_wheel from setuptools.command.build_ext import build_ext -from wheel.bdist_wheel import bdist_wheel - -# These were only added to setuptools in 59.0.1. -try: - from setuptools.errors import ( - CCompilerError, - DistutilsExecError, - DistutilsPlatformError, - ) -except ImportError: - from distutils.errors import ( - CCompilerError, - DistutilsExecError, - DistutilsPlatformError, - ) +from setuptools.errors import CCompilerError, ExecError, PlatformError cmdclass = {} PYPY = hasattr(sys, "pypy_version_info") @@ -74,7 +61,7 @@ # Cargo cult code for installing extension with pure Python fallback. # Taken from SQLAlchemy, but this same basic code exists in many modules. -ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError) +ext_errors = (CCompilerError, ExecError, PlatformError) class BuildFailed(Exception): @@ -88,7 +75,7 @@ class ve_build_ext(build_ext): def run(self) -> None: try: build_ext.run(self) - except DistutilsPlatformError: + except PlatformError: raise BuildFailed def build_extension(self, ext) -> None: diff --git a/tests/threading_test.py b/tests/threading_test.py new file mode 100644 index 0000000..d4afad4 --- /dev/null +++ b/tests/threading_test.py @@ -0,0 +1,274 @@ +"""Tests for thread-safety and free-threading support.""" + +from __future__ import annotations + +import threading +import time +import unittest +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from maxminddb.types import Record + +try: + import maxminddb.extension # noqa: F401 + + HAS_EXTENSION = True +except ImportError: + HAS_EXTENSION = False + +from maxminddb import open_database +from maxminddb.const import MODE_MMAP_EXT + + +@unittest.skipIf( + not HAS_EXTENSION, + "No C extension module found. Skipping threading tests", +) +class TestThreadSafety(unittest.TestCase): + """Test thread safety of the C extension.""" + + def test_concurrent_reads(self) -> None: + """Test multiple threads reading concurrently.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + results: list[Record | None] = [None] * 100 + errors: list[Exception] = [] + + def lookup(index: int, ip: str) -> None: + try: + results[index] = reader.get(ip) + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [] + for i in range(100): + ip = f"1.1.1.{(i % 32) + 1}" + t = threading.Thread(target=lookup, args=(i, ip)) + threads.append(t) + t.start() + + for t in threads: + t.join() + + reader.close() + + self.assertEqual(len(errors), 0, f"Errors during concurrent reads: {errors}") + # All lookups should have completed + self.assertNotIn(None, results) + + def test_read_during_close(self) -> None: + """Test that close is safe when reads are happening concurrently.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + errors: list[Exception] = [] + should_stop = threading.Event() + + def continuous_reader() -> None: + # Keep reading until signaled to stop or reader is closed + while not should_stop.is_set(): + try: + reader.get("1.1.1.1") + except ValueError as e: # noqa: PERF203 + # Expected once close() is called + if "closed MaxMind DB" not in str(e): + errors.append(e) + break + except Exception as e: # noqa: BLE001 + errors.append(e) + break + + # Start multiple readers + threads = [threading.Thread(target=continuous_reader) for _ in range(10)] + for t in threads: + t.start() + + # Let readers run for a bit + time.sleep(0.05) + + # Close while reads are happening + reader.close() + + # Signal threads to stop + should_stop.set() + + # Wait for all threads + for t in threads: + t.join(timeout=1.0) + + self.assertEqual(len(errors), 0, f"Errors during close test: {errors}") + + def test_read_after_close(self) -> None: + """Test that reads after close raise appropriate error.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + reader.close() + + with self.assertRaisesRegex( + ValueError, + "Attempt to read from a closed MaxMind DB", + ): + reader.get("1.1.1.1") + + def test_concurrent_reads_and_metadata(self) -> None: + """Test concurrent reads and metadata access.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + errors: list[Exception] = [] + results: list[bool] = [] + + def do_reads() -> None: + try: + for _ in range(50): + reader.get("1.1.1.1") + results.append(True) + except Exception as e: # noqa: BLE001 + errors.append(e) + + def do_metadata() -> None: + try: + for _ in range(50): + reader.metadata() + results.append(True) + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [] + for _ in range(5): + threads.append(threading.Thread(target=do_reads)) + threads.append(threading.Thread(target=do_metadata)) + + for t in threads: + t.start() + + for t in threads: + t.join() + + reader.close() + + self.assertEqual( + len(errors), 0, f"Errors during concurrent operations: {errors}" + ) + self.assertEqual(len(results), 10, "All threads should complete") + + def test_concurrent_iteration(self) -> None: + """Test that iteration is thread-safe.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + errors: list[Exception] = [] + counts: list[int] = [] + + def iterate() -> None: + try: + count = 0 + for _ in reader: + count += 1 + counts.append(count) + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [threading.Thread(target=iterate) for _ in range(10)] + + for t in threads: + t.start() + + for t in threads: + t.join() + + reader.close() + + self.assertEqual(len(errors), 0, f"Errors during iteration: {errors}") + # All threads should see the same number of entries + self.assertEqual(len(set(counts)), 1, "All threads should see same entry count") + + def test_stress_test(self) -> None: + """Stress test with many threads and operations.""" + reader = open_database( + "tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb", + MODE_MMAP_EXT, + ) + + errors: list[Exception] = [] + operations_completed = threading.Event() + + def random_operations() -> None: + try: + for i in range(100): + # Mix different operations + if i % 3 == 0: + reader.get("1.1.1.1") + elif i % 3 == 1: + reader.metadata() + else: + reader.get_with_prefix_len("1.1.1.2") + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [threading.Thread(target=random_operations) for _ in range(20)] + + for t in threads: + t.start() + + for t in threads: + t.join() + + operations_completed.set() + reader.close() + + self.assertEqual(len(errors), 0, f"Errors during stress test: {errors}") + + def test_multiple_readers_different_databases(self) -> None: + """Test multiple readers on different databases in parallel.""" + errors: list[Exception] = [] + + def use_reader(filename: str) -> None: + try: + reader = open_database(filename, MODE_MMAP_EXT) + for _ in range(50): + reader.get("1.1.1.1") + reader.close() + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [ + threading.Thread( + target=use_reader, + args=("tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb",), + ) + for _ in range(5) + ] + threads.extend( + [ + threading.Thread( + target=use_reader, + args=("tests/data/test-data/MaxMind-DB-test-ipv6-24.mmdb",), + ) + for _ in range(5) + ] + ) + + for t in threads: + t.start() + + for t in threads: + t.join() + + self.assertEqual(len(errors), 0, f"Errors with multiple readers: {errors}") + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 6026f7b..e2dca93 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.9" +requires-python = ">=3.10" [[package]] name = "colorama" @@ -97,12 +97,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/11/040983fad5132d85914c874a2836252bbc57832065548885b5bb5b0d4359/mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb", size = 13326683, upload-time = "2025-09-19T00:09:55.572Z" }, { url = "https://files.pythonhosted.org/packages/e9/ba/89b2901dd77414dd7a8c8729985832a5735053be15b744c18e4586e506ef/mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075", size = 13514749, upload-time = "2025-09-19T00:10:44.827Z" }, { url = "https://files.pythonhosted.org/packages/25/bc/cc98767cffd6b2928ba680f3e5bc969c4152bf7c2d83f92f5a504b92b0eb/mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf", size = 9982959, upload-time = "2025-09-19T00:10:37.344Z" }, - { url = "https://files.pythonhosted.org/packages/3f/a6/490ff491d8ecddf8ab91762d4f67635040202f76a44171420bcbe38ceee5/mypy-1.18.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:25a9c8fb67b00599f839cf472713f54249a62efd53a54b565eb61956a7e3296b", size = 12807230, upload-time = "2025-09-19T00:09:49.471Z" }, - { url = "https://files.pythonhosted.org/packages/eb/2e/60076fc829645d167ece9e80db9e8375648d210dab44cc98beb5b322a826/mypy-1.18.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2b9c7e284ee20e7598d6f42e13ca40b4928e6957ed6813d1ab6348aa3f47133", size = 11895666, upload-time = "2025-09-19T00:10:53.678Z" }, - { url = "https://files.pythonhosted.org/packages/97/4a/1e2880a2a5dda4dc8d9ecd1a7e7606bc0b0e14813637eeda40c38624e037/mypy-1.18.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6985ed057513e344e43a26cc1cd815c7a94602fb6a3130a34798625bc2f07b6", size = 12499608, upload-time = "2025-09-19T00:09:36.204Z" }, - { url = "https://files.pythonhosted.org/packages/00/81/a117f1b73a3015b076b20246b1f341c34a578ebd9662848c6b80ad5c4138/mypy-1.18.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22f27105f1525ec024b5c630c0b9f36d5c1cc4d447d61fe51ff4bd60633f47ac", size = 13244551, upload-time = "2025-09-19T00:10:17.531Z" }, - { url = "https://files.pythonhosted.org/packages/9b/61/b9f48e1714ce87c7bf0358eb93f60663740ebb08f9ea886ffc670cea7933/mypy-1.18.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:030c52d0ea8144e721e49b1f68391e39553d7451f0c3f8a7565b59e19fcb608b", size = 13491552, upload-time = "2025-09-19T00:10:13.753Z" }, - { url = "https://files.pythonhosted.org/packages/c9/66/b2c0af3b684fa80d1b27501a8bdd3d2daa467ea3992a8aa612f5ca17c2db/mypy-1.18.2-cp39-cp39-win_amd64.whl", hash = "sha256:aa5e07ac1a60a253445797e42b8b2963c9675563a94f11291ab40718b016a7a0", size = 9765635, upload-time = "2025-09-19T00:10:30.993Z" }, { url = "https://files.pythonhosted.org/packages/87/e3/be76d87158ebafa0309946c4a73831974d4d6ab4f4ef40c3b53a385a66fd/mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e", size = 2352367, upload-time = "2025-09-19T00:10:15.489Z" }, ]