From 56d89dfde641acf66bd7f4564e81f71478a1b970 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sat, 7 Jun 2025 14:28:22 +0200 Subject: [PATCH 1/3] Add a backwards-compatibility shim for PyInit --- peps/pep-0793.rst | 39 ++++++++++++++++++++ peps/pep-0793/examplemodule.c | 2 +- peps/pep-0793/shim.c | 68 +++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 peps/pep-0793/shim.c diff --git a/peps/pep-0793.rst b/peps/pep-0793.rst index 4dcd8da1dcc..e14d857c9cd 100644 --- a/peps/pep-0793.rst +++ b/peps/pep-0793.rst @@ -412,6 +412,9 @@ Here is a guide to convert an existing module to the new API, including some tricky edge cases. It should be moved to a HOWTO in the documentation. +This guide is meant for hand-written modules. For code generators and language +wrappers, the :ref:`pep793-shim` below may be more useful. + #. Scan your code for uses of ``PyModule_GetDef``. This function will return ``NULL`` for modules that use the new mechanism. Instead: @@ -493,6 +496,40 @@ Once your module no longer supports lower versions, delete the ``PyInit_`` function and any unused data. +Backwards compatibility shim +---------------------------- + +It is possible to write generic function that implements the existing export +hook (``PyInit_``) in terms of the API proposed here. + +The following implemntation can be copy-pasted; only the names +``PyInit_examplemodule`` (twice) and ``PyModExport_examplemodule`` need adjusting. + +When added to the :ref:`pep793-example` below and compiled with a +non-free-threaded build of this PEP's reference implementation, the resulting +extension is compatible with regular builds 3.9+ in addition to a +free-threading build of the reference implementation. +(The module must be named without a version tag, e.g. ``examplemodule.so``, +and be placed on ``sys.path``.) + +Full support for creating such modules will require backports of some new +API, and support in build/install tools. This is out of scope of this PEP. +(In particular, the demo “cheats” uses a subset Limited API 3.15 that +*happens to work* on 3.9; it *should* use Limited API 3.9 with backport +shims for new API like ``Py_mod_name``.) + +This implementation places a few additional requirements on the slots array: + +- Slots that correspond to ``PyModuleDef`` members must come first. +- A ``Py_mod_name`` slot is required. +- Any ``Py_mod_token`` must be set to ``&module_def_and_token``, defined here. + +It also passes ``NULL`` as *spec* to the ``PyModExport`` export hook. +A proper implementation would pass ``None`` instead, . + +.. literalinclude:: pep-0793/shim.c + :language: c + Security Implications ===================== @@ -507,6 +544,8 @@ In addition to regular reference docs, the :ref:`pep793-porting-notes` should be added as a new HOWTO. +.. _pep793-example: + Example ======= diff --git a/peps/pep-0793/examplemodule.c b/peps/pep-0793/examplemodule.c index c05ceb73040..654d282db88 100644 --- a/peps/pep-0793/examplemodule.c +++ b/peps/pep-0793/examplemodule.c @@ -49,9 +49,9 @@ PyDoc_STRVAR(examplemodule_doc, "Example extension."); static PyModuleDef_Slot examplemodule_slots[] = { {Py_mod_name, "examplemodule"}, {Py_mod_doc, (char*)examplemodule_doc}, - {Py_mod_exec, (void*)examplemodule_exec}, {Py_mod_methods, examplemodule_methods}, {Py_mod_state_size, (void*)sizeof(examplemodule_state)}, + {Py_mod_exec, (void*)examplemodule_exec}, {0} }; diff --git a/peps/pep-0793/shim.c b/peps/pep-0793/shim.c new file mode 100644 index 00000000000..e310f2060a4 --- /dev/null +++ b/peps/pep-0793/shim.c @@ -0,0 +1,68 @@ +#include // memset + +PyMODINIT_FUNC PyInit_examplemodule(void); + +static PyModuleDef module_def_and_token; + +PyMODINIT_FUNC +PyInit_examplemodule(void) +{ + PyModuleDef_Slot *slot = PyModExport_examplemodule(NULL); + + if (module_def_and_token.m_name) { + // Take care to only set up the static PyModuleDef once. + // (PyModExport might theoretically return different data each time.) + return PyModuleDef_Init(&module_def_and_token); + } + int copying_slots = 1; + for (/* slot set above */; slot->slot; slot++) { + switch (slot->slot) { + // Set PyModuleDef members from slots. These slots must come first. +# define COPYSLOT_CASE(SLOT, MEMBER, TYPE) \ + case SLOT: \ + if (!copying_slots) { \ + PyErr_SetString(PyExc_SystemError, \ + #SLOT " must be specified earlier"); \ + goto error; \ + } \ + module_def_and_token.MEMBER = (TYPE)(slot->value); \ + break; \ + ///////////////////////////////////////////////////////////////// + COPYSLOT_CASE(Py_mod_name, m_name, char*) + COPYSLOT_CASE(Py_mod_doc, m_doc, char*) + COPYSLOT_CASE(Py_mod_state_size, m_size, Py_ssize_t) + COPYSLOT_CASE(Py_mod_methods, m_methods, PyMethodDef*) + COPYSLOT_CASE(Py_mod_state_traverse, m_traverse, traverseproc) + COPYSLOT_CASE(Py_mod_state_clear, m_clear, inquiry) + COPYSLOT_CASE(Py_mod_state_free, m_free, freefunc) + case Py_mod_token: + // With PyInit_, the PyModuleDef is used as the token. + if (slot->value != &module_def_and_token) { + PyErr_SetString(PyExc_SystemError, + "Py_mod_token must be set to " + "&module_def_and_token"); + goto error; + } + break; + default: + // The remaining slots become m_slots in the def. + // (`slot` now points to the "rest" of the original + // zero-terminated array.) + if (copying_slots) { + module_def_and_token.m_slots = slot; + } + copying_slots = 0; + break; + } + } + if (!module_def_and_token.m_name) { + // This function needs m_name as the "is initialized" marker. + PyErr_SetString(PyExc_SystemError, "Py_mod_name slot is required"); + goto error; + } + return PyModuleDef_Init(&module_def_and_token); + +error: + memset(&module_def_and_token, 0, sizeof(module_def_and_token)); + return NULL; +} From 8a2e5aae39ef3b62c38e800488082a3c8b151386 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sat, 7 Jun 2025 16:54:33 +0200 Subject: [PATCH 2/3] Soft-deprecate the old API, and make PyType_GetModuleByDef take token --- peps/pep-0387.rst | 2 + peps/pep-0793.rst | 103 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 86 insertions(+), 19 deletions(-) diff --git a/peps/pep-0387.rst b/peps/pep-0387.rst index e5132ba0f9a..f11d4c72ddd 100644 --- a/peps/pep-0387.rst +++ b/peps/pep-0387.rst @@ -114,6 +114,8 @@ Basic policy for backwards compatibility platforms). +.. _pep387-soft-deprecation: + Soft Deprecation ================ diff --git a/peps/pep-0793.rst b/peps/pep-0793.rst index e14d857c9cd..d99ab48a25a 100644 --- a/peps/pep-0793.rst +++ b/peps/pep-0793.rst @@ -26,6 +26,10 @@ To make this viable, we also specify new module slot types to replace We also add an API for defining modules from slots dynamically. +The existing API (``PyInit_*``) is soft-deprecated. +(That is: it will continue to work without warnings, and it'll fe fully +documented and supported, but we plan to not add any new features to it.) + Background & Motivation ======================= @@ -147,6 +151,27 @@ Unlike types, the import mechanism often has a pointer that's known to be suitable as a token value; in these cases it can provide a default token. Thus, module tokens do not need a variant of the inelegant ``Py_TP_USE_SPEC``. +To help extensions that straddle Python versions, ``PyModuleDef`` addresses +are used as default tokens, and where it's reasonable, they are made +interchangeable with tokens. + + +Soft-deprecating the existing export hook +----------------------------------------- + +The only reason for authors of *existing* extensions to switch to the +API proposed here is that it allows a single module for both free-threaded +and non-free-threaded builds. +It is important that Python *allows* that, but for many existing modules, +it is nowhere near worth losing compatibility with 3.14 and lower versions. + +It is much too early to plan deprecation of the old API. + +Instead, this PEP proposes to stop adding new features to the ``PyInit_*`` +scheme. +After all, the perfect time for extension authors to switch is when they want +to modify module initialization anyway. + Specification ============= @@ -264,6 +289,10 @@ If specified, using a new ``Py_mod_token`` slot, the module token must: (Typically, it should point to a static constant.) +When the address of a ``PyModuleDef`` is used as as a module's token, +the module should behave as if it was created from that ``PyModuleDef``. +In particular, the module state must have matching layout and semantics. + Modules created using the ``PyModule_FromSlotsAndSpec`` or the ``PyModExport_`` export hook can use a new ``Py_mod_token`` slot to set the token. @@ -288,8 +317,15 @@ will return 0 on success and -1 on failure: int PyModule_GetToken(PyObject *, void **token_p) A new ``PyType_GetModuleByToken`` function will be added, with a signature -like ``PyType_GetModuleByDef`` but a ``void *token`` argument, -and the same behaviour except matching tokens, rather than only defs. +like the existing ``PyType_GetModuleByDef`` but a ``void *token`` argument, +and the same behaviour except matching tokens rather than only defs. + +For easier backwards compatibility, the existing ``PyType_GetModuleByDef`` +will be changed to work exactly like ``PyType_GetModuleByToken`` -- that is, +it will allow a token (cast to a ``PyModuleDef *`` pointer) as the +*def* argument. +(The ``PyModule_GetDef`` function will not get a similar change, as users may +access members of its result.) New slots @@ -333,6 +369,14 @@ via a pointer; the function will return 0 on success and -1 on failure: int PyModule_GetStateSize(PyObject *, Py_ssize_t *result); +Soft-deprecating the existing export hook +----------------------------------------- + +The ``PyInit_*`` export hook will be +:ref:`soft-deprecated `. + + + .. _pep793-api-summary: New API summary @@ -383,9 +427,14 @@ If an existing module is ported to use the new mechanism, then We claim that how a module was defined is an implementation detail of that module, so this should not be considered a breaking change. -Similarly, ``PyType_GetModuleByDef`` will not match modules that are not -defined using a *def*. -The new ``PyType_GetModuleByToken`` function may be used instead. +Similarly, the ``PyType_GetModuleByDef`` function may stop matching modules +whose definition changed. Module authors may avoid this by explicitly +explicitly setting a *def* as the *token*. + +``PyType_GetModuleByDef`` will now accept a module token as the *def* argument. +We specify a suitable restriction on using ``PyModuleDef`` addresses as tokens, +and non-``PyModuleDef`` pointers were previously invalid input, +so this is not a backwards-compatibility issue. The ``Py_mod_create`` function may now be called with ``NULL`` for the second argument. @@ -428,11 +477,15 @@ wrappers, the :ref:`pep793-shim` below may be more useful. Later in this guide, you'll set the token to *be* the existing ``PyModuleDef`` structure. -#. Scan your code for uses of ``PyType_GetModuleByDef``, and replace them by - ``PyType_GetModuleByToken``. +#. Optionally, scan your code for uses of ``PyType_GetModuleByDef``, + and replace them by ``PyType_GetModuleByToken``. Later in this guide, you'll set the token to *be* the existing ``PyModuleDef`` structure. + (You may skip this step if targetting Python versions that don't expose + ``PyType_GetModuleByToken``, since ``PyType_GetModuleByDef`` is + backwards-compatible. ) + #. Look at the function identified by ``Py_mod_create``, if any. Make sure that it does not use its second argument (``PyModuleDef``), as it will be called with ``NULL``. @@ -467,18 +520,17 @@ wrappers, the :ref:`pep793-shim` below may be more useful. }; #. If you switched from ``PyModule_GetDef`` to ``PyModule_GetToken``, - and/or from ``PyType_GetModuleByDef`` to ``PyType_GetModuleByToken``, + and/or if you use ``PyType_GetModuleByDef`` or ``PyType_GetModuleByToken``, add a ``Py_mod_token`` slot pointing to the existing ``PyModuleDef`` struct: .. code-block:: c static PyModuleDef_Slot module_slots[] = { // ... (keep existing slots here) - {Py_mod_token, your_module_def}, + {Py_mod_token, &your_module_def}, {0} }; - #. Add a new export hook. .. code-block:: c @@ -492,31 +544,43 @@ wrappers, the :ref:`pep793-shim` below may be more useful. } The new export hook will be used on Python 3.15 and above. -Once your module no longer supports lower versions, delete the ``PyInit_`` -function and any unused data. +Once your module no longer supports lower versions: +#. Delete the ``PyInit_`` function. + +#. If the existing ``PyModuleDef`` struct is used *only* for ``Py_mod_token`` + and/or ``PyType_GetModuleByToken``, you may remove the ``Py_mod_token`` + line and replace ``&your_module_def`` by ``module_slots`` everywhere else. + +#. Delete any unused data. + The ``PyModuleDef`` struct and the original slots array are likely to be + unused. + + +.. _pep793-shim: Backwards compatibility shim ---------------------------- -It is possible to write generic function that implements the existing export +It is possible to write generic function that implements the “old” export hook (``PyInit_``) in terms of the API proposed here. -The following implemntation can be copy-pasted; only the names -``PyInit_examplemodule`` (twice) and ``PyModExport_examplemodule`` need adjusting. +The following implementation can be copied and pasted to a project; only the +names ``PyInit_examplemodule`` (twice) and ``PyModExport_examplemodule`` should +need adjusting. When added to the :ref:`pep793-example` below and compiled with a non-free-threaded build of this PEP's reference implementation, the resulting -extension is compatible with regular builds 3.9+ in addition to a +extension is compatible with non-free-threading 3.9+ builds, in addition to a free-threading build of the reference implementation. (The module must be named without a version tag, e.g. ``examplemodule.so``, and be placed on ``sys.path``.) Full support for creating such modules will require backports of some new API, and support in build/install tools. This is out of scope of this PEP. -(In particular, the demo “cheats” uses a subset Limited API 3.15 that -*happens to work* on 3.9; it *should* use Limited API 3.9 with backport -shims for new API like ``Py_mod_name``.) +(In particular, the demo “cheats” by using a subset of Limited API 3.15 that +*happens to work* on 3.9; a proper implementation would use Limited API 3.9 +with backport shims for new API like ``Py_mod_name``.) This implementation places a few additional requirements on the slots array: @@ -589,6 +653,7 @@ Possible Future Directions These ideas are out of scope for *this* proposal. + Improving slots in general -------------------------- From 1db9be2fb156d13043d80cacc962dae3900ade68 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 11 Jun 2025 12:35:52 +0200 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- peps/pep-0793.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/peps/pep-0793.rst b/peps/pep-0793.rst index d99ab48a25a..562278f0582 100644 --- a/peps/pep-0793.rst +++ b/peps/pep-0793.rst @@ -27,7 +27,7 @@ To make this viable, we also specify new module slot types to replace We also add an API for defining modules from slots dynamically. The existing API (``PyInit_*``) is soft-deprecated. -(That is: it will continue to work without warnings, and it'll fe fully +(That is: it will continue to work without warnings, and it'll be fully documented and supported, but we plan to not add any new features to it.) @@ -289,7 +289,7 @@ If specified, using a new ``Py_mod_token`` slot, the module token must: (Typically, it should point to a static constant.) -When the address of a ``PyModuleDef`` is used as as a module's token, +When the address of a ``PyModuleDef`` is used as a module's token, the module should behave as if it was created from that ``PyModuleDef``. In particular, the module state must have matching layout and semantics. @@ -429,7 +429,7 @@ module, so this should not be considered a breaking change. Similarly, the ``PyType_GetModuleByDef`` function may stop matching modules whose definition changed. Module authors may avoid this by explicitly -explicitly setting a *def* as the *token*. +setting a *def* as the *token*. ``PyType_GetModuleByDef`` will now accept a module token as the *def* argument. We specify a suitable restriction on using ``PyModuleDef`` addresses as tokens, @@ -478,13 +478,13 @@ wrappers, the :ref:`pep793-shim` below may be more useful. ``PyModuleDef`` structure. #. Optionally, scan your code for uses of ``PyType_GetModuleByDef``, - and replace them by ``PyType_GetModuleByToken``. + and replace them with ``PyType_GetModuleByToken``. Later in this guide, you'll set the token to *be* the existing ``PyModuleDef`` structure. (You may skip this step if targetting Python versions that don't expose ``PyType_GetModuleByToken``, since ``PyType_GetModuleByDef`` is - backwards-compatible. ) + backwards-compatible.) #. Look at the function identified by ``Py_mod_create``, if any. Make sure that it does not use its second argument (``PyModuleDef``), @@ -550,7 +550,7 @@ Once your module no longer supports lower versions: #. If the existing ``PyModuleDef`` struct is used *only* for ``Py_mod_token`` and/or ``PyType_GetModuleByToken``, you may remove the ``Py_mod_token`` - line and replace ``&your_module_def`` by ``module_slots`` everywhere else. + line and replace ``&your_module_def`` with ``module_slots`` everywhere else. #. Delete any unused data. The ``PyModuleDef`` struct and the original slots array are likely to be @@ -562,7 +562,7 @@ Once your module no longer supports lower versions: Backwards compatibility shim ---------------------------- -It is possible to write generic function that implements the “old” export +It is possible to write a generic function that implements the “old” export hook (``PyInit_``) in terms of the API proposed here. The following implementation can be copied and pasted to a project; only the @@ -589,7 +589,7 @@ This implementation places a few additional requirements on the slots array: - Any ``Py_mod_token`` must be set to ``&module_def_and_token``, defined here. It also passes ``NULL`` as *spec* to the ``PyModExport`` export hook. -A proper implementation would pass ``None`` instead, . +A proper implementation would pass ``None`` instead. .. literalinclude:: pep-0793/shim.c :language: c @@ -653,7 +653,6 @@ Possible Future Directions These ideas are out of scope for *this* proposal. - Improving slots in general --------------------------