From b9cd77a0b36aafa4c409ceadc200cd7c1c9e5d9a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Oct 2025 14:46:45 +0200 Subject: [PATCH 1/3] Updates based on discussion - Lose the *spec* argument - Add *const* to arguments we don't change - Add a section on tokens --- peps/pep-0793.rst | 102 +++++++++++++++++++++++++------ peps/pep-0793/examplemodule.c | 2 +- peps/pep-0793/getmodulebytoken.c | 97 +++++++++++++++++++++++++++++ peps/pep-0793/shim.c | 2 +- 4 files changed, 184 insertions(+), 19 deletions(-) create mode 100644 peps/pep-0793/getmodulebytoken.c diff --git a/peps/pep-0793.rst b/peps/pep-0793.rst index 356cbca0066..97c12e06acd 100644 --- a/peps/pep-0793.rst +++ b/peps/pep-0793.rst @@ -184,7 +184,7 @@ like this: .. code-block:: c - PyModuleDef_Slot *PyModExport_(PyObject *spec); + PyModuleDef_Slot *PyModExport_(void); where ```` is the name of the module. For non-ASCII names, it will instead look for ``PyModExportU_``, @@ -194,12 +194,7 @@ with ```` encoded as for existing ``PyInitU_*`` hooks If not found, the import will continue as in previous Python versions (that is, by looking up a ``PyInit_*`` or ``PyInitU_*`` function). -If found, Python will call the hook with the appropriate -``importlib.machinery.ModuleSpec`` object as *spec*. -To support duck-typing, extensions should not type-check this object, and -if possible, implement fallbacks for any missing attributes. -(The argument is mainly meant for introspection, testing, or use with -specialized loaders.) +If found, Python will call the hook with no arguments. On failure, the export hook must return NULL with an exception set. This will cause the import to fail. @@ -225,13 +220,17 @@ A new function will be added to create a module from an array of slots: .. code-block:: c - PyObject *PyModule_FromSlotsAndSpec(PyModuleDef_Slot *slots, PyObject *spec) + PyObject *PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) The *slots* argument must point to an array of ``PyModuleDef_Slot`` structures, terminated by a slot with ``slot=0`` (typically written as ``{0}`` in C). There are no required slots, though *slots* must not be ``NULL``. It follows that minimal input contains only the terminator slot. +.. note:: + + If :pep:`803` is accepted, the ``Py_mod_abi`` slot will be mandatory. + The *spec* argument is a duck-typed ModuleSpec-like object, meaning that any attributes defined for ``importlib.machinery.ModuleSpec`` have matching semantics. @@ -274,12 +273,23 @@ For modules created from a *def*, calling this is equivalent to calling ``PyModule_ExecDef(module, PyModule_GetDef(module))``. +.. _pep793-token: + Tokens ------ Module objects will optionally store a “token”: a ``void*`` pointer similar to ``Py_tp_token`` for types. +.. note:: + + This specialized functionality replace the ``PyType_GetModuleByDef`` + function; users that don't need ``PyType_GetModuleByDef`` will most likely + not need tokens either. + + This section contains the technical specification; + see :ref:`pep793-using-token` for example usage. + If specified, using a new ``Py_mod_token`` slot, the module token must: - outlive the module, so it's not reused for something else while the module @@ -317,7 +327,7 @@ will return 0 on success and -1 on failure: int PyModule_GetToken(PyObject *, void **token_p) A new ``PyType_GetModuleByToken`` function will be added, with a signature -like the existing ``PyType_GetModuleByDef`` but a ``void *token`` argument, +like the existing ``PyType_GetModuleByDef`` but a ``const void *token`` argument, and the same behaviour except matching tokens rather than only defs, and returning a strong reference. @@ -388,17 +398,17 @@ Python will load a new module export hook, with two variants: .. code-block:: c - PyModuleDef_Slot *PyModExport_(PyObject *spec); - PyModuleDef_Slot *PyModExportU_(PyObject *spec); + PyModuleDef_Slot *PyModExport_(void); + PyModuleDef_Slot *PyModExportU_(void); The following functions will be added: .. code-block:: c - PyObject *PyModule_FromSlotsAndSpec(PyModuleDef_Slot *, PyObject *spec) + PyObject *PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *, PyObject *spec) int PyModule_Exec(PyObject *) int PyModule_GetToken(PyObject *, void**) - PyObject *PyType_GetModuleByToken(PyTypeObject *type, void *token) + PyObject *PyType_GetModuleByToken(PyTypeObject *type, const void *token) int PyModule_GetStateSize(PyObject *, Py_ssize_t *result); A new macro will be added: @@ -541,7 +551,7 @@ wrappers, the :ref:`pep793-shim` below may be more useful. PyMODEXPORT_FUNC PyModExport_examplemodule(PyObject); PyMODEXPORT_FUNC - PyModExport_examplemodule(PyObject *spec) + PyModExport_examplemodule(void) { return module_slots; } @@ -591,9 +601,6 @@ This implementation places a few additional requirements on the slots array: - A ``Py_mod_name`` slot is required. - Any ``Py_mod_token`` must be set to ``&module_def_and_token``, defined here. -It also passes ``NULL`` as *spec* to the ``PyModExport`` export hook. -A proper implementation would pass ``None`` instead. - .. literalinclude:: pep-0793/shim.c :language: c @@ -611,6 +618,40 @@ In addition to regular reference docs, the :ref:`pep793-porting-notes` should be added as a new HOWTO. +.. _pep793-using-token: + +Using the module token +---------------------- + +One feature that this PEP talks about a lot, relative to its importance, +is :ref:`module tokens `. +This is a mechanism that allows an extension to check that a given module +object “belongs” to that extension -- that is, that the module was created from +a particular slots array. +(*Usually* the token will be the “defining” slots array; the possibility to +use something else is meant for advanced use cases of dynamic module creation.) + +We envision that the most common use case for modules is using +``PyType_GetModuleByToken`` in cases like the following method, +which would be specified with ``METH_NOARGS`` on a type created with +``PyType_FromModuleAndSpec``, and needs to access “its” module state to +retrieve an exception type to raise. +It cannot use ``PyModule_GetState(Py_TYPE(self))`` because ``Py_TYPE(self)`` +could be a *subclass* defined in a different module. + +.. literalinclude:: pep-0793/getmodulebytoken.c + :language: c + :start-after: /// example-start + :end-before: /// example-end + +(In this particular case, lookup could be avoided using +:external+python:c:type:`PyCMethod` and the defining class, but such options +aren't always available.) + +We'll add complete documentation for the feature, but it will not be mentioned +too prominently in HOWTOs and guides. + + .. _pep793-example: Example @@ -651,6 +692,33 @@ A function also allows the extension to introspect its environment in a limited way -- for example, to tailor the returned data to the current Python version. +Changing ``PyModuleDef`` to not be ``PyObject`` +----------------------------------------------- + +It is possible to change ``PyModuleDef`` to no longer include the ``PyObject`` +header, and continue using the current ``PyInit_*`` hook. +There are several issues with this approach: + +- The import machinery would need to examine bit-patterns in the objects to + distinguish between different memory layouts: + + - the “old” ``PyObject``-based ``PyModuleDef``, returned by current ``abi3`` + extensions, + - the new ``PyModuleDef``, + - ``PyObject``-based module objects, for single-phase initialization. + + This is fragile, and places constraints on future changes to ``PyObject``: + the memory layouts need to stay *distinguishable* until both single-phase + initialization and the current Stable ABI are no longer supported. + + +- ``PyModuleDef_Init`` is documented to “Ensure a module definition is a + properly initialized Python object that correctly reports its type and + a reference count.” + This would need to change without warning, breaking any user code that treats + ``PyModuleDef``\ s as Python objects. + + Possible Future Directions ========================== diff --git a/peps/pep-0793/examplemodule.c b/peps/pep-0793/examplemodule.c index 654d282db88..e0eb2475f15 100644 --- a/peps/pep-0793/examplemodule.c +++ b/peps/pep-0793/examplemodule.c @@ -59,7 +59,7 @@ static PyModuleDef_Slot examplemodule_slots[] = { PyMODEXPORT_FUNC PyModExport_examplemodule(PyObject *); PyMODEXPORT_FUNC -PyModExport_examplemodule(PyObject *spec) +PyModExport_examplemodule(void) { return examplemodule_slots; } diff --git a/peps/pep-0793/getmodulebytoken.c b/peps/pep-0793/getmodulebytoken.c new file mode 100644 index 00000000000..be1b91bbdbd --- /dev/null +++ b/peps/pep-0793/getmodulebytoken.c @@ -0,0 +1,97 @@ +#include + +typedef struct { + PyObject *exception; + PyTypeObject *type; +} SpamState; + +static const PyModuleDef_Slot spam_slots[]; + +/// example-start +static PyObject * +spamtype_raise_exc(PyObject *self, PyObject *unused) +{ + PyObject *module = PyType_GetModuleByToken(Py_TYPE(self), spam_slots); + if (!module) { + return NULL; + } + SpamState *state = PyModule_GetState(module); + if (!state) { + return NULL; + } + PyErr_SetString(state->exception, "failed!"); + return NULL; +} +/// example-end + +static PyType_Spec spamtype_spec = { + .name = "spam.SpamType", + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = (PyType_Slot[]) { + { + Py_tp_methods, + (PyMethodDef[]) { + {"raise_exc", spamtype_raise_exc, METH_NOARGS}, + {0}, + } + }, + {0}, + }, +}; + +static int +spam_exec(PyObject *self) +{ + SpamState *state = PyModule_GetState(self); + state->exception = PyErr_NewException("spam.SpamException", NULL, NULL); + if (!state->exception) { + return -1; + } + state->type = (PyTypeObject*)PyType_FromModuleAndSpec(self, &spamtype_spec, NULL); + if (!state->type) { + return -1; + } + if (PyModule_AddType(self, state->type) < 0) { + return -1; + } + return 0; +} + +static int +spam_traverse(PyObject *self, visitproc visit, void *arg) +{ + SpamState *state = PyModule_GetState(self); + Py_VISIT(state->exception); + Py_VISIT(state->type); + return 0; +} + +static int +spam_clear(PyObject *self) +{ + SpamState *state = PyModule_GetState(self); + Py_CLEAR(state->exception); + Py_CLEAR(state->type); + return 0; +} + +static void +spam_free(PyObject *self) +{ + spam_clear(self); +} + +static const PyModuleDef_Slot spam_slots[] = { + {Py_mod_exec, spam_exec}, + {Py_mod_state_size, (void*)sizeof(SpamState)}, + {Py_mod_state_traverse, spam_traverse}, + {Py_mod_state_clear, spam_clear}, + {Py_mod_state_free, spam_free}, + {0}, +}; + +PyMODEXPORT_FUNC +PyModExport_spam(void) +{ + return spam_slots; +} diff --git a/peps/pep-0793/shim.c b/peps/pep-0793/shim.c index e310f2060a4..4e6b67e805e 100644 --- a/peps/pep-0793/shim.c +++ b/peps/pep-0793/shim.c @@ -7,7 +7,7 @@ static PyModuleDef module_def_and_token; PyMODINIT_FUNC PyInit_examplemodule(void) { - PyModuleDef_Slot *slot = PyModExport_examplemodule(NULL); + PyModuleDef_Slot *slot = PyModExport_examplemodule(); if (module_def_and_token.m_name) { // Take care to only set up the static PyModuleDef once. From 364ae9c8fb2e3b868ce90b72149fdd89380fefa6 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Oct 2025 15:21:54 +0200 Subject: [PATCH 2/3] Put token use in the main example --- peps/pep-0793.rst | 46 +++------------ peps/pep-0793/examplemodule.c | 62 +++++++++++++++++++- peps/pep-0793/getmodulebytoken.c | 97 -------------------------------- 3 files changed, 66 insertions(+), 139 deletions(-) delete mode 100644 peps/pep-0793/getmodulebytoken.c diff --git a/peps/pep-0793.rst b/peps/pep-0793.rst index 97c12e06acd..5caee64aee5 100644 --- a/peps/pep-0793.rst +++ b/peps/pep-0793.rst @@ -283,12 +283,13 @@ similar to ``Py_tp_token`` for types. .. note:: - This specialized functionality replace the ``PyType_GetModuleByDef`` - function; users that don't need ``PyType_GetModuleByDef`` will most likely - not need tokens either. + This is specialized functionality meant replace the + ``PyType_GetModuleByDef`` function; users that don't need + ``PyType_GetModuleByDef`` will most likely not need tokens either. This section contains the technical specification; - see :ref:`pep793-using-token` for example usage. + for an example of intended usage, see ``exampletype_repr`` in the + :ref:`Example section `. If specified, using a new ``Py_mod_token`` slot, the module token must: @@ -297,7 +298,8 @@ If specified, using a new ``Py_mod_token`` slot, the module token must: - "belong" to the extension module where the module lives, so it will not clash with other extension modules. -(Typically, it should point to a static constant.) +(Typically, it should be the slots array or ``PyModuleDef`` that a module is +created from, or another static constant for dynamically created modules.) When the address of a ``PyModuleDef`` is used as a module's token, the module should behave as if it was created from that ``PyModuleDef``. @@ -618,40 +620,6 @@ In addition to regular reference docs, the :ref:`pep793-porting-notes` should be added as a new HOWTO. -.. _pep793-using-token: - -Using the module token ----------------------- - -One feature that this PEP talks about a lot, relative to its importance, -is :ref:`module tokens `. -This is a mechanism that allows an extension to check that a given module -object “belongs” to that extension -- that is, that the module was created from -a particular slots array. -(*Usually* the token will be the “defining” slots array; the possibility to -use something else is meant for advanced use cases of dynamic module creation.) - -We envision that the most common use case for modules is using -``PyType_GetModuleByToken`` in cases like the following method, -which would be specified with ``METH_NOARGS`` on a type created with -``PyType_FromModuleAndSpec``, and needs to access “its” module state to -retrieve an exception type to raise. -It cannot use ``PyModule_GetState(Py_TYPE(self))`` because ``Py_TYPE(self)`` -could be a *subclass* defined in a different module. - -.. literalinclude:: pep-0793/getmodulebytoken.c - :language: c - :start-after: /// example-start - :end-before: /// example-end - -(In this particular case, lookup could be avoided using -:external+python:c:type:`PyCMethod` and the defining class, but such options -aren't always available.) - -We'll add complete documentation for the feature, but it will not be mentioned -too prominently in HOWTOs and guides. - - .. _pep793-example: Example diff --git a/peps/pep-0793/examplemodule.c b/peps/pep-0793/examplemodule.c index e0eb2475f15..9031b27646a 100644 --- a/peps/pep-0793/examplemodule.c +++ b/peps/pep-0793/examplemodule.c @@ -1,6 +1,10 @@ /* -Example module with C-level module-global state, and a simple function to -update and query it. +Example module with C-level module-global state, and + +- a simple function that updates and queries the state +- a class wihose repr() queries the same module state (for an example of + PyType_GetModuleByToken) + Once compiled and renamed to not include a version tag (for example examplemodule.so on Linux), this will run succesfully on both regular and free-threaded builds. @@ -13,6 +17,13 @@ print(examplemodule.increment_value()) # 1 print(examplemodule.increment_value()) # 2 print(examplemodule.increment_value()) # 3 + +class Subclass(examplemodule.ExampleType): + pass + +instance = Subclass() +print(instance) # + */ // Avoid CPython-version-specific ABI (inline functions & macros): @@ -24,6 +35,10 @@ typedef struct { int value; } examplemodule_state; +static PyModuleDef_Slot examplemodule_slots[]; + +// increment_value function + static PyObject * increment_value(PyObject *module, PyObject *_ignored) { @@ -37,10 +52,51 @@ static PyMethodDef examplemodule_methods[] = { {NULL} }; +// ExampleType + +static PyObject * +exampletype_repr(PyObject *self) +{ + /* To get module state, we cannot use PyModule_GetState(Py_TYPE(self)), + * since Py_TYPE(self) might be a subclass defined in an unrelated module. + * So, use PyType_GetModuleByToken. + */ + PyObject *module = PyType_GetModuleByToken( + Py_TYPE(self), examplemodule_slots); + if (!module) { + return NULL; + } + examplemodule_state *state = PyModule_GetState(module); + if (!state) { + return NULL; + } + return PyUnicode_FromFormat("<%T object; module value = %d>", + self, state->value); +} + +static PyType_Spec exampletype_spec = { + .name = "examplemodule.ExampleType", + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = (PyType_Slot[]) { + {Py_tp_repr, exampletype_repr}, + {0}, + }, +}; + +// Module + static int examplemodule_exec(PyObject *module) { examplemodule_state *state = PyModule_GetState(module); state->value = -1; + PyTypeObject *type = (PyTypeObject*)PyType_FromModuleAndSpec( + module, &exampletype_spec, NULL); + if (!type) { + return -1; + } + if (PyModule_AddType(module, type) < 0) { + return -1; + } return 0; } @@ -56,7 +112,7 @@ static PyModuleDef_Slot examplemodule_slots[] = { }; // Avoid "implicit declaration of function" warning: -PyMODEXPORT_FUNC PyModExport_examplemodule(PyObject *); +PyMODEXPORT_FUNC PyModExport_examplemodule(void); PyMODEXPORT_FUNC PyModExport_examplemodule(void) diff --git a/peps/pep-0793/getmodulebytoken.c b/peps/pep-0793/getmodulebytoken.c deleted file mode 100644 index be1b91bbdbd..00000000000 --- a/peps/pep-0793/getmodulebytoken.c +++ /dev/null @@ -1,97 +0,0 @@ -#include - -typedef struct { - PyObject *exception; - PyTypeObject *type; -} SpamState; - -static const PyModuleDef_Slot spam_slots[]; - -/// example-start -static PyObject * -spamtype_raise_exc(PyObject *self, PyObject *unused) -{ - PyObject *module = PyType_GetModuleByToken(Py_TYPE(self), spam_slots); - if (!module) { - return NULL; - } - SpamState *state = PyModule_GetState(module); - if (!state) { - return NULL; - } - PyErr_SetString(state->exception, "failed!"); - return NULL; -} -/// example-end - -static PyType_Spec spamtype_spec = { - .name = "spam.SpamType", - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .slots = (PyType_Slot[]) { - { - Py_tp_methods, - (PyMethodDef[]) { - {"raise_exc", spamtype_raise_exc, METH_NOARGS}, - {0}, - } - }, - {0}, - }, -}; - -static int -spam_exec(PyObject *self) -{ - SpamState *state = PyModule_GetState(self); - state->exception = PyErr_NewException("spam.SpamException", NULL, NULL); - if (!state->exception) { - return -1; - } - state->type = (PyTypeObject*)PyType_FromModuleAndSpec(self, &spamtype_spec, NULL); - if (!state->type) { - return -1; - } - if (PyModule_AddType(self, state->type) < 0) { - return -1; - } - return 0; -} - -static int -spam_traverse(PyObject *self, visitproc visit, void *arg) -{ - SpamState *state = PyModule_GetState(self); - Py_VISIT(state->exception); - Py_VISIT(state->type); - return 0; -} - -static int -spam_clear(PyObject *self) -{ - SpamState *state = PyModule_GetState(self); - Py_CLEAR(state->exception); - Py_CLEAR(state->type); - return 0; -} - -static void -spam_free(PyObject *self) -{ - spam_clear(self); -} - -static const PyModuleDef_Slot spam_slots[] = { - {Py_mod_exec, spam_exec}, - {Py_mod_state_size, (void*)sizeof(SpamState)}, - {Py_mod_state_traverse, spam_traverse}, - {Py_mod_state_clear, spam_clear}, - {Py_mod_state_free, spam_free}, - {0}, -}; - -PyMODEXPORT_FUNC -PyModExport_spam(void) -{ - return spam_slots; -} From cc40c8fac17d4480c2ee62c1e0d3a9dbc93a1529 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Oct 2025 15:56:37 +0200 Subject: [PATCH 3/3] Fix refcounting in example --- peps/pep-0793/examplemodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/peps/pep-0793/examplemodule.c b/peps/pep-0793/examplemodule.c index 9031b27646a..c49fd0d628b 100644 --- a/peps/pep-0793/examplemodule.c +++ b/peps/pep-0793/examplemodule.c @@ -2,7 +2,7 @@ Example module with C-level module-global state, and - a simple function that updates and queries the state -- a class wihose repr() queries the same module state (for an example of +- a class wihose repr() queries the same module state (as an example of PyType_GetModuleByToken) Once compiled and renamed to not include a version tag (for example @@ -67,6 +67,7 @@ exampletype_repr(PyObject *self) return NULL; } examplemodule_state *state = PyModule_GetState(module); + Py_DECREF(module); if (!state) { return NULL; } @@ -95,8 +96,10 @@ examplemodule_exec(PyObject *module) { return -1; } if (PyModule_AddType(module, type) < 0) { + Py_DECREF(type); return -1; } + Py_DECREF(type); return 0; }