Add support for class methods, unbound methods, static methods and nested classes.

Author: serhiy-storchaka

Doc/whatsnew/3.15.rst

@@ -646,6 +646,13 @@ os.path
   (Contributed by Petr Viktorin for :cve:`2025-4517`.)
 
 
+pickle
+------
+
+* Add support for pickling private methods and nested classes.
+  (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.)
+
+
 resource
 --------
 

Include/internal/pycore_symtable.h

@@ -151,7 +151,12 @@ extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject
 extern void _PySymtable_Free(struct symtable *);
 
 extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name);
-extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
+
+// Export for '_pickle' shared extension
+PyAPI_FUNC(PyObject *)
+_Py_Mangle(PyObject *, PyObject *);
+PyAPI_FUNC(int)
+_Py_IsPrivateName(PyObject *);
 
 /* Flags for def-use information */
 

Lib/pickle.py

@@ -1175,6 +1175,17 @@ def save_global(self, obj, name=None):
             if name is None:
                 name = obj.__name__
 
+        if '.__' in name:
+            # Mangle names of private attributes.
+            dotted_path = name.split('.')
+            for i, subpath in enumerate(dotted_path):
+                if i and subpath.startswith('__') and not subpath.endswith('__'):
+                    prev = prev.lstrip('_')
+                    if prev:
+                        dotted_path[i] = f"_{prev.lstrip('_')}{subpath}"
+                prev = subpath
+            name = '.'.join(dotted_path)
+
         module_name = whichmodule(obj, name)
         if self.proto >= 2:
             code = _extension_registry.get((module_name, name), _NoValue)

Lib/test/picklecommon.py

@@ -419,3 +419,17 @@ def __private_staticmethod():
     @classmethod
     def get_staticmethod(cls):
         return cls.__private_staticmethod
+
+# For test_private_nested_classes
+class PrivateNestedClasses:
+    @classmethod
+    def get_nested(cls):
+        return cls.__Nested
+
+    class __Nested:
+        @classmethod
+        def get_nested2(cls):
+            return cls.__Nested2
+
+        class __Nested2:
+            pass

Lib/test/pickletester.py

@@ -4133,6 +4133,18 @@ def test_private_methods(self):
                 unpickled = self.loads(self.dumps(obj.get_staticmethod(), proto))
                 self.assertEqual(unpickled(), 44)
 
+    def test_private_nested_classes(self):
+        if self.py_version < (3, 15):
+            self.skipTest('not supported in Python < 3.15')
+        cls1 = PrivateNestedClasses.get_nested()
+        cls2 = cls1.get_nested2()
+        for proto in protocols:
+            with self.subTest(proto=proto):
+                unpickled = self.loads(self.dumps(cls1, proto))
+                self.assertIs(unpickled, cls1)
+                unpickled = self.loads(self.dumps(cls2, proto))
+                self.assertIs(unpickled, cls2)
+
     def test_object_with_attrs(self):
         obj = Object()
         obj.a = 1

…2020-07-14-23-54-18.bpo-33007.TyI3_Q.rst …07-14-23-54-18.gh-issue-77188.TyI3_Q.rstMisc/NEWS.d/next/Library/2020-07-14-23-54-18.bpo-33007.TyI3_Q.rst renamed to Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst Misc/NEWS.d/next/Library/2020-07-14-23-54-18.bpo-33007.TyI3_Q.rst renamed to Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst

@@ -19,6 +19,7 @@
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_runtime.h"       // _Py_ID()
 #include "pycore_setobject.h"     // _PySet_NextEntry()
+#include "pycore_symtable.h"      // _Py_Mangle()
 #include "pycore_sysmodule.h"     // _PySys_GetSizeOf()
 #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString()
 
@@ -1928,6 +1929,37 @@ get_dotted_path(PyObject *name)
     return PyUnicode_Split(name, _Py_LATIN1_CHR('.'), -1);
 }
 
+static PyObject *
+join_dotted_path(PyObject *dotted_path)
+{
+    return PyUnicode_Join(_Py_LATIN1_CHR('.'), dotted_path);
+}
+
+/* Returns -1 (with an exception set) on error, 0 if there were no changes,
+ * 1 if some names were mangled. */
+static int
+mangle_dotted_path(PyObject *dotted_path)
+{
+    int rc = 0;
+    Py_ssize_t n = PyList_GET_SIZE(dotted_path);
+    for (Py_ssize_t i = n-1; i > 0; i--) {
+        PyObject *subpath = PyList_GET_ITEM(dotted_path, i);
+        if (_Py_IsPrivateName(subpath)) {
+            PyObject *parent = PyList_GET_ITEM(dotted_path, i-1);
+            PyObject *mangled = _Py_Mangle(parent, subpath);
+            if (mangled == NULL) {
+                return -1;
+            }
+            if (mangled != subpath) {
+                rc = 1;
+            }
+            PyList_SET_ITEM(dotted_path, i, mangled);
+            Py_DECREF(subpath);
+        }
+    }
+    return rc;
+}
+
 static int
 check_dotted_path(PickleState *st, PyObject *obj, PyObject *dotted_path)
 {
@@ -3809,6 +3841,15 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
     dotted_path = get_dotted_path(global_name);
     if (dotted_path == NULL)
         goto error;
+    switch (mangle_dotted_path(dotted_path)) {
+        case -1:
+            goto error;
+        case 1:
+            Py_SETREF(global_name, join_dotted_path(dotted_path));
+            if (global_name == NULL) {
+                goto error;
+            }
+    }
     module_name = whichmodule(st, obj, global_name, dotted_path);
     if (module_name == NULL)
         goto error;

Modules/_pickle.c

@@ -141,25 +141,19 @@ method___reduce___impl(PyMethodObject *self)
     PyObject *funcself = PyMethod_GET_SELF(self);
     PyObject *func = PyMethod_GET_FUNCTION(self);
     PyObject *funcname = PyObject_GetAttr(func, &_Py_ID(__name__));
-    Py_ssize_t len;
     if (funcname == NULL) {
         return NULL;
     }
-    if (PyUnicode_Check(funcname) &&
-        (len = PyUnicode_GET_LENGTH(funcname)) > 2 &&
-        PyUnicode_READ_CHAR(funcname, 0) == '_' &&
-        PyUnicode_READ_CHAR(funcname, 1) == '_' &&
-        !(PyUnicode_READ_CHAR(funcname, len-1) == '_' &&
-          PyUnicode_READ_CHAR(funcname, len-2) == '_'))
-    {
-        PyObject *name = PyObject_GetAttr((PyObject *)Py_TYPE(funcself),
-                                          &_Py_ID(__name__));
-        if (name == NULL) {
+    if (_Py_IsPrivateName(funcname)) {
+        PyObject *classname = PyType_Check(funcself)
+            ? PyType_GetName((PyTypeObject *)funcself)
+            : PyType_GetName(Py_TYPE(funcself));
+        if (classname == NULL) {
             Py_DECREF(funcname);
             return NULL;
         }
-        Py_SETREF(funcname, _Py_Mangle(name, funcname));
-        Py_DECREF(name);
+        Py_SETREF(funcname, _Py_Mangle(classname, funcname));
+        Py_DECREF(classname);
         if (funcname == NULL) {
             return NULL;
         }

Objects/classobject.c

@@ -3183,6 +3183,27 @@ _Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name)
     return _Py_Mangle(privateobj, name);
 }
 
+int
+_Py_IsPrivateName(PyObject *ident)
+{
+    if (!PyUnicode_Check(ident)) {
+        return 0;
+    }
+    Py_ssize_t nlen = PyUnicode_GET_LENGTH(ident);
+    if (nlen < 3 ||
+        PyUnicode_READ_CHAR(ident, 0) != '_' ||
+        PyUnicode_READ_CHAR(ident, 1) != '_')
+    {
+        return 0;
+    }
+    if (PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&
+        PyUnicode_READ_CHAR(ident, nlen-2) == '_')
+    {
+        return 0; /* Don't mangle __whatever__ */
+    }
+    return 1;
+}
+
 PyObject *
 _Py_Mangle(PyObject *privateobj, PyObject *ident)
 {