From 61fff28590f571d656331b826bb7ddfe0f8b7de5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:23:36 +0000 Subject: [PATCH 1/6] Initial plan From 976e1be370050b67b3b45607c801939ad21214ff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:33:16 +0000 Subject: [PATCH 2/6] Fix pickling ABC classes with annotations for Python 3.13+ In Python 3.13+ (PEP 649), classes with annotations have an __annotate__ function that may have closures referencing unpicklable objects in the class namespace (like _abc._abc_data in _abc_impl). This fix removes __annotate__ from the pickled class state, allowing Python to regenerate it from __annotations__ when the class is reconstructed. Co-authored-by: ogrisel <89061+ogrisel@users.noreply.github.com> --- cloudpickle/cloudpickle.py | 6 +++++ tests/cloudpickle_test.py | 46 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 4f4d857a..647f57dd 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -750,6 +750,12 @@ def _function_getstate(func): def _class_getstate(obj): clsdict = _extract_class_dict(obj) clsdict.pop("__weakref__", None) + + # In Python 3.13+, classes with annotations have an __annotate__ function + # that may have closures referencing unpicklable objects in the class + # namespace. We don't need to pickle it as Python will regenerate it from + # __annotations__ when the class is reconstructed. + clsdict.pop("__annotate__", None) if issubclass(type(obj), abc.ABCMeta): # If obj is an instance of an ABCMeta subclass, don't pickle the diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 45e8c592..e7080116 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1429,6 +1429,52 @@ def some_method(self): self.assertRaises(TypeError, IncompleteBaseSubclass) + def test_abc_with_annotations(self): + # Test pickling abstract classes with type annotations. + # This is a regression test for Python 3.14+ where __annotate__ + # functions can have closures that reference _abc_impl. + # See: https://github.com/cloudpipe/cloudpickle/issues/??? + + def class_factory(): + class Model(abc.ABC): + field: int + return Model + + Model = class_factory() + + # This should not raise TypeError about unpicklable _abc._abc_data + depickled = pickle_depickle(Model, protocol=self.protocol) + + # Verify the annotation is preserved + self.assertEqual(depickled.__annotations__, {'field': int}) + + # Verify it's still an ABC + self.assertTrue(issubclass(depickled, abc.ABC)) + + # Test with a concrete subclass + class ConcreteModel(Model): + def __init__(self, field: int): + self.field = field + + instance = ConcreteModel(42) + depickled_instance = pickle_depickle(instance, protocol=self.protocol) + self.assertEqual(depickled_instance.field, 42) + + # Test multiple annotations + def multi_annotation_factory(): + class MultiModel(abc.ABC): + field1: int + field2: str + field3: float + return MultiModel + + MultiModel = multi_annotation_factory() + depickled_multi = pickle_depickle(MultiModel, protocol=self.protocol) + self.assertEqual( + depickled_multi.__annotations__, + {'field1': int, 'field2': str, 'field3': float} + ) + def test_weakset_identity_preservation(self): # Test that weaksets don't lose all their inhabitants if they're # pickled in a larger data structure that includes other references to From c2254adfd635670091553aeb6185aacdbebb3422 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:38:13 +0000 Subject: [PATCH 3/6] Simplify test_abc_with_annotations to directly test multiple annotations Co-authored-by: ogrisel <89061+ogrisel@users.noreply.github.com> --- cloudpickle/cloudpickle.py | 8 +-- tests/cloudpickle_test.py | 99 ++++++++++++++++++++------------------ 2 files changed, 56 insertions(+), 51 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 647f57dd..21704ad0 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -79,7 +79,6 @@ # cloudpickle. See: tests/test_backward_compat.py from types import CellType # noqa: F401 - # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor # communication speed over compatibility: @@ -193,9 +192,12 @@ def _is_registered_pickle_by_value(module): if sys.version_info >= (3, 14): + def _getattribute(obj, name): - return _pickle_getattribute(obj, name.split('.')) + return _pickle_getattribute(obj, name.split(".")) + else: + def _getattribute(obj, name): return _pickle_getattribute(obj, name)[0] @@ -750,7 +752,7 @@ def _function_getstate(func): def _class_getstate(obj): clsdict = _extract_class_dict(obj) clsdict.pop("__weakref__", None) - + # In Python 3.13+, classes with annotations have an __annotate__ function # that may have closures referencing unpicklable objects in the class # namespace. We don't need to pickle it as Python will regenerate it from diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index e7080116..561efd9e 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -55,7 +55,6 @@ from .testutils import assert_run_python_script from .testutils import check_deterministic_pickle - _TEST_GLOBAL_VARIABLE = "default_value" _TEST_GLOBAL_VARIABLE2 = "another_value" @@ -354,7 +353,6 @@ class A: assert hasattr(A_roundtrip, "__firstlineno__") assert A_roundtrip.__firstlineno__ == A.__firstlineno__ - def test_dynamically_generated_class_that_uses_super(self): class Base: def method(self): @@ -1067,7 +1065,9 @@ def test_extended_arg(self): def f(): x = {tup} return zlib.crc32(bytes(bytearray(x))) - """.format(tup=", ".join(names)) + """.format( + tup=", ".join(names) + ) exec(textwrap.dedent(code), d, d) f = d["f"] res = f() @@ -1206,7 +1206,9 @@ def check_logger(self, name): logging.basicConfig(level=logging.INFO) logger = cloudpickle.loads(base64.b32decode(b'{}')) logger.info('hello') - """.format(base64.b32encode(dumped).decode("ascii")) + """.format( + base64.b32encode(dumped).decode("ascii") + ) proc = subprocess.Popen( [sys.executable, "-W ignore", "-c", code], stdout=subprocess.PIPE, @@ -1434,47 +1436,29 @@ def test_abc_with_annotations(self): # This is a regression test for Python 3.14+ where __annotate__ # functions can have closures that reference _abc_impl. # See: https://github.com/cloudpipe/cloudpickle/issues/??? - + def class_factory(): class Model(abc.ABC): - field: int + field1: int + field2: str + field3: float + return Model - + Model = class_factory() - + # This should not raise TypeError about unpicklable _abc._abc_data depickled = pickle_depickle(Model, protocol=self.protocol) - - # Verify the annotation is preserved - self.assertEqual(depickled.__annotations__, {'field': int}) - - # Verify it's still an ABC - self.assertTrue(issubclass(depickled, abc.ABC)) - - # Test with a concrete subclass - class ConcreteModel(Model): - def __init__(self, field: int): - self.field = field - - instance = ConcreteModel(42) - depickled_instance = pickle_depickle(instance, protocol=self.protocol) - self.assertEqual(depickled_instance.field, 42) - - # Test multiple annotations - def multi_annotation_factory(): - class MultiModel(abc.ABC): - field1: int - field2: str - field3: float - return MultiModel - - MultiModel = multi_annotation_factory() - depickled_multi = pickle_depickle(MultiModel, protocol=self.protocol) + + # Verify the annotations are preserved self.assertEqual( - depickled_multi.__annotations__, - {'field1': int, 'field2': str, 'field3': float} + depickled.__annotations__, + {"field1": int, "field2": str, "field3": float}, ) + # Verify it's still an ABC + self.assertTrue(issubclass(depickled, abc.ABC)) + def test_weakset_identity_preservation(self): # Test that weaksets don't lose all their inhabitants if they're # pickled in a larger data structure that includes other references to @@ -1552,7 +1536,8 @@ def __getattr__(self, name): def test_importing_multiprocessing_does_not_impact_whichmodule(self): # non-regression test for #528 - script = textwrap.dedent(""" + script = textwrap.dedent( + """ import multiprocessing import cloudpickle from cloudpickle.cloudpickle import dumps @@ -1562,7 +1547,8 @@ def test_importing_multiprocessing_does_not_impact_whichmodule(self): dumps.__module__ = None print(cloudpickle.cloudpickle._whichmodule(dumps, dumps.__name__)) - """) + """ + ) script_path = Path(self.tmpdir) / "whichmodule_and_multiprocessing.py" with open(script_path, mode="w") as f: f.write(script) @@ -1576,7 +1562,6 @@ def test_importing_multiprocessing_does_not_impact_whichmodule(self): self.assertEqual(proc.wait(), 0, msg="Stdout: " + str(out)) self.assertEqual(out.strip(), b"cloudpickle.cloudpickle") - def test_unrelated_faulty_module(self): # Check that pickling a dynamically defined function or class does not # fail when introspecting the currently loaded modules in sys.modules @@ -1779,7 +1764,9 @@ def f5(x): cloned = subprocess_pickle_echo(f5, protocol={protocol}) assert cloned(7) == f5(7) == 7 - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(textwrap.dedent(code)) def test_interactively_defined_global_variable(self): @@ -1918,7 +1905,9 @@ def interactive_function(x): # previous definition of `interactive_function`: assert w.run(wrapper_func, 41) == 40 - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(code) def test_interactive_remote_function_calls_no_side_effect(self): @@ -1962,7 +1951,9 @@ def is_in_main(name): assert is_in_main("GLOBAL_VARIABLE") assert not w.run(is_in_main, "GLOBAL_VARIABLE") - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(code) def test_interactive_dynamic_type_and_remote_instances(self): @@ -2001,7 +1992,9 @@ def echo(*args): assert isinstance(c1, CustomCounter) assert isinstance(c2, CustomCounter) - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(code) def test_interactive_dynamic_type_and_stored_remote_instances(self): @@ -2078,7 +2071,9 @@ class A: # method: assert w.run(lambda obj_id: lookup(obj_id).echo(43), id2) == 43 - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(code) def test_dynamic_func_deterministic_roundtrip(self): @@ -2266,7 +2261,9 @@ def process_data(): # iterations instead of 100 as used now (100x more data) assert growth < 5e7, growth - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(code) def test_pickle_reraise(self): @@ -2468,7 +2465,9 @@ def check_positive(x): result = w.run(check_positive, 1) assert result is Color.BLUE - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(code) def test_relative_import_inside_function(self): @@ -2522,7 +2521,9 @@ def f(a, /, b=1): with pytest.raises(TypeError): func(a=2) - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(textwrap.dedent(code)) def test___reduce___returns_string(self): @@ -3079,7 +3080,9 @@ def echo(*args): cloned_value, cloned_type = w.run(echo, value, SampleDataclass) assert cloned_type is SampleDataclass assert isinstance(cloned_value, SampleDataclass) - """.format(protocol=self.protocol) + """.format( + protocol=self.protocol + ) assert_run_python_script(code) From baf9548c0c7e7ac44be948ce806e6769aff99b91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:40:28 +0000 Subject: [PATCH 4/6] Address code review feedback: update test comments Co-authored-by: ogrisel <89061+ogrisel@users.noreply.github.com> --- tests/cloudpickle_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 561efd9e..47a67197 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1433,9 +1433,8 @@ def some_method(self): def test_abc_with_annotations(self): # Test pickling abstract classes with type annotations. - # This is a regression test for Python 3.14+ where __annotate__ + # This is a regression test for Python 3.13+ (PEP 649) where __annotate__ # functions can have closures that reference _abc_impl. - # See: https://github.com/cloudpipe/cloudpickle/issues/??? def class_factory(): class Model(abc.ABC): From ae9aac4d1719e0ebe8e8938575b39fe0cbef2b10 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:54:17 +0000 Subject: [PATCH 5/6] Also handle __annotate_func__ for Python 3.14 compatibility Add removal of __annotate_func__ in addition to __annotate__ to ensure compatibility with Python 3.14 where both attribute names may be present. Co-authored-by: ogrisel <89061+ogrisel@users.noreply.github.com> --- cloudpickle/cloudpickle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 21704ad0..b432909a 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -758,6 +758,8 @@ def _class_getstate(obj): # namespace. We don't need to pickle it as Python will regenerate it from # __annotations__ when the class is reconstructed. clsdict.pop("__annotate__", None) + # Also handle __annotate_func__ which may appear in some Python 3.14 builds + clsdict.pop("__annotate_func__", None) if issubclass(type(obj), abc.ABCMeta): # If obj is an instance of an ABCMeta subclass, don't pickle the From 361947a03913e91b474344a69d67860e22b39e5f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Oct 2025 17:07:58 +0000 Subject: [PATCH 6/6] Remove unrelated formatting changes, keep only the bug fix Reverted all unrelated formatting changes and kept only the substantive fix for ABC classes with annotations. The changes now only include: - Addition of __annotate__ and __annotate_func__ removal in _class_getstate() - Addition of test_abc_with_annotations() test Co-authored-by: ogrisel <89061+ogrisel@users.noreply.github.com> --- cloudpickle/cloudpickle.py | 6 ++--- tests/cloudpickle_test.py | 53 ++++++++++++-------------------------- 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index b432909a..5a74cce5 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -79,6 +79,7 @@ # cloudpickle. See: tests/test_backward_compat.py from types import CellType # noqa: F401 + # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor # communication speed over compatibility: @@ -192,12 +193,9 @@ def _is_registered_pickle_by_value(module): if sys.version_info >= (3, 14): - def _getattribute(obj, name): - return _pickle_getattribute(obj, name.split(".")) - + return _pickle_getattribute(obj, name.split('.')) else: - def _getattribute(obj, name): return _pickle_getattribute(obj, name)[0] diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 47a67197..7640cf4e 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -55,6 +55,7 @@ from .testutils import assert_run_python_script from .testutils import check_deterministic_pickle + _TEST_GLOBAL_VARIABLE = "default_value" _TEST_GLOBAL_VARIABLE2 = "another_value" @@ -353,6 +354,7 @@ class A: assert hasattr(A_roundtrip, "__firstlineno__") assert A_roundtrip.__firstlineno__ == A.__firstlineno__ + def test_dynamically_generated_class_that_uses_super(self): class Base: def method(self): @@ -1065,9 +1067,7 @@ def test_extended_arg(self): def f(): x = {tup} return zlib.crc32(bytes(bytearray(x))) - """.format( - tup=", ".join(names) - ) + """.format(tup=", ".join(names)) exec(textwrap.dedent(code), d, d) f = d["f"] res = f() @@ -1206,9 +1206,7 @@ def check_logger(self, name): logging.basicConfig(level=logging.INFO) logger = cloudpickle.loads(base64.b32decode(b'{}')) logger.info('hello') - """.format( - base64.b32encode(dumped).decode("ascii") - ) + """.format(base64.b32encode(dumped).decode("ascii")) proc = subprocess.Popen( [sys.executable, "-W ignore", "-c", code], stdout=subprocess.PIPE, @@ -1535,8 +1533,7 @@ def __getattr__(self, name): def test_importing_multiprocessing_does_not_impact_whichmodule(self): # non-regression test for #528 - script = textwrap.dedent( - """ + script = textwrap.dedent(""" import multiprocessing import cloudpickle from cloudpickle.cloudpickle import dumps @@ -1546,8 +1543,7 @@ def test_importing_multiprocessing_does_not_impact_whichmodule(self): dumps.__module__ = None print(cloudpickle.cloudpickle._whichmodule(dumps, dumps.__name__)) - """ - ) + """) script_path = Path(self.tmpdir) / "whichmodule_and_multiprocessing.py" with open(script_path, mode="w") as f: f.write(script) @@ -1561,6 +1557,7 @@ def test_importing_multiprocessing_does_not_impact_whichmodule(self): self.assertEqual(proc.wait(), 0, msg="Stdout: " + str(out)) self.assertEqual(out.strip(), b"cloudpickle.cloudpickle") + def test_unrelated_faulty_module(self): # Check that pickling a dynamically defined function or class does not # fail when introspecting the currently loaded modules in sys.modules @@ -1763,9 +1760,7 @@ def f5(x): cloned = subprocess_pickle_echo(f5, protocol={protocol}) assert cloned(7) == f5(7) == 7 - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(textwrap.dedent(code)) def test_interactively_defined_global_variable(self): @@ -1904,9 +1899,7 @@ def interactive_function(x): # previous definition of `interactive_function`: assert w.run(wrapper_func, 41) == 40 - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(code) def test_interactive_remote_function_calls_no_side_effect(self): @@ -1950,9 +1943,7 @@ def is_in_main(name): assert is_in_main("GLOBAL_VARIABLE") assert not w.run(is_in_main, "GLOBAL_VARIABLE") - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(code) def test_interactive_dynamic_type_and_remote_instances(self): @@ -1991,9 +1982,7 @@ def echo(*args): assert isinstance(c1, CustomCounter) assert isinstance(c2, CustomCounter) - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(code) def test_interactive_dynamic_type_and_stored_remote_instances(self): @@ -2070,9 +2059,7 @@ class A: # method: assert w.run(lambda obj_id: lookup(obj_id).echo(43), id2) == 43 - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(code) def test_dynamic_func_deterministic_roundtrip(self): @@ -2260,9 +2247,7 @@ def process_data(): # iterations instead of 100 as used now (100x more data) assert growth < 5e7, growth - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(code) def test_pickle_reraise(self): @@ -2464,9 +2449,7 @@ def check_positive(x): result = w.run(check_positive, 1) assert result is Color.BLUE - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(code) def test_relative_import_inside_function(self): @@ -2520,9 +2503,7 @@ def f(a, /, b=1): with pytest.raises(TypeError): func(a=2) - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(textwrap.dedent(code)) def test___reduce___returns_string(self): @@ -3079,9 +3060,7 @@ def echo(*args): cloned_value, cloned_type = w.run(echo, value, SampleDataclass) assert cloned_type is SampleDataclass assert isinstance(cloned_value, SampleDataclass) - """.format( - protocol=self.protocol - ) + """.format(protocol=self.protocol) assert_run_python_script(code)