From af143d6b62fd8db3f32c4e56e177fc8cfd32dcdc Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 9 Nov 2025 20:04:54 -0800 Subject: [PATCH] [SPARK-54269][PYTHON] Upgrade `cloudpickle` to 3.1.2 --- python/pyspark/cloudpickle/__init__.py | 2 +- python/pyspark/cloudpickle/cloudpickle.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/python/pyspark/cloudpickle/__init__.py b/python/pyspark/cloudpickle/__init__.py index bdb1738611b3..052b6e975a77 100644 --- a/python/pyspark/cloudpickle/__init__.py +++ b/python/pyspark/cloudpickle/__init__.py @@ -3,7 +3,7 @@ __doc__ = cloudpickle.__doc__ -__version__ = "3.1.1" +__version__ = "3.1.2" __all__ = [ # noqa "__version__", diff --git a/python/pyspark/cloudpickle/cloudpickle.py b/python/pyspark/cloudpickle/cloudpickle.py index 4d532e5de9f2..e600b35f2842 100644 --- a/python/pyspark/cloudpickle/cloudpickle.py +++ b/python/pyspark/cloudpickle/cloudpickle.py @@ -783,6 +783,12 @@ def _class_getstate(obj): clsdict.pop("__dict__", None) # unpicklable property object + if sys.version_info >= (3, 14): + # PEP-649/749: __annotate_func__ contains a closure that references the class + # dict. We need to exclude it from pickling. Python will recreate it when + # __annotations__ is accessed at unpickling time. + clsdict.pop("__annotate_func__", None) + return (clsdict, {}) @@ -1190,6 +1196,10 @@ def _class_setstate(obj, state): for subclass in registry: obj.register(subclass) + # PEP-649/749: During pickling, we excluded the __annotate_func__ attribute but it + # will be created by Python. Subsequently, annotations will be recreated when + # __annotations__ is accessed. + return obj @@ -1301,12 +1311,9 @@ def _function_getnewargs(self, func): def dump(self, obj): try: return super().dump(obj) - except RuntimeError as e: - if len(e.args) > 0 and "recursion" in e.args[0]: - msg = "Could not pickle object as excessively deep recursion required." - raise pickle.PicklingError(msg) from e - else: - raise + except RecursionError as e: + msg = "Could not pickle object as excessively deep recursion required." + raise pickle.PicklingError(msg) from e def __init__(self, file, protocol=None, buffer_callback=None): if protocol is None: