From 7ad91312485a2ae5bddbc24629ae618174c695fc Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 28 Mar 2025 13:43:33 +0000 Subject: [PATCH 01/40] Proposed code changes for [PEP XXX](link). Signed-off-by: Matthew A Johnson --- .gitignore | 3 + Doc/conf.py | 1 + Doc/data/stable_abi.dat | 1 + Include/cpython/pyerrors.h | 7 + Include/internal/pycore_dict.h | 15 +- Include/internal/pycore_frame.h | 2 +- .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + Include/internal/pycore_immutability.h | 18 + Include/internal/pycore_object.h | 19 +- .../internal/pycore_runtime_init_generated.h | 2 + Include/object.h | 42 +- Include/pyerrors.h | 1 + Lib/_compat_pickle.py | 1 + Lib/test/exception_hierarchy.txt | 1 + Lib/test/test_descr.py | 4 +- Lib/test/test_freeze.py | 473 ++++++++++++++ Lib/test/test_importlib/test_api.py | 2 + Lib/test/test_stable_abi_ctypes.py | 1 + Makefile.pre.in | 1 + Misc/stable_abi.toml | 2 + Objects/abstract.c | 82 +++ Objects/cellobject.c | 16 + Objects/dictobject.c | 382 ++++++++--- Objects/exceptions.c | 6 + Objects/frameobject.c | 4 + Objects/listobject.c | 64 ++ Objects/longobject.c | 2 + Objects/moduleobject.c | 8 +- Objects/object.c | 22 +- Objects/setobject.c | 42 ++ Objects/tupleobject.c | 7 + Objects/typeobject.c | 32 +- PC/python3dll.c | 1 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 6 + Python/bltinmodule.c | 38 ++ Python/bytecodes.c | 53 +- Python/ceval.c | 21 +- Python/clinic/bltinmodule.c.h | 20 +- Python/errors.c | 32 + Python/generated_cases.c.h | 607 +++++++++--------- Python/immutability.c | 376 +++++++++++ Python/opcode_metadata.h | 6 +- Python/pylifecycle.c | 3 + Tools/c-analyzer/cpython/ignored.tsv | 2 + Tools/gdb/libpython.py | 3 +- 49 files changed, 2031 insertions(+), 409 deletions(-) create mode 100644 Include/internal/pycore_immutability.h create mode 100644 Lib/test/test_freeze.py create mode 100644 Python/immutability.c diff --git a/.gitignore b/.gitignore index bde596a7a0298b..15f4e81ce895c5 100644 --- a/.gitignore +++ b/.gitignore @@ -156,3 +156,6 @@ Python/frozen_modules/MANIFEST # Ignore ./python binary on Unix but still look into ./Python/ directory. /python !/Python/ + +# Ignore the build directory. +build* \ No newline at end of file diff --git a/Doc/conf.py b/Doc/conf.py index 9f01bd89a35d3c..dbd75012988442 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -215,6 +215,7 @@ ('c:data', 'PyExc_UnicodeError'), ('c:data', 'PyExc_UnicodeTranslateError'), ('c:data', 'PyExc_ValueError'), + ('c:data', 'PyExc_NotWriteableError'), ('c:data', 'PyExc_ZeroDivisionError'), # C API: Standard Python warning classes ('c:data', 'PyExc_BytesWarning'), diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index f112d268129fd1..595b999d750648 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -238,6 +238,7 @@ var,PyExc_ModuleNotFoundError,3.6,, var,PyExc_NameError,3.2,, var,PyExc_NotADirectoryError,3.7,, var,PyExc_NotImplementedError,3.2,, +var,PyExc_NotWriteableError,3.12,, var,PyExc_OSError,3.2,, var,PyExc_OverflowError,3.2,, var,PyExc_PendingDeprecationWarning,3.2,, diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 156665cbdb1ba4..2f5914c4d385b8 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -168,6 +168,13 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFormat( const char *format, ...); +PyAPI_FUNC(PyObject *) _PyErr_WriteToImmutable(const char* file, int line, PyObject *obj); +#define PyErr_WriteToImmutable(obj) _PyErr_WriteToImmutable(__FILE__, __LINE__, _PyObject_CAST(obj)) + +PyAPI_FUNC(PyObject *) _PyErr_WriteToImmutableKey(const char* file, int line, PyObject *key); +#define PyErr_WriteToImmutableKey(key) _PyErr_WriteToImmutableKey(__FILE__, __LINE__, _PyObject_CAST(key)) + + extern PyObject *_PyErr_SetImportErrorWithNameFrom( PyObject *, PyObject *, diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6253e0841ad349..a9e29df8bf8dce 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -24,14 +24,24 @@ typedef struct { /* Cached hash code of me_key. */ Py_hash_t me_hash; PyObject *me_key; - PyObject *me_value; /* This field is only meaningful for combined tables */ + PyObject *_me_value; /* This field is only meaningful for combined tables */ } PyDictKeyEntry; typedef struct { PyObject *me_key; /* The key must be Unicode and have hash. */ - PyObject *me_value; /* This field is only meaningful for combined tables */ + PyObject *_me_value; /* This field is only meaningful for combined tables */ } PyDictUnicodeEntry; +#define _PyDictEntry_IsImmutable(entry) (((uintptr_t)((entry)->_me_value)) & 0x1) +#define _PyDictEntry_SetImmutable(entry) ((entry)->_me_value = (PyObject*)((uintptr_t)(entry)->_me_value | 0x1)) +#define _PyDictEntry_Hash(entry) ((entry)->me_hash) +#define _PyDictEntry_Key(entry) ((entry)->me_key) +#define _PyDictEntry_Value(entry) ((PyObject*)((((uintptr_t)((entry)->_me_value)) >> 1) << 1)) +#define _PyDictEntry_SetValue(entry, value) ((entry)->_me_value = value) +#define _PyDictEntry_IsEmpty(entry) ((entry)->_me_value == NULL) + +extern PyObject *_PyDict_IsKeyImmutable(PyObject* op, PyObject* key); + extern PyDictKeysObject *_PyDict_NewKeysForClass(void); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); @@ -50,6 +60,7 @@ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t has extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +extern PyObject *_PyDict_SetKeyImmutable(PyDictObject *mp, PyObject *key); /* Consumes references to key and value */ extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 158db2cf9df82e..78719efdca0c0f 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -198,7 +198,7 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily * creating it if necessary. - * Returns a borrowed referennce */ + * Returns a borrowed reference */ static inline PyFrameObject * _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) { diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 439f47a263dfa1..cac1b4ed3be807 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -562,10 +562,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(namedtuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(shim_name)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(startswith)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 0c84999cbf8127..4bc9854b914509 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -48,10 +48,12 @@ struct _Py_global_strings { STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(namedtuple, "namedtuple") STRUCT_FOR_STR(newline, "\n") STRUCT_FOR_STR(open_br, "{") STRUCT_FOR_STR(percent, "%") STRUCT_FOR_STR(shim_name, "") + STRUCT_FOR_STR(startswith, "startswith") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h new file mode 100644 index 00000000000000..426ac5ff7564da --- /dev/null +++ b/Include/internal/pycore_immutability.h @@ -0,0 +1,18 @@ +#ifndef Py_FREEZE_H +#define Py_FREEZE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyObject* _Py_Freeze(PyObject*); +#define Py_Freeze(op) _Py_Freeze(_PyObject_CAST(op)) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_FREEZE_H */ \ No newline at end of file diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7a2f13a21bda76..81584822a39f11 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -71,7 +71,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) static inline void _Py_SetImmortal(PyObject *op) { if (op) { - op->ob_refcnt = _Py_IMMORTAL_REFCNT; + op->ob_refcnt = (op->ob_refcnt & _Py_IMMUTABLE_MASK) | _Py_IMMORTAL_REFCNT; } } #define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) @@ -80,7 +80,8 @@ static inline void _Py_SetImmortal(PyObject *op) static inline void _Py_ClearImmortal(PyObject *op) { if (op) { - assert(op->ob_refcnt == _Py_IMMORTAL_REFCNT); + assert((op->ob_refcnt & _Py_REFCNT_MASK) == _Py_IMMORTAL_REFCNT); + // note this also clears the _Py_IMMUTABLE_FLAG, if set op->ob_refcnt = 1; Py_DECREF(op); } @@ -91,6 +92,17 @@ static inline void _Py_ClearImmortal(PyObject *op) op = NULL; \ } while (0) +static inline void _Py_SetImmutable(PyObject *op) +{ + if(op) { + op->ob_refcnt |= _Py_IMMUTABLE_FLAG; + } +} +#define _Py_SetImmutable(op) _Py_SetImmutable(_PyObject_CAST(op)) + +#define Py_CHECKWRITE(op) ((op) && (!_Py_IsImmutable(op) || _Py_IsFinalizing())) +#define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} + static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { @@ -101,7 +113,8 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) #ifdef Py_REF_DEBUG _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); #endif - if (--op->ob_refcnt != 0) { + op->ob_refcnt -= 1; + if ((op->ob_refcnt & _Py_REFCNT_MASK) != 0) { assert(op->ob_refcnt > 0); } else { diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 07f237b2905864..efd75a9fd08bb8 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -554,10 +554,12 @@ extern "C" { INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(namedtuple, "namedtuple"), \ INIT_STR(newline, "\n"), \ INIT_STR(open_br, "{"), \ INIT_STR(percent, "%"), \ INIT_STR(shim_name, ""), \ + INIT_STR(startswith, "startswith"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } diff --git a/Include/object.h b/Include/object.h index 5c30c77bc26a40..d3c332d6f96e2d 100644 --- a/Include/object.h +++ b/Include/object.h @@ -78,6 +78,25 @@ whose size is determined when the object is allocated. /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; +/* +Immutability: + +Immutability is tracked in the top bit of the reference count. The immutability +system also uses the second-to-top bit for managing immutable graphs. +*/ + +#if SIZEOF_VOID_P > 4 +#define _Py_REFCNT_MASK 0xFFFFFFFF +#define _Py_IMMUTABLE_MASK 0xC000000000 +#define _Py_IMMUTABLE_FLAG 0x4000000000 +#define _Py_IMMUTABLE_SCC_FLAG 0x8000000000 +#else +#define _Py_REFCNT_MASK 0x3FFFFFFF +#define _Py_IMMUTABLE_MASK 0xC0000000 +#define _Py_IMMUTABLE_FLAG 0x40000000 +#define _Py_IMMUTABLE_SCC_FLAG 0x80000000 +#endif + /* Immortalization: @@ -112,9 +131,9 @@ be done by checking the bit sign flag in the lower 32 bits. #else /* In 32 bit systems, an object will be marked as immortal by setting all of the -lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF +lower 28 bits of the reference count field, which is equal to: 0x0FFFFFFF. -Using the lower 30 bits makes the value backwards compatible by allowing +Using the lower 28 bits makes the value backwards compatible by allowing C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely increase and decrease the objects reference count. The object would lose its immortality, but the execution would still be correct. @@ -122,7 +141,7 @@ immortality, but the execution would still be correct. Reference count increases and decreases will first go through an immortality check by comparing the reference count field to the immortality reference count. */ -#define _Py_IMMORTAL_REFCNT (UINT_MAX >> 2) +#define _Py_IMMORTAL_REFCNT (UINT_MAX >> 4) #endif // Make all internal uses of PyObject_HEAD_INIT immortal while preserving the @@ -208,7 +227,7 @@ PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); static inline Py_ssize_t Py_REFCNT(PyObject *ob) { - return ob->ob_refcnt; + return ob->ob_refcnt & _Py_REFCNT_MASK; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) @@ -242,7 +261,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) #if SIZEOF_VOID_P > 4 return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; #else - return op->ob_refcnt == _Py_IMMORTAL_REFCNT; + return (op->ob_refcnt & _Py_REFCNT_MASK) == _Py_IMMORTAL_REFCNT; #endif } #define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) @@ -254,6 +273,11 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { # define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), (type)) #endif +static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) +{ + return (op->ob_refcnt & _Py_IMMUTABLE_FLAG) > 0; +} +#define _Py_IsImmutable(op) _Py_IsImmutable(_PyObject_CAST(op)) static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { // This immortal check is for code that is unaware of immortal objects. @@ -263,7 +287,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { if (_Py_IsImmortal(ob)) { return; } - ob->ob_refcnt = refcnt; + ob->ob_refcnt = (ob->ob_refcnt & _Py_IMMUTABLE_MASK) | (refcnt & _Py_REFCNT_MASK); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) @@ -687,7 +711,8 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) } _Py_DECREF_STAT_INC(); _Py_DECREF_DecRefTotal(); - if (--op->ob_refcnt == 0) { + op->ob_refcnt -= 1; + if ((op->ob_refcnt & _Py_REFCNT_MASK) == 0) { _Py_Dealloc(op); } } @@ -702,7 +727,8 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) return; } _Py_DECREF_STAT_INC(); - if (--op->ob_refcnt == 0) { + op->ob_refcnt -= 1; + if ((op->ob_refcnt & _Py_REFCNT_MASK) == 0) { _Py_Dealloc(op); } } diff --git a/Include/pyerrors.h b/Include/pyerrors.h index d089fa71779330..9dd230d3adcd60 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -121,6 +121,7 @@ PyAPI_DATA(PyObject *) PyExc_UnicodeDecodeError; PyAPI_DATA(PyObject *) PyExc_UnicodeTranslateError; PyAPI_DATA(PyObject *) PyExc_ValueError; PyAPI_DATA(PyObject *) PyExc_ZeroDivisionError; +PyAPI_DATA(PyObject *) PyExc_NotWriteableError; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_DATA(PyObject *) PyExc_BlockingIOError; diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 65a94b6b1bdfd5..e034427ecea908 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -137,6 +137,7 @@ "UnicodeWarning", "UserWarning", "ValueError", + "NotWriteableError", "Warning", "ZeroDivisionError", ) diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 1eca123be0fecb..c987419663409e 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -53,6 +53,7 @@ BaseException │ ├── UnicodeDecodeError │ ├── UnicodeEncodeError │ └── UnicodeTranslateError + ├── NotWriteableError └── Warning ├── BytesWarning ├── DeprecationWarning diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 9d8b1497330f0a..3cbfa8428513fe 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3282,13 +3282,15 @@ class F(D, E): pass def cant(x, C): try: x.__class__ = C + except NotWriteableError: + pass except TypeError: pass else: self.fail("shouldn't allow %r.__class__ = %r" % (x, C)) try: delattr(x, "__class__") - except (TypeError, AttributeError): + except (TypeError, AttributeError, NotWriteableError): pass else: self.fail("shouldn't allow del %r.__class__" % x) diff --git a/Lib/test/test_freeze.py b/Lib/test/test_freeze.py new file mode 100644 index 00000000000000..77949ce881ec19 --- /dev/null +++ b/Lib/test/test_freeze.py @@ -0,0 +1,473 @@ +import unittest + +# This is a canary to check that global variables are not made immutable +# when others are made immutable +global_canary = {} + +global0 = 0 + +global1 = 2 +def global1_inc(): + global global1 + global1 += 1 + return global1 + +class MutableGlobalTest(unittest.TestCase): + # Add initial test to confirm that global_canary is mutable + def test_global_mutable(self): + self.assertTrue(not isimmutable(global_canary)) + +class BaseObjectTest(unittest.TestCase): + def __init__(self, *args, obj=None, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + self.obj = obj + + def setUp(self): + freeze(self.obj) + + def test_immutable(self): + self.assertTrue(isimmutable(self.obj)) + + def test_add_attribute(self): + freeze(self.obj) + with self.assertRaises(NotWriteableError): + self.obj.new_attribute = 'value' + + def test_type_immutable(self): + self.assertTrue(isimmutable(type(self.obj))) + + +class TestBasicObject(BaseObjectTest): + class C: + pass + + def __init__(self, *args, **kwargs): + BaseObjectTest.__init__(self, *args, obj=self.C(), **kwargs) + + +class TestFloat(unittest.TestCase): + def test_freeze_float(self): + obj = 0.0 + freeze(obj) + self.assertTrue(isimmutable(obj)) + +class TestFloatType(unittest.TestCase): + def test_float_type_immutable(self): + obj = 0.0 + c = obj.__class__ + self.assertTrue(isimmutable(c)) + +class TestList(BaseObjectTest): + class C: + pass + + def __init__(self, *args, **kwargs): + obj = [self.C(), self.C(), 1, "two", None] + BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) + + def test_set_item(self): + with self.assertRaises(NotWriteableError): + self.obj[0] = None + + def test_set_slice(self): + with self.assertRaises(NotWriteableError): + self.obj[1:3] = [None, None] + + def test_append(self): + with self.assertRaises(NotWriteableError): + self.obj.append(TestList.C()) + + def test_extend(self): + with self.assertRaises(NotWriteableError): + self.obj.extend([TestList.C()]) + + def test_insert(self): + with self.assertRaises(NotWriteableError): + self.obj.insert(0, TestList.C()) + + def test_pop(self): + with self.assertRaises(NotWriteableError): + self.obj.pop() + + def test_remove(self): + with self.assertRaises(NotWriteableError): + self.obj.remove(1) + + def test_reverse(self): + with self.assertRaises(NotWriteableError): + self.obj.reverse() + + def test_clear(self): + with self.assertRaises(NotWriteableError): + self.obj.clear() + + def test_sort(self): + with self.assertRaises(NotWriteableError): + self.obj.sort() + + +class TestDict(BaseObjectTest): + class C: + pass + + def __init__(self, *args, **kwargs): + obj = {1: self.C(), "two": self.C()} + BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) + + def test_set_item_exists(self): + with self.assertRaises(NotWriteableError): + self.obj[1] = None + + def test_set_item_new(self): + with self.assertRaises(NotWriteableError): + self.obj["three"] = TestDict.C() + + def test_del_item(self): + with self.assertRaises(NotWriteableError): + del self.obj[1] + + def test_clear(self): + with self.assertRaises(NotWriteableError): + self.obj.clear() + + def test_pop(self): + with self.assertRaises(NotWriteableError): + self.obj.pop(1) + + def test_popitem(self): + with self.assertRaises(NotWriteableError): + self.obj.popitem() + + def test_setdefault(self): + with self.assertRaises(NotWriteableError): + self.obj.setdefault("three", TestDict.C()) + + def test_update(self): + with self.assertRaises(NotWriteableError): + self.obj.update({1: None}) + + +class TestSet(BaseObjectTest): + def __init__(self, *args, **kwargs): + obj = {1, "two", None, True} + BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) + + def test_add(self): + with self.assertRaises(NotWriteableError): + self.obj.add(1) + + def test_clear(self): + with self.assertRaises(NotWriteableError): + self.obj.clear() + + def test_discard(self): + with self.assertRaises(NotWriteableError): + self.obj.discard(1) + + def test_pop(self): + with self.assertRaises(NotWriteableError): + self.obj.pop() + + def test_remove(self): + with self.assertRaises(NotWriteableError): + self.obj.remove(1) + + def test_update(self): + with self.assertRaises(NotWriteableError): + self.obj.update([1, 2]) + + +class TestMultiLevel(unittest.TestCase): + def setUp(self): + class C: + const = 1 + + self.obj = C() + self.obj.a = C() + self.obj.a.b = "c" + self.obj.d = [C(), None] + self.obj.d[0].e = "f" + self.obj.g = {1: C(), "two": C()} + self.obj.g[1].h = True + self.obj.g["two"].i = False + freeze(self.obj) + + def test_immutable(self): + self.assertTrue(isimmutable(self.obj)) + self.assertTrue(isimmutable(self.obj.a)) + self.assertTrue(isimmutable(self.obj.a.b)) + self.assertTrue(isimmutable(self.obj.d)) + self.assertTrue(isimmutable(self.obj.d[0])) + self.assertTrue(isimmutable(self.obj.d[0].e)) + self.assertTrue(isimmutable(self.obj.g)) + self.assertTrue(isimmutable(self.obj.g[1])) + self.assertTrue(isimmutable(self.obj.g[1].h)) + self.assertTrue(isimmutable(self.obj.g["two"])) + self.assertTrue(isimmutable(self.obj.g["two"].i)) + + def test_set_const(self): + with self.assertRaises(NotWriteableError): + self.obj.const = 1 + + def test_type_immutable(self): + self.assertTrue(isimmutable(type(self.obj))) + self.assertTrue(isimmutable(type(self.obj).const)) + + +class TestFunctions(unittest.TestCase): + def setUp(self): + def a(): + return 1 + + self.obj = a + freeze(self.obj) + + def test_new_function(self): + def b(): + return 1 + + self.assertEqual(b(), 1) + + def test_nonlocal(self): + def c(): + v = 0 + + def inc(): + nonlocal v + v += 1 + return v + + return inc + + test = c() + self.assertEqual(test(), 1) + self.assertEqual(test(), 2) + freeze(test) + self.assertRaises(NotWriteableError, test) + + def test_global(self): + def d(): + global global0 + global0 += 1 + return global0 + + self.assertEqual(d(), 1) + freeze(d) + self.assertTrue(isimmutable(global0)) + self.assertFalse(isimmutable(global_canary)) + self.assertRaises(NotWriteableError, d) + + def test_builtins(self): + def e(): + test = list(range(5)) + return sum(test) + + freeze(e) + self.assertTrue(isimmutable(list)) + self.assertTrue(isimmutable(range)) + self.assertTrue(isimmutable(sum)) + + def test_builtins_nested(self): + def g(): + def nested_test(): + test = list(range(10)) + return sum(test) + + return nested_test() + + freeze(g) + self.assertTrue(isimmutable(list)) + self.assertTrue(isimmutable(range)) + self.assertTrue(isimmutable(sum)) + + def test_global_fun(self): + def d(): + return global1_inc() + + freeze(d) + self.assertTrue(isimmutable(global1)) + self.assertTrue(isimmutable(global1_inc)) + self.assertFalse(isimmutable(global_canary)) + self.assertRaises(NotWriteableError, d) + + def test_globals_copy(self): + def f(): + global global0 + return global0 + + expected = f() + freeze(f) + self.assertEqual(f(), expected) + global0 = 10 + self.assertEqual(f(), expected) + + +class TestMethods(unittest.TestCase): + class C: + def __init__(self): + self.val = -1 + + def a(self): + return abs(self.val) + + def b(self, x): + self.val = self.val + x + + def test_lambda(self): + obj = TestMethods.C() + obj.c = lambda x: pow(x, 2) + freeze(obj) + self.assertTrue(isimmutable(TestMethods.C)) + self.assertTrue(isimmutable(pow)) + self.assertRaises(NotWriteableError, obj.b, 1) + self.assertEqual(obj.c(2), 4) + + def test_method(self): + obj = TestMethods.C() + freeze(obj) + self.assertEqual(obj.a(), 1) + self.assertTrue(isimmutable(obj)) + self.assertTrue(isimmutable(abs)) + self.assertTrue(isimmutable(obj.val)) + self.assertRaises(NotWriteableError, obj.b, 1) + # Second test as the byte code can be changed by the first call + self.assertRaises(NotWriteableError, obj.b, 1) + + +class TestLocals(unittest.TestCase): + class C: + def __init__(self): + self.val = 0 + def a(self, locs): + self.l = locs + def test_locals(self): + # Inner scope used to prevent locals() containing self, + # and preventing the test updating state. + def inner(): + obj = TestLocals.C() + obj2 = TestLocals.C() + l = locals() + obj.a(l) + obj3 = TestLocals.C() + freeze(obj) + return obj, obj2, obj3 + obj, obj2, obj3 = inner() + self.assertTrue(isimmutable(obj)) + self.assertTrue(isimmutable(obj2)) + self.assertFalse(isimmutable(obj3)) + +class TestDictMutation(unittest.TestCase): + class C: + def __init__(self): + self.x = 0 + + def get(self): + return self.x + + def set(self, x): + d = self.__dict__ + d['x'] = x + + def test_dict_mutation(self): + obj = TestDictMutation.C() + freeze(obj) + self.assertTrue(isimmutable(obj)) + self.assertRaises(NotWriteableError, obj.set, 1) + self.assertEqual(obj.get(), 0) + + def test_dict_mutation2(self): + obj = TestDictMutation.C() + obj.set(1) + self.assertEqual(obj.get(), 1) + freeze(obj) + self.assertEqual(obj.get(), 1) + self.assertTrue(isimmutable(obj)) + self.assertRaises(NotWriteableError, obj.set, 1) + +class TestWeakRef(unittest.TestCase): + class B: + pass + + class C: + # Function that takes a object, and stores it in a weakref field. + def __init__(self, obj): + import weakref + self.obj = weakref.ref(obj) + def val(self): + return self.obj() + + def test_weakref(self): + obj = TestWeakRef.B() + c = TestWeakRef.C(obj) + freeze(c) + self.assertTrue(isimmutable(c)) + self.assertTrue(c.val() is obj) + # Following line is not true in the current implementation + # self.assertTrue(isimmutable(c.val())) + self.assertFalse(isimmutable(c.val())) + obj = None + # Following line is not true in the current implementation + # this means me can get a race on weak references + # self.assertTrue(c.val() is obj) + self.assertIsNone(c.val()) + +class TestStackCapture(unittest.TestCase): + def test_stack_capture(self): + import sys + x = {} + x["frame"] = sys._getframe() + freeze(x) + self.assertTrue(isimmutable(x)) + self.assertTrue(isimmutable(x["frame"])) + +global_test_dict = 0 +class TestGlobalDictMutation(unittest.TestCase): + def g(): + def f1(): + globals()["global_test_dict"] += 1 + return globals()["global_test_dict"] + freeze(f1) + return f1 + + def test_global_dict_mutation(self): + f1 = TestGlobalDictMutation.g() + self.assertTrue(isimmutable(f1)) + self.assertRaises(NotWriteableError, f1) + + +class TestSubclass(unittest.TestCase): + def test_subclass(self): + class C: + def __init__(self, val): + self.val = val + + def a(self): + return self.val + + def b(self, val): + self.val = val + + c_obj = C(1) + freeze(c_obj) + self.assertTrue(isimmutable(c_obj)) + self.assertTrue(isimmutable(C)) + class D(C): + def __init__(self, val): + super().__init__(val) + self.val2 = val * 2 + + def b(self): + return self.val2 + + def c(self, val): + self.val = val + + d_obj = D(1) + self.assertEqual(d_obj.a(), 1) + self.assertEqual(d_obj.b(), 2) + self.assertTrue(isinstance(d_obj, C)) + self.assertTrue(issubclass(D, C)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index ecf2c47c462e23..62b6edf299b4ec 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -135,6 +135,8 @@ def test_sys_modules_loader_is_not_set(self): del module.__spec__.loader except AttributeError: pass + except NotWriteableError: + pass sys.modules[name] = module with self.assertRaises(ValueError): self.util.find_spec(name) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 8cad71c7c34545..95eb05a1e6e39a 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -264,6 +264,7 @@ def test_windows_feature_macros(self): "PyExc_NameError", "PyExc_NotADirectoryError", "PyExc_NotImplementedError", + "PyExc_NotWriteableError", "PyExc_OSError", "PyExc_OverflowError", "PyExc_PendingDeprecationWarning", diff --git a/Makefile.pre.in b/Makefile.pre.in index 09ceccda1dcde5..7e479cc1552495 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -395,6 +395,7 @@ PYTHON_OBJS= \ Python/hashtable.o \ Python/import.o \ Python/importdl.o \ + Python/immutability.o \ Python/initconfig.o \ Python/instrumentation.o \ Python/intrinsics.o \ diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 48299e9b35ff97..c55e979663d59a 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2406,3 +2406,5 @@ added = '3.12' [const.Py_TPFLAGS_ITEMS_AT_END] added = '3.12' +[data.PyExc_NotWriteableError] + added = '3.12' diff --git a/Objects/abstract.c b/Objects/abstract.c index e95785900c9c5f..92a690e26035ff 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -34,6 +34,17 @@ null_error(void) return NULL; } +static PyObject * +immutable_error(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (!_PyErr_Occurred(tstate)) { + _PyErr_SetString(tstate, PyExc_NotWriteableError, + "cannot modify immutable instance"); + } + return NULL; +} + /* Operations on any object */ PyObject * @@ -209,6 +220,11 @@ PyObject_SetItem(PyObject *o, PyObject *key, PyObject *value) PyMappingMethods *m = Py_TYPE(o)->tp_as_mapping; if (m && m->mp_ass_subscript) { + if(!Py_CHECKWRITE(o)){ + immutable_error(); + return -1; + } + int res = m->mp_ass_subscript(o, key, value); assert(_Py_CheckSlotResult(o, "__setitem__", res >= 0)); return res; @@ -216,6 +232,11 @@ PyObject_SetItem(PyObject *o, PyObject *key, PyObject *value) if (Py_TYPE(o)->tp_as_sequence) { if (_PyIndex_Check(key)) { + if(!Py_CHECKWRITE(o)){ + immutable_error(); + return -1; + } + Py_ssize_t key_value; key_value = PyNumber_AsSsize_t(key, PyExc_IndexError); if (key_value == -1 && PyErr_Occurred()) @@ -243,6 +264,11 @@ PyObject_DelItem(PyObject *o, PyObject *key) PyMappingMethods *m = Py_TYPE(o)->tp_as_mapping; if (m && m->mp_ass_subscript) { + if(!Py_CHECKWRITE(o)){ + immutable_error(); + return -1; + } + int res = m->mp_ass_subscript(o, key, (PyObject*)NULL); assert(_Py_CheckSlotResult(o, "__delitem__", res >= 0)); return res; @@ -250,6 +276,11 @@ PyObject_DelItem(PyObject *o, PyObject *key) if (Py_TYPE(o)->tp_as_sequence) { if (_PyIndex_Check(key)) { + if(!Py_CHECKWRITE(o)){ + immutable_error(); + return -1; + } + Py_ssize_t key_value; key_value = PyNumber_AsSsize_t(key, PyExc_IndexError); if (key_value == -1 && PyErr_Occurred()) @@ -359,6 +390,12 @@ int PyObject_AsWriteBuffer(PyObject *obj, null_error(); return -1; } + + if(!Py_CHECKWRITE(obj)){ + immutable_error(); + return -1; + } + pb = Py_TYPE(obj)->tp_as_buffer; if (pb == NULL || pb->bf_getbuffer == NULL || @@ -387,8 +424,15 @@ PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) Py_TYPE(obj)->tp_name); return -1; } + + if((flags & PyBUF_WRITABLE) && !Py_CHECKWRITE(obj)){ + immutable_error(); + return -1; + } + int res = (*pb->bf_getbuffer)(obj, view, flags); assert(_Py_CheckSlotResult(obj, "getbuffer", res >= 0)); + return res; } @@ -1176,6 +1220,11 @@ binary_iop1(PyObject *v, PyObject *w, const int iop_slot, const int op_slot if (mv != NULL) { binaryfunc slot = NB_BINOP(mv, iop_slot); if (slot) { + if(!Py_CHECKWRITE(v)){ + immutable_error(); + return NULL; + } + PyObject *x = (slot)(v, w); assert(_Py_CheckSlotResult(v, op_name, x != NULL)); if (x != Py_NotImplemented) { @@ -1217,6 +1266,11 @@ ternary_iop(PyObject *v, PyObject *w, PyObject *z, const int iop_slot, const int if (mv != NULL) { ternaryfunc slot = NB_TERNOP(mv, iop_slot); if (slot) { + if(!Py_CHECKWRITE(v)){ + immutable_error(); + return NULL; + } + PyObject *x = (slot)(v, w, z); if (x != Py_NotImplemented) { return x; @@ -1803,6 +1857,10 @@ PySequence_InPlaceConcat(PyObject *s, PyObject *o) PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; if (m && m->sq_inplace_concat) { + if(!Py_CHECKWRITE(s)){ + return immutable_error(); + } + PyObject *res = m->sq_inplace_concat(s, o); assert(_Py_CheckSlotResult(s, "+=", res != NULL)); return res; @@ -1832,6 +1890,10 @@ PySequence_InPlaceRepeat(PyObject *o, Py_ssize_t count) PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; if (m && m->sq_inplace_repeat) { + if (!Py_CHECKWRITE(o)){ + return immutable_error(); + } + PyObject *res = m->sq_inplace_repeat(o, count); assert(_Py_CheckSlotResult(o, "*=", res != NULL)); return res; @@ -1919,6 +1981,11 @@ PySequence_SetItem(PyObject *s, Py_ssize_t i, PyObject *o) PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; if (m && m->sq_ass_item) { + if (!Py_CHECKWRITE(s)){ + immutable_error(); + return -1; + } + if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); @@ -1952,6 +2019,11 @@ PySequence_DelItem(PyObject *s, Py_ssize_t i) PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence; if (m && m->sq_ass_item) { + if(!Py_CHECKWRITE(s)){ + immutable_error(); + return -1; + } + if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); @@ -1985,6 +2057,11 @@ PySequence_SetSlice(PyObject *s, Py_ssize_t i1, Py_ssize_t i2, PyObject *o) PyMappingMethods *mp = Py_TYPE(s)->tp_as_mapping; if (mp && mp->mp_ass_subscript) { + if (!Py_CHECKWRITE(s)){ + immutable_error(); + return -1; + } + PyObject *slice = _PySlice_FromIndices(i1, i2); if (!slice) return -1; @@ -2008,6 +2085,11 @@ PySequence_DelSlice(PyObject *s, Py_ssize_t i1, Py_ssize_t i2) PyMappingMethods *mp = Py_TYPE(s)->tp_as_mapping; if (mp && mp->mp_ass_subscript) { + if(!Py_CHECKWRITE(s)){ + immutable_error(); + return -1; + } + PyObject *slice = _PySlice_FromIndices(i1, i2); if (!slice) { return -1; diff --git a/Objects/cellobject.c b/Objects/cellobject.c index f516707f6f8086..430a7efe0dcfe2 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -66,6 +66,12 @@ PyCell_Set(PyObject *op, PyObject *value) PyErr_BadInternalCall(); return -1; } + + if (!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return -1; + } + PyObject *old_value = PyCell_GET(op); PyCell_SET(op, Py_XNewRef(value)); Py_XDECREF(old_value); @@ -121,6 +127,11 @@ cell_traverse(PyCellObject *op, visitproc visit, void *arg) static int cell_clear(PyCellObject *op) { + if(!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return -1; + } + Py_CLEAR(op->ob_ref); return 0; } @@ -139,6 +150,11 @@ cell_get_contents(PyCellObject *op, void *closure) static int cell_set_contents(PyCellObject *op, PyObject *obj, void *Py_UNUSED(ignored)) { + if(!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return -1; + } + Py_XSETREF(op->ob_ref, Py_XNewRef(obj)); return 0; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 254cd9ad2f9bda..56d1b520b6ab18 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -497,11 +497,11 @@ dump_entries(PyDictKeysObject *dk) for (Py_ssize_t i = 0; i < dk->dk_nentries; i++) { if (DK_IS_UNICODE(dk)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(dk)[i]; - printf("key=%p value=%p\n", ep->me_key, ep->me_value); + printf("key=%p value=%p\n", ep->me_key, _PyDictEntry_Value(ep)); } else { PyDictKeyEntry *ep = &DK_ENTRIES(dk)[i]; - printf("key=%p hash=%lx value=%p\n", ep->me_key, ep->me_hash, ep->me_value); + printf("key=%p hash=%lx value=%p\n", ep->me_key, ep->me_hash, _PyDictEntry_Value(ep)); } } } @@ -551,7 +551,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) if (key != NULL) { /* test_dict fails if PyObject_Hash() is called again */ CHECK(entry->me_hash != -1); - CHECK(entry->me_value != NULL); + CHECK(_PyDictEntry_Value(entry) != NULL); if (PyUnicode_CheckExact(key)) { Py_hash_t hash = unicode_get_hash(key); @@ -571,12 +571,12 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) Py_hash_t hash = unicode_get_hash(key); CHECK(hash != -1); if (!splitted) { - CHECK(entry->me_value != NULL); + CHECK(_PyDictEntry_Value(entry) != NULL); } } if (splitted) { - CHECK(entry->me_value == NULL); + CHECK(_PyDictEntry_Value(entry) == NULL); } } } @@ -670,7 +670,7 @@ free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) Py_ssize_t i, n; for (i = 0, n = keys->dk_nentries; i < n; i++) { Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); + Py_XDECREF(_PyDictEntry_Value(entries + i)); } } else { @@ -678,7 +678,7 @@ free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) Py_ssize_t i, n; for (i = 0, n = keys->dk_nentries; i < n; i++) { Py_XDECREF(entries[i].me_key); - Py_XDECREF(entries[i].me_value); + Py_XDECREF(_PyDictEntry_Value(entries + i)); } } #if PyDict_MAXFREELIST > 0 @@ -812,19 +812,19 @@ clone_combined_dict_keys(PyDictObject *orig) if (DK_IS_UNICODE(orig->ma_keys)) { PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(keys); pkey = &ep0->me_key; - pvalue = &ep0->me_value; + pvalue = &ep0->_me_value; offs = sizeof(PyDictUnicodeEntry) / sizeof(PyObject*); } else { PyDictKeyEntry *ep0 = DK_ENTRIES(keys); pkey = &ep0->me_key; - pvalue = &ep0->me_value; + pvalue = &ep0->_me_value; offs = sizeof(PyDictKeyEntry) / sizeof(PyObject*); } Py_ssize_t n = keys->dk_nentries; for (Py_ssize_t i = 0; i < n; i++) { - PyObject *value = *pvalue; + PyObject *value = (PyObject*)(((uintptr_t)*pvalue >> 1) << 1); if (value != NULL) { Py_INCREF(value); Py_INCREF(*pkey); @@ -1075,7 +1075,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu *value_addr = mp->ma_values->values[ix]; } else { - *value_addr = DK_UNICODE_ENTRIES(dk)[ix].me_value; + *value_addr = _PyDictEntry_Value(DK_UNICODE_ENTRIES(dk) + ix); } } else { @@ -1088,7 +1088,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu goto start; } if (ix >= 0) { - *value_addr = DK_ENTRIES(dk)[ix].me_value; + *value_addr = _PyDictEntry_Value(DK_ENTRIES(dk) + ix); } else { *value_addr = NULL; @@ -1146,21 +1146,21 @@ _PyDict_MaybeUntrack(PyObject *op) } else { if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++) { - if ((value = ep0[i].me_value) == NULL) + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); + for (i = 0; i < numentries; i++, ep++) { + if ((value = _PyDictEntry_Value(ep)) == NULL) continue; if (_PyObject_GC_MAY_BE_TRACKED(value)) return; } } else { - PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++) { - if ((value = ep0[i].me_value) == NULL) + PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); + for (i = 0; i < numentries; i++, ep++) { + if ((value = _PyDictEntry_Value(ep)) == NULL) continue; if (_PyObject_GC_MAY_BE_TRACKED(value) || - _PyObject_GC_MAY_BE_TRACKED(ep0[i].me_key)) + _PyObject_GC_MAY_BE_TRACKED(_PyDictEntry_Key(ep))) return; } } @@ -1238,6 +1238,11 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, { PyObject *old_value; + if (!Py_CHECKWRITE(mp)){ + PyErr_WriteToImmutable(mp); + goto Fail; + } + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { if (insertion_resize(interp, mp, 0) < 0) goto Fail; @@ -1276,7 +1281,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, mp->ma_values->values[index] = value; } else { - ep->me_value = value; + _PyDictEntry_SetValue(ep, value); } } else { @@ -1284,7 +1289,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; ep->me_key = key; ep->me_hash = hash; - ep->me_value = value; + _PyDictEntry_SetValue(ep, value); } mp->ma_used++; mp->ma_version_tag = new_version; @@ -1296,6 +1301,22 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } if (old_value != value) { + if(DK_IS_UNICODE(mp->ma_keys)){ + PyDictUnicodeEntry *ep; + ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + if(_PyDictEntry_IsImmutable(ep)){ + PyErr_WriteToImmutableKey(_PyDictEntry_Key(ep)); + goto Fail; + } + }else{ + PyDictKeyEntry* ep; + ep = &DK_ENTRIES(mp->ma_keys)[ix]; + if(_PyDictEntry_IsImmutable(ep)){ + PyErr_WriteToImmutableKey(_PyDictEntry_Key(ep)); + goto Fail; + } + } + uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_MODIFIED, mp, key, value); if (_PyDict_HasSplitTable(mp)) { @@ -1308,10 +1329,10 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, else { assert(old_value != NULL); if (DK_IS_UNICODE(mp->ma_keys)) { - DK_UNICODE_ENTRIES(mp->ma_keys)[ix].me_value = value; + _PyDictEntry_SetValue(DK_UNICODE_ENTRIES(mp->ma_keys) + ix, value); } else { - DK_ENTRIES(mp->ma_keys)[ix].me_value = value; + _PyDictEntry_SetValue(DK_ENTRIES(mp->ma_keys) + ix, value); } } mp->ma_version_tag = new_version; @@ -1335,6 +1356,13 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, { assert(mp->ma_keys == Py_EMPTY_KEYS); + if (!Py_CHECKWRITE(mp)){ + PyErr_WriteToImmutable(mp); + Py_DECREF(key); + Py_DECREF(value); + return -1; + } + uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, value); @@ -1357,13 +1385,13 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, if (unicode) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); ep->me_key = key; - ep->me_value = value; + _PyDictEntry_SetValue(ep, value); } else { PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); ep->me_key = key; ep->me_hash = hash; - ep->me_value = value; + _PyDictEntry_SetValue(ep, value); } mp->ma_used++; mp->ma_version_tag = new_version; @@ -1464,26 +1492,34 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, if (mp->ma_keys->dk_kind == DICT_KEYS_GENERAL) { // split -> generic PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *new_ep = newentries; - for (Py_ssize_t i = 0; i < numentries; i++) { + for (Py_ssize_t i = 0; i < numentries; i++, new_ep++) { int index = get_index_from_order(mp, i); PyDictUnicodeEntry *ep = &oldentries[index]; assert(oldvalues->values[index] != NULL); - newentries[i].me_key = Py_NewRef(ep->me_key); - newentries[i].me_hash = unicode_get_hash(ep->me_key); - newentries[i].me_value = oldvalues->values[index]; + new_ep->me_key = Py_NewRef(ep->me_key); + new_ep->me_hash = unicode_get_hash(ep->me_key); + _PyDictEntry_SetValue(new_ep, oldvalues->values[index]); + if(_PyDictEntry_IsImmutable(ep)){ + _PyDictEntry_SetImmutable(new_ep); + } } build_indices_generic(mp->ma_keys, newentries, numentries); } else { // split -> combined unicode PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); + PyDictUnicodeEntry *new_ep = newentries; - for (Py_ssize_t i = 0; i < numentries; i++) { + for (Py_ssize_t i = 0; i < numentries; i++, new_ep++) { int index = get_index_from_order(mp, i); PyDictUnicodeEntry *ep = &oldentries[index]; assert(oldvalues->values[index] != NULL); - newentries[i].me_key = Py_NewRef(ep->me_key); - newentries[i].me_value = oldvalues->values[index]; + new_ep->me_key = Py_NewRef(ep->me_key); + _PyDictEntry_SetValue(new_ep, oldvalues->values[index]); + if(_PyDictEntry_IsImmutable(ep)){ + _PyDictEntry_SetImmutable(new_ep); + } } build_indices_unicode(mp->ma_keys, newentries, numentries); } @@ -1503,7 +1539,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, else { PyDictKeyEntry *ep = oldentries; for (Py_ssize_t i = 0; i < numentries; i++) { - while (ep->me_value == NULL) + while (_PyDictEntry_IsEmpty(ep)) ep++; newentries[i] = *ep++; } @@ -1520,7 +1556,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, else { PyDictUnicodeEntry *ep = oldentries; for (Py_ssize_t i = 0; i < numentries; i++) { - while (ep->me_value == NULL) + while (_PyDictEntry_IsEmpty(ep)) ep++; newentries[i] = *ep++; } @@ -1529,14 +1565,17 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } else { // combined unicode -> generic PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); + PyDictKeyEntry *new_ep = newentries; PyDictUnicodeEntry *ep = oldentries; - for (Py_ssize_t i = 0; i < numentries; i++) { - while (ep->me_value == NULL) + for (Py_ssize_t i = 0; i < numentries; i++, ep++, new_ep++) { + while (_PyDictEntry_IsEmpty(ep)) ep++; - newentries[i].me_key = ep->me_key; - newentries[i].me_hash = unicode_get_hash(ep->me_key); - newentries[i].me_value = ep->me_value; - ep++; + new_ep->me_key = ep->me_key; + new_ep->me_hash = unicode_get_hash(ep->me_key); + _PyDictEntry_SetValue(new_ep, _PyDictEntry_Value(ep)); + if(_PyDictEntry_IsImmutable(ep)){ + _PyDictEntry_SetImmutable(new_ep); + } } build_indices_generic(mp->ma_keys, newentries, numentries); } @@ -1841,6 +1880,40 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) return value; } +PyObject *_PyDict_SetKeyImmutable(PyDictObject* dict, PyObject *key) +{ + Py_ssize_t ix; + Py_hash_t hash; + PyObject *value; + + if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { + hash = PyObject_Hash(key); + if (hash == -1) + return NULL; + } + + /* namespace 1: globals */ + ix = _Py_dict_lookup(dict, key, hash, &value); + if (ix == DKIX_ERROR){ + return NULL; + } + + if(ix == DKIX_EMPTY){ + _PyErr_SetKeyError(key); + return NULL; + } + + if(DK_IS_UNICODE(dict->ma_keys)){ + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(dict->ma_keys)[ix]; + _PyDictEntry_SetImmutable(ep); + }else{ + PyDictKeyEntry *ep = &DK_ENTRIES(dict->ma_keys)[ix]; + _PyDictEntry_SetImmutable(ep); + } + + return value; +} + /* Consumes references to key and value */ int _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) @@ -1947,15 +2020,25 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY); if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + if(_PyDictEntry_IsImmutable(ep)){ + PyErr_WriteToImmutableKey(ep->me_key); + return -1; + } + old_key = ep->me_key; ep->me_key = NULL; - ep->me_value = NULL; + _PyDictEntry_SetValue(ep, NULL); } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; + if(_PyDictEntry_IsImmutable(ep)){ + PyErr_WriteToImmutableKey(ep->me_key); + return -1; + } + old_key = ep->me_key; ep->me_key = NULL; - ep->me_value = NULL; + _PyDictEntry_SetValue(ep, NULL); ep->me_hash = 0; } Py_DECREF(old_key); @@ -2002,6 +2085,11 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return -1; } + if(!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return -1; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, mp, key, NULL); @@ -2065,6 +2153,11 @@ PyDict_Clear(PyObject *op) PyDictValues *oldvalues; Py_ssize_t i, n; + if(!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return; + } + if (!PyDict_Check(op)) return; mp = ((PyDictObject *)op); @@ -2132,7 +2225,7 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, return 0; if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(mp->ma_keys)[i]; - while (i < n && entry_ptr->me_value == NULL) { + while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { entry_ptr++; i++; } @@ -2140,11 +2233,11 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, return 0; key = entry_ptr->me_key; hash = unicode_get_hash(entry_ptr->me_key); - value = entry_ptr->me_value; + value = _PyDictEntry_Value(entry_ptr); } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(mp->ma_keys)[i]; - while (i < n && entry_ptr->me_value == NULL) { + while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { entry_ptr++; i++; } @@ -2152,7 +2245,7 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, return 0; key = entry_ptr->me_key; hash = entry_ptr->me_hash; - value = entry_ptr->me_value; + value = _PyDictEntry_Value(entry_ptr); } } *ppos = i+1; @@ -2703,6 +2796,10 @@ dict_update_common(PyObject *self, PyObject *args, PyObject *kwds, static PyObject * dict_update(PyObject *self, PyObject *args, PyObject *kwds) { + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (dict_update_common(self, args, kwds, "update") != -1) Py_RETURN_NONE; return NULL; @@ -3164,12 +3261,12 @@ dict_equal(PyDictObject *a, PyDictObject *b) if (a->ma_values) aval = a->ma_values->values[i]; else - aval = ep->me_value; + aval = _PyDictEntry_Value(ep); } else { PyDictKeyEntry *ep = &DK_ENTRIES(a->ma_keys)[i]; key = ep->me_key; - aval = ep->me_value; + aval = _PyDictEntry_Value(ep); hash = ep->me_hash; } if (aval != NULL) { @@ -3299,6 +3396,10 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; } + if(!Py_CHECKWRITE(d)){ + return PyErr_WriteToImmutable(d); + } + if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { hash = PyObject_Hash(key); if (hash == -1) @@ -3347,14 +3448,14 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) _PyDictValues_AddToInsertionOrder(mp->ma_values, index); } else { - ep->me_value = Py_NewRef(value); + _PyDictEntry_SetValue(ep, Py_NewRef(value)); } } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; ep->me_key = Py_NewRef(key); ep->me_hash = hash; - ep->me_value = Py_NewRef(value); + _PyDictEntry_SetValue(ep, Py_NewRef(value)); } MAINTAIN_TRACKING(mp, key, value); mp->ma_used++; @@ -3406,6 +3507,10 @@ dict_setdefault_impl(PyDictObject *self, PyObject *key, static PyObject * dict_clear(PyDictObject *mp, PyObject *Py_UNUSED(ignored)) { + if(!Py_CHECKWRITE(mp)){ + return PyErr_WriteToImmutable(mp); + } + PyDict_Clear((PyObject *)mp); Py_RETURN_NONE; } @@ -3427,6 +3532,10 @@ static PyObject * dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value) /*[clinic end generated code: output=3abb47b89f24c21c input=e221baa01044c44c]*/ { + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + return _PyDict_Pop((PyObject*)self, key, default_value); } @@ -3448,6 +3557,10 @@ dict_popitem_impl(PyDictObject *self) uint64_t new_version; PyInterpreterState *interp = _PyInterpreterState_GET(); + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which * could empty the dict, so if we checked the size first and that @@ -3480,35 +3593,43 @@ dict_popitem_impl(PyDictObject *self) if (DK_IS_UNICODE(self->ma_keys)) { PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(self->ma_keys); i = self->ma_keys->dk_nentries - 1; - while (i >= 0 && ep0[i].me_value == NULL) { + while (i >= 0 && _PyDictEntry_IsEmpty(ep0 + i)) { i--; } assert(i >= 0); + if(_PyDictEntry_IsImmutable(ep0 + i)){ + return PyErr_WriteToImmutableKey(ep0[i].me_key); + } + key = ep0[i].me_key; new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); - value = ep0[i].me_value; + value = _PyDictEntry_Value(ep0 + i); ep0[i].me_key = NULL; - ep0[i].me_value = NULL; + _PyDictEntry_SetValue(ep0 + i, NULL); } else { PyDictKeyEntry *ep0 = DK_ENTRIES(self->ma_keys); i = self->ma_keys->dk_nentries - 1; - while (i >= 0 && ep0[i].me_value == NULL) { + while (i >= 0 && _PyDictEntry_IsEmpty(ep0 + i)) { i--; } assert(i >= 0); + if(_PyDictEntry_IsImmutable(ep0 + i)){ + return PyErr_WriteToImmutableKey(ep0[i].me_key); + } + key = ep0[i].me_key; new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; - value = ep0[i].me_value; + value = _PyDictEntry_Value(ep0 + i); ep0[i].me_key = NULL; ep0[i].me_hash = -1; - ep0[i].me_value = NULL; + _PyDictEntry_SetValue(ep0 + i, NULL); } j = lookdict_index(self->ma_keys, hash, i); @@ -3542,15 +3663,15 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) else { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); for (i = 0; i < n; i++) { - Py_VISIT(entries[i].me_value); + Py_VISIT(_PyDictEntry_Value(entries + i)); } } } else { PyDictKeyEntry *entries = DK_ENTRIES(keys); for (i = 0; i < n; i++) { - if (entries[i].me_value != NULL) { - Py_VISIT(entries[i].me_value); + if (!_PyDictEntry_IsEmpty(entries + i)) { + Py_VISIT(_PyDictEntry_Value(entries + i)); Py_VISIT(entries[i].me_key); } } @@ -4069,7 +4190,7 @@ dictiter_iternextkey(dictiterobject *di) Py_ssize_t n = k->dk_nentries; if (DK_IS_UNICODE(k)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; - while (i < n && entry_ptr->me_value == NULL) { + while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { entry_ptr++; i++; } @@ -4079,7 +4200,7 @@ dictiter_iternextkey(dictiterobject *di) } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; - while (i < n && entry_ptr->me_value == NULL) { + while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { entry_ptr++; i++; } @@ -4168,23 +4289,23 @@ dictiter_iternextvalue(dictiterobject *di) Py_ssize_t n = d->ma_keys->dk_nentries; if (DK_IS_UNICODE(d->ma_keys)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(d->ma_keys)[i]; - while (i < n && entry_ptr->me_value == NULL) { + while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { entry_ptr++; i++; } if (i >= n) goto fail; - value = entry_ptr->me_value; + value = _PyDictEntry_Value(entry_ptr); } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[i]; - while (i < n && entry_ptr->me_value == NULL) { + while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { entry_ptr++; i++; } if (i >= n) goto fail; - value = entry_ptr->me_value; + value = _PyDictEntry_Value(entry_ptr); } } // We found an element, but did not expect it @@ -4268,25 +4389,25 @@ dictiter_iternextitem(dictiterobject *di) Py_ssize_t n = d->ma_keys->dk_nentries; if (DK_IS_UNICODE(d->ma_keys)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(d->ma_keys)[i]; - while (i < n && entry_ptr->me_value == NULL) { + while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { entry_ptr++; i++; } if (i >= n) goto fail; key = entry_ptr->me_key; - value = entry_ptr->me_value; + value = _PyDictEntry_Value(entry_ptr); } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[i]; - while (i < n && entry_ptr->me_value == NULL) { + while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { entry_ptr++; i++; } if (i >= n) goto fail; key = entry_ptr->me_key; - value = entry_ptr->me_value; + value = _PyDictEntry_Value(entry_ptr); } } // We found an element, but did not expect it @@ -4396,25 +4517,25 @@ dictreviter_iternext(dictiterobject *di) else { if (DK_IS_UNICODE(k)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; - while (entry_ptr->me_value == NULL) { + while (_PyDictEntry_IsEmpty(entry_ptr)) { if (--i < 0) { goto fail; } entry_ptr--; } key = entry_ptr->me_key; - value = entry_ptr->me_value; + value = _PyDictEntry_Value(entry_ptr); } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; - while (entry_ptr->me_value == NULL) { + while (_PyDictEntry_IsEmpty(entry_ptr)) { if (--i < 0) { goto fail; } entry_ptr--; } key = entry_ptr->me_key; - value = entry_ptr->me_value; + value = _PyDictEntry_Value(entry_ptr); } } di->di_pos = i-1; @@ -5428,6 +5549,12 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, assert(keys != NULL); assert(values != NULL); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + + if(!Py_CHECKWRITE(obj)){ + PyErr_WriteToImmutable(obj); + return -1; + } + Py_ssize_t ix = DKIX_EMPTY; if (PyUnicode_CheckExact(name)) { ix = insert_into_dictkeys(keys, name); @@ -5625,6 +5752,11 @@ _PyObject_ClearManagedDict(PyObject *obj) PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { + // TODO: Pyrona: This method does not change the representation in the frozen case. + // However, repeated calls to __dict__ on a frozen object will all create a new dict. + // Alternative designs + // * Error on this case + // * Introduce locking, and return the same dict. PyObject *dict; PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); @@ -5636,7 +5768,12 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = make_dict_from_instance_attributes( interp, CACHED_KEYS(tp), values); if (dict != NULL) { - dorv_ptr->dict = dict; + if (_Py_IsImmutable(obj)) { + _Py_SetImmutable(dict); + } + else { + dorv_ptr->dict = dict; + } } } else { @@ -5644,7 +5781,12 @@ PyObject_GenericGetDict(PyObject *obj, void *context) if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - dorv_ptr->dict = dict; + if (_Py_IsImmutable(obj)) { + _Py_SetImmutable(dict); + } + else { + dorv_ptr->dict = dict; + } } } } @@ -5660,11 +5802,17 @@ PyObject_GenericGetDict(PyObject *obj, void *context) PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys( + dict = new_dict_with_shared_keys( interp, CACHED_KEYS(tp)); } else { - *dictptr = dict = PyDict_New(); + dict = PyDict_New(); + } + if (_Py_IsImmutable(obj)) { + _Py_SetImmutable(dict); + } + else { + *dictptr = dict; } } } @@ -5848,3 +5996,83 @@ _PyDict_SendEvent(int watcher_bits, watcher_bits >>= 1; } } + +PyObject * +_PyDict_IsKeyImmutable(PyObject* op, PyObject* key) +{ + PyDictKeysObject *dk; + DictKeysKind kind; + Py_ssize_t ix; + PyDictObject* mp; + Py_hash_t hash; + PyThreadState* tstate; + PyObject *exc; + + if (!PyDict_Check(op)) { + return NULL; + } + mp = (PyDictObject *)op; + + if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { + hash = PyObject_Hash(key); + if (hash == -1) { + PyErr_Clear(); + return NULL; + } + } + + tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG + // bpo-40839: Before Python 3.10, it was possible to call PyDict_GetItem() + // with the GIL released. + _Py_EnsureTstateNotNULL(tstate); +#endif + + /* Preserve the existing exception */ + exc = _PyErr_GetRaisedException(tstate); + +start: + dk = mp->ma_keys; + kind = dk->dk_kind; + + if (kind != DICT_KEYS_GENERAL) { + if (PyUnicode_CheckExact(key)) { + ix = unicodekeys_lookup_unicode(dk, key, hash); + } + else { + ix = unicodekeys_lookup_generic(mp, dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + goto start; + } + } + } + else { + ix = dictkeys_generic_lookup(mp, dk, key, hash); + if (ix == DKIX_KEY_CHANGED) { + goto start; + } + } + + /* Ignore any exception raised by the lookup */ + _PyErr_SetRaisedException(tstate, exc); + + if (ix == DKIX_ERROR){ + return NULL; + } + + if (DK_IS_UNICODE(mp->ma_keys)) { + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys) + ix; + if (_PyDictEntry_IsImmutable(ep)){ + Py_RETURN_TRUE; + }else{ + Py_RETURN_FALSE; + } + } else { + PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys) + ix; + if(_PyDictEntry_IsImmutable(ep)){ + Py_RETURN_TRUE; + }else{ + Py_RETURN_FALSE; + } + } +} \ No newline at end of file diff --git a/Objects/exceptions.c b/Objects/exceptions.c index f376ff24386a37..9dbab8669527e4 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3430,6 +3430,11 @@ PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; */ SimpleExtendsException(PyExc_Exception, BufferError, "Buffer error."); +/* + * NotWriteableError extends Exception + */ +SimpleExtendsException(PyExc_Exception, NotWriteableError, "Object is not writeable."); + /* Warning category docstrings */ @@ -3615,6 +3620,7 @@ static struct static_exception static_exceptions[] = { ITEM(SystemError), ITEM(TypeError), ITEM(ValueError), + ITEM(NotWriteableError), ITEM(Warning), // Level 4: ArithmeticError(Exception) subclasses diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 30c8d3c5270cad..fd2b39c791dd43 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1433,6 +1433,10 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) if (cell != NULL) { oldvalue = PyCell_GET(cell); if (value != oldvalue) { + if (!Py_CHECKWRITE(cell)) { + PyErr_WriteToImmutable(cell); + return; + } PyCell_SET(cell, Py_XNewRef(value)); Py_XDECREF(oldvalue); } diff --git a/Objects/listobject.c b/Objects/listobject.c index f1edfb3a9a039d..fd94aec47a8453 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -265,6 +265,13 @@ PyList_SetItem(PyObject *op, Py_ssize_t i, PyErr_BadInternalCall(); return -1; } + + if(!Py_CHECKWRITE(op)){ + Py_XDECREF(newitem); + PyErr_WriteToImmutable(op); + return -1; + } + if (!valid_index(i, Py_SIZE(op))) { Py_XDECREF(newitem); PyErr_SetString(PyExc_IndexError, @@ -311,6 +318,12 @@ PyList_Insert(PyObject *op, Py_ssize_t where, PyObject *newitem) PyErr_BadInternalCall(); return -1; } + + if (!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return -1; + } + return ins1((PyListObject *)op, where, newitem); } @@ -331,6 +344,11 @@ _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) int PyList_Append(PyObject *op, PyObject *newitem) { + if (!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return -1; + } + if (PyList_Check(op) && (newitem != NULL)) { return _PyList_AppendTakeRef((PyListObject *)op, Py_NewRef(newitem)); } @@ -731,6 +749,12 @@ PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) PyErr_BadInternalCall(); return -1; } + + if(!Py_CHECKWRITE(a)){ + PyErr_WriteToImmutable(a); + return -1; + } + return list_ass_slice((PyListObject *)a, ilow, ihigh, v); } @@ -793,6 +817,10 @@ static PyObject * list_insert_impl(PyListObject *self, Py_ssize_t index, PyObject *object) /*[clinic end generated code: output=7f35e32f60c8cb78 input=858514cf894c7eab]*/ { + if (!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (ins1(self, index, object) == 0) Py_RETURN_NONE; return NULL; @@ -808,6 +836,10 @@ static PyObject * list_clear_impl(PyListObject *self) /*[clinic end generated code: output=67a1896c01f74362 input=ca3c1646856742f6]*/ { + if (!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + _list_clear(self); Py_RETURN_NONE; } @@ -838,6 +870,11 @@ static PyObject * list_append(PyListObject *self, PyObject *object) /*[clinic end generated code: output=7c096003a29c0eae input=43a3fe48a7066e91]*/ { + if (!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return NULL; + } + if (_PyList_AppendTakeRef(self, Py_NewRef(object)) < 0) { return NULL; } @@ -863,6 +900,11 @@ list_extend(PyListObject *self, PyObject *iterable) Py_ssize_t i; PyObject *(*iternext)(PyObject *); + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return NULL; + } + /* Special cases: 1) lists and tuples which can use PySequence_Fast ops 2) extending self to self requires making a copy first @@ -1012,6 +1054,10 @@ list_pop_impl(PyListObject *self, Py_ssize_t index) PyObject *v; int status; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (Py_SIZE(self) == 0) { /* Special-case most common failure cause */ PyErr_SetString(PyExc_IndexError, "pop from empty list"); @@ -2261,6 +2307,10 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) Py_ssize_t i; PyObject **keys; + if (!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + assert(self != NULL); assert(PyList_Check(self)); if (keyfunc == Py_None) @@ -2534,6 +2584,10 @@ static PyObject * list_reverse_impl(PyListObject *self) /*[clinic end generated code: output=482544fc451abea9 input=eefd4c3ae1bc9887]*/ { + if (!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (Py_SIZE(self) > 1) reverse_slice(self->ob_item, self->ob_item + Py_SIZE(self)); Py_RETURN_NONE; @@ -2548,6 +2602,12 @@ PyList_Reverse(PyObject *v) PyErr_BadInternalCall(); return -1; } + + if (!Py_CHECKWRITE(v)){ + PyErr_WriteToImmutable(v); + return -1; + } + if (Py_SIZE(self) > 1) reverse_slice(self->ob_item, self->ob_item + Py_SIZE(self)); return 0; @@ -2676,6 +2736,10 @@ static PyObject * list_remove(PyListObject *self, PyObject *value) /*[clinic end generated code: output=f087e1951a5e30d1 input=2dc2ba5bb2fb1f82]*/ { + if (!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + Py_ssize_t i; for (i = 0; i < Py_SIZE(self); i++) { diff --git a/Objects/longobject.c b/Objects/longobject.c index 5d9b413861478a..abffcb68db0c7e 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1775,6 +1775,7 @@ long_to_decimal_string_internal(PyObject *aa, } size_a = _PyLong_DigitCount(a); negative = _PyLong_IsNegative(a); + kind = 0; /* quick and dirty pre-check for overflowing the decimal digit limit, based on the inequality 10/3 >= log2(10) @@ -2006,6 +2007,7 @@ long_format_binary(PyObject *aa, int base, int alternate, } size_a = _PyLong_DigitCount(a); negative = _PyLong_IsNegative(a); + kind = 0; /* Compute a rough upper bound for the length of the string */ switch (base) { diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 4daf1a929e0549..5d44796a3ef092 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -7,6 +7,7 @@ #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_dict.h" // _PyDict_IsKeyImmutable() #include "structmember.h" // PyMemberDef @@ -600,8 +601,9 @@ void _PyModule_Clear(PyObject *m) { PyObject *d = ((PyModuleObject *)m)->md_dict; - if (d != NULL) + if (d != NULL){ _PyModule_ClearDict(d); + } } void @@ -622,7 +624,7 @@ _PyModule_ClearDict(PyObject *d) /* First, clear only names starting with a single underscore */ pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { - if (value != Py_None && PyUnicode_Check(key)) { + if (value != Py_None && PyUnicode_Check(key) && Py_IsFalse(_PyDict_IsKeyImmutable(d, key))) { if (PyUnicode_READ_CHAR(key, 0) == '_' && PyUnicode_READ_CHAR(key, 1) != '_') { if (verbose > 1) { @@ -642,7 +644,7 @@ _PyModule_ClearDict(PyObject *d) /* Next, clear all names except for __builtins__ */ pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { - if (value != Py_None && PyUnicode_Check(key)) { + if (value != Py_None && PyUnicode_Check(key) && Py_IsFalse(_PyDict_IsKeyImmutable(d, key))) { if (PyUnicode_READ_CHAR(key, 0) != '_' || !_PyUnicode_EqualToASCIIString(key, "__builtins__")) { diff --git a/Objects/object.c b/Objects/object.c index 8a4010fb13669b..83f4d006f32eb0 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1172,7 +1172,13 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) PyUnicode_InternInPlace(&name); if (tp->tp_setattro != NULL) { - err = (*tp->tp_setattro)(v, name, value); + if(Py_CHECKWRITE(v)){ + err = (*tp->tp_setattro)(v, name, value); + }else{ + PyErr_WriteToImmutable(v); + err = -1; + } + Py_DECREF(name); return err; } @@ -1182,7 +1188,14 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) Py_DECREF(name); return -1; } - err = (*tp->tp_setattr)(v, (char *)name_str, value); + + if(Py_CHECKWRITE(v)){ + err = (*tp->tp_setattr)(v, (char *)name_str, value); + }else{ + PyErr_WriteToImmutable(v); + err = -1; + } + Py_DECREF(name); return err; } @@ -1622,6 +1635,11 @@ PyObject_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) int PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { + if(!Py_CHECKWRITE(obj)){ + PyErr_WriteToImmutable(obj); + return -1; + } + PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) && diff --git a/Objects/setobject.c b/Objects/setobject.c index 58f0ae73c0c403..6474568d3f8205 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -633,6 +633,10 @@ set_pop(PySetObject *so, PyObject *Py_UNUSED(ignored)) setentry *limit = so->table + so->mask; PyObject *key; + if(!Py_CHECKWRITE(so)){ + return PyErr_WriteToImmutable(so); + } + if (so->used == 0) { PyErr_SetString(PyExc_KeyError, "pop from an empty set"); return NULL; @@ -928,6 +932,10 @@ set_update(PySetObject *so, PyObject *args) { Py_ssize_t i; + if(!Py_CHECKWRITE(so)){ + return PyErr_WriteToImmutable(so); + } + for (i=0 ; i= Py_SIZE(op)) { Py_XDECREF(newitem); PyErr_SetString(PyExc_IndexError, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 5c71c28f751504..75f5e4998a63b7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5227,21 +5227,33 @@ PyDoc_STRVAR(type_doc, static int type_traverse(PyTypeObject *type, visitproc visit, void *arg) { - /* Because of type_is_gc(), the collector only calls this - for heaptypes. */ - if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - char msg[200]; - sprintf(msg, "type_traverse() called on non-heap type '%.100s'", - type->tp_name); - _PyObject_ASSERT_FAILED_MSG((PyObject *)type, msg); - } + // TODO: Immutability + // The following code and comment are not correct with how Pyrona wants to + // use traverse. It is about finding all the objects that are accessible, not just + // those that can particpate in cycles. + // We have adapted the following code to only visit `ht_module` in heap types. + // + // We should consider if other code has optimised for the cycle detector behaviour? + // + // We should check this with the core team. + + + // /* Because of type_is_gc(), the collector only calls this + // for heaptypes. */ + // if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + // char msg[200]; + // sprintf(msg, "type_traverse() called on non-heap type '%.100s'", + // type->tp_name); + // _PyObject_ASSERT_FAILED_MSG((PyObject *)type, msg); + // } Py_VISIT(type->tp_dict); - Py_VISIT(type->tp_cache); Py_VISIT(type->tp_mro); Py_VISIT(type->tp_bases); Py_VISIT(type->tp_base); - Py_VISIT(((PyHeapTypeObject *)type)->ht_module); + if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_VISIT(((PyHeapTypeObject *)type)->ht_module); + } /* There's no need to visit others because they can't be involved in cycles: diff --git a/PC/python3dll.c b/PC/python3dll.c index 505feef4b986c4..1dd359fe04a7cf 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -817,6 +817,7 @@ EXPORT_DATA(PyExc_ModuleNotFoundError) EXPORT_DATA(PyExc_NameError) EXPORT_DATA(PyExc_NotADirectoryError) EXPORT_DATA(PyExc_NotImplementedError) +EXPORT_DATA(PyExc_NotWriteableError) EXPORT_DATA(PyExc_OSError) EXPORT_DATA(PyExc_OverflowError) EXPORT_DATA(PyExc_PendingDeprecationWarning) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 3df0a072045208..8a8a153d0190fc 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -208,6 +208,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index d98a4c5ae4e2bb..62d0144fa66444 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -208,6 +208,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b265264dc161ac..1c7af9630faba0 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -534,6 +534,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 67a32f653de1f8..17400e52f0909f 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -99,6 +99,9 @@ Include + + Include + Include @@ -1181,6 +1184,9 @@ Python + + Python + Python diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 7f366b43599ae5..085d4d253b1568 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -11,6 +11,8 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ceval.h" // _PyEval_Vector() +#include "pycore_immutability.h" // _PyFreeze() +#include "pycore_dict.h" // _PyDict_SetGlobalImmutable() #include "clinic/bltinmodule.c.h" @@ -2737,6 +2739,40 @@ builtin_issubclass_impl(PyObject *module, PyObject *cls, return PyBool_FromLong(retval); } +/*[clinic input] +isimmutable as builtin_isimmutable + + obj: object + / + +Return whether 'obj' is immutable. +[clinic start generated code]*/ + +static PyObject * +builtin_isimmutable(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=80c746a3bd7adb46 input=15c0c9d2da47bc15]*/ +{ + return PyBool_FromLong(_Py_IsImmutable(obj)); +} + + +/*[clinic input] +freeze as builtin_freeze + + obj: object + / + +Make 'obj' and its entire reachable object graph immutable. +[clinic start generated code]*/ + +static PyObject * +builtin_freeze(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=c02caad61e252698 input=81af120172984a86]*/ +{ + return Py_Freeze(obj); +} + + typedef struct { PyObject_HEAD Py_ssize_t tuplesize; @@ -3035,6 +3071,8 @@ static PyMethodDef builtin_methods[] = { BUILTIN_INPUT_METHODDEF BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF + BUILTIN_ISIMMUTABLE_METHODDEF + BUILTIN_FREEZE_METHODDEF BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF diff --git a/Python/bytecodes.c b/Python/bytecodes.c index dfebaaa300fc8a..7c65202cc87f29 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1360,7 +1360,7 @@ dummy_func( DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; + res = _PyDictEntry_Value(entries + index); DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); @@ -1377,7 +1377,7 @@ dummy_func( DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; + res = _PyDictEntry_Value(entries + index); DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); @@ -1410,6 +1410,12 @@ dummy_func( format_exc_unbound(tstate, frame->f_code, oparg); goto error; } + + if (!Py_CHECKWRITE(cell)){ + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } + PyCell_SET(cell, NULL); Py_DECREF(oldobj); } @@ -1464,6 +1470,12 @@ dummy_func( inst(STORE_DEREF, (v --)) { PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); + + if(!Py_CHECKWRITE(cell)){ + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } + PyCell_SET(cell, v); Py_XDECREF(oldobj); } @@ -1826,7 +1838,7 @@ dummy_func( assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = ep->me_value; + res = _PyDictEntry_Value(ep); DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); @@ -1850,12 +1862,12 @@ dummy_func( if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + res = _PyDictEntry_Value(ep); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + res = _PyDictEntry_Value(ep); } DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); @@ -1954,6 +1966,11 @@ dummy_func( PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); STAT_INC(STORE_ATTR, hit); + if (!Py_CHECKWRITE(owner)) + { + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; values->values[index] = value; @@ -1975,6 +1992,11 @@ dummy_func( DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); + if (!Py_CHECKWRITE(owner)) + { + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(frame->f_code->co_names, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); @@ -1983,18 +2005,26 @@ dummy_func( if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; + old_value = _PyDictEntry_Value(ep); DEOPT_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + if(_PyDictEntry_IsImmutable(ep)){ + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } + _PyDictEntry_SetValue(ep, value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; + old_value = _PyDictEntry_Value(ep); DEOPT_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + if(_PyDictEntry_IsImmutable(ep)){ + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } + _PyDictEntry_SetValue(ep, value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); @@ -2011,6 +2041,11 @@ dummy_func( PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + if (!Py_CHECKWRITE(owner)) + { + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; diff --git a/Python/ceval.c b/Python/ceval.c index fdb5b72e6c0f7b..ecf574c633c1e8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -60,7 +60,8 @@ break; \ } \ _Py_DECREF_STAT_INC(); \ - if (--op->ob_refcnt == 0) { \ + op->ob_refcnt -= 1; \ + if ((op->ob_refcnt & _Py_REFCNT_MASK) == 0) { \ destructor dealloc = Py_TYPE(op)->tp_dealloc; \ (*dealloc)(op); \ } \ @@ -87,7 +88,8 @@ break; \ } \ _Py_DECREF_STAT_INC(); \ - if (--op->ob_refcnt == 0) { \ + op->ob_refcnt -= 1; \ + if ((op->ob_refcnt & _Py_REFCNT_MASK) == 0) { \ destructor d = (destructor)(dealloc); \ d(op); \ } \ @@ -205,6 +207,7 @@ static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); static PyObject * import_from(PyThreadState *, PyObject *, PyObject *); static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *); +static void format_exc_notwriteable(PyThreadState *tstate, PyCodeObject *co, int oparg); static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg); static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static int check_except_type_valid(PyThreadState *tstate, PyObject* right); @@ -227,6 +230,8 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); #define UNBOUNDFREE_ERROR_MSG \ "cannot access free variable '%s' where it is not associated with a" \ " value in enclosing scope" +#define NOT_WRITEABLE_ERROR_MSG \ + "cannot write to local variable '%s'" #ifdef HAVE_ERRNO_H #include @@ -2726,6 +2731,18 @@ format_exc_check_arg(PyThreadState *tstate, PyObject *exc, } } +static void +format_exc_notwriteable(PyThreadState *tstate, PyCodeObject *co, int oparg) +{ + PyObject *name; + /* Don't stomp existing exception */ + if (_PyErr_Occurred(tstate)) + return; + name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); + format_exc_check_arg(tstate, PyExc_NotWriteableError, + NOT_WRITEABLE_ERROR_MSG, name); +} + static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg) { diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index b77b4a1e4b410e..f21a0159aa7440 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1409,4 +1409,22 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=84a04e7446debf58 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(builtin_isimmutable__doc__, +"isimmutable($module, obj, /)\n" +"--\n" +"\n" +"Return whether \'obj\' is immutable."); + +#define BUILTIN_ISIMMUTABLE_METHODDEF \ + {"isimmutable", (PyCFunction)builtin_isimmutable, METH_O, builtin_isimmutable__doc__}, + +PyDoc_STRVAR(builtin_freeze__doc__, +"freeze($module, obj, /)\n" +"--\n" +"\n" +"Make \'obj\' and its entire reachable object graph immutable."); + +#define BUILTIN_FREEZE_METHODDEF \ + {"freeze", (PyCFunction)builtin_freeze, METH_O, builtin_freeze__doc__}, +/*[clinic end generated code: output=2967e6388573aa0b input=a9049054013a1b77]*/ diff --git a/Python/errors.c b/Python/errors.c index 6c46d1f2136654..27ff884c4b934e 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1950,6 +1950,38 @@ PyErr_ProgramTextObject(PyObject *filename, int lineno) return _PyErr_ProgramDecodedTextObject(filename, lineno, NULL); } +PyObject * +_PyErr_WriteToImmutable(const char* filename, int lineno, PyObject* obj) +{ + PyObject* string; + PyThreadState *tstate = _PyThreadState_GET(); + if (!_PyErr_Occurred(tstate)) { + string = PyUnicode_FromFormat("object of type %s is immutable at %s:%d", + obj->ob_type->tp_name, filename, lineno); + if (string != NULL) { + _PyErr_SetObject(tstate, PyExc_NotWriteableError, string); + Py_DECREF(string); + } + } + return NULL; +} + +PyObject * +_PyErr_WriteToImmutableKey(const char* filename, int lineno, PyObject* key) +{ + PyObject* string; + PyThreadState *tstate = _PyThreadState_GET(); + if (!_PyErr_Occurred(tstate)) { + string = PyUnicode_FromFormat("key %R is marked as immutable at %s:%d", + key, filename, lineno); + if (string != NULL) { + _PyErr_SetObject(tstate, PyExc_NotWriteableError, string); + Py_DECREF(string); + } + } + return NULL; +} + #ifdef __cplusplus } #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e4cff7bdc3384b..e805c2f9d76665 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1842,7 +1842,7 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = entries[index].me_value; + res = _PyDictEntry_Value(entries + index); DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); @@ -1872,7 +1872,7 @@ DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = entries[index].me_value; + res = _PyDictEntry_Value(entries + index); DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); @@ -1919,16 +1919,22 @@ format_exc_unbound(tstate, frame->f_code, oparg); goto error; } + + if (!Py_CHECKWRITE(cell)){ + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } + PyCell_SET(cell, NULL); Py_DECREF(oldobj); - #line 1925 "Python/generated_cases.c.h" + #line 1931 "Python/generated_cases.c.h" DISPATCH(); } TARGET(LOAD_FROM_DICT_OR_DEREF) { PyObject *class_dict = stack_pointer[-1]; PyObject *value; - #line 1418 "Python/bytecodes.c" + #line 1424 "Python/bytecodes.c" PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus); @@ -1963,14 +1969,14 @@ } Py_INCREF(value); } - #line 1967 "Python/generated_cases.c.h" + #line 1973 "Python/generated_cases.c.h" stack_pointer[-1] = value; DISPATCH(); } TARGET(LOAD_DEREF) { PyObject *value; - #line 1455 "Python/bytecodes.c" + #line 1461 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); if (value == NULL) { @@ -1978,7 +1984,7 @@ if (true) goto error; } Py_INCREF(value); - #line 1982 "Python/generated_cases.c.h" + #line 1988 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = value; DISPATCH(); @@ -1986,18 +1992,24 @@ TARGET(STORE_DEREF) { PyObject *v = stack_pointer[-1]; - #line 1465 "Python/bytecodes.c" + #line 1471 "Python/bytecodes.c" PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); + + if(!Py_CHECKWRITE(cell)){ + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } + PyCell_SET(cell, v); Py_XDECREF(oldobj); - #line 1995 "Python/generated_cases.c.h" + #line 2007 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #line 1472 "Python/bytecodes.c" + #line 1484 "Python/bytecodes.c" /* Copy closure variables to free variables */ PyCodeObject *co = frame->f_code; assert(PyFunction_Check(frame->f_funcobj)); @@ -2008,22 +2020,22 @@ PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); } - #line 2012 "Python/generated_cases.c.h" + #line 2024 "Python/generated_cases.c.h" DISPATCH(); } TARGET(BUILD_STRING) { PyObject **pieces = (stack_pointer - oparg); PyObject *str; - #line 1485 "Python/bytecodes.c" + #line 1497 "Python/bytecodes.c" str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - #line 2021 "Python/generated_cases.c.h" + #line 2033 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - #line 1487 "Python/bytecodes.c" + #line 1499 "Python/bytecodes.c" if (str == NULL) { STACK_SHRINK(oparg); goto error; } - #line 2027 "Python/generated_cases.c.h" + #line 2039 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = str; @@ -2033,10 +2045,10 @@ TARGET(BUILD_TUPLE) { PyObject **values = (stack_pointer - oparg); PyObject *tup; - #line 1491 "Python/bytecodes.c" + #line 1503 "Python/bytecodes.c" tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - #line 2040 "Python/generated_cases.c.h" + #line 2052 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = tup; @@ -2046,10 +2058,10 @@ TARGET(BUILD_LIST) { PyObject **values = (stack_pointer - oparg); PyObject *list; - #line 1496 "Python/bytecodes.c" + #line 1508 "Python/bytecodes.c" list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } - #line 2053 "Python/generated_cases.c.h" + #line 2065 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = list; @@ -2059,7 +2071,7 @@ TARGET(LIST_EXTEND) { PyObject *iterable = stack_pointer[-1]; PyObject *list = stack_pointer[-(2 + (oparg-1))]; - #line 1501 "Python/bytecodes.c" + #line 1513 "Python/bytecodes.c" PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -2070,13 +2082,13 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - #line 2074 "Python/generated_cases.c.h" + #line 2086 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1512 "Python/bytecodes.c" + #line 1524 "Python/bytecodes.c" if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); - #line 2080 "Python/generated_cases.c.h" + #line 2092 "Python/generated_cases.c.h" Py_DECREF(iterable); STACK_SHRINK(1); DISPATCH(); @@ -2085,13 +2097,13 @@ TARGET(SET_UPDATE) { PyObject *iterable = stack_pointer[-1]; PyObject *set = stack_pointer[-(2 + (oparg-1))]; - #line 1519 "Python/bytecodes.c" + #line 1531 "Python/bytecodes.c" int err = _PySet_Update(set, iterable); - #line 2091 "Python/generated_cases.c.h" + #line 2103 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 1521 "Python/bytecodes.c" + #line 1533 "Python/bytecodes.c" if (err < 0) goto pop_1_error; - #line 2095 "Python/generated_cases.c.h" + #line 2107 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2099,7 +2111,7 @@ TARGET(BUILD_SET) { PyObject **values = (stack_pointer - oparg); PyObject *set; - #line 1525 "Python/bytecodes.c" + #line 1537 "Python/bytecodes.c" set = PySet_New(NULL); if (set == NULL) goto error; @@ -2114,7 +2126,7 @@ Py_DECREF(set); if (true) { STACK_SHRINK(oparg); goto error; } } - #line 2118 "Python/generated_cases.c.h" + #line 2130 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_GROW(1); stack_pointer[-1] = set; @@ -2124,7 +2136,7 @@ TARGET(BUILD_MAP) { PyObject **values = (stack_pointer - oparg*2); PyObject *map; - #line 1542 "Python/bytecodes.c" + #line 1554 "Python/bytecodes.c" map = _PyDict_FromItems( values, 2, values+1, 2, @@ -2132,13 +2144,13 @@ if (map == NULL) goto error; - #line 2136 "Python/generated_cases.c.h" + #line 2148 "Python/generated_cases.c.h" for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - #line 1550 "Python/bytecodes.c" + #line 1562 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - #line 2142 "Python/generated_cases.c.h" + #line 2154 "Python/generated_cases.c.h" STACK_SHRINK(oparg*2); STACK_GROW(1); stack_pointer[-1] = map; @@ -2146,7 +2158,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #line 1554 "Python/bytecodes.c" + #line 1566 "Python/bytecodes.c" int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2186,7 +2198,7 @@ Py_DECREF(ann_dict); } } - #line 2190 "Python/generated_cases.c.h" + #line 2202 "Python/generated_cases.c.h" DISPATCH(); } @@ -2194,7 +2206,7 @@ PyObject *keys = stack_pointer[-1]; PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; - #line 1596 "Python/bytecodes.c" + #line 1608 "Python/bytecodes.c" if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2204,14 +2216,14 @@ map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); - #line 2208 "Python/generated_cases.c.h" + #line 2220 "Python/generated_cases.c.h" for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - #line 1606 "Python/bytecodes.c" + #line 1618 "Python/bytecodes.c" if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - #line 2215 "Python/generated_cases.c.h" + #line 2227 "Python/generated_cases.c.h" STACK_SHRINK(oparg); stack_pointer[-1] = map; DISPATCH(); @@ -2219,7 +2231,7 @@ TARGET(DICT_UPDATE) { PyObject *update = stack_pointer[-1]; - #line 1610 "Python/bytecodes.c" + #line 1622 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2227,12 +2239,12 @@ "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - #line 2231 "Python/generated_cases.c.h" + #line 2243 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1618 "Python/bytecodes.c" + #line 1630 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2236 "Python/generated_cases.c.h" + #line 2248 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); DISPATCH(); @@ -2240,17 +2252,17 @@ TARGET(DICT_MERGE) { PyObject *update = stack_pointer[-1]; - #line 1624 "Python/bytecodes.c" + #line 1636 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { format_kwargs_error(tstate, PEEK(3 + oparg), update); - #line 2249 "Python/generated_cases.c.h" + #line 2261 "Python/generated_cases.c.h" Py_DECREF(update); - #line 1629 "Python/bytecodes.c" + #line 1641 "Python/bytecodes.c" if (true) goto pop_1_error; } - #line 2254 "Python/generated_cases.c.h" + #line 2266 "Python/generated_cases.c.h" Py_DECREF(update); STACK_SHRINK(1); PREDICT(CALL_FUNCTION_EX); @@ -2260,26 +2272,26 @@ TARGET(MAP_ADD) { PyObject *value = stack_pointer[-1]; PyObject *key = stack_pointer[-2]; - #line 1636 "Python/bytecodes.c" + #line 1648 "Python/bytecodes.c" PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - #line 2270 "Python/generated_cases.c.h" + #line 2282 "Python/generated_cases.c.h" STACK_SHRINK(2); PREDICT(JUMP_BACKWARD); DISPATCH(); } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - #line 1645 "Python/bytecodes.c" + #line 1657 "Python/bytecodes.c" _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); - #line 2283 "Python/generated_cases.c.h" + #line 2295 "Python/generated_cases.c.h" } TARGET(LOAD_SUPER_ATTR) { @@ -2290,7 +2302,7 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2 = NULL; PyObject *res; - #line 1659 "Python/bytecodes.c" + #line 1671 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -2332,16 +2344,16 @@ } } } - #line 2336 "Python/generated_cases.c.h" + #line 2348 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - #line 1701 "Python/bytecodes.c" + #line 1713 "Python/bytecodes.c" if (super == NULL) goto pop_3_error; res = PyObject_GetAttr(super, name); Py_DECREF(super); if (res == NULL) goto pop_3_error; - #line 2345 "Python/generated_cases.c.h" + #line 2357 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2356,20 +2368,20 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2 = NULL; PyObject *res; - #line 1708 "Python/bytecodes.c" + #line 1720 "Python/bytecodes.c" assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - #line 2367 "Python/generated_cases.c.h" + #line 2379 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - #line 1715 "Python/bytecodes.c" + #line 1727 "Python/bytecodes.c" if (res == NULL) goto pop_3_error; - #line 2373 "Python/generated_cases.c.h" + #line 2385 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2384,7 +2396,7 @@ PyObject *global_super = stack_pointer[-3]; PyObject *res2; PyObject *res; - #line 1719 "Python/bytecodes.c" + #line 1731 "Python/bytecodes.c" assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -2407,7 +2419,7 @@ res = res2; res2 = NULL; } - #line 2411 "Python/generated_cases.c.h" + #line 2423 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -2421,7 +2433,7 @@ PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; - #line 1758 "Python/bytecodes.c" + #line 1770 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2455,9 +2467,9 @@ NULL | meth | arg1 | ... | argN */ - #line 2459 "Python/generated_cases.c.h" + #line 2471 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1792 "Python/bytecodes.c" + #line 1804 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; res2 = NULL; res = meth; @@ -2466,12 +2478,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2470 "Python/generated_cases.c.h" + #line 2482 "Python/generated_cases.c.h" Py_DECREF(owner); - #line 1801 "Python/bytecodes.c" + #line 1813 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2475 "Python/generated_cases.c.h" + #line 2487 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2485,7 +2497,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1806 "Python/bytecodes.c" + #line 1818 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2498,7 +2510,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2502 "Python/generated_cases.c.h" + #line 2514 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2513,7 +2525,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1822 "Python/bytecodes.c" + #line 1834 "Python/bytecodes.c" DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2521,12 +2533,12 @@ assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = ep->me_value; + res = _PyDictEntry_Value(ep); DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2530 "Python/generated_cases.c.h" + #line 2542 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2541,7 +2553,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1838 "Python/bytecodes.c" + #line 1850 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2557,18 +2569,18 @@ if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + res = _PyDictEntry_Value(ep); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + res = _PyDictEntry_Value(ep); } DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2572 "Python/generated_cases.c.h" + #line 2584 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2583,7 +2595,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1868 "Python/bytecodes.c" + #line 1880 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2593,7 +2605,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2597 "Python/generated_cases.c.h" + #line 2609 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2608,7 +2620,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 1881 "Python/bytecodes.c" + #line 1893 "Python/bytecodes.c" DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, @@ -2620,7 +2632,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2624 "Python/generated_cases.c.h" + #line 2636 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2634,7 +2646,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); - #line 1896 "Python/bytecodes.c" + #line 1908 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2658,7 +2670,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2662 "Python/generated_cases.c.h" + #line 2674 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2666,7 +2678,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); - #line 1922 "Python/bytecodes.c" + #line 1934 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2692,7 +2704,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2696 "Python/generated_cases.c.h" + #line 2708 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2700,7 +2712,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 1950 "Python/bytecodes.c" + #line 1962 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2708,6 +2720,11 @@ PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); STAT_INC(STORE_ATTR, hit); + if (!Py_CHECKWRITE(owner)) + { + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } PyDictValues *values = _PyDictOrValues_GetValues(dorv); PyObject *old_value = values->values[index]; values->values[index] = value; @@ -2718,7 +2735,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2722 "Python/generated_cases.c.h" + #line 2739 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2729,7 +2746,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); - #line 1970 "Python/bytecodes.c" + #line 1987 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2738,6 +2755,11 @@ DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(dict == NULL, STORE_ATTR); + if (!Py_CHECKWRITE(owner)) + { + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(frame->f_code->co_names, oparg); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); @@ -2746,18 +2768,26 @@ if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; + old_value = _PyDictEntry_Value(ep); DEOPT_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + if(_PyDictEntry_IsImmutable(ep)){ + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } + _PyDictEntry_SetValue(ep, value); } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = ep->me_value; + old_value = _PyDictEntry_Value(ep); DEOPT_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - ep->me_value = value; + if(_PyDictEntry_IsImmutable(ep)){ + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } + _PyDictEntry_SetValue(ep, value); } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); @@ -2768,7 +2798,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2772 "Python/generated_cases.c.h" + #line 2802 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2779,17 +2809,22 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 2011 "Python/bytecodes.c" + #line 2041 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + if (!Py_CHECKWRITE(owner)) + { + format_exc_notwriteable(tstate, frame->f_code, oparg); + goto error; + } char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2793 "Python/generated_cases.c.h" + #line 2828 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2801,7 +2836,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 2030 "Python/bytecodes.c" + #line 2065 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2814,12 +2849,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2818 "Python/generated_cases.c.h" + #line 2853 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 2043 "Python/bytecodes.c" + #line 2078 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2823 "Python/generated_cases.c.h" + #line 2858 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2830,7 +2865,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 2047 "Python/bytecodes.c" + #line 2082 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2841,7 +2876,7 @@ _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; - #line 2845 "Python/generated_cases.c.h" + #line 2880 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2852,7 +2887,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 2061 "Python/bytecodes.c" + #line 2096 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2867,7 +2902,7 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; - #line 2871 "Python/generated_cases.c.h" + #line 2906 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2878,7 +2913,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 2079 "Python/bytecodes.c" + #line 2114 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2890,7 +2925,7 @@ assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - #line 2894 "Python/generated_cases.c.h" + #line 2929 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2901,14 +2936,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 2093 "Python/bytecodes.c" + #line 2128 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2907 "Python/generated_cases.c.h" + #line 2942 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 2095 "Python/bytecodes.c" + #line 2130 "Python/bytecodes.c" b = res ? Py_True : Py_False; - #line 2912 "Python/generated_cases.c.h" + #line 2947 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2918,15 +2953,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 2099 "Python/bytecodes.c" + #line 2134 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2924 "Python/generated_cases.c.h" + #line 2959 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 2101 "Python/bytecodes.c" + #line 2136 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; - #line 2930 "Python/generated_cases.c.h" + #line 2965 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2937,12 +2972,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 2106 "Python/bytecodes.c" + #line 2141 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2943 "Python/generated_cases.c.h" + #line 2978 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 2108 "Python/bytecodes.c" + #line 2143 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2950,10 +2985,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2954 "Python/generated_cases.c.h" + #line 2989 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 2116 "Python/bytecodes.c" + #line 2151 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2962,7 +2997,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2966 "Python/generated_cases.c.h" + #line 3001 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2972,21 +3007,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 2127 "Python/bytecodes.c" + #line 2162 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2979 "Python/generated_cases.c.h" + #line 3014 "Python/generated_cases.c.h" Py_DECREF(right); - #line 2130 "Python/bytecodes.c" + #line 2165 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2986 "Python/generated_cases.c.h" + #line 3021 "Python/generated_cases.c.h" Py_DECREF(right); - #line 2135 "Python/bytecodes.c" + #line 2170 "Python/bytecodes.c" b = res ? Py_True : Py_False; - #line 2990 "Python/generated_cases.c.h" + #line 3025 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2995,15 +3030,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 2139 "Python/bytecodes.c" + #line 2174 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 3002 "Python/generated_cases.c.h" + #line 3037 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 2142 "Python/bytecodes.c" + #line 2177 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 3007 "Python/generated_cases.c.h" + #line 3042 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -3012,29 +3047,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 2146 "Python/bytecodes.c" + #line 2181 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 3020 "Python/generated_cases.c.h" + #line 3055 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 2152 "Python/bytecodes.c" + #line 2187 "Python/bytecodes.c" JUMPBY(oparg); - #line 3029 "Python/generated_cases.c.h" + #line 3064 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 2156 "Python/bytecodes.c" + #line 2191 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 3038 "Python/generated_cases.c.h" + #line 3073 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3042,15 +3077,15 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 2162 "Python/bytecodes.c" + #line 2197 "Python/bytecodes.c" if (Py_IsFalse(cond)) { JUMPBY(oparg); } else if (!Py_IsTrue(cond)) { int err = PyObject_IsTrue(cond); - #line 3052 "Python/generated_cases.c.h" + #line 3087 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2168 "Python/bytecodes.c" + #line 2203 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -3058,22 +3093,22 @@ if (err < 0) goto pop_1_error; } } - #line 3062 "Python/generated_cases.c.h" + #line 3097 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 2178 "Python/bytecodes.c" + #line 2213 "Python/bytecodes.c" if (Py_IsTrue(cond)) { JUMPBY(oparg); } else if (!Py_IsFalse(cond)) { int err = PyObject_IsTrue(cond); - #line 3075 "Python/generated_cases.c.h" + #line 3110 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2184 "Python/bytecodes.c" + #line 2219 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -3081,63 +3116,63 @@ if (err < 0) goto pop_1_error; } } - #line 3085 "Python/generated_cases.c.h" + #line 3120 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2194 "Python/bytecodes.c" + #line 2229 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 3094 "Python/generated_cases.c.h" + #line 3129 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2196 "Python/bytecodes.c" + #line 2231 "Python/bytecodes.c" JUMPBY(oparg); } - #line 3099 "Python/generated_cases.c.h" + #line 3134 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2201 "Python/bytecodes.c" + #line 2236 "Python/bytecodes.c" if (Py_IsNone(value)) { JUMPBY(oparg); } else { - #line 3111 "Python/generated_cases.c.h" + #line 3146 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2206 "Python/bytecodes.c" + #line 2241 "Python/bytecodes.c" } - #line 3115 "Python/generated_cases.c.h" + #line 3150 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2210 "Python/bytecodes.c" + #line 2245 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 3128 "Python/generated_cases.c.h" + #line 3163 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2219 "Python/bytecodes.c" + #line 2254 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 3141 "Python/generated_cases.c.h" + #line 3176 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3148,16 +3183,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2227 "Python/bytecodes.c" + #line 2262 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3157 "Python/generated_cases.c.h" + #line 3192 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2232 "Python/bytecodes.c" + #line 2267 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -3165,7 +3200,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_None; // Failure! } - #line 3169 "Python/generated_cases.c.h" + #line 3204 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3174,10 +3209,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2242 "Python/bytecodes.c" + #line 2277 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - #line 3181 "Python/generated_cases.c.h" + #line 3216 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3187,10 +3222,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2248 "Python/bytecodes.c" + #line 2283 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - #line 3194 "Python/generated_cases.c.h" + #line 3229 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3201,11 +3236,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2254 "Python/bytecodes.c" + #line 2289 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3209 "Python/generated_cases.c.h" + #line 3244 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3214,14 +3249,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2260 "Python/bytecodes.c" + #line 2295 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3221 "Python/generated_cases.c.h" + #line 3256 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2263 "Python/bytecodes.c" + #line 2298 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3225 "Python/generated_cases.c.h" + #line 3260 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3229,7 +3264,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2267 "Python/bytecodes.c" + #line 2302 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3252,11 +3287,11 @@ if (iter == NULL) { goto error; } - #line 3256 "Python/generated_cases.c.h" + #line 3291 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2290 "Python/bytecodes.c" + #line 2325 "Python/bytecodes.c" } - #line 3260 "Python/generated_cases.c.h" + #line 3295 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3267,7 +3302,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2309 "Python/bytecodes.c" + #line 2344 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3298,7 +3333,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3302 "Python/generated_cases.c.h" + #line 3337 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3306,7 +3341,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2342 "Python/bytecodes.c" + #line 2377 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3332,14 +3367,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3336 "Python/generated_cases.c.h" + #line 3371 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2370 "Python/bytecodes.c" + #line 2405 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3359,7 +3394,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3363 "Python/generated_cases.c.h" + #line 3398 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3369,7 +3404,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2392 "Python/bytecodes.c" + #line 2427 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3389,7 +3424,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3393 "Python/generated_cases.c.h" + #line 3428 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3399,7 +3434,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2414 "Python/bytecodes.c" + #line 2449 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3417,7 +3452,7 @@ if (next == NULL) { goto error; } - #line 3421 "Python/generated_cases.c.h" + #line 3456 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3426,7 +3461,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2434 "Python/bytecodes.c" + #line 2469 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -3442,14 +3477,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3446 "Python/generated_cases.c.h" + #line 3481 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2452 "Python/bytecodes.c" + #line 2487 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3472,16 +3507,16 @@ Py_DECREF(enter); goto error; } - #line 3476 "Python/generated_cases.c.h" + #line 3511 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2475 "Python/bytecodes.c" + #line 2510 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3485 "Python/generated_cases.c.h" + #line 3520 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3493,7 +3528,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2485 "Python/bytecodes.c" + #line 2520 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3519,16 +3554,16 @@ Py_DECREF(enter); goto error; } - #line 3523 "Python/generated_cases.c.h" + #line 3558 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2511 "Python/bytecodes.c" + #line 2546 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3532 "Python/generated_cases.c.h" + #line 3567 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3540,7 +3575,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2520 "Python/bytecodes.c" + #line 2555 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3566,7 +3601,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3570 "Python/generated_cases.c.h" + #line 3605 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3575,7 +3610,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2548 "Python/bytecodes.c" + #line 2583 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3585,7 +3620,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3589 "Python/generated_cases.c.h" + #line 3624 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3599,7 +3634,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2560 "Python/bytecodes.c" + #line 2595 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3616,7 +3651,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3620 "Python/generated_cases.c.h" + #line 3655 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3630,7 +3665,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2579 "Python/bytecodes.c" + #line 2614 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3640,7 +3675,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3644 "Python/generated_cases.c.h" + #line 3679 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3654,7 +3689,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2591 "Python/bytecodes.c" + #line 2626 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3668,7 +3703,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3672 "Python/generated_cases.c.h" + #line 3707 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3677,16 +3712,16 @@ } TARGET(KW_NAMES) { - #line 2607 "Python/bytecodes.c" + #line 2642 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3685 "Python/generated_cases.c.h" + #line 3720 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2613 "Python/bytecodes.c" + #line 2648 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3699,7 +3734,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3703 "Python/generated_cases.c.h" + #line 3738 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3709,7 +3744,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2658 "Python/bytecodes.c" + #line 2693 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3791,7 +3826,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3795 "Python/generated_cases.c.h" + #line 3830 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3803,7 +3838,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2746 "Python/bytecodes.c" + #line 2781 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3813,7 +3848,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3817 "Python/generated_cases.c.h" + #line 3852 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3822,7 +3857,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2758 "Python/bytecodes.c" + #line 2793 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3848,7 +3883,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3852 "Python/generated_cases.c.h" + #line 3887 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3856,7 +3891,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2786 "Python/bytecodes.c" + #line 2821 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3892,7 +3927,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3896 "Python/generated_cases.c.h" + #line 3931 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3900,7 +3935,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2824 "Python/bytecodes.c" + #line 2859 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3910,7 +3945,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3914 "Python/generated_cases.c.h" + #line 3949 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3923,7 +3958,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2836 "Python/bytecodes.c" + #line 2871 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3934,7 +3969,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3938 "Python/generated_cases.c.h" + #line 3973 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3948,7 +3983,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2850 "Python/bytecodes.c" + #line 2885 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3959,7 +3994,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3963 "Python/generated_cases.c.h" + #line 3998 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3973,7 +4008,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2864 "Python/bytecodes.c" + #line 2899 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3995,7 +4030,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3999 "Python/generated_cases.c.h" + #line 4034 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4009,7 +4044,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2889 "Python/bytecodes.c" + #line 2924 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4037,7 +4072,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4041 "Python/generated_cases.c.h" + #line 4076 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4051,7 +4086,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2920 "Python/bytecodes.c" + #line 2955 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4083,7 +4118,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4087 "Python/generated_cases.c.h" + #line 4122 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4097,7 +4132,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2955 "Python/bytecodes.c" + #line 2990 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -4129,7 +4164,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4133 "Python/generated_cases.c.h" + #line 4168 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4143,7 +4178,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2990 "Python/bytecodes.c" + #line 3025 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4168,7 +4203,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4172 "Python/generated_cases.c.h" + #line 4207 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4181,7 +4216,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3017 "Python/bytecodes.c" + #line 3052 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4208,7 +4243,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4212 "Python/generated_cases.c.h" + #line 4247 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4220,7 +4255,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 3047 "Python/bytecodes.c" + #line 3082 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -4238,14 +4273,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4242 "Python/generated_cases.c.h" + #line 4277 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3067 "Python/bytecodes.c" + #line 3102 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4276,7 +4311,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4280 "Python/generated_cases.c.h" + #line 4315 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4289,7 +4324,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3101 "Python/bytecodes.c" + #line 3136 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4318,7 +4353,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4322 "Python/generated_cases.c.h" + #line 4357 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4331,7 +4366,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3133 "Python/bytecodes.c" + #line 3168 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4360,7 +4395,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4364 "Python/generated_cases.c.h" + #line 4399 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4373,7 +4408,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3165 "Python/bytecodes.c" + #line 3200 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4401,7 +4436,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4405 "Python/generated_cases.c.h" + #line 4440 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4411,9 +4446,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3196 "Python/bytecodes.c" + #line 3231 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4417 "Python/generated_cases.c.h" + #line 4452 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4422,7 +4457,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3200 "Python/bytecodes.c" + #line 3235 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4484,14 +4519,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4488 "Python/generated_cases.c.h" + #line 4523 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3262 "Python/bytecodes.c" + #line 3297 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4495 "Python/generated_cases.c.h" + #line 4530 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4506,7 +4541,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3272 "Python/bytecodes.c" + #line 3307 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4535,14 +4570,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4539 "Python/generated_cases.c.h" + #line 4574 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3303 "Python/bytecodes.c" + #line 3338 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4563,7 +4598,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4567 "Python/generated_cases.c.h" + #line 4602 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4571,15 +4606,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3326 "Python/bytecodes.c" + #line 3361 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4577 "Python/generated_cases.c.h" + #line 4612 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3328 "Python/bytecodes.c" + #line 3363 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4583 "Python/generated_cases.c.h" + #line 4618 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4590,7 +4625,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3332 "Python/bytecodes.c" + #line 3367 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4625,7 +4660,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4629 "Python/generated_cases.c.h" + #line 4664 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4634,10 +4669,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3369 "Python/bytecodes.c" + #line 3404 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4641 "Python/generated_cases.c.h" + #line 4676 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4649,7 +4684,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3374 "Python/bytecodes.c" + #line 3409 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4664,12 +4699,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4668 "Python/generated_cases.c.h" + #line 4703 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3389 "Python/bytecodes.c" + #line 3424 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4673 "Python/generated_cases.c.h" + #line 4708 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4679,16 +4714,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3394 "Python/bytecodes.c" + #line 3429 "Python/bytecodes.c" assert(oparg >= 2); - #line 4685 "Python/generated_cases.c.h" + #line 4720 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3398 "Python/bytecodes.c" + #line 3433 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4700,26 +4735,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4704 "Python/generated_cases.c.h" + #line 4739 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3412 "Python/bytecodes.c" + #line 3447 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4710 "Python/generated_cases.c.h" + #line 4745 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3416 "Python/bytecodes.c" + #line 3451 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4717 "Python/generated_cases.c.h" + #line 4752 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3421 "Python/bytecodes.c" + #line 3456 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4728,12 +4763,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4732 "Python/generated_cases.c.h" + #line 4767 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3432 "Python/bytecodes.c" + #line 3467 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4742,12 +4777,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4746 "Python/generated_cases.c.h" + #line 4781 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3443 "Python/bytecodes.c" + #line 3478 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4759,12 +4794,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4763 "Python/generated_cases.c.h" + #line 4798 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3457 "Python/bytecodes.c" + #line 3492 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4776,30 +4811,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4780 "Python/generated_cases.c.h" + #line 4815 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3471 "Python/bytecodes.c" + #line 3506 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4791 "Python/generated_cases.c.h" + #line 4826 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3479 "Python/bytecodes.c" + #line 3514 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4798 "Python/generated_cases.c.h" + #line 4833 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3484 "Python/bytecodes.c" + #line 3519 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4805 "Python/generated_cases.c.h" + #line 4840 "Python/generated_cases.c.h" } diff --git a/Python/immutability.c b/Python/immutability.c new file mode 100644 index 00000000000000..fc957206e2b933 --- /dev/null +++ b/Python/immutability.c @@ -0,0 +1,376 @@ + +#include "Python.h" +#include +#include +#include +#include "pycore_dict.h" +#include "pycore_object.h" +#include "pycore_immutability.h" + + +static PyObject* pop(PyObject* s){ + Py_ssize_t size = PyList_Size(s); + if(size == 0){ + return NULL; + } + + PyObject* item = PyList_GetItem(s, size - 1); + if(item == NULL){ + return NULL; + } + + if(PyList_SetSlice(s, size - 1, size, NULL)){ + return NULL; + } + + Py_INCREF(item); + return item; +} + +static bool is_c_wrapper(PyObject* obj){ + return PyCFunction_Check(obj) || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) || Py_IS_TYPE(obj, &PyWrapperDescr_Type); +} + +#define _Py_VISIT_FUNC_ATTR(attr, frontier) do { \ + if(attr != NULL && !_Py_IsImmutable(attr)){ \ + if(PyList_Append(frontier, attr)){ \ + return PyErr_NoMemory(); \ + } \ + } \ +} while(0) + + +/** + * Special function for walking the reachable graph of a function object. + * + * This is necessary because the function object has a pointer to the global + * object, and this is problematic because freezing any function will make the + * global object immutable, which is not always the desired behaviour. + * + * This function attempts to find the globals that a function will use, and freeze + * just those, and prevent those keys from being updated in the global dictionary + * from this point onwards. + */ +static PyObject* walk_function(PyObject* op, PyObject* frontier) +{ + PyObject* builtins = NULL; + PyObject* frozen_builtins = NULL; + PyObject* globals = NULL; + PyObject* frozen_globals = NULL; + PyObject* module = NULL; + PyObject* module_dict = NULL; + PyFunctionObject* f = NULL; + PyObject* f_ptr = NULL; + PyCodeObject* f_code = NULL; + Py_ssize_t size; + PyObject* f_stack = NULL; + bool check_globals = false; + + _PyObject_ASSERT(op, PyFunction_Check(op)); + + _Py_SetImmutable(op); + + f = (PyFunctionObject*)op; + + globals = f->func_globals; + builtins = f->func_builtins; + + module = PyImport_Import(f->func_module); + if(module == NULL){ + // clear the exception so we can check if the module is a namedtuple + PyObject* exc = PyErr_GetRaisedException(); + _Py_DECLARE_STR(namedtuple, "namedtuple"); + _Py_DECLARE_STR(startswith, "startswith"); + PyObject* res = PyObject_CallMethodOneArg(f->func_module, &_Py_STR(startswith), &_Py_STR(namedtuple)); + if(Py_IsTrue(res)){ + // namedtuple creates a fake module, which cannot be imported + Py_RETURN_NONE; + } + + // not a namedtuple, so we need to set the exception + PyErr_SetRaisedException(exc); + return NULL; + } + + if(PyModule_Check(module)){ + module_dict = PyModule_GetDict(module); + }else{ + module_dict = NULL; + } + + _Py_VISIT_FUNC_ATTR(f->func_defaults, frontier); + _Py_VISIT_FUNC_ATTR(f->func_kwdefaults, frontier); + _Py_VISIT_FUNC_ATTR(f->func_doc, frontier); + _Py_VISIT_FUNC_ATTR(f->func_name, frontier); + _Py_VISIT_FUNC_ATTR(f->func_dict, frontier); + _Py_VISIT_FUNC_ATTR(f->func_closure, frontier); + _Py_VISIT_FUNC_ATTR(f->func_annotations, frontier); + _Py_VISIT_FUNC_ATTR(f->func_typeparams, frontier); + _Py_VISIT_FUNC_ATTR(f->func_qualname, frontier); + + f_stack = PyList_New(0); + if(f_stack == NULL){ + return PyErr_NoMemory(); + } + + f_ptr = f->func_code; + if(PyList_Append(f_stack, f_ptr)){ + goto nomemory; + } + + frozen_builtins = PyDict_New(); + if(frozen_builtins == NULL){ + goto nomemory; + } + + frozen_globals = PyDict_New(); + if(frozen_globals == NULL){ + goto nomemory; + } + + while(PyList_Size(f_stack) != 0){ + f_ptr = pop(f_stack); // fp.rc = x + 1 + _PyObject_ASSERT(f_ptr, PyCode_Check(f_ptr)); + f_code = (PyCodeObject*)f_ptr; + + size = 0; + if (f_code->co_names != NULL) + size = PySequence_Fast_GET_SIZE(f_code->co_names); + for(Py_ssize_t i = 0; i < size; i++){ + PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); // name.rc = x + + if(PyUnicode_CompareWithASCIIString(name, "globals") == 0){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. + check_globals = true; + } + + if(PyDict_Contains(globals, name)){ + PyObject* value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(frozen_globals, name, value)){ + Py_DECREF(frozen_builtins); + Py_DECREF(frozen_globals); + Py_DECREF(f_stack); + return NULL; + } + }else if(PyDict_Contains(builtins, name)){ + PyObject* value = PyDict_GetItem(builtins, name); + if(PyDict_SetItem(frozen_builtins, name, value)){ + Py_DECREF(frozen_builtins); + Py_DECREF(frozen_globals); + Py_DECREF(f_stack); + return NULL; + } + }else if(PyDict_Contains(module_dict, name)){ + PyObject* value = PyDict_GetItem(module_dict, name); // value.rc = x + + _PyDict_SetKeyImmutable((PyDictObject*)module_dict, name); + + if(!_Py_IsImmutable(value)){ + if(PyList_Append(frontier, value)){ + goto nomemory; + } + } + } + } + + size = PySequence_Fast_GET_SIZE(f_code->co_consts); + for(Py_ssize_t i = 0; i < size; i++){ + PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); // value.rc = x + if(!_Py_IsImmutable(value)){ + if(PyCode_Check(value)){ + _Py_SetImmutable(value); + + if(PyList_Append(f_stack, value)){ + goto nomemory; + } + }else{ + if(PyList_Append(frontier, value)){ + goto nomemory; + } + } + } + + if(check_globals && PyUnicode_Check(value)){ + PyObject* name = value; + if(PyDict_Contains(globals, name)){ + value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(frozen_globals, name, value)){ + Py_DECREF(frozen_builtins); + Py_DECREF(frozen_globals); + Py_DECREF(f_stack); + return NULL; + } + } + } + } + } + + Py_DECREF(f_stack); + + if(check_globals){ + size = 0; + if(f->func_closure != NULL) + size = PySequence_Fast_GET_SIZE(f->func_closure); + + for(Py_ssize_t i=0; i < size; ++i){ + PyObject* cellvar = PySequence_Fast_GET_ITEM(f->func_closure, i); // cellvar.rc = x + PyObject* value = PyCell_GET(cellvar); // value.rc = x + + if(PyUnicode_Check(value)){ + PyObject* name = value; + if(PyDict_Contains(globals, name)){ + value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(frozen_globals, name, value)){ + Py_DECREF(frozen_builtins); + Py_DECREF(frozen_globals); + return NULL; + } + } + } + } + } + + if(PyList_Append(frontier, frozen_globals)){ + goto nomemory; + } + + f->func_globals = frozen_globals; + Py_DECREF(globals); + + if(PyList_Append(frontier, frozen_builtins)){ + goto nomemory; + } + + f->func_builtins = frozen_builtins; + Py_DECREF(builtins); + + Py_RETURN_NONE; + +nomemory: + Py_XDECREF(frozen_builtins); + Py_XDECREF(frozen_globals); + Py_XDECREF(f_stack); + return PyErr_NoMemory(); +} + +static int freeze_visit(PyObject* obj, void* frontier) +{ + if(!_Py_IsImmutable(obj)){ + if(PyList_Append((PyObject*)frontier, obj)){ + PyErr_NoMemory(); + return -1; + } + } + + return 0; +} + +PyObject* _Py_Freeze(PyObject* obj) +{ + if(_Py_IsImmutable(obj) && _Py_IsImmutable(Py_TYPE(obj))){ + Py_RETURN_NONE; + } + + PyObject* frontier = PyList_New(0); + if(frontier == NULL){ + return PyErr_NoMemory(); + } + + if(PyList_Append(frontier, obj)){ + Py_DECREF(frontier); + return PyErr_NoMemory(); + } + + PyObject* frozen_importlib = PyImport_ImportModule("_frozen_importlib"); + if(frozen_importlib == NULL){ + Py_DECREF(frontier); + return NULL; + } + + PyObject* blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); + if(blocking_on == NULL){ + Py_DECREF(frozen_importlib); + Py_DECREF(frontier); + return NULL; + } + + PyObject* module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); + if(module_locks == NULL){ + Py_DECREF(blocking_on); + Py_DECREF(frozen_importlib); + Py_DECREF(frontier); + return NULL; + } + + Py_DECREF(frozen_importlib); + + while(PyList_Size(frontier) != 0){ + PyObject* item = pop(frontier); // item.rc = x + 1 + + if(item == blocking_on || + item == module_locks){ + // the module lock and blocking on dictionaries must remain mutable or else + // we will not be able to import modules + continue; + } + + PyTypeObject* type = Py_TYPE(item); + traverseproc traverse; + PyObject* type_op = NULL; + + if(_Py_IsImmutable(item)){ + goto handle_type; + } + + _Py_SetImmutable(item); + + + if(is_c_wrapper(item)) { + // C functions are not mutable, so we can skip them. + continue; + } + + if(PyFunction_Check(item)){ + PyObject* err = walk_function(item, frontier); + if(!Py_IsNone(err)){ + Py_DECREF(blocking_on); + Py_DECREF(module_locks); + Py_DECREF(frontier); + return err; + } + goto handle_type; + } + + traverse = type->tp_traverse; + if(traverse != NULL){ + if(traverse(item, (visitproc)freeze_visit, frontier)){ + Py_DECREF(blocking_on); + Py_DECREF(module_locks); + Py_DECREF(frontier); + return NULL; + } + } + +handle_type: + type_op = _PyObject_CAST(item->ob_type); + if (!_Py_IsImmutable(type_op)){ + if (PyList_Append(frontier, type_op)) + { + Py_DECREF(blocking_on); + Py_DECREF(module_locks); + Py_DECREF(frontier); + return PyErr_NoMemory(); + } + } + } + + Py_DECREF(blocking_on); + Py_DECREF(module_locks); + Py_DECREF(frontier); + + Py_RETURN_NONE; +} \ No newline at end of file diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index f9b1c928cd4845..5a0b7c52aa1258 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -795,7 +795,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC }; struct opcode_metadata { bool valid_entry; enum InstructionFormat instr_format; @@ -918,9 +918,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000 }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000 }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC000 }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000 }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000 }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IBC000 }, [COMPARE_OP] = { true, INSTR_FMT_IBC }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC }, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 29771e07ae6a2c..c392b478f2b52c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1497,6 +1497,7 @@ finalize_modules_clear_weaklist(PyInterpreterState *interp, if (verbose && PyUnicode_Check(name)) { PySys_FormatStderr("# cleanup[3] wiping %U\n", name); } + _PyModule_Clear(mod); Py_DECREF(mod); } @@ -1531,6 +1532,7 @@ finalize_modules(PyThreadState *tstate) // Already done return; } + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; // Delete some special builtins._ and sys attributes first. These are @@ -1859,6 +1861,7 @@ Py_FinalizeEx(void) PyGC_Collect(); /* Destroy all modules */ + _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 6a7c14ebb220a8..a898d3c5a9e556 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -306,6 +306,8 @@ Modules/timemodule.c init_timezone YEAR - Objects/bytearrayobject.c - _PyByteArray_empty_string - Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - +Objects/exceptions.c - _PyExc_NotWriteableError - +Objects/exceptions.c - PyExc_NotWriteableError - Objects/genobject.c - ASYNC_GEN_IGNORED_EXIT_MSG - Objects/genobject.c - NON_INIT_CORO_MSG - Objects/longobject.c - _PyLong_DigitValue - diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index e38bd59e20a305..a308f515b59b99 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -782,7 +782,8 @@ def iteritems(self): entries, nentries = self._get_entries(keys) for i in safe_range(nentries): ep = entries[i] - pyop_value = PyObjectPtr.from_pyobject_ptr(ep['me_value']) + value_ptr = ep['_me_value'].reinterpret_cast(gdb.lookup_type('uintptr_t')) + pyop_value = PyObjectPtr.from_pyobject_ptr((value_ptr >> 1) << 1) if not pyop_value.is_null(): pyop_key = PyObjectPtr.from_pyobject_ptr(ep['me_key']) yield (pyop_key, pyop_value) From 3b832e6f0221c9ffa813b63be5acc1f67a7c2f64 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 3 Apr 2025 10:41:19 +0100 Subject: [PATCH 02/40] Addressing PR comments Signed-off-by: Matthew A Johnson --- Include/internal/pycore_dict.h | 11 ++++-- Include/internal/pycore_object.h | 2 + Include/object.h | 12 +++--- Lib/test/test_freeze.py | 2 + Python/immutability.c | 67 +++++++++++++++++++------------- Tools/gdb/libpython.py | 1 + 6 files changed, 59 insertions(+), 36 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index a9e29df8bf8dce..68b9846e4e7048 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -32,13 +32,16 @@ typedef struct { PyObject *_me_value; /* This field is only meaningful for combined tables */ } PyDictUnicodeEntry; -#define _PyDictEntry_IsImmutable(entry) (((uintptr_t)((entry)->_me_value)) & 0x1) -#define _PyDictEntry_SetImmutable(entry) ((entry)->_me_value = (PyObject*)((uintptr_t)(entry)->_me_value | 0x1)) +#define _PyDictEntry_IMMUTABLE_FLAG 0x1 +#define _PyDictEntry_VALUE_MASK ~_PyDictEntry_IMMUTABLE_FLAG + +#define _PyDictEntry_IsImmutable(entry) (((uintptr_t)((entry)->_me_value)) & _PyDictEntry_IMMUTABLE_FLAG) +#define _PyDictEntry_SetImmutable(entry) ((entry)->_me_value = (PyObject*)((uintptr_t)(entry)->_me_value | _PyDictEntry_IMMUTABLE_FLAG)) #define _PyDictEntry_Hash(entry) ((entry)->me_hash) #define _PyDictEntry_Key(entry) ((entry)->me_key) -#define _PyDictEntry_Value(entry) ((PyObject*)((((uintptr_t)((entry)->_me_value)) >> 1) << 1)) +#define _PyDictEntry_Value(entry) ((PyObject*)(((uintptr_t)((entry)->_me_value)) & _PyDictEntry_VALUE_MASK)) #define _PyDictEntry_SetValue(entry, value) ((entry)->_me_value = value) -#define _PyDictEntry_IsEmpty(entry) ((entry)->_me_value == NULL) +#define _PyDictEntry_IsEmpty(entry) ((((uintptr_t)(entry)->_me_value) & _PyDictEntry_VALUE_MASK) == (uintptr_t)NULL) extern PyObject *_PyDict_IsKeyImmutable(PyObject* op, PyObject* key); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 81584822a39f11..936557b3e01c96 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -100,6 +100,8 @@ static inline void _Py_SetImmutable(PyObject *op) } #define _Py_SetImmutable(op) _Py_SetImmutable(_PyObject_CAST(op)) +// Check whether an object is writeable. +// Note that during runtime finalization, all objects must be mutable #define Py_CHECKWRITE(op) ((op) && (!_Py_IsImmutable(op) || _Py_IsFinalizing())) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} diff --git a/Include/object.h b/Include/object.h index d3c332d6f96e2d..8d3443f400e6c4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -87,16 +87,16 @@ system also uses the second-to-top bit for managing immutable graphs. #if SIZEOF_VOID_P > 4 #define _Py_REFCNT_MASK 0xFFFFFFFF -#define _Py_IMMUTABLE_MASK 0xC000000000 #define _Py_IMMUTABLE_FLAG 0x4000000000 #define _Py_IMMUTABLE_SCC_FLAG 0x8000000000 #else -#define _Py_REFCNT_MASK 0x3FFFFFFF -#define _Py_IMMUTABLE_MASK 0xC0000000 -#define _Py_IMMUTABLE_FLAG 0x40000000 -#define _Py_IMMUTABLE_SCC_FLAG 0x80000000 +#define _Py_REFCNT_MASK 0x1FFFFFFF +#define _Py_IMMUTABLE_FLAG 0x20000000 +#define _Py_IMMUTABLE_SCC_FLAG 0x40000000 #endif +#define _Py_IMMUTABLE_MASK (_Py_IMMUTABLE_SCC_FLAG | _Py_IMMUTABLE_FLAG) + /* Immortalization: @@ -275,7 +275,7 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) { - return (op->ob_refcnt & _Py_IMMUTABLE_FLAG) > 0; + return (op->ob_refcnt & _Py_IMMUTABLE_MASK) > 0; } #define _Py_IsImmutable(op) _Py_IsImmutable(_PyObject_CAST(op)) diff --git a/Lib/test/test_freeze.py b/Lib/test/test_freeze.py index 77949ce881ec19..1c2812efa4b3fb 100644 --- a/Lib/test/test_freeze.py +++ b/Lib/test/test_freeze.py @@ -293,6 +293,8 @@ def d(): def test_globals_copy(self): def f(): global global0 + ref_1 = global0 + ref_2 = global0 return global0 expected = f() diff --git a/Python/immutability.c b/Python/immutability.c index fc957206e2b933..8518d38174ee51 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -19,11 +19,13 @@ static PyObject* pop(PyObject* s){ return NULL; } + Py_INCREF(item); + if(PyList_SetSlice(s, size - 1, size, NULL)){ + Py_DECREF(item); return NULL; } - Py_INCREF(item); return item; } @@ -44,12 +46,14 @@ static bool is_c_wrapper(PyObject* obj){ * Special function for walking the reachable graph of a function object. * * This is necessary because the function object has a pointer to the global - * object, and this is problematic because freezing any function will make the - * global object immutable, which is not always the desired behaviour. + * dictionary, and this is problematic because freezing any function directly + * (as we do with other objects) would make all globals immutable. * - * This function attempts to find the globals that a function will use, and freeze - * just those, and prevent those keys from being updated in the global dictionary - * from this point onwards. + * Instead, we walk the function and find any places where it references + * global variables or builtins, and then freeze just those objects. The globals + * and builtins dictionaries for the function are then replaced with frozen + * copies containing just those globals and builtins we were able to determine + * the function uses. */ static PyObject* walk_function(PyObject* op, PyObject* frontier) { @@ -129,7 +133,7 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) } while(PyList_Size(f_stack) != 0){ - f_ptr = pop(f_stack); // fp.rc = x + 1 + f_ptr = pop(f_stack); _PyObject_ASSERT(f_ptr, PyCode_Check(f_ptr)); f_code = (PyCodeObject*)f_ptr; @@ -137,7 +141,7 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) if (f_code->co_names != NULL) size = PySequence_Fast_GET_SIZE(f_code->co_names); for(Py_ssize_t i = 0; i < size; i++){ - PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); // name.rc = x + PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); if(PyUnicode_CompareWithASCIIString(name, "globals") == 0){ // if the code calls the globals() builtin, then any @@ -165,7 +169,7 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) return NULL; } }else if(PyDict_Contains(module_dict, name)){ - PyObject* value = PyDict_GetItem(module_dict, name); // value.rc = x + PyObject* value = PyDict_GetItem(module_dict, name); _PyDict_SetKeyImmutable((PyDictObject*)module_dict, name); @@ -179,7 +183,7 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) size = PySequence_Fast_GET_SIZE(f_code->co_consts); for(Py_ssize_t i = 0; i < size; i++){ - PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); // value.rc = x + PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); if(!_Py_IsImmutable(value)){ if(PyCode_Check(value)){ _Py_SetImmutable(value); @@ -195,6 +199,11 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) } if(check_globals && PyUnicode_Check(value)){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. PyObject* name = value; if(PyDict_Contains(globals, name)){ value = PyDict_GetItem(globals, name); @@ -212,13 +221,20 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) Py_DECREF(f_stack); if(check_globals){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. + // we need to check the closure for any cellvars that are not + // referenced in the code object, but are still used in the function size = 0; if(f->func_closure != NULL) size = PySequence_Fast_GET_SIZE(f->func_closure); for(Py_ssize_t i=0; i < size; ++i){ - PyObject* cellvar = PySequence_Fast_GET_ITEM(f->func_closure, i); // cellvar.rc = x - PyObject* value = PyCell_GET(cellvar); // value.rc = x + PyObject* cellvar = PySequence_Fast_GET_ITEM(f->func_closure, i); + PyObject* value = PyCell_GET(cellvar); if(PyUnicode_Check(value)){ PyObject* name = value; @@ -271,7 +287,7 @@ static int freeze_visit(PyObject* obj, void* frontier) PyObject* _Py_Freeze(PyObject* obj) { - if(_Py_IsImmutable(obj) && _Py_IsImmutable(Py_TYPE(obj))){ + if(_Py_IsImmutable(obj)){ Py_RETURN_NONE; } @@ -309,7 +325,7 @@ PyObject* _Py_Freeze(PyObject* obj) Py_DECREF(frozen_importlib); while(PyList_Size(frontier) != 0){ - PyObject* item = pop(frontier); // item.rc = x + 1 + PyObject* item = pop(frontier); if(item == blocking_on || item == module_locks){ @@ -323,12 +339,11 @@ PyObject* _Py_Freeze(PyObject* obj) PyObject* type_op = NULL; if(_Py_IsImmutable(item)){ - goto handle_type; + continue; } _Py_SetImmutable(item); - if(is_c_wrapper(item)) { // C functions are not mutable, so we can skip them. continue; @@ -342,20 +357,20 @@ PyObject* _Py_Freeze(PyObject* obj) Py_DECREF(frontier); return err; } - goto handle_type; } - - traverse = type->tp_traverse; - if(traverse != NULL){ - if(traverse(item, (visitproc)freeze_visit, frontier)){ - Py_DECREF(blocking_on); - Py_DECREF(module_locks); - Py_DECREF(frontier); - return NULL; + else + { + traverse = type->tp_traverse; + if(traverse != NULL){ + if(traverse(item, (visitproc)freeze_visit, frontier)){ + Py_DECREF(blocking_on); + Py_DECREF(module_locks); + Py_DECREF(frontier); + return NULL; + } } } -handle_type: type_op = _PyObject_CAST(item->ob_type); if (!_Py_IsImmutable(type_op)){ if (PyList_Append(frontier, type_op)) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index a308f515b59b99..ea51c682ca8f9f 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -783,6 +783,7 @@ def iteritems(self): for i in safe_range(nentries): ep = entries[i] value_ptr = ep['_me_value'].reinterpret_cast(gdb.lookup_type('uintptr_t')) + # Clear the immutability flag pyop_value = PyObjectPtr.from_pyobject_ptr((value_ptr >> 1) << 1) if not pyop_value.is_null(): pyop_key = PyObjectPtr.from_pyobject_ptr(ep['me_key']) From d52b2229f45c5d7de96a76a45f7591737b4db908 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 3 Apr 2025 16:33:16 +0100 Subject: [PATCH 03/40] Cleaning up the immutability code Signed-off-by: Matthew A Johnson --- Include/internal/pycore_object.h | 2 +- Lib/test/test_freeze.py | 79 +++++++++++++++++++++ Objects/listobject.c | 6 ++ Python/immutability.c | 116 +++++++++++++++++-------------- 4 files changed, 150 insertions(+), 53 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 936557b3e01c96..26659af8cd15a3 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -101,7 +101,7 @@ static inline void _Py_SetImmutable(PyObject *op) #define _Py_SetImmutable(op) _Py_SetImmutable(_PyObject_CAST(op)) // Check whether an object is writeable. -// Note that during runtime finalization, all objects must be mutable +// This check will always succeed during runtime finalization. #define Py_CHECKWRITE(op) ((op) && (!_Py_IsImmutable(op) || _Py_IsFinalizing())) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} diff --git a/Lib/test/test_freeze.py b/Lib/test/test_freeze.py index 1c2812efa4b3fb..3632fe0468999c 100644 --- a/Lib/test/test_freeze.py +++ b/Lib/test/test_freeze.py @@ -1,4 +1,6 @@ import unittest +from collections import deque +from array import array # This is a canary to check that global variables are not made immutable # when others are made immutable @@ -97,6 +99,83 @@ def test_reverse(self): with self.assertRaises(NotWriteableError): self.obj.reverse() + def test_inplace_repeat(self): + with self.assertRaises(NotWriteableError): + self.obj *= 2 + + def test_inplace_concat(self): + with self.assertRaises(NotWriteableError): + self.obj += [TestList.C()] + + def test_clear(self): + with self.assertRaises(NotWriteableError): + self.obj.clear() + + def test_sort(self): + with self.assertRaises(NotWriteableError): + self.obj.sort() + + +class TestDeque(BaseObjectTest): + class C: + pass + + def __init__(self, *args, **kwargs): + obj = deque([self.C(), self.C(), 1, "two", None]) + BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) + + def test_set_item(self): + with self.assertRaises(NotWriteableError): + self.obj[0] = None + + def test_set_slice(self): + with self.assertRaises(NotWriteableError): + self.obj[1:3] = [None, None] + + def test_append(self): + with self.assertRaises(NotWriteableError): + self.obj.append(TestList.C()) + + def test_appendleft(self): + with self.assertRaises(NotWriteableError): + self.obj.appendleft(TestList.C()) + + def test_extend(self): + with self.assertRaises(NotWriteableError): + self.obj.extend([TestList.C()]) + + def test_extendleft(self): + with self.assertRaises(NotWriteableError): + self.obj.extendleft([TestList.C()]) + + def test_insert(self): + with self.assertRaises(NotWriteableError): + self.obj.insert(0, TestList.C()) + + def test_pop(self): + with self.assertRaises(NotWriteableError): + self.obj.pop() + + def test_popleft(self): + with self.assertRaises(NotWriteableError): + self.obj.popleft() + + def test_remove(self): + with self.assertRaises(NotWriteableError): + self.obj.remove(1) + + def test_inplace_repeat(self): + with self.assertRaises(NotWriteableError): + self.obj *= 2 + + def test_inplace_concat(self): + with self.assertRaises(NotWriteableError): + self.obj += [TestList.C()] + + def test_reverse(self): + with self.assertRaises(NotWriteableError): + self.obj.reverse() + def test_clear(self): with self.assertRaises(NotWriteableError): self.obj.clear() diff --git a/Objects/listobject.c b/Objects/listobject.c index fd94aec47a8453..3b066bae8d09b4 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -774,6 +774,12 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n) if (input_size > PY_SSIZE_T_MAX / n) { return PyErr_NoMemory(); } + + if(!Py_CHECKWRITE(self)) + { + return PyErr_WriteToImmutable(self); + } + Py_ssize_t output_size = input_size * n; if (list_resize(self, output_size) < 0) diff --git a/Python/immutability.c b/Python/immutability.c index 8518d38174ee51..7d0dfbcde79c8c 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -8,19 +8,32 @@ #include "pycore_immutability.h" +static int push(PyObject* s, PyObject* item){ + if(item == NULL){ + return 0; + } + + if(!PyList_Check(s)){ + PyErr_SetString(PyExc_TypeError, "Expected a list"); + return -1; + } + + return _PyList_AppendTakeRef(_PyList_CAST(s), item); +} + static PyObject* pop(PyObject* s){ + PyObject* item; Py_ssize_t size = PyList_Size(s); if(size == 0){ return NULL; } - PyObject* item = PyList_GetItem(s, size - 1); + item = PyList_GetItem(s, size - 1); if(item == NULL){ return NULL; } Py_INCREF(item); - if(PyList_SetSlice(s, size - 1, size, NULL)){ Py_DECREF(item); return NULL; @@ -35,7 +48,7 @@ static bool is_c_wrapper(PyObject* obj){ #define _Py_VISIT_FUNC_ATTR(attr, frontier) do { \ if(attr != NULL && !_Py_IsImmutable(attr)){ \ - if(PyList_Append(frontier, attr)){ \ + if(push((frontier), (attr))){ \ return PyErr_NoMemory(); \ } \ } \ @@ -118,7 +131,7 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) } f_ptr = f->func_code; - if(PyList_Append(f_stack, f_ptr)){ + if(push(f_stack, f_ptr)){ goto nomemory; } @@ -174,7 +187,7 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) _PyDict_SetKeyImmutable((PyDictObject*)module_dict, name); if(!_Py_IsImmutable(value)){ - if(PyList_Append(frontier, value)){ + if(push(frontier, value)){ goto nomemory; } } @@ -188,11 +201,11 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) if(PyCode_Check(value)){ _Py_SetImmutable(value); - if(PyList_Append(f_stack, value)){ + if(push(f_stack, value)){ goto nomemory; } }else{ - if(PyList_Append(frontier, value)){ + if(push(frontier, value)){ goto nomemory; } } @@ -250,14 +263,14 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) } } - if(PyList_Append(frontier, frozen_globals)){ + if(push(frontier, frozen_globals)){ goto nomemory; } f->func_globals = frozen_globals; Py_DECREF(globals); - if(PyList_Append(frontier, frozen_builtins)){ + if(push(frontier, frozen_builtins)){ goto nomemory; } @@ -276,7 +289,7 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) static int freeze_visit(PyObject* obj, void* frontier) { if(!_Py_IsImmutable(obj)){ - if(PyList_Append((PyObject*)frontier, obj)){ + if(push(frontier, obj)){ PyErr_NoMemory(); return -1; } @@ -287,44 +300,49 @@ static int freeze_visit(PyObject* obj, void* frontier) PyObject* _Py_Freeze(PyObject* obj) { + PyObject* frontier = NULL; + PyObject* frozen_importlib = NULL; + PyObject* blocking_on = NULL; + PyObject* module_locks = NULL; + PyObject* result = Py_None; + if(_Py_IsImmutable(obj)){ - Py_RETURN_NONE; + return result; } - PyObject* frontier = PyList_New(0); + frontier = PyList_New(0); if(frontier == NULL){ - return PyErr_NoMemory(); + result = PyErr_NoMemory(); + goto cleanup; } - if(PyList_Append(frontier, obj)){ - Py_DECREF(frontier); - return PyErr_NoMemory(); + if(push(frontier, obj)){ + result = PyErr_NoMemory(); + goto cleanup; } - PyObject* frozen_importlib = PyImport_ImportModule("_frozen_importlib"); + frozen_importlib = PyImport_ImportModule("_frozen_importlib"); if(frozen_importlib == NULL){ - Py_DECREF(frontier); - return NULL; + result = NULL; + goto cleanup; } - PyObject* blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); + blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); if(blocking_on == NULL){ - Py_DECREF(frozen_importlib); - Py_DECREF(frontier); - return NULL; + result = NULL; + goto cleanup; } - PyObject* module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); + module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); if(module_locks == NULL){ - Py_DECREF(blocking_on); - Py_DECREF(frozen_importlib); - Py_DECREF(frontier); - return NULL; + result = NULL; + goto cleanup; } - Py_DECREF(frozen_importlib); - while(PyList_Size(frontier) != 0){ + PyTypeObject* type; + PyObject* type_op; + traverseproc traverse; PyObject* item = pop(frontier); if(item == blocking_on || @@ -334,9 +352,8 @@ PyObject* _Py_Freeze(PyObject* obj) continue; } - PyTypeObject* type = Py_TYPE(item); - traverseproc traverse; - PyObject* type_op = NULL; + type = Py_TYPE(item); + type_op = NULL; if(_Py_IsImmutable(item)){ continue; @@ -350,12 +367,9 @@ PyObject* _Py_Freeze(PyObject* obj) } if(PyFunction_Check(item)){ - PyObject* err = walk_function(item, frontier); - if(!Py_IsNone(err)){ - Py_DECREF(blocking_on); - Py_DECREF(module_locks); - Py_DECREF(frontier); - return err; + result = walk_function(item, frontier); + if(!Py_IsNone(result)){ + goto cleanup; } } else @@ -363,29 +377,27 @@ PyObject* _Py_Freeze(PyObject* obj) traverse = type->tp_traverse; if(traverse != NULL){ if(traverse(item, (visitproc)freeze_visit, frontier)){ - Py_DECREF(blocking_on); - Py_DECREF(module_locks); - Py_DECREF(frontier); - return NULL; + result = NULL; + goto cleanup; } } } type_op = _PyObject_CAST(item->ob_type); if (!_Py_IsImmutable(type_op)){ - if (PyList_Append(frontier, type_op)) + if(push(frontier, type_op)) { - Py_DECREF(blocking_on); - Py_DECREF(module_locks); - Py_DECREF(frontier); - return PyErr_NoMemory(); + result = PyErr_NoMemory(); + goto cleanup; } } } - Py_DECREF(blocking_on); - Py_DECREF(module_locks); - Py_DECREF(frontier); +cleanup: + Py_XDECREF(blocking_on); + Py_XDECREF(module_locks); + Py_XDECREF(frozen_importlib); + Py_XDECREF(frontier); - Py_RETURN_NONE; + return result; } \ No newline at end of file From 80b648c4bb0002da40e58d8c7efe472d51c2a076 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 4 Apr 2025 14:02:36 +0100 Subject: [PATCH 04/40] deque working Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze.py | 20 ++++++++----- Modules/_collectionsmodule.c | 58 ++++++++++++++++++++++++++++++++++++ Objects/typeobject.c | 5 ++++ 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_freeze.py b/Lib/test/test_freeze.py index 3632fe0468999c..d9b4b66f2a71e3 100644 --- a/Lib/test/test_freeze.py +++ b/Lib/test/test_freeze.py @@ -95,6 +95,10 @@ def test_remove(self): with self.assertRaises(NotWriteableError): self.obj.remove(1) + def test_delete(self): + with self.assertRaises(NotWriteableError): + del self.obj[0] + def test_reverse(self): with self.assertRaises(NotWriteableError): self.obj.reverse() @@ -128,10 +132,6 @@ def test_set_item(self): with self.assertRaises(NotWriteableError): self.obj[0] = None - def test_set_slice(self): - with self.assertRaises(NotWriteableError): - self.obj[1:3] = [None, None] - def test_append(self): with self.assertRaises(NotWriteableError): self.obj.append(TestList.C()) @@ -164,6 +164,10 @@ def test_remove(self): with self.assertRaises(NotWriteableError): self.obj.remove(1) + def test_delete(self): + with self.assertRaises(NotWriteableError): + del self.obj[0] + def test_inplace_repeat(self): with self.assertRaises(NotWriteableError): self.obj *= 2 @@ -176,13 +180,13 @@ def test_reverse(self): with self.assertRaises(NotWriteableError): self.obj.reverse() - def test_clear(self): + def test_rotate(self): with self.assertRaises(NotWriteableError): - self.obj.clear() + self.obj.rotate(1) - def test_sort(self): + def test_clear(self): with self.assertRaises(NotWriteableError): - self.obj.sort() + self.obj.clear() class TestDict(BaseObjectTest): diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 9a81531bdffb16..1c127a3c4dc5dc 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -3,6 +3,7 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_object.h" // Py_CHECKWRITE #include "structmember.h" // PyMemberDef #include @@ -224,6 +225,10 @@ deque_pop(dequeobject *deque, PyObject *unused) PyObject *item; block *prevblock; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + if (Py_SIZE(deque) == 0) { PyErr_SetString(PyExc_IndexError, "pop from an empty deque"); return NULL; @@ -261,6 +266,10 @@ deque_popleft(dequeobject *deque, PyObject *unused) PyObject *item; block *prevblock; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + if (Py_SIZE(deque) == 0) { PyErr_SetString(PyExc_IndexError, "pop from an empty deque"); return NULL; @@ -336,6 +345,10 @@ deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) static PyObject * deque_append(dequeobject *deque, PyObject *item) { + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + if (deque_append_internal(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; @@ -372,6 +385,10 @@ deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) static PyObject * deque_appendleft(dequeobject *deque, PyObject *item) { + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + if (deque_appendleft_internal(deque, Py_NewRef(item), deque->maxlen) < 0) return NULL; Py_RETURN_NONE; @@ -416,6 +433,10 @@ deque_extend(dequeobject *deque, PyObject *iterable) PyObject *(*iternext)(PyObject *); Py_ssize_t maxlen = deque->maxlen; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { PyObject *result; @@ -463,6 +484,10 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) PyObject *(*iternext)(PyObject *); Py_ssize_t maxlen = deque->maxlen; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { PyObject *result; @@ -508,6 +533,10 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) { PyObject *result; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + result = deque_extend(deque, other); if (result == NULL) return result; @@ -678,6 +707,10 @@ deque_clear(dequeobject *deque) static PyObject * deque_clearmethod(dequeobject *deque, PyObject *Py_UNUSED(ignored)) { + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + deque_clear(deque); Py_RETURN_NONE; } @@ -691,6 +724,10 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) PyObject *seq; PyObject *rv; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + size = Py_SIZE(deque); if (size == 0 || n == 1) { return Py_NewRef(deque); @@ -929,6 +966,10 @@ deque_rotate(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) { Py_ssize_t n=1; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + if (!_PyArg_CheckPositional("deque.rotate", nargs, 0, 1)) { return NULL; } @@ -962,6 +1003,10 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_ssize_t n = Py_SIZE(deque) >> 1; PyObject *tmp; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + while (--n >= 0) { /* Validate that pointers haven't met in the middle */ assert(leftblock != rightblock || leftindex < rightindex); @@ -1157,6 +1202,10 @@ deque_insert(dequeobject *deque, PyObject *const *args, Py_ssize_t nargs) PyObject *value; PyObject *rv; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + if (!_PyArg_ParseStack(args, nargs, "nO:insert", &index, &value)) { return NULL; } @@ -1261,6 +1310,10 @@ deque_remove(dequeobject *deque, PyObject *value) size_t start_state = deque->state; int cmp, rv; + if(!Py_CHECKWRITE(deque)){ + return PyErr_WriteToImmutable(deque); + } + for (i = 0 ; i < n; i++) { item = Py_NewRef(b->data[index]); cmp = PyObject_RichCompareBool(item, value, Py_EQ); @@ -1299,6 +1352,11 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) block *b; Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i; + if(!Py_CHECKWRITE(deque)){ + PyErr_WriteToImmutable(deque); + return -1; + } + if (!valid_index(i, len)) { PyErr_SetString(PyExc_IndexError, "deque index out of range"); return -1; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 75f5e4998a63b7..c0982deda1f98f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5251,9 +5251,14 @@ type_traverse(PyTypeObject *type, visitproc visit, void *arg) Py_VISIT(type->tp_mro); Py_VISIT(type->tp_bases); Py_VISIT(type->tp_base); + + /* + TODO Do we need to visit the module? This can potentially freeze the entire + standard library. What do we gain? if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_VISIT(((PyHeapTypeObject *)type)->ht_module); } + */ /* There's no need to visit others because they can't be involved in cycles: From 697d3b1dd4cc6621a2fe8959b0b861c8a85165ae Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 4 Apr 2025 14:21:52 +0100 Subject: [PATCH 05/40] Adding write barriers for array Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze.py | 54 +++++++++++++++++++++++++++++++++++++ Modules/arraymodule.c | 59 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/Lib/test/test_freeze.py b/Lib/test/test_freeze.py index d9b4b66f2a71e3..f244169b25d9e5 100644 --- a/Lib/test/test_freeze.py +++ b/Lib/test/test_freeze.py @@ -189,6 +189,60 @@ def test_clear(self): self.obj.clear() +class TestArray(BaseObjectTest): + def __init__(self, *args, **kwargs): + obj = array('i', [1, 2, 3, 4]) + BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) + + def test_set_item(self): + with self.assertRaises(NotWriteableError): + self.obj[0] = 5 + + def test_set_slice(self): + with self.assertRaises(NotWriteableError): + self.obj[1:3] = [6, 7] + + def test_append(self): + with self.assertRaises(NotWriteableError): + self.obj.append(8) + + def test_extend(self): + with self.assertRaises(NotWriteableError): + self.obj.extend(array('i', [9])) + + def test_insert(self): + with self.assertRaises(NotWriteableError): + self.obj.insert(0, 10) + + def test_pop(self): + with self.assertRaises(NotWriteableError): + self.obj.pop() + + def test_remove(self): + with self.assertRaises(NotWriteableError): + self.obj.remove(1) + + def test_delete(self): + with self.assertRaises(NotWriteableError): + del self.obj[0] + + def test_reverse(self): + with self.assertRaises(NotWriteableError): + self.obj.reverse() + + def test_inplace_repeat(self): + with self.assertRaises(NotWriteableError): + self.obj *= 2 + + def test_inplace_concat(self): + with self.assertRaises(NotWriteableError): + self.obj += array('i', [11]) + + def test_byteswap(self): + with self.assertRaises(NotWriteableError): + self.obj.byteswap() + + class TestDict(BaseObjectTest): class C: pass diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 6680820d8e61bc..68709b5b7dee16 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -11,6 +11,7 @@ #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_bytesobject.h" // _PyBytes_Repeat +#include "pycore_object.h" // Py_CHECKWRITE #include "structmember.h" // PyMemberDef #include // offsetof() @@ -1048,6 +1049,10 @@ array_inplace_concat(arrayobject *self, PyObject *bb) { array_state *state = find_array_state_by_type(Py_TYPE(self)); + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (!array_Check(bb, state)) { PyErr_Format(PyExc_TypeError, "can only extend array with array (not \"%.200s\")", @@ -1064,6 +1069,10 @@ array_inplace_repeat(arrayobject *self, Py_ssize_t n) { const Py_ssize_t array_size = Py_SIZE(self); + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (array_size > 0 && n != 1 ) { if (n < 0) n = 0; @@ -1087,6 +1096,10 @@ array_inplace_repeat(arrayobject *self, Py_ssize_t n) static PyObject * ins(arrayobject *self, Py_ssize_t where, PyObject *v) { + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (ins1(self, where, v) != 0) return NULL; Py_RETURN_NONE; @@ -1205,6 +1218,10 @@ array_array_remove(arrayobject *self, PyObject *v) { Py_ssize_t i; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + for (i = 0; i < Py_SIZE(self); i++) { PyObject *selfi; int cmp; @@ -1243,6 +1260,10 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) { PyObject *v; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (Py_SIZE(self) == 0) { /* Special-case most common failure cause */ PyErr_SetString(PyExc_IndexError, "pop from empty array"); @@ -1278,6 +1299,10 @@ static PyObject * array_array_extend_impl(arrayobject *self, PyTypeObject *cls, PyObject *bb) /*[clinic end generated code: output=e65eb7588f0bc266 input=8eb6817ec4d2cb62]*/ { + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + array_state *state = get_array_state_by_class(cls); if (array_do_extend(state, self, bb) == -1) @@ -1351,6 +1376,10 @@ static PyObject * array_array_append(arrayobject *self, PyObject *v) /*[clinic end generated code: output=745a0669bf8db0e2 input=0b98d9d78e78f0fa]*/ { + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + return ins(self, Py_SIZE(self), v); } @@ -1370,6 +1399,10 @@ array_array_byteswap_impl(arrayobject *self) char *p; Py_ssize_t i; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + switch (self->ob_descr->itemsize) { case 1: break; @@ -1430,6 +1463,10 @@ array_array_reverse_impl(arrayobject *self) char tmp[256]; /* 8 is probably enough -- but why skimp */ assert((size_t)itemsize <= sizeof(tmp)); + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (Py_SIZE(self) > 1) { for (p = self->ob_item, q = self->ob_item + (Py_SIZE(self) - 1)*itemsize; @@ -1468,6 +1505,10 @@ array_array_fromfile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f, Py_ssize_t nbytes; int not_enough_bytes; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (n < 0) { PyErr_SetString(PyExc_ValueError, "negative count"); return NULL; @@ -1575,6 +1616,10 @@ array_array_fromlist(arrayobject *self, PyObject *list) { Py_ssize_t n; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (!PyList_Check(list)) { PyErr_SetString(PyExc_TypeError, "arg must be list"); return NULL; @@ -1636,6 +1681,11 @@ frombytes(arrayobject *self, Py_buffer *buffer) { int itemsize = self->ob_descr->itemsize; Py_ssize_t n; + + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (buffer->itemsize != 1) { PyBuffer_Release(buffer); PyErr_SetString(PyExc_TypeError, "a bytes-like object is required"); @@ -1725,6 +1775,10 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) return NULL; } + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + Py_ssize_t ustr_length = PyUnicode_AsWideChar(ustr, NULL, 0); assert(ustr_length > 0); if (ustr_length > 1) { @@ -2401,6 +2455,11 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) arrayobject* other; int itemsize; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); From 4024677730ae45a6575422d10ed34a697d4436e8 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Wed, 9 Apr 2025 14:08:30 +0100 Subject: [PATCH 06/40] Cleaning up function freezing + more tests Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze.py | 106 ++++++++++++++++++ Python/immutability.c | 240 ++++++++++++---------------------------- 2 files changed, 176 insertions(+), 170 deletions(-) diff --git a/Lib/test/test_freeze.py b/Lib/test/test_freeze.py index f244169b25d9e5..ddbfd7019573ba 100644 --- a/Lib/test/test_freeze.py +++ b/Lib/test/test_freeze.py @@ -394,6 +394,19 @@ def d(): self.assertFalse(isimmutable(global_canary)) self.assertRaises(NotWriteableError, d) + def test_hidden_global(self): + global global0 + def hide_access(): + global global0 + global0 += 1 + return global0 + def d(): + return hide_access() + global0 = 0 + self.assertEqual(d(), 1) + freeze(d) + self.assertRaises(NotWriteableError, d) + def test_builtins(self): def e(): test = list(range(5)) @@ -607,6 +620,99 @@ def c(self, val): self.assertTrue(isinstance(d_obj, C)) self.assertTrue(issubclass(D, C)) +class TestImport(unittest.TestCase): + def test_import(self): + def f(): + import sys + pass + + freeze(f) + + # The following should not fail, but we + # have removed __import__ from globals + # during freeze + f() + +class TestFunctionAttributes(unittest.TestCase): + def test_function_attributes(self): + def f(): + pass + + freeze(f) + + with self.assertRaises(NotWriteableError): + f.__annotations__ = {} + + # The following should raise an exception, + # but doesn't as the __annotations__ gets + # replaced with a new dict that is not + # immutable. + with self.assertRaises(NotWriteableError): + f.__annotations__["foo"] = 2 + + with self.assertRaises(NotWriteableError): + f.__builtins__ = {} + + with self.assertRaises(NotWriteableError): + f.__builtins__["foo"] = 2 + + with self.assertRaises(NotWriteableError): + def g(): + pass + f.__code__ = g.__code__ + + with self.assertRaises(NotWriteableError): + f.__defaults__ = (1,2) + + with self.assertRaises(NotWriteableError): + f.__dict__ = {} + + with self.assertRaises(NotWriteableError): + f.__dict__["foo"] = {} + + with self.assertRaises(NotWriteableError): + f.__doc__ = "foo" + + with self.assertRaises(NotWriteableError): + f.__globals__ = {} + + with self.assertRaises(NotWriteableError): + f.__globals__["foo"] = 2 + + with self.assertRaises(NotWriteableError): + f.__kwdefaults__ = {} + + with self.assertRaises(NotWriteableError): + f.__module__ = "foo" + + with self.assertRaises(NotWriteableError): + f.__name__ = "foo" + + with self.assertRaises(NotWriteableError): + f.__qualname__ = "foo" + + with self.assertRaises(NotWriteableError): + f.__type_params__ = (1,2) + +class TestFunctionDefaults(unittest.TestCase): + def test_function_defaults(self): + bdef = {} + def f(b=bdef): + return b + + freeze(f) + + self.assertTrue(isimmutable(bdef)) + + def test_function_kwdefaults(self): + bdef = {} + def f(a, **b): + return a, b + f.__kwdefaults__ = bdef + + freeze(f) + + self.assertTrue(isimmutable(bdef)) if __name__ == '__main__': unittest.main() diff --git a/Python/immutability.c b/Python/immutability.c index 7d0dfbcde79c8c..a9293f797e421e 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -18,7 +18,7 @@ static int push(PyObject* s, PyObject* item){ return -1; } - return _PyList_AppendTakeRef(_PyList_CAST(s), item); + return _PyList_AppendTakeRef(_PyList_CAST(s), Py_NewRef(item)); } static PyObject* pop(PyObject* s){ @@ -33,9 +33,7 @@ static PyObject* pop(PyObject* s){ return NULL; } - Py_INCREF(item); if(PyList_SetSlice(s, size - 1, size, NULL)){ - Py_DECREF(item); return NULL; } @@ -46,17 +44,8 @@ static bool is_c_wrapper(PyObject* obj){ return PyCFunction_Check(obj) || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) || Py_IS_TYPE(obj, &PyWrapperDescr_Type); } -#define _Py_VISIT_FUNC_ATTR(attr, frontier) do { \ - if(attr != NULL && !_Py_IsImmutable(attr)){ \ - if(push((frontier), (attr))){ \ - return PyErr_NoMemory(); \ - } \ - } \ -} while(0) - - /** - * Special function for walking the reachable graph of a function object. + * Special function for replacing globals and builtins with a copy of just what they use. * * This is necessary because the function object has a pointer to the global * dictionary, and this is problematic because freezing any function directly @@ -64,175 +53,97 @@ static bool is_c_wrapper(PyObject* obj){ * * Instead, we walk the function and find any places where it references * global variables or builtins, and then freeze just those objects. The globals - * and builtins dictionaries for the function are then replaced with frozen + * and builtins dictionaries for the function are then replaced with * copies containing just those globals and builtins we were able to determine * the function uses. */ -static PyObject* walk_function(PyObject* op, PyObject* frontier) +static PyObject* shadow_function_globals(PyObject* op) { PyObject* builtins = NULL; - PyObject* frozen_builtins = NULL; + PyObject* shadow_builtins = NULL; PyObject* globals = NULL; - PyObject* frozen_globals = NULL; - PyObject* module = NULL; - PyObject* module_dict = NULL; + PyObject* shadow_globals = NULL; PyFunctionObject* f = NULL; PyObject* f_ptr = NULL; PyCodeObject* f_code = NULL; Py_ssize_t size; - PyObject* f_stack = NULL; bool check_globals = false; _PyObject_ASSERT(op, PyFunction_Check(op)); - _Py_SetImmutable(op); - f = (PyFunctionObject*)op; globals = f->func_globals; builtins = f->func_builtins; - module = PyImport_Import(f->func_module); - if(module == NULL){ - // clear the exception so we can check if the module is a namedtuple - PyObject* exc = PyErr_GetRaisedException(); - _Py_DECLARE_STR(namedtuple, "namedtuple"); - _Py_DECLARE_STR(startswith, "startswith"); - PyObject* res = PyObject_CallMethodOneArg(f->func_module, &_Py_STR(startswith), &_Py_STR(namedtuple)); - if(Py_IsTrue(res)){ - // namedtuple creates a fake module, which cannot be imported - Py_RETURN_NONE; - } - - // not a namedtuple, so we need to set the exception - PyErr_SetRaisedException(exc); - return NULL; - } - - if(PyModule_Check(module)){ - module_dict = PyModule_GetDict(module); - }else{ - module_dict = NULL; - } - - _Py_VISIT_FUNC_ATTR(f->func_defaults, frontier); - _Py_VISIT_FUNC_ATTR(f->func_kwdefaults, frontier); - _Py_VISIT_FUNC_ATTR(f->func_doc, frontier); - _Py_VISIT_FUNC_ATTR(f->func_name, frontier); - _Py_VISIT_FUNC_ATTR(f->func_dict, frontier); - _Py_VISIT_FUNC_ATTR(f->func_closure, frontier); - _Py_VISIT_FUNC_ATTR(f->func_annotations, frontier); - _Py_VISIT_FUNC_ATTR(f->func_typeparams, frontier); - _Py_VISIT_FUNC_ATTR(f->func_qualname, frontier); - - f_stack = PyList_New(0); - if(f_stack == NULL){ - return PyErr_NoMemory(); - } - f_ptr = f->func_code; - if(push(f_stack, f_ptr)){ - goto nomemory; - } - frozen_builtins = PyDict_New(); - if(frozen_builtins == NULL){ + shadow_builtins = PyDict_New(); + if(shadow_builtins == NULL){ goto nomemory; } - frozen_globals = PyDict_New(); - if(frozen_globals == NULL){ + shadow_globals = PyDict_New(); + if(shadow_globals == NULL){ goto nomemory; } - while(PyList_Size(f_stack) != 0){ - f_ptr = pop(f_stack); - _PyObject_ASSERT(f_ptr, PyCode_Check(f_ptr)); - f_code = (PyCodeObject*)f_ptr; + _PyObject_ASSERT(f_ptr, PyCode_Check(f_ptr)); + f_code = (PyCodeObject*)f_ptr; + + size = 0; + if (f_code->co_names != NULL) + size = PySequence_Fast_GET_SIZE(f_code->co_names); + for(Py_ssize_t i = 0; i < size; i++){ + PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); + + if(PyUnicode_CompareWithASCIIString(name, "globals") == 0){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. + check_globals = true; + } - size = 0; - if (f_code->co_names != NULL) - size = PySequence_Fast_GET_SIZE(f_code->co_names); - for(Py_ssize_t i = 0; i < size; i++){ - PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); - - if(PyUnicode_CompareWithASCIIString(name, "globals") == 0){ - // if the code calls the globals() builtin, then any - // cellvar or const in the function could, potentially, refer to - // a global variable. As such, we need to check if the globals - // dictionary contains that key and then make it immutable - // from this point forwards. - check_globals = true; + if(PyDict_Contains(globals, name)){ + PyObject* value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(shadow_globals, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return NULL; } - - if(PyDict_Contains(globals, name)){ - PyObject* value = PyDict_GetItem(globals, name); - if(PyDict_SetItem(frozen_globals, name, value)){ - Py_DECREF(frozen_builtins); - Py_DECREF(frozen_globals); - Py_DECREF(f_stack); - return NULL; - } - }else if(PyDict_Contains(builtins, name)){ - PyObject* value = PyDict_GetItem(builtins, name); - if(PyDict_SetItem(frozen_builtins, name, value)){ - Py_DECREF(frozen_builtins); - Py_DECREF(frozen_globals); - Py_DECREF(f_stack); - return NULL; - } - }else if(PyDict_Contains(module_dict, name)){ - PyObject* value = PyDict_GetItem(module_dict, name); - - _PyDict_SetKeyImmutable((PyDictObject*)module_dict, name); - - if(!_Py_IsImmutable(value)){ - if(push(frontier, value)){ - goto nomemory; - } - } + }else if(PyDict_Contains(builtins, name)){ + PyObject* value = PyDict_GetItem(builtins, name); + if(PyDict_SetItem(shadow_builtins, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return NULL; } } + } - size = PySequence_Fast_GET_SIZE(f_code->co_consts); - for(Py_ssize_t i = 0; i < size; i++){ - PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); - if(!_Py_IsImmutable(value)){ - if(PyCode_Check(value)){ - _Py_SetImmutable(value); - - if(push(f_stack, value)){ - goto nomemory; - } - }else{ - if(push(frontier, value)){ - goto nomemory; - } - } - } - - if(check_globals && PyUnicode_Check(value)){ - // if the code calls the globals() builtin, then any - // cellvar or const in the function could, potentially, refer to - // a global variable. As such, we need to check if the globals - // dictionary contains that key and then make it immutable - // from this point forwards. - PyObject* name = value; - if(PyDict_Contains(globals, name)){ - value = PyDict_GetItem(globals, name); - if(PyDict_SetItem(frozen_globals, name, value)){ - Py_DECREF(frozen_builtins); - Py_DECREF(frozen_globals); - Py_DECREF(f_stack); - return NULL; - } + size = PySequence_Fast_GET_SIZE(f_code->co_consts); + for(Py_ssize_t i = 0; i < size; i++){ + PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); + if(check_globals && PyUnicode_Check(value)){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. + PyObject* name = value; + if(PyDict_Contains(globals, name)){ + value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(shadow_globals, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return NULL; } } } } - Py_DECREF(f_stack); - if(check_globals){ // if the code calls the globals() builtin, then any // cellvar or const in the function could, potentially, refer to @@ -253,9 +164,9 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) PyObject* name = value; if(PyDict_Contains(globals, name)){ value = PyDict_GetItem(globals, name); - if(PyDict_SetItem(frozen_globals, name, value)){ - Py_DECREF(frozen_builtins); - Py_DECREF(frozen_globals); + if(PyDict_SetItem(shadow_globals, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); return NULL; } } @@ -263,26 +174,17 @@ static PyObject* walk_function(PyObject* op, PyObject* frontier) } } - if(push(frontier, frozen_globals)){ - goto nomemory; - } - - f->func_globals = frozen_globals; + f->func_globals = shadow_globals; Py_DECREF(globals); - if(push(frontier, frozen_builtins)){ - goto nomemory; - } - - f->func_builtins = frozen_builtins; + f->func_builtins = shadow_builtins; Py_DECREF(builtins); Py_RETURN_NONE; nomemory: - Py_XDECREF(frozen_builtins); - Py_XDECREF(frozen_globals); - Py_XDECREF(f_stack); + Py_XDECREF(shadow_builtins); + Py_XDECREF(shadow_globals); return PyErr_NoMemory(); } @@ -367,19 +269,17 @@ PyObject* _Py_Freeze(PyObject* obj) } if(PyFunction_Check(item)){ - result = walk_function(item, frontier); + result = shadow_function_globals(item); if(!Py_IsNone(result)){ goto cleanup; } } - else - { - traverse = type->tp_traverse; - if(traverse != NULL){ - if(traverse(item, (visitproc)freeze_visit, frontier)){ - result = NULL; - goto cleanup; - } + + traverse = type->tp_traverse; + if(traverse != NULL){ + if(traverse(item, (visitproc)freeze_visit, frontier)){ + result = NULL; + goto cleanup; } } From 91362463ea557ea029a4fe49f15d3fed413d3f27 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Wed, 9 Apr 2025 14:08:58 +0100 Subject: [PATCH 07/40] Immutability for blake2 + test_freeze module Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/__init__.py | 27 ++++ Lib/test/test_freeze/__main__.py | 4 + Lib/test/test_freeze/test_array.py | 56 +++++++ Lib/test/test_freeze/test_collections.py | 71 +++++++++ .../test_core.py} | 147 +----------------- Lib/test/test_freeze/test_hashlib.py | 18 +++ Modules/_blake2/blake2b_impl.c | 5 + Modules/_blake2/blake2s_impl.c | 5 + 8 files changed, 189 insertions(+), 144 deletions(-) create mode 100644 Lib/test/test_freeze/__init__.py create mode 100644 Lib/test/test_freeze/__main__.py create mode 100644 Lib/test/test_freeze/test_array.py create mode 100644 Lib/test/test_freeze/test_collections.py rename Lib/test/{test_freeze.py => test_freeze/test_core.py} (79%) create mode 100644 Lib/test/test_freeze/test_hashlib.py diff --git a/Lib/test/test_freeze/__init__.py b/Lib/test/test_freeze/__init__.py new file mode 100644 index 00000000000000..38eb722baebac8 --- /dev/null +++ b/Lib/test/test_freeze/__init__.py @@ -0,0 +1,27 @@ +import os +from test.support import load_package_tests +import unittest + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) + + +class BaseObjectTest(unittest.TestCase): + def __init__(self, *args, obj=None, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + self.obj = obj + + def setUp(self): + freeze(self.obj) + + def test_immutable(self): + self.assertTrue(isimmutable(self.obj)) + + def test_add_attribute(self): + freeze(self.obj) + with self.assertRaises(NotWriteableError): + self.obj.new_attribute = 'value' + + def test_type_immutable(self): + self.assertTrue(isimmutable(type(self.obj))) diff --git a/Lib/test/test_freeze/__main__.py b/Lib/test/test_freeze/__main__.py new file mode 100644 index 00000000000000..40a23a297ec2b4 --- /dev/null +++ b/Lib/test/test_freeze/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_freeze/test_array.py b/Lib/test/test_freeze/test_array.py new file mode 100644 index 00000000000000..907d1066f82487 --- /dev/null +++ b/Lib/test/test_freeze/test_array.py @@ -0,0 +1,56 @@ +from array import array +from . import BaseObjectTest + + +class TestArray(BaseObjectTest): + def __init__(self, *args, **kwargs): + obj = array('i', [1, 2, 3, 4]) + BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) + + def test_set_item(self): + with self.assertRaises(NotWriteableError): + self.obj[0] = 5 + + def test_set_slice(self): + with self.assertRaises(NotWriteableError): + self.obj[1:3] = [6, 7] + + def test_append(self): + with self.assertRaises(NotWriteableError): + self.obj.append(8) + + def test_extend(self): + with self.assertRaises(NotWriteableError): + self.obj.extend(array('i', [9])) + + def test_insert(self): + with self.assertRaises(NotWriteableError): + self.obj.insert(0, 10) + + def test_pop(self): + with self.assertRaises(NotWriteableError): + self.obj.pop() + + def test_remove(self): + with self.assertRaises(NotWriteableError): + self.obj.remove(1) + + def test_delete(self): + with self.assertRaises(NotWriteableError): + del self.obj[0] + + def test_reverse(self): + with self.assertRaises(NotWriteableError): + self.obj.reverse() + + def test_inplace_repeat(self): + with self.assertRaises(NotWriteableError): + self.obj *= 2 + + def test_inplace_concat(self): + with self.assertRaises(NotWriteableError): + self.obj += array('i', [11]) + + def test_byteswap(self): + with self.assertRaises(NotWriteableError): + self.obj.byteswap() diff --git a/Lib/test/test_freeze/test_collections.py b/Lib/test/test_freeze/test_collections.py new file mode 100644 index 00000000000000..cd9359d7510a91 --- /dev/null +++ b/Lib/test/test_freeze/test_collections.py @@ -0,0 +1,71 @@ +from collections import deque + +from . import BaseObjectTest + +class TestDeque(BaseObjectTest): + class C: + pass + + def __init__(self, *args, **kwargs): + obj = deque([self.C(), self.C(), 1, "two", None]) + BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) + + def test_set_item(self): + with self.assertRaises(NotWriteableError): + self.obj[0] = None + + def test_append(self): + with self.assertRaises(NotWriteableError): + self.obj.append(TestDeque.C()) + + def test_appendleft(self): + with self.assertRaises(NotWriteableError): + self.obj.appendleft(TestDeque.C()) + + def test_extend(self): + with self.assertRaises(NotWriteableError): + self.obj.extend([TestDeque.C()]) + + def test_extendleft(self): + with self.assertRaises(NotWriteableError): + self.obj.extendleft([TestDeque.C()]) + + def test_insert(self): + with self.assertRaises(NotWriteableError): + self.obj.insert(0, TestDeque.C()) + + def test_pop(self): + with self.assertRaises(NotWriteableError): + self.obj.pop() + + def test_popleft(self): + with self.assertRaises(NotWriteableError): + self.obj.popleft() + + def test_remove(self): + with self.assertRaises(NotWriteableError): + self.obj.remove(1) + + def test_delete(self): + with self.assertRaises(NotWriteableError): + del self.obj[0] + + def test_inplace_repeat(self): + with self.assertRaises(NotWriteableError): + self.obj *= 2 + + def test_inplace_concat(self): + with self.assertRaises(NotWriteableError): + self.obj += [TestDeque.C()] + + def test_reverse(self): + with self.assertRaises(NotWriteableError): + self.obj.reverse() + + def test_rotate(self): + with self.assertRaises(NotWriteableError): + self.obj.rotate(1) + + def test_clear(self): + with self.assertRaises(NotWriteableError): + self.obj.clear() diff --git a/Lib/test/test_freeze.py b/Lib/test/test_freeze/test_core.py similarity index 79% rename from Lib/test/test_freeze.py rename to Lib/test/test_freeze/test_core.py index ddbfd7019573ba..dd33cf93420f3b 100644 --- a/Lib/test/test_freeze.py +++ b/Lib/test/test_freeze/test_core.py @@ -1,6 +1,7 @@ import unittest -from collections import deque -from array import array + +from . import BaseObjectTest + # This is a canary to check that global variables are not made immutable # when others are made immutable @@ -19,25 +20,6 @@ class MutableGlobalTest(unittest.TestCase): def test_global_mutable(self): self.assertTrue(not isimmutable(global_canary)) -class BaseObjectTest(unittest.TestCase): - def __init__(self, *args, obj=None, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - self.obj = obj - - def setUp(self): - freeze(self.obj) - - def test_immutable(self): - self.assertTrue(isimmutable(self.obj)) - - def test_add_attribute(self): - freeze(self.obj) - with self.assertRaises(NotWriteableError): - self.obj.new_attribute = 'value' - - def test_type_immutable(self): - self.assertTrue(isimmutable(type(self.obj))) - class TestBasicObject(BaseObjectTest): class C: @@ -120,129 +102,6 @@ def test_sort(self): self.obj.sort() -class TestDeque(BaseObjectTest): - class C: - pass - - def __init__(self, *args, **kwargs): - obj = deque([self.C(), self.C(), 1, "two", None]) - BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) - - def test_set_item(self): - with self.assertRaises(NotWriteableError): - self.obj[0] = None - - def test_append(self): - with self.assertRaises(NotWriteableError): - self.obj.append(TestList.C()) - - def test_appendleft(self): - with self.assertRaises(NotWriteableError): - self.obj.appendleft(TestList.C()) - - def test_extend(self): - with self.assertRaises(NotWriteableError): - self.obj.extend([TestList.C()]) - - def test_extendleft(self): - with self.assertRaises(NotWriteableError): - self.obj.extendleft([TestList.C()]) - - def test_insert(self): - with self.assertRaises(NotWriteableError): - self.obj.insert(0, TestList.C()) - - def test_pop(self): - with self.assertRaises(NotWriteableError): - self.obj.pop() - - def test_popleft(self): - with self.assertRaises(NotWriteableError): - self.obj.popleft() - - def test_remove(self): - with self.assertRaises(NotWriteableError): - self.obj.remove(1) - - def test_delete(self): - with self.assertRaises(NotWriteableError): - del self.obj[0] - - def test_inplace_repeat(self): - with self.assertRaises(NotWriteableError): - self.obj *= 2 - - def test_inplace_concat(self): - with self.assertRaises(NotWriteableError): - self.obj += [TestList.C()] - - def test_reverse(self): - with self.assertRaises(NotWriteableError): - self.obj.reverse() - - def test_rotate(self): - with self.assertRaises(NotWriteableError): - self.obj.rotate(1) - - def test_clear(self): - with self.assertRaises(NotWriteableError): - self.obj.clear() - - -class TestArray(BaseObjectTest): - def __init__(self, *args, **kwargs): - obj = array('i', [1, 2, 3, 4]) - BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) - - def test_set_item(self): - with self.assertRaises(NotWriteableError): - self.obj[0] = 5 - - def test_set_slice(self): - with self.assertRaises(NotWriteableError): - self.obj[1:3] = [6, 7] - - def test_append(self): - with self.assertRaises(NotWriteableError): - self.obj.append(8) - - def test_extend(self): - with self.assertRaises(NotWriteableError): - self.obj.extend(array('i', [9])) - - def test_insert(self): - with self.assertRaises(NotWriteableError): - self.obj.insert(0, 10) - - def test_pop(self): - with self.assertRaises(NotWriteableError): - self.obj.pop() - - def test_remove(self): - with self.assertRaises(NotWriteableError): - self.obj.remove(1) - - def test_delete(self): - with self.assertRaises(NotWriteableError): - del self.obj[0] - - def test_reverse(self): - with self.assertRaises(NotWriteableError): - self.obj.reverse() - - def test_inplace_repeat(self): - with self.assertRaises(NotWriteableError): - self.obj *= 2 - - def test_inplace_concat(self): - with self.assertRaises(NotWriteableError): - self.obj += array('i', [11]) - - def test_byteswap(self): - with self.assertRaises(NotWriteableError): - self.obj.byteswap() - - class TestDict(BaseObjectTest): class C: pass diff --git a/Lib/test/test_freeze/test_hashlib.py b/Lib/test/test_freeze/test_hashlib.py new file mode 100644 index 00000000000000..9943a3bf0f5a11 --- /dev/null +++ b/Lib/test/test_freeze/test_hashlib.py @@ -0,0 +1,18 @@ +from hashlib import blake2b, blake2s +import unittest + + +class TestHashlib(unittest.TestCase): + def test_blake2b(self): + h = blake2b(digest_size=32) + h.update(b'Hello world') + freeze(h) + with self.assertRaises(NotWriteableError): + h.update(b'!') + + def test_blake2s(self): + h = blake2s(digest_size=32) + h.update(b'Hello world') + freeze(h) + with self.assertRaises(NotWriteableError): + h.update(b'!') diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index c2cac98c7529eb..6398064f8c529f 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -19,6 +19,7 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() +#include "pycore_object.h" // Py_CHECKWRITE #include "../hashlib.h" #include "blake2module.h" @@ -276,6 +277,10 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) { Py_buffer buf; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 1c47328ece13e8..9a0be0adbe5061 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -19,6 +19,7 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() +#include "pycore_object.h" // Py_CHECKWRITE #include "../hashlib.h" #include "blake2module.h" @@ -276,6 +277,10 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) { Py_buffer buf; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) From 61a0400f8eda13ffbf4ee55b3c30a14482cc7488 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Wed, 9 Apr 2025 16:58:38 +0100 Subject: [PATCH 08/40] Fixing some holes in object/typeobject Signed-off-by: Matthew A Johnson --- Objects/funcobject.c | 10 ++++++++++ Objects/object.c | 5 +++++ Objects/typeobject.c | 46 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index f43e3a2787b846..72a624de01341c 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -633,7 +633,12 @@ func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) op->func_annotations = PyDict_New(); if (op->func_annotations == NULL) return NULL; + + if(!Py_CHECKWRITE(op)){ + _Py_SetImmutable(op->func_annotations); + } } + PyObject *d = func_get_annotation_dict(op); return Py_XNewRef(d); } @@ -641,6 +646,11 @@ func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) static int func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) { + if(!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return -1; + } + if (value == Py_None) value = NULL; /* Legal to del f.func_annotations. diff --git a/Objects/object.c b/Objects/object.c index 83f4d006f32eb0..01de54ff1a5885 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1554,6 +1554,11 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, return -1; } + if(!Py_CHECKWRITE(obj)){ + PyErr_WriteToImmutable(obj); + return -1; + } + Py_INCREF(name); Py_INCREF(tp); descr = _PyType_Lookup(tp, name); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index c0982deda1f98f..90be936ba385f6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1015,6 +1015,11 @@ type_set_name(PyTypeObject *type, PyObject *value, void *context) const char *tp_name; Py_ssize_t name_size; + if(!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } + if (!check_set_special_type_attr(type, value, "__name__")) return -1; if (!PyUnicode_Check(value)) { @@ -1044,6 +1049,11 @@ type_set_qualname(PyTypeObject *type, PyObject *value, void *context) { PyHeapTypeObject* et; + if(!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } + if (!check_set_special_type_attr(type, value, "__qualname__")) return -1; if (!PyUnicode_Check(value)) { @@ -1092,6 +1102,11 @@ type_module(PyTypeObject *type, void *context) static int type_set_module(PyTypeObject *type, PyObject *value, void *context) { + if(!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } + if (!check_set_special_type_attr(type, value, "__module__")) return -1; @@ -1123,6 +1138,10 @@ type_abstractmethods(PyTypeObject *type, void *context) static int type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) { + if(!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } /* __abstractmethods__ should only be set once on a type, in abc.ABCMeta.__new__, so this function doesn't do anything special to update subclasses. @@ -1257,6 +1276,11 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) static int type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) { + if(!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } + // Check arguments if (!check_set_special_type_attr(type, new_bases, "__bases__")) { return -1; @@ -1433,6 +1457,11 @@ type_get_text_signature(PyTypeObject *type, void *context) static int type_set_doc(PyTypeObject *type, PyObject *value, void *context) { + if(!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } + if (!check_set_special_type_attr(type, value, "__doc__")) return -1; PyType_Modified(type); @@ -1477,6 +1506,11 @@ type_get_annotations(PyTypeObject *type, void *context) static int type_set_annotations(PyTypeObject *type, PyObject *value, void *context) { + if(!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } + if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { PyErr_Format(PyExc_TypeError, "cannot set '__annotations__' attribute of immutable type '%s'", @@ -1521,6 +1555,11 @@ type_get_type_params(PyTypeObject *type, void *context) static int type_set_type_params(PyTypeObject *type, PyObject *value, void *context) { + if(!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } + if (!check_set_special_type_attr(type, value, "__type_params__")) { return -1; } @@ -3358,6 +3397,7 @@ static int type_new_set_name(const type_new_ctx *ctx, PyTypeObject *type) { Py_ssize_t name_size; + type->tp_name = PyUnicode_AsUTF8AndSize(ctx->name, &name_size); if (!type->tp_name) { return -1; @@ -4901,6 +4941,12 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) name, type->tp_name); return -1; } + + if (!Py_CHECKWRITE(type)){ + PyErr_WriteToImmutable(type); + return -1; + } + if (PyUnicode_Check(name)) { if (PyUnicode_CheckExact(name)) { if (PyUnicode_READY(name) == -1) From 6a6b04bff0fa351d698d0b8b1b3f1abcdf43c8d4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 10 Apr 2025 15:15:05 +0100 Subject: [PATCH 09/40] Remove immutable keys from dictionary as no longer required --- Include/cpython/pyerrors.h | 4 - Include/internal/pycore_dict.h | 18 +- .../pycore_global_objects_fini_generated.h | 2 - Include/internal/pycore_global_strings.h | 2 - .../internal/pycore_runtime_init_generated.h | 2 - Objects/dictobject.c | 305 +++---------- Objects/moduleobject.c | 8 +- Python/bytecodes.c | 26 +- Python/errors.c | 16 - Python/generated_cases.c.h | 416 +++++++++--------- Python/pylifecycle.c | 3 - Tools/gdb/libpython.py | 4 +- 12 files changed, 292 insertions(+), 514 deletions(-) diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 2f5914c4d385b8..74cf3f1d1de680 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -171,10 +171,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFormat( PyAPI_FUNC(PyObject *) _PyErr_WriteToImmutable(const char* file, int line, PyObject *obj); #define PyErr_WriteToImmutable(obj) _PyErr_WriteToImmutable(__FILE__, __LINE__, _PyObject_CAST(obj)) -PyAPI_FUNC(PyObject *) _PyErr_WriteToImmutableKey(const char* file, int line, PyObject *key); -#define PyErr_WriteToImmutableKey(key) _PyErr_WriteToImmutableKey(__FILE__, __LINE__, _PyObject_CAST(key)) - - extern PyObject *_PyErr_SetImportErrorWithNameFrom( PyObject *, PyObject *, diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 68b9846e4e7048..6253e0841ad349 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -24,27 +24,14 @@ typedef struct { /* Cached hash code of me_key. */ Py_hash_t me_hash; PyObject *me_key; - PyObject *_me_value; /* This field is only meaningful for combined tables */ + PyObject *me_value; /* This field is only meaningful for combined tables */ } PyDictKeyEntry; typedef struct { PyObject *me_key; /* The key must be Unicode and have hash. */ - PyObject *_me_value; /* This field is only meaningful for combined tables */ + PyObject *me_value; /* This field is only meaningful for combined tables */ } PyDictUnicodeEntry; -#define _PyDictEntry_IMMUTABLE_FLAG 0x1 -#define _PyDictEntry_VALUE_MASK ~_PyDictEntry_IMMUTABLE_FLAG - -#define _PyDictEntry_IsImmutable(entry) (((uintptr_t)((entry)->_me_value)) & _PyDictEntry_IMMUTABLE_FLAG) -#define _PyDictEntry_SetImmutable(entry) ((entry)->_me_value = (PyObject*)((uintptr_t)(entry)->_me_value | _PyDictEntry_IMMUTABLE_FLAG)) -#define _PyDictEntry_Hash(entry) ((entry)->me_hash) -#define _PyDictEntry_Key(entry) ((entry)->me_key) -#define _PyDictEntry_Value(entry) ((PyObject*)(((uintptr_t)((entry)->_me_value)) & _PyDictEntry_VALUE_MASK)) -#define _PyDictEntry_SetValue(entry, value) ((entry)->_me_value = value) -#define _PyDictEntry_IsEmpty(entry) ((((uintptr_t)(entry)->_me_value) & _PyDictEntry_VALUE_MASK) == (uintptr_t)NULL) - -extern PyObject *_PyDict_IsKeyImmutable(PyObject* op, PyObject* key); - extern PyDictKeysObject *_PyDict_NewKeysForClass(void); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); @@ -63,7 +50,6 @@ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t has extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); -extern PyObject *_PyDict_SetKeyImmutable(PyDictObject *mp, PyObject *key); /* Consumes references to key and value */ extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index cac1b4ed3be807..439f47a263dfa1 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -562,12 +562,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(namedtuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(shim_name)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(startswith)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 4bc9854b914509..0c84999cbf8127 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -48,12 +48,10 @@ struct _Py_global_strings { STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") - STRUCT_FOR_STR(namedtuple, "namedtuple") STRUCT_FOR_STR(newline, "\n") STRUCT_FOR_STR(open_br, "{") STRUCT_FOR_STR(percent, "%") STRUCT_FOR_STR(shim_name, "") - STRUCT_FOR_STR(startswith, "startswith") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index efd75a9fd08bb8..07f237b2905864 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -554,12 +554,10 @@ extern "C" { INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ - INIT_STR(namedtuple, "namedtuple"), \ INIT_STR(newline, "\n"), \ INIT_STR(open_br, "{"), \ INIT_STR(percent, "%"), \ INIT_STR(shim_name, ""), \ - INIT_STR(startswith, "startswith"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 56d1b520b6ab18..57253bb542ed7f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -497,11 +497,11 @@ dump_entries(PyDictKeysObject *dk) for (Py_ssize_t i = 0; i < dk->dk_nentries; i++) { if (DK_IS_UNICODE(dk)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(dk)[i]; - printf("key=%p value=%p\n", ep->me_key, _PyDictEntry_Value(ep)); + printf("key=%p value=%p\n", ep->me_key, ep->me_value); } else { PyDictKeyEntry *ep = &DK_ENTRIES(dk)[i]; - printf("key=%p hash=%lx value=%p\n", ep->me_key, ep->me_hash, _PyDictEntry_Value(ep)); + printf("key=%p hash=%lx value=%p\n", ep->me_key, ep->me_hash, ep->me_value); } } } @@ -551,7 +551,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) if (key != NULL) { /* test_dict fails if PyObject_Hash() is called again */ CHECK(entry->me_hash != -1); - CHECK(_PyDictEntry_Value(entry) != NULL); + CHECK(entry->me_value != NULL); if (PyUnicode_CheckExact(key)) { Py_hash_t hash = unicode_get_hash(key); @@ -571,12 +571,12 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) Py_hash_t hash = unicode_get_hash(key); CHECK(hash != -1); if (!splitted) { - CHECK(_PyDictEntry_Value(entry) != NULL); + CHECK(entry->me_value != NULL); } } if (splitted) { - CHECK(_PyDictEntry_Value(entry) == NULL); + CHECK(entry->me_value == NULL); } } } @@ -670,7 +670,7 @@ free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) Py_ssize_t i, n; for (i = 0, n = keys->dk_nentries; i < n; i++) { Py_XDECREF(entries[i].me_key); - Py_XDECREF(_PyDictEntry_Value(entries + i)); + Py_XDECREF(entries[i].me_value); } } else { @@ -678,7 +678,7 @@ free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) Py_ssize_t i, n; for (i = 0, n = keys->dk_nentries; i < n; i++) { Py_XDECREF(entries[i].me_key); - Py_XDECREF(_PyDictEntry_Value(entries + i)); + Py_XDECREF(entries[i].me_value); } } #if PyDict_MAXFREELIST > 0 @@ -812,19 +812,19 @@ clone_combined_dict_keys(PyDictObject *orig) if (DK_IS_UNICODE(orig->ma_keys)) { PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(keys); pkey = &ep0->me_key; - pvalue = &ep0->_me_value; + pvalue = &ep0->me_value; offs = sizeof(PyDictUnicodeEntry) / sizeof(PyObject*); } else { PyDictKeyEntry *ep0 = DK_ENTRIES(keys); pkey = &ep0->me_key; - pvalue = &ep0->_me_value; + pvalue = &ep0->me_value; offs = sizeof(PyDictKeyEntry) / sizeof(PyObject*); } Py_ssize_t n = keys->dk_nentries; for (Py_ssize_t i = 0; i < n; i++) { - PyObject *value = (PyObject*)(((uintptr_t)*pvalue >> 1) << 1); + PyObject *value = *pvalue; if (value != NULL) { Py_INCREF(value); Py_INCREF(*pkey); @@ -1075,7 +1075,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu *value_addr = mp->ma_values->values[ix]; } else { - *value_addr = _PyDictEntry_Value(DK_UNICODE_ENTRIES(dk) + ix); + *value_addr = DK_UNICODE_ENTRIES(dk)[ix].me_value; } } else { @@ -1088,7 +1088,7 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu goto start; } if (ix >= 0) { - *value_addr = _PyDictEntry_Value(DK_ENTRIES(dk) + ix); + *value_addr = DK_ENTRIES(dk)[ix].me_value; } else { *value_addr = NULL; @@ -1146,21 +1146,21 @@ _PyDict_MaybeUntrack(PyObject *op) } else { if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++, ep++) { - if ((value = _PyDictEntry_Value(ep)) == NULL) + PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(mp->ma_keys); + for (i = 0; i < numentries; i++) { + if ((value = ep0[i].me_value) == NULL) continue; if (_PyObject_GC_MAY_BE_TRACKED(value)) return; } } else { - PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); - for (i = 0; i < numentries; i++, ep++) { - if ((value = _PyDictEntry_Value(ep)) == NULL) + PyDictKeyEntry *ep0 = DK_ENTRIES(mp->ma_keys); + for (i = 0; i < numentries; i++) { + if ((value = ep0[i].me_value) == NULL) continue; if (_PyObject_GC_MAY_BE_TRACKED(value) || - _PyObject_GC_MAY_BE_TRACKED(_PyDictEntry_Key(ep))) + _PyObject_GC_MAY_BE_TRACKED(ep0[i].me_key)) return; } } @@ -1281,7 +1281,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, mp->ma_values->values[index] = value; } else { - _PyDictEntry_SetValue(ep, value); + ep->me_value = value; } } else { @@ -1289,7 +1289,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; ep->me_key = key; ep->me_hash = hash; - _PyDictEntry_SetValue(ep, value); + ep->me_value = value; } mp->ma_used++; mp->ma_version_tag = new_version; @@ -1301,22 +1301,6 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } if (old_value != value) { - if(DK_IS_UNICODE(mp->ma_keys)){ - PyDictUnicodeEntry *ep; - ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - if(_PyDictEntry_IsImmutable(ep)){ - PyErr_WriteToImmutableKey(_PyDictEntry_Key(ep)); - goto Fail; - } - }else{ - PyDictKeyEntry* ep; - ep = &DK_ENTRIES(mp->ma_keys)[ix]; - if(_PyDictEntry_IsImmutable(ep)){ - PyErr_WriteToImmutableKey(_PyDictEntry_Key(ep)); - goto Fail; - } - } - uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_MODIFIED, mp, key, value); if (_PyDict_HasSplitTable(mp)) { @@ -1329,10 +1313,10 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, else { assert(old_value != NULL); if (DK_IS_UNICODE(mp->ma_keys)) { - _PyDictEntry_SetValue(DK_UNICODE_ENTRIES(mp->ma_keys) + ix, value); + DK_UNICODE_ENTRIES(mp->ma_keys)[ix].me_value = value; } else { - _PyDictEntry_SetValue(DK_ENTRIES(mp->ma_keys) + ix, value); + DK_ENTRIES(mp->ma_keys)[ix].me_value = value; } } mp->ma_version_tag = new_version; @@ -1385,13 +1369,13 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, if (unicode) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys); ep->me_key = key; - _PyDictEntry_SetValue(ep, value); + ep->me_value = value; } else { PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys); ep->me_key = key; ep->me_hash = hash; - _PyDictEntry_SetValue(ep, value); + ep->me_value = value; } mp->ma_used++; mp->ma_version_tag = new_version; @@ -1492,34 +1476,26 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, if (mp->ma_keys->dk_kind == DICT_KEYS_GENERAL) { // split -> generic PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); - PyDictKeyEntry *new_ep = newentries; - for (Py_ssize_t i = 0; i < numentries; i++, new_ep++) { + for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); PyDictUnicodeEntry *ep = &oldentries[index]; assert(oldvalues->values[index] != NULL); - new_ep->me_key = Py_NewRef(ep->me_key); - new_ep->me_hash = unicode_get_hash(ep->me_key); - _PyDictEntry_SetValue(new_ep, oldvalues->values[index]); - if(_PyDictEntry_IsImmutable(ep)){ - _PyDictEntry_SetImmutable(new_ep); - } + newentries[i].me_key = Py_NewRef(ep->me_key); + newentries[i].me_hash = unicode_get_hash(ep->me_key); + newentries[i].me_value = oldvalues->values[index]; } build_indices_generic(mp->ma_keys, newentries, numentries); } else { // split -> combined unicode PyDictUnicodeEntry *newentries = DK_UNICODE_ENTRIES(mp->ma_keys); - PyDictUnicodeEntry *new_ep = newentries; - for (Py_ssize_t i = 0; i < numentries; i++, new_ep++) { + for (Py_ssize_t i = 0; i < numentries; i++) { int index = get_index_from_order(mp, i); PyDictUnicodeEntry *ep = &oldentries[index]; assert(oldvalues->values[index] != NULL); - new_ep->me_key = Py_NewRef(ep->me_key); - _PyDictEntry_SetValue(new_ep, oldvalues->values[index]); - if(_PyDictEntry_IsImmutable(ep)){ - _PyDictEntry_SetImmutable(new_ep); - } + newentries[i].me_key = Py_NewRef(ep->me_key); + newentries[i].me_value = oldvalues->values[index]; } build_indices_unicode(mp->ma_keys, newentries, numentries); } @@ -1539,7 +1515,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, else { PyDictKeyEntry *ep = oldentries; for (Py_ssize_t i = 0; i < numentries; i++) { - while (_PyDictEntry_IsEmpty(ep)) + while (ep->me_value == NULL) ep++; newentries[i] = *ep++; } @@ -1556,7 +1532,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, else { PyDictUnicodeEntry *ep = oldentries; for (Py_ssize_t i = 0; i < numentries; i++) { - while (_PyDictEntry_IsEmpty(ep)) + while (ep->me_value == NULL) ep++; newentries[i] = *ep++; } @@ -1565,17 +1541,14 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } else { // combined unicode -> generic PyDictKeyEntry *newentries = DK_ENTRIES(mp->ma_keys); - PyDictKeyEntry *new_ep = newentries; PyDictUnicodeEntry *ep = oldentries; - for (Py_ssize_t i = 0; i < numentries; i++, ep++, new_ep++) { - while (_PyDictEntry_IsEmpty(ep)) + for (Py_ssize_t i = 0; i < numentries; i++) { + while (ep->me_value == NULL) ep++; - new_ep->me_key = ep->me_key; - new_ep->me_hash = unicode_get_hash(ep->me_key); - _PyDictEntry_SetValue(new_ep, _PyDictEntry_Value(ep)); - if(_PyDictEntry_IsImmutable(ep)){ - _PyDictEntry_SetImmutable(new_ep); - } + newentries[i].me_key = ep->me_key; + newentries[i].me_hash = unicode_get_hash(ep->me_key); + newentries[i].me_value = ep->me_value; + ep++; } build_indices_generic(mp->ma_keys, newentries, numentries); } @@ -1880,40 +1853,6 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) return value; } -PyObject *_PyDict_SetKeyImmutable(PyDictObject* dict, PyObject *key) -{ - Py_ssize_t ix; - Py_hash_t hash; - PyObject *value; - - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) - return NULL; - } - - /* namespace 1: globals */ - ix = _Py_dict_lookup(dict, key, hash, &value); - if (ix == DKIX_ERROR){ - return NULL; - } - - if(ix == DKIX_EMPTY){ - _PyErr_SetKeyError(key); - return NULL; - } - - if(DK_IS_UNICODE(dict->ma_keys)){ - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(dict->ma_keys)[ix]; - _PyDictEntry_SetImmutable(ep); - }else{ - PyDictKeyEntry *ep = &DK_ENTRIES(dict->ma_keys)[ix]; - _PyDictEntry_SetImmutable(ep); - } - - return value; -} - /* Consumes references to key and value */ int _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) @@ -2020,25 +1959,15 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY); if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - if(_PyDictEntry_IsImmutable(ep)){ - PyErr_WriteToImmutableKey(ep->me_key); - return -1; - } - old_key = ep->me_key; ep->me_key = NULL; - _PyDictEntry_SetValue(ep, NULL); + ep->me_value = NULL; } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; - if(_PyDictEntry_IsImmutable(ep)){ - PyErr_WriteToImmutableKey(ep->me_key); - return -1; - } - old_key = ep->me_key; ep->me_key = NULL; - _PyDictEntry_SetValue(ep, NULL); + ep->me_value = NULL; ep->me_hash = 0; } Py_DECREF(old_key); @@ -2225,7 +2154,7 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, return 0; if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(mp->ma_keys)[i]; - while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { + while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; i++; } @@ -2233,11 +2162,11 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, return 0; key = entry_ptr->me_key; hash = unicode_get_hash(entry_ptr->me_key); - value = _PyDictEntry_Value(entry_ptr); + value = entry_ptr->me_value; } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(mp->ma_keys)[i]; - while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { + while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; i++; } @@ -2245,7 +2174,7 @@ _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, return 0; key = entry_ptr->me_key; hash = entry_ptr->me_hash; - value = _PyDictEntry_Value(entry_ptr); + value = entry_ptr->me_value; } } *ppos = i+1; @@ -3261,12 +3190,12 @@ dict_equal(PyDictObject *a, PyDictObject *b) if (a->ma_values) aval = a->ma_values->values[i]; else - aval = _PyDictEntry_Value(ep); + aval = ep->me_value; } else { PyDictKeyEntry *ep = &DK_ENTRIES(a->ma_keys)[i]; key = ep->me_key; - aval = _PyDictEntry_Value(ep); + aval = ep->me_value; hash = ep->me_hash; } if (aval != NULL) { @@ -3448,14 +3377,14 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) _PyDictValues_AddToInsertionOrder(mp->ma_values, index); } else { - _PyDictEntry_SetValue(ep, Py_NewRef(value)); + ep->me_value = Py_NewRef(value); } } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[mp->ma_keys->dk_nentries]; ep->me_key = Py_NewRef(key); ep->me_hash = hash; - _PyDictEntry_SetValue(ep, Py_NewRef(value)); + ep->me_value = Py_NewRef(value); } MAINTAIN_TRACKING(mp, key, value); mp->ma_used++; @@ -3593,43 +3522,35 @@ dict_popitem_impl(PyDictObject *self) if (DK_IS_UNICODE(self->ma_keys)) { PyDictUnicodeEntry *ep0 = DK_UNICODE_ENTRIES(self->ma_keys); i = self->ma_keys->dk_nentries - 1; - while (i >= 0 && _PyDictEntry_IsEmpty(ep0 + i)) { + while (i >= 0 && ep0[i].me_value == NULL) { i--; } assert(i >= 0); - if(_PyDictEntry_IsImmutable(ep0 + i)){ - return PyErr_WriteToImmutableKey(ep0[i].me_key); - } - key = ep0[i].me_key; new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); - value = _PyDictEntry_Value(ep0 + i); + value = ep0[i].me_value; ep0[i].me_key = NULL; - _PyDictEntry_SetValue(ep0 + i, NULL); + ep0[i].me_value = NULL; } else { PyDictKeyEntry *ep0 = DK_ENTRIES(self->ma_keys); i = self->ma_keys->dk_nentries - 1; - while (i >= 0 && _PyDictEntry_IsEmpty(ep0 + i)) { + while (i >= 0 && ep0[i].me_value == NULL) { i--; } assert(i >= 0); - if(_PyDictEntry_IsImmutable(ep0 + i)){ - return PyErr_WriteToImmutableKey(ep0[i].me_key); - } - key = ep0[i].me_key; new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; - value = _PyDictEntry_Value(ep0 + i); + value = ep0[i].me_value; ep0[i].me_key = NULL; ep0[i].me_hash = -1; - _PyDictEntry_SetValue(ep0 + i, NULL); + ep0[i].me_value = NULL; } j = lookdict_index(self->ma_keys, hash, i); @@ -3663,15 +3584,15 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) else { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); for (i = 0; i < n; i++) { - Py_VISIT(_PyDictEntry_Value(entries + i)); + Py_VISIT(entries[i].me_value); } } } else { PyDictKeyEntry *entries = DK_ENTRIES(keys); for (i = 0; i < n; i++) { - if (!_PyDictEntry_IsEmpty(entries + i)) { - Py_VISIT(_PyDictEntry_Value(entries + i)); + if (entries[i].me_value != NULL) { + Py_VISIT(entries[i].me_value); Py_VISIT(entries[i].me_key); } } @@ -4190,7 +4111,7 @@ dictiter_iternextkey(dictiterobject *di) Py_ssize_t n = k->dk_nentries; if (DK_IS_UNICODE(k)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; - while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { + while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; i++; } @@ -4200,7 +4121,7 @@ dictiter_iternextkey(dictiterobject *di) } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; - while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { + while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; i++; } @@ -4289,23 +4210,23 @@ dictiter_iternextvalue(dictiterobject *di) Py_ssize_t n = d->ma_keys->dk_nentries; if (DK_IS_UNICODE(d->ma_keys)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(d->ma_keys)[i]; - while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { + while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; i++; } if (i >= n) goto fail; - value = _PyDictEntry_Value(entry_ptr); + value = entry_ptr->me_value; } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[i]; - while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { + while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; i++; } if (i >= n) goto fail; - value = _PyDictEntry_Value(entry_ptr); + value = entry_ptr->me_value; } } // We found an element, but did not expect it @@ -4389,25 +4310,25 @@ dictiter_iternextitem(dictiterobject *di) Py_ssize_t n = d->ma_keys->dk_nentries; if (DK_IS_UNICODE(d->ma_keys)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(d->ma_keys)[i]; - while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { + while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; i++; } if (i >= n) goto fail; key = entry_ptr->me_key; - value = _PyDictEntry_Value(entry_ptr); + value = entry_ptr->me_value; } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(d->ma_keys)[i]; - while (i < n && _PyDictEntry_IsEmpty(entry_ptr)) { + while (i < n && entry_ptr->me_value == NULL) { entry_ptr++; i++; } if (i >= n) goto fail; key = entry_ptr->me_key; - value = _PyDictEntry_Value(entry_ptr); + value = entry_ptr->me_value; } } // We found an element, but did not expect it @@ -4517,25 +4438,25 @@ dictreviter_iternext(dictiterobject *di) else { if (DK_IS_UNICODE(k)) { PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(k)[i]; - while (_PyDictEntry_IsEmpty(entry_ptr)) { + while (entry_ptr->me_value == NULL) { if (--i < 0) { goto fail; } entry_ptr--; } key = entry_ptr->me_key; - value = _PyDictEntry_Value(entry_ptr); + value = entry_ptr->me_value; } else { PyDictKeyEntry *entry_ptr = &DK_ENTRIES(k)[i]; - while (_PyDictEntry_IsEmpty(entry_ptr)) { + while (entry_ptr->me_value == NULL) { if (--i < 0) { goto fail; } entry_ptr--; } key = entry_ptr->me_key; - value = _PyDictEntry_Value(entry_ptr); + value = entry_ptr->me_value; } } di->di_pos = i-1; @@ -5996,83 +5917,3 @@ _PyDict_SendEvent(int watcher_bits, watcher_bits >>= 1; } } - -PyObject * -_PyDict_IsKeyImmutable(PyObject* op, PyObject* key) -{ - PyDictKeysObject *dk; - DictKeysKind kind; - Py_ssize_t ix; - PyDictObject* mp; - Py_hash_t hash; - PyThreadState* tstate; - PyObject *exc; - - if (!PyDict_Check(op)) { - return NULL; - } - mp = (PyDictObject *)op; - - if (!PyUnicode_CheckExact(key) || (hash = unicode_get_hash(key)) == -1) { - hash = PyObject_Hash(key); - if (hash == -1) { - PyErr_Clear(); - return NULL; - } - } - - tstate = _PyThreadState_GET(); -#ifdef Py_DEBUG - // bpo-40839: Before Python 3.10, it was possible to call PyDict_GetItem() - // with the GIL released. - _Py_EnsureTstateNotNULL(tstate); -#endif - - /* Preserve the existing exception */ - exc = _PyErr_GetRaisedException(tstate); - -start: - dk = mp->ma_keys; - kind = dk->dk_kind; - - if (kind != DICT_KEYS_GENERAL) { - if (PyUnicode_CheckExact(key)) { - ix = unicodekeys_lookup_unicode(dk, key, hash); - } - else { - ix = unicodekeys_lookup_generic(mp, dk, key, hash); - if (ix == DKIX_KEY_CHANGED) { - goto start; - } - } - } - else { - ix = dictkeys_generic_lookup(mp, dk, key, hash); - if (ix == DKIX_KEY_CHANGED) { - goto start; - } - } - - /* Ignore any exception raised by the lookup */ - _PyErr_SetRaisedException(tstate, exc); - - if (ix == DKIX_ERROR){ - return NULL; - } - - if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys) + ix; - if (_PyDictEntry_IsImmutable(ep)){ - Py_RETURN_TRUE; - }else{ - Py_RETURN_FALSE; - } - } else { - PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys) + ix; - if(_PyDictEntry_IsImmutable(ep)){ - Py_RETURN_TRUE; - }else{ - Py_RETURN_FALSE; - } - } -} \ No newline at end of file diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 5d44796a3ef092..4daf1a929e0549 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -7,7 +7,6 @@ #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_moduleobject.h" // _PyModule_GetDef() -#include "pycore_dict.h" // _PyDict_IsKeyImmutable() #include "structmember.h" // PyMemberDef @@ -601,9 +600,8 @@ void _PyModule_Clear(PyObject *m) { PyObject *d = ((PyModuleObject *)m)->md_dict; - if (d != NULL){ + if (d != NULL) _PyModule_ClearDict(d); - } } void @@ -624,7 +622,7 @@ _PyModule_ClearDict(PyObject *d) /* First, clear only names starting with a single underscore */ pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { - if (value != Py_None && PyUnicode_Check(key) && Py_IsFalse(_PyDict_IsKeyImmutable(d, key))) { + if (value != Py_None && PyUnicode_Check(key)) { if (PyUnicode_READ_CHAR(key, 0) == '_' && PyUnicode_READ_CHAR(key, 1) != '_') { if (verbose > 1) { @@ -644,7 +642,7 @@ _PyModule_ClearDict(PyObject *d) /* Next, clear all names except for __builtins__ */ pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { - if (value != Py_None && PyUnicode_Check(key) && Py_IsFalse(_PyDict_IsKeyImmutable(d, key))) { + if (value != Py_None && PyUnicode_Check(key)) { if (PyUnicode_READ_CHAR(key, 0) != '_' || !_PyUnicode_EqualToASCIIString(key, "__builtins__")) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7c65202cc87f29..bca36ec5db5876 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1360,7 +1360,7 @@ dummy_func( DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = _PyDictEntry_Value(entries + index); + res = entries[index].me_value; DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); @@ -1377,7 +1377,7 @@ dummy_func( DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = _PyDictEntry_Value(entries + index); + res = entries[index].me_value; DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); @@ -1838,7 +1838,7 @@ dummy_func( assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = _PyDictEntry_Value(ep); + res = ep->me_value; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); @@ -1862,12 +1862,12 @@ dummy_func( if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = _PyDictEntry_Value(ep); + res = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = _PyDictEntry_Value(ep); + res = ep->me_value; } DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); @@ -2005,26 +2005,18 @@ dummy_func( if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = _PyDictEntry_Value(ep); + old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - if(_PyDictEntry_IsImmutable(ep)){ - format_exc_notwriteable(tstate, frame->f_code, oparg); - goto error; - } - _PyDictEntry_SetValue(ep, value); + ep->me_value = value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = _PyDictEntry_Value(ep); + old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - if(_PyDictEntry_IsImmutable(ep)){ - format_exc_notwriteable(tstate, frame->f_code, oparg); - goto error; - } - _PyDictEntry_SetValue(ep, value); + ep->me_value = value; } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); diff --git a/Python/errors.c b/Python/errors.c index 27ff884c4b934e..7126df2b8088a9 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1966,22 +1966,6 @@ _PyErr_WriteToImmutable(const char* filename, int lineno, PyObject* obj) return NULL; } -PyObject * -_PyErr_WriteToImmutableKey(const char* filename, int lineno, PyObject* key) -{ - PyObject* string; - PyThreadState *tstate = _PyThreadState_GET(); - if (!_PyErr_Occurred(tstate)) { - string = PyUnicode_FromFormat("key %R is marked as immutable at %s:%d", - key, filename, lineno); - if (string != NULL) { - _PyErr_SetObject(tstate, PyExc_NotWriteableError, string); - Py_DECREF(string); - } - } - return NULL; -} - #ifdef __cplusplus } #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e805c2f9d76665..54c2fe8ddea36e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1842,7 +1842,7 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); - res = _PyDictEntry_Value(entries + index); + res = entries[index].me_value; DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); @@ -1872,7 +1872,7 @@ DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL); assert(DK_IS_UNICODE(bdict->ma_keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); - res = _PyDictEntry_Value(entries + index); + res = entries[index].me_value; DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); @@ -2533,7 +2533,7 @@ assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = _PyDictEntry_Value(ep); + res = ep->me_value; DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); @@ -2569,12 +2569,12 @@ if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = _PyDictEntry_Value(ep); + res = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = _PyDictEntry_Value(ep); + res = ep->me_value; } DEOPT_IF(res == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); @@ -2768,26 +2768,18 @@ if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = _PyDictEntry_Value(ep); + old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - if(_PyDictEntry_IsImmutable(ep)){ - format_exc_notwriteable(tstate, frame->f_code, oparg); - goto error; - } - _PyDictEntry_SetValue(ep, value); + ep->me_value = value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, STORE_ATTR); - old_value = _PyDictEntry_Value(ep); + old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); - if(_PyDictEntry_IsImmutable(ep)){ - format_exc_notwriteable(tstate, frame->f_code, oparg); - goto error; - } - _PyDictEntry_SetValue(ep, value); + ep->me_value = value; } Py_DECREF(old_value); STAT_INC(STORE_ATTR, hit); @@ -2798,7 +2790,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2802 "Python/generated_cases.c.h" + #line 2794 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2809,7 +2801,7 @@ PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); - #line 2041 "Python/bytecodes.c" + #line 2033 "Python/bytecodes.c" PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2824,7 +2816,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2828 "Python/generated_cases.c.h" + #line 2820 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2836,7 +2828,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 2065 "Python/bytecodes.c" + #line 2057 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2849,12 +2841,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2853 "Python/generated_cases.c.h" + #line 2845 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 2078 "Python/bytecodes.c" + #line 2070 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2858 "Python/generated_cases.c.h" + #line 2850 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2865,7 +2857,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 2082 "Python/bytecodes.c" + #line 2074 "Python/bytecodes.c" DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2876,7 +2868,7 @@ _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; - #line 2880 "Python/generated_cases.c.h" + #line 2872 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2887,7 +2879,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 2096 "Python/bytecodes.c" + #line 2088 "Python/bytecodes.c" DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2902,7 +2894,7 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; - #line 2906 "Python/generated_cases.c.h" + #line 2898 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2913,7 +2905,7 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *res; - #line 2114 "Python/bytecodes.c" + #line 2106 "Python/bytecodes.c" DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2925,7 +2917,7 @@ assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; - #line 2929 "Python/generated_cases.c.h" + #line 2921 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2936,14 +2928,14 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 2128 "Python/bytecodes.c" + #line 2120 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2942 "Python/generated_cases.c.h" + #line 2934 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 2130 "Python/bytecodes.c" + #line 2122 "Python/bytecodes.c" b = res ? Py_True : Py_False; - #line 2947 "Python/generated_cases.c.h" + #line 2939 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2953,15 +2945,15 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 2134 "Python/bytecodes.c" + #line 2126 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2959 "Python/generated_cases.c.h" + #line 2951 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); - #line 2136 "Python/bytecodes.c" + #line 2128 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; - #line 2965 "Python/generated_cases.c.h" + #line 2957 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2972,12 +2964,12 @@ PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; - #line 2141 "Python/bytecodes.c" + #line 2133 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2978 "Python/generated_cases.c.h" + #line 2970 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 2143 "Python/bytecodes.c" + #line 2135 "Python/bytecodes.c" if (true) goto pop_2_error; } @@ -2985,10 +2977,10 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2989 "Python/generated_cases.c.h" + #line 2981 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); - #line 2151 "Python/bytecodes.c" + #line 2143 "Python/bytecodes.c" if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); @@ -2997,7 +2989,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 3001 "Python/generated_cases.c.h" + #line 2993 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -3007,21 +2999,21 @@ PyObject *right = stack_pointer[-1]; PyObject *left = stack_pointer[-2]; PyObject *b; - #line 2162 "Python/bytecodes.c" + #line 2154 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 3014 "Python/generated_cases.c.h" + #line 3006 "Python/generated_cases.c.h" Py_DECREF(right); - #line 2165 "Python/bytecodes.c" + #line 2157 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 3021 "Python/generated_cases.c.h" + #line 3013 "Python/generated_cases.c.h" Py_DECREF(right); - #line 2170 "Python/bytecodes.c" + #line 2162 "Python/bytecodes.c" b = res ? Py_True : Py_False; - #line 3025 "Python/generated_cases.c.h" + #line 3017 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -3030,15 +3022,15 @@ PyObject *fromlist = stack_pointer[-1]; PyObject *level = stack_pointer[-2]; PyObject *res; - #line 2174 "Python/bytecodes.c" + #line 2166 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 3037 "Python/generated_cases.c.h" + #line 3029 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); - #line 2177 "Python/bytecodes.c" + #line 2169 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 3042 "Python/generated_cases.c.h" + #line 3034 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -3047,29 +3039,29 @@ TARGET(IMPORT_FROM) { PyObject *from = stack_pointer[-1]; PyObject *res; - #line 2181 "Python/bytecodes.c" + #line 2173 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 3055 "Python/generated_cases.c.h" + #line 3047 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); } TARGET(JUMP_FORWARD) { - #line 2187 "Python/bytecodes.c" + #line 2179 "Python/bytecodes.c" JUMPBY(oparg); - #line 3064 "Python/generated_cases.c.h" + #line 3056 "Python/generated_cases.c.h" DISPATCH(); } TARGET(JUMP_BACKWARD) { PREDICTED(JUMP_BACKWARD); - #line 2191 "Python/bytecodes.c" + #line 2183 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 3073 "Python/generated_cases.c.h" + #line 3065 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3077,15 +3069,15 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); PyObject *cond = stack_pointer[-1]; - #line 2197 "Python/bytecodes.c" + #line 2189 "Python/bytecodes.c" if (Py_IsFalse(cond)) { JUMPBY(oparg); } else if (!Py_IsTrue(cond)) { int err = PyObject_IsTrue(cond); - #line 3087 "Python/generated_cases.c.h" + #line 3079 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2203 "Python/bytecodes.c" + #line 2195 "Python/bytecodes.c" if (err == 0) { JUMPBY(oparg); } @@ -3093,22 +3085,22 @@ if (err < 0) goto pop_1_error; } } - #line 3097 "Python/generated_cases.c.h" + #line 3089 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { PyObject *cond = stack_pointer[-1]; - #line 2213 "Python/bytecodes.c" + #line 2205 "Python/bytecodes.c" if (Py_IsTrue(cond)) { JUMPBY(oparg); } else if (!Py_IsFalse(cond)) { int err = PyObject_IsTrue(cond); - #line 3110 "Python/generated_cases.c.h" + #line 3102 "Python/generated_cases.c.h" Py_DECREF(cond); - #line 2219 "Python/bytecodes.c" + #line 2211 "Python/bytecodes.c" if (err > 0) { JUMPBY(oparg); } @@ -3116,63 +3108,63 @@ if (err < 0) goto pop_1_error; } } - #line 3120 "Python/generated_cases.c.h" + #line 3112 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { PyObject *value = stack_pointer[-1]; - #line 2229 "Python/bytecodes.c" + #line 2221 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 3129 "Python/generated_cases.c.h" + #line 3121 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2231 "Python/bytecodes.c" + #line 2223 "Python/bytecodes.c" JUMPBY(oparg); } - #line 3134 "Python/generated_cases.c.h" + #line 3126 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { PyObject *value = stack_pointer[-1]; - #line 2236 "Python/bytecodes.c" + #line 2228 "Python/bytecodes.c" if (Py_IsNone(value)) { JUMPBY(oparg); } else { - #line 3146 "Python/generated_cases.c.h" + #line 3138 "Python/generated_cases.c.h" Py_DECREF(value); - #line 2241 "Python/bytecodes.c" + #line 2233 "Python/bytecodes.c" } - #line 3150 "Python/generated_cases.c.h" + #line 3142 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #line 2245 "Python/bytecodes.c" + #line 2237 "Python/bytecodes.c" /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ JUMPBY(-oparg); - #line 3163 "Python/generated_cases.c.h" + #line 3155 "Python/generated_cases.c.h" DISPATCH(); } TARGET(GET_LEN) { PyObject *obj = stack_pointer[-1]; PyObject *len_o; - #line 2254 "Python/bytecodes.c" + #line 2246 "Python/bytecodes.c" // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 3176 "Python/generated_cases.c.h" + #line 3168 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3183,16 +3175,16 @@ PyObject *type = stack_pointer[-2]; PyObject *subject = stack_pointer[-3]; PyObject *attrs; - #line 2262 "Python/bytecodes.c" + #line 2254 "Python/bytecodes.c" // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3192 "Python/generated_cases.c.h" + #line 3184 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); - #line 2267 "Python/bytecodes.c" + #line 2259 "Python/bytecodes.c" if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -3200,7 +3192,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_None; // Failure! } - #line 3204 "Python/generated_cases.c.h" + #line 3196 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3209,10 +3201,10 @@ TARGET(MATCH_MAPPING) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2277 "Python/bytecodes.c" + #line 2269 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - #line 3216 "Python/generated_cases.c.h" + #line 3208 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3222,10 +3214,10 @@ TARGET(MATCH_SEQUENCE) { PyObject *subject = stack_pointer[-1]; PyObject *res; - #line 2283 "Python/bytecodes.c" + #line 2275 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - #line 3229 "Python/generated_cases.c.h" + #line 3221 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3236,11 +3228,11 @@ PyObject *keys = stack_pointer[-1]; PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; - #line 2289 "Python/bytecodes.c" + #line 2281 "Python/bytecodes.c" // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3244 "Python/generated_cases.c.h" + #line 3236 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3249,14 +3241,14 @@ TARGET(GET_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2295 "Python/bytecodes.c" + #line 2287 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3256 "Python/generated_cases.c.h" + #line 3248 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2298 "Python/bytecodes.c" + #line 2290 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3260 "Python/generated_cases.c.h" + #line 3252 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3264,7 +3256,7 @@ TARGET(GET_YIELD_FROM_ITER) { PyObject *iterable = stack_pointer[-1]; PyObject *iter; - #line 2302 "Python/bytecodes.c" + #line 2294 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -3287,11 +3279,11 @@ if (iter == NULL) { goto error; } - #line 3291 "Python/generated_cases.c.h" + #line 3283 "Python/generated_cases.c.h" Py_DECREF(iterable); - #line 2325 "Python/bytecodes.c" + #line 2317 "Python/bytecodes.c" } - #line 3295 "Python/generated_cases.c.h" + #line 3287 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3302,7 +3294,7 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2344 "Python/bytecodes.c" + #line 2336 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3333,7 +3325,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3337 "Python/generated_cases.c.h" + #line 3329 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3341,7 +3333,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #line 2377 "Python/bytecodes.c" + #line 2369 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -3367,14 +3359,14 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3371 "Python/generated_cases.c.h" + #line 3363 "Python/generated_cases.c.h" DISPATCH(); } TARGET(FOR_ITER_LIST) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2405 "Python/bytecodes.c" + #line 2397 "Python/bytecodes.c" DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); _PyListIterObject *it = (_PyListIterObject *)iter; STAT_INC(FOR_ITER, hit); @@ -3394,7 +3386,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3398 "Python/generated_cases.c.h" + #line 3390 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3404,7 +3396,7 @@ TARGET(FOR_ITER_TUPLE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2427 "Python/bytecodes.c" + #line 2419 "Python/bytecodes.c" _PyTupleIterObject *it = (_PyTupleIterObject *)iter; DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3424,7 +3416,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3428 "Python/generated_cases.c.h" + #line 3420 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3434,7 +3426,7 @@ TARGET(FOR_ITER_RANGE) { PyObject *iter = stack_pointer[-1]; PyObject *next; - #line 2449 "Python/bytecodes.c" + #line 2441 "Python/bytecodes.c" _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); @@ -3452,7 +3444,7 @@ if (next == NULL) { goto error; } - #line 3456 "Python/generated_cases.c.h" + #line 3448 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3461,7 +3453,7 @@ TARGET(FOR_ITER_GEN) { PyObject *iter = stack_pointer[-1]; - #line 2469 "Python/bytecodes.c" + #line 2461 "Python/bytecodes.c" DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -3477,14 +3469,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3481 "Python/generated_cases.c.h" + #line 3473 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2487 "Python/bytecodes.c" + #line 2479 "Python/bytecodes.c" PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3507,16 +3499,16 @@ Py_DECREF(enter); goto error; } - #line 3511 "Python/generated_cases.c.h" + #line 3503 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2510 "Python/bytecodes.c" + #line 2502 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3520 "Python/generated_cases.c.h" + #line 3512 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3528,7 +3520,7 @@ PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; - #line 2520 "Python/bytecodes.c" + #line 2512 "Python/bytecodes.c" /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3554,16 +3546,16 @@ Py_DECREF(enter); goto error; } - #line 3558 "Python/generated_cases.c.h" + #line 3550 "Python/generated_cases.c.h" Py_DECREF(mgr); - #line 2546 "Python/bytecodes.c" + #line 2538 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3567 "Python/generated_cases.c.h" + #line 3559 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3575,7 +3567,7 @@ PyObject *lasti = stack_pointer[-3]; PyObject *exit_func = stack_pointer[-4]; PyObject *res; - #line 2555 "Python/bytecodes.c" + #line 2547 "Python/bytecodes.c" /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3601,7 +3593,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3605 "Python/generated_cases.c.h" + #line 3597 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3610,7 +3602,7 @@ TARGET(PUSH_EXC_INFO) { PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; - #line 2583 "Python/bytecodes.c" + #line 2575 "Python/bytecodes.c" _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3620,7 +3612,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3624 "Python/generated_cases.c.h" + #line 3616 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3634,7 +3626,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2595 "Python/bytecodes.c" + #line 2587 "Python/bytecodes.c" /* Cached method object */ PyTypeObject *self_cls = Py_TYPE(self); assert(type_version != 0); @@ -3651,7 +3643,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3655 "Python/generated_cases.c.h" + #line 3647 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3665,7 +3657,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2614 "Python/bytecodes.c" + #line 2606 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); assert(self_cls->tp_dictoffset == 0); @@ -3675,7 +3667,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3679 "Python/generated_cases.c.h" + #line 3671 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3689,7 +3681,7 @@ PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - #line 2626 "Python/bytecodes.c" + #line 2618 "Python/bytecodes.c" PyTypeObject *self_cls = Py_TYPE(self); DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); Py_ssize_t dictoffset = self_cls->tp_dictoffset; @@ -3703,7 +3695,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3707 "Python/generated_cases.c.h" + #line 3699 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3712,16 +3704,16 @@ } TARGET(KW_NAMES) { - #line 2642 "Python/bytecodes.c" + #line 2634 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3720 "Python/generated_cases.c.h" + #line 3712 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #line 2648 "Python/bytecodes.c" + #line 2640 "Python/bytecodes.c" int is_meth = PEEK(oparg+2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(total_args + 1); @@ -3734,7 +3726,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3738 "Python/generated_cases.c.h" + #line 3730 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3744,7 +3736,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2693 "Python/bytecodes.c" + #line 2685 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3826,7 +3818,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3830 "Python/generated_cases.c.h" + #line 3822 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3838,7 +3830,7 @@ TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2781 "Python/bytecodes.c" + #line 2773 "Python/bytecodes.c" DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3848,7 +3840,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3852 "Python/generated_cases.c.h" + #line 3844 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3857,7 +3849,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2793 "Python/bytecodes.c" + #line 2785 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3883,7 +3875,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3887 "Python/generated_cases.c.h" + #line 3879 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3891,7 +3883,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - #line 2821 "Python/bytecodes.c" + #line 2813 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -3927,7 +3919,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3931 "Python/generated_cases.c.h" + #line 3923 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3935,7 +3927,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2859 "Python/bytecodes.c" + #line 2851 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3945,7 +3937,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3949 "Python/generated_cases.c.h" + #line 3941 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3958,7 +3950,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2871 "Python/bytecodes.c" + #line 2863 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3969,7 +3961,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3973 "Python/generated_cases.c.h" + #line 3965 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3983,7 +3975,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2885 "Python/bytecodes.c" + #line 2877 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3994,7 +3986,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3998 "Python/generated_cases.c.h" + #line 3990 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4008,7 +4000,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2899 "Python/bytecodes.c" + #line 2891 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4030,7 +4022,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4034 "Python/generated_cases.c.h" + #line 4026 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4044,7 +4036,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2924 "Python/bytecodes.c" + #line 2916 "Python/bytecodes.c" /* Builtin METH_O functions */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4072,7 +4064,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4076 "Python/generated_cases.c.h" + #line 4068 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4086,7 +4078,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2955 "Python/bytecodes.c" + #line 2947 "Python/bytecodes.c" /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); int is_meth = method != NULL; @@ -4118,7 +4110,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4122 "Python/generated_cases.c.h" + #line 4114 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4132,7 +4124,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2990 "Python/bytecodes.c" + #line 2982 "Python/bytecodes.c" /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -4164,7 +4156,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4168 "Python/generated_cases.c.h" + #line 4160 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4178,7 +4170,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3025 "Python/bytecodes.c" + #line 3017 "Python/bytecodes.c" assert(kwnames == NULL); /* len(o) */ int is_meth = method != NULL; @@ -4203,7 +4195,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4207 "Python/generated_cases.c.h" + #line 4199 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4216,7 +4208,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3052 "Python/bytecodes.c" + #line 3044 "Python/bytecodes.c" assert(kwnames == NULL); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4243,7 +4235,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4247 "Python/generated_cases.c.h" + #line 4239 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4255,7 +4247,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 3082 "Python/bytecodes.c" + #line 3074 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -4273,14 +4265,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4277 "Python/generated_cases.c.h" + #line 4269 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3102 "Python/bytecodes.c" + #line 3094 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4311,7 +4303,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4315 "Python/generated_cases.c.h" + #line 4307 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4324,7 +4316,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3136 "Python/bytecodes.c" + #line 3128 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4353,7 +4345,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4357 "Python/generated_cases.c.h" + #line 4349 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4366,7 +4358,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3168 "Python/bytecodes.c" + #line 3160 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4395,7 +4387,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4399 "Python/generated_cases.c.h" + #line 4391 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4408,7 +4400,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 3200 "Python/bytecodes.c" + #line 3192 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -4436,7 +4428,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4440 "Python/generated_cases.c.h" + #line 4432 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4446,9 +4438,9 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #line 3231 "Python/bytecodes.c" + #line 3223 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4452 "Python/generated_cases.c.h" + #line 4444 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4457,7 +4449,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 3235 "Python/bytecodes.c" + #line 3227 "Python/bytecodes.c" // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4519,14 +4511,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4523 "Python/generated_cases.c.h" + #line 4515 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3297 "Python/bytecodes.c" + #line 3289 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4530 "Python/generated_cases.c.h" + #line 4522 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4541,7 +4533,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3307 "Python/bytecodes.c" + #line 3299 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4570,14 +4562,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4574 "Python/generated_cases.c.h" + #line 4566 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3338 "Python/bytecodes.c" + #line 3330 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4598,7 +4590,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4602 "Python/generated_cases.c.h" + #line 4594 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4606,15 +4598,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3361 "Python/bytecodes.c" + #line 3353 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4612 "Python/generated_cases.c.h" + #line 4604 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3363 "Python/bytecodes.c" + #line 3355 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4618 "Python/generated_cases.c.h" + #line 4610 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4625,7 +4617,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3367 "Python/bytecodes.c" + #line 3359 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4660,7 +4652,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4664 "Python/generated_cases.c.h" + #line 4656 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4669,10 +4661,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3404 "Python/bytecodes.c" + #line 3396 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4676 "Python/generated_cases.c.h" + #line 4668 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4684,7 +4676,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3409 "Python/bytecodes.c" + #line 3401 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4699,12 +4691,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4703 "Python/generated_cases.c.h" + #line 4695 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3424 "Python/bytecodes.c" + #line 3416 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4708 "Python/generated_cases.c.h" + #line 4700 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4714,16 +4706,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3429 "Python/bytecodes.c" + #line 3421 "Python/bytecodes.c" assert(oparg >= 2); - #line 4720 "Python/generated_cases.c.h" + #line 4712 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3433 "Python/bytecodes.c" + #line 3425 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4735,26 +4727,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4739 "Python/generated_cases.c.h" + #line 4731 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3447 "Python/bytecodes.c" + #line 3439 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4745 "Python/generated_cases.c.h" + #line 4737 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3451 "Python/bytecodes.c" + #line 3443 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4752 "Python/generated_cases.c.h" + #line 4744 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3456 "Python/bytecodes.c" + #line 3448 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4763,12 +4755,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4767 "Python/generated_cases.c.h" + #line 4759 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3467 "Python/bytecodes.c" + #line 3459 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4777,12 +4769,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4781 "Python/generated_cases.c.h" + #line 4773 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3478 "Python/bytecodes.c" + #line 3470 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4794,12 +4786,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4798 "Python/generated_cases.c.h" + #line 4790 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3492 "Python/bytecodes.c" + #line 3484 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4811,30 +4803,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4815 "Python/generated_cases.c.h" + #line 4807 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3506 "Python/bytecodes.c" + #line 3498 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4826 "Python/generated_cases.c.h" + #line 4818 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3514 "Python/bytecodes.c" + #line 3506 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4833 "Python/generated_cases.c.h" + #line 4825 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3519 "Python/bytecodes.c" + #line 3511 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4840 "Python/generated_cases.c.h" + #line 4832 "Python/generated_cases.c.h" } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c392b478f2b52c..29771e07ae6a2c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1497,7 +1497,6 @@ finalize_modules_clear_weaklist(PyInterpreterState *interp, if (verbose && PyUnicode_Check(name)) { PySys_FormatStderr("# cleanup[3] wiping %U\n", name); } - _PyModule_Clear(mod); Py_DECREF(mod); } @@ -1532,7 +1531,6 @@ finalize_modules(PyThreadState *tstate) // Already done return; } - int verbose = _PyInterpreterState_GetConfig(interp)->verbose; // Delete some special builtins._ and sys attributes first. These are @@ -1861,7 +1859,6 @@ Py_FinalizeEx(void) PyGC_Collect(); /* Destroy all modules */ - _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index ea51c682ca8f9f..e38bd59e20a305 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -782,9 +782,7 @@ def iteritems(self): entries, nentries = self._get_entries(keys) for i in safe_range(nentries): ep = entries[i] - value_ptr = ep['_me_value'].reinterpret_cast(gdb.lookup_type('uintptr_t')) - # Clear the immutability flag - pyop_value = PyObjectPtr.from_pyobject_ptr((value_ptr >> 1) << 1) + pyop_value = PyObjectPtr.from_pyobject_ptr(ep['me_value']) if not pyop_value.is_null(): pyop_key = PyObjectPtr.from_pyobject_ptr(ep['me_key']) yield (pyop_key, pyop_value) From 0caa7888b23844a81b7a889cbf9408348e44202a Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 11 Apr 2025 10:37:29 +0100 Subject: [PATCH 10/40] Dealing with buffer immutability Signed-off-by: Matthew A Johnson --- Objects/abstract.c | 11 +++++++++++ Objects/funcobject.c | 4 ---- Python/immutability.c | 7 +++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 92a690e26035ff..f6d17d71b87e3d 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -611,6 +611,11 @@ PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char *ptr; const char *src; + if(!Py_CHECKWRITE(view->obj)){ + PyErr_WriteToImmutable(view->obj); + return -1; + } + if (len > view->len) { len = view->len; } @@ -670,6 +675,12 @@ int PyObject_CopyData(PyObject *dest, PyObject *src) return -1; } + + if(!Py_CHECKWRITE(dest)){ + PyErr_WriteToImmutable(dest); + return -1; + } + if (PyObject_GetBuffer(dest, &view_dest, PyBUF_FULL) != 0) return -1; if (PyObject_GetBuffer(src, &view_src, PyBUF_FULL_RO) != 0) { PyBuffer_Release(&view_dest); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 72a624de01341c..94801de7c71105 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -633,10 +633,6 @@ func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored)) op->func_annotations = PyDict_New(); if (op->func_annotations == NULL) return NULL; - - if(!Py_CHECKWRITE(op)){ - _Py_SetImmutable(op->func_annotations); - } } PyObject *d = func_get_annotation_dict(op); diff --git a/Python/immutability.c b/Python/immutability.c index a9293f797e421e..c8613ee179c7b2 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -180,6 +180,13 @@ static PyObject* shadow_function_globals(PyObject* op) f->func_builtins = shadow_builtins; Py_DECREF(builtins); + if(f->func_annotations == NULL){ + f->func_annotations = PyDict_New(); + if(f->func_annotations == NULL){ + goto nomemory; + } + } + Py_RETURN_NONE; nomemory: From a5ceb515fa90d13332569c0e1f15708741a850ba Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 11 Apr 2025 12:49:22 +0100 Subject: [PATCH 11/40] _ctypes immutability Signed-off-by: Matthew A Johnson --- Include/internal/pycore_object.h | 5 -- Include/object.h | 5 ++ Lib/test/test_freeze/test_ctypes.py | 131 ++++++++++++++++++++++++++++ Modules/_blake2/blake2b_impl.c | 1 - Modules/_blake2/blake2s_impl.c | 1 - Modules/_collectionsmodule.c | 1 - Modules/_ctypes/_ctypes.c | 71 +++++++++++++++ Modules/arraymodule.c | 1 - 8 files changed, 207 insertions(+), 9 deletions(-) create mode 100644 Lib/test/test_freeze/test_ctypes.py diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 26659af8cd15a3..16078406cea384 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -100,11 +100,6 @@ static inline void _Py_SetImmutable(PyObject *op) } #define _Py_SetImmutable(op) _Py_SetImmutable(_PyObject_CAST(op)) -// Check whether an object is writeable. -// This check will always succeed during runtime finalization. -#define Py_CHECKWRITE(op) ((op) && (!_Py_IsImmutable(op) || _Py_IsFinalizing())) -#define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} - static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { diff --git a/Include/object.h b/Include/object.h index 8d3443f400e6c4..ac03b0c8d57078 100644 --- a/Include/object.h +++ b/Include/object.h @@ -279,6 +279,11 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) } #define _Py_IsImmutable(op) _Py_IsImmutable(_PyObject_CAST(op)) +// Check whether an object is writeable. +// This check will always succeed during runtime finalization. +#define Py_CHECKWRITE(op) ((op) && (!_Py_IsImmutable(op) || _Py_IsFinalizing())) +#define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} + static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { // This immortal check is for code that is unaware of immortal objects. // The runtime tracks these objects and we should avoid as much diff --git a/Lib/test/test_freeze/test_ctypes.py b/Lib/test/test_freeze/test_ctypes.py new file mode 100644 index 00000000000000..b357175aa0c7a5 --- /dev/null +++ b/Lib/test/test_freeze/test_ctypes.py @@ -0,0 +1,131 @@ +import ctypes +import unittest + +from . import BaseObjectTest + + +class TestCharArray(BaseObjectTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=ctypes.create_string_buffer(b"hello"), **kwargs) + + def test_raw(self): + with self.assertRaises(NotWriteableError): + self.obj.raw = b"world" + + self.assertEqual(self.obj.raw, b"hello\x00") + + def test_value(self): + with self.assertRaises(NotWriteableError): + self.obj.value = b"world" + + self.assertEqual(self.obj.value, b"hello") + + +class TestWCharArray(BaseObjectTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=ctypes.create_unicode_buffer("hello"), **kwargs) + + def test_value(self): + with self.assertRaises(NotWriteableError): + self.obj.value = "world" + + self.assertEqual(self.obj.value, "hello") + + +class TestStructure(BaseObjectTest): + class POINT(ctypes.Structure): + _fields_ = [("x", ctypes.c_int), ("y", ctypes.c_int)] + + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=TestStructure.POINT(1, 2), **kwargs) + + def test_modify_field(self): + with self.assertRaises(NotWriteableError): + self.obj.x = 3 + + self.assertEqual(self.obj.x, 1) + + +class TestPointer(BaseObjectTest): + class POINT(ctypes.Structure): + _fields_ = [("x", ctypes.c_int), ("y", ctypes.c_int)] + + def __init__(self, *args, **kwargs): + self.a = TestPointer.POINT(1, 2) + super().__init__(*args, obj=ctypes.pointer(self.a), **kwargs) + + def test_contents_immutable(self): + self.assertTrue(isimmutable(self.a)) + self.assertTrue(isimmutable(TestPointer.POINT)) + + def test_set_contents(self): + b = TestPointer.POINT(3, 4) + with self.assertRaises(NotWriteableError): + self.obj.contents = b + + self.assertEqual(self.obj.contents.x, self.a.x) + self.assertEqual(self.obj.contents.y, self.a.y) + + +class TestArray(BaseObjectTest): + class POINT(ctypes.Structure): + _fields_ = [("x", ctypes.c_int), ("y", ctypes.c_int)] + + def __init__(self, *args, **kwargs): + TenPointsArrayType = TestArray.POINT * 10 + super().__init__(*args, obj=TenPointsArrayType(), **kwargs) + + def test_point_immutable(self): + self.assertTrue(isimmutable(self.obj[0])) + self.assertTrue(isimmutable(TestArray.POINT)) + + def test_modify_item(self): + with self.assertRaises(NotWriteableError): + self.obj[0].x = 1 + + self.assertEqual(self.obj[0].x, 0) + + def test_ass_item(self): + with self.assertRaises(NotWriteableError): + self.obj[0] = TestArray.POINT(1, 2) + + def test_ass_subscript(self): + TwoPointsArrayType = TestArray.POINT * 2 + a = TwoPointsArrayType() + with self.assertRaises(NotWriteableError): + self.obj[:2] = a + + +class TestUnion(BaseObjectTest): + class INTPARTS(ctypes.Union): + class SHORTS(ctypes.Structure): + _fields_ = [("high", ctypes.c_short), + ("low", ctypes.c_short)] + + _fields_ = [("parts", SHORTS), + ("value", ctypes.c_int)] + + def __init__(self, *args, **kwargs): + a = TestUnion.INTPARTS() + a.value = ctypes.c_int(0x00FF00FF) + super().__init__(*args, obj=a, **kwargs) + + def test_assign_part(self): + with self.assertRaises(NotWriteableError): + self.obj.parts.high = 0 + + self.assertEqual(self.obj.parts.high, 0xFF) + + with self.assertRaises(NotWriteableError): + self.obj.parts.low = 0 + + self.assertEqual(self.obj.parts.low, 0xFF) + self.assertEqual(self.obj.value, 0x00FF00FF) + + def test_assign_value(self): + with self.assertRaises(NotWriteableError): + self.obj.value = 0x00FF00FF + + self.assertEqual(self.obj.value, 0x00FF00FF) + self.assertEqual(self.obj.parts.high, 0xFF) + self.assertEqual(self.obj.parts.low, 0xFF) diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index 6398064f8c529f..9e62c7cf6d8e1c 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -19,7 +19,6 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() -#include "pycore_object.h" // Py_CHECKWRITE #include "../hashlib.h" #include "blake2module.h" diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 9a0be0adbe5061..b9eb9dae0f2085 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -19,7 +19,6 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() -#include "pycore_object.h" // Py_CHECKWRITE #include "../hashlib.h" #include "blake2module.h" diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 1c127a3c4dc5dc..c27338818f5968 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -3,7 +3,6 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "pycore_object.h" // Py_CHECKWRITE #include "structmember.h" // PyMemberDef #include diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 534ef8c1d6cf8f..d4677d50dc6873 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -111,6 +111,7 @@ bytes(cdata) #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_object.h" // _Py_SetImmutable() #include "structmember.h" // PyMemberDef #include @@ -1158,6 +1159,9 @@ PyCPointerType_set_type(PyTypeObject *self, PyObject *type) { StgDictObject *dict; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } dict = PyType_stgdict((PyObject *)self); if (!dict) { @@ -1293,6 +1297,11 @@ CharArray_set_raw(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) Py_ssize_t size; Py_buffer view; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (value == NULL) { PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); return -1; @@ -1339,6 +1348,11 @@ CharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored) const char *ptr; Py_ssize_t size; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); @@ -1391,6 +1405,11 @@ WCharArray_get_value(CDataObject *self, void *Py_UNUSED(ignored)) static int WCharArray_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) { + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); @@ -2996,6 +3015,11 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) memcpy(cmem->b_ptr, adr, dict->size); cmem->b_index = index; } + + if(_Py_IsImmutable(base)) { + _Py_SetImmutable(cmem); + } + return (PyObject *)cmem; } @@ -3182,6 +3206,11 @@ PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, return -1; } + if(!Py_CHECKWRITE(dst)){ + PyErr_WriteToImmutable(dst); + return -1; + } + result = _PyCData_set(mem, type, setfunc, value, size, ptr); if (result == NULL) @@ -3232,6 +3261,11 @@ GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int PyCFuncPtr_set_errcheck(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) { + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (ob && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "the errcheck attribute must be callable"); @@ -3255,6 +3289,12 @@ static int PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) { PyObject *checker, *oldchecker; + + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (ob == NULL) { oldchecker = self->checker; self->checker = NULL; @@ -3299,6 +3339,11 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ig { PyObject *converters; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (ob == NULL || ob == Py_None) { Py_CLEAR(self->converters); Py_CLEAR(self->argtypes); @@ -4674,6 +4719,11 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) StgDictObject *stgdict; char *ptr; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Array does not support item deletion"); @@ -4700,6 +4750,11 @@ Array_ass_subscript(PyObject *myself, PyObject *item, PyObject *value) { CDataObject *self = (CDataObject *)myself; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Array does not support item deletion"); @@ -4905,6 +4960,12 @@ static int Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) { PyObject *result; + + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + StgDictObject *dict = PyObject_stgdict((PyObject *)self); if (value == NULL) { @@ -5089,6 +5150,11 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) StgDictObject *stgdict, *itemdict; PyObject *proto; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Pointer does not support item deletion"); @@ -5143,6 +5209,11 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) CDataObject *dst; PyObject *keep; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Pointer does not support item deletion"); diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 68709b5b7dee16..97dd0b6ecb42a2 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -11,7 +11,6 @@ #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_bytesobject.h" // _PyBytes_Repeat -#include "pycore_object.h" // Py_CHECKWRITE #include "structmember.h" // PyMemberDef #include // offsetof() From 2e54f321d204153e74c1e633bebf8c4958c4b293 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 11 Apr 2025 13:21:15 +0100 Subject: [PATCH 12/40] _decimal Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_ctypes.py | 1 - Lib/test/test_freeze/test_decimal.py | 32 ++++++++++ Modules/_decimal/_decimal.c | 95 +++++++++++++++++++++++++++- 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 Lib/test/test_freeze/test_decimal.py diff --git a/Lib/test/test_freeze/test_ctypes.py b/Lib/test/test_freeze/test_ctypes.py index b357175aa0c7a5..9fbc8135c7c2f5 100644 --- a/Lib/test/test_freeze/test_ctypes.py +++ b/Lib/test/test_freeze/test_ctypes.py @@ -1,5 +1,4 @@ import ctypes -import unittest from . import BaseObjectTest diff --git a/Lib/test/test_freeze/test_decimal.py b/Lib/test/test_freeze/test_decimal.py new file mode 100644 index 00000000000000..bd140ff9f1b164 --- /dev/null +++ b/Lib/test/test_freeze/test_decimal.py @@ -0,0 +1,32 @@ +import decimal + +from . import BaseObjectTest + + +class TestContext(BaseObjectTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=decimal.Context(), **kwargs) + + def test_prec(self): + with self.assertRaises(NotWriteableError): + self.obj.prec = 10 + + def test_emax(self): + with self.assertRaises(NotWriteableError): + self.obj.Emax = 10 + + def test_emin(self): + with self.assertRaises(NotWriteableError): + self.obj.Emin = -10 + + def test_rounding(self): + with self.assertRaises(NotWriteableError): + self.obj.rounding = decimal.ROUND_DOWN + + def test_capitals(self): + with self.assertRaises(NotWriteableError): + self.obj.capitals = 0 + + def test_clamp(self): + with self.assertRaises(NotWriteableError): + self.obj.clamp = 1 diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 70b13982bb0cc4..ee866b45190d98 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -588,6 +588,11 @@ signaldict_setitem(PyObject *self, PyObject *key, PyObject *value) uint32_t flag; int x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (SdFlagAddr(self) == NULL) { return value_error_int(INVALID_SIGNALDICT_ERROR_MSG); } @@ -810,6 +815,11 @@ context_setprec(PyObject *self, PyObject *value, void *closure UNUSED) mpd_context_t *ctx; mpd_ssize_t x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { return -1; @@ -830,6 +840,11 @@ context_setemin(PyObject *self, PyObject *value, void *closure UNUSED) mpd_context_t *ctx; mpd_ssize_t x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { return -1; @@ -850,6 +865,11 @@ context_setemax(PyObject *self, PyObject *value, void *closure UNUSED) mpd_context_t *ctx; mpd_ssize_t x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { return -1; @@ -871,6 +891,11 @@ context_unsafe_setprec(PyObject *self, PyObject *value) mpd_context_t *ctx = CTX(self); mpd_ssize_t x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { return NULL; @@ -891,6 +916,11 @@ context_unsafe_setemin(PyObject *self, PyObject *value) mpd_context_t *ctx = CTX(self); mpd_ssize_t x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { return NULL; @@ -911,6 +941,11 @@ context_unsafe_setemax(PyObject *self, PyObject *value) mpd_context_t *ctx = CTX(self); mpd_ssize_t x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { return NULL; @@ -932,6 +967,11 @@ context_setround(PyObject *self, PyObject *value, void *closure UNUSED) mpd_context_t *ctx; int x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = getround(value); if (x == -1) { return -1; @@ -950,6 +990,11 @@ context_setcapitals(PyObject *self, PyObject *value, void *closure UNUSED) { mpd_ssize_t x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { return -1; @@ -971,6 +1016,11 @@ context_settraps(PyObject *self, PyObject *value, void *closure UNUSED) mpd_context_t *ctx; uint32_t flags; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + flags = long_as_flags(value); if (flags & DEC_ERRORS) { return -1; @@ -991,6 +1041,11 @@ context_settraps_list(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + flags = list_as_flags(value); if (flags & DEC_ERRORS) { return -1; @@ -1010,6 +1065,11 @@ context_settraps_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (PyDecSignalDict_Check(value)) { flags = SdFlags(value); } @@ -1035,6 +1095,11 @@ context_setstatus(PyObject *self, PyObject *value, void *closure UNUSED) mpd_context_t *ctx; uint32_t flags; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + flags = long_as_flags(value); if (flags & DEC_ERRORS) { return -1; @@ -1055,6 +1120,11 @@ context_setstatus_list(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + flags = list_as_flags(value); if (flags & DEC_ERRORS) { return -1; @@ -1074,6 +1144,11 @@ context_setstatus_dict(PyObject *self, PyObject *value) mpd_context_t *ctx; uint32_t flags; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + if (PyDecSignalDict_Check(value)) { flags = SdFlags(value); } @@ -1098,6 +1173,11 @@ context_setclamp(PyObject *self, PyObject *value, void *closure UNUSED) mpd_context_t *ctx; mpd_ssize_t x; + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } + x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { return -1; @@ -1117,7 +1197,12 @@ static int context_setallcr(PyObject *self, PyObject *value, void *closure UNUSED) { mpd_context_t *ctx; - mpd_ssize_t x; + mpd_ssize_t x;j + + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + return -1; + } x = PyLong_AsSsize_t(value); if (x == -1 && PyErr_Occurred()) { @@ -1238,6 +1323,10 @@ context_setattrs(PyObject *self, PyObject *prec, PyObject *rounding, static PyObject * context_clear_traps(PyObject *self, PyObject *dummy UNUSED) { + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + CTX(self)->traps = 0; Py_RETURN_NONE; } @@ -1245,6 +1334,10 @@ context_clear_traps(PyObject *self, PyObject *dummy UNUSED) static PyObject * context_clear_flags(PyObject *self, PyObject *dummy UNUSED) { + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + CTX(self)->status = 0; Py_RETURN_NONE; } From 32d3fe7874cfdf4a88cb8546e0a8dd4236d885ba Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Mon, 14 Apr 2025 11:02:29 +0100 Subject: [PATCH 13/40] Adding a NotFreezable type Signed-off-by: Matthew A Johnson --- Include/Python.h | 1 + Include/cpython/immutability.h | 6 ++++++ Include/immutability.h | 20 ++++++++++++++++++ Include/internal/pycore_immutability.h | 18 ---------------- Lib/test/test_freeze/test_notfreezable.py | 13 ++++++++++++ Makefile.pre.in | 2 ++ Objects/memoryobject.c | 4 ++++ Objects/object.c | 1 + PC/python3dll.c | 1 + PCbuild/pythoncore.vcxproj | 2 ++ PCbuild/pythoncore.vcxproj.filters | 3 +++ Python/bltinmodule.c | 2 +- Python/immutability.c | 25 ++++++++++++++++++++++- Tools/c-analyzer/cpython/_parser.py | 1 + 14 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 Include/cpython/immutability.h create mode 100644 Include/immutability.h delete mode 100644 Include/internal/pycore_immutability.h create mode 100644 Lib/test/test_freeze/test_notfreezable.py diff --git a/Include/Python.h b/Include/Python.h index 52a7aac6ba6cb6..eb6b17a4beba9b 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -105,5 +105,6 @@ #include "fileutils.h" #include "cpython/pyfpe.h" #include "tracemalloc.h" +#include "immutability.h" #endif /* !Py_PYTHON_H */ diff --git a/Include/cpython/immutability.h b/Include/cpython/immutability.h new file mode 100644 index 00000000000000..54ab13a80a4792 --- /dev/null +++ b/Include/cpython/immutability.h @@ -0,0 +1,6 @@ +#ifndef Py_CPYTHON_IMMUTABILITY_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(PyObject *) _Py_Freeze(PyObject*); +#define Py_Freeze(op) _Py_Freeze(_PyObject_CAST(op)) diff --git a/Include/immutability.h b/Include/immutability.h new file mode 100644 index 00000000000000..4cc60971e54a38 --- /dev/null +++ b/Include/immutability.h @@ -0,0 +1,20 @@ +#ifndef Py_IMMUTABILITY_H +#define Py_IMMUTABILITY_H + +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_DATA(PyTypeObject) PyNotFreezable_Type; + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_IMMUTABILITY_H +# include "cpython/immutability.h" +# undef Py_CPYTHON_IMMUTABILITY_H +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_IMMUTABILITY_H */ diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h deleted file mode 100644 index 426ac5ff7564da..00000000000000 --- a/Include/internal/pycore_immutability.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef Py_FREEZE_H -#define Py_FREEZE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -PyObject* _Py_Freeze(PyObject*); -#define Py_Freeze(op) _Py_Freeze(_PyObject_CAST(op)) - -#ifdef __cplusplus -} -#endif -#endif /* !Py_FREEZE_H */ \ No newline at end of file diff --git a/Lib/test/test_freeze/test_notfreezable.py b/Lib/test/test_freeze/test_notfreezable.py new file mode 100644 index 00000000000000..c754d0c389f3a4 --- /dev/null +++ b/Lib/test/test_freeze/test_notfreezable.py @@ -0,0 +1,13 @@ +import unittest + +class Mock(notfreezable): + def __init__(self, name): + self.name = name + + +class TestNotFreezable(unittest.TestCase): + def test_not_freezable(self): + # Test that the NotFreezable class raises an exception when trying to freeze + mock = Mock("test") + with self.assertRaises(TypeError): + freeze(mock) diff --git a/Makefile.pre.in b/Makefile.pre.in index 7e479cc1552495..a9659e7aa3cc52 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1617,6 +1617,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/floatobject.h \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/import.h \ + $(srcdir)/Include/immutability.h \ $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ @@ -1687,6 +1688,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/funcobject.h \ $(srcdir)/Include/cpython/genobject.h \ $(srcdir)/Include/cpython/import.h \ + $(srcdir)/Include/cpython/immutability.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/listobject.h \ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index b0168044d9f85a..d6a605d4bc2d15 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -99,6 +99,10 @@ _PyManagedBuffer_FromObject(PyObject *base, int flags) return NULL; } + if(_Py_IsImmutable(base)){ + _Py_SetImmutable(mbuf); + } + return (PyObject *)mbuf; } diff --git a/Objects/object.c b/Objects/object.c index 01de54ff1a5885..bba1e37a960a5a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2102,6 +2102,7 @@ static PyTypeObject* static_types[] = { &PyMethod_Type, &PyModuleDef_Type, &PyModule_Type, + &PyNotFreezable_Type, &PyODictIter_Type, &PyPickleBuffer_Type, &PyProperty_Type, diff --git a/PC/python3dll.c b/PC/python3dll.c index 1dd359fe04a7cf..5479e32b408a23 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -863,6 +863,7 @@ EXPORT_DATA(PyMemoryView_Type) EXPORT_DATA(PyMethodDescr_Type) EXPORT_DATA(PyModule_Type) EXPORT_DATA(PyModuleDef_Type) +EXPORT_DATA(PyNotFreezable_Type) EXPORT_DATA(PyOS_InputHook) EXPORT_DATA(PyProperty_Type) EXPORT_DATA(PyRange_Type) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 1c7af9630faba0..04dcd6cdbf91f1 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -151,6 +151,7 @@ + @@ -194,6 +195,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 17400e52f0909f..54aa040db67799 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -465,6 +465,9 @@ Include + + Include\cpython + Include\cpython diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 085d4d253b1568..c9c7863c70cc79 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -11,7 +11,6 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ceval.h" // _PyEval_Vector() -#include "pycore_immutability.h" // _PyFreeze() #include "pycore_dict.h" // _PyDict_SetGlobalImmutable() #include "clinic/bltinmodule.c.h" @@ -3164,6 +3163,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("filter", &PyFilter_Type); SETBUILTIN("float", &PyFloat_Type); SETBUILTIN("frozenset", &PyFrozenSet_Type); + SETBUILTIN("notfreezable", &PyNotFreezable_Type); SETBUILTIN("property", &PyProperty_Type); SETBUILTIN("int", &PyLong_Type); SETBUILTIN("list", &PyList_Type); diff --git a/Python/immutability.c b/Python/immutability.c index c8613ee179c7b2..bc40b29fb80fa2 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -5,7 +5,23 @@ #include #include "pycore_dict.h" #include "pycore_object.h" -#include "pycore_immutability.h" + + +PyDoc_STRVAR(notfreezable_doc, + "NotFreezable()\n\ + \n\ + Indicate that a type cannot be frozen."); + + +PyTypeObject PyNotFreezable_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "NotFreezable", + .tp_doc = notfreezable_doc, + .tp_basicsize = sizeof(PyObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_BASETYPE, + .tp_new = PyType_GenericNew +}; static int push(PyObject* s, PyObject* item){ @@ -262,6 +278,13 @@ PyObject* _Py_Freeze(PyObject* obj) } type = Py_TYPE(item); + + if(PyType_IsSubtype(type, &PyNotFreezable_Type)){ + PyErr_SetString(PyExc_TypeError, "Cannot freeze a NotFreezable object"); + result = NULL; + goto cleanup; + } + type_op = NULL; if(_Py_IsImmutable(item)){ diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 1587a19716fe0b..d65a0fa0e8edec 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -227,6 +227,7 @@ def clean_lines(text): Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 +Include/cpython/immutability.h Py_CPYTHON_IMMUTABILITY_H 1 Include/cpython/interpreteridobject.h Py_CPYTHON_INTERPRETERIDOBJECT_H 1 Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 From 37f2d76c0a304375c34f55e9bc42ea2108a67423 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Mon, 14 Apr 2025 11:41:44 +0100 Subject: [PATCH 14/40] _io Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/__init__.py | 10 ++++++ Lib/test/test_freeze/test_io.py | 37 +++++++++++++++++++++++ Lib/test/test_freeze/test_notfreezable.py | 13 -------- Modules/_io/_iomodule.c | 6 ++-- 4 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 Lib/test/test_freeze/test_io.py delete mode 100644 Lib/test/test_freeze/test_notfreezable.py diff --git a/Lib/test/test_freeze/__init__.py b/Lib/test/test_freeze/__init__.py index 38eb722baebac8..f3dde340c81ae4 100644 --- a/Lib/test/test_freeze/__init__.py +++ b/Lib/test/test_freeze/__init__.py @@ -25,3 +25,13 @@ def test_add_attribute(self): def test_type_immutable(self): self.assertTrue(isimmutable(type(self.obj))) + + +class BaseNotFreezableTest(unittest.TestCase): + def __init__(self, *args, obj=notfreezable(), **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + self.obj = obj + + def test_not_freezable(self): + with self.assertRaises(TypeError): + freeze(self.obj) diff --git a/Lib/test/test_freeze/test_io.py b/Lib/test/test_freeze/test_io.py new file mode 100644 index 00000000000000..885bd3fbfb6dcb --- /dev/null +++ b/Lib/test/test_freeze/test_io.py @@ -0,0 +1,37 @@ +import io + +from . import BaseNotFreezableTest + + +class BytesIOTest(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=io.BytesIO(), **kwargs) + + def tearDown(self): + self.obj.close() + + +class StringIOTest(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=io.StringIO(), **kwargs) + + def tearDown(self): + self.obj.close() + + +class TextWrapperTest(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + handle = open('test_file.txt', 'w') + super().__init__(*args, obj=handle, **kwargs) + + def tearDown(self): + self.obj.close() + + +class RawWrapperTest(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + handle = open('test_file.txt', 'wb') + super().__init__(*args, obj=handle, **kwargs) + + def tearDown(self): + self.obj.close() diff --git a/Lib/test/test_freeze/test_notfreezable.py b/Lib/test/test_freeze/test_notfreezable.py deleted file mode 100644 index c754d0c389f3a4..00000000000000 --- a/Lib/test/test_freeze/test_notfreezable.py +++ /dev/null @@ -1,13 +0,0 @@ -import unittest - -class Mock(notfreezable): - def __init__(self, name): - self.name = name - - -class TestNotFreezable(unittest.TestCase): - def test_not_freezable(self): - # Test that the NotFreezable class raises an exception when trying to freeze - mock = Mock("test") - with self.assertRaises(TypeError): - freeze(mock) diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 7b06c1bee5a832..6cafc622313926 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -674,9 +674,9 @@ iomodule_exec(PyObject *m) } // Base classes - ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); - ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); - ADD_TYPE(m, state->PyIOBase_Type, &iobase_spec, NULL); + ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, &PyNotFreezable_Type); + ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, &PyNotFreezable_Type); + ADD_TYPE(m, state->PyIOBase_Type, &iobase_spec, &PyNotFreezable_Type); // PyIOBase_Type subclasses ADD_TYPE(m, state->PyTextIOBase_Type, &textiobase_spec, From fe5de3403fb095b602f059cb50133e8cf1fd76cb Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Mon, 14 Apr 2025 12:45:46 +0100 Subject: [PATCH 15/40] Fixing some broken tests Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/__init__.py | 4 ++- Lib/test/test_freeze/test_io.py | 4 +-- Modules/_ctypes/_ctypes.c | 2 +- Objects/typeobject.c | 37 ++++++--------------- Python/immutability.c | 56 +++++++++++++++++++++----------- 5 files changed, 53 insertions(+), 50 deletions(-) diff --git a/Lib/test/test_freeze/__init__.py b/Lib/test/test_freeze/__init__.py index f3dde340c81ae4..59f1fb1caf4153 100644 --- a/Lib/test/test_freeze/__init__.py +++ b/Lib/test/test_freeze/__init__.py @@ -19,7 +19,6 @@ def test_immutable(self): self.assertTrue(isimmutable(self.obj)) def test_add_attribute(self): - freeze(self.obj) with self.assertRaises(NotWriteableError): self.obj.new_attribute = 'value' @@ -32,6 +31,9 @@ def __init__(self, *args, obj=notfreezable(), **kwargs): unittest.TestCase.__init__(self, *args, **kwargs) self.obj = obj + def test_isinstance(self): + self.assertTrue(isinstance(self.obj, notfreezable)) + def test_not_freezable(self): with self.assertRaises(TypeError): freeze(self.obj) diff --git a/Lib/test/test_freeze/test_io.py b/Lib/test/test_freeze/test_io.py index 885bd3fbfb6dcb..f07dbe30ce752c 100644 --- a/Lib/test/test_freeze/test_io.py +++ b/Lib/test/test_freeze/test_io.py @@ -21,7 +21,7 @@ def tearDown(self): class TextWrapperTest(BaseNotFreezableTest): def __init__(self, *args, **kwargs): - handle = open('test_file.txt', 'w') + handle = open(__file__, 'r') super().__init__(*args, obj=handle, **kwargs) def tearDown(self): @@ -30,7 +30,7 @@ def tearDown(self): class RawWrapperTest(BaseNotFreezableTest): def __init__(self, *args, **kwargs): - handle = open('test_file.txt', 'wb') + handle = open(__file__, 'rb') super().__init__(*args, obj=handle, **kwargs) def tearDown(self): diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index d4677d50dc6873..67343a001eb10f 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3016,7 +3016,7 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) cmem->b_index = index; } - if(_Py_IsImmutable(base)) { + if(base && _Py_IsImmutable(base)) { _Py_SetImmutable(cmem); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 90be936ba385f6..542c6673117efe 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5273,38 +5273,21 @@ PyDoc_STRVAR(type_doc, static int type_traverse(PyTypeObject *type, visitproc visit, void *arg) { - // TODO: Immutability - // The following code and comment are not correct with how Pyrona wants to - // use traverse. It is about finding all the objects that are accessible, not just - // those that can particpate in cycles. - // We have adapted the following code to only visit `ht_module` in heap types. - // - // We should consider if other code has optimised for the cycle detector behaviour? - // - // We should check this with the core team. - - - // /* Because of type_is_gc(), the collector only calls this - // for heaptypes. */ - // if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - // char msg[200]; - // sprintf(msg, "type_traverse() called on non-heap type '%.100s'", - // type->tp_name); - // _PyObject_ASSERT_FAILED_MSG((PyObject *)type, msg); - // } + /* Because of type_is_gc(), the collector only calls this + for heaptypes. */ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + char msg[200]; + sprintf(msg, "type_traverse() called on non-heap type '%.100s'", + type->tp_name); + _PyObject_ASSERT_FAILED_MSG((PyObject *)type, msg); + } Py_VISIT(type->tp_dict); + Py_VISIT(type->tp_cache); Py_VISIT(type->tp_mro); Py_VISIT(type->tp_bases); Py_VISIT(type->tp_base); - - /* - TODO Do we need to visit the module? This can potentially freeze the entire - standard library. What do we gain? - if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { - Py_VISIT(((PyHeapTypeObject *)type)->ht_module); - } - */ + Py_VISIT(((PyHeapTypeObject *)type)->ht_module); /* There's no need to visit others because they can't be involved in cycles: diff --git a/Python/immutability.c b/Python/immutability.c index bc40b29fb80fa2..331742ddb6b0e0 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -15,7 +15,7 @@ PyDoc_STRVAR(notfreezable_doc, PyTypeObject PyNotFreezable_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "NotFreezable", + .tp_name = "notfreezable", .tp_doc = notfreezable_doc, .tp_basicsize = sizeof(PyObject), .tp_itemsize = 0, @@ -265,9 +265,6 @@ PyObject* _Py_Freeze(PyObject* obj) } while(PyList_Size(frontier) != 0){ - PyTypeObject* type; - PyObject* type_op; - traverseproc traverse; PyObject* item = pop(frontier); if(item == blocking_on || @@ -277,16 +274,14 @@ PyObject* _Py_Freeze(PyObject* obj) continue; } - type = Py_TYPE(item); - - if(PyType_IsSubtype(type, &PyNotFreezable_Type)){ - PyErr_SetString(PyExc_TypeError, "Cannot freeze a NotFreezable object"); + if(PyObject_IsInstance(item, (PyObject*)&PyNotFreezable_Type)){ + // the object is not freezable, so we can skip it + PyObject* error_msg = PyUnicode_FromFormat("Cannot freeze object of type %s", Py_TYPE(item)->tp_name); + PyErr_SetObject(PyExc_TypeError, error_msg); result = NULL; goto cleanup; } - type_op = NULL; - if(_Py_IsImmutable(item)){ continue; } @@ -294,7 +289,8 @@ PyObject* _Py_Freeze(PyObject* obj) _Py_SetImmutable(item); if(is_c_wrapper(item)) { - // C functions are not mutable, so we can skip them. + // C functions are not mutable + // Types are manually traversed continue; } @@ -305,18 +301,40 @@ PyObject* _Py_Freeze(PyObject* obj) } } - traverse = type->tp_traverse; - if(traverse != NULL){ - if(traverse(item, (visitproc)freeze_visit, frontier)){ - result = NULL; + if(PyType_Check(item)){ + PyTypeObject* type = (PyTypeObject*)item; + if(push(frontier, type->tp_dict)) + { + result = PyErr_NoMemory(); + goto cleanup; + } + if(push(frontier, type->tp_mro)) + { + result = PyErr_NoMemory(); + goto cleanup; + } + if(push(frontier, type->tp_bases)) + { + result = PyErr_NoMemory(); + goto cleanup; + } + if(push(frontier, _PyObject_CAST(type->tp_base))) + { + result = PyErr_NoMemory(); goto cleanup; } } + else + { + traverseproc traverse = Py_TYPE(item)->tp_traverse; + if(traverse != NULL){ + if(traverse(item, (visitproc)freeze_visit, frontier)){ + result = NULL; + goto cleanup; + } + } - type_op = _PyObject_CAST(item->ob_type); - if (!_Py_IsImmutable(type_op)){ - if(push(frontier, type_op)) - { + if(push(frontier, _PyObject_CAST(Py_TYPE(item)))){ result = PyErr_NoMemory(); goto cleanup; } From fef1fdb74d9c3d1eb7346ff6e7f89349babf12f7 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Mon, 14 Apr 2025 15:04:11 +0100 Subject: [PATCH 16/40] Fixing the name of NotWritableError Signed-off-by: Matthew A Johnson --- Doc/conf.py | 2 +- Doc/data/stable_abi.dat | 2 +- Include/pyerrors.h | 2 +- Lib/_compat_pickle.py | 2 +- Lib/test/exception_hierarchy.txt | 2 +- Lib/test/test_descr.py | 4 +- Lib/test/test_freeze/__init__.py | 2 +- Lib/test/test_freeze/test_array.py | 24 ++--- Lib/test/test_freeze/test_collections.py | 30 +++---- Lib/test/test_freeze/test_core.py | 108 +++++++++++------------ Lib/test/test_freeze/test_ctypes.py | 22 ++--- Lib/test/test_freeze/test_decimal.py | 12 +-- Lib/test/test_freeze/test_hashlib.py | 4 +- Lib/test/test_importlib/test_api.py | 2 +- Lib/test/test_io.py | 4 +- Lib/test/test_stable_abi_ctypes.py | 2 +- Misc/stable_abi.toml | 2 +- Objects/abstract.c | 2 +- Objects/exceptions.c | 6 +- PC/python3dll.c | 2 +- Python/ceval.c | 2 +- Python/errors.c | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 4 +- 23 files changed, 122 insertions(+), 122 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index dbd75012988442..8d88d4d29e6b81 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -215,7 +215,7 @@ ('c:data', 'PyExc_UnicodeError'), ('c:data', 'PyExc_UnicodeTranslateError'), ('c:data', 'PyExc_ValueError'), - ('c:data', 'PyExc_NotWriteableError'), + ('c:data', 'PyExc_NotWritableError'), ('c:data', 'PyExc_ZeroDivisionError'), # C API: Standard Python warning classes ('c:data', 'PyExc_BytesWarning'), diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 595b999d750648..947e1f5ddee238 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -238,7 +238,7 @@ var,PyExc_ModuleNotFoundError,3.6,, var,PyExc_NameError,3.2,, var,PyExc_NotADirectoryError,3.7,, var,PyExc_NotImplementedError,3.2,, -var,PyExc_NotWriteableError,3.12,, +var,PyExc_NotWritableError,3.12,, var,PyExc_OSError,3.2,, var,PyExc_OverflowError,3.2,, var,PyExc_PendingDeprecationWarning,3.2,, diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 9dd230d3adcd60..59bc75c6b65491 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -121,7 +121,7 @@ PyAPI_DATA(PyObject *) PyExc_UnicodeDecodeError; PyAPI_DATA(PyObject *) PyExc_UnicodeTranslateError; PyAPI_DATA(PyObject *) PyExc_ValueError; PyAPI_DATA(PyObject *) PyExc_ZeroDivisionError; -PyAPI_DATA(PyObject *) PyExc_NotWriteableError; +PyAPI_DATA(PyObject *) PyExc_NotWritableError; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_DATA(PyObject *) PyExc_BlockingIOError; diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index e034427ecea908..cb39eb7e937e8b 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -137,7 +137,6 @@ "UnicodeWarning", "UserWarning", "ValueError", - "NotWriteableError", "Warning", "ZeroDivisionError", ) @@ -222,6 +221,7 @@ ('http.server', 'CGIHTTPRequestHandler'): ('CGIHTTPServer', 'CGIHTTPRequestHandler'), ('_socket', 'socket'): ('socket', '_socketobject'), + ('builtins', 'NotWritableError'): ('exceptions', 'StandardError'), }) PYTHON3_OSERROR_EXCEPTIONS = ( diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index c987419663409e..68db5620ebc872 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -53,7 +53,7 @@ BaseException │ ├── UnicodeDecodeError │ ├── UnicodeEncodeError │ └── UnicodeTranslateError - ├── NotWriteableError + ├── NotWritableError └── Warning ├── BytesWarning ├── DeprecationWarning diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 3cbfa8428513fe..15ea2b196766f8 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3282,7 +3282,7 @@ class F(D, E): pass def cant(x, C): try: x.__class__ = C - except NotWriteableError: + except NotWritableError: pass except TypeError: pass @@ -3290,7 +3290,7 @@ def cant(x, C): self.fail("shouldn't allow %r.__class__ = %r" % (x, C)) try: delattr(x, "__class__") - except (TypeError, AttributeError, NotWriteableError): + except (TypeError, AttributeError, NotWritableError): pass else: self.fail("shouldn't allow del %r.__class__" % x) diff --git a/Lib/test/test_freeze/__init__.py b/Lib/test/test_freeze/__init__.py index 59f1fb1caf4153..588066c73c7982 100644 --- a/Lib/test/test_freeze/__init__.py +++ b/Lib/test/test_freeze/__init__.py @@ -19,7 +19,7 @@ def test_immutable(self): self.assertTrue(isimmutable(self.obj)) def test_add_attribute(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.new_attribute = 'value' def test_type_immutable(self): diff --git a/Lib/test/test_freeze/test_array.py b/Lib/test/test_freeze/test_array.py index 907d1066f82487..a345029761ace3 100644 --- a/Lib/test/test_freeze/test_array.py +++ b/Lib/test/test_freeze/test_array.py @@ -8,49 +8,49 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[0] = 5 def test_set_slice(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[1:3] = [6, 7] def test_append(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.append(8) def test_extend(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.extend(array('i', [9])) def test_insert(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.insert(0, 10) def test_pop(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.pop() def test_remove(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.remove(1) def test_delete(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): del self.obj[0] def test_reverse(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.reverse() def test_inplace_repeat(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj *= 2 def test_inplace_concat(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj += array('i', [11]) def test_byteswap(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.byteswap() diff --git a/Lib/test/test_freeze/test_collections.py b/Lib/test/test_freeze/test_collections.py index cd9359d7510a91..dadf5885f632f8 100644 --- a/Lib/test/test_freeze/test_collections.py +++ b/Lib/test/test_freeze/test_collections.py @@ -11,61 +11,61 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[0] = None def test_append(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.append(TestDeque.C()) def test_appendleft(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.appendleft(TestDeque.C()) def test_extend(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.extend([TestDeque.C()]) def test_extendleft(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.extendleft([TestDeque.C()]) def test_insert(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.insert(0, TestDeque.C()) def test_pop(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.pop() def test_popleft(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.popleft() def test_remove(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.remove(1) def test_delete(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): del self.obj[0] def test_inplace_repeat(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj *= 2 def test_inplace_concat(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj += [TestDeque.C()] def test_reverse(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.reverse() def test_rotate(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.rotate(1) def test_clear(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.clear() diff --git a/Lib/test/test_freeze/test_core.py b/Lib/test/test_freeze/test_core.py index dd33cf93420f3b..fa3ea8cc6bcb03 100644 --- a/Lib/test/test_freeze/test_core.py +++ b/Lib/test/test_freeze/test_core.py @@ -50,55 +50,55 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[0] = None def test_set_slice(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[1:3] = [None, None] def test_append(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.append(TestList.C()) def test_extend(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.extend([TestList.C()]) def test_insert(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.insert(0, TestList.C()) def test_pop(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.pop() def test_remove(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.remove(1) def test_delete(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): del self.obj[0] def test_reverse(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.reverse() def test_inplace_repeat(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj *= 2 def test_inplace_concat(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj += [TestList.C()] def test_clear(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.clear() def test_sort(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.sort() @@ -111,35 +111,35 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item_exists(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[1] = None def test_set_item_new(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj["three"] = TestDict.C() def test_del_item(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): del self.obj[1] def test_clear(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.clear() def test_pop(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.pop(1) def test_popitem(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.popitem() def test_setdefault(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.setdefault("three", TestDict.C()) def test_update(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.update({1: None}) @@ -149,27 +149,27 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_add(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.add(1) def test_clear(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.clear() def test_discard(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.discard(1) def test_pop(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.pop() def test_remove(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.remove(1) def test_update(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.update([1, 2]) @@ -202,7 +202,7 @@ def test_immutable(self): self.assertTrue(isimmutable(self.obj.g["two"].i)) def test_set_const(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.const = 1 def test_type_immutable(self): @@ -239,7 +239,7 @@ def inc(): self.assertEqual(test(), 1) self.assertEqual(test(), 2) freeze(test) - self.assertRaises(NotWriteableError, test) + self.assertRaises(NotWritableError, test) def test_global(self): def d(): @@ -251,7 +251,7 @@ def d(): freeze(d) self.assertTrue(isimmutable(global0)) self.assertFalse(isimmutable(global_canary)) - self.assertRaises(NotWriteableError, d) + self.assertRaises(NotWritableError, d) def test_hidden_global(self): global global0 @@ -264,7 +264,7 @@ def d(): global0 = 0 self.assertEqual(d(), 1) freeze(d) - self.assertRaises(NotWriteableError, d) + self.assertRaises(NotWritableError, d) def test_builtins(self): def e(): @@ -297,7 +297,7 @@ def d(): self.assertTrue(isimmutable(global1)) self.assertTrue(isimmutable(global1_inc)) self.assertFalse(isimmutable(global_canary)) - self.assertRaises(NotWriteableError, d) + self.assertRaises(NotWritableError, d) def test_globals_copy(self): def f(): @@ -330,7 +330,7 @@ def test_lambda(self): freeze(obj) self.assertTrue(isimmutable(TestMethods.C)) self.assertTrue(isimmutable(pow)) - self.assertRaises(NotWriteableError, obj.b, 1) + self.assertRaises(NotWritableError, obj.b, 1) self.assertEqual(obj.c(2), 4) def test_method(self): @@ -340,9 +340,9 @@ def test_method(self): self.assertTrue(isimmutable(obj)) self.assertTrue(isimmutable(abs)) self.assertTrue(isimmutable(obj.val)) - self.assertRaises(NotWriteableError, obj.b, 1) + self.assertRaises(NotWritableError, obj.b, 1) # Second test as the byte code can be changed by the first call - self.assertRaises(NotWriteableError, obj.b, 1) + self.assertRaises(NotWritableError, obj.b, 1) class TestLocals(unittest.TestCase): @@ -383,7 +383,7 @@ def test_dict_mutation(self): obj = TestDictMutation.C() freeze(obj) self.assertTrue(isimmutable(obj)) - self.assertRaises(NotWriteableError, obj.set, 1) + self.assertRaises(NotWritableError, obj.set, 1) self.assertEqual(obj.get(), 0) def test_dict_mutation2(self): @@ -393,7 +393,7 @@ def test_dict_mutation2(self): freeze(obj) self.assertEqual(obj.get(), 1) self.assertTrue(isimmutable(obj)) - self.assertRaises(NotWriteableError, obj.set, 1) + self.assertRaises(NotWritableError, obj.set, 1) class TestWeakRef(unittest.TestCase): class B: @@ -443,7 +443,7 @@ def f1(): def test_global_dict_mutation(self): f1 = TestGlobalDictMutation.g() self.assertTrue(isimmutable(f1)) - self.assertRaises(NotWriteableError, f1) + self.assertRaises(NotWritableError, f1) class TestSubclass(unittest.TestCase): @@ -499,58 +499,58 @@ def f(): freeze(f) - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__annotations__ = {} # The following should raise an exception, # but doesn't as the __annotations__ gets # replaced with a new dict that is not # immutable. - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__annotations__["foo"] = 2 - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__builtins__ = {} - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__builtins__["foo"] = 2 - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): def g(): pass f.__code__ = g.__code__ - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__defaults__ = (1,2) - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__dict__ = {} - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__dict__["foo"] = {} - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__doc__ = "foo" - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__globals__ = {} - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__globals__["foo"] = 2 - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__kwdefaults__ = {} - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__module__ = "foo" - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__name__ = "foo" - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__qualname__ = "foo" - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): f.__type_params__ = (1,2) class TestFunctionDefaults(unittest.TestCase): diff --git a/Lib/test/test_freeze/test_ctypes.py b/Lib/test/test_freeze/test_ctypes.py index 9fbc8135c7c2f5..fa7683afd89430 100644 --- a/Lib/test/test_freeze/test_ctypes.py +++ b/Lib/test/test_freeze/test_ctypes.py @@ -8,13 +8,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=ctypes.create_string_buffer(b"hello"), **kwargs) def test_raw(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.raw = b"world" self.assertEqual(self.obj.raw, b"hello\x00") def test_value(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.value = b"world" self.assertEqual(self.obj.value, b"hello") @@ -25,7 +25,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=ctypes.create_unicode_buffer("hello"), **kwargs) def test_value(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.value = "world" self.assertEqual(self.obj.value, "hello") @@ -39,7 +39,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=TestStructure.POINT(1, 2), **kwargs) def test_modify_field(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.x = 3 self.assertEqual(self.obj.x, 1) @@ -59,7 +59,7 @@ def test_contents_immutable(self): def test_set_contents(self): b = TestPointer.POINT(3, 4) - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.contents = b self.assertEqual(self.obj.contents.x, self.a.x) @@ -79,19 +79,19 @@ def test_point_immutable(self): self.assertTrue(isimmutable(TestArray.POINT)) def test_modify_item(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[0].x = 1 self.assertEqual(self.obj[0].x, 0) def test_ass_item(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[0] = TestArray.POINT(1, 2) def test_ass_subscript(self): TwoPointsArrayType = TestArray.POINT * 2 a = TwoPointsArrayType() - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj[:2] = a @@ -110,19 +110,19 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=a, **kwargs) def test_assign_part(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.parts.high = 0 self.assertEqual(self.obj.parts.high, 0xFF) - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.parts.low = 0 self.assertEqual(self.obj.parts.low, 0xFF) self.assertEqual(self.obj.value, 0x00FF00FF) def test_assign_value(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.value = 0x00FF00FF self.assertEqual(self.obj.value, 0x00FF00FF) diff --git a/Lib/test/test_freeze/test_decimal.py b/Lib/test/test_freeze/test_decimal.py index bd140ff9f1b164..be7314adc8bdd9 100644 --- a/Lib/test/test_freeze/test_decimal.py +++ b/Lib/test/test_freeze/test_decimal.py @@ -8,25 +8,25 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=decimal.Context(), **kwargs) def test_prec(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.prec = 10 def test_emax(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.Emax = 10 def test_emin(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.Emin = -10 def test_rounding(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.rounding = decimal.ROUND_DOWN def test_capitals(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.capitals = 0 def test_clamp(self): - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): self.obj.clamp = 1 diff --git a/Lib/test/test_freeze/test_hashlib.py b/Lib/test/test_freeze/test_hashlib.py index 9943a3bf0f5a11..c7f656f6e5571f 100644 --- a/Lib/test/test_freeze/test_hashlib.py +++ b/Lib/test/test_freeze/test_hashlib.py @@ -7,12 +7,12 @@ def test_blake2b(self): h = blake2b(digest_size=32) h.update(b'Hello world') freeze(h) - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): h.update(b'!') def test_blake2s(self): h = blake2s(digest_size=32) h.update(b'Hello world') freeze(h) - with self.assertRaises(NotWriteableError): + with self.assertRaises(NotWritableError): h.update(b'!') diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 62b6edf299b4ec..450b910f0907f3 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -135,7 +135,7 @@ def test_sys_modules_loader_is_not_set(self): del module.__spec__.loader except AttributeError: pass - except NotWriteableError: + except NotWritableError: pass sys.modules[name] = module with self.assertRaises(ValueError): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index e032325fbe2578..14e65a439ffe63 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2110,11 +2110,11 @@ def readable(self): self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO()) def test_constructor_with_not_writeable(self): - class NotWriteable(MockRawIO): + class NotWritable(MockRawIO): def writable(self): return False - self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable()) + self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWritable()) def test_read(self): pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 95eb05a1e6e39a..a232d8e143ab47 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -264,7 +264,7 @@ def test_windows_feature_macros(self): "PyExc_NameError", "PyExc_NotADirectoryError", "PyExc_NotImplementedError", - "PyExc_NotWriteableError", + "PyExc_NotWritableError", "PyExc_OSError", "PyExc_OverflowError", "PyExc_PendingDeprecationWarning", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index c55e979663d59a..f64ec7ace4c571 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2406,5 +2406,5 @@ added = '3.12' [const.Py_TPFLAGS_ITEMS_AT_END] added = '3.12' -[data.PyExc_NotWriteableError] +[data.PyExc_NotWritableError] added = '3.12' diff --git a/Objects/abstract.c b/Objects/abstract.c index f6d17d71b87e3d..39ad2ac6a5949a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -39,7 +39,7 @@ immutable_error(void) { PyThreadState *tstate = _PyThreadState_GET(); if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_NotWriteableError, + _PyErr_SetString(tstate, PyExc_NotWritableError, "cannot modify immutable instance"); } return NULL; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 9dbab8669527e4..7f675eaa2802c6 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3431,9 +3431,9 @@ PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; SimpleExtendsException(PyExc_Exception, BufferError, "Buffer error."); /* - * NotWriteableError extends Exception + * NotWritableError extends Exception */ -SimpleExtendsException(PyExc_Exception, NotWriteableError, "Object is not writeable."); +SimpleExtendsException(PyExc_Exception, NotWritableError, "Object is not writeable."); /* Warning category docstrings */ @@ -3620,7 +3620,7 @@ static struct static_exception static_exceptions[] = { ITEM(SystemError), ITEM(TypeError), ITEM(ValueError), - ITEM(NotWriteableError), + ITEM(NotWritableError), ITEM(Warning), // Level 4: ArithmeticError(Exception) subclasses diff --git a/PC/python3dll.c b/PC/python3dll.c index 5479e32b408a23..4ebb6b481198a8 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -817,7 +817,7 @@ EXPORT_DATA(PyExc_ModuleNotFoundError) EXPORT_DATA(PyExc_NameError) EXPORT_DATA(PyExc_NotADirectoryError) EXPORT_DATA(PyExc_NotImplementedError) -EXPORT_DATA(PyExc_NotWriteableError) +EXPORT_DATA(PyExc_NotWritableError) EXPORT_DATA(PyExc_OSError) EXPORT_DATA(PyExc_OverflowError) EXPORT_DATA(PyExc_PendingDeprecationWarning) diff --git a/Python/ceval.c b/Python/ceval.c index ecf574c633c1e8..9a88c800cb7401 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2739,7 +2739,7 @@ format_exc_notwriteable(PyThreadState *tstate, PyCodeObject *co, int oparg) if (_PyErr_Occurred(tstate)) return; name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); - format_exc_check_arg(tstate, PyExc_NotWriteableError, + format_exc_check_arg(tstate, PyExc_NotWritableError, NOT_WRITEABLE_ERROR_MSG, name); } diff --git a/Python/errors.c b/Python/errors.c index 7126df2b8088a9..c98d86d5660095 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1959,7 +1959,7 @@ _PyErr_WriteToImmutable(const char* filename, int lineno, PyObject* obj) string = PyUnicode_FromFormat("object of type %s is immutable at %s:%d", obj->ob_type->tp_name, filename, lineno); if (string != NULL) { - _PyErr_SetObject(tstate, PyExc_NotWriteableError, string); + _PyErr_SetObject(tstate, PyExc_NotWritableError, string); Py_DECREF(string); } } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a898d3c5a9e556..83afe37cf2440c 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -306,8 +306,8 @@ Modules/timemodule.c init_timezone YEAR - Objects/bytearrayobject.c - _PyByteArray_empty_string - Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - -Objects/exceptions.c - _PyExc_NotWriteableError - -Objects/exceptions.c - PyExc_NotWriteableError - +Objects/exceptions.c - _PyExc_NotWritableError - +Objects/exceptions.c - PyExc_NotWritableError - Objects/genobject.c - ASYNC_GEN_IGNORED_EXIT_MSG - Objects/genobject.c - NON_INIT_CORO_MSG - Objects/longobject.c - _PyLong_DigitValue - From ab10e214c9e1c2aec97d329f421f02c8317ff485 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Mon, 14 Apr 2025 15:44:58 +0100 Subject: [PATCH 17/40] _multiprocessing _sqlite3 Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/__init__.py | 3 -- Lib/test/test_freeze/test_multiprocessing.py | 10 ++++++ Lib/test/test_freeze/test_sqllite3.py | 37 ++++++++++++++++++++ Modules/_multiprocessing/multiprocessing.c | 2 +- Modules/_sqlite/blob.c | 2 +- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/cursor.c | 2 +- 7 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 Lib/test/test_freeze/test_multiprocessing.py create mode 100644 Lib/test/test_freeze/test_sqllite3.py diff --git a/Lib/test/test_freeze/__init__.py b/Lib/test/test_freeze/__init__.py index 588066c73c7982..a6da8bbde5e531 100644 --- a/Lib/test/test_freeze/__init__.py +++ b/Lib/test/test_freeze/__init__.py @@ -31,9 +31,6 @@ def __init__(self, *args, obj=notfreezable(), **kwargs): unittest.TestCase.__init__(self, *args, **kwargs) self.obj = obj - def test_isinstance(self): - self.assertTrue(isinstance(self.obj, notfreezable)) - def test_not_freezable(self): with self.assertRaises(TypeError): freeze(self.obj) diff --git a/Lib/test/test_freeze/test_multiprocessing.py b/Lib/test/test_freeze/test_multiprocessing.py new file mode 100644 index 00000000000000..60a7f2b82cf6fc --- /dev/null +++ b/Lib/test/test_freeze/test_multiprocessing.py @@ -0,0 +1,10 @@ +from _multiprocessing import SemLock + +from . import BaseNotFreezableTest + +SEMAPHORE = 1 +SEM_VALUE_MAX = SemLock.SEM_VALUE_MAX + +class TestSemLock(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=SemLock(SEMAPHORE, 0, SEM_VALUE_MAX, "mock", True), **kwargs) diff --git a/Lib/test/test_freeze/test_sqllite3.py b/Lib/test/test_freeze/test_sqllite3.py new file mode 100644 index 00000000000000..540e74701714a6 --- /dev/null +++ b/Lib/test/test_freeze/test_sqllite3.py @@ -0,0 +1,37 @@ +from test.support import import_helper + +import_helper.import_module('_sqlite3') + +import sqlite3 + +from . import BaseNotFreezableTest + + +class TestConnection(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=sqlite3.connect(':memory:'), **kwargs) + + def tearDown(self): + self.obj.close() + + +class TestCursor(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + self.con = sqlite3.connect(':memory:') + super().__init__(*args, obj=self.con.cursor(), **kwargs) + + def tearDown(self): + self.con.close() + + +class TestBlob(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + self.con = sqlite3.connect(':memory:') + self.con.execute("CREATE TABLE test(blob_col blob)") + self.con.execute("INSERT INTO test(blob_col) VALUES(zeroblob(13))") + + blob = self.con.blobopen("test", "blob_col", 1) + super().__init__(*args, obj=blob, **kwargs) + + def tearDown(self): + self.con.close() diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 8f9daa5c3de0cc..42ea827761c128 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -197,7 +197,7 @@ multiprocessing_exec(PyObject *module) #ifdef HAVE_MP_SEMAPHORE PyTypeObject *semlock_type = (PyTypeObject *)PyType_FromModuleAndSpec( - module, &_PyMp_SemLockType_spec, NULL); + module, &_PyMp_SemLockType_spec, (PyObject *)&PyNotFreezable_Type); if (semlock_type == NULL) { return -1; diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 76d261baf00f38..aa53fe58a0fc52 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -607,7 +607,7 @@ static PyType_Spec blob_spec = { int pysqlite_blob_setup_types(PyObject *mod) { - PyObject *type = PyType_FromModuleAndSpec(mod, &blob_spec, NULL); + PyObject *type = PyType_FromModuleAndSpec(mod, &blob_spec, (PyObject *)(&PyNotFreezable_Type)); if (type == NULL) { return -1; } diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 12e5c135aafa50..812bb210b4d524 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -2649,7 +2649,7 @@ static PyType_Spec connection_spec = { int pysqlite_connection_setup_types(PyObject *module) { - PyObject *type = PyType_FromModuleAndSpec(module, &connection_spec, NULL); + PyObject *type = PyType_FromModuleAndSpec(module, &connection_spec, (PyObject *)(&PyNotFreezable_Type)); if (type == NULL) { return -1; } diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index caeedbddb8d88b..26ec2c41945754 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -1356,7 +1356,7 @@ static PyType_Spec cursor_spec = { int pysqlite_cursor_setup_types(PyObject *module) { - PyObject *type = PyType_FromModuleAndSpec(module, &cursor_spec, NULL); + PyObject *type = PyType_FromModuleAndSpec(module, &cursor_spec, (PyObject *)(&PyNotFreezable_Type)); if (type == NULL) { return -1; } From a7dbd1a5cb2dc729c0cd7ea91f5d00b833d83ed2 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Mon, 14 Apr 2025 16:28:17 +0100 Subject: [PATCH 18/40] cjkcodecs Signed-off-by: Matthew A Johnson --- Modules/cjkcodecs/multibytecodec.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index b501e4fb923232..0b6e1e8180d1aa 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -589,6 +589,12 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *errorcb, *r, *ucvt; Py_ssize_t datalen; + if(self->codec->encinit != NULL){ + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + } + if (PyUnicode_Check(input)) ucvt = NULL; else { @@ -660,6 +666,12 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, const char *data; Py_ssize_t datalen; + if(self->codec->decinit != NULL){ + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + } + data = input->buf; datalen = input->len; From dd611e4d887882f1a35841b229bb48c7b35bddd4 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Wed, 16 Apr 2025 14:21:59 +0100 Subject: [PATCH 19/40] _abc Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_abc.py | 82 ++++++++++++++++++++++++++++++++ Modules/_abc.c | 10 ++++ 2 files changed, 92 insertions(+) create mode 100644 Lib/test/test_freeze/test_abc.py diff --git a/Lib/test/test_freeze/test_abc.py b/Lib/test/test_freeze/test_abc.py new file mode 100644 index 00000000000000..71012cff86ce1f --- /dev/null +++ b/Lib/test/test_freeze/test_abc.py @@ -0,0 +1,82 @@ +from abc import ABC, abstractmethod + + +from . import BaseObjectTest + + +class TestABC(BaseObjectTest): + def __init__(self, *args, **kwargs): + class A(ABC): + @abstractmethod + def foo(self): + pass + + class B(A): + def foo(self): + print("foo") + + self.A = A + super().__init__(*args, obj=B(), **kwargs) + + def test_abstract_immutable(self): + self.assertTrue(isimmutable(self.A)) + + def test_register(self): + class C(ABC): + @abstractmethod + def bar(self): + pass + + with self.assertRaises(NotWritableError): + self.A.register(C) + + def test_invalid_cache(self): + class D(ABC): + @abstractmethod + def baz(self): + pass + + class E(ABC): + @abstractmethod + def qux(self): + pass + + self.assertFalse(issubclass(E, D)) + + D.register(E) + + class F(D): + def baz(self): + pass + + x = F() + freeze(x) + self.assertTrue(isimmutable(D)) + with self.assertRaises(NotWritableError): + # the caches are invalidated but cannot be updated + # because the class is frozen + isinstance(x, D) + + def test_valid_cache(self): + class DD(ABC): + @abstractmethod + def baz(self): + pass + + class EE(ABC): + @abstractmethod + def qux(self): + pass + + DD.register(EE) + + self.assertTrue(issubclass(EE, DD)) + + class FF(DD): + def baz(self): + pass + + x = FF() + freeze(x) + self.assertTrue(isimmutable(DD)) + self.assertTrue(isinstance(x, DD)) diff --git a/Modules/_abc.c b/Modules/_abc.c index d3e405dadb664a..64e6a9a9878255 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -542,6 +542,11 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) if (result < 0) { return NULL; } + + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + _abc_data *impl = _get_impl(module, self); if (impl == NULL) { return NULL; @@ -694,6 +699,11 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, state = get_abc_state(module); /* 2. Check negative cache; may have to invalidate. */ if (impl->_abc_negative_cache_version < state->abc_invalidation_counter) { + if(!Py_CHECKWRITE(self)){ + PyErr_WriteToImmutable(self); + goto end; + } + /* Invalidate the negative cache. */ if (impl->_abc_negative_cache != NULL && PySet_Clear(impl->_abc_negative_cache) < 0) From 47fab24440d4a08335709ff2f404d97cf3a5d15d Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Wed, 16 Apr 2025 14:54:09 +0100 Subject: [PATCH 20/40] _asyncio Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_asyncio.py | 22 ++++++++++++++++++++++ Modules/_asynciomodule.c | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 Lib/test/test_freeze/test_asyncio.py diff --git a/Lib/test/test_freeze/test_asyncio.py b/Lib/test/test_freeze/test_asyncio.py new file mode 100644 index 00000000000000..3d7d70b273c3da --- /dev/null +++ b/Lib/test/test_freeze/test_asyncio.py @@ -0,0 +1,22 @@ +import asyncio +import unittest + + +class TestFuture(unittest.TestCase): + def test_future(self): + async def set_after(fut, delay, value): + await asyncio.sleep(delay) + fut.set_result(value) + + async def main(): + loop = asyncio.get_running_loop() + fut = loop.create_future() + loop.create_task( + set_after(fut, .1, '... world')) + + with self.assertRaises(TypeError): + freeze(fut) + + await fut + + asyncio.run(main()) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a465090bfaaa38..dffc916cac383e 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3756,8 +3756,8 @@ module_exec(PyObject *mod) } while (0) CREATE_TYPE(mod, state->TaskStepMethWrapper_Type, &TaskStepMethWrapper_spec, NULL); - CREATE_TYPE(mod, state->FutureIterType, &FutureIter_spec, NULL); - CREATE_TYPE(mod, state->FutureType, &Future_spec, NULL); + CREATE_TYPE(mod, state->FutureIterType, &FutureIter_spec, &PyNotFreezable_Type); + CREATE_TYPE(mod, state->FutureType, &Future_spec, &PyNotFreezable_Type); CREATE_TYPE(mod, state->TaskType, &Task_spec, state->FutureType); #undef CREATE_TYPE From 99f259c676582717e41bbf7856f4b6056a8b6326 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Wed, 16 Apr 2025 16:36:12 +0100 Subject: [PATCH 21/40] Fixing import error Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/mock.py | 1 + Lib/test/test_freeze/test_core.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 Lib/test/test_freeze/mock.py diff --git a/Lib/test/test_freeze/mock.py b/Lib/test/test_freeze/mock.py new file mode 100644 index 00000000000000..1337a530cbc1bd --- /dev/null +++ b/Lib/test/test_freeze/mock.py @@ -0,0 +1 @@ +a = 1 diff --git a/Lib/test/test_freeze/test_core.py b/Lib/test/test_freeze/test_core.py index fa3ea8cc6bcb03..61192aab2f0d05 100644 --- a/Lib/test/test_freeze/test_core.py +++ b/Lib/test/test_freeze/test_core.py @@ -482,15 +482,16 @@ def c(self, val): class TestImport(unittest.TestCase): def test_import(self): def f(): - import sys - pass + # immutable objects are not allowed to import + # modules. This will result in an ImportError. + from . import mock + return mock.a freeze(f) - # The following should not fail, but we - # have removed __import__ from globals - # during freeze - f() + with self.assertRaises(ImportError): + f() + class TestFunctionAttributes(unittest.TestCase): def test_function_attributes(self): From 4f7816ad1665df7929dc358937ee407dabdeca5d Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 17 Apr 2025 09:15:18 +0100 Subject: [PATCH 22/40] bz2 Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_bz2.py | 13 +++++++++++++ Modules/_bz2module.c | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 Lib/test/test_freeze/test_bz2.py diff --git a/Lib/test/test_freeze/test_bz2.py b/Lib/test/test_freeze/test_bz2.py new file mode 100644 index 00000000000000..9d2154f9cc050b --- /dev/null +++ b/Lib/test/test_freeze/test_bz2.py @@ -0,0 +1,13 @@ +from bz2 import BZ2Compressor, BZ2Decompressor + +from . import BaseNotFreezableTest + + +class TestBZ2Compressor(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=BZ2Compressor(), **kwargs) + + +class TestBZ2Decompressor(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=BZ2Decompressor(), **kwargs) diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 97bd44b4ac9694..6d40f4cff23655 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -753,7 +753,7 @@ _bz2_exec(PyObject *module) { _bz2_state *state = get_module_state(module); state->bz2_compressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, - &bz2_compressor_type_spec, NULL); + &bz2_compressor_type_spec, (PyObject *)&PyNotFreezable_Type); if (state->bz2_compressor_type == NULL) { return -1; } @@ -762,7 +762,7 @@ _bz2_exec(PyObject *module) } state->bz2_decompressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, - &bz2_decompressor_type_spec, NULL); + &bz2_decompressor_type_spec, (PyObject *)&PyNotFreezable_Type); if (state->bz2_decompressor_type == NULL) { return -1; } From 09fb9a7468acadf7a4f2585fca556154a930a0be Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 17 Apr 2025 09:40:40 +0100 Subject: [PATCH 23/40] _collections _csv Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_collections.py | 50 +++++++++++++++++++++++- Lib/test/test_freeze/test_csv.py | 15 +++++++ Modules/_collectionsmodule.c | 4 +- Modules/_csv.c | 4 +- 4 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 Lib/test/test_freeze/test_csv.py diff --git a/Lib/test/test_freeze/test_collections.py b/Lib/test/test_freeze/test_collections.py index dadf5885f632f8..612f78965a4292 100644 --- a/Lib/test/test_freeze/test_collections.py +++ b/Lib/test/test_freeze/test_collections.py @@ -1,7 +1,8 @@ -from collections import deque +from collections import defaultdict, deque from . import BaseObjectTest + class TestDeque(BaseObjectTest): class C: pass @@ -69,3 +70,50 @@ def test_rotate(self): def test_clear(self): with self.assertRaises(NotWritableError): self.obj.clear() + + def test_iter(self): + it = iter(self.obj) + with self.assertRaises(TypeError): + freeze(it) + + def test_reviter(self): + it = reversed(self.obj) + with self.assertRaises(TypeError): + freeze(it) + + +class TestDefaultDict(BaseObjectTest): + def __init__(self, *args, **kwargs): + s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] + obj = defaultdict(list) + for k, v in s: + obj[k].append(v) + BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) + + def test_set_item_exists(self): + with self.assertRaises(NotWritableError): + self.obj[1] = None + + def test_set_item_new(self): + with self.assertRaises(NotWritableError): + self.obj["three"] = 5 + + def test_del_item(self): + with self.assertRaises(NotWritableError): + del self.obj[1] + + def test_clear(self): + with self.assertRaises(NotWritableError): + self.obj.clear() + + def test_pop(self): + with self.assertRaises(NotWritableError): + self.obj.pop(1) + + def test_popitem(self): + with self.assertRaises(NotWritableError): + self.obj.popitem() + + def test_update(self): + with self.assertRaises(NotWritableError): + self.obj.update({1: None}) diff --git a/Lib/test/test_freeze/test_csv.py b/Lib/test/test_freeze/test_csv.py new file mode 100644 index 00000000000000..14271761cb8ea1 --- /dev/null +++ b/Lib/test/test_freeze/test_csv.py @@ -0,0 +1,15 @@ +import csv +from io import BytesIO + +from . import BaseNotFreezableTest + + +class TestCSVReader(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=csv.reader([]), **kwargs) + + +class TestCSVWriter(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + self.buffer = BytesIO() + super().__init__(*args, obj=csv.writer(self.buffer), **kwargs) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c27338818f5968..0bec947de1dae3 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2613,8 +2613,8 @@ collections_exec(PyObject *module) { collections_state *state = get_module_state(module); ADD_TYPE(module, &deque_spec, state->deque_type, NULL); ADD_TYPE(module, &defdict_spec, state->defdict_type, &PyDict_Type); - ADD_TYPE(module, &dequeiter_spec, state->dequeiter_type, NULL); - ADD_TYPE(module, &dequereviter_spec, state->dequereviter_type, NULL); + ADD_TYPE(module, &dequeiter_spec, state->dequeiter_type, &PyNotFreezable_Type); + ADD_TYPE(module, &dequereviter_spec, state->dequereviter_type, &PyNotFreezable_Type); ADD_TYPE(module, &tuplegetter_spec, state->tuplegetter_type, NULL); if (PyModule_AddType(module, &PyODict_Type) < 0) { diff --git a/Modules/_csv.c b/Modules/_csv.c index 9ab2ad266c2739..ed426507614f00 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1740,13 +1740,13 @@ csv_exec(PyObject *module) { return -1; } - temp = PyType_FromModuleAndSpec(module, &Reader_Type_spec, NULL); + temp = PyType_FromModuleAndSpec(module, &Reader_Type_spec, (PyObject *)&PyNotFreezable_Type); module_state->reader_type = (PyTypeObject *)temp; if (PyModule_AddObjectRef(module, "Reader", temp) < 0) { return -1; } - temp = PyType_FromModuleAndSpec(module, &Writer_Type_spec, NULL); + temp = PyType_FromModuleAndSpec(module, &Writer_Type_spec, (PyObject *)&PyNotFreezable_Type); module_state->writer_type = (PyTypeObject *)temp; if (PyModule_AddObjectRef(module, "Writer", temp) < 0) { return -1; From 11119d0e8b21988de85693f7bbb36eb6b6460f0c Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 17 Apr 2025 11:52:53 +0100 Subject: [PATCH 24/40] Moving Py_Freeze to public API Signed-off-by: Matthew A Johnson --- Include/cpython/immutability.h | 6 -- Include/immutability.h | 8 +- Makefile.pre.in | 1 - Modules/_ctypes/_ctypes.c | 6 +- Objects/dictobject.c | 17 ++-- Objects/memoryobject.c | 6 +- Python/immutability.c | 118 ++++++++++++++++++++++++---- Tools/c-analyzer/cpython/_parser.py | 1 - 8 files changed, 124 insertions(+), 39 deletions(-) delete mode 100644 Include/cpython/immutability.h diff --git a/Include/cpython/immutability.h b/Include/cpython/immutability.h deleted file mode 100644 index 54ab13a80a4792..00000000000000 --- a/Include/cpython/immutability.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef Py_CPYTHON_IMMUTABILITY_H -# error "this header file must not be included directly" -#endif - -PyAPI_FUNC(PyObject *) _Py_Freeze(PyObject*); -#define Py_Freeze(op) _Py_Freeze(_PyObject_CAST(op)) diff --git a/Include/immutability.h b/Include/immutability.h index 4cc60971e54a38..55fbb6b83fa003 100644 --- a/Include/immutability.h +++ b/Include/immutability.h @@ -7,12 +7,8 @@ extern "C" { PyAPI_DATA(PyTypeObject) PyNotFreezable_Type; -#ifndef Py_LIMITED_API -# define Py_CPYTHON_IMMUTABILITY_H -# include "cpython/immutability.h" -# undef Py_CPYTHON_IMMUTABILITY_H -#endif - +PyAPI_FUNC(PyObject *) _Py_Freeze(PyObject*); +#define Py_Freeze(op) _Py_Freeze(_PyObject_CAST(op)) #ifdef __cplusplus } diff --git a/Makefile.pre.in b/Makefile.pre.in index a9659e7aa3cc52..70ac4433ea8e4f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1688,7 +1688,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/funcobject.h \ $(srcdir)/Include/cpython/genobject.h \ $(srcdir)/Include/cpython/import.h \ - $(srcdir)/Include/cpython/immutability.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/listobject.h \ diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 67343a001eb10f..0f73404e7c570a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -111,7 +111,6 @@ bytes(cdata) #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() -#include "pycore_object.h" // _Py_SetImmutable() #include "structmember.h" // PyMemberDef #include @@ -3017,7 +3016,10 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) } if(base && _Py_IsImmutable(base)) { - _Py_SetImmutable(cmem); + if(Py_Freeze(cmem) == NULL){ + Py_DECREF(cmem); + return NULL; + } } return (PyObject *)cmem; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 57253bb542ed7f..ae95847695ba4a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5673,11 +5673,6 @@ _PyObject_ClearManagedDict(PyObject *obj) PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { - // TODO: Pyrona: This method does not change the representation in the frozen case. - // However, repeated calls to __dict__ on a frozen object will all create a new dict. - // Alternative designs - // * Error on this case - // * Introduce locking, and return the same dict. PyObject *dict; PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); @@ -5690,7 +5685,9 @@ PyObject_GenericGetDict(PyObject *obj, void *context) interp, CACHED_KEYS(tp), values); if (dict != NULL) { if (_Py_IsImmutable(obj)) { - _Py_SetImmutable(dict); + if(Py_Freeze(dict) == NULL){ + return NULL; + } } else { dorv_ptr->dict = dict; @@ -5703,7 +5700,9 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dictkeys_incref(CACHED_KEYS(tp)); dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); if (_Py_IsImmutable(obj)) { - _Py_SetImmutable(dict); + if(Py_Freeze(dict) == NULL){ + return NULL; + } } else { dorv_ptr->dict = dict; @@ -5730,7 +5729,9 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = PyDict_New(); } if (_Py_IsImmutable(obj)) { - _Py_SetImmutable(dict); + if(Py_Freeze(dict) == NULL){ + return NULL; + } } else { *dictptr = dict; diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index d6a605d4bc2d15..dacf5c72628743 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -100,7 +100,11 @@ _PyManagedBuffer_FromObject(PyObject *base, int flags) } if(_Py_IsImmutable(base)){ - _Py_SetImmutable(mbuf); + if(Py_Freeze(mbuf) == NULL){ + PyBuffer_Release(&mbuf->master); + Py_DECREF(mbuf); + return NULL; + } } return (PyObject *)mbuf; diff --git a/Python/immutability.c b/Python/immutability.c index 331742ddb6b0e0..ea16ad70e75c86 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -104,6 +104,12 @@ static PyObject* shadow_function_globals(PyObject* op) goto nomemory; } + if(PyDict_SetItemString(shadow_globals, "__builtins__", shadow_builtins)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return NULL; + } + _PyObject_ASSERT(f_ptr, PyCode_Check(f_ptr)); f_code = (PyCodeObject*)f_ptr; @@ -223,12 +229,106 @@ static int freeze_visit(PyObject* obj, void* frontier) return 0; } +static PyObject* get_frozen_importlib(PyInterpreterState* interp) +{ + PyObject* frozen_importlib = NULL; + PyObject* interp_dict = PyInterpreterState_GetDict(interp); + if(interp_dict == NULL){ + return NULL; + } + + frozen_importlib = PyDict_GetItemString(interp_dict, "_frozen_importlib"); + if(frozen_importlib != NULL){ + return frozen_importlib; + } + + frozen_importlib = PyImport_ImportModule("_frozen_importlib"); + if(frozen_importlib == NULL){ + return NULL; + } + + if(PyDict_SetItemString(interp_dict, "_frozen_importlib", frozen_importlib)){ + Py_DECREF(frozen_importlib); + return NULL; + } + + Py_DECREF(frozen_importlib); + return frozen_importlib; +} + +static PyObject* get_blocking_on(PyInterpreterState* interp) +{ + PyObject* frozen_importlib = NULL; + PyObject* blocking_on = NULL; + PyObject* interp_dict = PyInterpreterState_GetDict(interp); + if(interp_dict == NULL){ + return NULL; + } + + blocking_on = PyDict_GetItemString(interp_dict, "_blocking_on"); + if(blocking_on != NULL){ + return blocking_on; + } + + frozen_importlib = get_frozen_importlib(interp); + if(frozen_importlib == NULL){ + return NULL; + } + + blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); + if(blocking_on == NULL){ + return NULL; + } + + if(PyDict_SetItemString(interp_dict, "_blocking_on", blocking_on)){ + Py_DECREF(blocking_on); + return NULL; + } + + Py_DECREF(blocking_on); + return blocking_on; +} + +static PyObject* get_module_locks(PyInterpreterState* interp) +{ + PyObject* frozen_importlib = NULL; + PyObject* module_locks = NULL; + PyObject* interp_dict = PyInterpreterState_GetDict(interp); + if(interp_dict == NULL){ + return NULL; + } + + module_locks = PyDict_GetItemString(interp_dict, "_module_locks"); + if(module_locks != NULL){ + return module_locks; + } + + frozen_importlib = get_frozen_importlib(interp); + if(frozen_importlib == NULL){ + return NULL; + } + + module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); + if(module_locks == NULL){ + return NULL; + } + + if(PyDict_SetItemString(interp_dict, "_module_locks", module_locks)){ + Py_DECREF(module_locks); + return NULL; + } + + Py_DECREF(module_locks); + return module_locks; +} + + PyObject* _Py_Freeze(PyObject* obj) { PyObject* frontier = NULL; - PyObject* frozen_importlib = NULL; PyObject* blocking_on = NULL; PyObject* module_locks = NULL; + PyInterpreterState* interp = PyInterpreterState_Get(); PyObject* result = Py_None; if(_Py_IsImmutable(obj)){ @@ -246,19 +346,13 @@ PyObject* _Py_Freeze(PyObject* obj) goto cleanup; } - frozen_importlib = PyImport_ImportModule("_frozen_importlib"); - if(frozen_importlib == NULL){ - result = NULL; - goto cleanup; - } - - blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); + blocking_on = get_blocking_on(interp); if(blocking_on == NULL){ result = NULL; goto cleanup; } - module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); + module_locks = get_module_locks(interp); if(module_locks == NULL){ result = NULL; goto cleanup; @@ -295,8 +389,7 @@ PyObject* _Py_Freeze(PyObject* obj) } if(PyFunction_Check(item)){ - result = shadow_function_globals(item); - if(!Py_IsNone(result)){ + if(shadow_function_globals(item) == NULL){ goto cleanup; } } @@ -342,9 +435,6 @@ PyObject* _Py_Freeze(PyObject* obj) } cleanup: - Py_XDECREF(blocking_on); - Py_XDECREF(module_locks); - Py_XDECREF(frozen_importlib); Py_XDECREF(frontier); return result; diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index d65a0fa0e8edec..1587a19716fe0b 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -227,7 +227,6 @@ def clean_lines(text): Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 -Include/cpython/immutability.h Py_CPYTHON_IMMUTABILITY_H 1 Include/cpython/interpreteridobject.h Py_CPYTHON_INTERPRETERIDOBJECT_H 1 Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 From baff5f803d3315b09ea7b8e0e531a26ab503ab15 Mon Sep 17 00:00:00 2001 From: Fridtjof Stoldt Date: Thu, 17 Apr 2025 13:57:45 +0200 Subject: [PATCH 25/40] xxlimited, xxsubtype, zlib Signed-off-by: Matthew A Johnson --- Include/object.h | 4 +++ Lib/test/test_freeze/test_xxlimited.py | 41 ++++++++++++++++++++++++++ Lib/test/test_freeze/test_xxsubtype.py | 24 +++++++++++++++ Lib/test/test_freeze/test_zlib.py | 15 ++++++++++ Modules/xxlimited.c | 21 ++++++++++++- Modules/xxlimited_35.c | 12 ++++++++ Modules/xxmodule.c | 4 +++ Modules/xxsubtype.c | 9 ++++++ Modules/zlibmodule.c | 14 +++++++-- 9 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 Lib/test/test_freeze/test_xxlimited.py create mode 100644 Lib/test/test_freeze/test_xxsubtype.py create mode 100644 Lib/test/test_freeze/test_zlib.py diff --git a/Include/object.h b/Include/object.h index ac03b0c8d57078..b5b81bd80dd62b 100644 --- a/Include/object.h +++ b/Include/object.h @@ -281,7 +281,11 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) // Check whether an object is writeable. // This check will always succeed during runtime finalization. +#ifndef Py_LIMITED_API #define Py_CHECKWRITE(op) ((op) && (!_Py_IsImmutable(op) || _Py_IsFinalizing())) +#else +#define Py_CHECKWRITE(op) ((op) && !_Py_IsImmutable(op)) +#endif #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { diff --git a/Lib/test/test_freeze/test_xxlimited.py b/Lib/test/test_freeze/test_xxlimited.py new file mode 100644 index 00000000000000..638f0fce68f318 --- /dev/null +++ b/Lib/test/test_freeze/test_xxlimited.py @@ -0,0 +1,41 @@ +import unittest +from test.support import import_helper +import types + +xxlimited = import_helper.import_module('xxlimited') +xxlimited_35 = import_helper.import_module('xxlimited_35') + +class CommonTests: + module: types.ModuleType + + def test_xxo_set_attribute(self): + xxo = self.module.Xxo() + + freeze(xxo) + + with self.assertRaises(NotWritableError): + xxo.foo = 1234 + + def test_xxo_del_attribute(self): + xxo = self.module.Xxo() + + freeze(xxo) + + with self.assertRaises(NotWritableError): + del xxo.foo + + +class TestXXLimited(CommonTests, unittest.TestCase): + module = xxlimited + + def test_buffer(self): + xxo = self.module.Xxo() + + freeze(xxo) + + # Creating a buffer into immutable memory should be fine. + b1 = memoryview(xxo) + del b1 + +class TestXXLimited35(CommonTests, unittest.TestCase): + module = xxlimited_35 diff --git a/Lib/test/test_freeze/test_xxsubtype.py b/Lib/test/test_freeze/test_xxsubtype.py new file mode 100644 index 00000000000000..c32d5a5bf01d9a --- /dev/null +++ b/Lib/test/test_freeze/test_xxsubtype.py @@ -0,0 +1,24 @@ +import unittest +from test.support import import_helper + +xxsubtype = import_helper.import_module('xxsubtype') + +class xxsubtypelib(unittest.TestCase): + + def test_spamdict_setstate(self): + import xxsubtype as spam + a = spam.spamlist() + + freeze(a) + + with self.assertRaises(NotWritableError): + a.setstate(17) + + def test_spamdict_setstate(self): + import xxsubtype as spam + a = spam.spamdict() + + freeze(a) + + with self.assertRaises(NotWritableError): + a.setstate(17) diff --git a/Lib/test/test_freeze/test_zlib.py b/Lib/test/test_freeze/test_zlib.py new file mode 100644 index 00000000000000..e2035ac7497e8f --- /dev/null +++ b/Lib/test/test_freeze/test_zlib.py @@ -0,0 +1,15 @@ +import zlib + +from . import BaseNotFreezableTest + +class ZlibCompressTest(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=zlib.compressobj(), **kwargs) + +class ZlibDecompressTest(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=zlib.decompressobj(), **kwargs) + +class ZlibDecompressorTest(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=zlib._ZlibDecompressor(), **kwargs) diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 3935c00fc26530..3a098e3ea19726 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -65,6 +65,8 @@ #define Py_LIMITED_API 0x030b0000 #include "Python.h" +#include "object.h" +#include "pylifecycle.h" #include #define BUFSIZE 10 @@ -167,6 +169,11 @@ Xxo_getattro(XxoObject *self, PyObject *name) static int Xxo_setattro(XxoObject *self, PyObject *name, PyObject *v) { + if (!Py_CHECKWRITE(self)) { + PyErr_SetObject(PyExc_NotWritableError, (PyObject *)self); + return -1; + } + if (self->x_attr == NULL) { // prepare the attribute dict self->x_attr = PyDict_New(); @@ -235,7 +242,12 @@ Xxo_getbuffer(XxoObject *self, Py_buffer *view, int flags) (void *)self->x_buffer, BUFSIZE, 0, flags); if (res == 0) { - self->x_exports++; + // Not incrementing the counter is safe, since it's only used to prevent + // memory reallocation, which should never happen for an immutable object + // anyways. + if (Py_CHECKWRITE(self)) { + self->x_exports++; + } } return res; } @@ -243,6 +255,13 @@ Xxo_getbuffer(XxoObject *self, Py_buffer *view, int flags) static void Xxo_releasebuffer(XxoObject *self, Py_buffer *view) { + // Not decrementing the counter is safe, since it's only used to prevent + // memory reallocation, which should never happen for an immutable object + // anyways. + if (!Py_CHECKWRITE(self)) { + return; + } + self->x_exports--; } diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 1ff3ef1cb6f296..1baee7c977d05c 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -46,6 +46,12 @@ Xxo_traverse(XxoObject *self, visitproc visit, void *arg) static int Xxo_clear(XxoObject *self) { + // TODO: Pyrona: + // if (!Py_CHECKWRITE(self)) { + // PyErr_WriteToImmutable(self); + // return -1; + // } + Py_CLEAR(self->x_attr); return 0; } @@ -93,6 +99,12 @@ Xxo_getattro(XxoObject *self, PyObject *name) static int Xxo_setattr(XxoObject *self, const char *name, PyObject *v) { + // TODO: Pyrona: + // if (!Py_CHECKWRITE(self)) { + // PyErr_WriteToImmutable(self); + // return -1; + // } + if (self->x_attr == NULL) { self->x_attr = PyDict_New(); if (self->x_attr == NULL) diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index 1e4e0ea3743ce3..d02ac3c68094ef 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -79,6 +79,10 @@ Xxo_getattro(XxoObject *self, PyObject *name) static int Xxo_setattr(XxoObject *self, const char *name, PyObject *v) { + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (self->x_attr == NULL) { self->x_attr = PyDict_New(); if (self->x_attr == NULL) diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 9e4a3d66ef41bd..aca60c7be93020 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -1,5 +1,6 @@ #include "Python.h" #include "structmember.h" // PyMemberDef +#include "pyerrors.h" // PyErr_WriteToImmutable PyDoc_STRVAR(xxsubtype__doc__, "xxsubtype is an example module showing how to subtype builtin types from C.\n" @@ -36,6 +37,10 @@ spamlist_setstate(spamlistobject *self, PyObject *args) { int state; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (!PyArg_ParseTuple(args, "i:setstate", &state)) return NULL; self->state = state; @@ -157,6 +162,10 @@ spamdict_setstate(spamdictobject *self, PyObject *args) { int state; + if(!Py_CHECKWRITE(self)){ + return PyErr_WriteToImmutable(self); + } + if (!PyArg_ParseTuple(args, "i:setstate", &state)) return NULL; self->state = state; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index b67844a67c315c..8f450392978b04 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2010,20 +2010,28 @@ zlib_exec(PyObject *mod) { zlibstate *state = get_zlib_state(mod); + // These types store a `z_stream` which is modified in basically + // every call. This prevents them from being immutable. However, + // it seems like some functions could be refactored to create the + // z_stream on demand, which would allow these types to be used + // even after being frozen. + // + // Removing the `PyNotFreezable_Type` type later shouldn't be a breaking + // change. state->Comptype = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, &Comptype_spec, NULL); + mod, &Comptype_spec, (PyObject*)&PyNotFreezable_Type); if (state->Comptype == NULL) { return -1; } state->Decomptype = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, &Decomptype_spec, NULL); + mod, &Decomptype_spec, (PyObject*)&PyNotFreezable_Type); if (state->Decomptype == NULL) { return -1; } state->ZlibDecompressorType = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, &ZlibDecompressor_type_spec, NULL); + mod, &ZlibDecompressor_type_spec, (PyObject*)&PyNotFreezable_Type); if (state->ZlibDecompressorType == NULL) { return -1; } From 93eca6d963c8c257e2db6e4fd91fde5c13ebdf7e Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Tue, 22 Apr 2025 11:49:56 +0100 Subject: [PATCH 26/40] _dbm etree Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_etree.py | 58 ++++++++++++++++++++++++ Modules/_dbmmodule.c | 2 +- Modules/_elementtree.c | 71 ++++++++++++++++++++++++------ 3 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 Lib/test/test_freeze/test_etree.py diff --git a/Lib/test/test_freeze/test_etree.py b/Lib/test/test_freeze/test_etree.py new file mode 100644 index 00000000000000..56893904b11dcc --- /dev/null +++ b/Lib/test/test_freeze/test_etree.py @@ -0,0 +1,58 @@ +from xml.etree.ElementTree import ElementTree, Element, XMLParser +import unittest + + +from . import BaseNotFreezableTest, BaseObjectTest + + +class TestElementTree(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=ElementTree(), **kwargs) + + +class TestXMLParser(BaseNotFreezableTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=XMLParser(), **kwargs) + + +class TestElement(BaseObjectTest): + def __init__(self, *args, **kwargs): + obj = Element("tag", {"key": "value"}) + super().__init__(*args, obj=obj, **kwargs) + + def test_set(self): + with self.assertRaises(NotWritableError): + self.obj.set("key", "value") + + def test_setitem(self): + with self.assertRaises(NotWritableError): + self.obj["key"] = "value" + + def test_delitem(self): + with self.assertRaises(NotWritableError): + del self.obj["key"] + + def test_clear(self): + with self.assertRaises(NotWritableError): + self.obj.clear() + + def test_append(self): + with self.assertRaises(NotWritableError): + self.obj.append(Element("child")) + + def test_insert(self): + with self.assertRaises(NotWritableError): + self.obj.insert(0, Element("child")) + + def test_remove(self): + with self.assertRaises(NotWritableError): + self.obj.remove(Element("child")) + + def test_iter(self): + it = self.obj.iter() + with self.assertRaises(TypeError): + freeze(it) + + +if __name__ == '__main__': + unittest.main() diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 9908174c94c450..b2d2a9dc23b235 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -540,7 +540,7 @@ _dbm_exec(PyObject *module) { _dbm_state *state = get_dbm_state(module); state->dbm_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, - &dbmtype_spec, NULL); + &dbmtype_spec, (PyObject *)&PyNotFreezable_Type); if (state->dbm_type == NULL) { return -1; } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 6244fcc2064c33..49073e79976305 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -697,6 +697,16 @@ element_dealloc(ElementObject* self) /* -------------------------------------------------------------------- */ +/* macro for writable error checking */ +#define _CHECK_IS_WRITABLE(op) \ + if (!Py_CHECKWRITE(op)) { \ + PyErr_SetObject( \ + PyExc_NotWritableError, \ + (PyObject *)(op)); \ + return NULL; \ + } + + /*[clinic input] _elementtree.Element.append @@ -711,6 +721,8 @@ _elementtree_Element_append_impl(ElementObject *self, PyTypeObject *cls, PyObject *subelement) /*[clinic end generated code: output=d00923711ea317fc input=8baf92679f9717b8]*/ { + _CHECK_IS_WRITABLE(self); + elementtreestate *st = get_elementtree_state_by_cls(cls); if (element_add_subelement(st, self, subelement) < 0) return NULL; @@ -727,6 +739,8 @@ static PyObject * _elementtree_Element_clear_impl(ElementObject *self) /*[clinic end generated code: output=8bcd7a51f94cfff6 input=3c719ff94bf45dd6]*/ { + _CHECK_IS_WRITABLE(self); + clear_extra(self); _set_joined_ptr(&self->text, Py_NewRef(Py_None)); @@ -995,6 +1009,7 @@ _elementtree_Element___getstate___impl(ElementObject *self) PICKLED_TAIL, JOIN_OBJ(self->tail)); } + static PyObject * element_setstate_from_attributes(elementtreestate *st, ElementObject *self, @@ -1007,6 +1022,8 @@ element_setstate_from_attributes(elementtreestate *st, Py_ssize_t i, nchildren; ElementObjectExtra *oldextra = NULL; + _CHECK_IS_WRITABLE(self); + if (!tag) { PyErr_SetString(PyExc_TypeError, "tag may not be NULL"); return NULL; @@ -1096,6 +1113,8 @@ element_setstate_from_Python(elementtreestate *st, ElementObject *self, PyObject *tag, *attrib, *text, *tail, *children; PyObject *retval; + _CHECK_IS_WRITABLE(self); + tag = attrib = text = tail = children = NULL; args = PyTuple_New(0); if (!args) @@ -1210,6 +1229,8 @@ _elementtree_Element_extend_impl(ElementObject *self, PyTypeObject *cls, PyObject* seq; Py_ssize_t i; + _CHECK_IS_WRITABLE(self); + seq = PySequence_Fast(elements, ""); if (!seq) { PyErr_Format( @@ -1529,6 +1550,8 @@ _elementtree_Element_insert_impl(ElementObject *self, Py_ssize_t index, { Py_ssize_t i; + _CHECK_IS_WRITABLE(self); + if (!self->extra) { if (create_extra(self, NULL) < 0) return NULL; @@ -1639,6 +1662,8 @@ _elementtree_Element_remove_impl(ElementObject *self, PyObject *subelement) int rc; PyObject *found; + _CHECK_IS_WRITABLE(self); + if (!self->extra) { /* element has no children, so raise exception */ PyErr_SetString( @@ -1715,6 +1740,8 @@ _elementtree_Element_set_impl(ElementObject *self, PyObject *key, { PyObject* attrib; + _CHECK_IS_WRITABLE(self); + if (!self->extra) { if (create_extra(self, NULL) < 0) return NULL; @@ -1730,6 +1757,15 @@ _elementtree_Element_set_impl(ElementObject *self, PyObject *key, Py_RETURN_NONE; } +/* macro for writable validation */ +#define _VALIDATE_WRITABLE(op) \ + if (!Py_CHECKWRITE(op)) { \ + PyErr_SetObject( \ + PyExc_NotWritableError, \ + (PyObject *)((op))); \ + return -1; \ + } + static int element_setitem(PyObject* self_, Py_ssize_t index, PyObject* item) { @@ -1737,6 +1773,8 @@ element_setitem(PyObject* self_, Py_ssize_t index, PyObject* item) Py_ssize_t i; PyObject* old; + _VALIDATE_WRITABLE(self); + if (!self->extra || index < 0 || index >= self->extra->length) { PyErr_SetString( PyExc_IndexError, @@ -1822,6 +1860,8 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) { ElementObject* self = (ElementObject*) self_; + _VALIDATE_WRITABLE(self); + if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -2049,6 +2089,7 @@ element_attrib_getter(ElementObject *self, void *closure) static int element_tag_setter(ElementObject *self, PyObject *value, void *closure) { + _VALIDATE_WRITABLE(self); _VALIDATE_ATTR_VALUE(value); Py_SETREF(self->tag, Py_NewRef(value)); return 0; @@ -2057,6 +2098,7 @@ element_tag_setter(ElementObject *self, PyObject *value, void *closure) static int element_text_setter(ElementObject *self, PyObject *value, void *closure) { + _VALIDATE_WRITABLE(self); _VALIDATE_ATTR_VALUE(value); _set_joined_ptr(&self->text, Py_NewRef(value)); return 0; @@ -2065,6 +2107,7 @@ element_text_setter(ElementObject *self, PyObject *value, void *closure) static int element_tail_setter(ElementObject *self, PyObject *value, void *closure) { + _VALIDATE_WRITABLE(self); _VALIDATE_ATTR_VALUE(value); _set_joined_ptr(&self->tail, Py_NewRef(value)); return 0; @@ -2073,6 +2116,7 @@ element_tail_setter(ElementObject *self, PyObject *value, void *closure) static int element_attrib_setter(ElementObject *self, PyObject *value, void *closure) { + _VALIDATE_WRITABLE(self); _VALIDATE_ATTR_VALUE(value); if (!PyDict_Check(value)) { PyErr_Format(PyExc_TypeError, @@ -2618,6 +2662,7 @@ treebuilder_flush_data(TreeBuilderObject* self) if (!self->data) { return 0; } + elementtreestate *st = self->state; if (!self->last_for_tail) { PyObject *element = self->last; @@ -4315,15 +4360,15 @@ static PyMethodDef _functions[] = { {NULL, NULL} }; -#define CREATE_TYPE(module, type, spec) \ -do { \ - if (type != NULL) { \ - break; \ - } \ - type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, NULL); \ - if (type == NULL) { \ - goto error; \ - } \ +#define CREATE_TYPE(module, type, spec, base) \ +do { \ + if (type != NULL) { \ + break; \ + } \ + type = (PyTypeObject *)PyType_FromModuleAndSpec(module, spec, (PyObject *)base); \ + if (type == NULL) { \ + goto error; \ + } \ } while (0) static int @@ -4332,10 +4377,10 @@ module_exec(PyObject *m) elementtreestate *st = get_elementtree_state(m); /* Initialize object types */ - CREATE_TYPE(m, st->ElementIter_Type, &elementiter_spec); - CREATE_TYPE(m, st->TreeBuilder_Type, &treebuilder_spec); - CREATE_TYPE(m, st->Element_Type, &element_spec); - CREATE_TYPE(m, st->XMLParser_Type, &xmlparser_spec); + CREATE_TYPE(m, st->ElementIter_Type, &elementiter_spec, &PyNotFreezable_Type); + CREATE_TYPE(m, st->TreeBuilder_Type, &treebuilder_spec, &PyNotFreezable_Type); + CREATE_TYPE(m, st->Element_Type, &element_spec, NULL); + CREATE_TYPE(m, st->XMLParser_Type, &xmlparser_spec, &PyNotFreezable_Type); st->deepcopy_obj = _PyImport_GetModuleAttrString("copy", "deepcopy"); if (st->deepcopy_obj == NULL) { From 2ab77d3040a92fb4eb3269e8def63c3bb9129001 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Mon, 28 Apr 2025 13:21:44 +0100 Subject: [PATCH 27/40] immutable module Signed-off-by: Matthew A Johnson --- Doc/conf.py | 1 - Doc/data/stable_abi.dat | 1 - Include/cpython/immutability.h | 8 + Include/cpython/object.h | 1 + Include/cpython/pyerrors.h | 3 - Include/immutability.h | 9 +- Include/internal/pycore_frame.h | 2 +- Include/internal/pycore_immutability.h | 20 ++ Include/internal/pycore_interp.h | 2 + Include/pyerrors.h | 3 +- Lib/_compat_pickle.py | 1 - Lib/test/exception_hierarchy.txt | 1 - Lib/test/test_descr.py | 4 +- Lib/test/test_freeze/__init__.py | 9 +- Lib/test/test_freeze/test_abc.py | 4 +- Lib/test/test_freeze/test_array.py | 24 +-- Lib/test/test_freeze/test_collections.py | 44 ++--- Lib/test/test_freeze/test_core.py | 203 +++++++++---------- Lib/test/test_freeze/test_ctypes.py | 22 +-- Lib/test/test_freeze/test_decimal.py | 12 +- Lib/test/test_freeze/test_etree.py | 14 +- Lib/test/test_freeze/test_hashlib.py | 4 +- Lib/test/test_freeze/test_xxlimited.py | 4 +- Lib/test/test_freeze/test_xxsubtype.py | 11 +- Lib/test/test_importlib/test_api.py | 2 +- Lib/test/test_io.py | 4 +- Lib/test/test_stable_abi_ctypes.py | 1 - Makefile.pre.in | 1 + Misc/stable_abi.toml | 2 - Modules/Setup | 1 + Modules/Setup.stdlib.in | 1 + Modules/_asynciomodule.c | 4 +- Modules/_bz2module.c | 4 +- Modules/_collectionsmodule.c | 4 +- Modules/_csv.c | 4 +- Modules/_ctypes/_ctypes.c | 2 +- Modules/_dbmmodule.c | 2 +- Modules/_elementtree.c | 12 +- Modules/_io/_iomodule.c | 6 +- Modules/_multiprocessing/multiprocessing.c | 2 +- Modules/_sqlite/blob.c | 2 +- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/cursor.c | 2 +- Modules/clinic/immutablemodule.c.h | 37 ++++ Modules/immutablemodule.c | 185 ++++++++++++++++++ Modules/xxlimited.c | 21 +- Modules/xxlimited_35.c | 12 -- Modules/xxmodule.c | 4 - Modules/xxsubtype.c | 9 - Modules/zlibmodule.c | 14 +- Objects/abstract.c | 2 +- Objects/dictobject.c | 6 +- Objects/exceptions.c | 6 - Objects/memoryobject.c | 2 +- Objects/object.c | 1 - Objects/typeobject.c | 87 +++++++++ PC/python3dll.c | 2 - PCbuild/pythoncore.vcxproj | 2 + PCbuild/pythoncore.vcxproj.filters | 8 +- Python/bltinmodule.c | 38 ---- Python/ceval.c | 2 +- Python/clinic/bltinmodule.c.h | 20 +- Python/errors.c | 8 +- Python/immutability.c | 214 +++++++++++---------- Python/pystate.c | 4 + Python/stdlib_module_names.h | 1 + Tools/c-analyzer/cpython/_parser.py | 1 + Tools/c-analyzer/cpython/ignored.tsv | 2 - configure | 28 +++ configure.ac | 1 + 70 files changed, 726 insertions(+), 456 deletions(-) create mode 100644 Include/cpython/immutability.h create mode 100644 Include/internal/pycore_immutability.h create mode 100644 Modules/clinic/immutablemodule.c.h create mode 100644 Modules/immutablemodule.c diff --git a/Doc/conf.py b/Doc/conf.py index 8d88d4d29e6b81..9f01bd89a35d3c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -215,7 +215,6 @@ ('c:data', 'PyExc_UnicodeError'), ('c:data', 'PyExc_UnicodeTranslateError'), ('c:data', 'PyExc_ValueError'), - ('c:data', 'PyExc_NotWritableError'), ('c:data', 'PyExc_ZeroDivisionError'), # C API: Standard Python warning classes ('c:data', 'PyExc_BytesWarning'), diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 947e1f5ddee238..f112d268129fd1 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -238,7 +238,6 @@ var,PyExc_ModuleNotFoundError,3.6,, var,PyExc_NameError,3.2,, var,PyExc_NotADirectoryError,3.7,, var,PyExc_NotImplementedError,3.2,, -var,PyExc_NotWritableError,3.12,, var,PyExc_OSError,3.2,, var,PyExc_OverflowError,3.2,, var,PyExc_PendingDeprecationWarning,3.2,, diff --git a/Include/cpython/immutability.h b/Include/cpython/immutability.h new file mode 100644 index 00000000000000..c6729243fede06 --- /dev/null +++ b/Include/cpython/immutability.h @@ -0,0 +1,8 @@ +#ifndef Py_CPYTHON_IMMUTABLE_H +# error "this header file must not be included directly" +#endif + +PyAPI_DATA(PyTypeObject) PyNotFreezable_Type; + +PyAPI_FUNC(PyObject *) _PyImmutability_Freeze(PyObject*); +PyAPI_FUNC(PyObject *) _PyImmutability_RegisterFreezable(PyObject*); diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ae7f780a93182a..50b34253039203 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -273,6 +273,7 @@ typedef struct _heaptypeobject { PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *); +PyAPI_FUNC(PyObject *) _PyType_UsesDefaultSlots(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); #ifndef Py_BUILD_CORE // Backward compatibility for 3rd-party extensions diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 74cf3f1d1de680..156665cbdb1ba4 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -168,9 +168,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFormat( const char *format, ...); -PyAPI_FUNC(PyObject *) _PyErr_WriteToImmutable(const char* file, int line, PyObject *obj); -#define PyErr_WriteToImmutable(obj) _PyErr_WriteToImmutable(__FILE__, __LINE__, _PyObject_CAST(obj)) - extern PyObject *_PyErr_SetImportErrorWithNameFrom( PyObject *, PyObject *, diff --git a/Include/immutability.h b/Include/immutability.h index 55fbb6b83fa003..4735ada6cc9a5b 100644 --- a/Include/immutability.h +++ b/Include/immutability.h @@ -5,10 +5,13 @@ extern "C" { #endif -PyAPI_DATA(PyTypeObject) PyNotFreezable_Type; -PyAPI_FUNC(PyObject *) _Py_Freeze(PyObject*); -#define Py_Freeze(op) _Py_Freeze(_PyObject_CAST(op)) +#ifndef Py_LIMITED_API +# define Py_CPYTHON_IMMUTABLE_H +# include "cpython/immutability.h" +# undef Py_CPYTHON_IMMUTABLE_H +#endif + #ifdef __cplusplus } diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 78719efdca0c0f..158db2cf9df82e 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -198,7 +198,7 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily * creating it if necessary. - * Returns a borrowed reference */ + * Returns a borrowed referennce */ static inline PyFrameObject * _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) { diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h new file mode 100644 index 00000000000000..67ac1ebe736494 --- /dev/null +++ b/Include/internal/pycore_immutability.h @@ -0,0 +1,20 @@ +#ifndef Py_INTERNAL_IMMUTABILITY_H +#define Py_INTERNAL_IMMUTABILITY_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "Py_BUILD_CORE must be defined to include this header" +#endif + +struct _Py_immutability_state { + PyObject *module_locks; + PyObject *blocking_on; + PyObject *freezable_types; +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_IMMUTABILITY_H */ \ No newline at end of file diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1db23145a539cb..3225f2bbcb3855 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -25,6 +25,7 @@ extern "C" { #include "pycore_gc.h" // struct _gc_runtime_state #include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_import.h" // struct _import_state +#include "pycore_immutability.h" // struct _immutability_runtime_state #include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_object_state.h" // struct _py_object_state @@ -176,6 +177,7 @@ struct _is { struct _Py_async_gen_state async_gen; struct _Py_context_state context; struct _Py_exc_state exc_state; + struct _Py_immutability_state immutability; struct ast_state ast; struct types_state types; diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 59bc75c6b65491..8eee5a780afd1b 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -121,7 +121,6 @@ PyAPI_DATA(PyObject *) PyExc_UnicodeDecodeError; PyAPI_DATA(PyObject *) PyExc_UnicodeTranslateError; PyAPI_DATA(PyObject *) PyExc_ValueError; PyAPI_DATA(PyObject *) PyExc_ZeroDivisionError; -PyAPI_DATA(PyObject *) PyExc_NotWritableError; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_DATA(PyObject *) PyExc_BlockingIOError; @@ -169,6 +168,8 @@ PyAPI_DATA(PyObject *) PyExc_ResourceWarning; PyAPI_FUNC(int) PyErr_BadArgument(void); PyAPI_FUNC(PyObject *) PyErr_NoMemory(void); PyAPI_FUNC(PyObject *) PyErr_SetFromErrno(PyObject *); +PyAPI_FUNC(PyObject *) _PyErr_WriteToImmutable(PyObject *); +#define PyErr_WriteToImmutable(op) _PyErr_WriteToImmutable(_PyObject_CAST(op)) PyAPI_FUNC(PyObject *) PyErr_SetFromErrnoWithFilenameObject( PyObject *, PyObject *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03040000 diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index cb39eb7e937e8b..65a94b6b1bdfd5 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -221,7 +221,6 @@ ('http.server', 'CGIHTTPRequestHandler'): ('CGIHTTPServer', 'CGIHTTPRequestHandler'), ('_socket', 'socket'): ('socket', '_socketobject'), - ('builtins', 'NotWritableError'): ('exceptions', 'StandardError'), }) PYTHON3_OSERROR_EXCEPTIONS = ( diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 68db5620ebc872..1eca123be0fecb 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -53,7 +53,6 @@ BaseException │ ├── UnicodeDecodeError │ ├── UnicodeEncodeError │ └── UnicodeTranslateError - ├── NotWritableError └── Warning ├── BytesWarning ├── DeprecationWarning diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 15ea2b196766f8..9d8b1497330f0a 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3282,15 +3282,13 @@ class F(D, E): pass def cant(x, C): try: x.__class__ = C - except NotWritableError: - pass except TypeError: pass else: self.fail("shouldn't allow %r.__class__ = %r" % (x, C)) try: delattr(x, "__class__") - except (TypeError, AttributeError, NotWritableError): + except (TypeError, AttributeError): pass else: self.fail("shouldn't allow del %r.__class__" % x) diff --git a/Lib/test/test_freeze/__init__.py b/Lib/test/test_freeze/__init__.py index a6da8bbde5e531..285e31be224866 100644 --- a/Lib/test/test_freeze/__init__.py +++ b/Lib/test/test_freeze/__init__.py @@ -1,6 +1,7 @@ import os from test.support import load_package_tests import unittest +from immutable import freeze, isfrozen, NotFreezable def load_tests(*args): @@ -16,18 +17,18 @@ def setUp(self): freeze(self.obj) def test_immutable(self): - self.assertTrue(isimmutable(self.obj)) + self.assertTrue(isfrozen(self.obj)) def test_add_attribute(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.new_attribute = 'value' def test_type_immutable(self): - self.assertTrue(isimmutable(type(self.obj))) + self.assertTrue(isfrozen(type(self.obj))) class BaseNotFreezableTest(unittest.TestCase): - def __init__(self, *args, obj=notfreezable(), **kwargs): + def __init__(self, *args, obj=NotFreezable(), **kwargs): unittest.TestCase.__init__(self, *args, **kwargs) self.obj = obj diff --git a/Lib/test/test_freeze/test_abc.py b/Lib/test/test_freeze/test_abc.py index 71012cff86ce1f..523122c769d449 100644 --- a/Lib/test/test_freeze/test_abc.py +++ b/Lib/test/test_freeze/test_abc.py @@ -27,7 +27,7 @@ class C(ABC): def bar(self): pass - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.A.register(C) def test_invalid_cache(self): @@ -52,7 +52,7 @@ def baz(self): x = F() freeze(x) self.assertTrue(isimmutable(D)) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): # the caches are invalidated but cannot be updated # because the class is frozen isinstance(x, D) diff --git a/Lib/test/test_freeze/test_array.py b/Lib/test/test_freeze/test_array.py index a345029761ace3..fa6bbb0b6f77c5 100644 --- a/Lib/test/test_freeze/test_array.py +++ b/Lib/test/test_freeze/test_array.py @@ -8,49 +8,49 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[0] = 5 def test_set_slice(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[1:3] = [6, 7] def test_append(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.append(8) def test_extend(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.extend(array('i', [9])) def test_insert(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.insert(0, 10) def test_pop(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.pop() def test_remove(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.remove(1) def test_delete(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): del self.obj[0] def test_reverse(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.reverse() def test_inplace_repeat(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj *= 2 def test_inplace_concat(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj += array('i', [11]) def test_byteswap(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.byteswap() diff --git a/Lib/test/test_freeze/test_collections.py b/Lib/test/test_freeze/test_collections.py index 612f78965a4292..dba48eed747b89 100644 --- a/Lib/test/test_freeze/test_collections.py +++ b/Lib/test/test_freeze/test_collections.py @@ -12,63 +12,63 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[0] = None def test_append(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.append(TestDeque.C()) def test_appendleft(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.appendleft(TestDeque.C()) def test_extend(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.extend([TestDeque.C()]) def test_extendleft(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.extendleft([TestDeque.C()]) def test_insert(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.insert(0, TestDeque.C()) def test_pop(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.pop() def test_popleft(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.popleft() def test_remove(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.remove(1) def test_delete(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): del self.obj[0] def test_inplace_repeat(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj *= 2 def test_inplace_concat(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj += [TestDeque.C()] def test_reverse(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.reverse() def test_rotate(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.rotate(1) def test_clear(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.clear() def test_iter(self): @@ -91,29 +91,29 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item_exists(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[1] = None def test_set_item_new(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj["three"] = 5 def test_del_item(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): del self.obj[1] def test_clear(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.clear() def test_pop(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.pop(1) def test_popitem(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.popitem() def test_update(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.update({1: None}) diff --git a/Lib/test/test_freeze/test_core.py b/Lib/test/test_freeze/test_core.py index 61192aab2f0d05..b063fd862f849d 100644 --- a/Lib/test/test_freeze/test_core.py +++ b/Lib/test/test_freeze/test_core.py @@ -1,4 +1,5 @@ import unittest +from immutable import freeze, isfrozen from . import BaseObjectTest @@ -18,7 +19,7 @@ def global1_inc(): class MutableGlobalTest(unittest.TestCase): # Add initial test to confirm that global_canary is mutable def test_global_mutable(self): - self.assertTrue(not isimmutable(global_canary)) + self.assertTrue(not isfrozen(global_canary)) class TestBasicObject(BaseObjectTest): @@ -33,13 +34,13 @@ class TestFloat(unittest.TestCase): def test_freeze_float(self): obj = 0.0 freeze(obj) - self.assertTrue(isimmutable(obj)) + self.assertTrue(isfrozen(obj)) class TestFloatType(unittest.TestCase): def test_float_type_immutable(self): obj = 0.0 c = obj.__class__ - self.assertTrue(isimmutable(c)) + self.assertTrue(isfrozen(c)) class TestList(BaseObjectTest): class C: @@ -50,55 +51,55 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[0] = None def test_set_slice(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[1:3] = [None, None] def test_append(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.append(TestList.C()) def test_extend(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.extend([TestList.C()]) def test_insert(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.insert(0, TestList.C()) def test_pop(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.pop() def test_remove(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.remove(1) def test_delete(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): del self.obj[0] def test_reverse(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.reverse() def test_inplace_repeat(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj *= 2 def test_inplace_concat(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj += [TestList.C()] def test_clear(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.clear() def test_sort(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.sort() @@ -111,35 +112,35 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_set_item_exists(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[1] = None def test_set_item_new(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj["three"] = TestDict.C() def test_del_item(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): del self.obj[1] def test_clear(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.clear() def test_pop(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.pop(1) def test_popitem(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.popitem() def test_setdefault(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.setdefault("three", TestDict.C()) def test_update(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.update({1: None}) @@ -149,27 +150,27 @@ def __init__(self, *args, **kwargs): BaseObjectTest.__init__(self, *args, obj=obj, **kwargs) def test_add(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.add(1) def test_clear(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.clear() def test_discard(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.discard(1) def test_pop(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.pop() def test_remove(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.remove(1) def test_update(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.update([1, 2]) @@ -189,25 +190,25 @@ class C: freeze(self.obj) def test_immutable(self): - self.assertTrue(isimmutable(self.obj)) - self.assertTrue(isimmutable(self.obj.a)) - self.assertTrue(isimmutable(self.obj.a.b)) - self.assertTrue(isimmutable(self.obj.d)) - self.assertTrue(isimmutable(self.obj.d[0])) - self.assertTrue(isimmutable(self.obj.d[0].e)) - self.assertTrue(isimmutable(self.obj.g)) - self.assertTrue(isimmutable(self.obj.g[1])) - self.assertTrue(isimmutable(self.obj.g[1].h)) - self.assertTrue(isimmutable(self.obj.g["two"])) - self.assertTrue(isimmutable(self.obj.g["two"].i)) + self.assertTrue(isfrozen(self.obj)) + self.assertTrue(isfrozen(self.obj.a)) + self.assertTrue(isfrozen(self.obj.a.b)) + self.assertTrue(isfrozen(self.obj.d)) + self.assertTrue(isfrozen(self.obj.d[0])) + self.assertTrue(isfrozen(self.obj.d[0].e)) + self.assertTrue(isfrozen(self.obj.g)) + self.assertTrue(isfrozen(self.obj.g[1])) + self.assertTrue(isfrozen(self.obj.g[1].h)) + self.assertTrue(isfrozen(self.obj.g["two"])) + self.assertTrue(isfrozen(self.obj.g["two"].i)) def test_set_const(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.const = 1 def test_type_immutable(self): - self.assertTrue(isimmutable(type(self.obj))) - self.assertTrue(isimmutable(type(self.obj).const)) + self.assertTrue(isfrozen(type(self.obj))) + self.assertTrue(isfrozen(type(self.obj).const)) class TestFunctions(unittest.TestCase): @@ -239,7 +240,7 @@ def inc(): self.assertEqual(test(), 1) self.assertEqual(test(), 2) freeze(test) - self.assertRaises(NotWritableError, test) + self.assertRaises(TypeError, test) def test_global(self): def d(): @@ -249,9 +250,9 @@ def d(): self.assertEqual(d(), 1) freeze(d) - self.assertTrue(isimmutable(global0)) - self.assertFalse(isimmutable(global_canary)) - self.assertRaises(NotWritableError, d) + self.assertTrue(isfrozen(global0)) + self.assertFalse(isfrozen(global_canary)) + self.assertRaises(TypeError, d) def test_hidden_global(self): global global0 @@ -264,7 +265,7 @@ def d(): global0 = 0 self.assertEqual(d(), 1) freeze(d) - self.assertRaises(NotWritableError, d) + self.assertRaises(TypeError, d) def test_builtins(self): def e(): @@ -272,9 +273,9 @@ def e(): return sum(test) freeze(e) - self.assertTrue(isimmutable(list)) - self.assertTrue(isimmutable(range)) - self.assertTrue(isimmutable(sum)) + self.assertTrue(isfrozen(list)) + self.assertTrue(isfrozen(range)) + self.assertTrue(isfrozen(sum)) def test_builtins_nested(self): def g(): @@ -285,19 +286,19 @@ def nested_test(): return nested_test() freeze(g) - self.assertTrue(isimmutable(list)) - self.assertTrue(isimmutable(range)) - self.assertTrue(isimmutable(sum)) + self.assertTrue(isfrozen(list)) + self.assertTrue(isfrozen(range)) + self.assertTrue(isfrozen(sum)) def test_global_fun(self): def d(): return global1_inc() freeze(d) - self.assertTrue(isimmutable(global1)) - self.assertTrue(isimmutable(global1_inc)) - self.assertFalse(isimmutable(global_canary)) - self.assertRaises(NotWritableError, d) + self.assertTrue(isfrozen(global1)) + self.assertTrue(isfrozen(global1_inc)) + self.assertFalse(isfrozen(global_canary)) + self.assertRaises(TypeError, d) def test_globals_copy(self): def f(): @@ -328,21 +329,21 @@ def test_lambda(self): obj = TestMethods.C() obj.c = lambda x: pow(x, 2) freeze(obj) - self.assertTrue(isimmutable(TestMethods.C)) - self.assertTrue(isimmutable(pow)) - self.assertRaises(NotWritableError, obj.b, 1) + self.assertTrue(isfrozen(TestMethods.C)) + self.assertTrue(isfrozen(pow)) + self.assertRaises(TypeError, obj.b, 1) self.assertEqual(obj.c(2), 4) def test_method(self): obj = TestMethods.C() freeze(obj) self.assertEqual(obj.a(), 1) - self.assertTrue(isimmutable(obj)) - self.assertTrue(isimmutable(abs)) - self.assertTrue(isimmutable(obj.val)) - self.assertRaises(NotWritableError, obj.b, 1) + self.assertTrue(isfrozen(obj)) + self.assertTrue(isfrozen(abs)) + self.assertTrue(isfrozen(obj.val)) + self.assertRaises(TypeError, obj.b, 1) # Second test as the byte code can be changed by the first call - self.assertRaises(NotWritableError, obj.b, 1) + self.assertRaises(TypeError, obj.b, 1) class TestLocals(unittest.TestCase): @@ -363,9 +364,9 @@ def inner(): freeze(obj) return obj, obj2, obj3 obj, obj2, obj3 = inner() - self.assertTrue(isimmutable(obj)) - self.assertTrue(isimmutable(obj2)) - self.assertFalse(isimmutable(obj3)) + self.assertTrue(isfrozen(obj)) + self.assertTrue(isfrozen(obj2)) + self.assertFalse(isfrozen(obj3)) class TestDictMutation(unittest.TestCase): class C: @@ -382,8 +383,8 @@ def set(self, x): def test_dict_mutation(self): obj = TestDictMutation.C() freeze(obj) - self.assertTrue(isimmutable(obj)) - self.assertRaises(NotWritableError, obj.set, 1) + self.assertTrue(isfrozen(obj)) + self.assertRaises(TypeError, obj.set, 1) self.assertEqual(obj.get(), 0) def test_dict_mutation2(self): @@ -392,8 +393,8 @@ def test_dict_mutation2(self): self.assertEqual(obj.get(), 1) freeze(obj) self.assertEqual(obj.get(), 1) - self.assertTrue(isimmutable(obj)) - self.assertRaises(NotWritableError, obj.set, 1) + self.assertTrue(isfrozen(obj)) + self.assertRaises(TypeError, obj.set, 1) class TestWeakRef(unittest.TestCase): class B: @@ -411,11 +412,11 @@ def test_weakref(self): obj = TestWeakRef.B() c = TestWeakRef.C(obj) freeze(c) - self.assertTrue(isimmutable(c)) + self.assertTrue(isfrozen(c)) self.assertTrue(c.val() is obj) # Following line is not true in the current implementation - # self.assertTrue(isimmutable(c.val())) - self.assertFalse(isimmutable(c.val())) + # self.assertTrue(isfrozen(c.val())) + self.assertFalse(isfrozen(c.val())) obj = None # Following line is not true in the current implementation # this means me can get a race on weak references @@ -428,8 +429,8 @@ def test_stack_capture(self): x = {} x["frame"] = sys._getframe() freeze(x) - self.assertTrue(isimmutable(x)) - self.assertTrue(isimmutable(x["frame"])) + self.assertTrue(isfrozen(x)) + self.assertTrue(isfrozen(x["frame"])) global_test_dict = 0 class TestGlobalDictMutation(unittest.TestCase): @@ -442,8 +443,8 @@ def f1(): def test_global_dict_mutation(self): f1 = TestGlobalDictMutation.g() - self.assertTrue(isimmutable(f1)) - self.assertRaises(NotWritableError, f1) + self.assertTrue(isfrozen(f1)) + self.assertRaises(TypeError, f1) class TestSubclass(unittest.TestCase): @@ -460,8 +461,8 @@ def b(self, val): c_obj = C(1) freeze(c_obj) - self.assertTrue(isimmutable(c_obj)) - self.assertTrue(isimmutable(C)) + self.assertTrue(isfrozen(c_obj)) + self.assertTrue(isfrozen(C)) class D(C): def __init__(self, val): super().__init__(val) @@ -500,58 +501,58 @@ def f(): freeze(f) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__annotations__ = {} # The following should raise an exception, # but doesn't as the __annotations__ gets # replaced with a new dict that is not # immutable. - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__annotations__["foo"] = 2 - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__builtins__ = {} - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__builtins__["foo"] = 2 - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): def g(): pass f.__code__ = g.__code__ - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__defaults__ = (1,2) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__dict__ = {} - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__dict__["foo"] = {} - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__doc__ = "foo" - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__globals__ = {} - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__globals__["foo"] = 2 - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__kwdefaults__ = {} - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__module__ = "foo" - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__name__ = "foo" - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__qualname__ = "foo" - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): f.__type_params__ = (1,2) class TestFunctionDefaults(unittest.TestCase): @@ -562,7 +563,7 @@ def f(b=bdef): freeze(f) - self.assertTrue(isimmutable(bdef)) + self.assertTrue(isfrozen(bdef)) def test_function_kwdefaults(self): bdef = {} @@ -572,7 +573,7 @@ def f(a, **b): freeze(f) - self.assertTrue(isimmutable(bdef)) + self.assertTrue(isfrozen(bdef)) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_freeze/test_ctypes.py b/Lib/test/test_freeze/test_ctypes.py index fa7683afd89430..fb005eb45d7918 100644 --- a/Lib/test/test_freeze/test_ctypes.py +++ b/Lib/test/test_freeze/test_ctypes.py @@ -8,13 +8,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=ctypes.create_string_buffer(b"hello"), **kwargs) def test_raw(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.raw = b"world" self.assertEqual(self.obj.raw, b"hello\x00") def test_value(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.value = b"world" self.assertEqual(self.obj.value, b"hello") @@ -25,7 +25,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=ctypes.create_unicode_buffer("hello"), **kwargs) def test_value(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.value = "world" self.assertEqual(self.obj.value, "hello") @@ -39,7 +39,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=TestStructure.POINT(1, 2), **kwargs) def test_modify_field(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.x = 3 self.assertEqual(self.obj.x, 1) @@ -59,7 +59,7 @@ def test_contents_immutable(self): def test_set_contents(self): b = TestPointer.POINT(3, 4) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.contents = b self.assertEqual(self.obj.contents.x, self.a.x) @@ -79,19 +79,19 @@ def test_point_immutable(self): self.assertTrue(isimmutable(TestArray.POINT)) def test_modify_item(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[0].x = 1 self.assertEqual(self.obj[0].x, 0) def test_ass_item(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[0] = TestArray.POINT(1, 2) def test_ass_subscript(self): TwoPointsArrayType = TestArray.POINT * 2 a = TwoPointsArrayType() - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj[:2] = a @@ -110,19 +110,19 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=a, **kwargs) def test_assign_part(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.parts.high = 0 self.assertEqual(self.obj.parts.high, 0xFF) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.parts.low = 0 self.assertEqual(self.obj.parts.low, 0xFF) self.assertEqual(self.obj.value, 0x00FF00FF) def test_assign_value(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.value = 0x00FF00FF self.assertEqual(self.obj.value, 0x00FF00FF) diff --git a/Lib/test/test_freeze/test_decimal.py b/Lib/test/test_freeze/test_decimal.py index be7314adc8bdd9..be2df236657f74 100644 --- a/Lib/test/test_freeze/test_decimal.py +++ b/Lib/test/test_freeze/test_decimal.py @@ -8,25 +8,25 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=decimal.Context(), **kwargs) def test_prec(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.prec = 10 def test_emax(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.Emax = 10 def test_emin(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.Emin = -10 def test_rounding(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.rounding = decimal.ROUND_DOWN def test_capitals(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.capitals = 0 def test_clamp(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.clamp = 1 diff --git a/Lib/test/test_freeze/test_etree.py b/Lib/test/test_freeze/test_etree.py index 56893904b11dcc..2cf7db0afc3b14 100644 --- a/Lib/test/test_freeze/test_etree.py +++ b/Lib/test/test_freeze/test_etree.py @@ -21,31 +21,31 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=obj, **kwargs) def test_set(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.set("key", "value") def test_setitem(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj["key"] = "value" def test_delitem(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): del self.obj["key"] def test_clear(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.clear() def test_append(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.append(Element("child")) def test_insert(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.insert(0, Element("child")) def test_remove(self): - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): self.obj.remove(Element("child")) def test_iter(self): diff --git a/Lib/test/test_freeze/test_hashlib.py b/Lib/test/test_freeze/test_hashlib.py index c7f656f6e5571f..2eb5742d79744b 100644 --- a/Lib/test/test_freeze/test_hashlib.py +++ b/Lib/test/test_freeze/test_hashlib.py @@ -7,12 +7,12 @@ def test_blake2b(self): h = blake2b(digest_size=32) h.update(b'Hello world') freeze(h) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): h.update(b'!') def test_blake2s(self): h = blake2s(digest_size=32) h.update(b'Hello world') freeze(h) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): h.update(b'!') diff --git a/Lib/test/test_freeze/test_xxlimited.py b/Lib/test/test_freeze/test_xxlimited.py index 638f0fce68f318..b51e0957fea55a 100644 --- a/Lib/test/test_freeze/test_xxlimited.py +++ b/Lib/test/test_freeze/test_xxlimited.py @@ -13,7 +13,7 @@ def test_xxo_set_attribute(self): freeze(xxo) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): xxo.foo = 1234 def test_xxo_del_attribute(self): @@ -21,7 +21,7 @@ def test_xxo_del_attribute(self): freeze(xxo) - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): del xxo.foo diff --git a/Lib/test/test_freeze/test_xxsubtype.py b/Lib/test/test_freeze/test_xxsubtype.py index c32d5a5bf01d9a..c2ca6e6b3906b8 100644 --- a/Lib/test/test_freeze/test_xxsubtype.py +++ b/Lib/test/test_freeze/test_xxsubtype.py @@ -11,14 +11,5 @@ def test_spamdict_setstate(self): freeze(a) - with self.assertRaises(NotWritableError): - a.setstate(17) - - def test_spamdict_setstate(self): - import xxsubtype as spam - a = spam.spamdict() - - freeze(a) - - with self.assertRaises(NotWritableError): + with self.assertRaises(TypeError): a.setstate(17) diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 450b910f0907f3..e8efd32ddabe22 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -135,7 +135,7 @@ def test_sys_modules_loader_is_not_set(self): del module.__spec__.loader except AttributeError: pass - except NotWritableError: + except TypeError: pass sys.modules[name] = module with self.assertRaises(ValueError): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 14e65a439ffe63..e032325fbe2578 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2110,11 +2110,11 @@ def readable(self): self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO()) def test_constructor_with_not_writeable(self): - class NotWritable(MockRawIO): + class NotWriteable(MockRawIO): def writable(self): return False - self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWritable()) + self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable()) def test_read(self): pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index a232d8e143ab47..8cad71c7c34545 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -264,7 +264,6 @@ def test_windows_feature_macros(self): "PyExc_NameError", "PyExc_NotADirectoryError", "PyExc_NotImplementedError", - "PyExc_NotWritableError", "PyExc_OSError", "PyExc_OverflowError", "PyExc_PendingDeprecationWarning", diff --git a/Makefile.pre.in b/Makefile.pre.in index 70ac4433ea8e4f..a9659e7aa3cc52 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1688,6 +1688,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/funcobject.h \ $(srcdir)/Include/cpython/genobject.h \ $(srcdir)/Include/cpython/import.h \ + $(srcdir)/Include/cpython/immutability.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/listobject.h \ diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index f64ec7ace4c571..48299e9b35ff97 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2406,5 +2406,3 @@ added = '3.12' [const.Py_TPFLAGS_ITEMS_AT_END] added = '3.12' -[data.PyExc_NotWritableError] - added = '3.12' diff --git a/Modules/Setup b/Modules/Setup index a8faa1d1028da5..4bfffbca4a8cdc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -152,6 +152,7 @@ PYTHONPATH=$(COREPYTHONPATH) #array arraymodule.c #audioop audioop.c #binascii binascii.c +#immutable immutablemodule.c #cmath cmathmodule.c #math mathmodule.c #mmap mmapmodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 3e01e25056dfa8..3a842982330160 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -34,6 +34,7 @@ @MODULE__CONTEXTVARS_TRUE@_contextvars _contextvarsmodule.c @MODULE__CSV_TRUE@_csv _csv.c @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c +@MODULE_IMMUTABLE_TRUE@immutable immutablemodule.c @MODULE__JSON_TRUE@_json _json.c @MODULE__LSPROF_TRUE@_lsprof _lsprof.c rotatingtree.c @MODULE__OPCODE_TRUE@_opcode _opcode.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index dffc916cac383e..a465090bfaaa38 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3756,8 +3756,8 @@ module_exec(PyObject *mod) } while (0) CREATE_TYPE(mod, state->TaskStepMethWrapper_Type, &TaskStepMethWrapper_spec, NULL); - CREATE_TYPE(mod, state->FutureIterType, &FutureIter_spec, &PyNotFreezable_Type); - CREATE_TYPE(mod, state->FutureType, &Future_spec, &PyNotFreezable_Type); + CREATE_TYPE(mod, state->FutureIterType, &FutureIter_spec, NULL); + CREATE_TYPE(mod, state->FutureType, &Future_spec, NULL); CREATE_TYPE(mod, state->TaskType, &Task_spec, state->FutureType); #undef CREATE_TYPE diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 6d40f4cff23655..97bd44b4ac9694 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -753,7 +753,7 @@ _bz2_exec(PyObject *module) { _bz2_state *state = get_module_state(module); state->bz2_compressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, - &bz2_compressor_type_spec, (PyObject *)&PyNotFreezable_Type); + &bz2_compressor_type_spec, NULL); if (state->bz2_compressor_type == NULL) { return -1; } @@ -762,7 +762,7 @@ _bz2_exec(PyObject *module) } state->bz2_decompressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, - &bz2_decompressor_type_spec, (PyObject *)&PyNotFreezable_Type); + &bz2_decompressor_type_spec, NULL); if (state->bz2_decompressor_type == NULL) { return -1; } diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 0bec947de1dae3..c27338818f5968 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2613,8 +2613,8 @@ collections_exec(PyObject *module) { collections_state *state = get_module_state(module); ADD_TYPE(module, &deque_spec, state->deque_type, NULL); ADD_TYPE(module, &defdict_spec, state->defdict_type, &PyDict_Type); - ADD_TYPE(module, &dequeiter_spec, state->dequeiter_type, &PyNotFreezable_Type); - ADD_TYPE(module, &dequereviter_spec, state->dequereviter_type, &PyNotFreezable_Type); + ADD_TYPE(module, &dequeiter_spec, state->dequeiter_type, NULL); + ADD_TYPE(module, &dequereviter_spec, state->dequereviter_type, NULL); ADD_TYPE(module, &tuplegetter_spec, state->tuplegetter_type, NULL); if (PyModule_AddType(module, &PyODict_Type) < 0) { diff --git a/Modules/_csv.c b/Modules/_csv.c index ed426507614f00..9ab2ad266c2739 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1740,13 +1740,13 @@ csv_exec(PyObject *module) { return -1; } - temp = PyType_FromModuleAndSpec(module, &Reader_Type_spec, (PyObject *)&PyNotFreezable_Type); + temp = PyType_FromModuleAndSpec(module, &Reader_Type_spec, NULL); module_state->reader_type = (PyTypeObject *)temp; if (PyModule_AddObjectRef(module, "Reader", temp) < 0) { return -1; } - temp = PyType_FromModuleAndSpec(module, &Writer_Type_spec, (PyObject *)&PyNotFreezable_Type); + temp = PyType_FromModuleAndSpec(module, &Writer_Type_spec, NULL); module_state->writer_type = (PyTypeObject *)temp; if (PyModule_AddObjectRef(module, "Writer", temp) < 0) { return -1; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 0f73404e7c570a..50b4795002eba6 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3016,7 +3016,7 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) } if(base && _Py_IsImmutable(base)) { - if(Py_Freeze(cmem) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(cmem)) == NULL){ Py_DECREF(cmem); return NULL; } diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index b2d2a9dc23b235..9908174c94c450 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -540,7 +540,7 @@ _dbm_exec(PyObject *module) { _dbm_state *state = get_dbm_state(module); state->dbm_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, - &dbmtype_spec, (PyObject *)&PyNotFreezable_Type); + &dbmtype_spec, NULL); if (state->dbm_type == NULL) { return -1; } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 49073e79976305..ee85201eeb475d 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -701,7 +701,7 @@ element_dealloc(ElementObject* self) #define _CHECK_IS_WRITABLE(op) \ if (!Py_CHECKWRITE(op)) { \ PyErr_SetObject( \ - PyExc_NotWritableError, \ + PyExc_TypeError, \ (PyObject *)(op)); \ return NULL; \ } @@ -740,7 +740,7 @@ _elementtree_Element_clear_impl(ElementObject *self) /*[clinic end generated code: output=8bcd7a51f94cfff6 input=3c719ff94bf45dd6]*/ { _CHECK_IS_WRITABLE(self); - + clear_extra(self); _set_joined_ptr(&self->text, Py_NewRef(Py_None)); @@ -1761,7 +1761,7 @@ _elementtree_Element_set_impl(ElementObject *self, PyObject *key, #define _VALIDATE_WRITABLE(op) \ if (!Py_CHECKWRITE(op)) { \ PyErr_SetObject( \ - PyExc_NotWritableError, \ + PyExc_TypeError, \ (PyObject *)((op))); \ return -1; \ } @@ -4377,10 +4377,10 @@ module_exec(PyObject *m) elementtreestate *st = get_elementtree_state(m); /* Initialize object types */ - CREATE_TYPE(m, st->ElementIter_Type, &elementiter_spec, &PyNotFreezable_Type); - CREATE_TYPE(m, st->TreeBuilder_Type, &treebuilder_spec, &PyNotFreezable_Type); + CREATE_TYPE(m, st->ElementIter_Type, &elementiter_spec, NULL); + CREATE_TYPE(m, st->TreeBuilder_Type, &treebuilder_spec, NULL); CREATE_TYPE(m, st->Element_Type, &element_spec, NULL); - CREATE_TYPE(m, st->XMLParser_Type, &xmlparser_spec, &PyNotFreezable_Type); + CREATE_TYPE(m, st->XMLParser_Type, &xmlparser_spec, NULL); st->deepcopy_obj = _PyImport_GetModuleAttrString("copy", "deepcopy"); if (st->deepcopy_obj == NULL) { diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 6cafc622313926..7b06c1bee5a832 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -674,9 +674,9 @@ iomodule_exec(PyObject *m) } // Base classes - ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, &PyNotFreezable_Type); - ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, &PyNotFreezable_Type); - ADD_TYPE(m, state->PyIOBase_Type, &iobase_spec, &PyNotFreezable_Type); + ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); + ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); + ADD_TYPE(m, state->PyIOBase_Type, &iobase_spec, NULL); // PyIOBase_Type subclasses ADD_TYPE(m, state->PyTextIOBase_Type, &textiobase_spec, diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 42ea827761c128..8f9daa5c3de0cc 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -197,7 +197,7 @@ multiprocessing_exec(PyObject *module) #ifdef HAVE_MP_SEMAPHORE PyTypeObject *semlock_type = (PyTypeObject *)PyType_FromModuleAndSpec( - module, &_PyMp_SemLockType_spec, (PyObject *)&PyNotFreezable_Type); + module, &_PyMp_SemLockType_spec, NULL); if (semlock_type == NULL) { return -1; diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index aa53fe58a0fc52..76d261baf00f38 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -607,7 +607,7 @@ static PyType_Spec blob_spec = { int pysqlite_blob_setup_types(PyObject *mod) { - PyObject *type = PyType_FromModuleAndSpec(mod, &blob_spec, (PyObject *)(&PyNotFreezable_Type)); + PyObject *type = PyType_FromModuleAndSpec(mod, &blob_spec, NULL); if (type == NULL) { return -1; } diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 812bb210b4d524..12e5c135aafa50 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -2649,7 +2649,7 @@ static PyType_Spec connection_spec = { int pysqlite_connection_setup_types(PyObject *module) { - PyObject *type = PyType_FromModuleAndSpec(module, &connection_spec, (PyObject *)(&PyNotFreezable_Type)); + PyObject *type = PyType_FromModuleAndSpec(module, &connection_spec, NULL); if (type == NULL) { return -1; } diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 26ec2c41945754..caeedbddb8d88b 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -1356,7 +1356,7 @@ static PyType_Spec cursor_spec = { int pysqlite_cursor_setup_types(PyObject *module) { - PyObject *type = PyType_FromModuleAndSpec(module, &cursor_spec, (PyObject *)(&PyNotFreezable_Type)); + PyObject *type = PyType_FromModuleAndSpec(module, &cursor_spec, NULL); if (type == NULL) { return -1; } diff --git a/Modules/clinic/immutablemodule.c.h b/Modules/clinic/immutablemodule.c.h new file mode 100644 index 00000000000000..fea1b2373e3af1 --- /dev/null +++ b/Modules/clinic/immutablemodule.c.h @@ -0,0 +1,37 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(immutable_register_freezable__doc__, +"register_freezable($module, obj, /)\n" +"--\n" +"\n" +"Register a type as freezable."); + +#define IMMUTABLE_REGISTER_FREEZABLE_METHODDEF \ + {"register_freezable", (PyCFunction)immutable_register_freezable, METH_O, immutable_register_freezable__doc__}, + +PyDoc_STRVAR(immutable_freeze__doc__, +"freeze($module, obj, /)\n" +"--\n" +"\n" +"Freeze an object and its graph."); + +#define IMMUTABLE_FREEZE_METHODDEF \ + {"freeze", (PyCFunction)immutable_freeze, METH_O, immutable_freeze__doc__}, + +PyDoc_STRVAR(immutable_isfrozen__doc__, +"isfrozen($module, obj, /)\n" +"--\n" +"\n" +"Check if an object is frozen."); + +#define IMMUTABLE_ISFROZEN_METHODDEF \ + {"isfrozen", (PyCFunction)immutable_isfrozen, METH_O, immutable_isfrozen__doc__}, +/*[clinic end generated code: output=0edcea6c15426dec input=a9049054013a1b77]*/ diff --git a/Modules/immutablemodule.c b/Modules/immutablemodule.c new file mode 100644 index 00000000000000..d0e5fdb1001a00 --- /dev/null +++ b/Modules/immutablemodule.c @@ -0,0 +1,185 @@ +/* immutable module */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#define MODULE_VERSION "1.0" + +#include "Python.h" +#include +#include "pycore_object.h" + +/*[clinic input] +module immutable +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=46b92e14e140418a]*/ + +#include "clinic/immutablemodule.c.h" + +typedef struct { + PyObject *not_freezable_error_obj; +} immutable_state; + +static struct PyModuleDef immutablemodule; + +static inline immutable_state* +get_immutable_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (immutable_state *)state; +} + +static int +immutable_clear(PyObject *module) +{ + immutable_state *module_state = PyModule_GetState(module); + Py_CLEAR(module_state->not_freezable_error_obj); + return 0; +} + +static int +immutable_traverse(PyObject *module, visitproc visit, void *arg) +{ + immutable_state *module_state = PyModule_GetState(module); + Py_VISIT(module_state->not_freezable_error_obj); + return 0; +} + +static void +immutable_free(void *module) +{ + immutable_clear((PyObject *)module); +} + +/*[clinic input] +immutable.register_freezable + obj: object + / + +Register a type as freezable. +[clinic start generated code]*/ + +static PyObject * +immutable_register_freezable(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=1afbb9a860e2bde9 input=fbb7f42f02d27a88]*/ +{ + return _PyImmutability_RegisterFreezable(obj); +} + +/*[clinic input] +immutable.freeze + obj: object + / + +Freeze an object and its graph. +[clinic start generated code]*/ + +static PyObject * +immutable_freeze(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=76b9e6c577ec3841 input=d7090b2d52afbb4b]*/ +{ + return _PyImmutability_Freeze(obj); +} + +/*[clinic input] +immutable.isfrozen + obj: object + / + +Check if an object is frozen. +[clinic start generated code]*/ + +static PyObject * +immutable_isfrozen(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=7c10bf6e5f8e4639 input=23d5da80f538c315]*/ +{ + if(_Py_IsImmutable(obj)){ + Py_RETURN_TRUE; + } + + Py_RETURN_FALSE; +} + +static PyType_Slot not_freezable_error_slots[] = { + {0, NULL}, +}; + +PyType_Spec not_freezable_error_spec = { + .name = "immutable.NotFreezableError", + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = not_freezable_error_slots, +}; + +/* + * MODULE + */ + + +PyDoc_STRVAR(immutable_module_doc, ""); + +static struct PyMethodDef immutable_methods[] = { + IMMUTABLE_REGISTER_FREEZABLE_METHODDEF + IMMUTABLE_FREEZE_METHODDEF + IMMUTABLE_ISFROZEN_METHODDEF + { NULL, NULL } +}; + + +static int +immutable_exec(PyObject *module) { + PyObject* frozen_importlib = NULL; + immutable_state *module_state = get_immutable_state(module); + + /* Add version to the module. */ + if (PyModule_AddStringConstant(module, "__version__", + MODULE_VERSION) == -1) { + return -1; + } + + PyObject *bases = PyTuple_Pack(1, PyExc_TypeError); + if (bases == NULL) { + return -1; + } + module_state->not_freezable_error_obj = PyType_FromModuleAndSpec(module, ¬_freezable_error_spec, + bases); + Py_DECREF(bases); + if (module_state->not_freezable_error_obj == NULL) { + return -1; + } + + if (PyModule_AddType(module, (PyTypeObject *)module_state->not_freezable_error_obj) != 0) { + return -1; + } + + if (PyModule_AddType(module, &PyNotFreezable_Type) != 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot immutable_slots[] = { + {Py_mod_exec, immutable_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + +static struct PyModuleDef immutablemodule = { + PyModuleDef_HEAD_INIT, + "immutable", + immutable_module_doc, + sizeof(immutable_state), + immutable_methods, + immutable_slots, + immutable_traverse, + immutable_clear, + immutable_free +}; + +PyMODINIT_FUNC +PyInit_immutable(void) +{ + return PyModuleDef_Init(&immutablemodule); +} diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 3a098e3ea19726..3935c00fc26530 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -65,8 +65,6 @@ #define Py_LIMITED_API 0x030b0000 #include "Python.h" -#include "object.h" -#include "pylifecycle.h" #include #define BUFSIZE 10 @@ -169,11 +167,6 @@ Xxo_getattro(XxoObject *self, PyObject *name) static int Xxo_setattro(XxoObject *self, PyObject *name, PyObject *v) { - if (!Py_CHECKWRITE(self)) { - PyErr_SetObject(PyExc_NotWritableError, (PyObject *)self); - return -1; - } - if (self->x_attr == NULL) { // prepare the attribute dict self->x_attr = PyDict_New(); @@ -242,12 +235,7 @@ Xxo_getbuffer(XxoObject *self, Py_buffer *view, int flags) (void *)self->x_buffer, BUFSIZE, 0, flags); if (res == 0) { - // Not incrementing the counter is safe, since it's only used to prevent - // memory reallocation, which should never happen for an immutable object - // anyways. - if (Py_CHECKWRITE(self)) { - self->x_exports++; - } + self->x_exports++; } return res; } @@ -255,13 +243,6 @@ Xxo_getbuffer(XxoObject *self, Py_buffer *view, int flags) static void Xxo_releasebuffer(XxoObject *self, Py_buffer *view) { - // Not decrementing the counter is safe, since it's only used to prevent - // memory reallocation, which should never happen for an immutable object - // anyways. - if (!Py_CHECKWRITE(self)) { - return; - } - self->x_exports--; } diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 1baee7c977d05c..1ff3ef1cb6f296 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -46,12 +46,6 @@ Xxo_traverse(XxoObject *self, visitproc visit, void *arg) static int Xxo_clear(XxoObject *self) { - // TODO: Pyrona: - // if (!Py_CHECKWRITE(self)) { - // PyErr_WriteToImmutable(self); - // return -1; - // } - Py_CLEAR(self->x_attr); return 0; } @@ -99,12 +93,6 @@ Xxo_getattro(XxoObject *self, PyObject *name) static int Xxo_setattr(XxoObject *self, const char *name, PyObject *v) { - // TODO: Pyrona: - // if (!Py_CHECKWRITE(self)) { - // PyErr_WriteToImmutable(self); - // return -1; - // } - if (self->x_attr == NULL) { self->x_attr = PyDict_New(); if (self->x_attr == NULL) diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index d02ac3c68094ef..1e4e0ea3743ce3 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -79,10 +79,6 @@ Xxo_getattro(XxoObject *self, PyObject *name) static int Xxo_setattr(XxoObject *self, const char *name, PyObject *v) { - if(!Py_CHECKWRITE(self)){ - return PyErr_WriteToImmutable(self); - } - if (self->x_attr == NULL) { self->x_attr = PyDict_New(); if (self->x_attr == NULL) diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index aca60c7be93020..9e4a3d66ef41bd 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -1,6 +1,5 @@ #include "Python.h" #include "structmember.h" // PyMemberDef -#include "pyerrors.h" // PyErr_WriteToImmutable PyDoc_STRVAR(xxsubtype__doc__, "xxsubtype is an example module showing how to subtype builtin types from C.\n" @@ -37,10 +36,6 @@ spamlist_setstate(spamlistobject *self, PyObject *args) { int state; - if(!Py_CHECKWRITE(self)){ - return PyErr_WriteToImmutable(self); - } - if (!PyArg_ParseTuple(args, "i:setstate", &state)) return NULL; self->state = state; @@ -162,10 +157,6 @@ spamdict_setstate(spamdictobject *self, PyObject *args) { int state; - if(!Py_CHECKWRITE(self)){ - return PyErr_WriteToImmutable(self); - } - if (!PyArg_ParseTuple(args, "i:setstate", &state)) return NULL; self->state = state; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 8f450392978b04..b67844a67c315c 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2010,28 +2010,20 @@ zlib_exec(PyObject *mod) { zlibstate *state = get_zlib_state(mod); - // These types store a `z_stream` which is modified in basically - // every call. This prevents them from being immutable. However, - // it seems like some functions could be refactored to create the - // z_stream on demand, which would allow these types to be used - // even after being frozen. - // - // Removing the `PyNotFreezable_Type` type later shouldn't be a breaking - // change. state->Comptype = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, &Comptype_spec, (PyObject*)&PyNotFreezable_Type); + mod, &Comptype_spec, NULL); if (state->Comptype == NULL) { return -1; } state->Decomptype = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, &Decomptype_spec, (PyObject*)&PyNotFreezable_Type); + mod, &Decomptype_spec, NULL); if (state->Decomptype == NULL) { return -1; } state->ZlibDecompressorType = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, &ZlibDecompressor_type_spec, (PyObject*)&PyNotFreezable_Type); + mod, &ZlibDecompressor_type_spec, NULL); if (state->ZlibDecompressorType == NULL) { return -1; } diff --git a/Objects/abstract.c b/Objects/abstract.c index 39ad2ac6a5949a..fb9c878d7c0823 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -39,7 +39,7 @@ immutable_error(void) { PyThreadState *tstate = _PyThreadState_GET(); if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_NotWritableError, + _PyErr_SetString(tstate, PyExc_TypeError, "cannot modify immutable instance"); } return NULL; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ae95847695ba4a..50e1e562aab9c0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5685,7 +5685,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) interp, CACHED_KEYS(tp), values); if (dict != NULL) { if (_Py_IsImmutable(obj)) { - if(Py_Freeze(dict) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(dict)) == NULL){ return NULL; } } @@ -5700,7 +5700,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dictkeys_incref(CACHED_KEYS(tp)); dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); if (_Py_IsImmutable(obj)) { - if(Py_Freeze(dict) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(dict)) == NULL){ return NULL; } } @@ -5729,7 +5729,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = PyDict_New(); } if (_Py_IsImmutable(obj)) { - if(Py_Freeze(dict) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(dict)) == NULL){ return NULL; } } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 7f675eaa2802c6..f376ff24386a37 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3430,11 +3430,6 @@ PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; */ SimpleExtendsException(PyExc_Exception, BufferError, "Buffer error."); -/* - * NotWritableError extends Exception - */ -SimpleExtendsException(PyExc_Exception, NotWritableError, "Object is not writeable."); - /* Warning category docstrings */ @@ -3620,7 +3615,6 @@ static struct static_exception static_exceptions[] = { ITEM(SystemError), ITEM(TypeError), ITEM(ValueError), - ITEM(NotWritableError), ITEM(Warning), // Level 4: ArithmeticError(Exception) subclasses diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index dacf5c72628743..c31b56f67d72ae 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -100,7 +100,7 @@ _PyManagedBuffer_FromObject(PyObject *base, int flags) } if(_Py_IsImmutable(base)){ - if(Py_Freeze(mbuf) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(mbuf)) == NULL){ PyBuffer_Release(&mbuf->master); Py_DECREF(mbuf); return NULL; diff --git a/Objects/object.c b/Objects/object.c index bba1e37a960a5a..01de54ff1a5885 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2102,7 +2102,6 @@ static PyTypeObject* static_types[] = { &PyMethod_Type, &PyModuleDef_Type, &PyModule_Type, - &PyNotFreezable_Type, &PyODictIter_Type, &PyPickleBuffer_Type, &PyProperty_Type, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 542c6673117efe..f8fea715f31223 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3986,6 +3986,7 @@ type_vectorcall(PyObject *metatype, PyObject *const *args, return _PyObject_MakeTpCall(tstate, metatype, args, nargs, kwnames); } + /* An array of type slot offsets corresponding to Py_tp_* constants, * for use in e.g. PyType_Spec and PyType_GetSlot. * Each entry has two offsets: "slot_offset" and "subslot_offset". @@ -6614,6 +6615,92 @@ PyTypeObject PyBaseObject_Type = { }; +PyObject * +_PyType_UsesDefaultSlots(PyTypeObject *tp) +{ + PyNumberMethods *nb = tp->tp_as_number; + PySequenceMethods *sq = tp->tp_as_sequence; + PyMappingMethods *mp = tp->tp_as_mapping; + PyBufferProcs *bp = tp->tp_as_buffer; + + if(!(nb == NULL || + (nb->nb_index == NULL && + nb->nb_int == NULL && + nb->nb_float == NULL))) + { + printf("custom number slots\n"); + Py_RETURN_FALSE; + } + + if(!(sq == NULL || sq->sq_item == NULL)) + { + printf("custom sequence slots\n"); + Py_RETURN_FALSE; + } + + if(!(mp == NULL || mp->mp_subscript == NULL)) + { + printf("custom mapping slots\n"); + Py_RETURN_FALSE; + } + + if(!(bp == NULL || bp->bf_getbuffer == NULL)) + { + printf("custom buffer slots\n"); + Py_RETURN_FALSE; + } + + if(tp->tp_getattr != NULL || + tp->tp_setattr != NULL || + tp->tp_methods != NULL) + { + printf("custom slots\n"); + Py_RETURN_FALSE; + } + + if (!(tp->tp_setattro == PyObject_GenericSetAttr || tp->tp_setattro == NULL)){ + printf("custom setattro\n"); + Py_RETURN_FALSE; + } + + if (!(tp->tp_getattro == PyObject_GenericGetAttr || tp->tp_getattro == NULL)){ + printf("custom getattro\n"); + Py_RETURN_FALSE; + } + + if(!(tp->tp_getset == NULL || + tp->tp_getset == subtype_getsets_full || + tp->tp_getset == subtype_getsets_weakref_only || + tp->tp_getset == subtype_getsets_dict_only)) + { + printf("custom getset\n"); + Py_RETURN_FALSE; + } + + if(!(tp->tp_str == NULL || tp->tp_str == object_str)){ + printf("custom str\n"); + Py_RETURN_FALSE; + } + + if(!(tp->tp_repr == NULL || tp->tp_repr == object_repr)){ + printf("custom repr\n"); + Py_RETURN_FALSE; + } + + if(!(tp->tp_hash == NULL || tp->tp_hash == (hashfunc)_Py_HashPointer)){ + printf("custom hash\n"); + Py_RETURN_FALSE; + } + + if(!(tp->tp_richcompare == NULL || tp->tp_richcompare == object_richcompare)){ + printf("custom richcompare\n"); + Py_RETURN_FALSE; + } + + Py_RETURN_TRUE; +} + + static int type_add_method(PyTypeObject *type, PyMethodDef *meth) { diff --git a/PC/python3dll.c b/PC/python3dll.c index 4ebb6b481198a8..505feef4b986c4 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -817,7 +817,6 @@ EXPORT_DATA(PyExc_ModuleNotFoundError) EXPORT_DATA(PyExc_NameError) EXPORT_DATA(PyExc_NotADirectoryError) EXPORT_DATA(PyExc_NotImplementedError) -EXPORT_DATA(PyExc_NotWritableError) EXPORT_DATA(PyExc_OSError) EXPORT_DATA(PyExc_OverflowError) EXPORT_DATA(PyExc_PendingDeprecationWarning) @@ -863,7 +862,6 @@ EXPORT_DATA(PyMemoryView_Type) EXPORT_DATA(PyMethodDescr_Type) EXPORT_DATA(PyModule_Type) EXPORT_DATA(PyModuleDef_Type) -EXPORT_DATA(PyNotFreezable_Type) EXPORT_DATA(PyOS_InputHook) EXPORT_DATA(PyProperty_Type) EXPORT_DATA(PyRange_Type) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 04dcd6cdbf91f1..5675189ebde192 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -235,6 +235,7 @@ + @@ -408,6 +409,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 54aa040db67799..2f0888d0a443d8 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -465,7 +465,7 @@ Include - + Include\cpython @@ -609,6 +609,9 @@ Include\internal + + Include\internal + Include\internal @@ -866,6 +869,9 @@ Modules + + Modules + Modules diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c9c7863c70cc79..7f366b43599ae5 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -11,7 +11,6 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ceval.h" // _PyEval_Vector() -#include "pycore_dict.h" // _PyDict_SetGlobalImmutable() #include "clinic/bltinmodule.c.h" @@ -2738,40 +2737,6 @@ builtin_issubclass_impl(PyObject *module, PyObject *cls, return PyBool_FromLong(retval); } -/*[clinic input] -isimmutable as builtin_isimmutable - - obj: object - / - -Return whether 'obj' is immutable. -[clinic start generated code]*/ - -static PyObject * -builtin_isimmutable(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=80c746a3bd7adb46 input=15c0c9d2da47bc15]*/ -{ - return PyBool_FromLong(_Py_IsImmutable(obj)); -} - - -/*[clinic input] -freeze as builtin_freeze - - obj: object - / - -Make 'obj' and its entire reachable object graph immutable. -[clinic start generated code]*/ - -static PyObject * -builtin_freeze(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=c02caad61e252698 input=81af120172984a86]*/ -{ - return Py_Freeze(obj); -} - - typedef struct { PyObject_HEAD Py_ssize_t tuplesize; @@ -3070,8 +3035,6 @@ static PyMethodDef builtin_methods[] = { BUILTIN_INPUT_METHODDEF BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF - BUILTIN_ISIMMUTABLE_METHODDEF - BUILTIN_FREEZE_METHODDEF BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF @@ -3163,7 +3126,6 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("filter", &PyFilter_Type); SETBUILTIN("float", &PyFloat_Type); SETBUILTIN("frozenset", &PyFrozenSet_Type); - SETBUILTIN("notfreezable", &PyNotFreezable_Type); SETBUILTIN("property", &PyProperty_Type); SETBUILTIN("int", &PyLong_Type); SETBUILTIN("list", &PyList_Type); diff --git a/Python/ceval.c b/Python/ceval.c index 9a88c800cb7401..e978434717d8ac 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2739,7 +2739,7 @@ format_exc_notwriteable(PyThreadState *tstate, PyCodeObject *co, int oparg) if (_PyErr_Occurred(tstate)) return; name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); - format_exc_check_arg(tstate, PyExc_NotWritableError, + format_exc_check_arg(tstate, PyExc_TypeError, NOT_WRITEABLE_ERROR_MSG, name); } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index f21a0159aa7440..b77b4a1e4b410e 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1409,22 +1409,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } - -PyDoc_STRVAR(builtin_isimmutable__doc__, -"isimmutable($module, obj, /)\n" -"--\n" -"\n" -"Return whether \'obj\' is immutable."); - -#define BUILTIN_ISIMMUTABLE_METHODDEF \ - {"isimmutable", (PyCFunction)builtin_isimmutable, METH_O, builtin_isimmutable__doc__}, - -PyDoc_STRVAR(builtin_freeze__doc__, -"freeze($module, obj, /)\n" -"--\n" -"\n" -"Make \'obj\' and its entire reachable object graph immutable."); - -#define BUILTIN_FREEZE_METHODDEF \ - {"freeze", (PyCFunction)builtin_freeze, METH_O, builtin_freeze__doc__}, -/*[clinic end generated code: output=2967e6388573aa0b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=84a04e7446debf58 input=a9049054013a1b77]*/ diff --git a/Python/errors.c b/Python/errors.c index c98d86d5660095..58d05a440d0807 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1951,15 +1951,15 @@ PyErr_ProgramTextObject(PyObject *filename, int lineno) } PyObject * -_PyErr_WriteToImmutable(const char* filename, int lineno, PyObject* obj) +_PyErr_WriteToImmutable(PyObject* obj) { PyObject* string; PyThreadState *tstate = _PyThreadState_GET(); if (!_PyErr_Occurred(tstate)) { - string = PyUnicode_FromFormat("object of type %s is immutable at %s:%d", - obj->ob_type->tp_name, filename, lineno); + string = PyUnicode_FromFormat("object of type %s is immutable", + obj->ob_type->tp_name); if (string != NULL) { - _PyErr_SetObject(tstate, PyExc_NotWritableError, string); + _PyErr_SetObject(tstate, PyExc_TypeError, string); Py_DECREF(string); } } diff --git a/Python/immutability.c b/Python/immutability.c index ea16ad70e75c86..a5cf91d9bc9c9b 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -3,8 +3,8 @@ #include #include #include -#include "pycore_dict.h" #include "pycore_object.h" +#include "pycore_immutability.h" PyDoc_STRVAR(notfreezable_doc, @@ -15,7 +15,7 @@ PyDoc_STRVAR(notfreezable_doc, PyTypeObject PyNotFreezable_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "notfreezable", + .tp_name = "NotFreezable", .tp_doc = notfreezable_doc, .tp_basicsize = sizeof(PyObject), .tp_itemsize = 0, @@ -229,107 +229,146 @@ static int freeze_visit(PyObject* obj, void* frontier) return 0; } -static PyObject* get_frozen_importlib(PyInterpreterState* interp) +static bool can_freeze(PyTypeObject* type, PyObject* freezable_types) { - PyObject* frozen_importlib = NULL; - PyObject* interp_dict = PyInterpreterState_GetDict(interp); - if(interp_dict == NULL){ - return NULL; - } + PyObject* type_op = _PyObject_CAST(type); + if(_Py_IsImmutable(type_op)){ + return true; + } + + if(type == &PyType_Type || + type == &PyBaseObject_Type || + type == &PyFunction_Type || + type == &_PyNone_Type || + type == &PyBool_Type || + type == &PyLong_Type || + type == &PyFloat_Type || + type == &PyComplex_Type || + type == &PyBytes_Type || + type == &PyUnicode_Type || + type == &PyTuple_Type || + type == &PyList_Type || + type == &PyDict_Type || + type == &PySet_Type || + type == &PyMemoryView_Type || + type == &PyByteArray_Type || + type == &PyRange_Type || + type == &PyGetSetDescr_Type || + type == &PyMemberDescr_Type || + type == &PyProperty_Type || + type == &PyWrapperDescr_Type || + type == &PyMethodDescr_Type || + type == &PyMethod_Type || + type == &PyCFunction_Type || + type == &PyCapsule_Type || + type == &PyCode_Type || + type == &PyCell_Type || + type == &PyFrame_Type || + type == &_PyWeakref_RefType) + { + return true; + } + + if(type == &PyNotFreezable_Type || PyType_IsSubtype(type, &PyNotFreezable_Type)){ + PyErr_SetString(PyExc_TypeError, "Cannot freeze NotFreezable type"); + return false; + } + + if(freezable_types == NULL){ + return false; + } + + type_op = _PyObject_CAST(type); + if (PySet_Contains(freezable_types, type_op) == 1) { + return true; + } + + if(Py_IsFalse(_PyType_UsesDefaultSlots(type))){ + return false; + } + + return true; +} - frozen_importlib = PyDict_GetItemString(interp_dict, "_frozen_importlib"); - if(frozen_importlib != NULL){ - return frozen_importlib; - } + +static int init_state(struct _Py_immutability_state *state) +{ + PyObject* frozen_importlib = NULL; frozen_importlib = PyImport_ImportModule("_frozen_importlib"); if(frozen_importlib == NULL){ - return NULL; + return -1; } - if(PyDict_SetItemString(interp_dict, "_frozen_importlib", frozen_importlib)){ + state->module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); + if(state->module_locks == NULL){ Py_DECREF(frozen_importlib); - return NULL; - } - - Py_DECREF(frozen_importlib); - return frozen_importlib; -} - -static PyObject* get_blocking_on(PyInterpreterState* interp) -{ - PyObject* frozen_importlib = NULL; - PyObject* blocking_on = NULL; - PyObject* interp_dict = PyInterpreterState_GetDict(interp); - if(interp_dict == NULL){ - return NULL; - } - - blocking_on = PyDict_GetItemString(interp_dict, "_blocking_on"); - if(blocking_on != NULL){ - return blocking_on; + return -1; } - frozen_importlib = get_frozen_importlib(interp); - if(frozen_importlib == NULL){ - return NULL; + state->blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); + if(state->blocking_on == NULL){ + Py_DECREF(frozen_importlib); + return -1; } - blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); - if(blocking_on == NULL){ - return NULL; + state->freezable_types = PySet_New(NULL); + if(state->freezable_types == NULL){ + Py_DECREF(frozen_importlib); + return -1; } - if(PyDict_SetItemString(interp_dict, "_blocking_on", blocking_on)){ - Py_DECREF(blocking_on); - return NULL; + if(PyDict_SetItemString(PyModule_GetDict(frozen_importlib), "_freezable_types", state->freezable_types)){ + Py_DECREF(frozen_importlib); + return -1; } - Py_DECREF(blocking_on); - return blocking_on; + return 0; } -static PyObject* get_module_locks(PyInterpreterState* interp) +static struct _Py_immutability_state* get_immutable_state(PyObject* module) { - PyObject* frozen_importlib = NULL; - PyObject* module_locks = NULL; - PyObject* interp_dict = PyInterpreterState_GetDict(interp); - if(interp_dict == NULL){ - return NULL; + PyInterpreterState* interp = PyInterpreterState_Get(); + struct _Py_immutability_state *state = &interp->immutability; + if(state->freezable_types == NULL){ + if(init_state(state) == -1){ + return NULL; + } } - module_locks = PyDict_GetItemString(interp_dict, "_module_locks"); - if(module_locks != NULL){ - return module_locks; - } + return state; +} - frozen_importlib = get_frozen_importlib(interp); - if(frozen_importlib == NULL){ +PyObject* _PyImmutability_RegisterFreezable(PyObject* obj) +{ + struct _Py_immutability_state *state = get_immutable_state(obj); + if(state == NULL){ + PyErr_SetString(PyExc_RuntimeError, "Failed to initialize immutability state"); return NULL; } - module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); - if(module_locks == NULL){ + if(!PyType_Check(obj)){ + PyErr_SetString(PyExc_TypeError, "Expected a type"); return NULL; } - if(PyDict_SetItemString(interp_dict, "_module_locks", module_locks)){ - Py_DECREF(module_locks); + if(PySet_Add(state->freezable_types, obj) == -1){ return NULL; } - Py_DECREF(module_locks); - return module_locks; + Py_RETURN_NONE; } -PyObject* _Py_Freeze(PyObject* obj) +PyObject* _PyImmutability_Freeze(PyObject* obj) { PyObject* frontier = NULL; - PyObject* blocking_on = NULL; - PyObject* module_locks = NULL; - PyInterpreterState* interp = PyInterpreterState_Get(); PyObject* result = Py_None; + struct _Py_immutability_state* state = get_immutable_state(obj); + if(state == NULL){ + PyErr_SetString(PyExc_RuntimeError, "Failed to initialize immutability state"); + return NULL; + } if(_Py_IsImmutable(obj)){ return result; @@ -346,30 +385,15 @@ PyObject* _Py_Freeze(PyObject* obj) goto cleanup; } - blocking_on = get_blocking_on(interp); - if(blocking_on == NULL){ - result = NULL; - goto cleanup; - } - - module_locks = get_module_locks(interp); - if(module_locks == NULL){ - result = NULL; - goto cleanup; - } - while(PyList_Size(frontier) != 0){ PyObject* item = pop(frontier); - if(item == blocking_on || - item == module_locks){ - // the module lock and blocking on dictionaries must remain mutable or else - // we will not be able to import modules + if(item == state->blocking_on || + item == state->module_locks){ continue; } - if(PyObject_IsInstance(item, (PyObject*)&PyNotFreezable_Type)){ - // the object is not freezable, so we can skip it + if(!can_freeze(item->ob_type, state->freezable_types)){ PyObject* error_msg = PyUnicode_FromFormat("Cannot freeze object of type %s", Py_TYPE(item)->tp_name); PyErr_SetObject(PyExc_TypeError, error_msg); result = NULL; @@ -401,20 +425,14 @@ PyObject* _Py_Freeze(PyObject* obj) result = PyErr_NoMemory(); goto cleanup; } - if(push(frontier, type->tp_mro)) - { - result = PyErr_NoMemory(); - goto cleanup; - } - if(push(frontier, type->tp_bases)) - { - result = PyErr_NoMemory(); - goto cleanup; - } - if(push(frontier, _PyObject_CAST(type->tp_base))) - { - result = PyErr_NoMemory(); - goto cleanup; + + if(PySet_Contains(state->freezable_types, item) != 1){ + // type is not explicit freezable, so we need to check its bases + if(push(frontier, type->tp_mro)) + { + result = PyErr_NoMemory(); + goto cleanup; + } } } else diff --git a/Python/pystate.c b/Python/pystate.c index 2ee16e3de25da3..4f87046ec9e927 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -857,6 +857,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) assert(interp->imports.importlib == NULL); assert(interp->imports.import_func == NULL); + Py_CLEAR(interp->immutability.module_locks); + Py_CLEAR(interp->immutability.blocking_on); + Py_CLEAR(interp->immutability.freezable_types); + Py_CLEAR(interp->sysdict_copy); Py_CLEAR(interp->builtins_copy); Py_CLEAR(interp->dict); diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index ed4a0ac2dd32de..ac8c520b754bd9 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -166,6 +166,7 @@ static const char* _Py_stdlib_module_names[] = { "imaplib", "imghdr", "importlib", +"immutable", "inspect", "io", "ipaddress", diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 1587a19716fe0b..6de9ff683f4866 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -202,6 +202,7 @@ def clean_lines(text): Modules/gcmodule.c Py_BUILD_CORE 1 Modules/getpath.c Py_BUILD_CORE 1 Modules/getpath_noop.c Py_BUILD_CORE 1 +Modules/immutablemodule.c Py_BUILD_CORE 1 Modules/itertoolsmodule.c Py_BUILD_CORE 1 Modules/main.c Py_BUILD_CORE 1 Modules/mathmodule.c Py_BUILD_CORE 1 diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 83afe37cf2440c..6a7c14ebb220a8 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -306,8 +306,6 @@ Modules/timemodule.c init_timezone YEAR - Objects/bytearrayobject.c - _PyByteArray_empty_string - Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - -Objects/exceptions.c - _PyExc_NotWritableError - -Objects/exceptions.c - PyExc_NotWritableError - Objects/genobject.c - ASYNC_GEN_IGNORED_EXIT_MSG - Objects/genobject.c - NON_INIT_CORO_MSG - Objects/longobject.c - _PyLong_DigitValue - diff --git a/configure b/configure index b6f90bcd8c7300..80743b0f9c02af 100755 --- a/configure +++ b/configure @@ -801,6 +801,8 @@ MODULE__LSPROF_FALSE MODULE__LSPROF_TRUE MODULE__JSON_FALSE MODULE__JSON_TRUE +MODULE_IMMUTABLE_FALSE +MODULE_IMMUTABLE_TRUE MODULE__HEAPQ_FALSE MODULE__HEAPQ_TRUE MODULE__CSV_FALSE @@ -28793,6 +28795,28 @@ then : +fi + + + if test "$py_cv_module_immutable" != "n/a" +then : + py_cv_module_immutable=yes +fi + if test "$py_cv_module_immutable" = yes; then + MODULE_IMMUTABLE_TRUE= + MODULE_IMMUTABLE_FALSE='#' +else + MODULE_IMMUTABLE_TRUE='#' + MODULE_IMMUTABLE_FALSE= +fi + + as_fn_append MODULE_BLOCK "MODULE_IMMUTABLE_STATE=$py_cv_module_immutable$as_nl" + if test "x$py_cv_module_immutable" = xyes +then : + + + + fi @@ -31380,6 +31404,10 @@ if test -z "${MODULE__HEAPQ_TRUE}" && test -z "${MODULE__HEAPQ_FALSE}"; then as_fn_error $? "conditional \"MODULE__HEAPQ\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE_IMMUTABLE_TRUE}" && test -z "${MODULE_IMMUTABLE_FALSE}"; then + as_fn_error $? "conditional \"MODULE_IMMUTABLE\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__JSON_TRUE}" && test -z "${MODULE__JSON_FALSE}"; then as_fn_error $? "conditional \"MODULE__JSON\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index ba768aea930714..95f8f60f56503e 100644 --- a/configure.ac +++ b/configure.ac @@ -7336,6 +7336,7 @@ PY_STDLIB_MOD_SIMPLE([_bisect]) PY_STDLIB_MOD_SIMPLE([_contextvars]) PY_STDLIB_MOD_SIMPLE([_csv]) PY_STDLIB_MOD_SIMPLE([_heapq]) +PY_STDLIB_MOD_SIMPLE([immutable]) PY_STDLIB_MOD_SIMPLE([_json]) PY_STDLIB_MOD_SIMPLE([_lsprof]) PY_STDLIB_MOD_SIMPLE([_opcode]) From a4d42da67c971d1964339b9a81d0f29fa7052618 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Tue, 29 Apr 2025 16:13:14 +0100 Subject: [PATCH 28/40] All tests passing again Signed-off-by: Matthew A Johnson --- Include/cpython/immutability.h | 7 +- Include/cpython/object.h | 2 +- Lib/test/test_freeze/__init__.py | 31 -- Lib/test/test_freeze/test_abc.py | 82 ---- Lib/test/test_freeze/test_array.py | 2 +- Lib/test/test_freeze/test_asyncio.py | 22 - Lib/test/test_freeze/test_bz2.py | 7 +- Lib/test/test_freeze/test_collections.py | 3 +- Lib/test/test_freeze/test_common.py | 36 ++ Lib/test/test_freeze/test_core.py | 3 +- Lib/test/test_freeze/test_csv.py | 2 +- Lib/test/test_freeze/test_ctypes.py | 16 +- Lib/test/test_freeze/test_decimal.py | 2 +- Lib/test/test_freeze/test_etree.py | 8 +- Lib/test/test_freeze/test_hashlib.py | 1 + Lib/test/test_freeze/test_io.py | 2 +- Lib/test/test_freeze/test_multiprocessing.py | 2 +- Lib/test/test_freeze/test_sqllite3.py | 2 +- Lib/test/test_freeze/test_xxlimited.py | 41 -- Lib/test/test_freeze/test_xxsubtype.py | 15 - Lib/test/test_freeze/test_zlib.py | 2 +- Modules/_blake2/blake2module.c | 18 + Modules/_collectionsmodule.c | 8 + Modules/_ctypes/_ctypes.c | 8 +- Modules/_decimal/_decimal.c | 3 + Modules/_elementtree.c | 11 + Modules/arraymodule.c | 5 + Modules/clinic/immutablemodule.c.h | 11 +- Modules/immutablemodule.c | 45 +- Objects/dictobject.c | 6 +- Objects/memoryobject.c | 2 +- Objects/typeobject.c | 489 +++++++++++++++---- Python/immutability.c | 299 +++++++----- 33 files changed, 756 insertions(+), 437 deletions(-) delete mode 100644 Lib/test/test_freeze/test_abc.py delete mode 100644 Lib/test/test_freeze/test_asyncio.py create mode 100644 Lib/test/test_freeze/test_common.py delete mode 100644 Lib/test/test_freeze/test_xxlimited.py delete mode 100644 Lib/test/test_freeze/test_xxsubtype.py diff --git a/Include/cpython/immutability.h b/Include/cpython/immutability.h index c6729243fede06..2d203c0fd03fc7 100644 --- a/Include/cpython/immutability.h +++ b/Include/cpython/immutability.h @@ -2,7 +2,8 @@ # error "this header file must not be included directly" #endif -PyAPI_DATA(PyTypeObject) PyNotFreezable_Type; +PyAPI_DATA(PyTypeObject) _PyNotFreezable_Type; -PyAPI_FUNC(PyObject *) _PyImmutability_Freeze(PyObject*); -PyAPI_FUNC(PyObject *) _PyImmutability_RegisterFreezable(PyObject*); +PyAPI_FUNC(int) _PyImmutability_Freeze(PyObject*); +PyAPI_FUNC(int) _PyImmutability_RegisterFreezable(PyTypeObject*); +PyAPI_FUNC(int) _PyImmutability_IsFreezable(PyObject*); diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 50b34253039203..37f19c21b7ba5a 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -273,7 +273,7 @@ typedef struct _heaptypeobject { PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *); -PyAPI_FUNC(PyObject *) _PyType_UsesDefaultSlots(PyTypeObject *); +PyAPI_FUNC(int) _PyType_HasExtensionSlots(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); #ifndef Py_BUILD_CORE // Backward compatibility for 3rd-party extensions diff --git a/Lib/test/test_freeze/__init__.py b/Lib/test/test_freeze/__init__.py index 285e31be224866..ca273763bed98d 100644 --- a/Lib/test/test_freeze/__init__.py +++ b/Lib/test/test_freeze/__init__.py @@ -1,37 +1,6 @@ import os from test.support import load_package_tests -import unittest -from immutable import freeze, isfrozen, NotFreezable def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) - - -class BaseObjectTest(unittest.TestCase): - def __init__(self, *args, obj=None, **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - self.obj = obj - - def setUp(self): - freeze(self.obj) - - def test_immutable(self): - self.assertTrue(isfrozen(self.obj)) - - def test_add_attribute(self): - with self.assertRaises(TypeError): - self.obj.new_attribute = 'value' - - def test_type_immutable(self): - self.assertTrue(isfrozen(type(self.obj))) - - -class BaseNotFreezableTest(unittest.TestCase): - def __init__(self, *args, obj=NotFreezable(), **kwargs): - unittest.TestCase.__init__(self, *args, **kwargs) - self.obj = obj - - def test_not_freezable(self): - with self.assertRaises(TypeError): - freeze(self.obj) diff --git a/Lib/test/test_freeze/test_abc.py b/Lib/test/test_freeze/test_abc.py deleted file mode 100644 index 523122c769d449..00000000000000 --- a/Lib/test/test_freeze/test_abc.py +++ /dev/null @@ -1,82 +0,0 @@ -from abc import ABC, abstractmethod - - -from . import BaseObjectTest - - -class TestABC(BaseObjectTest): - def __init__(self, *args, **kwargs): - class A(ABC): - @abstractmethod - def foo(self): - pass - - class B(A): - def foo(self): - print("foo") - - self.A = A - super().__init__(*args, obj=B(), **kwargs) - - def test_abstract_immutable(self): - self.assertTrue(isimmutable(self.A)) - - def test_register(self): - class C(ABC): - @abstractmethod - def bar(self): - pass - - with self.assertRaises(TypeError): - self.A.register(C) - - def test_invalid_cache(self): - class D(ABC): - @abstractmethod - def baz(self): - pass - - class E(ABC): - @abstractmethod - def qux(self): - pass - - self.assertFalse(issubclass(E, D)) - - D.register(E) - - class F(D): - def baz(self): - pass - - x = F() - freeze(x) - self.assertTrue(isimmutable(D)) - with self.assertRaises(TypeError): - # the caches are invalidated but cannot be updated - # because the class is frozen - isinstance(x, D) - - def test_valid_cache(self): - class DD(ABC): - @abstractmethod - def baz(self): - pass - - class EE(ABC): - @abstractmethod - def qux(self): - pass - - DD.register(EE) - - self.assertTrue(issubclass(EE, DD)) - - class FF(DD): - def baz(self): - pass - - x = FF() - freeze(x) - self.assertTrue(isimmutable(DD)) - self.assertTrue(isinstance(x, DD)) diff --git a/Lib/test/test_freeze/test_array.py b/Lib/test/test_freeze/test_array.py index fa6bbb0b6f77c5..9aa3cf539d36f6 100644 --- a/Lib/test/test_freeze/test_array.py +++ b/Lib/test/test_freeze/test_array.py @@ -1,5 +1,5 @@ from array import array -from . import BaseObjectTest +from .test_common import BaseObjectTest class TestArray(BaseObjectTest): diff --git a/Lib/test/test_freeze/test_asyncio.py b/Lib/test/test_freeze/test_asyncio.py deleted file mode 100644 index 3d7d70b273c3da..00000000000000 --- a/Lib/test/test_freeze/test_asyncio.py +++ /dev/null @@ -1,22 +0,0 @@ -import asyncio -import unittest - - -class TestFuture(unittest.TestCase): - def test_future(self): - async def set_after(fut, delay, value): - await asyncio.sleep(delay) - fut.set_result(value) - - async def main(): - loop = asyncio.get_running_loop() - fut = loop.create_future() - loop.create_task( - set_after(fut, .1, '... world')) - - with self.assertRaises(TypeError): - freeze(fut) - - await fut - - asyncio.run(main()) diff --git a/Lib/test/test_freeze/test_bz2.py b/Lib/test/test_freeze/test_bz2.py index 9d2154f9cc050b..c37c2e0985d2eb 100644 --- a/Lib/test/test_freeze/test_bz2.py +++ b/Lib/test/test_freeze/test_bz2.py @@ -1,6 +1,7 @@ from bz2 import BZ2Compressor, BZ2Decompressor +import unittest -from . import BaseNotFreezableTest +from .test_common import BaseNotFreezableTest class TestBZ2Compressor(BaseNotFreezableTest): @@ -11,3 +12,7 @@ def __init__(self, *args, **kwargs): class TestBZ2Decompressor(BaseNotFreezableTest): def __init__(self, *args, **kwargs): super().__init__(*args, obj=BZ2Decompressor(), **kwargs) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_freeze/test_collections.py b/Lib/test/test_freeze/test_collections.py index dba48eed747b89..5ccfe19a85f036 100644 --- a/Lib/test/test_freeze/test_collections.py +++ b/Lib/test/test_freeze/test_collections.py @@ -1,6 +1,7 @@ from collections import defaultdict, deque +from immutable import freeze -from . import BaseObjectTest +from .test_common import BaseObjectTest class TestDeque(BaseObjectTest): diff --git a/Lib/test/test_freeze/test_common.py b/Lib/test/test_freeze/test_common.py new file mode 100644 index 00000000000000..b5fe14b9ac72ee --- /dev/null +++ b/Lib/test/test_freeze/test_common.py @@ -0,0 +1,36 @@ +import unittest +from immutable import freeze, isfrozen, isfreezable, NotFreezable + + +class BaseObjectTest(unittest.TestCase): + def __init__(self, *args, obj=None, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + self.obj = obj + + def setUp(self): + self.assertTrue(isfreezable(self.obj)) + freeze(self.obj) + + def test_immutable(self): + self.assertTrue(isfrozen(self.obj)) + + def test_add_attribute(self): + with self.assertRaises(TypeError): + self.obj.new_attribute = 'value' + + def test_type_immutable(self): + self.assertTrue(isfrozen(type(self.obj))) + + +class BaseNotFreezableTest(unittest.TestCase): + def __init__(self, *args, obj=NotFreezable(), **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + self.obj = obj + + def test_not_freezable(self): + with self.assertRaises(TypeError): + freeze(self.obj) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_freeze/test_core.py b/Lib/test/test_freeze/test_core.py index b063fd862f849d..33d0053c00ca6e 100644 --- a/Lib/test/test_freeze/test_core.py +++ b/Lib/test/test_freeze/test_core.py @@ -1,7 +1,7 @@ import unittest from immutable import freeze, isfrozen -from . import BaseObjectTest +from .test_common import BaseObjectTest # This is a canary to check that global variables are not made immutable @@ -575,5 +575,6 @@ def f(a, **b): self.assertTrue(isfrozen(bdef)) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_freeze/test_csv.py b/Lib/test/test_freeze/test_csv.py index 14271761cb8ea1..e04ada5401fbf0 100644 --- a/Lib/test/test_freeze/test_csv.py +++ b/Lib/test/test_freeze/test_csv.py @@ -1,7 +1,7 @@ import csv from io import BytesIO -from . import BaseNotFreezableTest +from .test_common import BaseNotFreezableTest class TestCSVReader(BaseNotFreezableTest): diff --git a/Lib/test/test_freeze/test_ctypes.py b/Lib/test/test_freeze/test_ctypes.py index fb005eb45d7918..2bfb2b166c6b39 100644 --- a/Lib/test/test_freeze/test_ctypes.py +++ b/Lib/test/test_freeze/test_ctypes.py @@ -1,6 +1,8 @@ import ctypes +import unittest +from immutable import freeze, isfrozen, isfrozen -from . import BaseObjectTest +from .test_common import BaseObjectTest class TestCharArray(BaseObjectTest): @@ -54,8 +56,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=ctypes.pointer(self.a), **kwargs) def test_contents_immutable(self): - self.assertTrue(isimmutable(self.a)) - self.assertTrue(isimmutable(TestPointer.POINT)) + self.assertTrue(isfrozen(self.a)) + self.assertTrue(isfrozen(TestPointer.POINT)) def test_set_contents(self): b = TestPointer.POINT(3, 4) @@ -75,8 +77,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, obj=TenPointsArrayType(), **kwargs) def test_point_immutable(self): - self.assertTrue(isimmutable(self.obj[0])) - self.assertTrue(isimmutable(TestArray.POINT)) + self.assertTrue(isfrozen(self.obj[0])) + self.assertTrue(isfrozen(TestArray.POINT)) def test_modify_item(self): with self.assertRaises(TypeError): @@ -128,3 +130,7 @@ def test_assign_value(self): self.assertEqual(self.obj.value, 0x00FF00FF) self.assertEqual(self.obj.parts.high, 0xFF) self.assertEqual(self.obj.parts.low, 0xFF) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_freeze/test_decimal.py b/Lib/test/test_freeze/test_decimal.py index be2df236657f74..15382dbd7b39e3 100644 --- a/Lib/test/test_freeze/test_decimal.py +++ b/Lib/test/test_freeze/test_decimal.py @@ -1,6 +1,6 @@ import decimal -from . import BaseObjectTest +from .test_common import BaseObjectTest class TestContext(BaseObjectTest): diff --git a/Lib/test/test_freeze/test_etree.py b/Lib/test/test_freeze/test_etree.py index 2cf7db0afc3b14..2932178029ab97 100644 --- a/Lib/test/test_freeze/test_etree.py +++ b/Lib/test/test_freeze/test_etree.py @@ -1,8 +1,9 @@ +from immutable import freeze from xml.etree.ElementTree import ElementTree, Element, XMLParser import unittest -from . import BaseNotFreezableTest, BaseObjectTest +from .test_common import BaseNotFreezableTest, BaseObjectTest class TestElementTree(BaseNotFreezableTest): @@ -48,11 +49,6 @@ def test_remove(self): with self.assertRaises(TypeError): self.obj.remove(Element("child")) - def test_iter(self): - it = self.obj.iter() - with self.assertRaises(TypeError): - freeze(it) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_freeze/test_hashlib.py b/Lib/test/test_freeze/test_hashlib.py index 2eb5742d79744b..d64baff6b7a337 100644 --- a/Lib/test/test_freeze/test_hashlib.py +++ b/Lib/test/test_freeze/test_hashlib.py @@ -1,4 +1,5 @@ from hashlib import blake2b, blake2s +from immutable import freeze import unittest diff --git a/Lib/test/test_freeze/test_io.py b/Lib/test/test_freeze/test_io.py index f07dbe30ce752c..d7c0ac950f034d 100644 --- a/Lib/test/test_freeze/test_io.py +++ b/Lib/test/test_freeze/test_io.py @@ -1,6 +1,6 @@ import io -from . import BaseNotFreezableTest +from .test_common import BaseNotFreezableTest class BytesIOTest(BaseNotFreezableTest): diff --git a/Lib/test/test_freeze/test_multiprocessing.py b/Lib/test/test_freeze/test_multiprocessing.py index 60a7f2b82cf6fc..9cce1d2377f756 100644 --- a/Lib/test/test_freeze/test_multiprocessing.py +++ b/Lib/test/test_freeze/test_multiprocessing.py @@ -1,6 +1,6 @@ from _multiprocessing import SemLock -from . import BaseNotFreezableTest +from .test_common import BaseNotFreezableTest SEMAPHORE = 1 SEM_VALUE_MAX = SemLock.SEM_VALUE_MAX diff --git a/Lib/test/test_freeze/test_sqllite3.py b/Lib/test/test_freeze/test_sqllite3.py index 540e74701714a6..60db5fa4e96661 100644 --- a/Lib/test/test_freeze/test_sqllite3.py +++ b/Lib/test/test_freeze/test_sqllite3.py @@ -4,7 +4,7 @@ import sqlite3 -from . import BaseNotFreezableTest +from .test_common import BaseNotFreezableTest class TestConnection(BaseNotFreezableTest): diff --git a/Lib/test/test_freeze/test_xxlimited.py b/Lib/test/test_freeze/test_xxlimited.py deleted file mode 100644 index b51e0957fea55a..00000000000000 --- a/Lib/test/test_freeze/test_xxlimited.py +++ /dev/null @@ -1,41 +0,0 @@ -import unittest -from test.support import import_helper -import types - -xxlimited = import_helper.import_module('xxlimited') -xxlimited_35 = import_helper.import_module('xxlimited_35') - -class CommonTests: - module: types.ModuleType - - def test_xxo_set_attribute(self): - xxo = self.module.Xxo() - - freeze(xxo) - - with self.assertRaises(TypeError): - xxo.foo = 1234 - - def test_xxo_del_attribute(self): - xxo = self.module.Xxo() - - freeze(xxo) - - with self.assertRaises(TypeError): - del xxo.foo - - -class TestXXLimited(CommonTests, unittest.TestCase): - module = xxlimited - - def test_buffer(self): - xxo = self.module.Xxo() - - freeze(xxo) - - # Creating a buffer into immutable memory should be fine. - b1 = memoryview(xxo) - del b1 - -class TestXXLimited35(CommonTests, unittest.TestCase): - module = xxlimited_35 diff --git a/Lib/test/test_freeze/test_xxsubtype.py b/Lib/test/test_freeze/test_xxsubtype.py deleted file mode 100644 index c2ca6e6b3906b8..00000000000000 --- a/Lib/test/test_freeze/test_xxsubtype.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest -from test.support import import_helper - -xxsubtype = import_helper.import_module('xxsubtype') - -class xxsubtypelib(unittest.TestCase): - - def test_spamdict_setstate(self): - import xxsubtype as spam - a = spam.spamlist() - - freeze(a) - - with self.assertRaises(TypeError): - a.setstate(17) diff --git a/Lib/test/test_freeze/test_zlib.py b/Lib/test/test_freeze/test_zlib.py index e2035ac7497e8f..2c028e461f666e 100644 --- a/Lib/test/test_freeze/test_zlib.py +++ b/Lib/test/test_freeze/test_zlib.py @@ -1,6 +1,6 @@ import zlib -from . import BaseNotFreezableTest +from .test_common import BaseNotFreezableTest class ZlibCompressTest(BaseNotFreezableTest): def __init__(self, *args, **kwargs): diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 0d1d88c6603684..cdc2147f2c008f 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -89,6 +89,7 @@ blake2_exec(PyObject *m) return -1; } + PyObject *d = st->blake2b_type->tp_dict; ADD_INT(d, "SALT_SIZE", BLAKE2B_SALTBYTES); ADD_INT(d, "PERSON_SIZE", BLAKE2B_PERSONALBYTES); @@ -111,6 +112,23 @@ blake2_exec(PyObject *m) return -1; } + PyObject *register_freezable = _PyImport_GetModuleAttrString("immutable", "register_freezable"); + if(register_freezable != NULL){ + PyObject *result = PyObject_CallFunctionObjArgs(register_freezable, st->blake2b_type, NULL); + if(result == NULL){ + Py_DECREF(register_freezable); + return -1; + } + + result = PyObject_CallFunctionObjArgs(register_freezable, st->blake2s_type, NULL); + if(result == NULL){ + Py_DECREF(register_freezable); + return -1; + } + + Py_DECREF(register_freezable); + } + d = st->blake2s_type->tp_dict; ADD_INT(d, "SALT_SIZE", BLAKE2S_SALTBYTES); ADD_INT(d, "PERSON_SIZE", BLAKE2S_PERSONALBYTES); diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c27338818f5968..b084728b5e0b51 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2617,6 +2617,14 @@ collections_exec(PyObject *module) { ADD_TYPE(module, &dequereviter_spec, state->dequereviter_type, NULL); ADD_TYPE(module, &tuplegetter_spec, state->tuplegetter_type, NULL); + if(_PyImmutability_RegisterFreezable(state->deque_type) < 0){ + return -1; + } + + if(_PyImmutability_RegisterFreezable(state->defdict_type) < 0){ + return -1; + } + if (PyModule_AddType(module, &PyODict_Type) < 0) { return -1; } diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 50b4795002eba6..0ff1df7942e39b 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3016,7 +3016,7 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) } if(base && _Py_IsImmutable(base)) { - if(_PyImmutability_Freeze(_PyObject_CAST(cmem)) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(cmem)) < 0){ Py_DECREF(cmem); return NULL; } @@ -5718,6 +5718,9 @@ _ctypes_add_types(PyObject *mod) PyTypeObject *type = (TYPE_EXPR); \ type->tp_base = (TP_BASE); \ TYPE_READY(type); \ + if(_PyImmutability_RegisterFreezable(type) < 0){ \ + return -1; \ + } \ } while (0) #define MOD_ADD_TYPE(TYPE_EXPR, TP_TYPE, TP_BASE) \ @@ -5780,6 +5783,9 @@ _ctypes_add_types(PyObject *mod) */ CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec, NULL); + if(_PyImmutability_RegisterFreezable(st->PyCField_Type) < 0){ + return -1; + } /************************************************* * diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index ee866b45190d98..238e071c8acc96 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -5991,6 +5991,9 @@ PyInit__decimal(void) CHECK_INT(PyModule_AddObjectRef(m, "Context", (PyObject *)&PyDecContext_Type)); CHECK_INT(PyModule_AddObjectRef(m, "DecimalTuple", (PyObject *)DecimalTuple)); + CHECK_INT(_PyImmutability_RegisterFreezable(&PyDec_Type)); + CHECK_INT(_PyImmutability_RegisterFreezable(&PyDecContext_Type)); + /* Create top level exception */ ASSIGN_PTR(DecimalException, PyErr_NewException( "decimal.DecimalException", diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index ee85201eeb475d..f1173eb86a5486 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4382,6 +4382,17 @@ module_exec(PyObject *m) CREATE_TYPE(m, st->Element_Type, &element_spec, NULL); CREATE_TYPE(m, st->XMLParser_Type, &xmlparser_spec, NULL); + PyObject *register_freezable = _PyImport_GetModuleAttrString("immutable", "register_freezable"); + if(register_freezable != NULL) + { + PyObject* result = PyObject_CallOneArg(register_freezable, (PyObject *)st->Element_Type); + if(result == NULL){ + goto error; + } + + Py_DECREF(register_freezable); + } + st->deepcopy_obj = _PyImport_GetModuleAttrString("copy", "deepcopy"); if (st->deepcopy_obj == NULL) { goto error; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 97dd0b6ecb42a2..14b6dd281bb470 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3137,6 +3137,11 @@ array_modexec(PyObject *m) return -1; } + if(_PyImmutability_RegisterFreezable(state->ArrayType) < 0){ + Py_DECREF((PyObject *)state->ArrayType); + return -1; + } + PyObject *mutablesequence = _PyImport_GetModuleAttrString( "collections.abc", "MutableSequence"); if (!mutablesequence) { diff --git a/Modules/clinic/immutablemodule.c.h b/Modules/clinic/immutablemodule.c.h index fea1b2373e3af1..9218704d2803a7 100644 --- a/Modules/clinic/immutablemodule.c.h +++ b/Modules/clinic/immutablemodule.c.h @@ -34,4 +34,13 @@ PyDoc_STRVAR(immutable_isfrozen__doc__, #define IMMUTABLE_ISFROZEN_METHODDEF \ {"isfrozen", (PyCFunction)immutable_isfrozen, METH_O, immutable_isfrozen__doc__}, -/*[clinic end generated code: output=0edcea6c15426dec input=a9049054013a1b77]*/ + +PyDoc_STRVAR(immutable_isfreezable__doc__, +"isfreezable($module, obj, /)\n" +"--\n" +"\n" +"Check if an object can be frozen."); + +#define IMMUTABLE_ISFREEZABLE_METHODDEF \ + {"isfreezable", (PyCFunction)immutable_isfreezable, METH_O, immutable_isfreezable__doc__}, +/*[clinic end generated code: output=b26b154bf5fbe7c9 input=a9049054013a1b77]*/ diff --git a/Modules/immutablemodule.c b/Modules/immutablemodule.c index d0e5fdb1001a00..bc6178b2b0c90b 100644 --- a/Modules/immutablemodule.c +++ b/Modules/immutablemodule.c @@ -65,7 +65,16 @@ static PyObject * immutable_register_freezable(PyObject *module, PyObject *obj) /*[clinic end generated code: output=1afbb9a860e2bde9 input=fbb7f42f02d27a88]*/ { - return _PyImmutability_RegisterFreezable(obj); + if(!PyType_Check(obj)){ + PyErr_SetString(PyExc_TypeError, "Expected a type"); + return NULL; + } + + if(_PyImmutability_RegisterFreezable((PyTypeObject *)obj) < 0){ + return NULL; + } + + Py_RETURN_NONE; } /*[clinic input] @@ -80,7 +89,11 @@ static PyObject * immutable_freeze(PyObject *module, PyObject *obj) /*[clinic end generated code: output=76b9e6c577ec3841 input=d7090b2d52afbb4b]*/ { - return _PyImmutability_Freeze(obj); + if(_PyImmutability_Freeze(obj) < 0){ + return NULL; + } + + Py_RETURN_NONE; } /*[clinic input] @@ -102,6 +115,30 @@ immutable_isfrozen(PyObject *module, PyObject *obj) Py_RETURN_FALSE; } +/*[clinic input] +immutable.isfreezable + obj: object + / + +Check if an object can be frozen. +[clinic start generated code]*/ + +static PyObject * +immutable_isfreezable(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=ddc1a85bd9279882 input=9b06ae134ca053ea]*/ +{ + int result = _PyImmutability_IsFreezable(obj); + if(result == -1){ + return NULL; + } + + if(result){ + Py_RETURN_TRUE; + } + + Py_RETURN_FALSE; +} + static PyType_Slot not_freezable_error_slots[] = { {0, NULL}, }; @@ -123,13 +160,13 @@ static struct PyMethodDef immutable_methods[] = { IMMUTABLE_REGISTER_FREEZABLE_METHODDEF IMMUTABLE_FREEZE_METHODDEF IMMUTABLE_ISFROZEN_METHODDEF + IMMUTABLE_ISFREEZABLE_METHODDEF { NULL, NULL } }; static int immutable_exec(PyObject *module) { - PyObject* frozen_importlib = NULL; immutable_state *module_state = get_immutable_state(module); /* Add version to the module. */ @@ -153,7 +190,7 @@ immutable_exec(PyObject *module) { return -1; } - if (PyModule_AddType(module, &PyNotFreezable_Type) != 0) { + if (PyModule_AddType(module, &_PyNotFreezable_Type) != 0) { return -1; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 50e1e562aab9c0..ae95a58b0c582e 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5685,7 +5685,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) interp, CACHED_KEYS(tp), values); if (dict != NULL) { if (_Py_IsImmutable(obj)) { - if(_PyImmutability_Freeze(_PyObject_CAST(dict)) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(dict)) < 0){ return NULL; } } @@ -5700,7 +5700,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dictkeys_incref(CACHED_KEYS(tp)); dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); if (_Py_IsImmutable(obj)) { - if(_PyImmutability_Freeze(_PyObject_CAST(dict)) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(dict)) < 0){ return NULL; } } @@ -5729,7 +5729,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = PyDict_New(); } if (_Py_IsImmutable(obj)) { - if(_PyImmutability_Freeze(_PyObject_CAST(dict)) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(dict)) < 0){ return NULL; } } diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index c31b56f67d72ae..cc91cb125452c3 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -100,7 +100,7 @@ _PyManagedBuffer_FromObject(PyObject *base, int flags) } if(_Py_IsImmutable(base)){ - if(_PyImmutability_Freeze(_PyObject_CAST(mbuf)) == NULL){ + if(_PyImmutability_Freeze(_PyObject_CAST(mbuf)) < 0){ PyBuffer_Release(&mbuf->master); Py_DECREF(mbuf); return NULL; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f8fea715f31223..02f5f808f1ac11 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6615,92 +6615,6 @@ PyTypeObject PyBaseObject_Type = { }; -PyObject * -_PyType_UsesDefaultSlots(PyTypeObject *tp) -{ - PyNumberMethods *nb = tp->tp_as_number; - PySequenceMethods *sq = tp->tp_as_sequence; - PyMappingMethods *mp = tp->tp_as_mapping; - PyBufferProcs *bp = tp->tp_as_buffer; - - if(!(nb == NULL || - (nb->nb_index == NULL && - nb->nb_int == NULL && - nb->nb_float == NULL))) - { - printf("custom number slots\n"); - Py_RETURN_FALSE; - } - - if(!(sq == NULL || sq->sq_item == NULL)) - { - printf("custom sequence slots\n"); - Py_RETURN_FALSE; - } - - if(!(mp == NULL || mp->mp_subscript == NULL)) - { - printf("custom mapping slots\n"); - Py_RETURN_FALSE; - } - - if(!(bp == NULL || bp->bf_getbuffer == NULL)) - { - printf("custom buffer slots\n"); - Py_RETURN_FALSE; - } - - if(tp->tp_getattr != NULL || - tp->tp_setattr != NULL || - tp->tp_methods != NULL) - { - printf("custom slots\n"); - Py_RETURN_FALSE; - } - - if (!(tp->tp_setattro == PyObject_GenericSetAttr || tp->tp_setattro == NULL)){ - printf("custom setattro\n"); - Py_RETURN_FALSE; - } - - if (!(tp->tp_getattro == PyObject_GenericGetAttr || tp->tp_getattro == NULL)){ - printf("custom getattro\n"); - Py_RETURN_FALSE; - } - - if(!(tp->tp_getset == NULL || - tp->tp_getset == subtype_getsets_full || - tp->tp_getset == subtype_getsets_weakref_only || - tp->tp_getset == subtype_getsets_dict_only)) - { - printf("custom getset\n"); - Py_RETURN_FALSE; - } - - if(!(tp->tp_str == NULL || tp->tp_str == object_str)){ - printf("custom str\n"); - Py_RETURN_FALSE; - } - - if(!(tp->tp_repr == NULL || tp->tp_repr == object_repr)){ - printf("custom repr\n"); - Py_RETURN_FALSE; - } - - if(!(tp->tp_hash == NULL || tp->tp_hash == (hashfunc)_Py_HashPointer)){ - printf("custom hash\n"); - Py_RETURN_FALSE; - } - - if(!(tp->tp_richcompare == NULL || tp->tp_richcompare == object_richcompare)){ - printf("custom richcompare\n"); - Py_RETURN_FALSE; - } - - Py_RETURN_TRUE; -} - - static int type_add_method(PyTypeObject *type, PyMethodDef *meth) { @@ -10768,3 +10682,406 @@ PyTypeObject PySuper_Type = { PyObject_GC_Del, /* tp_free */ .tp_vectorcall = (vectorcallfunc)super_vectorcall, }; + + +int +_PyType_HasExtensionSlots(PyTypeObject *tp) +{ + PyNumberMethods *nb = tp->tp_as_number; + PySequenceMethods *sq = tp->tp_as_sequence; + PyMappingMethods *mp = tp->tp_as_mapping; + PyAsyncMethods *am = tp->tp_as_async; + PyBufferProcs *bf = tp->tp_as_buffer; + Py_ssize_t mro_size = PyTuple_GET_SIZE(tp->tp_mro); + + #define EXT_FLAG(PREFIX, NAME) bool NAME##_ext = PREFIX->PREFIX##_##NAME != NULL + #define SLOT_EXT_FLAG(PREFIX, NAME) bool NAME##_ext = !(PREFIX->PREFIX##_##NAME == NULL || PREFIX->PREFIX##_##NAME == slot_##PREFIX##_##NAME) + #define EXT_TEST(PREFIX, NAME) if(PREFIX->PREFIX##_##NAME == base_##PREFIX->PREFIX##_##NAME){NAME##_ext = false;} + + if(!(nb == NULL || + (nb->nb_index == NULL && + nb->nb_int == NULL && + nb->nb_float == NULL))) + { + SLOT_EXT_FLAG(nb, add); + SLOT_EXT_FLAG(nb, subtract); + SLOT_EXT_FLAG(nb, multiply); + SLOT_EXT_FLAG(nb, floor_divide); + SLOT_EXT_FLAG(nb, true_divide); + SLOT_EXT_FLAG(nb, remainder); + SLOT_EXT_FLAG(nb, divmod); + SLOT_EXT_FLAG(nb, power); + SLOT_EXT_FLAG(nb, lshift); + SLOT_EXT_FLAG(nb, rshift); + SLOT_EXT_FLAG(nb, and); + SLOT_EXT_FLAG(nb, xor); + SLOT_EXT_FLAG(nb, or); + SLOT_EXT_FLAG(nb, matrix_multiply); + SLOT_EXT_FLAG(nb, index); + SLOT_EXT_FLAG(nb, int); + SLOT_EXT_FLAG(nb, float); + SLOT_EXT_FLAG(nb, inplace_add); + SLOT_EXT_FLAG(nb, inplace_subtract); + SLOT_EXT_FLAG(nb, inplace_multiply); + SLOT_EXT_FLAG(nb, inplace_floor_divide); + SLOT_EXT_FLAG(nb, inplace_true_divide); + SLOT_EXT_FLAG(nb, inplace_remainder); + SLOT_EXT_FLAG(nb, inplace_power); + SLOT_EXT_FLAG(nb, inplace_lshift); + SLOT_EXT_FLAG(nb, inplace_rshift); + SLOT_EXT_FLAG(nb, inplace_and); + SLOT_EXT_FLAG(nb, inplace_xor); + SLOT_EXT_FLAG(nb, inplace_or); + SLOT_EXT_FLAG(nb, inplace_matrix_multiply); + + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(base->tp_as_number != NULL) + { + PyNumberMethods *base_nb = base->tp_as_number; + EXT_TEST(nb, add); + EXT_TEST(nb, subtract); + EXT_TEST(nb, multiply); + EXT_TEST(nb, floor_divide); + EXT_TEST(nb, true_divide); + EXT_TEST(nb, remainder); + EXT_TEST(nb, divmod); + EXT_TEST(nb, power); + EXT_TEST(nb, lshift); + EXT_TEST(nb, rshift); + EXT_TEST(nb, and); + EXT_TEST(nb, xor); + EXT_TEST(nb, or); + EXT_TEST(nb, matrix_multiply); + EXT_TEST(nb, index); + EXT_TEST(nb, int); + EXT_TEST(nb, float); + EXT_TEST(nb, inplace_add); + EXT_TEST(nb, inplace_subtract); + EXT_TEST(nb, inplace_multiply); + EXT_TEST(nb, inplace_floor_divide); + EXT_TEST(nb, inplace_true_divide); + EXT_TEST(nb, inplace_remainder); + EXT_TEST(nb, inplace_power); + EXT_TEST(nb, inplace_lshift); + EXT_TEST(nb, inplace_rshift); + EXT_TEST(nb, inplace_and); + EXT_TEST(nb, inplace_xor); + EXT_TEST(nb, inplace_or); + EXT_TEST(nb, inplace_matrix_multiply); + } + } + + if(add_ext || + subtract_ext || + multiply_ext || + floor_divide_ext || + true_divide_ext || + remainder_ext || + divmod_ext || + power_ext || + lshift_ext || + rshift_ext || + and_ext || + xor_ext || + or_ext || + matrix_multiply_ext || + index_ext || + int_ext || + float_ext || + inplace_add_ext || + inplace_subtract_ext || + inplace_multiply_ext || + inplace_floor_divide_ext || + inplace_true_divide_ext || + inplace_remainder_ext || + inplace_power_ext || + inplace_lshift_ext || + inplace_rshift_ext || + inplace_and_ext || + inplace_xor_ext || + inplace_or_ext || + inplace_matrix_multiply_ext) + { + return 1; + } + } + + if(!(sq == NULL || sq->sq_item == NULL)) + { + SLOT_EXT_FLAG(sq, length); + EXT_FLAG(sq, concat); + EXT_FLAG(sq, repeat); + SLOT_EXT_FLAG(sq, item); + SLOT_EXT_FLAG(sq, ass_item); + SLOT_EXT_FLAG(sq, contains); + EXT_FLAG(sq, inplace_concat); + EXT_FLAG(sq, inplace_repeat); + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(base->tp_as_sequence != NULL) + { + PySequenceMethods *base_sq = base->tp_as_sequence; + EXT_TEST(sq, length); + EXT_TEST(sq, concat); + EXT_TEST(sq, item); + EXT_TEST(sq, ass_item); + EXT_TEST(sq, contains); + EXT_TEST(sq, inplace_concat); + EXT_TEST(sq, inplace_repeat); + } + } + + if(length_ext || + concat_ext || + repeat_ext || + item_ext || + ass_item_ext || + contains_ext || + inplace_concat_ext || + inplace_repeat_ext) + { + return 1; + } + } + + if(!(mp == NULL || mp->mp_subscript == NULL)) + { + SLOT_EXT_FLAG(mp, length); + SLOT_EXT_FLAG(mp, subscript); + SLOT_EXT_FLAG(mp, ass_subscript); + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(base->tp_as_mapping != NULL) + { + PyMappingMethods *base_mp = base->tp_as_mapping; + EXT_TEST(mp, length); + EXT_TEST(mp, subscript); + EXT_TEST(mp, ass_subscript); + } + } + + if(length_ext || + subscript_ext || + ass_subscript_ext) + { + return 1; + } + } + + if(!(am == NULL || (am->am_await != NULL || am->am_aiter != NULL || am->am_anext != NULL || am->am_send != NULL))) + { + SLOT_EXT_FLAG(am, await); + SLOT_EXT_FLAG(am, aiter); + SLOT_EXT_FLAG(am, anext); + EXT_FLAG(am, send); + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(base->tp_as_async != NULL) + { + PyAsyncMethods *base_am = base->tp_as_async; + EXT_TEST(am, await); + EXT_TEST(am, aiter); + EXT_TEST(am, anext); + EXT_TEST(am, send); + } + } + + if(await_ext || + aiter_ext || + anext_ext || + send_ext) + { + return 1; + } + } + + if(!(bf == NULL || bf->bf_getbuffer == NULL)) + { + SLOT_EXT_FLAG(bf, getbuffer); + SLOT_EXT_FLAG(bf, releasebuffer); + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(base->tp_as_buffer != NULL) + { + PyBufferProcs *base_bf = base->tp_as_buffer; + EXT_TEST(bf, getbuffer); + EXT_TEST(bf, releasebuffer); + } + } + + if(getbuffer_ext || + releasebuffer_ext) + { + return 1; + } + } + + if(tp->tp_getattr != NULL || + tp->tp_setattr != NULL || + tp->tp_methods != NULL) + { + bool getattr_ext = tp->tp_getattr != NULL; + bool setattr_ext = tp->tp_setattr != NULL; + bool methods_ext = tp->tp_methods != NULL; + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(tp->tp_getattr == base->tp_getattr) + { + getattr_ext = false; + } + if(tp->tp_setattr == base->tp_setattr) + { + setattr_ext = false; + } + if(tp->tp_methods == base->tp_methods) + { + methods_ext = false; + } + + if(!(getattr_ext || setattr_ext || methods_ext)) + { + break; + } + } + + if(getattr_ext || setattr_ext || methods_ext) + { + return 1; + } + } + + if (!(tp->tp_setattro == PyObject_GenericSetAttr || tp->tp_setattro == NULL)){ + bool setattro_ext = true; + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(tp->tp_setattro == base->tp_setattro) + { + setattro_ext = false; + break; + } + } + + if(setattro_ext) + { + return 1; + } + } + + if (!(tp->tp_getattro == PyObject_GenericGetAttr || tp->tp_getattro == NULL)){ + bool getattro_ext = true; + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(tp->tp_getattro == base->tp_getattro) + { + getattro_ext = false; + break; + } + } + + if(getattro_ext) + { + return 1; + } + } + + if(!(tp->tp_getset == NULL || + tp->tp_getset == subtype_getsets_full || + tp->tp_getset == subtype_getsets_weakref_only || + tp->tp_getset == subtype_getsets_dict_only)) + { + bool getset_ext = true; + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(tp->tp_getset == base->tp_getset) + { + getset_ext = false; + break; + } + } + + if(getset_ext) + { + return 1; + } + } + + if(!(tp->tp_str == NULL || tp->tp_str == object_str)){ + bool str_ext = true; + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(tp->tp_str == base->tp_str) + { + str_ext = false; + break; + } + } + if(str_ext) + { + return 1; + } + } + + if(!(tp->tp_repr == NULL || tp->tp_repr == object_repr)){ + bool repr_ext = true; + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(tp->tp_repr == base->tp_repr) + { + repr_ext = false; + break; + } + } + if(repr_ext) + { + return 1; + } + } + + if(!(tp->tp_richcompare == NULL || tp->tp_richcompare == object_richcompare)){ + bool richcompare_ext = true; + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(tp->tp_richcompare == base->tp_richcompare) + { + richcompare_ext = false; + break; + } + } + if(richcompare_ext) + { + return 1; + } + } + + if(!(tp->tp_hash == NULL || tp->tp_hash == (hashfunc)_Py_HashPointer)){ + bool hash_ext = true; + for(Py_ssize_t i=1; i < mro_size; i++) + { + PyTypeObject *base = (PyTypeObject *)PyTuple_GET_ITEM(tp->tp_mro, i); + if(tp->tp_hash == base->tp_hash) + { + hash_ext = false; + break; + } + } + if(hash_ext) + { + return 1; + } + } + + return 0; +} diff --git a/Python/immutability.c b/Python/immutability.c index a5cf91d9bc9c9b..90abcf81acc8e8 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -7,13 +7,63 @@ #include "pycore_immutability.h" + +static int init_state(struct _Py_immutability_state *state) +{ + PyObject* frozen_importlib = NULL; + + frozen_importlib = PyImport_ImportModule("_frozen_importlib"); + if(frozen_importlib == NULL){ + return -1; + } + + state->module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); + if(state->module_locks == NULL){ + Py_DECREF(frozen_importlib); + return -1; + } + + state->blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); + if(state->blocking_on == NULL){ + Py_DECREF(frozen_importlib); + return -1; + } + + state->freezable_types = PySet_New(NULL); + if(state->freezable_types == NULL){ + Py_DECREF(frozen_importlib); + return -1; + } + + if(PyDict_SetItemString(PyModule_GetDict(frozen_importlib), "_freezable_types", state->freezable_types)){ + Py_DECREF(frozen_importlib); + return -1; + } + + return 0; +} + +static struct _Py_immutability_state* get_immutable_state(void) +{ + PyInterpreterState* interp = PyInterpreterState_Get(); + struct _Py_immutability_state *state = &interp->immutability; + if(state->freezable_types == NULL){ + if(init_state(state) == -1){ + return NULL; + } + } + + return state; +} + + PyDoc_STRVAR(notfreezable_doc, "NotFreezable()\n\ \n\ Indicate that a type cannot be frozen."); -PyTypeObject PyNotFreezable_Type = { +PyTypeObject _PyNotFreezable_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "NotFreezable", .tp_doc = notfreezable_doc, @@ -229,145 +279,127 @@ static int freeze_visit(PyObject* obj, void* frontier) return 0; } -static bool can_freeze(PyTypeObject* type, PyObject* freezable_types) +static bool is_freezable_builtin(PyTypeObject *type) { - PyObject* type_op = _PyObject_CAST(type); - if(_Py_IsImmutable(type_op)){ - return true; - } - if(type == &PyType_Type || - type == &PyBaseObject_Type || - type == &PyFunction_Type || - type == &_PyNone_Type || - type == &PyBool_Type || - type == &PyLong_Type || - type == &PyFloat_Type || - type == &PyComplex_Type || - type == &PyBytes_Type || - type == &PyUnicode_Type || - type == &PyTuple_Type || - type == &PyList_Type || - type == &PyDict_Type || - type == &PySet_Type || - type == &PyMemoryView_Type || - type == &PyByteArray_Type || - type == &PyRange_Type || - type == &PyGetSetDescr_Type || - type == &PyMemberDescr_Type || - type == &PyProperty_Type || - type == &PyWrapperDescr_Type || - type == &PyMethodDescr_Type || - type == &PyMethod_Type || - type == &PyCFunction_Type || - type == &PyCapsule_Type || - type == &PyCode_Type || - type == &PyCell_Type || - type == &PyFrame_Type || - type == &_PyWeakref_RefType) - { - return true; - } - - if(type == &PyNotFreezable_Type || PyType_IsSubtype(type, &PyNotFreezable_Type)){ - PyErr_SetString(PyExc_TypeError, "Cannot freeze NotFreezable type"); - return false; - } - - if(freezable_types == NULL){ - return false; - } - - type_op = _PyObject_CAST(type); - if (PySet_Contains(freezable_types, type_op) == 1) { - return true; - } - - if(Py_IsFalse(_PyType_UsesDefaultSlots(type))){ - return false; - } - - return true; + type == &PyBaseObject_Type || + type == &PyFunction_Type || + type == &_PyNone_Type || + type == &PyBool_Type || + type == &PyLong_Type || + type == &PyFloat_Type || + type == &PyComplex_Type || + type == &PyBytes_Type || + type == &PyUnicode_Type || + type == &PyTuple_Type || + type == &PyList_Type || + type == &PyDict_Type || + type == &PySet_Type || + type == &PyFrozenSet_Type || + type == &PyMemoryView_Type || + type == &PyByteArray_Type || + type == &PyRange_Type || + type == &PyGetSetDescr_Type || + type == &PyMemberDescr_Type || + type == &PyProperty_Type || + type == &PyWrapperDescr_Type || + type == &PyMethodDescr_Type || + type == &PyClassMethodDescr_Type || + type == &PyMethod_Type || + type == &PyCFunction_Type || + type == &PyCapsule_Type || + type == &PyCode_Type || + type == &PyCell_Type || + type == &PyFrame_Type || + type == &_PyWeakref_RefType) + { + return true; + } + + return false; } -static int init_state(struct _Py_immutability_state *state) -{ - PyObject* frozen_importlib = NULL; +typedef enum { + VALID_BUILTIN, + VALID_EXPLICIT, + VALID_IMPLICIT, + INVALID_NOT_FREEZABLE, + INVALID_C_EXTENSIONS +} FreezableCheck; - frozen_importlib = PyImport_ImportModule("_frozen_importlib"); - if(frozen_importlib == NULL){ - return -1; - } - state->module_locks = PyObject_GetAttrString(frozen_importlib, "_module_locks"); - if(state->module_locks == NULL){ - Py_DECREF(frozen_importlib); - return -1; +static FreezableCheck check_freezable(PyObject* obj, PyObject* freezable_types) +{ + if(PyObject_IsInstance(obj, (PyObject*)&_PyNotFreezable_Type)){ + return INVALID_NOT_FREEZABLE; } - state->blocking_on = PyObject_GetAttrString(frozen_importlib, "_blocking_on"); - if(state->blocking_on == NULL){ - Py_DECREF(frozen_importlib); - return -1; + if(is_freezable_builtin(obj->ob_type)){ + return VALID_BUILTIN; } - state->freezable_types = PySet_New(NULL); - if(state->freezable_types == NULL){ - Py_DECREF(frozen_importlib); - return -1; + PyObject* type = (PyObject*)obj->ob_type; + if(PySet_Contains(freezable_types, type) == 1){ + return VALID_EXPLICIT; } - if(PyDict_SetItemString(PyModule_GetDict(frozen_importlib), "_freezable_types", state->freezable_types)){ - Py_DECREF(frozen_importlib); - return -1; + if(_PyType_HasExtensionSlots(obj->ob_type)){ + return INVALID_C_EXTENSIONS; } - return 0; + return VALID_IMPLICIT; } -static struct _Py_immutability_state* get_immutable_state(PyObject* module) + +int _PyImmutability_IsFreezable(PyObject *obj) { - PyInterpreterState* interp = PyInterpreterState_Get(); - struct _Py_immutability_state *state = &interp->immutability; - if(state->freezable_types == NULL){ - if(init_state(state) == -1){ - return NULL; - } + struct _Py_immutability_state *state = get_immutable_state(); + if(state == NULL){ + PyErr_SetString(PyExc_RuntimeError, "Failed to initialize immutability state"); + return 0; } - return state; + switch(check_freezable(obj, state->freezable_types)) + { + case VALID_BUILTIN: + case VALID_EXPLICIT: + case VALID_IMPLICIT: + return 1; + case INVALID_NOT_FREEZABLE: + case INVALID_C_EXTENSIONS: + return 0; + } + + PyErr_SetString(PyExc_RuntimeError, "Unknown state"); + return -1; } -PyObject* _PyImmutability_RegisterFreezable(PyObject* obj) +int _PyImmutability_RegisterFreezable(PyTypeObject* tp) { - struct _Py_immutability_state *state = get_immutable_state(obj); + struct _Py_immutability_state *state = get_immutable_state(); if(state == NULL){ PyErr_SetString(PyExc_RuntimeError, "Failed to initialize immutability state"); - return NULL; - } - - if(!PyType_Check(obj)){ - PyErr_SetString(PyExc_TypeError, "Expected a type"); - return NULL; + return -1; } - if(PySet_Add(state->freezable_types, obj) == -1){ - return NULL; + if(PySet_Add(state->freezable_types, _PyObject_CAST(tp)) == -1){ + return -1; } - Py_RETURN_NONE; + return 0; } -PyObject* _PyImmutability_Freeze(PyObject* obj) +int _PyImmutability_Freeze(PyObject* obj) { PyObject* frontier = NULL; - PyObject* result = Py_None; - struct _Py_immutability_state* state = get_immutable_state(obj); + int result = 0; + + struct _Py_immutability_state* state = get_immutable_state(); if(state == NULL){ PyErr_SetString(PyExc_RuntimeError, "Failed to initialize immutability state"); - return NULL; + return -1; } if(_Py_IsImmutable(obj)){ @@ -376,28 +408,43 @@ PyObject* _PyImmutability_Freeze(PyObject* obj) frontier = PyList_New(0); if(frontier == NULL){ - result = PyErr_NoMemory(); - goto cleanup; + goto error; } if(push(frontier, obj)){ - result = PyErr_NoMemory(); - goto cleanup; + goto error; } while(PyList_Size(frontier) != 0){ PyObject* item = pop(frontier); + FreezableCheck check; if(item == state->blocking_on || item == state->module_locks){ continue; } - if(!can_freeze(item->ob_type, state->freezable_types)){ - PyObject* error_msg = PyUnicode_FromFormat("Cannot freeze object of type %s", Py_TYPE(item)->tp_name); - PyErr_SetObject(PyExc_TypeError, error_msg); - result = NULL; - goto cleanup; + check = check_freezable(item, state->freezable_types); + switch(check){ + case INVALID_NOT_FREEZABLE: + PyErr_SetString(PyExc_TypeError, "Invalid freeze request: instance of NotFreezable"); + goto error; + + case INVALID_C_EXTENSIONS: + PyObject* error_msg = PyUnicode_FromFormat( + "Cannot freeze instance of type %s due to custom functionality implemented in C", + (item->ob_type->tp_name)); + PyErr_SetObject(PyExc_TypeError, error_msg); + goto error; + + case VALID_BUILTIN: + case VALID_EXPLICIT: + case VALID_IMPLICIT: + break; + + default: + PyErr_SetString(PyExc_RuntimeError, "Unknown freezable check value"); + goto error; } if(_Py_IsImmutable(item)){ @@ -414,24 +461,23 @@ PyObject* _PyImmutability_Freeze(PyObject* obj) if(PyFunction_Check(item)){ if(shadow_function_globals(item) == NULL){ - goto cleanup; + goto error; } } if(PyType_Check(item)){ PyTypeObject* type = (PyTypeObject*)item; + if(push(frontier, type->tp_dict)) { - result = PyErr_NoMemory(); - goto cleanup; + goto error; } - if(PySet_Contains(state->freezable_types, item) != 1){ - // type is not explicit freezable, so we need to check its bases + if(check == VALID_IMPLICIT) + { if(push(frontier, type->tp_mro)) { - result = PyErr_NoMemory(); - goto cleanup; + goto error; } } } @@ -440,19 +486,22 @@ PyObject* _PyImmutability_Freeze(PyObject* obj) traverseproc traverse = Py_TYPE(item)->tp_traverse; if(traverse != NULL){ if(traverse(item, (visitproc)freeze_visit, frontier)){ - result = NULL; - goto cleanup; + goto error; } } if(push(frontier, _PyObject_CAST(Py_TYPE(item)))){ - result = PyErr_NoMemory(); - goto cleanup; + goto error; } } } -cleanup: + goto finally; + +error: + result = -1; + +finally: Py_XDECREF(frontier); return result; From 5a0b5b765b1b8448d3655406056f099d4f24f31e Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 1 May 2025 10:42:25 +0100 Subject: [PATCH 29/40] _datetime Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_datetime.py | 19 +++++++++++++++++++ Modules/_datetimemodule.c | 4 ++++ 2 files changed, 23 insertions(+) create mode 100644 Lib/test/test_freeze/test_datetime.py diff --git a/Lib/test/test_freeze/test_datetime.py b/Lib/test/test_freeze/test_datetime.py new file mode 100644 index 00000000000000..7dbed5f67e3fee --- /dev/null +++ b/Lib/test/test_freeze/test_datetime.py @@ -0,0 +1,19 @@ + +import unittest +from datetime import datetime, timedelta + +from .test_common import BaseObjectTest + + +class TestDatetime(BaseObjectTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=datetime.now(), **kwargs) + + +class TestDatetimeTimeDelta(BaseObjectTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=timedelta(days=1), **kwargs) + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index d8183c63f44976..d05b64269f644b 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -6732,6 +6732,10 @@ _datetime_exec(PyObject *module) if (PyModule_AddType(module, types[i]) < 0) { return -1; } + + if(_PyImmutability_RegisterFreezable(types[i]) < 0) { + return -1; + } } if (PyType_Ready(&PyDateTime_IsoCalendarDateType) < 0) { From 2fab5cdcff5fc8bfbe92d77be8be5fdcedff9d83 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 1 May 2025 10:59:53 +0100 Subject: [PATCH 30/40] _struct Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_datetime.py | 1 - Lib/test/test_freeze/test_struct.py | 13 +++++++++++++ Modules/_elementtree.c | 3 +-- Modules/_struct.c | 3 +++ 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 Lib/test/test_freeze/test_struct.py diff --git a/Lib/test/test_freeze/test_datetime.py b/Lib/test/test_freeze/test_datetime.py index 7dbed5f67e3fee..373a8b07719fda 100644 --- a/Lib/test/test_freeze/test_datetime.py +++ b/Lib/test/test_freeze/test_datetime.py @@ -1,4 +1,3 @@ - import unittest from datetime import datetime, timedelta diff --git a/Lib/test/test_freeze/test_struct.py b/Lib/test/test_freeze/test_struct.py new file mode 100644 index 00000000000000..e513e42bd55b55 --- /dev/null +++ b/Lib/test/test_freeze/test_struct.py @@ -0,0 +1,13 @@ +import unittest +from struct import Struct + +from .test_common import BaseObjectTest + + +class TestStruct(BaseObjectTest): + def __init__(self, *args, **kwargs): + super().__init__(*args, obj=Struct("i"), **kwargs) + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f1173eb86a5486..4daf3bf71ec692 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4386,11 +4386,10 @@ module_exec(PyObject *m) if(register_freezable != NULL) { PyObject* result = PyObject_CallOneArg(register_freezable, (PyObject *)st->Element_Type); + Py_DECREF(register_freezable); if(result == NULL){ goto error; } - - Py_DECREF(register_freezable); } st->deepcopy_obj = _PyImport_GetModuleAttrString("copy", "deepcopy"); diff --git a/Modules/_struct.c b/Modules/_struct.c index 4f9478bd98095d..9bc3a5bd81083a 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2503,6 +2503,9 @@ _structmodule_exec(PyObject *m) if (PyModule_AddType(m, (PyTypeObject *)state->PyStructType) < 0) { return -1; } + if (_PyImmutability_RegisterFreezable((PyTypeObject *)state->PyStructType) < 0){ + return -1; + } state->unpackiter_type = PyType_FromModuleAndSpec( m, &unpackiter_type_spec, NULL); From f621a0cf4f700ecdc996950b24ac091bf71c1534 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 1 May 2025 12:35:14 +0100 Subject: [PATCH 31/40] Making freezable types a weakset Signed-off-by: Matthew A Johnson --- Include/internal/pycore_immutability.h | 1 + Python/immutability.c | 88 ++++++++++++++++++++++---- Python/pystate.c | 1 + 3 files changed, 79 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_immutability.h b/Include/internal/pycore_immutability.h index 67ac1ebe736494..415c5c1d430055 100644 --- a/Include/internal/pycore_immutability.h +++ b/Include/internal/pycore_immutability.h @@ -12,6 +12,7 @@ struct _Py_immutability_state { PyObject *module_locks; PyObject *blocking_on; PyObject *freezable_types; + PyObject *destroy_cb; }; #ifdef __cplusplus diff --git a/Python/immutability.c b/Python/immutability.c index 90abcf81acc8e8..9e602795153102 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -7,8 +7,38 @@ #include "pycore_immutability.h" +static PyObject * +_destroy(PyObject* set, PyObject *objweakref) +{ + Py_INCREF(set); + if (PySet_Discard(set, objweakref) < 0) { + Py_DECREF(set); + return NULL; + } + Py_DECREF(set); + + Py_RETURN_NONE; +} + +static PyMethodDef _destroy_def = { + "_destroy", (PyCFunction) _destroy, METH_O +}; -static int init_state(struct _Py_immutability_state *state) +static PyObject * +type_weakref(struct _Py_immutability_state *state, PyObject *obj) +{ + if(state->destroy_cb == NULL){ + state->destroy_cb = PyCFunction_NewEx(&_destroy_def, state->freezable_types, NULL); + if (state->destroy_cb == NULL) { + return NULL; + } + } + + return PyWeakref_NewRef(obj, state->destroy_cb); +} + +static +int init_state(struct _Py_immutability_state *state) { PyObject* frozen_importlib = NULL; @@ -40,6 +70,8 @@ static int init_state(struct _Py_immutability_state *state) return -1; } + Py_DECREF(frozen_importlib); + return 0; } @@ -279,7 +311,8 @@ static int freeze_visit(PyObject* obj, void* frontier) return 0; } -static bool is_freezable_builtin(PyTypeObject *type) +static bool +is_freezable_builtin(PyTypeObject *type) { if(type == &PyType_Type || type == &PyBaseObject_Type || @@ -319,19 +352,39 @@ static bool is_freezable_builtin(PyTypeObject *type) return false; } +static int +is_explicitly_freezable(struct _Py_immutability_state *state, PyObject *obj) +{ + int result = 0; + PyObject *ref = type_weakref(state, (PyObject *)obj->ob_type); + if(ref == NULL){ + return -1; + } + + result = PySet_Contains(state->freezable_types, ref); + Py_DECREF(ref); + return result; +} typedef enum { VALID_BUILTIN, VALID_EXPLICIT, VALID_IMPLICIT, INVALID_NOT_FREEZABLE, - INVALID_C_EXTENSIONS + INVALID_C_EXTENSIONS, + ERROR } FreezableCheck; -static FreezableCheck check_freezable(PyObject* obj, PyObject* freezable_types) +static FreezableCheck check_freezable(struct _Py_immutability_state *state, PyObject* obj) { - if(PyObject_IsInstance(obj, (PyObject*)&_PyNotFreezable_Type)){ + int result = 0; + + result = PyObject_IsInstance(obj, (PyObject*)&_PyNotFreezable_Type); + if(result == -1){ + return ERROR; + } + else if(result == 1){ return INVALID_NOT_FREEZABLE; } @@ -339,8 +392,11 @@ static FreezableCheck check_freezable(PyObject* obj, PyObject* freezable_types) return VALID_BUILTIN; } - PyObject* type = (PyObject*)obj->ob_type; - if(PySet_Contains(freezable_types, type) == 1){ + result = is_explicitly_freezable(state, obj); + if(result == -1){ + return ERROR; + } + else if(result == 1){ return VALID_EXPLICIT; } @@ -360,7 +416,7 @@ int _PyImmutability_IsFreezable(PyObject *obj) return 0; } - switch(check_freezable(obj, state->freezable_types)) + switch(check_freezable(state, obj)) { case VALID_BUILTIN: case VALID_EXPLICIT: @@ -369,6 +425,8 @@ int _PyImmutability_IsFreezable(PyObject *obj) case INVALID_NOT_FREEZABLE: case INVALID_C_EXTENSIONS: return 0; + case ERROR: + return -1; } PyErr_SetString(PyExc_RuntimeError, "Unknown state"); @@ -377,17 +435,22 @@ int _PyImmutability_IsFreezable(PyObject *obj) int _PyImmutability_RegisterFreezable(PyTypeObject* tp) { + PyObject *ref; + int result; struct _Py_immutability_state *state = get_immutable_state(); if(state == NULL){ PyErr_SetString(PyExc_RuntimeError, "Failed to initialize immutability state"); return -1; } - if(PySet_Add(state->freezable_types, _PyObject_CAST(tp)) == -1){ + ref = type_weakref(state, (PyObject*)tp); + if(ref == NULL){ return -1; } - return 0; + result = PySet_Add(state->freezable_types, ref); + Py_DECREF(ref); + return result; } @@ -424,7 +487,7 @@ int _PyImmutability_Freeze(PyObject* obj) continue; } - check = check_freezable(item, state->freezable_types); + check = check_freezable(state, item); switch(check){ case INVALID_NOT_FREEZABLE: PyErr_SetString(PyExc_TypeError, "Invalid freeze request: instance of NotFreezable"); @@ -441,6 +504,9 @@ int _PyImmutability_Freeze(PyObject* obj) case VALID_EXPLICIT: case VALID_IMPLICIT: break; + + case ERROR: + goto error; default: PyErr_SetString(PyExc_RuntimeError, "Unknown freezable check value"); diff --git a/Python/pystate.c b/Python/pystate.c index 4f87046ec9e927..5f4ff64f2a48f6 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -860,6 +860,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->immutability.module_locks); Py_CLEAR(interp->immutability.blocking_on); Py_CLEAR(interp->immutability.freezable_types); + Py_CLEAR(interp->immutability.destroy_cb); Py_CLEAR(interp->sysdict_copy); Py_CLEAR(interp->builtins_copy); From a31e962aeaf8a649651f6cce61e1feacd6978610 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Thu, 1 May 2025 12:47:25 +0100 Subject: [PATCH 32/40] These don't need to be freezable Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_hashlib.py | 19 ------------------- Modules/_blake2/blake2b_impl.c | 4 ---- Modules/_blake2/blake2module.c | 18 ------------------ Modules/_blake2/blake2s_impl.c | 4 ---- 4 files changed, 45 deletions(-) delete mode 100644 Lib/test/test_freeze/test_hashlib.py diff --git a/Lib/test/test_freeze/test_hashlib.py b/Lib/test/test_freeze/test_hashlib.py deleted file mode 100644 index d64baff6b7a337..00000000000000 --- a/Lib/test/test_freeze/test_hashlib.py +++ /dev/null @@ -1,19 +0,0 @@ -from hashlib import blake2b, blake2s -from immutable import freeze -import unittest - - -class TestHashlib(unittest.TestCase): - def test_blake2b(self): - h = blake2b(digest_size=32) - h.update(b'Hello world') - freeze(h) - with self.assertRaises(TypeError): - h.update(b'!') - - def test_blake2s(self): - h = blake2s(digest_size=32) - h.update(b'Hello world') - freeze(h) - with self.assertRaises(TypeError): - h.update(b'!') diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index 9e62c7cf6d8e1c..c2cac98c7529eb 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -276,10 +276,6 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) { Py_buffer buf; - if(!Py_CHECKWRITE(self)){ - return PyErr_WriteToImmutable(self); - } - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index cdc2147f2c008f..0d1d88c6603684 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -89,7 +89,6 @@ blake2_exec(PyObject *m) return -1; } - PyObject *d = st->blake2b_type->tp_dict; ADD_INT(d, "SALT_SIZE", BLAKE2B_SALTBYTES); ADD_INT(d, "PERSON_SIZE", BLAKE2B_PERSONALBYTES); @@ -112,23 +111,6 @@ blake2_exec(PyObject *m) return -1; } - PyObject *register_freezable = _PyImport_GetModuleAttrString("immutable", "register_freezable"); - if(register_freezable != NULL){ - PyObject *result = PyObject_CallFunctionObjArgs(register_freezable, st->blake2b_type, NULL); - if(result == NULL){ - Py_DECREF(register_freezable); - return -1; - } - - result = PyObject_CallFunctionObjArgs(register_freezable, st->blake2s_type, NULL); - if(result == NULL){ - Py_DECREF(register_freezable); - return -1; - } - - Py_DECREF(register_freezable); - } - d = st->blake2s_type->tp_dict; ADD_INT(d, "SALT_SIZE", BLAKE2S_SALTBYTES); ADD_INT(d, "PERSON_SIZE", BLAKE2S_PERSONALBYTES); diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index b9eb9dae0f2085..1c47328ece13e8 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -276,10 +276,6 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) { Py_buffer buf; - if(!Py_CHECKWRITE(self)){ - return PyErr_WriteToImmutable(self); - } - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) From adc13191d4310d5c7efc571622f45dcf1eaa5822 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 1 May 2025 15:09:05 +0100 Subject: [PATCH 33/40] Make mutable during finalisation. (#6) Signed-off-by: Matthew A Johnson --- Include/internal/pycore_object.h | 15 +++++++++-- Include/object.h | 41 ++++++++++++++++++++++++++--- Lib/test/test_freeze/test_ctypes.py | 2 +- Python/ceval.c | 9 ++++++- Python/immutability.c | 20 ++++++++++++++ 5 files changed, 80 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 16078406cea384..71e0b7cc6ded89 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -103,7 +103,14 @@ static inline void _Py_SetImmutable(PyObject *op) static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { - if (_Py_IsImmortal(op)) { + if (_Py_IsImmortalOrImmutable(op)) { + if (_Py_IsImmortal(op)) { + return; + } + assert(_Py_IsImmutable(op)); + if (_Py_DecRef_Immutable(op)) { + destruct(op); + } return; } _Py_DECREF_STAT_INC(); @@ -125,7 +132,11 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) static inline void _Py_DECREF_NO_DEALLOC(PyObject *op) { - if (_Py_IsImmortal(op)) { + if (_Py_IsImmortalOrImmutable(op)) { + if (_Py_IsImmortal(op)) { + return; + } + _Py_DecRef_Immutable(op); return; } _Py_DECREF_STAT_INC(); diff --git a/Include/object.h b/Include/object.h index b5b81bd80dd62b..4652afd1581a82 100644 --- a/Include/object.h +++ b/Include/object.h @@ -288,12 +288,30 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) #endif #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} +static inline Py_ALWAYS_INLINE int _Py_IsImmortalOrImmutable(PyObject *op) +{ + // TODO(Immutable): Does this work for both 32 and 64bit? + return op->ob_refcnt >= _Py_IMMORTAL_REFCNT; +} +#define _Py_IsImmortalOrImmutable(op) _Py_IsImmortalOrImmutable(_PyObject_CAST(op)) + static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { // This immortal check is for code that is unaware of immortal objects. // The runtime tracks these objects and we should avoid as much // as possible having extensions inadvertently change the refcnt // of an immortalized object. - if (_Py_IsImmortal(ob)) { + if (_Py_IsImmortalOrImmutable(ob)) + { + if (_Py_IsImmortal(ob)) { + return; + } + assert(_Py_IsImmutable(ob)); + // TODO(Immutable): It is dangerous to set the reference count of an + // immutable object. The majority of calls appear to be where the rc + // has reached 0 and a finalizer is running. This seems a reasonable + // place to allow the refcnt to be set to 1, and clear the immutable flag. + assert((ob->ob_refcnt & _Py_REFCNT_MASK) == 0); + ob->ob_refcnt = refcnt; return; } ob->ob_refcnt = (ob->ob_refcnt & _Py_IMMUTABLE_MASK) | (refcnt & _Py_REFCNT_MASK); @@ -655,6 +673,8 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *); PyAPI_FUNC(void) _Py_IncRef(PyObject *); PyAPI_FUNC(void) _Py_DecRef(PyObject *); +PyAPI_FUNC(int) _Py_DecRef_Immutable(PyObject *op); + static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { #if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) @@ -715,7 +735,15 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) if (op->ob_refcnt <= 0) { _Py_NegativeRefcount(filename, lineno, op); } - if (_Py_IsImmortal(op)) { + + if (_Py_IsImmortalOrImmutable(op)) + { + if (_Py_IsImmortal(op)) { + return; + } + assert(_Py_IsImmutable(op)); + if (_Py_DecRef_Immutable(op)) + _Py_Dealloc(op); return; } _Py_DECREF_STAT_INC(); @@ -732,7 +760,14 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) { // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. - if (_Py_IsImmortal(op)) { + if (_Py_IsImmortalOrImmutable(op)) + { + if (_Py_IsImmortal(op)) { + return; + } + assert(_Py_IsImmutable(op)); + if (_Py_DecRef_Immutable(op)) + _Py_Dealloc(op); return; } _Py_DECREF_STAT_INC(); diff --git a/Lib/test/test_freeze/test_ctypes.py b/Lib/test/test_freeze/test_ctypes.py index 2bfb2b166c6b39..78ff70aeef1c0d 100644 --- a/Lib/test/test_freeze/test_ctypes.py +++ b/Lib/test/test_freeze/test_ctypes.py @@ -1,6 +1,6 @@ import ctypes import unittest -from immutable import freeze, isfrozen, isfrozen +from immutable import isfrozen from .test_common import BaseObjectTest diff --git a/Python/ceval.c b/Python/ceval.c index e978434717d8ac..28caf60057af65 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -84,7 +84,14 @@ #define _Py_DECREF_SPECIALIZED(arg, dealloc) \ do { \ PyObject *op = _PyObject_CAST(arg); \ - if (_Py_IsImmortal(op)) { \ + if (_Py_IsImmortalOrImmutable(op)) { \ + if (_Py_IsImmortal(op)) { \ + break; \ + } \ + if (_Py_DecRef_Immutable(op)) { \ + destructor d = (destructor)(dealloc); \ + d(op); \ + } \ break; \ } \ _Py_DECREF_STAT_INC(); \ diff --git a/Python/immutability.c b/Python/immutability.c index 9e602795153102..de12da6a3ba440 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -453,6 +453,26 @@ int _PyImmutability_RegisterFreezable(PyTypeObject* tp) return result; } +// Perform a decref on an immutable object +// returns true if the object should be deallocated. +int _Py_DecRef_Immutable(PyObject *op) +{ + // Decrement the reference count of an immutable object without + // deallocating it. + assert(_Py_IsImmutable(op)); + + // TODO(Immutable): This needs to be atomic. + op->ob_refcnt -= 1; + if ((op->ob_refcnt & _Py_REFCNT_MASK) != 0) + // Context does not to dealloc this object. + return false; + + // Clear the immutable flag so that finalisers can run correctly. + assert((op->ob_refcnt & _Py_REFCNT_MASK) == 0); + op->ob_refcnt = 0; + return true; +} + int _PyImmutability_Freeze(PyObject* obj) { From ccb893c2f0f2febb98ad1fd1e7a6a170170f9b63 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 2 May 2025 11:35:38 +0100 Subject: [PATCH 34/40] Dealing with more finalisation bugs Signed-off-by: Matthew A Johnson --- Include/cpython/pylifecycle.h | 1 - Include/object.h | 6 +----- Include/pylifecycle.h | 1 + Modules/_io/fileio.c | 2 +- Objects/abstract.c | 2 +- Objects/dictobject.c | 23 +++++++++++++++-------- Objects/typeobject.c | 5 +---- Python/pylifecycle.c | 2 +- Python/sysmodule.c | 2 +- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 4daea33bf80114..9a51e0c454117f 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -51,7 +51,6 @@ Py_DEPRECATED(3.11) PyAPI_FUNC(void) _Py_SetProgramFullPath(const wchar_t *); PyAPI_FUNC(const char *) _Py_gitidentifier(void); PyAPI_FUNC(const char *) _Py_gitversion(void); -PyAPI_FUNC(int) _Py_IsFinalizing(void); PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp); /* Random */ diff --git a/Include/object.h b/Include/object.h index 4652afd1581a82..fc31bc94ced749 100644 --- a/Include/object.h +++ b/Include/object.h @@ -281,11 +281,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) // Check whether an object is writeable. // This check will always succeed during runtime finalization. -#ifndef Py_LIMITED_API -#define Py_CHECKWRITE(op) ((op) && (!_Py_IsImmutable(op) || _Py_IsFinalizing())) -#else -#define Py_CHECKWRITE(op) ((op) && !_Py_IsImmutable(op)) -#endif +#define Py_CHECKWRITE(op) ((op) && (!_Py_IsImmutable(op) || Py_IsFinalizing())) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} static inline Py_ALWAYS_INLINE int _Py_IsImmortalOrImmutable(PyObject *op) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index e4c3b09c963fe8..fc0c818bc9254a 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -16,6 +16,7 @@ PyAPI_FUNC(void) Py_Finalize(void); PyAPI_FUNC(int) Py_FinalizeEx(void); #endif PyAPI_FUNC(int) Py_IsInitialized(void); +PyAPI_FUNC(int) Py_IsFinalizing(void); /* Subinterpreter support */ PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index c8be9982890b97..bab68077a2144a 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -153,7 +153,7 @@ _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) return res; } - PyObject *exc; + PyObject *exc = NULL; if (res == NULL) { exc = PyErr_GetRaisedException(); } diff --git a/Objects/abstract.c b/Objects/abstract.c index fb9c878d7c0823..a3014109f066c0 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -611,7 +611,7 @@ PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char *ptr; const char *src; - if(!Py_CHECKWRITE(view->obj)){ + if(view->obj && !Py_CHECKWRITE(view->obj)){ PyErr_WriteToImmutable(view->obj); return -1; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ae95a58b0c582e..199c6af1edf39c 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2074,19 +2074,14 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, } -void -PyDict_Clear(PyObject *op) +static void +_dict_clear(PyObject *op) { PyDictObject *mp; PyDictKeysObject *oldkeys; PyDictValues *oldvalues; Py_ssize_t i, n; - if(!Py_CHECKWRITE(op)){ - PyErr_WriteToImmutable(op); - return; - } - if (!PyDict_Check(op)) return; mp = ((PyDictObject *)op); @@ -2119,6 +2114,18 @@ PyDict_Clear(PyObject *op) ASSERT_CONSISTENT(mp); } + +void +PyDict_Clear(PyObject *op) +{ + if(!Py_CHECKWRITE(op)){ + PyErr_WriteToImmutable(op); + return; + } + + _dict_clear(op); +} + /* Internal version of PyDict_Next that returns a hash value in addition * to the key and value. * Return 1 on success, return 0 when the reached the end of the dictionary @@ -3603,7 +3610,7 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) static int dict_tp_clear(PyObject *op) { - PyDict_Clear(op); + _dict_clear(op); return 0; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 02f5f808f1ac11..f528301503df32 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5337,10 +5337,7 @@ type_clear(PyTypeObject *type) */ PyType_Modified(type); - PyObject *dict = lookup_tp_dict(type); - if (dict) { - PyDict_Clear(dict); - } + clear_tp_dict(type); Py_CLEAR(((PyHeapTypeObject *)type)->ht_module); Py_CLEAR(type->tp_mro); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 29771e07ae6a2c..bac69a7d2c9080 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -125,7 +125,7 @@ _PyRuntime_Finalize(void) } int -_Py_IsFinalizing(void) +Py_IsFinalizing(void) { return _PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 4bd38b4b267873..b9be70e4a0aca7 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2075,7 +2075,7 @@ static PyObject * sys_is_finalizing_impl(PyObject *module) /*[clinic end generated code: output=735b5ff7962ab281 input=f0df747a039948a5]*/ { - return PyBool_FromLong(_Py_IsFinalizing()); + return PyBool_FromLong(Py_IsFinalizing()); } #ifdef Py_STATS From 30dd83f0078cf62bda3ce753551a6c038b534540 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 2 May 2025 12:54:34 +0100 Subject: [PATCH 35/40] Shadow cell Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_core.py | 19 ++++++++++ Python/immutability.c | 62 ++++++++++++++++++------------- 2 files changed, 55 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_freeze/test_core.py b/Lib/test/test_freeze/test_core.py index 33d0053c00ca6e..9a9bc594b37a8c 100644 --- a/Lib/test/test_freeze/test_core.py +++ b/Lib/test/test_freeze/test_core.py @@ -242,6 +242,25 @@ def inc(): freeze(test) self.assertRaises(TypeError, test) + def test_nonlocal_changed(self): + v = 0 + def c(): + nonlocal v + v += 1 + + def inc(): + return v + 1 + + return inc + + test = c() + self.assertEqual(test(), 2) + test = c() + self.assertEqual(test(), 3) + freeze(test) + v = 5 + self.assertEqual(test(), 3) + def test_global(self): def d(): global global0 diff --git a/Python/immutability.c b/Python/immutability.c index de12da6a3ba440..8ead7b6c4a0a93 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -248,31 +248,41 @@ static PyObject* shadow_function_globals(PyObject* op) } } - if(check_globals){ - // if the code calls the globals() builtin, then any - // cellvar or const in the function could, potentially, refer to - // a global variable. As such, we need to check if the globals - // dictionary contains that key and then make it immutable - // from this point forwards. - // we need to check the closure for any cellvars that are not - // referenced in the code object, but are still used in the function - size = 0; - if(f->func_closure != NULL) - size = PySequence_Fast_GET_SIZE(f->func_closure); - - for(Py_ssize_t i=0; i < size; ++i){ - PyObject* cellvar = PySequence_Fast_GET_ITEM(f->func_closure, i); - PyObject* value = PyCell_GET(cellvar); - - if(PyUnicode_Check(value)){ - PyObject* name = value; - if(PyDict_Contains(globals, name)){ - value = PyDict_GetItem(globals, name); - if(PyDict_SetItem(shadow_globals, name, value)){ - Py_DECREF(shadow_builtins); - Py_DECREF(shadow_globals); - return NULL; - } + size = 0; + if(f->func_closure != NULL){ + size = PyTuple_Size(f->func_closure); + if(size == -1){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return NULL; + } + } + + for(Py_ssize_t i=0; i < size; ++i){ + PyObject* cellvar = PyTuple_GET_ITEM(f->func_closure, i); + PyObject* value = PyCell_GET(cellvar); + + PyObject* shadow_cellvar = PyCell_New(value); + if(PyTuple_SetItem(f->func_closure, i, shadow_cellvar) == -1){ + Py_DECREF(shadow_cellvar); + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return NULL; + } + + if(PyUnicode_Check(value) && check_globals){ + // if the code calls the globals() builtin, then any + // cellvar or const in the function could, potentially, refer to + // a global variable. As such, we need to check if the globals + // dictionary contains that key and then make it immutable + // from this point forwards. + PyObject* name = value; + if(PyDict_Contains(globals, name)){ + value = PyDict_GetItem(globals, name); + if(PyDict_SetItem(shadow_globals, name, value)){ + Py_DECREF(shadow_builtins); + Py_DECREF(shadow_globals); + return NULL; } } } @@ -524,7 +534,7 @@ int _PyImmutability_Freeze(PyObject* obj) case VALID_EXPLICIT: case VALID_IMPLICIT: break; - + case ERROR: goto error; From 57df4db7b03c521415ca901657cf3c4ddf187940 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 2 May 2025 14:12:11 +0100 Subject: [PATCH 36/40] Removing isfreezable and adding some TODOs Signed-off-by: Matthew A Johnson --- Include/cpython/immutability.h | 1 - Lib/test/test_freeze/test_common.py | 8 ++++-- Lib/test/test_freeze/test_core.py | 13 ++++++++-- Modules/clinic/immutablemodule.c.h | 11 +-------- Modules/immutablemodule.c | 25 ------------------- Python/immutability.c | 38 +++++++++-------------------- Python/stdlib_module_names.h | 2 +- 7 files changed, 30 insertions(+), 68 deletions(-) diff --git a/Include/cpython/immutability.h b/Include/cpython/immutability.h index 2d203c0fd03fc7..e88c4c39547d7d 100644 --- a/Include/cpython/immutability.h +++ b/Include/cpython/immutability.h @@ -6,4 +6,3 @@ PyAPI_DATA(PyTypeObject) _PyNotFreezable_Type; PyAPI_FUNC(int) _PyImmutability_Freeze(PyObject*); PyAPI_FUNC(int) _PyImmutability_RegisterFreezable(PyTypeObject*); -PyAPI_FUNC(int) _PyImmutability_IsFreezable(PyObject*); diff --git a/Lib/test/test_freeze/test_common.py b/Lib/test/test_freeze/test_common.py index b5fe14b9ac72ee..107d91c70aaddf 100644 --- a/Lib/test/test_freeze/test_common.py +++ b/Lib/test/test_freeze/test_common.py @@ -1,5 +1,5 @@ import unittest -from immutable import freeze, isfrozen, isfreezable, NotFreezable +from immutable import freeze, isfrozen, NotFreezable class BaseObjectTest(unittest.TestCase): @@ -8,7 +8,6 @@ def __init__(self, *args, obj=None, **kwargs): self.obj = obj def setUp(self): - self.assertTrue(isfreezable(self.obj)) freeze(self.obj) def test_immutable(self): @@ -31,6 +30,11 @@ def test_not_freezable(self): with self.assertRaises(TypeError): freeze(self.obj) + # Immutability(TODO) + # this test currently fails due to the lack of a walk-back functionality + # for failed freeze attempts + #self.assertFalse(isfrozen(self.obj)) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_freeze/test_core.py b/Lib/test/test_freeze/test_core.py index 9a9bc594b37a8c..fac16435c23aec 100644 --- a/Lib/test/test_freeze/test_core.py +++ b/Lib/test/test_freeze/test_core.py @@ -1,7 +1,7 @@ import unittest -from immutable import freeze, isfrozen +from immutable import freeze, NotFreezable, isfrozen -from .test_common import BaseObjectTest +from .test_common import BaseNotFreezableTest, BaseObjectTest # This is a canary to check that global variables are not made immutable @@ -595,5 +595,14 @@ def f(a, **b): self.assertTrue(isfrozen(bdef)) +class TestNotFreezable(BaseNotFreezableTest): + class C(NotFreezable): + pass + + def __init__(self, *args, **kwargs): + obj = self.C() + super().__init__(*args, obj=obj, **kwargs) + + if __name__ == '__main__': unittest.main() diff --git a/Modules/clinic/immutablemodule.c.h b/Modules/clinic/immutablemodule.c.h index 9218704d2803a7..fea1b2373e3af1 100644 --- a/Modules/clinic/immutablemodule.c.h +++ b/Modules/clinic/immutablemodule.c.h @@ -34,13 +34,4 @@ PyDoc_STRVAR(immutable_isfrozen__doc__, #define IMMUTABLE_ISFROZEN_METHODDEF \ {"isfrozen", (PyCFunction)immutable_isfrozen, METH_O, immutable_isfrozen__doc__}, - -PyDoc_STRVAR(immutable_isfreezable__doc__, -"isfreezable($module, obj, /)\n" -"--\n" -"\n" -"Check if an object can be frozen."); - -#define IMMUTABLE_ISFREEZABLE_METHODDEF \ - {"isfreezable", (PyCFunction)immutable_isfreezable, METH_O, immutable_isfreezable__doc__}, -/*[clinic end generated code: output=b26b154bf5fbe7c9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0edcea6c15426dec input=a9049054013a1b77]*/ diff --git a/Modules/immutablemodule.c b/Modules/immutablemodule.c index bc6178b2b0c90b..103d735f573ea5 100644 --- a/Modules/immutablemodule.c +++ b/Modules/immutablemodule.c @@ -115,30 +115,6 @@ immutable_isfrozen(PyObject *module, PyObject *obj) Py_RETURN_FALSE; } -/*[clinic input] -immutable.isfreezable - obj: object - / - -Check if an object can be frozen. -[clinic start generated code]*/ - -static PyObject * -immutable_isfreezable(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=ddc1a85bd9279882 input=9b06ae134ca053ea]*/ -{ - int result = _PyImmutability_IsFreezable(obj); - if(result == -1){ - return NULL; - } - - if(result){ - Py_RETURN_TRUE; - } - - Py_RETURN_FALSE; -} - static PyType_Slot not_freezable_error_slots[] = { {0, NULL}, }; @@ -160,7 +136,6 @@ static struct PyMethodDef immutable_methods[] = { IMMUTABLE_REGISTER_FREEZABLE_METHODDEF IMMUTABLE_FREEZE_METHODDEF IMMUTABLE_ISFROZEN_METHODDEF - IMMUTABLE_ISFREEZABLE_METHODDEF { NULL, NULL } }; diff --git a/Python/immutability.c b/Python/immutability.c index 8ead7b6c4a0a93..5a64485587083e 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -390,7 +390,16 @@ static FreezableCheck check_freezable(struct _Py_immutability_state *state, PyOb { int result = 0; - result = PyObject_IsInstance(obj, (PyObject*)&_PyNotFreezable_Type); + /* + Immutable(TODO) + This is technically all that is needed, but without the ability to back out + the immutability, the instance will still be frozen, which is why the alternative code + is used for now. + if(obj == (PyObject *)&_PyNotFreezable_Type){ + return INVALID_NOT_FREEZABLE; + } + */ + result = PyObject_IsInstance(obj, (PyObject *)&_PyNotFreezable_Type); if(result == -1){ return ERROR; } @@ -418,31 +427,6 @@ static FreezableCheck check_freezable(struct _Py_immutability_state *state, PyOb } -int _PyImmutability_IsFreezable(PyObject *obj) -{ - struct _Py_immutability_state *state = get_immutable_state(); - if(state == NULL){ - PyErr_SetString(PyExc_RuntimeError, "Failed to initialize immutability state"); - return 0; - } - - switch(check_freezable(state, obj)) - { - case VALID_BUILTIN: - case VALID_EXPLICIT: - case VALID_IMPLICIT: - return 1; - case INVALID_NOT_FREEZABLE: - case INVALID_C_EXTENSIONS: - return 0; - case ERROR: - return -1; - } - - PyErr_SetString(PyExc_RuntimeError, "Unknown state"); - return -1; -} - int _PyImmutability_RegisterFreezable(PyTypeObject* tp) { PyObject *ref; @@ -569,7 +553,7 @@ int _PyImmutability_Freeze(PyObject* obj) goto error; } - if(check == VALID_IMPLICIT) + if(check != VALID_EXPLICIT) { if(push(frontier, type->tp_mro)) { diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index ac8c520b754bd9..e0dcb1af3232c9 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -165,8 +165,8 @@ static const char* _Py_stdlib_module_names[] = { "idlelib", "imaplib", "imghdr", -"importlib", "immutable", +"importlib", "inspect", "io", "ipaddress", From 4866d044349c0ba5c9baebdb7380908043bc51b1 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Fri, 2 May 2025 15:19:58 +0100 Subject: [PATCH 37/40] Using import helper Signed-off-by: Matthew A Johnson --- Lib/test/test_freeze/test_bz2.py | 5 +++- Lib/test/test_freeze/test_sqllite3.py | 37 --------------------------- Lib/test/test_freeze/test_zlib.py | 15 ----------- 3 files changed, 4 insertions(+), 53 deletions(-) delete mode 100644 Lib/test/test_freeze/test_sqllite3.py delete mode 100644 Lib/test/test_freeze/test_zlib.py diff --git a/Lib/test/test_freeze/test_bz2.py b/Lib/test/test_freeze/test_bz2.py index c37c2e0985d2eb..d7530eef06eee0 100644 --- a/Lib/test/test_freeze/test_bz2.py +++ b/Lib/test/test_freeze/test_bz2.py @@ -1,8 +1,11 @@ -from bz2 import BZ2Compressor, BZ2Decompressor +from test.support import import_helper import unittest from .test_common import BaseNotFreezableTest +bz2 = import_helper.import_module('bz2') +from bz2 import BZ2Compressor, BZ2Decompressor + class TestBZ2Compressor(BaseNotFreezableTest): def __init__(self, *args, **kwargs): diff --git a/Lib/test/test_freeze/test_sqllite3.py b/Lib/test/test_freeze/test_sqllite3.py deleted file mode 100644 index 60db5fa4e96661..00000000000000 --- a/Lib/test/test_freeze/test_sqllite3.py +++ /dev/null @@ -1,37 +0,0 @@ -from test.support import import_helper - -import_helper.import_module('_sqlite3') - -import sqlite3 - -from .test_common import BaseNotFreezableTest - - -class TestConnection(BaseNotFreezableTest): - def __init__(self, *args, **kwargs): - super().__init__(*args, obj=sqlite3.connect(':memory:'), **kwargs) - - def tearDown(self): - self.obj.close() - - -class TestCursor(BaseNotFreezableTest): - def __init__(self, *args, **kwargs): - self.con = sqlite3.connect(':memory:') - super().__init__(*args, obj=self.con.cursor(), **kwargs) - - def tearDown(self): - self.con.close() - - -class TestBlob(BaseNotFreezableTest): - def __init__(self, *args, **kwargs): - self.con = sqlite3.connect(':memory:') - self.con.execute("CREATE TABLE test(blob_col blob)") - self.con.execute("INSERT INTO test(blob_col) VALUES(zeroblob(13))") - - blob = self.con.blobopen("test", "blob_col", 1) - super().__init__(*args, obj=blob, **kwargs) - - def tearDown(self): - self.con.close() diff --git a/Lib/test/test_freeze/test_zlib.py b/Lib/test/test_freeze/test_zlib.py deleted file mode 100644 index 2c028e461f666e..00000000000000 --- a/Lib/test/test_freeze/test_zlib.py +++ /dev/null @@ -1,15 +0,0 @@ -import zlib - -from .test_common import BaseNotFreezableTest - -class ZlibCompressTest(BaseNotFreezableTest): - def __init__(self, *args, **kwargs): - super().__init__(*args, obj=zlib.compressobj(), **kwargs) - -class ZlibDecompressTest(BaseNotFreezableTest): - def __init__(self, *args, **kwargs): - super().__init__(*args, obj=zlib.decompressobj(), **kwargs) - -class ZlibDecompressorTest(BaseNotFreezableTest): - def __init__(self, *args, **kwargs): - super().__init__(*args, obj=zlib._ZlibDecompressor(), **kwargs) From 0387c128338e17d132454f91671dc6c9a5847235 Mon Sep 17 00:00:00 2001 From: Matthew A Johnson Date: Mon, 12 May 2025 10:24:17 +0100 Subject: [PATCH 38/40] Adding the bases tuple as discussed Signed-off-by: Matthew A Johnson --- Python/immutability.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Python/immutability.c b/Python/immutability.c index 5a64485587083e..093d5b3fd4df6c 100644 --- a/Python/immutability.c +++ b/Python/immutability.c @@ -559,6 +559,13 @@ int _PyImmutability_Freeze(PyObject* obj) { goto error; } + + // We need to freeze the tuple object, even though the types + // within will have been frozen already. + if(push(frontier, type->tp_bases)) + { + goto error; + } } } else From c86727155f82e93d8299de9961986434d617ae07 Mon Sep 17 00:00:00 2001 From: Tage Johansson Date: Thu, 10 Jul 2025 15:41:33 +0200 Subject: [PATCH 39/40] Initial simple Python module. --- Modules/bocmodule.c | 63 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 Modules/bocmodule.c diff --git a/Modules/bocmodule.c b/Modules/bocmodule.c new file mode 100644 index 00000000000000..11716049731ab9 --- /dev/null +++ b/Modules/bocmodule.c @@ -0,0 +1,63 @@ +#define PY_SSIZE_T_CLEAN // Not necessary since Python 3.13 +#include + +// SpamError exception +static PyObject *SpamError = NULL; + +static int +spam_module_exec(PyObject *m) +{ + if (SpamError != NULL) { + PyErr_SetString(PyExc_ImportError, + "cannot initialise spam module more than once"); + return -1; + } + SpamError = PyErr_NewException("spam.error", NULL, NULL); + if (PyModule_AddObjectRef(m, "SpamError", SpamError)) { + return -1; + } + + return 0; +} + +static PyObject * +spam_system(PyObject *self, PyObject *args) +{ + const char *command; + + if (!PyArg_ParseTuple(args, "s", &command)) { + return NULL; + } + + int sts = system(command); + if (sts < 0) { + PyErr_SetString(SpamError, "System command failed"); + return NULL; + } + + return PyLong_FromLong(sts); +} + +static PyMethodDef spam_methods[] = { + { "system", spam_system, METH_VARARGS, "Execute a shell command." }, + { NULL, NULL, 0, NULL } // Sentinel +}; + +static PyModuleDef_Slot spam_module_slots[] = { + { Py_mod_exec, spam_module_exec }, + { 0, NULL } +}; + +static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + .m_size = 0, // non-negative + .m_methods = spam_methods, + .m_slots = spam_module_slots, +}; + +PyMODINIT_FUNC +PyInit_spam(void) +{ + return PyModuleDef_Init(&spam_module); +} From 81aebf3dcb7ef3101ec49509601d04b25fb218dd Mon Sep 17 00:00:00 2001 From: Tage Johansson Date: Thu, 10 Jul 2025 16:37:21 +0200 Subject: [PATCH 40/40] Add .clang-format file. Source: --- .clang-format | 232 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000000000..83f482cc1ef543 --- /dev/null +++ b/.clang-format @@ -0,0 +1,232 @@ +# Reference: https://clang.llvm.org/docs/ClangFormatStyleOptions.html +Language: Cpp +BasedOnStyle: Google +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignArrayOfStructures: Left +AlignConsecutiveAssignments: None +AlignConsecutiveBitFields: AcrossEmptyLines +AlignConsecutiveDeclarations: None +AlignConsecutiveMacros: Consecutive +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: + Kind: Always + OverEmptyLines: 1 +AllowAllArgumentsOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowBreakBeforeNoexceptSpecifier: OnlyWithParen +AllowShortBlocksOnASingleLine: Never +AllowShortCaseExpressionOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: false +AllowShortCompoundRequirementOnASingleLine: true +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakBeforeMultilineStrings: true +BinPackArguments: true +BinPackParameters: false +BitFieldColonSpacing: None +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterExternBlock: false + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: true + BeforeElse: true + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakAdjacentStringLiterals: true +BreakAfterAttributes: Leave +BreakAfterReturnType: AllDefinitions +BreakBeforeBinaryOperators: NonAssignment +BreakBeforeConceptDeclarations: Always +BreakBeforeBraces: Custom +BreakBeforeInlineASMColon: OnlyMultiline +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakFunctionDefinitionParameters: false +BreakInheritanceList: BeforeColon +BreakStringLiterals: true +BreakTemplateDeclarations: Yes +ColumnLimit: 79 +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: false +DerivePointerAlignment: true +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +FixNamespaceComments: true +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^[<"]Python\.h[">]$' + Priority: 2 + CaseSensitive: true + - Regex: '^<[[:alnum:]_/]+(\.h)?>$' + Priority: 1 + - Regex: '^"cpython/.*\.h"$' + Priority: 3 + CaseSensitive: true + - Regex: '^"pycore.*\.h"$' + Priority: 4 + CaseSensitive: true + - Regex: '.*' + Priority: 5 +IndentAccessModifiers: false +IndentCaseBlocks: false +IndentCaseLabels: true +IndentExternBlock: NoIndent +IndentGotoLabels: false +IndentPPDirectives: None +IndentRequiresClause: true +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertBraces: true +InsertNewlineAtEOF: true +InsertTrailingCommas: None +KeepEmptyLines: + AtEndOfFile: false + AtStartOfBlock: false + AtStartOfFile: false +LambdaBodyIndentation: Signature +LineEnding: DeriveLF +MacroBlockBegin: '' +MacroBlockEnd: '' +Macros: + - >- + PyObject_HEAD_INIT(type)={ + /* this is not exactly match with PyObject_HEAD_INIT in Python source code + * but it is enough for clang-format */ + { 0xFFFFFFFF }, + (type) + }, + - >- + PyVarObject_HEAD_INIT(type, size)={ + { + /* manually expand PyObject_HEAD_INIT(type) above + * because clang-format do not support recursive expansion */ + { 0xFFFFFFFF }, + (type) + }, + (size) + }, + # Add indentation for aligning parameters when the definition spans multiple lines. + - PyAPI_FUNC(RTYPE)=extern RTYPE /* */ + # Add a qualifier to properly indent right aligned pointer macro PyAPI_DATA(PyObject *) + # This is a workaround for clang-format and has no effect on the formatted code. + - PyAPI_DATA(RTYPE)=extern RTYPE const + - PyMODINIT_FUNC=PyObject * +MainIncludeChar: Quote +MaxEmptyLinesToKeep: 2 +NamespaceIndentation: None +PackConstructorInitializers: NextLine +PenaltyBreakAssignment: 20 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakScopeResolution: 500 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyIndentedWhitespace: 0 +PenaltyReturnTypeOnItsOwnLine: 1000000000 +PointerAlignment: Right +PPIndentWidth: -1 +QualifierAlignment: Custom +QualifierOrder: + [friend, static, inline, const, constexpr, volatile, type, restrict] +ReferenceAlignment: Right +ReflowComments: false +RemoveBracesLLVM: false +RemoveParentheses: MultipleParentheses +RemoveSemicolon: true +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SkipMacroDefinitionBody: false +SortIncludes: CaseSensitive +SortUsingDeclarations: LexicographicNumeric +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceAroundPointerQualifiers: Default +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeJsonColon: false +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: Never +SpacesInContainerLiterals: true +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParens: Never +SpacesInSquareBrackets: false +Standard: c++11 +StatementAttributeLikeMacros: + # Defined in Include/pyport.h + - Py_DEPRECATED + - _Py_HOT_FUNCTION + - Py_ALWAYS_INLINE + - Py_NO_INLINE + - Py_ALIGNED + - Py_GCC_ATTRIBUTE + - _Py_NO_RETURN + - _Py_FALLTHROUGH +StatementMacros: + - PyObject_HEAD + - PyObject_VAR_HEAD + - PyException_HEAD + - _PyTZINFO_HEAD + - _PyDateTime_TIMEHEAD + - _PyDateTime_DATETIMEHEAD + - _PyGenObject_HEAD + - Window_NoArgNoReturnFunction + - Window_NoArgTrueFalseFunction + - Window_NoArgNoReturnVoidFunction + - Window_NoArg2TupleReturnFunction + - Window_OneArgNoReturnVoidFunction + - Window_OneArgNoReturnFunction + - Window_TwoArgNoReturnFunction +TableGenBreakInsideDAGArg: DontBreak +TabWidth: 4 +TypeNames: + # Defined in Include/pytypedefs.h + - PyModuleDef + - PyModuleDef_Slot + - PyMethodDef + - PyGetSetDef + - PyMemberDef + - PyObject + - PyLongObject + - PyTypeObject + - PyCodeObject + - PyFrameObject + - PyThreadState + - PyInterpreterState +UseTab: Never +WhitespaceSensitiveMacros: + - _Py_XSTRINGIFY + - Py_STRINGIFY