Adding Deep Immutability to 3.12

.clang-format

@@ -0,0 +1,232 @@
+# Reference: https://clang.llvm.org/docs/ClangFormatStyleOptions.html
+Language: Cpp
+BasedOnStyle: Google
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignArrayOfStructures: Left
+AlignConsecutiveAssignments: None
+AlignConsecutiveBitFields: AcrossEmptyLines
+AlignConsecutiveDeclarations: None
+AlignConsecutiveMacros: Consecutive
+AlignEscapedNewlines: Right
+AlignOperands: Align
+AlignTrailingComments:
+  Kind: Always
+  OverEmptyLines: 1
+AllowAllArgumentsOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowBreakBeforeNoexceptSpecifier: OnlyWithParen
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseExpressionOnASingleLine: true
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortCompoundRequirementOnASingleLine: true
+AllowShortEnumsOnASingleLine: true
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLambdasOnASingleLine: All
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+BinPackArguments: true
+BinPackParameters: false
+BitFieldColonSpacing: None
+BraceWrapping:
+  AfterCaseLabel: false
+  AfterClass: false
+  AfterControlStatement: Never
+  AfterEnum: false
+  AfterExternBlock: false
+  AfterFunction: true
+  AfterNamespace: false
+  AfterObjCDeclaration: false
+  AfterStruct: false
+  AfterUnion: false
+  BeforeCatch: true
+  BeforeElse: true
+  BeforeLambdaBody: false
+  BeforeWhile: false
+  IndentBraces: false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakAdjacentStringLiterals: true
+BreakAfterAttributes: Leave
+BreakAfterReturnType: AllDefinitions
+BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeConceptDeclarations: Always
+BreakBeforeBraces: Custom
+BreakBeforeInlineASMColon: OnlyMultiline
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializers: BeforeColon
+BreakFunctionDefinitionParameters: false
+BreakInheritanceList: BeforeColon
+BreakStringLiterals: true
+BreakTemplateDeclarations: Yes
+ColumnLimit: 79
+CompactNamespaces: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: false
+DerivePointerAlignment: true
+EmptyLineAfterAccessModifier: Never
+EmptyLineBeforeAccessModifier: LogicalBlock
+FixNamespaceComments: true
+IncludeBlocks: Regroup
+IncludeCategories:
+  - Regex: '^[<"]Python\.h[">]$'
+    Priority: 2
+    CaseSensitive: true
+  - Regex: '^<[[:alnum:]_/]+(\.h)?>$'
+    Priority: 1
+  - Regex: '^"cpython/.*\.h"$'
+    Priority: 3
+    CaseSensitive: true
+  - Regex: '^"pycore.*\.h"$'
+    Priority: 4
+    CaseSensitive: true
+  - Regex: '.*'
+    Priority: 5
+IndentAccessModifiers: false
+IndentCaseBlocks: false
+IndentCaseLabels: true
+IndentExternBlock: NoIndent
+IndentGotoLabels: false
+IndentPPDirectives: None
+IndentRequiresClause: true
+IndentWidth: 4
+IndentWrappedFunctionNames: false
+InsertBraces: true
+InsertNewlineAtEOF: true
+InsertTrailingCommas: None
+KeepEmptyLines:
+  AtEndOfFile: false
+  AtStartOfBlock: false
+  AtStartOfFile: false
+LambdaBodyIndentation: Signature
+LineEnding: DeriveLF
+MacroBlockBegin: ''
+MacroBlockEnd: ''
+Macros:
+  - >-
+    PyObject_HEAD_INIT(type)={
+        /* this is not exactly match with PyObject_HEAD_INIT in Python source code
+         * but it is enough for clang-format */
+        { 0xFFFFFFFF },
+        (type)
+    },
+  - >-
+    PyVarObject_HEAD_INIT(type, size)={
+        {
+            /* manually expand PyObject_HEAD_INIT(type) above
+             * because clang-format do not support recursive expansion */
+            { 0xFFFFFFFF },
+            (type)
+        },
+        (size)
+    },
+  # Add indentation for aligning parameters when the definition spans multiple lines.
+  - PyAPI_FUNC(RTYPE)=extern RTYPE /*  */
+  # Add a qualifier to properly indent right aligned pointer macro PyAPI_DATA(PyObject *)
+  # This is a workaround for clang-format and has no effect on the formatted code.
+  - PyAPI_DATA(RTYPE)=extern RTYPE const
+  - PyMODINIT_FUNC=PyObject *
+MainIncludeChar: Quote
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: None
+PackConstructorInitializers: NextLine
+PenaltyBreakAssignment: 20
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakOpenParenthesis: 0
+PenaltyBreakScopeResolution: 500
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyIndentedWhitespace: 0
+PenaltyReturnTypeOnItsOwnLine: 1000000000
+PointerAlignment: Right
+PPIndentWidth: -1
+QualifierAlignment: Custom
+QualifierOrder:
+  [friend, static, inline, const, constexpr, volatile, type, restrict]
+ReferenceAlignment: Right
+ReflowComments: false
+RemoveBracesLLVM: false
+RemoveParentheses: MultipleParentheses
+RemoveSemicolon: true
+RequiresClausePosition: OwnLine
+RequiresExpressionIndentation: OuterScope
+SeparateDefinitionBlocks: Leave
+ShortNamespaceLines: 1
+SkipMacroDefinitionBody: false
+SortIncludes: CaseSensitive
+SortUsingDeclarations: LexicographicNumeric
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceAroundPointerQualifiers: Default
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCaseColon: false
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeJsonColon: false
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceBeforeSquareBrackets: false
+SpaceInEmptyBlock: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles: Never
+SpacesInContainerLiterals: true
+SpacesInLineCommentPrefix:
+  Minimum: 1
+  Maximum: -1
+SpacesInParens: Never
+SpacesInSquareBrackets: false
+Standard: c++11
+StatementAttributeLikeMacros:
+  # Defined in Include/pyport.h
+  - Py_DEPRECATED
+  - _Py_HOT_FUNCTION
+  - Py_ALWAYS_INLINE
+  - Py_NO_INLINE
+  - Py_ALIGNED
+  - Py_GCC_ATTRIBUTE
+  - _Py_NO_RETURN
+  - _Py_FALLTHROUGH
+StatementMacros:
+  - PyObject_HEAD
+  - PyObject_VAR_HEAD
+  - PyException_HEAD
+  - _PyTZINFO_HEAD
+  - _PyDateTime_TIMEHEAD
+  - _PyDateTime_DATETIMEHEAD
+  - _PyGenObject_HEAD
+  - Window_NoArgNoReturnFunction
+  - Window_NoArgTrueFalseFunction
+  - Window_NoArgNoReturnVoidFunction
+  - Window_NoArg2TupleReturnFunction
+  - Window_OneArgNoReturnVoidFunction
+  - Window_OneArgNoReturnFunction
+  - Window_TwoArgNoReturnFunction
+TableGenBreakInsideDAGArg: DontBreak
+TabWidth: 4
+TypeNames:
+  # Defined in Include/pytypedefs.h
+  - PyModuleDef
+  - PyModuleDef_Slot
+  - PyMethodDef
+  - PyGetSetDef
+  - PyMemberDef
+  - PyObject
+  - PyLongObject
+  - PyTypeObject
+  - PyCodeObject
+  - PyFrameObject
+  - PyThreadState
+  - PyInterpreterState
+UseTab: Never
+WhitespaceSensitiveMacros:
+  - _Py_XSTRINGIFY
+  - Py_STRINGIFY

.gitignore

@@ -156,3 +156,6 @@ Python/frozen_modules/MANIFEST
 # Ignore ./python binary on Unix but still look into ./Python/ directory.
 /python
 !/Python/
+
+# Ignore the build directory.
+build*

Include/Python.h

@@ -105,5 +105,6 @@
 #include "fileutils.h"
 #include "cpython/pyfpe.h"
 #include "tracemalloc.h"
+#include "immutability.h"
 
 #endif /* !Py_PYTHON_H */

Include/cpython/immutability.h

@@ -0,0 +1,8 @@
+#ifndef Py_CPYTHON_IMMUTABLE_H
+#  error "this header file must not be included directly"
+#endif
+
+PyAPI_DATA(PyTypeObject) _PyNotFreezable_Type;
+
+PyAPI_FUNC(int) _PyImmutability_Freeze(PyObject*);
+PyAPI_FUNC(int) _PyImmutability_RegisterFreezable(PyTypeObject*);

Include/cpython/object.h

@@ -273,6 +273,7 @@ typedef struct _heaptypeobject {
 PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *);
 PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *);
 PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *);
+PyAPI_FUNC(int) _PyType_HasExtensionSlots(PyTypeObject *);
 PyAPI_FUNC(PyObject *) _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *);
 #ifndef Py_BUILD_CORE
 // Backward compatibility for 3rd-party extensions

Include/cpython/pylifecycle.h

@@ -51,7 +51,6 @@ Py_DEPRECATED(3.11) PyAPI_FUNC(void) _Py_SetProgramFullPath(const wchar_t *);
 PyAPI_FUNC(const char *) _Py_gitidentifier(void);
 PyAPI_FUNC(const char *) _Py_gitversion(void);
 
-PyAPI_FUNC(int) _Py_IsFinalizing(void);
 PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp);
 
 /* Random */

Include/immutability.h

@@ -0,0 +1,19 @@
+#ifndef Py_IMMUTABILITY_H
+#define Py_IMMUTABILITY_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_IMMUTABLE_H
+#  include "cpython/immutability.h"
+#  undef Py_CPYTHON_IMMUTABLE_H
+#endif
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_IMMUTABILITY_H */

Include/internal/pycore_immutability.h

@@ -0,0 +1,21 @@
+#ifndef Py_INTERNAL_IMMUTABILITY_H
+#define Py_INTERNAL_IMMUTABILITY_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "Py_BUILD_CORE must be defined to include this header"
+#endif
+
+struct _Py_immutability_state {
+    PyObject *module_locks;
+    PyObject *blocking_on;
+    PyObject *freezable_types;
+    PyObject *destroy_cb;
+};
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_IMMUTABILITY_H */

Include/internal/pycore_interp.h

@@ -25,6 +25,7 @@ extern "C" {
 #include "pycore_gc.h"            // struct _gc_runtime_state
 #include "pycore_global_objects.h"  // struct _Py_interp_static_objects
 #include "pycore_import.h"        // struct _import_state
+#include "pycore_immutability.h"    // struct _immutability_runtime_state
 #include "pycore_instruments.h"   // _PY_MONITORING_EVENTS
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_object_state.h"   // struct _py_object_state
@@ -176,6 +177,7 @@ struct _is {
     struct _Py_async_gen_state async_gen;
     struct _Py_context_state context;
     struct _Py_exc_state exc_state;
+    struct _Py_immutability_state immutability;
 
     struct ast_state ast;
     struct types_state types;

Include/internal/pycore_object.h

@@ -71,7 +71,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n)
 static inline void _Py_SetImmortal(PyObject *op)
 {
     if (op) {
-        op->ob_refcnt = _Py_IMMORTAL_REFCNT;
+        op->ob_refcnt = (op->ob_refcnt & _Py_IMMUTABLE_MASK) | _Py_IMMORTAL_REFCNT;
     }
 }
 #define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op))
@@ -80,7 +80,8 @@ static inline void _Py_SetImmortal(PyObject *op)
 static inline void _Py_ClearImmortal(PyObject *op)
 {
     if (op) {
-        assert(op->ob_refcnt == _Py_IMMORTAL_REFCNT);
+        assert((op->ob_refcnt & _Py_REFCNT_MASK) == _Py_IMMORTAL_REFCNT);
+        // note this also clears the _Py_IMMUTABLE_FLAG, if set
         op->ob_refcnt = 1;
         Py_DECREF(op);
     }
@@ -91,17 +92,33 @@ static inline void _Py_ClearImmortal(PyObject *op)
         op = NULL; \
     } while (0)
 
+static inline void _Py_SetImmutable(PyObject *op)
+{
+    if(op) {
+        op->ob_refcnt |= _Py_IMMUTABLE_FLAG;
+    }
+}
+#define _Py_SetImmutable(op) _Py_SetImmutable(_PyObject_CAST(op))
+
 static inline void
 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 {
-    if (_Py_IsImmortal(op)) {
+    if (_Py_IsImmortalOrImmutable(op)) {
+        if (_Py_IsImmortal(op)) {
+            return;
+        }
+        assert(_Py_IsImmutable(op));
+        if (_Py_DecRef_Immutable(op)) {
+            destruct(op);
+        }
         return;
     }
     _Py_DECREF_STAT_INC();
 #ifdef Py_REF_DEBUG
     _Py_DEC_REFTOTAL(_PyInterpreterState_GET());
 #endif
-    if (--op->ob_refcnt != 0) {
+    op->ob_refcnt -= 1;
+    if ((op->ob_refcnt & _Py_REFCNT_MASK) != 0) {
         assert(op->ob_refcnt > 0);
     }
     else {
@@ -115,7 +132,11 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 static inline void
 _Py_DECREF_NO_DEALLOC(PyObject *op)
 {
-    if (_Py_IsImmortal(op)) {
+    if (_Py_IsImmortalOrImmutable(op)) {
+        if (_Py_IsImmortal(op)) {
+            return;
+        }
+        _Py_DecRef_Immutable(op);
         return;
     }
     _Py_DECREF_STAT_INC();