diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index e59611c07fa793..c9902ea0fd22a4 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -57,6 +57,13 @@ extern "C" { #define Py_TAGGED_SHIFT 2 +/* References to immortal objects always have their tag bit set to Py_TAG_REFCNT + * as they can (must) have their reclamation deferred */ + +#if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT +# error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT" +#endif + #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); @@ -502,7 +509,12 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj) assert(obj != NULL); // Make sure we don't take an already tagged value. assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - return (_PyStackRef){ .bits = (uintptr_t)obj }; + if (_Py_IsImmortal(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; + } + else { + return (_PyStackRef){ .bits = (uintptr_t)obj }; + } } # define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) @@ -511,7 +523,7 @@ PyStackRef_IsHeapSafe(_PyStackRef stackref) { if (PyStackRef_IsDeferred(stackref)) { PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); - return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj); + return obj == NULL || _PyObject_HasDeferredRefcount(obj); } return true; } @@ -631,13 +643,6 @@ PyStackRef_AsStrongReference(_PyStackRef stackref) // With GIL -/* References to immortal objects always have their tag bit set to Py_TAG_REFCNT - * as they can (must) have their reclamation deferred */ - -#if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT -# error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT" -#endif - #define BITS_TO_PTR(REF) ((PyObject *)((REF).bits)) #define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_REFCNT))) diff --git a/Objects/object.c b/Objects/object.c index 0540112d7d2acf..21a3e2fcb768cf 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2675,6 +2675,12 @@ _Py_SetImmortalUntracked(PyObject *op) assert(PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); } +#endif +#ifdef Py_GIL_DISABLED + // We set these bits for all immortals, even static one, to simplify + // further checks on hot path. In fact, if _Py_IsImmortal(op) == 1, + // then _PyObject_HasDeferredRefcount(op) == 1. + _Py_atomic_or_uint8(&op->ob_gc_bits, _PyGC_BITS_DEFERRED); #endif // Check if already immortal to avoid degrading from static immortal to plain immortal if (_Py_IsImmortal(op)) { @@ -2684,7 +2690,6 @@ _Py_SetImmortalUntracked(PyObject *op) op->ob_tid = _Py_UNOWNED_TID; op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; op->ob_ref_shared = 0; - _Py_atomic_or_uint8(&op->ob_gc_bits, _PyGC_BITS_DEFERRED); #elif SIZEOF_VOID_P > 4 op->ob_flags = _Py_IMMORTAL_FLAGS; op->ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT;