diff --git a/.github/workflows/build_min.yml b/.github/workflows/build_min.yml index c59e9fb3e7bbc9..0385280d8aa00a 100644 --- a/.github/workflows/build_min.yml +++ b/.github/workflows/build_min.yml @@ -134,7 +134,7 @@ jobs: check_generated_files: name: 'Check if generated files are up to date' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 0f58326f6c06b7..a74642cf921861 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -466,7 +466,7 @@ The following functions and structs are used to create * :c:member:`~PyTypeObject.tp_dict` * :c:member:`~PyTypeObject.tp_mro` - * :c:member:`~PyTypeObject.tp_cache` + * :c:member:`~PyTypeObject.tp_lock` * :c:member:`~PyTypeObject.tp_subclasses` * :c:member:`~PyTypeObject.tp_weaklist` * :c:member:`~PyTypeObject.tp_vectorcall` diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 0988090194a5fe..8214ad9806d750 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -133,7 +133,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | + | [:c:member:`~PyTypeObject.tp_lock`] | :c:type:`PyObject` * | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | [:c:member:`~PyTypeObject.tp_subclasses`] | void * | __subclasses__ | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -2013,9 +2013,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:func:`PyType_Ready`. -.. c:member:: PyObject* PyTypeObject.tp_cache +.. c:member:: PyObject* PyTypeObject.tp_lock - Unused. Internal use only. + Internal use only. **Inheritance:** diff --git a/Doc/data/python3.12.abi b/Doc/data/python3.12.abi index f4138bee406a27..645710c27f8c17 100644 --- a/Doc/data/python3.12.abi +++ b/Doc/data/python3.12.abi @@ -19363,7 +19363,7 @@ - + diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index ec939c28831c33..005ea65d86e7de 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -70,7 +70,7 @@ typedef struct _typeobject { inquiry tp_is_gc; /* For PyObject_IS_GC */ PyObject *tp_bases; PyObject *tp_mro; /* method resolution order */ - PyObject *tp_cache; + PyObject *tp_lock; PyObject *tp_subclasses; PyObject *tp_weaklist; destructor tp_del; diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ae7f780a93182a..bd2b6d8e28c6ea 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -215,7 +215,7 @@ struct _typeobject { inquiry tp_is_gc; /* For PyObject_IS_GC */ PyObject *tp_bases; PyObject *tp_mro; /* method resolution order */ - PyObject *tp_cache; /* no longer used */ + PyObject *tp_lock; /* used by immutable types */ void *tp_subclasses; /* for static builtin types this is an index */ PyObject *tp_weaklist; /* not used for static builtin types */ destructor tp_del; diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 8f044874121741..63d29974ef944f 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -40,7 +40,7 @@ typedef struct { #define _PyDictEntry_SetValue(entry, value) ((entry)->_me_value = value) #define _PyDictEntry_IsEmpty(entry) ((entry)->_me_value == NULL) -extern bool _PyDict_IsKeyImmutable(PyObject* op, PyObject* key); +extern PyObject *_PyDict_IsKeyImmutable(PyObject* op, PyObject* key); extern PyDictKeysObject *_PyDict_NewKeysForClass(void); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index ea075f96a3de9f..a41c9c8ddc7b02 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -17,6 +17,10 @@ extern "C" { PyObject* _Py_MakeImmutable(PyObject* obj); #define Py_MakeImmutable(op) _Py_MakeImmutable(_PyObject_CAST(op)) +PyObject* Py_MakeGlobalsImmutable(void); + +bool _PyGlobalsImmutable_Check(void); + #ifdef NDEBUG #define _Py_VPYDBG(fmt, ...) #define _Py_VPYDBGPRINT(fmt, ...) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index ec2e56f6ea9ca1..ec89c662b0bede 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -29,6 +29,7 @@ def _object_name(obj): # Bootstrap-related code ###################################################### # Modules injected manually by _setup() +_behaviors = None _thread = None _warnings = None _weakref = None @@ -407,6 +408,69 @@ def __repr__(self): return f'_DummyModuleLock({self.name!r}) at {id(self)}' +class _MultiInterpreterModuleLock: + """A recursive lock implementation which is able to detect deadlocks + across multiple subinterpreters. + """ + + def __init__(self, name): + self.lock = _behaviors.RLock() + self.wakeup = _behaviors.Lock() + self.name = name + self.owner = None + self.count = [] + self.waiters = [] + + def has_deadlock(self): + return _has_deadlocked( + # get the thread id of the current subinterpreter + target_id=_behaviors.get_ident(), + seen_ids=set(), + candidate_ids=[self.owner], + blocking_on=_blocking_on, + ) + + def acquire(self): + """ + Acquire the module lock. If a potential deadlock is detected, + a _DeadlockError is raised. + Otherwise, the lock is always acquired and True is returned. + """ + bid = _behaviors.get_ident() + with _BlockingOnManager(bid, self): + while True: + with self.lock: + if self.count == [] or self.owner == bid: + self.owner = bid + self.count.append(True) + return True + + if self.has_deadlock(): + raise _DeadlockError(f'deadlock detected by {self!r}') + + if self.wakeup.acquire(False): + self.waiters.append(None) + + self.wakeup.acquire() + self.wakeup.release() + + def release(self): + bid = _behaviors.get_ident() + with self.lock: + if self.owner != bid: + raise RuntimeError('cannot release un-acquired lock') + assert len(self.count) > 0 + self.count.pop() + if not len(self.count): + self.owner = None + if len(self.waiters) > 0: + self.waiters.pop() + self.wakeup.release() + + def __repr__(self): + return f'_MultiInterpreterModuleLock({self.name!r}) at {id(self)}' + + class _ModuleLockManager: def __init__(self, name): @@ -439,6 +503,8 @@ def _get_module_lock(name): if lock is None: if _thread is None: lock = _DummyModuleLock(name) + elif _behaviors is not None and _behaviors.running(): + lock = _MultiInterpreterModuleLock(name) else: lock = _ModuleLock(name) @@ -943,6 +1009,11 @@ def _load_unlocked(spec): finally: spec._initializing = False + if _behaviors is not None and _behaviors.running(): + # all modules must be made immutable upon load + if not isimmutable(module): + makeimmutable(module) + return module # A method used during testing of _load_unlocked() and by @@ -1518,7 +1589,7 @@ def _setup(sys_module, _imp_module): # Directly load built-in modules needed during bootstrap. self_module = sys.modules[__name__] - for builtin_name in ('_thread', '_warnings', '_weakref'): + for builtin_name in ('_thread', '_warnings', '_weakref', '_behaviors'): if builtin_name not in sys.modules: builtin_module = _builtin_from_name(builtin_name) else: diff --git a/Makefile.pre.in b/Makefile.pre.in index 0b80d67452011f..8f799bcecf13ac 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -431,6 +431,7 @@ PYTHON_OBJS= \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ + Python/regions.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ @@ -475,7 +476,6 @@ OBJECT_OBJS= \ Objects/obmalloc.o \ Objects/picklebufobject.o \ Objects/rangeobject.o \ - Objects/regions.o \ Objects/setobject.o \ Objects/sliceobject.o \ Objects/structseq.o \ diff --git a/Modules/Setup b/Modules/Setup index a8faa1d1028da5..9f767118f533f0 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -131,6 +131,7 @@ PYTHONPATH=$(COREPYTHONPATH) # Modules that should always be present (POSIX and Windows): #_asyncio _asynciomodule.c +#_behaviors _behaviorsmodule.c #_bisect _bisectmodule.c #_contextvars _contextvarsmodule.c #_csv _csv.c diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 8ef0f203a82a8e..31bf701b5383c9 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -13,6 +13,7 @@ _signal signalmodule.c _tracemalloc _tracemalloc.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig +_behaviors _behaviorsmodule.c _codecs _codecsmodule.c _collections _collectionsmodule.c errno errnomodule.c diff --git a/Modules/_behaviorsmodule.c b/Modules/_behaviorsmodule.c new file mode 100644 index 00000000000000..31d83fc44c6511 --- /dev/null +++ b/Modules/_behaviorsmodule.c @@ -0,0 +1,857 @@ +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include +#include +#include +#include "pycore_regions.h" + +/***************************************************************/ +/* Platform-specific Threading Aliases/Wrappers */ +/***************************************************************/ + +#if _WIN32 +#include + +typedef int PyBehaviorsLockStatus; +typedef LPCRITICAL_SECTION PyBehaviors_type_lock; + +#define PY_BEHAVIORS_LOCK_SUCCESS 0 +#define PY_BEHAVIORS_LOCK_BUSY 1 +#define PY_BEHAVIORS_LOCK_ERROR 2 + +PyBehaviors_type_lock PyBehaviors_allocate_lock(void) +{ + LPCRITICAL_SECTION lock = (LPCRITICAL_SECTION)PyMem_RawMalloc(sizeof(CRITICAL_SECTION)); + if (lock == NULL) + { + PyErr_NoMemory(); + return NULL; + } + + InitializeCriticalSection(lock); + return lock; +} + +void PyBehaviors_free_lock(PyBehaviors_type_lock lock) +{ + if (lock) + { + DeleteCriticalSection(lock); + PyMem_RawFree(lock); + } +} + +PyBehaviorsLockStatus PyBehaviors_acquire_lock(PyBehaviors_type_lock lock, int waitflag) +{ + if (waitflag) + { + EnterCriticalSection(lock); + return PY_BEHAVIORS_LOCK_SUCCESS; + } + + if (TryEnterCriticalSection(lock)) + { + return PY_BEHAVIORS_LOCK_SUCCESS; + } + + return PY_BEHAVIORS_LOCK_BUSY; +} + +PyBehaviorsLockStatus PyBehaviors_release_lock(PyBehaviors_type_lock lock) +{ + LeaveCriticalSection(lock); + return PY_BEHAVIORS_LOCK_SUCCESS; +} + +#elif __APPLE__ +#include + +typedef int PyBehaviorsLockStatus; +typedef pthread_mutex_t *PyBehaviors_type_lock; + +#define PY_BEHAVIORS_LOCK_SUCCESS 0 +#define PY_BEHAVIORS_LOCK_BUSY EBUSY +#define PY_BEHAVIORS_LOCK_ERROR EINVAL + +PyBehaviors_type_lock PyBehaviors_allocate_lock(void) +{ + pthread_mutex_t *lock = (pthread_mutex_t *)PyMem_RawMalloc(sizeof(pthread_mutex_t)); + if (lock == NULL) + { + PyErr_NoMemory(); + return NULL; + } + + pthread_mutex_init(lock, NULL); + return lock; +} + +void PyBehaviors_free_lock(PyBehaviors_type_lock lock) +{ + int r; + if (lock) + { + r = pthread_mutex_destroy(lock); + if (r == EBUSY) + { + PyErr_SetString(PyExc_RuntimeError, "Lock is busy"); + return; + } + + PyMem_RawFree(lock); + } +} + +PyBehaviorsLockStatus PyBehaviors_acquire_lock(PyBehaviors_type_lock lock, int waitflag) +{ + if (waitflag) + { + return pthread_mutex_lock(lock); + } + + return pthread_mutex_trylock(lock); +} + +PyBehaviorsLockStatus PyBehaviors_release_lock(PyBehaviors_type_lock lock) +{ + return pthread_mutex_unlock(lock); +} + +#else +#include +typedef int PyBehaviorsLockStatus; +typedef mtx_t *PyBehaviors_type_lock; + +#define PY_BEHAVIORS_LOCK_SUCCESS thrd_success +#define PY_BEHAVIORS_LOCK_BUSY thrd_busy +#define PY_BEHAVIORS_LOCK_ERROR thrd_error + +PyBehaviors_type_lock PyBehaviors_allocate_lock(void) +{ + mtx_t *lock = (mtx_t *)PyMem_RawMalloc(sizeof(mtx_t)); + if (lock == NULL) + { + PyErr_NoMemory(); + return NULL; + } + + if (mtx_init(lock, mtx_timed) == thrd_success) + { + return lock; + } + + PyMem_RawFree(lock); + PyErr_SetString(PyExc_RuntimeError, "Error initialising lock"); + return NULL; +} + +void PyBehaviors_free_lock(PyBehaviors_type_lock lock) +{ + if (lock) + { + mtx_destroy(lock); + PyMem_RawFree(lock); + } +} + +PyBehaviorsLockStatus PyBehaviors_acquire_lock(PyBehaviors_type_lock lock, int waitflag) +{ + if (waitflag) + { + return mtx_lock(lock); + } + + return mtx_trylock(lock); +} + +PyBehaviorsLockStatus PyBehaviors_release_lock(PyBehaviors_type_lock lock) +{ + return mtx_unlock(lock); +} +#endif + +static int +lock_acquire_parse_args(PyObject *args, PyObject *kwds, + bool *blocking) +{ + char *kwlist[] = {"blocking", NULL}; + *blocking = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|p:acquire", kwlist, + &blocking)) + return -1; + + return 0; +} + +/* lock object */ + +typedef struct behaviors_lock_object_s +{ + PyObject_HEAD + PyBehaviors_type_lock lock_lock; + PyObject *in_weakreflist; + char locked; /* for sanity checking */ +} lockobject; + +static int +lock_PyBehaviors_traverse(lockobject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +lock_PyBehaviors_dealloc(lockobject *self) +{ + PyObject_GC_UnTrack(self); + if (self->in_weakreflist != NULL) + { + PyObject_ClearWeakRefs((PyObject *)self); + } + if (self->lock_lock != NULL) + { + /* Unlock the lock so it's safe to free it */ + if (self->locked) + PyBehaviors_release_lock(self->lock_lock); + PyBehaviors_free_lock(self->lock_lock); + } + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +static PyObject * +lock_PyBehaviors_acquire_lock(lockobject *self, PyObject *args, PyObject *kwds) +{ + bool blocking; + if (lock_acquire_parse_args(args, kwds, &blocking) < 0) + return NULL; + + PyBehaviorsLockStatus r = PyBehaviors_acquire_lock(self->lock_lock, blocking); + if (r == PY_BEHAVIORS_LOCK_ERROR) + { + return NULL; + } + + if (r == PY_BEHAVIORS_LOCK_SUCCESS) + self->locked = 1; + return PyBool_FromLong(r == PY_BEHAVIORS_LOCK_SUCCESS); +} + +PyDoc_STRVAR(acquire_doc, + "acquire(blocking=True, timeout=-1) -> bool\n\ +(acquire_lock() is an obsolete synonym)\n\ +\n\ +Lock the lock. Without argument, this blocks if the lock is already\n\ +locked (even by the same thread), waiting for another thread to release\n\ +the lock, and return True once the lock is acquired.\n\ +With an argument, this will only block if the argument is true,\n\ +and the return value reflects whether the lock is acquired.\n\ +The blocking operation is interruptible."); + +static PyObject * +lock_PyBehaviors_release_lock(lockobject *self, PyObject *Py_UNUSED(ignored)) +{ + /* Sanity check: the lock must be locked */ + if (!self->locked) + { + PyErr_SetString(PyExc_RuntimeError, "release unlocked lock"); + return NULL; + } + + PyBehaviorsLockStatus r = PyBehaviors_release_lock(self->lock_lock); + if (r == PY_BEHAVIORS_LOCK_ERROR) + { + PyErr_SetString(PyExc_RuntimeError, "cannot release lock"); + return NULL; + } + + self->locked = 0; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(release_doc, + "release()\n\ +(release_lock() is an obsolete synonym)\n\ +\n\ +Release the lock, allowing another thread that is blocked waiting for\n\ +the lock to acquire the lock. The lock must be in the locked state,\n\ +but it needn't be locked by the same thread that unlocks it."); + +static PyObject * +lock_PyBehaviors_locked(lockobject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyBool_FromLong((long)self->locked); +} + +PyDoc_STRVAR(locked_doc, + "locked() -> bool\n\ +(locked_lock() is an obsolete synonym)\n\ +\n\ +Return whether the lock is in the locked state."); + +static PyObject * +lock_PyBehaviors_repr(lockobject *self) +{ + return PyUnicode_FromFormat("<%s %s object at %p>", + self->locked ? "locked" : "unlocked", Py_TYPE(self)->tp_name, self); +} + +static PyObject * +lock_PyBehaviors_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + lockobject *self = (lockobject *)type->tp_alloc(type, 0); + if (self == NULL) + { + return NULL; + } + + self->in_weakreflist = NULL; + self->locked = 0; + self->lock_lock = PyBehaviors_allocate_lock(); + + if (self->lock_lock == NULL) + { + Py_DECREF(self); + return NULL; + } + return (PyObject *)self; +} + +static PyMethodDef lock_methods[] = { + {"acquire_lock", _PyCFunction_CAST(lock_PyBehaviors_acquire_lock), + METH_VARARGS | METH_KEYWORDS, acquire_doc}, + {"acquire", _PyCFunction_CAST(lock_PyBehaviors_acquire_lock), + METH_VARARGS | METH_KEYWORDS, acquire_doc}, + {"release_lock", (PyCFunction)lock_PyBehaviors_release_lock, + METH_NOARGS, release_doc}, + {"release", (PyCFunction)lock_PyBehaviors_release_lock, + METH_NOARGS, release_doc}, + {"locked_lock", (PyCFunction)lock_PyBehaviors_locked, + METH_NOARGS, locked_doc}, + {"locked", (PyCFunction)lock_PyBehaviors_locked, + METH_NOARGS, locked_doc}, + {"__enter__", _PyCFunction_CAST(lock_PyBehaviors_acquire_lock), + METH_VARARGS | METH_KEYWORDS, acquire_doc}, + {"__exit__", (PyCFunction)lock_PyBehaviors_release_lock, + METH_VARARGS, release_doc}, + {NULL, NULL} /* sentinel */ +}; + +PyDoc_STRVAR(lock_doc, + "A lock object is a synchronization primitive. To create a lock,\n\ +call behaviors.Lock(). Methods are:\n\ +\n\ +acquire() -- lock the lock, possibly blocking until it can be obtained\n\ +release() -- unlock of the lock\n\ +locked() -- test whether the lock is currently locked\n\ +\n\ +A lock is not owned by the thread that locked it; another thread may\n\ +unlock it. A thread attempting to lock a lock that it has already locked\n\ +will block until another thread unlocks it. Deadlocks may ensue."); + +static PyMemberDef lock_type_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(lockobject, in_weakreflist), READONLY}, + {NULL}, +}; + +static PyType_Slot lock_type_slots[] = { + {Py_tp_dealloc, (destructor)lock_PyBehaviors_dealloc}, + {Py_tp_repr, (reprfunc)lock_PyBehaviors_repr}, + {Py_tp_doc, (void *)lock_doc}, + {Py_tp_methods, lock_methods}, + {Py_tp_new, lock_PyBehaviors_new}, + {Py_tp_traverse, lock_PyBehaviors_traverse}, + {Py_tp_members, lock_type_members}, + {0, 0}}; + +static PyType_Spec lock_type_spec = { + .name = "_behaviors.lock", + .basicsize = sizeof(lockobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), + .slots = lock_type_slots, +}; + +/* recursive lock */ + +typedef struct behaviors_rlock_object_s +{ + PyObject_HEAD + PyBehaviors_type_lock rlock_lock; + uint64_t rlock_interp; + unsigned long rlock_thread; + uint64_t rlock_count; + PyObject *in_weakreflist; +} rlockobject; + +static int +rlock_PyBehaviors_traverse(rlockobject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +rlock_PyBehaviors_dealloc(rlockobject *self) +{ + PyObject_GC_UnTrack(self); + if (self->in_weakreflist != NULL) + { + PyObject_ClearWeakRefs((PyObject *)self); + } + + if (self->rlock_count > 0) + { + PyBehaviors_release_lock(self->rlock_lock); + PyBehaviors_free_lock(self->rlock_lock); + } + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyObject * +rlock_PyBehaviors_acquire(PyObject *lock, PyObject *args, PyObject *kwds) +{ + uint64_t iid; + unsigned long tid; + bool blocking; + PyBehaviorsLockStatus r; + + rlockobject *self = (rlockobject *)lock; + + if (lock_acquire_parse_args(args, kwds, &blocking) < 0) + return NULL; + + iid = PyInterpreterState_GetID(PyInterpreterState_Get()); + tid = PyThread_get_thread_ident(); + if (self->rlock_count > 0 && iid == self->rlock_interp && tid == self->rlock_thread) + { + uint64_t count = self->rlock_count + 1; + if (count <= self->rlock_count) + { + PyErr_SetString(PyExc_OverflowError, + "Internal lock count overflowed"); + return NULL; + } + self->rlock_count = count; + Py_RETURN_TRUE; + } + + r = PyBehaviors_acquire_lock(self->rlock_lock, blocking); + + if (r == PY_BEHAVIORS_LOCK_SUCCESS) + { + assert(self->rlock_count == 0); + self->rlock_interp = iid; + self->rlock_thread = tid; + self->rlock_count = 1; + } + else if (r == PY_BEHAVIORS_LOCK_ERROR) + { + PyErr_SetString(PyExc_RuntimeError, + "error acquiring lock"); + return NULL; + } + + return PyBool_FromLong(r == PY_BEHAVIORS_LOCK_SUCCESS); +} + +PyDoc_STRVAR(rlock_acquire_doc, + "acquire(blocking=True) -> bool\n\ +\n\ +Lock the lock. `blocking` indicates whether we should wait\n\ +for the lock to be available or not. If `blocking` is False\n\ +and another thread holds the lock, the method will return False\n\ +immediately. If `blocking` is True and another thread holds\n\ +the lock, the method will wait for the lock to be released,\n\ +take it and then return True.\n\ +(note: the blocking operation is interruptible.)\n\ +\n\ +In all other cases, the method will return True immediately.\n\ +Precisely, if the current thread already holds the lock, its\n\ +internal counter is simply incremented. If nobody holds the lock,\n\ +the lock is taken and its internal counter initialized to 1."); + +static PyObject * +rlock_PyBehaviors_release(PyObject *lock, PyObject *Py_UNUSED(ignored)) +{ + rlockobject *self = (rlockobject *)lock; + uint64_t iid = PyInterpreterState_GetID(PyInterpreterState_Get()); + unsigned long tid = PyThread_get_thread_ident(); + + if (self->rlock_count == 0 || self->rlock_thread != tid || self->rlock_interp != iid) + { + PyErr_SetString(PyExc_RuntimeError, + "cannot release un-acquired lock"); + return NULL; + } + if (--self->rlock_count == 0) + { + self->rlock_interp = 0; + self->rlock_thread = 0; + if (PyBehaviors_release_lock(self->rlock_lock) == PY_BEHAVIORS_LOCK_ERROR) + { + PyErr_SetString(PyExc_RuntimeError, + "cannot release lock"); + return NULL; + } + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(rlock_release_doc, + "release()\n\ +\n\ +Release the lock, allowing another thread that is blocked waiting for\n\ +the lock to acquire the lock. The lock must be in the locked state,\n\ +and must be locked by the same thread that unlocks it; otherwise a\n\ +`RuntimeError` is raised.\n\ +\n\ +Do note that if the lock was acquire()d several times in a row by the\n\ +current thread, release() needs to be called as many times for the lock\n\ +to be available for other threads."); + +static PyObject * +rlock_PyBehaviors_acquire_restore(rlockobject *self, PyObject *args) +{ + uint64_t interp; + unsigned long thread; + unsigned long count; + PyBehaviorsLockStatus r; + + interp = PyThreadState_GetID(PyThreadState_GET()); + + if (!PyArg_ParseTuple(args, "(kk):_acquire_restore", &count, &thread)) + return NULL; + + r = PyBehaviors_acquire_lock(self->rlock_lock, 0); + if (r == PY_BEHAVIORS_LOCK_BUSY) + { + Py_BEGIN_ALLOW_THREADS + r = PyBehaviors_acquire_lock(self->rlock_lock, 1); + Py_END_ALLOW_THREADS + } + + if (r != PY_BEHAVIORS_LOCK_SUCCESS) + { + PyErr_SetString(PyExc_RuntimeError, "couldn't acquire lock"); + return NULL; + } + assert(self->rlock_count == 0); + self->rlock_interp = interp; + self->rlock_thread = thread; + self->rlock_count = count; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(rlock_acquire_restore_doc, + "_acquire_restore(state) -> None\n\ +\n\ +For internal use by `threading.Condition`."); + +static PyObject * +rlock_PyBehaviors_release_save(rlockobject *self, PyObject *Py_UNUSED(ignored)) +{ + uint64_t interp; + unsigned long thread; + uint64_t count; + PyBehaviorsLockStatus r; + + if (self->rlock_count == 0) + { + PyErr_SetString(PyExc_RuntimeError, + "cannot release un-acquired lock"); + return NULL; + } + + interp = self->rlock_interp; + thread = self->rlock_thread; + count = self->rlock_count; + self->rlock_count = 0; + self->rlock_thread = 0; + self->rlock_interp = 0; + r = PyBehaviors_release_lock(self->rlock_lock); + if (r != PY_BEHAVIORS_LOCK_SUCCESS) + { + PyErr_SetString(PyExc_RuntimeError, + "cannot release lock"); + return NULL; + } + + return Py_BuildValue("kkk", count, thread, interp); +} + +PyDoc_STRVAR(rlock_release_save_doc, + "_release_save() -> tuple\n\ +\n\ +For internal use by `threading.Condition`."); + +static PyObject * +rlock_PyBehaviors_is_owned(rlockobject *self, PyObject *Py_UNUSED(ignored)) +{ + uint64_t iid = PyInterpreterState_GetID(PyInterpreterState_Get()); + unsigned long tid = PyThread_get_thread_ident(); + + if (self->rlock_count > 0 && self->rlock_thread == tid && self->rlock_interp == iid) + { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(rlock_is_owned_doc, + "_is_owned() -> bool\n\ +\n\ +For internal use by `threading.Condition`."); + +static PyObject * +rlock_PyBehaviors_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + rlockobject *self = (rlockobject *)type->tp_alloc(type, 0); + if (self == NULL) + { + return NULL; + } + + self->in_weakreflist = NULL; + self->rlock_interp = 0; + self->rlock_thread = 0; + self->rlock_count = 0; + self->rlock_lock = PyBehaviors_allocate_lock(); + + if (self->rlock_lock == NULL) + { + Py_DECREF(self); + return NULL; + } + return (PyObject *)self; +} + +static PyObject * +rlock_PyBehaviors_repr(rlockobject *self) +{ + return PyUnicode_FromFormat("<%s %s object interp=%ld thread=%ld count=%lu at %p>", + self->rlock_count ? "locked" : "unlocked", + Py_TYPE(self)->tp_name, self->rlock_interp, self->rlock_thread, + self->rlock_count, self); +} + +/* + * This rlock implementation is meant, as much as possible, to be interchangeable with the Python Threads + * version of RLock while operating across interpreters. + */ + +static PyMethodDef rlock_methods[] = { + {"acquire", _PyCFunction_CAST(rlock_PyBehaviors_acquire), + METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, + {"release", (PyCFunction)rlock_PyBehaviors_release, + METH_NOARGS, rlock_release_doc}, + {"_is_owned", (PyCFunction)rlock_PyBehaviors_is_owned, + METH_NOARGS, rlock_is_owned_doc}, + {"_acquire_restore", (PyCFunction)rlock_PyBehaviors_acquire_restore, + METH_VARARGS, rlock_acquire_restore_doc}, + {"_release_save", (PyCFunction)rlock_PyBehaviors_release_save, + METH_NOARGS, rlock_release_save_doc}, + {"__enter__", _PyCFunction_CAST(rlock_PyBehaviors_acquire), + METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, + {"__exit__", (PyCFunction)rlock_PyBehaviors_release, + METH_VARARGS, rlock_release_doc}, + {NULL, NULL} /* Sentinel */ +}; + +static PyMemberDef rlock_type_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(rlockobject, in_weakreflist), READONLY}, + {NULL}, +}; + +static PyType_Slot rlock_type_slots[] = { + {Py_tp_dealloc, (destructor)rlock_PyBehaviors_dealloc}, + {Py_tp_repr, (reprfunc)rlock_PyBehaviors_repr}, + {Py_tp_methods, rlock_methods}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_new, rlock_PyBehaviors_new}, + {Py_tp_members, rlock_type_members}, + {Py_tp_traverse, rlock_PyBehaviors_traverse}, + {0, 0}, +}; + +static PyType_Spec rlock_type_spec = { + .name = "_behaviors.RLock", + .basicsize = sizeof(rlockobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), + .slots = rlock_type_slots, +}; + +typedef struct behaviors_state_s +{ + bool is_running; +} BehaviorsState; + +static PyObject *behaviors_start(PyObject *self, PyObject *noargs) +{ + PyObject *behaviors, *ret; + BehaviorsState *state; + + behaviors = PyImport_ImportModule("_behaviors"); + if (behaviors == NULL) + { + PyErr_SetString(PyExc_RuntimeError, "Unable to import behaviors module"); + return NULL; + } + + ret = Py_MakeGlobalsImmutable(); + if (ret == NULL) + { + Py_DECREF(behaviors); + return NULL; + } + + state = (BehaviorsState *)PyModule_GetState(behaviors); + state->is_running = true; + Py_DECREF(behaviors); + Py_RETURN_NONE; +} + +static PyObject *behaviors_running(PyObject *self, PyObject *noargs) +{ + PyObject *behaviors; + BehaviorsState *state; + bool is_running; + + behaviors = PyImport_ImportModule("_behaviors"); + if (behaviors == NULL) + { + // If we can't import the module, it's not running + PyErr_Clear(); + Py_RETURN_FALSE; + } + + state = (BehaviorsState *)PyModule_GetState(behaviors); + is_running = state->is_running; + Py_DECREF(behaviors); + + if (is_running) + { + Py_RETURN_TRUE; + } + else + { + Py_RETURN_FALSE; + } +} + +static PyObject *behaviors_wait(PyObject *self, PyObject *noargs) +{ + PyObject *behaviors; + BehaviorsState *state; + + behaviors = PyImport_ImportModule("_behaviors"); + if (behaviors == NULL) + { + PyErr_SetString(PyExc_RuntimeError, "Unable to import behaviors module"); + return NULL; + } + + state = (BehaviorsState *)PyModule_GetState(behaviors); + state->is_running = false; + Py_DECREF(behaviors); + Py_RETURN_NONE; +} + +static PyObject *behaviors_get_ident(PyObject *self, PyObject *noargs) +{ + int64_t iid = PyInterpreterState_GetID(PyInterpreterState_Get()); + unsigned long tid = PyThread_get_thread_ident(); + return Py_BuildValue("(k,k)", iid, tid); +} + +static PyMethodDef behaviors_methods[] = { + {"start", (PyCFunction)behaviors_start, METH_NOARGS, NULL}, + {"running", (PyCFunction)behaviors_running, METH_NOARGS, NULL}, + {"wait", (PyCFunction)behaviors_wait, METH_NOARGS, NULL}, + {"get_ident", (PyCFunction)behaviors_get_ident, METH_NOARGS, NULL}, + {NULL} /* Sentinel */ +}; + +static int behaviors_exec(PyObject *module) +{ + // Lock + PyTypeObject *lock_type = (PyTypeObject *)PyType_FromSpec(&lock_type_spec); + if (lock_type == NULL) + { + return -1; + } + if (PyModule_AddType(module, lock_type) < 0) + { + Py_DECREF(lock_type); + return -1; + } + Py_DECREF(lock_type); + + // RLock + PyTypeObject *rlock_type = (PyTypeObject *)PyType_FromSpec(&rlock_type_spec); + if (rlock_type == NULL) + { + return -1; + } + if (PyModule_AddType(module, rlock_type) < 0) + { + Py_DECREF(rlock_type); + return -1; + } + Py_DECREF(rlock_type); + + // BehaviorsState + BehaviorsState *state = (BehaviorsState *)PyModule_GetState(module); + state->is_running = false; + + return 0; +} + +static void behaviors_free(PyObject *module) +{ +} + +#ifdef Py_mod_exec +static PyModuleDef_Slot behaviors_slots[] = { + {Py_mod_exec, (void *)behaviors_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; +#endif + +static PyModuleDef behaviorsmoduledef = { + PyModuleDef_HEAD_INIT, + .m_name = "behaviors", + .m_methods = behaviors_methods, + .m_free = (freefunc)behaviors_free, +#ifdef Py_mod_exec + .m_slots = behaviors_slots, +#endif + .m_size = sizeof(BehaviorsState)}; + +PyMODINIT_FUNC PyInit__behaviors(void) +{ +#ifdef Py_mod_exec + return PyModuleDef_Init(&behaviorsmoduledef); +#else + PyObject *module; + module = PyModule_Create(&behaviorsmoduledef); + if (module == NULL) + return NULL; + + if (behaviors_exec(module) != 0) + { + Py_DECREF(module); + return NULL; + } + + return module; +#endif +} \ No newline at end of file diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7867ad70431b6..56d1b520b6ab18 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5997,7 +5997,7 @@ _PyDict_SendEvent(int watcher_bits, } } -bool +PyObject * _PyDict_IsKeyImmutable(PyObject* op, PyObject* key) { PyDictKeysObject *dk; @@ -6057,14 +6057,22 @@ _PyDict_IsKeyImmutable(PyObject* op, PyObject* key) _PyErr_SetRaisedException(tstate, exc); if (ix == DKIX_ERROR){ - return -1; + return NULL; } if (DK_IS_UNICODE(mp->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mp->ma_keys) + ix; - return _PyDictEntry_IsImmutable(ep); + if (_PyDictEntry_IsImmutable(ep)){ + Py_RETURN_TRUE; + }else{ + Py_RETURN_FALSE; + } } else { PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys) + ix; - return _PyDictEntry_IsImmutable(ep); + if(_PyDictEntry_IsImmutable(ep)){ + Py_RETURN_TRUE; + }else{ + Py_RETURN_FALSE; + } } } \ No newline at end of file diff --git a/Objects/genobject.c b/Objects/genobject.c index b13b52edf5c52d..70ad0b534f9968 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -870,7 +870,7 @@ PyTypeObject PyGen_Type = { 0, /* tp_is_gc */ 0, /* tp_bases */ 0, /* tp_mro */ - 0, /* tp_cache */ + 0, /* tp_lock */ 0, /* tp_subclasses */ 0, /* tp_weaklist */ 0, /* tp_del */ @@ -1220,7 +1220,7 @@ PyTypeObject PyCoro_Type = { 0, /* tp_is_gc */ 0, /* tp_bases */ 0, /* tp_mro */ - 0, /* tp_cache */ + 0, /* tp_lock */ 0, /* tp_subclasses */ 0, /* tp_weaklist */ 0, /* tp_del */ @@ -1636,7 +1636,7 @@ PyTypeObject PyAsyncGen_Type = { 0, /* tp_is_gc */ 0, /* tp_bases */ 0, /* tp_mro */ - 0, /* tp_cache */ + 0, /* tp_lock */ 0, /* tp_subclasses */ 0, /* tp_weaklist */ 0, /* tp_del */ diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 8397c2634712f9..98cb7ac537bb61 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -625,7 +625,7 @@ _PyModule_ClearDict(PyObject *d) /* First, clear only names starting with a single underscore */ pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { - if (value != Py_None && PyUnicode_Check(key) && !_PyDict_IsKeyImmutable(d, key)) { + if (value != Py_None && PyUnicode_Check(key) && Py_IsFalse(_PyDict_IsKeyImmutable(d, key))) { if (PyUnicode_READ_CHAR(key, 0) == '_' && PyUnicode_READ_CHAR(key, 1) != '_') { if (verbose > 1) { @@ -645,7 +645,7 @@ _PyModule_ClearDict(PyObject *d) /* Next, clear all names except for __builtins__ */ pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { - if (value != Py_None && PyUnicode_Check(key) && !_PyDict_IsKeyImmutable(d, key)) { + if (value != Py_None && PyUnicode_Check(key) && Py_IsFalse(_PyDict_IsKeyImmutable(d, key))) { if (PyUnicode_READ_CHAR(key, 0) != '_' || !_PyUnicode_EqualToASCIIString(key, "__builtins__")) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 8547f0f2a86277..8698a61612d9f7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -12,6 +12,7 @@ #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_regions.h" // Py_MakeImmutable() #include "pycore_typeobject.h" // struct type_cache #include "pycore_unionobject.h" // _Py_union_type_or #include "pycore_frame.h" // _PyInterpreterFrame @@ -446,8 +447,48 @@ _PyType_GetSubclasses(PyTypeObject *self) return NULL; } + PyObject* rlock = self->tp_lock; + PyObject* res; + if (rlock != NULL) { + PyObject* name = PyUnicode_FromString("acquire"); + if (name == NULL) { + Py_DECREF(list); + return NULL; + } + + res = PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + + if (res == NULL) { + Py_DECREF(list); + return NULL; + } + + if (Py_IsFalse(res)) { + Py_DECREF(list); + PyErr_SetString(PyExc_RuntimeError, + "cannot acquire lock"); + return NULL; + } + } + PyObject *subclasses = lookup_tp_subclasses(self); // borrowed ref if (subclasses == NULL) { + if (rlock != NULL) { + PyObject *name = PyUnicode_FromString("release"); + if (name == NULL) { + Py_DECREF(list); + return NULL; + } + + res = PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + + if(res == NULL) { + Py_DECREF(list); + return NULL; + } + } return list; } assert(PyDict_CheckExact(subclasses)); @@ -467,6 +508,27 @@ _PyType_GetSubclasses(PyTypeObject *self) return NULL; } } + + if (rlock != NULL) { + PyObject *name = PyUnicode_FromString("release"); + if (name == NULL) { + Py_DECREF(list); + return NULL; + } + + res = PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + + if (res == NULL) { + Py_DECREF(list); + return NULL; + } + } + + if(_Py_IsImmutable(self)) { + Py_MakeImmutable(list); + } + return list; } @@ -5008,7 +5070,7 @@ static void clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) { if (_Py_IsMainInterpreter(interp)) { - Py_CLEAR(type->tp_cache); + Py_CLEAR(type->tp_lock); } clear_tp_dict(type); clear_tp_bases(type); @@ -5056,7 +5118,7 @@ type_dealloc(PyTypeObject *type) Py_XDECREF(type->tp_dict); Py_XDECREF(type->tp_bases); Py_XDECREF(type->tp_mro); - Py_XDECREF(type->tp_cache); + Py_XDECREF(type->tp_lock); clear_tp_subclasses(type); /* A type's tp_doc is heap allocated, unlike the tp_doc slots @@ -5248,7 +5310,6 @@ type_traverse(PyTypeObject *type, visitproc visit, void *arg) // } Py_VISIT(type->tp_dict); - Py_VISIT(type->tp_cache); Py_VISIT(type->tp_mro); Py_VISIT(type->tp_bases); Py_VISIT(type->tp_base); @@ -5287,7 +5348,7 @@ type_clear(PyTypeObject *type) cleared, and here's why: - tp_cache: + tp_lock: Not used; if it were, it would be a dict. tp_bases, tp_base: @@ -7602,6 +7663,33 @@ add_subclass(PyTypeObject *base, PyTypeObject *type) return -1; } + PyObject* rlock = type->tp_lock; + PyObject* res; + if (rlock != NULL) { + PyObject *name = PyUnicode_FromString("acquire"); + if (name == NULL) { + Py_DECREF(key); + Py_DECREF(ref); + return -1; + } + + res = PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + + if (res == NULL) { + Py_DECREF(key); + Py_DECREF(ref); + return -1; + } + + if (Py_IsFalse(res)) { + PyErr_SetString(PyExc_RuntimeError, "unable to acquire lock"); + Py_DECREF(key); + Py_DECREF(ref); + return -1; + } + } + // Only get tp_subclasses after creating the key and value. // PyWeakref_NewRef() can trigger a garbage collection which can execute // arbitrary Python code and so modify base->tp_subclasses. @@ -7609,6 +7697,31 @@ add_subclass(PyTypeObject *base, PyTypeObject *type) if (subclasses == NULL) { subclasses = init_tp_subclasses(base); if (subclasses == NULL) { + if(rlock != NULL) { + PyObject *name = PyUnicode_FromString("release"); + if (name != NULL) { + PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + } + } + Py_DECREF(key); + Py_DECREF(ref); + return -1; + } + } + + if(rlock != NULL) { + PyObject* name = PyUnicode_FromString("release"); + if(name == NULL){ + Py_DECREF(key); + Py_DECREF(ref); + return -1; + } + + res = PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + + if (res == NULL) { Py_DECREF(key); Py_DECREF(ref); return -1; @@ -7667,8 +7780,36 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) static void remove_subclass(PyTypeObject *base, PyTypeObject *type) { + PyObject* res; + PyObject* rlock = base->tp_lock; + if(rlock != NULL) { + PyObject* name = PyUnicode_FromString("acquire"); + if(name == NULL){ + return; + } + + res = PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + + if(res == NULL){ + return; + } + + if(Py_IsFalse(res)){ + PyErr_SetString(PyExc_RuntimeError, "unable to acquire lock"); + return; + } + } + PyObject *subclasses = lookup_tp_subclasses(base); // borrowed ref if (subclasses == NULL) { + if(rlock != NULL) { + PyObject* name = PyUnicode_FromString("release"); + if(name != NULL){ + PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + } + } return; } assert(PyDict_CheckExact(subclasses)); @@ -7685,6 +7826,14 @@ remove_subclass(PyTypeObject *base, PyTypeObject *type) if (PyDict_Size(subclasses) == 0) { clear_tp_subclasses(base); } + + if(rlock != NULL) { + PyObject *name = PyUnicode_FromString("release"); + if(name != NULL){ + PyObject_CallMethodNoArgs(rlock, name); + Py_DECREF(name); + } + } } static void diff --git a/PC/config.c b/PC/config.c index 9d0fe6f87df69a..56190a05ed137b 100644 --- a/PC/config.c +++ b/PC/config.c @@ -8,6 +8,7 @@ extern PyObject* PyInit__abc(void); extern PyObject* PyInit_array(void); extern PyObject* PyInit_audioop(void); +extern PyObject* PyInit__behaviors(void); extern PyObject* PyInit_binascii(void); extern PyObject* PyInit_cmath(void); extern PyObject* PyInit_errno(void); @@ -90,6 +91,7 @@ struct _inittab _PyImport_Inittab[] = { {"array", PyInit_array}, {"_ast", PyInit__ast}, {"audioop", PyInit_audioop}, + {"_behaviors", PyInit__behaviors}, {"binascii", PyInit_binascii}, {"cmath", PyInit_cmath}, {"errno", PyInit_errno}, diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 31b94b81f5e889..39cf554b7bbb99 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -154,7 +154,6 @@ - @@ -234,6 +233,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 3366289ccd05ef..01636bfe192523 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -343,7 +343,7 @@ Source Files - + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 48d882a803a998..5826a8ce5f4165 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -368,6 +368,7 @@ + @@ -478,7 +479,6 @@ - @@ -560,6 +560,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4e4ba7b0b63330..bf057f3290996b 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -758,6 +758,9 @@ Modules + + Modules + Modules @@ -1052,9 +1055,6 @@ Objects - - Objects - Objects @@ -1256,6 +1256,9 @@ Python + + Python + Python diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d773fad7f8da11..a7e0742dc6fc08 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2754,7 +2754,7 @@ builtin_isimmutable(PyObject *module, PyObject *obj) { _Py_VPYDBG("isimmutable("); _Py_VPYDBGPRINT(obj); - _Py_VPYDBG(") region: %lu\n", Py_REGION(obj)); + _Py_VPYDBG(") region: %" PRIuPTR "\n", Py_REGION(obj)); return PyBool_FromLong(_Py_IsImmutable(obj)); } @@ -2775,6 +2775,20 @@ builtin_makeimmutable(PyObject *module, PyObject *obj) return Py_MakeImmutable(obj); } +/*[clinic input] +makeglobalsimmutable as builtin_makeglobalsimmutable + +Make all globals and the global dictionary immutable. +[clinic start generated code]*/ + +static PyObject * +builtin_makeglobalsimmutable_impl(PyObject *module) +/*[clinic end generated code: output=6e689380d1cd629c input=e31cfac504e8ba55]*/ +{ + return Py_MakeGlobalsImmutable(); +} + + typedef struct { PyObject_HEAD Py_ssize_t tuplesize; @@ -3075,6 +3089,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISSUBCLASS_METHODDEF BUILTIN_ISIMMUTABLE_METHODDEF BUILTIN_MAKEIMMUTABLE_METHODDEF + BUILTIN_MAKEGLOBALSIMMUTABLE_METHODDEF BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 5d52d7cf9e8575..c5353e7967cd67 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1427,4 +1427,22 @@ PyDoc_STRVAR(builtin_makeimmutable__doc__, #define BUILTIN_MAKEIMMUTABLE_METHODDEF \ {"makeimmutable", (PyCFunction)builtin_makeimmutable, METH_O, builtin_makeimmutable__doc__}, -/*[clinic end generated code: output=fd970a348a6896e0 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(builtin_makeglobalsimmutable__doc__, +"makeglobalsimmutable($module, /)\n" +"--\n" +"\n" +"Make all globals and the global dictionary immutable."); + +#define BUILTIN_MAKEGLOBALSIMMUTABLE_METHODDEF \ + {"makeglobalsimmutable", (PyCFunction)builtin_makeglobalsimmutable, METH_NOARGS, builtin_makeglobalsimmutable__doc__}, + +static PyObject * +builtin_makeglobalsimmutable_impl(PyObject *module); + +static PyObject * +builtin_makeglobalsimmutable(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return builtin_makeglobalsimmutable_impl(module); +} +/*[clinic end generated code: output=9f5cc3251dacf4c1 input=a9049054013a1b77]*/ diff --git a/Python/import.c b/Python/import.c index 3f3f2a2b681351..f0875418676507 100644 --- a/Python/import.c +++ b/Python/import.c @@ -13,6 +13,7 @@ #include "pycore_pylifecycle.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_regions.h" // _PyGlobalsImmutable_Check() #include "pycore_sysmodule.h" // _PySys_Audit() #include "marshal.h" // PyMarshal_ReadObjectFromString() #include "importdl.h" // _PyImport_DynLoadFiletab @@ -2789,12 +2790,21 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp; + PyThreadState *tstate; + if(_PyGlobalsImmutable_Check()){ + // If globals are immutable, we only import modules into a single place. + interp = PyInterpreterState_Main(); + tstate = PyInterpreterState_ThreadHead(interp); + }else{ + tstate = _PyThreadState_GET(); + interp = tstate->interp; + } + PyObject *abs_name = NULL; PyObject *final_mod = NULL; PyObject *mod = NULL; PyObject *package = NULL; - PyInterpreterState *interp = tstate->interp; int has_from; if (name == NULL) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 29771e07ae6a2c..8c4ee516687554 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1497,6 +1497,14 @@ finalize_modules_clear_weaklist(PyInterpreterState *interp, if (verbose && PyUnicode_Check(name)) { PySys_FormatStderr("# cleanup[3] wiping %U\n", name); } + + // before we can proceed, we need to restore mutability + mod->ob_region = _Py_DEFAULT_REGION; + PyObject *d = PyModule_GetDict(mod); + if (d != NULL) { + d->ob_region = _Py_DEFAULT_REGION; + } + _PyModule_Clear(mod); Py_DECREF(mod); } @@ -1531,6 +1539,10 @@ finalize_modules(PyThreadState *tstate) // Already done return; } + + // before we can proceed, we need to restore mutability + modules->ob_region = _Py_DEFAULT_REGION; + int verbose = _PyInterpreterState_GetConfig(interp)->verbose; // Delete some special builtins._ and sys attributes first. These are @@ -1859,6 +1871,11 @@ Py_FinalizeEx(void) PyGC_Collect(); /* Destroy all modules */ + + /* Module destruction requires that various dictionaries are mutable. */ + tstate->interp->sysdict->ob_region = _Py_DEFAULT_REGION; + tstate->interp->builtins->ob_region = _Py_DEFAULT_REGION; + _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); diff --git a/Objects/regions.c b/Python/regions.c similarity index 74% rename from Objects/regions.c rename to Python/regions.c index bf5f2d859474a4..b006e2f5cb98fe 100644 --- a/Objects/regions.c +++ b/Python/regions.c @@ -41,7 +41,7 @@ static bool stack_push(stack* s, PyObject* object){ _Py_VPYDBG("pushing "); _Py_VPYDBGPRINT(object); - _Py_VPYDBG(" [rc=%ld]\n", object->ob_refcnt); + _Py_VPYDBG(" [rc=%zd]\n", object->ob_refcnt); n->object = object; n->next = s->head; s->head = n; @@ -74,16 +74,6 @@ static bool stack_empty(stack* s){ return s->head == NULL; } -static void stack_print(stack* s){ - _Py_VPYDBG("stack: "); - node* n = s->head; - while(n != NULL){ - _Py_VPYDBGPRINT(n->object); - _Py_VPYDBG("[rc=%ld]\n", n->object->ob_refcnt); - n = n->next; - } -} - static bool is_c_wrapper(PyObject* obj){ return PyCFunction_Check(obj) || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) || Py_IS_TYPE(obj, &PyWrapperDescr_Type); } @@ -143,7 +133,7 @@ static PyObject* walk_function(PyObject* op, stack* frontier) _Py_VPYDBG("function: "); _Py_VPYDBGPRINT(op); - _Py_VPYDBG("[rc=%ld]\n", Py_REFCNT(op)); + _Py_VPYDBG("[rc=%zd]\n", Py_REFCNT(op)); _Py_SetImmutable(op); @@ -156,7 +146,21 @@ static PyObject* walk_function(PyObject* op, stack* frontier) // func_globals, func_builtins, and func_module can stay mutable, but depending on code we may need to make some keys immutable globals = f->func_globals; builtins = f->func_builtins; + _Py_VPYDBG("func_module: "); + _Py_VPYDBGPRINT(f->func_module); + _Py_VPYDBG("\n"); + + if(PyUnicode_CompareWithASCIIString(f->func_module, "_frozen_importlib") == 0){ + // we don't want to freeze the importlib module + _Py_VPYDBG("skipping importlib\n"); + Py_RETURN_NONE; + } + module = PyImport_Import(f->func_module); + if(module == NULL){ + return module; + } + if(PyModule_Check(module)){ module_dict = PyModule_GetDict(module); }else{ @@ -198,7 +202,7 @@ static PyObject* walk_function(PyObject* op, stack* frontier) size = 0; if (f_code->co_names != NULL) size = PySequence_Fast_GET_SIZE(f_code->co_names); - _Py_VPYDBG("Enumerating %ld names\n", size); + _Py_VPYDBG("Enumerating %zd names\n", size); for(Py_ssize_t i = 0; i < size; i++){ PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); // name.rc = x _Py_VPYDBG("name "); @@ -257,7 +261,7 @@ static PyObject* walk_function(PyObject* op, stack* frontier) } size = PySequence_Fast_GET_SIZE(f_code->co_consts); - _Py_VPYDBG("Enumerating %ld consts\n", size); + _Py_VPYDBG("Enumerating %zd consts\n", size); for(Py_ssize_t i = 0; i < size; i++){ PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); // value.rc = x _Py_VPYDBG("const "); @@ -319,7 +323,7 @@ static PyObject* walk_function(PyObject* op, stack* frontier) size = 0; if(f->func_closure != NULL) size = PySequence_Fast_GET_SIZE(f->func_closure); - _Py_VPYDBG("Enumerating %ld closure vars to check for global names\n", size); + _Py_VPYDBG("Enumerating %zd closure vars to check for global names\n", size); for(Py_ssize_t i=0; i < size; ++i){ PyObject* cellvar = PySequence_Fast_GET_ITEM(f->func_closure, i); // cellvar.rc = x PyObject* value = PyCell_GET(cellvar); // value.rc = x @@ -362,9 +366,25 @@ static PyObject* walk_function(PyObject* op, stack* frontier) static int _makeimmutable_visit(PyObject* obj, void* frontier) { + if(PyModule_Check(obj)){ + const char* name = PyModule_GetName(obj); + + if(strcmp(name, "_frozen_importlib_external") == 0){ + _Py_VPYDBG("skipping _frozen_importlib_external module\n"); + return 0; + } + + if(strcmp(name, "_frozen_importlib") == 0){ + _Py_VPYDBG("skipping _frozen_importlib module\n"); + return 0; + } + } + _Py_VPYDBG("visit("); _Py_VPYDBGPRINT(obj); - _Py_VPYDBG(") region: %lu rc: %ld\n", Py_REGION(obj), Py_REFCNT(obj)); + _Py_VPYDBG("["); + _Py_VPYDBGPRINT(obj->ob_type); + _Py_VPYDBG("]) region: %" PRIuPTR " rc: %zd\n", Py_REGION(obj), Py_REFCNT(obj)); if(!_Py_IsImmutable(obj)){ if(stack_push((stack*)frontier, obj)){ PyErr_NoMemory(); @@ -375,11 +395,29 @@ static int _makeimmutable_visit(PyObject* obj, void* frontier) return 0; } +static inline PyObject* allocate_rlock(void) +{ + PyObject* module = PyImport_ImportModule("_behaviors"); + if(module == NULL){ + return NULL; + } + + PyObject* rlock_t = PyObject_GetAttrString(module, "RLock"); + if(rlock_t == NULL){ + return NULL; + } + + PyObject* rlock = PyObject_CallNoArgs(rlock_t); + Py_DECREF(rlock_t); + Py_DECREF(module); + return rlock; +} + PyObject* _Py_MakeImmutable(PyObject* obj) { _Py_VPYDBG(">> makeimmutable("); _Py_VPYDBGPRINT(obj); - _Py_VPYDBG(") region: %lu rc: %ld\n", Py_REGION(obj), Py_REFCNT(obj)); + _Py_VPYDBG(") region: %" PRIuPTR " rc: %zd\n", Py_REGION(obj), Py_REFCNT(obj)); if(_Py_IsImmutable(obj) && _Py_IsImmutable(Py_TYPE(obj))){ return obj; } @@ -442,16 +480,17 @@ PyObject* _Py_MakeImmutable(PyObject* obj) stack_free(frontier); return NULL; } + + if(PyType_Check(item)){ + // we need to allocate and store the subclass lock + PyTypeObject* tp = (PyTypeObject *)PyObject_Type(item); // type_op.rc = x + 1 + if(tp->tp_subclasses == NULL){ + // subclasses were not requested prior to this point + // we will store the lock here + } + } }else{ _Py_VPYDBG("does not implements tp_traverse\n"); - // TODO: (mjp comment) These functions causes every character of - // a string to become an immutable object, which is is not the - // desired behavior. Commenting so we can discuss. I believe - // we should depend solely on the tp_traverse function to - // determine the objects an object depends on. - // - // _Py_MAKEIMMUTABLE_CALL(walk_sequence, item, frontier); - // _Py_MAKEIMMUTABLE_CALL(walk_mapping, item, frontier); } handle_type: @@ -464,6 +503,18 @@ PyObject* _Py_MakeImmutable(PyObject* obj) stack_free(frontier); return PyErr_NoMemory(); } + + // we need to allocate an rlock to control access + // to this type's subclass list + PyTypeObject* tp = (PyTypeObject*)type_op; + PyObject* rlock = allocate_rlock(); + if(rlock == NULL){ + Py_DECREF(item); + stack_free(frontier); + return PyErr_NoMemory(); + } + + tp->tp_lock = rlock; } else { Py_DECREF(type_op); // type_op.rc = x @@ -476,8 +527,93 @@ PyObject* _Py_MakeImmutable(PyObject* obj) stack_free(frontier); _Py_VPYDBGPRINT(obj); - _Py_VPYDBG(" region: %lu rc: %ld \n", Py_REGION(obj), Py_REFCNT(obj)); + _Py_VPYDBG(" region: %" PRIuPTR " rc: %zd \n", Py_REGION(obj), Py_REFCNT(obj)); _Py_VPYDBG("<< makeimmutable complete\n\n"); return obj; +} + +PyObject* Py_MakeGlobalsImmutable() +{ + PyObject* ret; + PyObject* main_dict; + PyInterpreterState* main; + + _Py_VPYDBG(">> makeglobalsimmutable\n"); + + // go module by module and freeze their global dictionaries + PyObject* modules = PyImport_GetModuleDict(); + Py_ssize_t size = PyDict_Size(modules); + PyObject* keys = PyDict_Keys(modules); + for(Py_ssize_t i = 0; i < size; i++){ + PyObject* key = PyList_GetItem(keys, i); + _Py_VPYDBG("module: "); + _Py_VPYDBGPRINT(key); + _Py_VPYDBG("\n"); + + if (PyUnicode_CompareWithASCIIString(key, "importlib") == 0){ + _Py_VPYDBG("skipping importlib module\n"); + continue; + } + + PyObject* module = PyDict_GetItem(modules, key); + PyObject* globals = PyModule_GetDict(module); + ret = _Py_MakeImmutable(globals); + if(ret == NULL){ + _Py_VPYDBG("<< makeglobalsimmutable failed\n"); + Py_DECREF(keys); + return NULL; + } + } + + main = PyInterpreterState_Main(); + if(main == NULL){ + _Py_VPYDBG("<< makeglobalsimmutable failed (cannot access main interpreter)\n"); + Py_DECREF(keys); + return NULL; + } + + main_dict = PyInterpreterState_GetDict(main); + if(main_dict == NULL){ + _Py_VPYDBG("<< makeglobalsimmutable failed (cannot access main interpreter dict)\n"); + Py_DECREF(keys); + return NULL; + } + + if(PyDict_SetItemString(main_dict, "__globals_immutable__", Py_True)) + { + _Py_VPYDBG("<< makeglobalsimmutable failed (cannot set __globals_immutable__)\n"); + Py_DECREF(keys); + return NULL; + } + + Py_DECREF(keys); + _Py_VPYDBG("<< makeglobalsimmutable complete\n"); + Py_RETURN_NONE; +} + +bool _PyGlobalsImmutable_Check() +{ + PyObject* main_dict; + PyInterpreterState* main; + PyObject* flag; + + main = PyInterpreterState_Main(); + if(main == NULL){ + _Py_VPYDBG("<< _PyGlobalsImmutable_Check failed (cannot access main interpreter)\n"); + return NULL; + } + + main_dict = PyInterpreterState_GetDict(main); + if(main_dict == NULL){ + _Py_VPYDBG("<< _PyGlobalsImmutable_Check failed (cannot access main interpreter dict)\n"); + return NULL; + } + + flag = PyDict_GetItemString(main_dict, "__globals_immutable__"); + if(flag == NULL){ + return false; + } + + return PyObject_IsTrue(flag); } \ No newline at end of file diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index ed4a0ac2dd32de..85044549e7edea 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -7,6 +7,7 @@ static const char* _Py_stdlib_module_names[] = { "_aix_support", "_ast", "_asyncio", +"_behaviors", "_bisect", "_blake2", "_bz2", diff --git a/TODO.md b/TODO.md index 6350c4e85bcfec..05374d751e1b7a 100644 --- a/TODO.md +++ b/TODO.md @@ -90,3 +90,17 @@ - [x] [PyCell_Set](Objects/cellobject.c#L63) - [x] [DELETE_DEREF](Python/bytecodes.c) - [x] [STORE_DEREF](Python/bytecodes.c) + +# Phase 2 + +## Notes +- Make all the keys of the globals dictionary immutable +- Module dictionaries are made immutable (`PyModule_GetDict()` should return an immutable dict) +- Implement `_PyBehaviorRuntime_CheckInit` which returns whether behaviors can run. +- Make all types immutable (except for subtype checks, see below). This includes static/built-in types. +- Altered behavior if `_PyBehaviorRuntime_CheckInit` is true: + * `Py_NewInterpreterFromConfig()` uses the immutable globals dictionary for new thread state + * `PyState_AddModule()/PyState_RemoveModule()` acquire a global lock. The resulting module is made immutable once it is loaded. + * `PyObject_GenericGetDict()` acquires the type lock if a dictionary is not already present. Dictionary is made immutable if created. + * `PyType_IsSubtype()/add_subclass()/remove_subclasses` acquire the type lock. + * `PyType_FromMetaclass()/PyType_FromModuleAndSpec()/PyType_FromSpecWithBases()/PyType_FromSpec()` create immutable types diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 1587a19716fe0b..0624901a9a6350 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -178,6 +178,7 @@ def clean_lines(text): Objects/**/*.h Py_BUILD_CORE 1 Modules/_asynciomodule.c Py_BUILD_CORE 1 +Modules/_behaviorsmodule.c Py_BUILD_CORE 1 Modules/_codecsmodule.c Py_BUILD_CORE 1 Modules/_collectionsmodule.c Py_BUILD_CORE 1 Modules/_ctypes/_ctypes.c Py_BUILD_CORE 1