From 607760754b6bcd7de5949522b137d649af200b69 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 18 Nov 2024 11:50:09 +0000 Subject: [PATCH 01/68] A first attempt at an invariant --- Include/internal/pycore_regions.h | 11 ++ Objects/regions.c | 166 ++++++++++++++++++++++++++++++ Python/bltinmodule.c | 45 +++++++- Python/ceval_gil.c | 6 ++ Python/ceval_macros.h | 2 + Python/clinic/bltinmodule.c.h | 58 ++++++++++- 6 files changed, 285 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index ea075f96a3de9f..0aba43b4e6859a 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -17,6 +17,15 @@ extern "C" { PyObject* _Py_MakeImmutable(PyObject* obj); #define Py_MakeImmutable(op) _Py_MakeImmutable(_PyObject_CAST(op)) +PyObject* _Py_InvariantSrcFailure(void); +#define Py_InvariantSrcFailure() _Py_InvariantSrcFailure() + +PyObject* _Py_InvariantTgtFailure(void); +#define Py_InvariantTgtFailure() _Py_InvariantTgtFailure() + +PyObject* _Py_EnableInvariant(void); +#define Py_EnableInvariant() _Py_EnableInvariant() + #ifdef NDEBUG #define _Py_VPYDBG(fmt, ...) #define _Py_VPYDBGPRINT(fmt, ...) @@ -25,6 +34,8 @@ PyObject* _Py_MakeImmutable(PyObject* obj); #define _Py_VPYDBGPRINT(op) PyObject_Print(_PyObject_CAST(op), stdout, 0) #endif +int _Py_CheckRegionInvariant(PyThreadState *tstate); + #ifdef __cplusplus } #endif diff --git a/Objects/regions.c b/Objects/regions.c index bbde367b9007b1..d5e48a2a9f0e89 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -4,6 +4,7 @@ #include #include #include "pycore_dict.h" +#include "pycore_interp.h" #include "pycore_object.h" #include "pycore_regions.h" @@ -88,6 +89,168 @@ bool is_c_wrapper(PyObject* obj){ return PyCFunction_Check(obj) || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) || Py_IS_TYPE(obj, &PyWrapperDescr_Type); } +/** + * Global status for performing the region check. + */ +bool do_region_check = false; + +// The src object for an edge that invalidated the invariant. +PyObject* error_src = Py_None; + +// The tgt object for an edge that invalidated the invariant. +PyObject* error_tgt = Py_None; + +// Once an error has occurred this is used to surpress further checking +bool error_occurred = false; + + +/** + * Enable the region check. + */ +void notify_regions_in_use(void) +{ + // Do not re-enable, if we have detected a fault. + if (!error_occurred) + do_region_check = true; +} + +PyObject* _Py_EnableInvariant(void) +{ + // Disable failure as program has explicitly requested invariant to be checked again. + error_occurred = false; + // Re-enable region check + do_region_check = true; + return Py_None; +} + +/** + * Set the global variables for a failure. + * This allows the interpreter to inspect what has failed. + */ +void set_failed_edge(PyObject* src, PyObject* tgt) +{ + Py_IncRef(src); + error_src = src; + Py_IncRef(tgt); + error_tgt = tgt; + printf("Error: %p -> %p: destination is not immutable\n", src, tgt); + // We have discovered a failure. + // Disable region check, until the program switches it back on. + do_region_check = false; + error_occurred = true; +} + +PyObject* _Py_InvariantSrcFailure(void) +{ + return Py_NewRef(error_src); +} + +PyObject* _Py_InvariantTgtFailure(void) +{ + return Py_NewRef(error_tgt); +} + + +// Lifted from gcmodule.c +typedef struct _gc_runtime_state GCState; +#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) +#define GC_NEXT _PyGCHead_NEXT +#define GC_PREV _PyGCHead_PREV +#define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head))) + + +/* A traversal callback for _Py_CheckRegionInvariant. + - op is the target of the reference we are checking, and + - parent is the source of the reference we are checking. +*/ +static int +visit_invariant_check(PyObject *op, void *parent) +{ + PyObject *src_op = _PyObject_CAST(parent); + // Check Immutable only reaches immutable + if ((src_op->ob_region == _Py_IMMUTABLE) + && (op->ob_region != _Py_IMMUTABLE)) + { + set_failed_edge(src_op, op); + return 0; + } + // TODO: More checks to go here as we add more region + // properties. + + return 0; +} + +/** + * This uses checks that the region topology is valid. + * + * It is currently implemented using the GC data. This + * means that not all objects are traversed as some objects + * are considered to not participate in cycles, and hence + * do not need to be understood for the cycle detector. + * + * This is not ideal for the region invariant, but is a good + * first approximation. We could actually walk the heap + * in a subsequent more elaborate invariant check. + * + * Returns non-zero if the invariant is violated. + */ +int _Py_CheckRegionInvariant(PyThreadState *tstate) +{ + // Check if we should perform the region invariant check + if(!do_region_check){ + return 0; + } + + // Use the GC data to find all the objects, and traverse them to + // confirm all their references satisfy the region invariant. + GCState *gcstate = &tstate->interp->gc; + + // There is an cyclic doubly linked list per generation of all the objects + // in that generation. + for (int i = NUM_GENERATIONS-1; i >= 0; i--) { + PyGC_Head *containers = GEN_HEAD(gcstate, i); + PyGC_Head *gc = GC_NEXT(containers); + // Walk doubly linked list of objects. + for (; gc != containers; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + // Local can point to anything. No invariant needed + if (op->ob_region == _Py_DEFAULT_REGION) + continue; + // Functions are complex. + // Removing from invariant initially. + // TODO provide custom traverse here. + if (PyFunction_Check(op)) + continue; + + // TODO the immutable code ignores c_wrappers + // review if this is correct. + if (is_c_wrapper(op)) + continue; + + // Use traverse proceduce to visit each field of the object. + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + (visitproc)visit_invariant_check, + op); + + // Also need to visit the type of the object + // As this isn't covered by the traverse. + PyObject* type_op = PyObject_Type(op); + visit_invariant_check(op, type_op); + Py_DECREF(type_op); + + // If we detected an error, stop so we don't + // write too much. + // TODO: The first error might not be the most useful. + // So might not need to build all error edges as a structure. + if (error_occurred) + return 1; + } + } + + return 0; +} + #define _Py_VISIT_FUNC_ATTR(attr, frontier) do { \ if(attr != NULL && !_Py_IsImmutable(attr)){ \ Py_INCREF(attr); \ @@ -377,6 +540,9 @@ int _makeimmutable_visit(PyObject* obj, void* frontier) PyObject* _Py_MakeImmutable(PyObject* obj) { + // We have started using regions, so notify to potentially enable checks. + notify_regions_in_use(); + _Py_VPYDBG(">> makeimmutable("); _Py_VPYDBGPRINT(obj); _Py_VPYDBG(") region: %lu rc: %ld\n", Py_REGION(obj), Py_REFCNT(obj)); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 7d92f94fccdd70..f02b557a25762b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2770,11 +2770,51 @@ Make 'obj' and its entire reachable object graph immutable. static PyObject * builtin_makeimmutable(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=4e665122542dfd24 input=21a50256fa4fb099]*/ +/*[clinic end generated code: output=4e665122542dfd24 input=bec4cf1797c848d4]*/ { return Py_MakeImmutable(obj); } +/*[clinic input] +invariant_failure_src as builtin_invariantsrcfailure + +Find the source of an invariant failure. +[clinic start generated code]*/ + +static PyObject * +builtin_invariantsrcfailure_impl(PyObject *module) +/*[clinic end generated code: output=8830901cbbefe8ba input=0266aae8308be0a4]*/ +{ + return Py_InvariantSrcFailure(); +} + +/*[clinic input] +invariant_failure_tgt as builtin_invarianttgtfailure + +Find the target of an invariant failure. +[clinic start generated code]*/ + +static PyObject * +builtin_invarianttgtfailure_impl(PyObject *module) +/*[clinic end generated code: output=f7c9cd7cb737bd13 input=9c79a563d1eb52f9]*/ +{ + return Py_InvariantTgtFailure(); +} + +/*[clinic input] +enableinvariant as builtin_enableinvariant + +Enable the checking of the region invariant. +[clinic start generated code]*/ + +static PyObject * +builtin_enableinvariant_impl(PyObject *module) +/*[clinic end generated code: output=a3a27509957788c2 input=cf5922b1eb45ef0e]*/ +{ + return Py_EnableInvariant(); +} + + typedef struct { PyObject_HEAD Py_ssize_t tuplesize; @@ -3061,6 +3101,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_DELATTR_METHODDEF BUILTIN_DIR_METHODDEF BUILTIN_DIVMOD_METHODDEF + BUILTIN_ENABLEINVARIANT_METHODDEF BUILTIN_EVAL_METHODDEF BUILTIN_EXEC_METHODDEF BUILTIN_FORMAT_METHODDEF @@ -3075,6 +3116,8 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISSUBCLASS_METHODDEF BUILTIN_ISIMMUTABLE_METHODDEF BUILTIN_MAKEIMMUTABLE_METHODDEF + BUILTIN_INVARIANTSRCFAILURE_METHODDEF + BUILTIN_INVARIANTTGTFAILURE_METHODDEF BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index aacf2b5c2e2c4f..4003553a7aae9c 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -7,6 +7,7 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // _Py_RunGC() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() +#include "pycore_regions.h" // _Py_CheckRegionInvariant() /* Notes about the implementation: @@ -1056,6 +1057,11 @@ _Py_HandlePending(PyThreadState *tstate) struct _ceval_runtime_state *ceval = &runtime->ceval; struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; + /* Check the region invariant if required. */ + if (_Py_CheckRegionInvariant(tstate) != 0) { + return -1; + } + /* Pending signals */ if (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)) { if (handle_signals(tstate) != 0) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index fccf9088cbd131..3cf4b8cf6b1322 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -91,6 +91,8 @@ /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ { \ + if (_Py_CheckRegionInvariant(tstate) != 0) \ + goto error; \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 0ffde42568666d..34d29e1b5c81a2 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1423,8 +1423,62 @@ PyDoc_STRVAR(builtin_makeimmutable__doc__, "makeimmutable($module, obj, /)\n" "--\n" "\n" -"Make \'obj\' and its entire graph immutable."); +"Make \'obj\' and its entire reachable object graph immutable."); #define BUILTIN_MAKEIMMUTABLE_METHODDEF \ {"makeimmutable", (PyCFunction)builtin_makeimmutable, METH_O, builtin_makeimmutable__doc__}, -/*[clinic end generated code: output=356f1513888beba0 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(builtin_invariantsrcfailure__doc__, +"invariant_failure_src($module, /)\n" +"--\n" +"\n" +"Find the source of an invariant failure."); + +#define BUILTIN_INVARIANTSRCFAILURE_METHODDEF \ + {"invariant_failure_src", (PyCFunction)builtin_invariantsrcfailure, METH_NOARGS, builtin_invariantsrcfailure__doc__}, + +static PyObject * +builtin_invariantsrcfailure_impl(PyObject *module); + +static PyObject * +builtin_invariantsrcfailure(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return builtin_invariantsrcfailure_impl(module); +} + +PyDoc_STRVAR(builtin_invarianttgtfailure__doc__, +"invariant_failure_tgt($module, /)\n" +"--\n" +"\n" +"Find the target of an invariant failure."); + +#define BUILTIN_INVARIANTTGTFAILURE_METHODDEF \ + {"invariant_failure_tgt", (PyCFunction)builtin_invarianttgtfailure, METH_NOARGS, builtin_invarianttgtfailure__doc__}, + +static PyObject * +builtin_invarianttgtfailure_impl(PyObject *module); + +static PyObject * +builtin_invarianttgtfailure(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return builtin_invarianttgtfailure_impl(module); +} + +PyDoc_STRVAR(builtin_enableinvariant__doc__, +"enableinvariant($module, /)\n" +"--\n" +"\n" +"Enable the checking of the region invariant."); + +#define BUILTIN_ENABLEINVARIANT_METHODDEF \ + {"enableinvariant", (PyCFunction)builtin_enableinvariant, METH_NOARGS, builtin_enableinvariant__doc__}, + +static PyObject * +builtin_enableinvariant_impl(PyObject *module); + +static PyObject * +builtin_enableinvariant(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return builtin_enableinvariant_impl(module); +} +/*[clinic end generated code: output=3a883fa08bbd248e input=a9049054013a1b77]*/ From 95896eafa2f56100885a750a44799558a231d162 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 25 Nov 2024 12:33:58 +0000 Subject: [PATCH 02/68] CR feedback --- Objects/regions.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index d5e48a2a9f0e89..ba6923815fc583 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -127,13 +127,15 @@ PyObject* _Py_EnableInvariant(void) * Set the global variables for a failure. * This allows the interpreter to inspect what has failed. */ -void set_failed_edge(PyObject* src, PyObject* tgt) +void set_failed_edge(PyObject* src, PyObject* tgt, const char* msg) { + Py_DecRef(error_src); Py_IncRef(src); error_src = src; + Py_DecRef(error_tgt); Py_IncRef(tgt); error_tgt = tgt; - printf("Error: %p -> %p: destination is not immutable\n", src, tgt); + printf("Error: Invalid edge %p -> %p: %s\n", src, tgt, msg); // We have discovered a failure. // Disable region check, until the program switches it back on. do_region_check = false; @@ -171,7 +173,7 @@ visit_invariant_check(PyObject *op, void *parent) if ((src_op->ob_region == _Py_IMMUTABLE) && (op->ob_region != _Py_IMMUTABLE)) { - set_failed_edge(src_op, op); + set_failed_edge(src_op, op, "Destination is not immutable"); return 0; } // TODO: More checks to go here as we add more region From 3c8ee1460f5eb9978d1388adab1038609deadeff Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 18 Nov 2024 16:33:42 +0100 Subject: [PATCH 03/68] Region: Create `RegionObject` and `RegionMetadata` objects Co-authored-by: Tobias Wrigstad --- Objects/object.c | 4 ++ Objects/regions.c | 91 +++++++++++++++++++++++++++++++++++++++++++- Python/bltinmodule.c | 4 +- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/Objects/object.c b/Objects/object.c index 5e76e3c8274c82..0bfe5b0285bcea 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2041,6 +2041,7 @@ extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyLineIterator; extern PyTypeObject _PyPositionsIterator; extern PyTypeObject _PyLegacyEventHandler_Type; +extern PyTypeObject Region_Type; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and @@ -2161,6 +2162,9 @@ static PyTypeObject* static_types[] = { &PyODictKeys_Type, // base=&PyDictKeys_Type &PyODictValues_Type, // base=&PyDictValues_Type &PyODict_Type, // base=&PyDict_Type + + // Pyrona Region: + &Region_Type, }; diff --git a/Objects/regions.c b/Objects/regions.c index ba6923815fc583..bb82ba626cdc5a 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -648,4 +648,93 @@ PyObject* _Py_MakeImmutable(PyObject* obj) _Py_VPYDBG("<< makeimmutable complete\n\n"); return obj; -} \ No newline at end of file +} + +typedef struct RegionObject RegionObject; + +typedef struct { + int lrc; // Integer field for "local reference count" + int osc; // Integer field for "open subregion count" + RegionObject* bridge; +} RegionMetadata; + +struct RegionObject { + PyObject_HEAD + RegionMetadata* metadata; + PyObject *name; // Optional string field for "name" +}; + +static void Region_dealloc(RegionObject *self) { + Py_XDECREF(self->name); + self->metadata->bridge = NULL; + // The lifetimes are joined for now + free(self->metadata); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int Region_init(RegionObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"name", NULL}; + self->metadata = (RegionMetadata*)calloc(1, sizeof(RegionMetadata)); + self->metadata->bridge = self; + self->name = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|U", kwlist, &self->name)) + return -1; + + Py_XINCREF(self->name); + return 0; +} + +static PyObject *Region_repr(RegionObject *self) { + if (Py_DebugFlag) { + RegionMetadata* data = self->metadata; + // Debug mode: include detailed representation + return PyUnicode_FromFormat( + "Region(lrc=%d, osc=%d, name=%S)", data->lrc, data->osc, self->name ? self->name : Py_None + ); + } else { + // Normal mode: simple representation + return PyUnicode_FromFormat("Region(name=%S)", self->name ? self->name : Py_None); + } +} + +PyTypeObject Region_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "Region", /* tp_name */ + sizeof(RegionObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)Region_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + (reprfunc)Region_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "TODO =^.^=", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)Region_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f02b557a25762b..f09f7f0c95657a 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3163,7 +3163,7 @@ static struct PyModuleDef builtinsmodule = { NULL }; - +extern PyTypeObject Region_Type; PyObject * _PyBuiltin_Init(PyInterpreterState *interp) { @@ -3224,6 +3224,8 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("tuple", &PyTuple_Type); SETBUILTIN("type", &PyType_Type); SETBUILTIN("zip", &PyZip_Type); + SETBUILTIN("Region", &Region_Type); + debug = PyBool_FromLong(config->optimization_level == 0); if (PyDict_SetItemString(dict, "__debug__", debug) < 0) { Py_DECREF(debug); From b9bca26b86b5869f227e520b07122ff6c970b488 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 18 Nov 2024 17:02:19 +0100 Subject: [PATCH 04/68] Region: Methods for adding, removing and checking obj membership Co-authored-by: Tobias Wrigstad --- Objects/regions.c | 144 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 136 insertions(+), 8 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index bb82ba626cdc5a..6b9855c4465ab0 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -651,12 +651,15 @@ PyObject* _Py_MakeImmutable(PyObject* obj) } typedef struct RegionObject RegionObject; +typedef struct RegionMetadata RegionMetadata; -typedef struct { +struct RegionMetadata { int lrc; // Integer field for "local reference count" int osc; // Integer field for "open subregion count" + int is_open; + RegionMetadata* parent; RegionObject* bridge; -} RegionMetadata; +}; struct RegionObject { PyObject_HEAD @@ -664,6 +667,63 @@ struct RegionObject { PyObject *name; // Optional string field for "name" }; +static void RegionMetadata_inc_lrc(RegionMetadata* data) { + data->lrc += 1; +} + +static void RegionMetadata_dec_lrc(RegionMetadata* data) { + data->lrc -= 1; +} + +static void RegionMetadata_inc_osc(RegionMetadata* data) { + data->osc += 1; +} + +static void RegionMetadata_dec_osc(RegionMetadata* data) { + data->osc -= 1; +} + +static void RegionMetadata_open(RegionMetadata* data) { + data->is_open = 1; +} + +static void RegionMetadata_close(RegionMetadata* data) { + data->is_open = 0; +} + +static bool RegionMetadata_is_open(RegionMetadata* data) { + return data->is_open == 0; +} + +static void RegionMetadata_set_parent(RegionMetadata* data, RegionMetadata* parent) { + data->parent = parent; +} + +static bool RegionMetadata_has_parent(RegionMetadata* data) { + return data->parent != NULL; +} + +static RegionMetadata* RegionMetadata_get_parent(RegionMetadata* data) { + return data->parent; +} + +static void RegionMetadata_unparent(RegionMetadata* data) { + RegionMetadata_set_parent(data, NULL); +} + +static PyObject* RegionMetadata_is_root(RegionMetadata* data) { + if (RegionMetadata_has_parent(data)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +static RegionMetadata* Region_get_metadata(RegionObject* obj) { + return obj->metadata; +} + + static void Region_dealloc(RegionObject *self) { Py_XDECREF(self->name); self->metadata->bridge = NULL; @@ -680,24 +740,92 @@ static int Region_init(RegionObject *self, PyObject *args, PyObject *kwds) { if (!PyArg_ParseTupleAndKeywords(args, kwds, "|U", kwlist, &self->name)) return -1; - + Py_XINCREF(self->name); return 0; } +// is_open method (returns True if the region is open, otherwise False) +static PyObject *Region_is_open(RegionObject *self, PyObject *args) { + if (RegionMetadata_is_open(self->metadata)) { + Py_RETURN_TRUE; // Return True if the region is open + } else { + Py_RETURN_FALSE; // Return False if the region is closed + } +} + +// Open method (sets the region to "open") +static PyObject *Region_open(RegionObject *self, PyObject *args) { + RegionMetadata_open(self->metadata); + Py_RETURN_NONE; // Return None (standard for methods with no return value) +} + +// Close method (sets the region to "closed") +static PyObject *Region_close(RegionObject *self, PyObject *args) { + RegionMetadata_close(self->metadata); // Mark as closed + Py_RETURN_NONE; // Return None (standard for methods with no return value) +} + +// Adds args object to self region +static PyObject *Region_add_object(RegionObject *self, PyObject *args) { + RegionMetadata* md = Region_get_metadata(self); + if (args->ob_region == _Py_DEFAULT_REGION) { + args->ob_region = (Py_uintptr_t) md; + Py_RETURN_NONE; + } else { + PyErr_SetString(PyExc_RuntimeError, "Object already had an owner or was immutable!"); + return NULL; + } +} + +// Remove args object to self region +static PyObject *Region_remove_object(RegionObject *self, PyObject *args) { + RegionMetadata* md = Region_get_metadata(self); + if (args->ob_region == (Py_uintptr_t) md) { + args->ob_region = _Py_DEFAULT_REGION; + Py_RETURN_NONE; + } else { + PyErr_SetString(PyExc_RuntimeError, "Object not a member of region!"); + return NULL; + } +} + +// Return True if args object is member of self region +static PyObject *Region_owns_object(RegionObject *self, PyObject *args) { + if ((Py_uintptr_t) Region_get_metadata(self) == args->ob_region) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + static PyObject *Region_repr(RegionObject *self) { + RegionMetadata* data = self->metadata; + // FIXME: deprecated flag, but config.parse_debug seems to not work? if (Py_DebugFlag) { - RegionMetadata* data = self->metadata; // Debug mode: include detailed representation return PyUnicode_FromFormat( - "Region(lrc=%d, osc=%d, name=%S)", data->lrc, data->osc, self->name ? self->name : Py_None + "Region(lrc=%d, osc=%d, name=%S, is_open=%d)", data->lrc, data->osc, self->name ? self->name : Py_None, data->is_open ); } else { // Normal mode: simple representation - return PyUnicode_FromFormat("Region(name=%S)", self->name ? self->name : Py_None); + return PyUnicode_FromFormat("Region(name=%S, is_open=%d)", self->name ? self->name : Py_None, data->is_open); } + Py_RETURN_NONE; } +// Define the RegionType with methods +static PyMethodDef Region_methods[] = { + {"open", (PyCFunction)Region_open, METH_NOARGS, "Open the region."}, + {"close", (PyCFunction)Region_close, METH_NOARGS, "Close the region."}, + {"is_open", (PyCFunction)Region_is_open, METH_NOARGS, "Check if the region is open."}, + {"add_object", (PyCFunction)Region_add_object, METH_O, "Add object to the region."}, + {"remove_object", (PyCFunction)Region_remove_object, METH_O, "Remove object from the region."}, + {"owns_object", (PyCFunction)Region_owns_object, METH_O, "Check if object is owned by the region."}, + {NULL} // Sentinel +}; + + PyTypeObject Region_Type = { PyVarObject_HEAD_INIT(NULL, 0) "Region", /* tp_name */ @@ -719,14 +847,14 @@ PyTypeObject Region_Type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - "TODO =^.^=", /* tp_doc */ + "TODO =^.^=", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - 0, /* tp_methods */ + Region_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ From 230a07e435b8eaae04c05ddfc29ef12252ce31de Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Mon, 25 Nov 2024 14:05:00 +0100 Subject: [PATCH 05/68] Region: Minimal test set for region ownership Co-authored-by: Tobias Wrigstad --- Lib/test/test_veronapy.py | 52 +++++++++++++++++++++++++++++ Objects/object.c | 4 +-- Objects/regions.c | 70 +++++++++++++++++++-------------------- Python/bltinmodule.c | 4 +-- 4 files changed, 91 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index 1b02a43959ca74..f95e4ab398dcc9 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -389,6 +389,58 @@ def test_weakref(self): # self.assertTrue(c.val() is obj) self.assertIsNone(c.val()) +class TestRegionOwnership(unittest.TestCase): + class A: + pass + + def test_default_ownership(self): + a = self.A() + r = Region() + self.assertFalse(r.owns_object(a)) + + def test_add_ownership(self): + a = self.A() + r = Region() + r.add_object(a) + self.assertTrue(r.owns_object(a)) + + def test_remove_ownership(self): + a = self.A() + r = Region() + r.add_object(a) + r.remove_object(a) + self.assertFalse(r.owns_object(a)) + + def test_add_ownership2(self): + a = self.A() + r1 = Region() + r2 = Region() + r1.add_object(a) + self.assertFalse(r2.owns_object(a)) + + def test_should_fail_add_ownership_twice_1(self): + a = self.A() + r = Region() + r.add_object(a) + try: + r.add_object(a) + except RuntimeError: + pass + else: + self.fail("Should not reach here") + + def test_should_fail_add_ownership_twice_2(self): + a = self.A() + r = Region() + r.add_object(a) + try: + r2 = Region() + r2.add_object(a) + except RuntimeError: + pass + else: + self.fail("Should not reach here") + # This test will make the Python environment unusable. # Should perhaps forbid making the frame immutable. # class TestStackCapture(unittest.TestCase): diff --git a/Objects/object.c b/Objects/object.c index 0bfe5b0285bcea..09b663f1f58e6f 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2041,7 +2041,7 @@ extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyLineIterator; extern PyTypeObject _PyPositionsIterator; extern PyTypeObject _PyLegacyEventHandler_Type; -extern PyTypeObject Region_Type; +extern PyTypeObject PyRegion_Type; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and @@ -2164,7 +2164,7 @@ static PyTypeObject* static_types[] = { &PyODict_Type, // base=&PyDict_Type // Pyrona Region: - &Region_Type, + &PyRegion_Type, }; diff --git a/Objects/regions.c b/Objects/regions.c index 6b9855c4465ab0..b30e046f648e63 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -650,68 +650,68 @@ PyObject* _Py_MakeImmutable(PyObject* obj) return obj; } -typedef struct RegionObject RegionObject; -typedef struct RegionMetadata RegionMetadata; +typedef struct PyRegionObject PyRegionObject; +typedef struct regionmetadata regionmetadata; -struct RegionMetadata { +struct regionmetadata { int lrc; // Integer field for "local reference count" int osc; // Integer field for "open subregion count" int is_open; - RegionMetadata* parent; - RegionObject* bridge; + regionmetadata* parent; + PyRegionObject* bridge; }; -struct RegionObject { +struct PyRegionObject { PyObject_HEAD - RegionMetadata* metadata; + regionmetadata* metadata; PyObject *name; // Optional string field for "name" }; -static void RegionMetadata_inc_lrc(RegionMetadata* data) { +static void RegionMetadata_inc_lrc(regionmetadata* data) { data->lrc += 1; } -static void RegionMetadata_dec_lrc(RegionMetadata* data) { +static void RegionMetadata_dec_lrc(regionmetadata* data) { data->lrc -= 1; } -static void RegionMetadata_inc_osc(RegionMetadata* data) { +static void RegionMetadata_inc_osc(regionmetadata* data) { data->osc += 1; } -static void RegionMetadata_dec_osc(RegionMetadata* data) { +static void RegionMetadata_dec_osc(regionmetadata* data) { data->osc -= 1; } -static void RegionMetadata_open(RegionMetadata* data) { +static void RegionMetadata_open(regionmetadata* data) { data->is_open = 1; } -static void RegionMetadata_close(RegionMetadata* data) { +static void RegionMetadata_close(regionmetadata* data) { data->is_open = 0; } -static bool RegionMetadata_is_open(RegionMetadata* data) { +static bool RegionMetadata_is_open(regionmetadata* data) { return data->is_open == 0; } -static void RegionMetadata_set_parent(RegionMetadata* data, RegionMetadata* parent) { +static void RegionMetadata_set_parent(regionmetadata* data, regionmetadata* parent) { data->parent = parent; } -static bool RegionMetadata_has_parent(RegionMetadata* data) { +static bool RegionMetadata_has_parent(regionmetadata* data) { return data->parent != NULL; } -static RegionMetadata* RegionMetadata_get_parent(RegionMetadata* data) { +static regionmetadata* RegionMetadata_get_parent(regionmetadata* data) { return data->parent; } -static void RegionMetadata_unparent(RegionMetadata* data) { +static void RegionMetadata_unparent(regionmetadata* data) { RegionMetadata_set_parent(data, NULL); } -static PyObject* RegionMetadata_is_root(RegionMetadata* data) { +static PyObject* RegionMetadata_is_root(regionmetadata* data) { if (RegionMetadata_has_parent(data)) { Py_RETURN_TRUE; } else { @@ -719,12 +719,12 @@ static PyObject* RegionMetadata_is_root(RegionMetadata* data) { } } -static RegionMetadata* Region_get_metadata(RegionObject* obj) { +static regionmetadata* Region_get_metadata(PyRegionObject* obj) { return obj->metadata; } -static void Region_dealloc(RegionObject *self) { +static void Region_dealloc(PyRegionObject *self) { Py_XDECREF(self->name); self->metadata->bridge = NULL; // The lifetimes are joined for now @@ -732,9 +732,9 @@ static void Region_dealloc(RegionObject *self) { Py_TYPE(self)->tp_free((PyObject *)self); } -static int Region_init(RegionObject *self, PyObject *args, PyObject *kwds) { +static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"name", NULL}; - self->metadata = (RegionMetadata*)calloc(1, sizeof(RegionMetadata)); + self->metadata = (regionmetadata*)calloc(1, sizeof(regionmetadata)); self->metadata->bridge = self; self->name = NULL; @@ -746,7 +746,7 @@ static int Region_init(RegionObject *self, PyObject *args, PyObject *kwds) { } // is_open method (returns True if the region is open, otherwise False) -static PyObject *Region_is_open(RegionObject *self, PyObject *args) { +static PyObject *Region_is_open(PyRegionObject *self, PyObject *args) { if (RegionMetadata_is_open(self->metadata)) { Py_RETURN_TRUE; // Return True if the region is open } else { @@ -755,20 +755,20 @@ static PyObject *Region_is_open(RegionObject *self, PyObject *args) { } // Open method (sets the region to "open") -static PyObject *Region_open(RegionObject *self, PyObject *args) { +static PyObject *Region_open(PyRegionObject *self, PyObject *args) { RegionMetadata_open(self->metadata); Py_RETURN_NONE; // Return None (standard for methods with no return value) } // Close method (sets the region to "closed") -static PyObject *Region_close(RegionObject *self, PyObject *args) { +static PyObject *Region_close(PyRegionObject *self, PyObject *args) { RegionMetadata_close(self->metadata); // Mark as closed Py_RETURN_NONE; // Return None (standard for methods with no return value) } // Adds args object to self region -static PyObject *Region_add_object(RegionObject *self, PyObject *args) { - RegionMetadata* md = Region_get_metadata(self); +static PyObject *Region_add_object(PyRegionObject *self, PyObject *args) { + regionmetadata* md = Region_get_metadata(self); if (args->ob_region == _Py_DEFAULT_REGION) { args->ob_region = (Py_uintptr_t) md; Py_RETURN_NONE; @@ -779,8 +779,8 @@ static PyObject *Region_add_object(RegionObject *self, PyObject *args) { } // Remove args object to self region -static PyObject *Region_remove_object(RegionObject *self, PyObject *args) { - RegionMetadata* md = Region_get_metadata(self); +static PyObject *Region_remove_object(PyRegionObject *self, PyObject *args) { + regionmetadata* md = Region_get_metadata(self); if (args->ob_region == (Py_uintptr_t) md) { args->ob_region = _Py_DEFAULT_REGION; Py_RETURN_NONE; @@ -791,7 +791,7 @@ static PyObject *Region_remove_object(RegionObject *self, PyObject *args) { } // Return True if args object is member of self region -static PyObject *Region_owns_object(RegionObject *self, PyObject *args) { +static PyObject *Region_owns_object(PyRegionObject *self, PyObject *args) { if ((Py_uintptr_t) Region_get_metadata(self) == args->ob_region) { Py_RETURN_TRUE; } else { @@ -799,8 +799,8 @@ static PyObject *Region_owns_object(RegionObject *self, PyObject *args) { } } -static PyObject *Region_repr(RegionObject *self) { - RegionMetadata* data = self->metadata; +static PyObject *Region_repr(PyRegionObject *self) { + regionmetadata* data = self->metadata; // FIXME: deprecated flag, but config.parse_debug seems to not work? if (Py_DebugFlag) { // Debug mode: include detailed representation @@ -826,10 +826,10 @@ static PyMethodDef Region_methods[] = { }; -PyTypeObject Region_Type = { +PyTypeObject PyRegion_Type = { PyVarObject_HEAD_INIT(NULL, 0) "Region", /* tp_name */ - sizeof(RegionObject), /* tp_basicsize */ + sizeof(PyRegionObject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)Region_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f09f7f0c95657a..11dd1e16f443e0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3163,7 +3163,7 @@ static struct PyModuleDef builtinsmodule = { NULL }; -extern PyTypeObject Region_Type; +extern PyTypeObject PyRegion_Type; PyObject * _PyBuiltin_Init(PyInterpreterState *interp) { @@ -3224,7 +3224,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("tuple", &PyTuple_Type); SETBUILTIN("type", &PyType_Type); SETBUILTIN("zip", &PyZip_Type); - SETBUILTIN("Region", &Region_Type); + SETBUILTIN("Region", &PyRegion_Type); debug = PyBool_FromLong(config->optimization_level == 0); if (PyDict_SetItemString(dict, "__debug__", debug) < 0) { From 7a43d6cdb22a42781cac8672a3de8643d0fa34d6 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 25 Nov 2024 15:06:19 +0000 Subject: [PATCH 06/68] Region: Correct Invariant and enable on Region creation Co-authored-by: Tobias Wrigstad --- Lib/test/test_veronapy.py | 4 ++++ Objects/regions.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index f95e4ab398dcc9..85c35c8235b6ba 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -393,6 +393,10 @@ class TestRegionOwnership(unittest.TestCase): class A: pass + def setUp(self): + # This freezes A and super and meta types of A namely `type` and `object` + makeimmutable(self.A) + def test_default_ownership(self): a = self.A() r = Region() diff --git a/Objects/regions.c b/Objects/regions.c index b30e046f648e63..4302a9d83449c4 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -238,7 +238,7 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) // Also need to visit the type of the object // As this isn't covered by the traverse. PyObject* type_op = PyObject_Type(op); - visit_invariant_check(op, type_op); + visit_invariant_check(type_op, op); Py_DECREF(type_op); // If we detected an error, stop so we don't @@ -733,6 +733,8 @@ static void Region_dealloc(PyRegionObject *self) { } static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { + notify_regions_in_use(); + static char *kwlist[] = {"name", NULL}; self->metadata = (regionmetadata*)calloc(1, sizeof(regionmetadata)); self->metadata->bridge = self; From 489e569fc7e732117211fcea3391946bf3dcedab Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 25 Nov 2024 15:58:04 +0000 Subject: [PATCH 07/68] Region: Catch invalid cross region refs Co-authored-by: Tobias Wrigstad --- Include/internal/pycore_regions.h | 5 +- Lib/test/test_veronapy.py | 93 ++++++++++++++++++++ Objects/regions.c | 140 +++++++++++++++++++++++++----- Python/bltinmodule.c | 1 - 4 files changed, 215 insertions(+), 24 deletions(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 0aba43b4e6859a..8909e896a8d6fb 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -26,6 +26,9 @@ PyObject* _Py_InvariantTgtFailure(void); PyObject* _Py_EnableInvariant(void); #define Py_EnableInvariant() _Py_EnableInvariant() +PyObject* _Py_ResetInvariant(void); +#define Py_ResetInvariant() _Py_ResetInvariant() + #ifdef NDEBUG #define _Py_VPYDBG(fmt, ...) #define _Py_VPYDBGPRINT(fmt, ...) @@ -39,4 +42,4 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate); #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_VERONAPY_H */ \ No newline at end of file +#endif /* !Py_INTERNAL_VERONAPY_H */ diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index 85c35c8235b6ba..03a5bb7483b628 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -1,4 +1,5 @@ import unittest +from gc import collect # This is a canary to check that global variables are not made immutable # when others are made immutable @@ -394,8 +395,10 @@ class A: pass def setUp(self): + collect() # This freezes A and super and meta types of A namely `type` and `object` makeimmutable(self.A) + enableinvariant() def test_default_ownership(self): a = self.A() @@ -445,6 +448,96 @@ def test_should_fail_add_ownership_twice_2(self): else: self.fail("Should not reach here") +class TestRegionInvariance(unittest.TestCase): + class A: + pass + + def setUp(self): + # This freezes A and super and meta types of A namely `type` and `object` + collect() + makeimmutable(self.A) + enableinvariant() + + def test_invalid_point_to_local(self): + # Create linked objects (a) -> (b) + a = self.A() + b = self.A() + a.b = b + + # Create a region and take ownership of a + r = Region() + try: + r.add_object(a) + except RuntimeError: + # Check that the errors are on the appropriate objects + self.assertFalse(r.owns_object(b)) + self.assertTrue(r.owns_object(a)) + self.assertEqual(invariant_failure_src(), a) + self.assertEqual(invariant_failure_tgt(), b) + # We have broken the heap -- need to fix it to continue testing + r.remove_object(a) + a.b = None + else: + self.fail("Should not reach here") + + def test_allow_bridge_object_ref(self): + # Create linked objects (a) -> (b) + a = self.A() + b = Region() + a.b = b + + # Create a region and take ownership of a + r = Region() + r.add_object(a) + self.assertFalse(r.owns_object(b)) + self.assertTrue(r.owns_object(a)) + +# def disabled_test_allow_bridge_object_ref(self): +# r1 = Region() +# r1a = self.A() +# r1.add_object(r1a) +# r2 = Region() +# r2a = self.A() +# r2.add_object(r2a) + +# # Make r2 a subregion of r1 +# r1a.f = r2 +# try: +# # Create a beautiful cycle +# r2a.f = r1 +# except RuntimeError: +# # These are currently true since the write barrier doesn't exist +# # and the exception is thrown by the invariance check +# if invariant_failure_src() == a: +# self.assertEqual(invariant_failure_tgt(), b) +# elif invariant_failure_tgt() == a: +# self.assertEqual(invariant_failure_src(), b) +# else: +# self.fail("Should not reach here") +# else: +# self.fail("Should not reach here") + + def test_should_fail_external_uniqueness(self): + a = self.A() + r = Region() + a.f = r + a.g = r + r2 = Region() + try: + r2.add_object(a) + except RuntimeError: + # We have broken the heap -- need to fix it to continue testing + r2.remove_object(a) + a.f = None + a.g = None + # Check that the errors are on the appropriate objects + self.assertEqual(invariant_failure_src(), a) + self.assertEqual(invariant_failure_tgt(), r) + else: + self.fail("Should not reach here -- external uniqueness validated but not caught by invariant checker") + + + # This test will make the Python environment unusable. # Should perhaps forbid making the frame immutable. # class TestStackCapture(unittest.TestCase): diff --git a/Objects/regions.c b/Objects/regions.c index 4302a9d83449c4..3a51c42c36be5b 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -2,12 +2,38 @@ #include "Python.h" #include #include +#include #include +#include "object.h" #include "pycore_dict.h" #include "pycore_interp.h" #include "pycore_object.h" #include "pycore_regions.h" +typedef struct PyRegionObject PyRegionObject; +typedef struct regionmetadata regionmetadata; + +struct PyRegionObject { + PyObject_HEAD + regionmetadata* metadata; + PyObject *name; // Optional string field for "name" +}; + +struct regionmetadata { + int lrc; // Integer field for "local reference count" + int osc; // Integer field for "open subregion count" + int is_open; + regionmetadata* parent; + PyRegionObject* bridge; + // TODO: make the following conditional of a debug build (or something) + // Intrinsic list for invariant checking + regionmetadata* next; +}; + +bool is_bridge_object(PyObject *op); +static int RegionMetadata_has_ancestor(regionmetadata* data, regionmetadata* other); +static regionmetadata* Region_get_metadata(PyRegionObject* obj); + /** * Simple implementation of stack for tracing during make immutable. * TODO: More efficient implementation @@ -103,6 +129,10 @@ PyObject* error_tgt = Py_None; // Once an error has occurred this is used to surpress further checking bool error_occurred = false; +// Start of a linked list of bridge objects used to check for external uniqueness +// Bridge objects appear in this list if they are captured +#define CAPTURED_SENTINEL 0xc0defefe +regionmetadata* captured = (regionmetadata*) CAPTURED_SENTINEL; /** * Enable the region check. @@ -120,6 +150,9 @@ PyObject* _Py_EnableInvariant(void) error_occurred = false; // Re-enable region check do_region_check = true; + // Reset the error state + error_src = Py_None; + error_tgt = Py_None; return Py_None; } @@ -135,7 +168,7 @@ void set_failed_edge(PyObject* src, PyObject* tgt, const char* msg) Py_DecRef(error_tgt); Py_IncRef(tgt); error_tgt = tgt; - printf("Error: Invalid edge %p -> %p: %s\n", src, tgt, msg); + PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p -> %p: %s\n", src, tgt, msg); // We have discovered a failure. // Disable region check, until the program switches it back on. do_region_check = false; @@ -160,22 +193,68 @@ typedef struct _gc_runtime_state GCState; #define GC_PREV _PyGCHead_PREV #define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head))) +static int +is_immutable_region(regionmetadata* r) +{ + return ((Py_uintptr_t) r) == _Py_IMMUTABLE; +} + +static int +is_default_region(regionmetadata* r) +{ + return ((Py_uintptr_t) r) == _Py_DEFAULT_REGION; +} /* A traversal callback for _Py_CheckRegionInvariant. - op is the target of the reference we are checking, and - parent is the source of the reference we are checking. */ static int -visit_invariant_check(PyObject *op, void *parent) +visit_invariant_check(PyObject *tgt, void *parent) { + // fprintf(stderr, "Visited %p from %p\n", tgt, parent); PyObject *src_op = _PyObject_CAST(parent); - // Check Immutable only reaches immutable - if ((src_op->ob_region == _Py_IMMUTABLE) - && (op->ob_region != _Py_IMMUTABLE)) - { - set_failed_edge(src_op, op, "Destination is not immutable"); + regionmetadata* src_region = (regionmetadata*) src_op->ob_region; + regionmetadata* tgt_region = (regionmetadata*) tgt->ob_region; + // Anything is allowed to point to immutable + if (is_immutable_region(tgt_region)) + return 0; + // Since tgt is not immutable, src also may not be as immutable may not point to mutable + if (is_immutable_region(src_region)) { + set_failed_edge(src_op, tgt, "Destination is not immutable"); + return 0; + // Borrowed references are unrestricted + } else if (is_default_region(src_region)) { + return 0; + // Check cross-region references + } else if (src_region != tgt_region) { + // Permitted cross-region references + fprintf(stderr, "%p <-- %p\n", tgt, parent); + if (is_bridge_object(tgt)) { + // We just followed a non-borrowed, external reference here, + // so this is a *capturing* reference (so we add it to the captured list) + regionmetadata* meta = Region_get_metadata((PyRegionObject*) tgt); + // Check if region is already added to captured list + if (meta->next == NULL) { + // First discovery of bridge -- add to list of captured bridge objects + meta->next = captured; + captured = meta; + } else { + // Bridge object was already captured + set_failed_edge(src_op, tgt, "Bridge object not externally unique"); + return 0; + } + // Forbid cycles in the region topology + if (RegionMetadata_has_ancestor(src_region, tgt_region)) { + set_failed_edge(src_op, tgt, "Region cycle detected"); + return 0; + } return 0; } + set_failed_edge(src_op, tgt, "Destination is not in the same region"); + return 0; + } + // TODO: More checks to go here as we add more region // properties. @@ -237,6 +316,7 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) // Also need to visit the type of the object // As this isn't covered by the traverse. + // TODO: this might be covered by tp_traverse? PyObject* type_op = PyObject_Type(op); visit_invariant_check(type_op, op); Py_DECREF(type_op); @@ -250,6 +330,14 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) } } + // Reset the captured list + while (captured != CAPTURED_SENTINEL) { + regionmetadata* m = captured; + captured = m->next; + m->next = NULL; + } + // fprintf(stderr, "----\n"); + return 0; } @@ -650,22 +738,18 @@ PyObject* _Py_MakeImmutable(PyObject* obj) return obj; } -typedef struct PyRegionObject PyRegionObject; -typedef struct regionmetadata regionmetadata; - -struct regionmetadata { - int lrc; // Integer field for "local reference count" - int osc; // Integer field for "open subregion count" - int is_open; - regionmetadata* parent; - PyRegionObject* bridge; -}; +bool is_bridge_object(PyObject *op) { + Py_uintptr_t region = op->ob_region; + if (region == _Py_IMMUTABLE || region == _Py_DEFAULT_REGION) { + return 0; + } -struct PyRegionObject { - PyObject_HEAD - regionmetadata* metadata; - PyObject *name; // Optional string field for "name" -}; + if (((regionmetadata*)region)->bridge == op) { + return 1; + } else { + return 0; + } +} static void RegionMetadata_inc_lrc(regionmetadata* data) { data->lrc += 1; @@ -719,6 +803,16 @@ static PyObject* RegionMetadata_is_root(regionmetadata* data) { } } +static int RegionMetadata_has_ancestor(regionmetadata* data, regionmetadata* other) { + do { + if (data == other) { + return 1; + } + data = RegionMetadata_get_parent(data); + } while (data); + return 0; +} + static regionmetadata* Region_get_metadata(PyRegionObject* obj) { return obj->metadata; } @@ -739,6 +833,8 @@ static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { self->metadata = (regionmetadata*)calloc(1, sizeof(regionmetadata)); self->metadata->bridge = self; self->name = NULL; + // Make the region an owner of the bridge object + self->ob_base.ob_region = (Py_uintptr_t) self->metadata; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|U", kwlist, &self->name)) return -1; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 11dd1e16f443e0..d7d95456e9f5ba 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2814,7 +2814,6 @@ builtin_enableinvariant_impl(PyObject *module) return Py_EnableInvariant(); } - typedef struct { PyObject_HEAD Py_ssize_t tuplesize; From e808d1f6591712049b812f6f99c4ab94cabe2191 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 26 Nov 2024 15:22:37 +0000 Subject: [PATCH 08/68] Region: Allow users to store data in Region fields Co-authored-by: Tobias Wrigstad --- Lib/test/test_veronapy.py | 116 ++++++++++++++------------- Objects/regions.c | 159 ++++++++++++++++++-------------------- 2 files changed, 130 insertions(+), 145 deletions(-) diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index 03a5bb7483b628..17d7b05fa7e0f5 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -1,5 +1,4 @@ import unittest -from gc import collect # This is a canary to check that global variables are not made immutable # when others are made immutable @@ -395,7 +394,6 @@ class A: pass def setUp(self): - collect() # This freezes A and super and meta types of A namely `type` and `object` makeimmutable(self.A) enableinvariant() @@ -429,24 +427,35 @@ def test_should_fail_add_ownership_twice_1(self): a = self.A() r = Region() r.add_object(a) - try: - r.add_object(a) - except RuntimeError: - pass - else: - self.fail("Should not reach here") + self.assertRaises(RuntimeError, r.add_object, a) def test_should_fail_add_ownership_twice_2(self): a = self.A() r = Region() r.add_object(a) - try: - r2 = Region() - r2.add_object(a) - except RuntimeError: - pass - else: - self.fail("Should not reach here") + r2 = Region() + self.assertRaises(RuntimeError, r2.add_object, a) + + def test_init_with_name(self): + r1 = Region() + r2 = Region("Super-name") + self.assertTrue("Super-name" in repr(r2)) + + r3_name = "Trevligt-Name" + r3a = Region(r3_name) + r3b = Region(r3_name) + self.assertTrue(r3_name in repr(r3a)) + self.assertTrue(r3_name in repr(r3b)) + self.assertTrue(isimmutable(r3_name)) + + def test_init_invalid_name(self): + self.assertRaises(TypeError, Region, 42) + + def test_init_same_name(self): + r1 = Region("Andy") + r2 = Region("Andy") + # Check that we reach the end of the test + self.assertTrue(True) class TestRegionInvariance(unittest.TestCase): class A: @@ -454,7 +463,6 @@ class A: def setUp(self): # This freezes A and super and meta types of A namely `type` and `object` - collect() makeimmutable(self.A) enableinvariant() @@ -466,19 +474,13 @@ def test_invalid_point_to_local(self): # Create a region and take ownership of a r = Region() - try: - r.add_object(a) - except RuntimeError: - # Check that the errors are on the appropriate objects - self.assertFalse(r.owns_object(b)) - self.assertTrue(r.owns_object(a)) - self.assertEqual(invariant_failure_src(), a) - self.assertEqual(invariant_failure_tgt(), b) - # We have broken the heap -- need to fix it to continue testing - r.remove_object(a) - a.b = None - else: - self.fail("Should not reach here") + self.assertRaises(RuntimeError, r.add_object, a) + + # Check that the errors are on the appropriate objects + self.assertFalse(r.owns_object(b)) + self.assertTrue(r.owns_object(a)) + self.assertEqual(invariant_failure_src(), a) + self.assertEqual(invariant_failure_tgt(), b) def test_allow_bridge_object_ref(self): # Create linked objects (a) -> (b) @@ -491,31 +493,31 @@ def test_allow_bridge_object_ref(self): r.add_object(a) self.assertFalse(r.owns_object(b)) self.assertTrue(r.owns_object(a)) - -# def disabled_test_allow_bridge_object_ref(self): -# r1 = Region() -# r1a = self.A() -# r1.add_object(r1a) -# r2 = Region() -# r2a = self.A() -# r2.add_object(r2a) - -# # Make r2 a subregion of r1 -# r1a.f = r2 -# try: -# # Create a beautiful cycle -# r2a.f = r1 -# except RuntimeError: -# # These are currently true since the write barrier doesn't exist -# # and the exception is thrown by the invariance check -# if invariant_failure_src() == a: -# self.assertEqual(invariant_failure_tgt(), b) -# elif invariant_failure_tgt() == a: -# self.assertEqual(invariant_failure_src(), b) -# else: -# self.fail("Should not reach here") -# else: -# self.fail("Should not reach here") + + def disabled_test_allow_bridge_object_ref(self): + r1 = Region() + r1a = self.A() + r1.add_object(r1a) + r2 = Region() + r2a = self.A() + r2.add_object(r2a) + + # Make r2 a subregion of r1 + r1a.f = r2 + try: + # Create a beautiful cycle + r2a.f = r1 + except RuntimeError: + # These are currently true since the write barrier doesn't exist + # and the exception is thrown by the invariance check + if invariant_failure_src() == a: + self.assertEqual(invariant_failure_tgt(), b) + elif invariant_failure_tgt() == a: + self.assertEqual(invariant_failure_src(), b) + else: + self.fail("Should not reach here") + else: + self.fail("Should not reach here") def test_should_fail_external_uniqueness(self): a = self.A() @@ -526,18 +528,12 @@ def test_should_fail_external_uniqueness(self): try: r2.add_object(a) except RuntimeError: - # We have broken the heap -- need to fix it to continue testing - r2.remove_object(a) - a.f = None - a.g = None # Check that the errors are on the appropriate objects self.assertEqual(invariant_failure_src(), a) self.assertEqual(invariant_failure_tgt(), r) else: self.fail("Should not reach here -- external uniqueness validated but not caught by invariant checker") - - # This test will make the Python environment unusable. # Should perhaps forbid making the frame immutable. # class TestStackCapture(unittest.TestCase): diff --git a/Objects/regions.c b/Objects/regions.c index 3a51c42c36be5b..697aa791261906 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -13,10 +13,14 @@ typedef struct PyRegionObject PyRegionObject; typedef struct regionmetadata regionmetadata; +static PyObject *Region_add_object(PyRegionObject *self, PyObject *args); +static PyObject *Region_remove_object(PyRegionObject *self, PyObject *args); + struct PyRegionObject { PyObject_HEAD regionmetadata* metadata; PyObject *name; // Optional string field for "name" + PyObject *dict; }; struct regionmetadata { @@ -66,9 +70,6 @@ bool stack_push(stack* s, PyObject* object){ return true; } - _Py_VPYDBG("pushing "); - _Py_VPYDBGPRINT(object); - _Py_VPYDBG(" [rc=%ld]\n", object->ob_refcnt); n->object = object; n->next = s->head; s->head = n; @@ -102,11 +103,8 @@ bool stack_empty(stack* s){ } void stack_print(stack* s){ - _Py_VPYDBG("stack: "); node* n = s->head; while(n != NULL){ - _Py_VPYDBGPRINT(n->object); - _Py_VPYDBG("[rc=%ld]\n", n->object->ob_refcnt); n = n->next; } } @@ -151,7 +149,9 @@ PyObject* _Py_EnableInvariant(void) // Re-enable region check do_region_check = true; // Reset the error state + Py_DecRef(error_src); error_src = Py_None; + Py_DecRef(error_tgt); error_tgt = Py_None; return Py_None; } @@ -212,7 +212,6 @@ is_default_region(regionmetadata* r) static int visit_invariant_check(PyObject *tgt, void *parent) { - // fprintf(stderr, "Visited %p from %p\n", tgt, parent); PyObject *src_op = _PyObject_CAST(parent); regionmetadata* src_region = (regionmetadata*) src_op->ob_region; regionmetadata* tgt_region = (regionmetadata*) tgt->ob_region; @@ -229,23 +228,19 @@ visit_invariant_check(PyObject *tgt, void *parent) // Check cross-region references } else if (src_region != tgt_region) { // Permitted cross-region references - fprintf(stderr, "%p <-- %p\n", tgt, parent); if (is_bridge_object(tgt)) { - // We just followed a non-borrowed, external reference here, - // so this is a *capturing* reference (so we add it to the captured list) - regionmetadata* meta = Region_get_metadata((PyRegionObject*) tgt); // Check if region is already added to captured list - if (meta->next == NULL) { + if (tgt_region->next == NULL) { // First discovery of bridge -- add to list of captured bridge objects - meta->next = captured; - captured = meta; + tgt_region->next = captured; + captured = tgt_region; } else { // Bridge object was already captured set_failed_edge(src_op, tgt, "Bridge object not externally unique"); return 0; } // Forbid cycles in the region topology - if (RegionMetadata_has_ancestor(src_region, tgt_region)) { + if (RegionMetadata_has_ancestor((regionmetadata*)src_region, (regionmetadata*)tgt_region)) { set_failed_edge(src_op, tgt, "Region cycle detected"); return 0; } @@ -336,7 +331,6 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) captured = m->next; m->next = NULL; } - // fprintf(stderr, "----\n"); return 0; } @@ -353,18 +347,13 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) PyObject* make_global_immutable(PyObject* globals, PyObject* name) { PyObject* value = PyDict_GetItem(globals, name); // value.rc = x - _Py_VPYDBG("value("); - _Py_VPYDBGPRINT(value); - _Py_VPYDBG(") -> "); _PyDict_SetKeyImmutable((PyDictObject*)globals, name); if(!_Py_IsImmutable(value)){ - _Py_VPYDBG("pushed\n"); Py_INCREF(value); return value; }else{ - _Py_VPYDBG("immutable\n"); Py_RETURN_NONE; } } @@ -394,9 +383,6 @@ PyObject* walk_function(PyObject* op, stack* frontier) bool check_globals = false; _PyObject_ASSERT(op, PyFunction_Check(op)); - _Py_VPYDBG("function: "); - _Py_VPYDBGPRINT(op); - _Py_VPYDBG("[rc=%ld]\n", Py_REFCNT(op)); _Py_SetImmutable(op); @@ -438,25 +424,17 @@ PyObject* walk_function(PyObject* op, stack* frontier) } Py_INCREF(f_ptr); // fp.rc = x + 1 - _Py_VPYDBG("function: adding captured vars/funcs/builtins\n"); while(!stack_empty(f_stack)){ f_ptr = stack_pop(f_stack); // fp.rc = x + 1 _PyObject_ASSERT(f_ptr, PyCode_Check(f_ptr)); f_code = (PyCodeObject*)f_ptr; - _Py_VPYDBG("analysing code: "); - _Py_VPYDBGPRINT(f_code->co_name); - _Py_VPYDBG("\n"); size = 0; if (f_code->co_names != NULL) size = PySequence_Fast_GET_SIZE(f_code->co_names); - _Py_VPYDBG("Enumerating %ld names\n", size); for(Py_ssize_t i = 0; i < size; i++){ PyObject* name = PySequence_Fast_GET_ITEM(f_code->co_names, i); // name.rc = x - _Py_VPYDBG("name "); - _Py_VPYDBGPRINT(name); - _Py_VPYDBG(": "); if(PyUnicode_CompareWithASCIIString(name, "globals") == 0){ // if the code calls the globals() builtin, then any @@ -477,7 +455,6 @@ PyObject* walk_function(PyObject* op, stack* frontier) } } }else if(PyDict_Contains(builtins, name)){ - _Py_VPYDBG("builtin\n"); _PyDict_SetKeyImmutable((PyDictObject*)builtins, name); @@ -487,9 +464,6 @@ PyObject* walk_function(PyObject* op, stack* frontier) } }else if(PyDict_Contains(module_dict, name)){ PyObject* value = PyDict_GetItem(module_dict, name); // value.rc = x - _Py_VPYDBG("module("); - _Py_VPYDBGPRINT(value); - _Py_VPYDBG(") -> "); _PyDict_SetKeyImmutable((PyDictObject*)module_dict, name); @@ -501,25 +475,18 @@ PyObject* walk_function(PyObject* op, stack* frontier) return PyErr_NoMemory(); } }else{ - _Py_VPYDBG("immutable\n"); } }else{ - _Py_VPYDBG("instance\n"); // TODO assert that it is an instance variable } } size = PySequence_Fast_GET_SIZE(f_code->co_consts); - _Py_VPYDBG("Enumerating %ld consts\n", size); for(Py_ssize_t i = 0; i < size; i++){ PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); // value.rc = x - _Py_VPYDBG("const "); - _Py_VPYDBGPRINT(value); - _Py_VPYDBG(": "); if(!_Py_IsImmutable(value)){ Py_INCREF(value); // value.rc = x + 1 if(PyCode_Check(value)){ - _Py_VPYDBG("nested_func\n"); _Py_SetImmutable(value); @@ -529,7 +496,6 @@ PyObject* walk_function(PyObject* op, stack* frontier) return PyErr_NoMemory(); } }else{ - _Py_VPYDBG("pushed\n"); if(stack_push(frontier, value)){ stack_free(f_stack); @@ -538,16 +504,11 @@ PyObject* walk_function(PyObject* op, stack* frontier) } } }else{ - _Py_VPYDBG("immutable\n"); } if(check_globals && PyUnicode_Check(value)){ - _Py_VPYDBG("checking if"); - _Py_VPYDBGPRINT(value); - _Py_VPYDBG(" is a global: "); PyObject* name = value; if(PyDict_Contains(globals, name)){ - _Py_VPYDBG(" true "); value = make_global_immutable(globals, name); if(!Py_IsNone(value)){ if(stack_push(frontier, value)){ @@ -557,7 +518,6 @@ PyObject* walk_function(PyObject* op, stack* frontier) } } }else{ - _Py_VPYDBG("false\n"); } } @@ -572,18 +532,13 @@ PyObject* walk_function(PyObject* op, stack* frontier) size = 0; if(f->func_closure != NULL) size = PySequence_Fast_GET_SIZE(f->func_closure); - _Py_VPYDBG("Enumerating %ld closure vars to check for global names\n", size); for(Py_ssize_t i=0; i < size; ++i){ PyObject* cellvar = PySequence_Fast_GET_ITEM(f->func_closure, i); // cellvar.rc = x PyObject* value = PyCell_GET(cellvar); // value.rc = x - _Py_VPYDBG("cellvar("); - _Py_VPYDBGPRINT(value); - _Py_VPYDBG(") is "); if(PyUnicode_Check(value)){ PyObject* name = value; if(PyDict_Contains(globals, name)){ - _Py_VPYDBG("a global "); value = make_global_immutable(globals, name); if(!Py_IsNone(value)){ if(stack_push(frontier, value)){ @@ -593,10 +548,8 @@ PyObject* walk_function(PyObject* op, stack* frontier) } } }else{ - _Py_VPYDBG("not a global\n"); } }else{ - _Py_VPYDBG("not a global\n"); } } } @@ -615,9 +568,6 @@ PyObject* walk_function(PyObject* op, stack* frontier) int _makeimmutable_visit(PyObject* obj, void* frontier) { - _Py_VPYDBG("visit("); - _Py_VPYDBGPRINT(obj); - _Py_VPYDBG(") region: %lu rc: %ld\n", Py_REGION(obj), Py_REFCNT(obj)); if(!_Py_IsImmutable(obj)){ if(stack_push((stack*)frontier, obj)){ PyErr_NoMemory(); @@ -630,12 +580,13 @@ int _makeimmutable_visit(PyObject* obj, void* frontier) PyObject* _Py_MakeImmutable(PyObject* obj) { + if (!obj) { + return NULL; + } + // We have started using regions, so notify to potentially enable checks. notify_regions_in_use(); - _Py_VPYDBG(">> makeimmutable("); - _Py_VPYDBGPRINT(obj); - _Py_VPYDBG(") region: %lu rc: %ld\n", Py_REGION(obj), Py_REFCNT(obj)); if(_Py_IsImmutable(obj) && _Py_IsImmutable(Py_TYPE(obj))){ return obj; } @@ -657,25 +608,16 @@ PyObject* _Py_MakeImmutable(PyObject* obj) traverseproc traverse; PyObject* type_op = NULL; - _Py_VPYDBG("item: "); - _Py_VPYDBGPRINT(item); if(_Py_IsImmutable(item)){ - _Py_VPYDBG(" already immutable!\n"); // Direct access like this is not recommended, but will be removed in the future as // this is just for debugging purposes. if(type->ob_base.ob_base.ob_region != _Py_IMMUTABLE){ // Why do we need to handle the type here, surely what ever made this immutable already did that? // Log so we can investigate. - _Py_VPYDBG("type "); - _Py_VPYDBGPRINT(type_op); - _Py_VPYDBG(" not immutable! but object is: "); - _Py_VPYDBGPRINT(item); - _Py_VPYDBG("\n"); } goto handle_type; } - _Py_VPYDBG("\n"); _Py_SetImmutable(item); @@ -692,14 +634,12 @@ PyObject* _Py_MakeImmutable(PyObject* obj) traverse = type->tp_traverse; if(traverse != NULL){ - _Py_VPYDBG("implements tp_traverse\n"); if(traverse(item, (visitproc)_makeimmutable_visit, frontier)){ Py_DECREF(item); stack_free(frontier); return NULL; } }else{ - _Py_VPYDBG("does not implements tp_traverse\n"); // TODO: (mjp comment) These functions causes every character of // a string to become an immutable object, which is is not the // desired behavior. Commenting so we can discuss. I believe @@ -731,9 +671,6 @@ PyObject* _Py_MakeImmutable(PyObject* obj) stack_free(frontier); - _Py_VPYDBGPRINT(obj); - _Py_VPYDBG(" region: %lu rc: %ld \n", Py_REGION(obj), Py_REFCNT(obj)); - _Py_VPYDBG("<< makeimmutable complete\n\n"); return obj; } @@ -744,7 +681,7 @@ bool is_bridge_object(PyObject *op) { return 0; } - if (((regionmetadata*)region)->bridge == op) { + if ((Py_uintptr_t)((regionmetadata*)region)->bridge == (Py_uintptr_t)op) { return 1; } else { return 0; @@ -819,8 +756,23 @@ static regionmetadata* Region_get_metadata(PyRegionObject* obj) { static void Region_dealloc(PyRegionObject *self) { + // Name is immutable and not in our region. Py_XDECREF(self->name); + self->name = NULL; self->metadata->bridge = NULL; + + // The dictionary can be NULL if the Region constructor crashed + if (self->dict) { + // We need to clear the ownership, since this dictionary might be + // returned to an object pool rather than freed. This would result + // in an error if the dictionary has the previous region. + Region_remove_object(self, (PyObject*)self->dict); + Py_DECREF(self->dict); + self->dict = NULL; + } + + PyObject_GC_UnTrack((PyObject *)self); + // The lifetimes are joined for now free(self->metadata); Py_TYPE(self)->tp_free((PyObject *)self); @@ -833,13 +785,42 @@ static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { self->metadata = (regionmetadata*)calloc(1, sizeof(regionmetadata)); self->metadata->bridge = self; self->name = NULL; - // Make the region an owner of the bridge object - self->ob_base.ob_region = (Py_uintptr_t) self->metadata; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|U", kwlist, &self->name)) return -1; + if (self->name) { + Py_XINCREF(self->name); + // Freeze the name and it's type. Short strings in python are inturned + // by default. This means that `id("AB") == id("AB")`. We therefore + // need to either clone the name object or freeze it to share it + // across regions. Freezing should be safe, since `+=` and other + // operators return new strings and keep the old one intact + _Py_MakeImmutable(self->name); + // FIXME: Implicit freezing should take care of this instead + if (!_Py_IsImmutable(self->name)) { + Region_add_object(self, self->name); + } + } + + // Make the region an owner of the bridge object + self->ob_base.ob_region = (Py_uintptr_t) self->metadata; + _Py_MakeImmutable(Py_TYPE(self)); + + // FIXME: Usually this is created on the fly. We need to do it manually to + // set the region and freeze the type + self->dict = PyDict_New(); + if (self->dict == NULL) { + return -1; // Propagate memory allocation failure + } + _Py_MakeImmutable(Py_TYPE(self->dict)); + Region_add_object(self, self->dict); - Py_XINCREF(self->name); + return 0; +} + +static int PyRegionObject_traverse(PyRegionObject *self, visitproc visit, void *arg) { + Py_VISIT(self->name); + Py_VISIT(self->dict); return 0; } @@ -866,6 +847,10 @@ static PyObject *Region_close(PyRegionObject *self, PyObject *args) { // Adds args object to self region static PyObject *Region_add_object(PyRegionObject *self, PyObject *args) { + if (!args) { + Py_RETURN_NONE; + } + regionmetadata* md = Region_get_metadata(self); if (args->ob_region == _Py_DEFAULT_REGION) { args->ob_region = (Py_uintptr_t) md; @@ -878,6 +863,10 @@ static PyObject *Region_add_object(PyRegionObject *self, PyObject *args) { // Remove args object to self region static PyObject *Region_remove_object(PyRegionObject *self, PyObject *args) { + if (!args) { + Py_RETURN_NONE; + } + regionmetadata* md = Region_get_metadata(self); if (args->ob_region == (Py_uintptr_t) md) { args->ob_region = _Py_DEFAULT_REGION; @@ -944,9 +933,9 @@ PyTypeObject PyRegion_Type = { 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "TODO =^.^=", /* tp_doc */ - 0, /* tp_traverse */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + "TODO =^.^=", /* tp_doc */ + (traverseproc)PyRegionObject_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -959,7 +948,7 @@ PyTypeObject PyRegion_Type = { 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ + offsetof(PyRegionObject, dict), /* tp_dictoffset */ (initproc)Region_init, /* tp_init */ 0, /* tp_alloc */ PyType_GenericNew, /* tp_new */ From 954e2c163178d82934d94047ea1e8a8e3248f7d5 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Thu, 28 Nov 2024 17:29:28 +0100 Subject: [PATCH 09/68] Region: Review logic changes --- Lib/test/test_veronapy.py | 29 ++------------ Objects/regions.c | 83 +++++++++++++++++++++------------------ Python/bltinmodule.c | 1 + 3 files changed, 48 insertions(+), 65 deletions(-) diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index 17d7b05fa7e0f5..deeff7ec925f9c 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -474,8 +474,10 @@ def test_invalid_point_to_local(self): # Create a region and take ownership of a r = Region() + # FIXME: Once the write barrier is implemented, this assert will fail. + # The code above should work without any errors. self.assertRaises(RuntimeError, r.add_object, a) - + # Check that the errors are on the appropriate objects self.assertFalse(r.owns_object(b)) self.assertTrue(r.owns_object(a)) @@ -494,31 +496,6 @@ def test_allow_bridge_object_ref(self): self.assertFalse(r.owns_object(b)) self.assertTrue(r.owns_object(a)) - def disabled_test_allow_bridge_object_ref(self): - r1 = Region() - r1a = self.A() - r1.add_object(r1a) - r2 = Region() - r2a = self.A() - r2.add_object(r2a) - - # Make r2 a subregion of r1 - r1a.f = r2 - try: - # Create a beautiful cycle - r2a.f = r1 - except RuntimeError: - # These are currently true since the write barrier doesn't exist - # and the exception is thrown by the invariance check - if invariant_failure_src() == a: - self.assertEqual(invariant_failure_tgt(), b) - elif invariant_failure_tgt() == a: - self.assertEqual(invariant_failure_src(), b) - else: - self.fail("Should not reach here") - else: - self.fail("Should not reach here") - def test_should_fail_external_uniqueness(self): a = self.A() r = Region() diff --git a/Objects/regions.c b/Objects/regions.c index 697aa791261906..4c83a3d7c483b0 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -29,7 +29,9 @@ struct regionmetadata { int is_open; regionmetadata* parent; PyRegionObject* bridge; - // TODO: make the following conditional of a debug build (or something) + // TODO: Currently only used for invariant checking. If it's not used for other things + // it might make sense to make this conditional in debug builds (or something) + // // Intrinsic list for invariant checking regionmetadata* next; }; @@ -196,13 +198,13 @@ typedef struct _gc_runtime_state GCState; static int is_immutable_region(regionmetadata* r) { - return ((Py_uintptr_t) r) == _Py_IMMUTABLE; + return ((Py_uintptr_t) r) == _Py_IMMUTABLE; } static int is_default_region(regionmetadata* r) { - return ((Py_uintptr_t) r) == _Py_DEFAULT_REGION; + return ((Py_uintptr_t) r) == _Py_DEFAULT_REGION; } /* A traversal callback for _Py_CheckRegionInvariant. @@ -215,47 +217,54 @@ visit_invariant_check(PyObject *tgt, void *parent) PyObject *src_op = _PyObject_CAST(parent); regionmetadata* src_region = (regionmetadata*) src_op->ob_region; regionmetadata* tgt_region = (regionmetadata*) tgt->ob_region; + // Internal references are always allowed + if (src_region == tgt_region) + return 0; // Anything is allowed to point to immutable if (is_immutable_region(tgt_region)) return 0; + // Borrowed references are unrestricted + if (is_default_region(src_region)) + return 0; // Since tgt is not immutable, src also may not be as immutable may not point to mutable if (is_immutable_region(src_region)) { set_failed_edge(src_op, tgt, "Destination is not immutable"); return 0; - // Borrowed references are unrestricted - } else if (is_default_region(src_region)) { - return 0; - // Check cross-region references - } else if (src_region != tgt_region) { - // Permitted cross-region references - if (is_bridge_object(tgt)) { - // Check if region is already added to captured list - if (tgt_region->next == NULL) { - // First discovery of bridge -- add to list of captured bridge objects - tgt_region->next = captured; - captured = tgt_region; - } else { - // Bridge object was already captured - set_failed_edge(src_op, tgt, "Bridge object not externally unique"); - return 0; - } - // Forbid cycles in the region topology - if (RegionMetadata_has_ancestor((regionmetadata*)src_region, (regionmetadata*)tgt_region)) { - set_failed_edge(src_op, tgt, "Region cycle detected"); - return 0; - } - return 0; - } + } + + // Cross-region references must be to a bridge + if (!is_bridge_object(tgt)) { set_failed_edge(src_op, tgt, "Destination is not in the same region"); return 0; } + // Check if region is already added to captured list + if (tgt_region->next != NULL) { + // Bridge object was already captured + set_failed_edge(src_op, tgt, "Bridge object not externally unique"); + return 0; + } + // Forbid cycles in the region topology + if (RegionMetadata_has_ancestor((regionmetadata*)src_region, (regionmetadata*)tgt_region)) { + set_failed_edge(src_op, tgt, "Region cycle detected"); + return 0; + } - // TODO: More checks to go here as we add more region - // properties. + // First discovery of bridge -- add to list of captured bridge objects + tgt_region->next = captured; + captured = tgt_region; return 0; } +void invariant_reset_captured_list() { + // Reset the captured list + while (captured != CAPTURED_SENTINEL) { + regionmetadata* m = captured; + captured = m->next; + m->next = NULL; + } +} + /** * This uses checks that the region topology is valid. * @@ -320,18 +329,14 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) // write too much. // TODO: The first error might not be the most useful. // So might not need to build all error edges as a structure. - if (error_occurred) + if (error_occurred) { + invariant_reset_captured_list(); return 1; + } } } - // Reset the captured list - while (captured != CAPTURED_SENTINEL) { - regionmetadata* m = captured; - captured = m->next; - m->next = NULL; - } - + invariant_reset_captured_list(); return 0; } @@ -760,7 +765,7 @@ static void Region_dealloc(PyRegionObject *self) { Py_XDECREF(self->name); self->name = NULL; self->metadata->bridge = NULL; - + // The dictionary can be NULL if the Region constructor crashed if (self->dict) { // We need to clear the ownership, since this dictionary might be @@ -866,7 +871,7 @@ static PyObject *Region_remove_object(PyRegionObject *self, PyObject *args) { if (!args) { Py_RETURN_NONE; } - + regionmetadata* md = Region_get_metadata(self); if (args->ob_region == (Py_uintptr_t) md) { args->ob_region = _Py_DEFAULT_REGION; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d7d95456e9f5ba..338bb088463a01 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3163,6 +3163,7 @@ static struct PyModuleDef builtinsmodule = { }; extern PyTypeObject PyRegion_Type; + PyObject * _PyBuiltin_Init(PyInterpreterState *interp) { From 2ef1ef3608ccc4e58bb32e2d7dea991f5c6700b2 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Thu, 28 Nov 2024 17:51:45 +0100 Subject: [PATCH 10/68] Region: No more warnings --- Objects/regions.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index 4c83a3d7c483b0..1390389d18c088 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -131,8 +131,8 @@ bool error_occurred = false; // Start of a linked list of bridge objects used to check for external uniqueness // Bridge objects appear in this list if they are captured -#define CAPTURED_SENTINEL 0xc0defefe -regionmetadata* captured = (regionmetadata*) CAPTURED_SENTINEL; +#define CAPTURED_SENTINEL ((regionmetadata*) 0xc0defefe) +regionmetadata* captured = CAPTURED_SENTINEL; /** * Enable the region check. @@ -256,7 +256,7 @@ visit_invariant_check(PyObject *tgt, void *parent) return 0; } -void invariant_reset_captured_list() { +void invariant_reset_captured_list(void) { // Reset the captured list while (captured != CAPTURED_SENTINEL) { regionmetadata* m = captured; @@ -693,18 +693,22 @@ bool is_bridge_object(PyObject *op) { } } +__attribute__((unused)) static void RegionMetadata_inc_lrc(regionmetadata* data) { data->lrc += 1; } +__attribute__((unused)) static void RegionMetadata_dec_lrc(regionmetadata* data) { data->lrc -= 1; } +__attribute__((unused)) static void RegionMetadata_inc_osc(regionmetadata* data) { data->osc += 1; } +__attribute__((unused)) static void RegionMetadata_dec_osc(regionmetadata* data) { data->osc -= 1; } @@ -733,10 +737,12 @@ static regionmetadata* RegionMetadata_get_parent(regionmetadata* data) { return data->parent; } +__attribute__((unused)) static void RegionMetadata_unparent(regionmetadata* data) { RegionMetadata_set_parent(data, NULL); } +__attribute__((unused)) static PyObject* RegionMetadata_is_root(regionmetadata* data) { if (RegionMetadata_has_parent(data)) { Py_RETURN_TRUE; @@ -809,7 +815,7 @@ static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { // Make the region an owner of the bridge object self->ob_base.ob_region = (Py_uintptr_t) self->metadata; - _Py_MakeImmutable(Py_TYPE(self)); + _Py_MakeImmutable((PyObject*)Py_TYPE(self)); // FIXME: Usually this is created on the fly. We need to do it manually to // set the region and freeze the type @@ -817,7 +823,7 @@ static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { if (self->dict == NULL) { return -1; // Propagate memory allocation failure } - _Py_MakeImmutable(Py_TYPE(self->dict)); + _Py_MakeImmutable((PyObject*)Py_TYPE(self->dict)); Region_add_object(self, self->dict); return 0; @@ -894,7 +900,10 @@ static PyObject *Region_owns_object(PyRegionObject *self, PyObject *args) { static PyObject *Region_repr(PyRegionObject *self) { regionmetadata* data = self->metadata; // FIXME: deprecated flag, but config.parse_debug seems to not work? +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" if (Py_DebugFlag) { +#pragma GCC diagnostic pop // Debug mode: include detailed representation return PyUnicode_FromFormat( "Region(lrc=%d, osc=%d, name=%S, is_open=%d)", data->lrc, data->osc, self->name ? self->name : Py_None, data->is_open From edebab339bf310f2a0c4869fe1db4349ba09634d Mon Sep 17 00:00:00 2001 From: xFrednet Date: Thu, 28 Nov 2024 17:59:50 +0100 Subject: [PATCH 11/68] Region: Review Formatting Adjustments --- Objects/regions.c | 166 +++++++++++++++++++++++----------------------- 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index 1390389d18c088..c8969f2805d78c 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -13,8 +13,8 @@ typedef struct PyRegionObject PyRegionObject; typedef struct regionmetadata regionmetadata; -static PyObject *Region_add_object(PyRegionObject *self, PyObject *args); -static PyObject *Region_remove_object(PyRegionObject *self, PyObject *args); +static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); +static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); struct PyRegionObject { PyObject_HEAD @@ -37,8 +37,8 @@ struct regionmetadata { }; bool is_bridge_object(PyObject *op); -static int RegionMetadata_has_ancestor(regionmetadata* data, regionmetadata* other); -static regionmetadata* Region_get_metadata(PyRegionObject* obj); +static int regionmetadata_has_ancestor(regionmetadata* data, regionmetadata* other); +static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj); /** * Simple implementation of stack for tracing during make immutable. @@ -244,7 +244,7 @@ visit_invariant_check(PyObject *tgt, void *parent) return 0; } // Forbid cycles in the region topology - if (RegionMetadata_has_ancestor((regionmetadata*)src_region, (regionmetadata*)tgt_region)) { + if (regionmetadata_has_ancestor((regionmetadata*)src_region, (regionmetadata*)tgt_region)) { set_failed_edge(src_op, tgt, "Region cycle detected"); return 0; } @@ -694,79 +694,79 @@ bool is_bridge_object(PyObject *op) { } __attribute__((unused)) -static void RegionMetadata_inc_lrc(regionmetadata* data) { +static void regionmetadata_inc_lrc(regionmetadata* data) { data->lrc += 1; } __attribute__((unused)) -static void RegionMetadata_dec_lrc(regionmetadata* data) { +static void regionmetadata_dec_lrc(regionmetadata* data) { data->lrc -= 1; } __attribute__((unused)) -static void RegionMetadata_inc_osc(regionmetadata* data) { +static void regionmetadata_inc_osc(regionmetadata* data) { data->osc += 1; } __attribute__((unused)) -static void RegionMetadata_dec_osc(regionmetadata* data) { +static void regionmetadata_dec_osc(regionmetadata* data) { data->osc -= 1; } -static void RegionMetadata_open(regionmetadata* data) { +static void regionmetadata_open(regionmetadata* data) { data->is_open = 1; } -static void RegionMetadata_close(regionmetadata* data) { +static void regionmetadata_close(regionmetadata* data) { data->is_open = 0; } -static bool RegionMetadata_is_open(regionmetadata* data) { +static bool regionmetadata_is_open(regionmetadata* data) { return data->is_open == 0; } -static void RegionMetadata_set_parent(regionmetadata* data, regionmetadata* parent) { +static void regionmetadata_set_parent(regionmetadata* data, regionmetadata* parent) { data->parent = parent; } -static bool RegionMetadata_has_parent(regionmetadata* data) { +static bool regionmetadata_has_parent(regionmetadata* data) { return data->parent != NULL; } -static regionmetadata* RegionMetadata_get_parent(regionmetadata* data) { +static regionmetadata* regionmetadata_get_parent(regionmetadata* data) { return data->parent; } __attribute__((unused)) -static void RegionMetadata_unparent(regionmetadata* data) { - RegionMetadata_set_parent(data, NULL); +static void regionmetadata_unparent(regionmetadata* data) { + regionmetadata_set_parent(data, NULL); } __attribute__((unused)) -static PyObject* RegionMetadata_is_root(regionmetadata* data) { - if (RegionMetadata_has_parent(data)) { +static PyObject* regionmetadata_is_root(regionmetadata* data) { + if (regionmetadata_has_parent(data)) { Py_RETURN_TRUE; } else { Py_RETURN_FALSE; } } -static int RegionMetadata_has_ancestor(regionmetadata* data, regionmetadata* other) { +static int regionmetadata_has_ancestor(regionmetadata* data, regionmetadata* other) { do { if (data == other) { return 1; } - data = RegionMetadata_get_parent(data); + data = regionmetadata_get_parent(data); } while (data); return 0; } -static regionmetadata* Region_get_metadata(PyRegionObject* obj) { +static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj) { return obj->metadata; } -static void Region_dealloc(PyRegionObject *self) { +static void PyRegion_dealloc(PyRegionObject *self) { // Name is immutable and not in our region. Py_XDECREF(self->name); self->name = NULL; @@ -777,7 +777,7 @@ static void Region_dealloc(PyRegionObject *self) { // We need to clear the ownership, since this dictionary might be // returned to an object pool rather than freed. This would result // in an error if the dictionary has the previous region. - Region_remove_object(self, (PyObject*)self->dict); + PyRegion_remove_object(self, (PyObject*)self->dict); Py_DECREF(self->dict); self->dict = NULL; } @@ -789,7 +789,7 @@ static void Region_dealloc(PyRegionObject *self) { Py_TYPE(self)->tp_free((PyObject *)self); } -static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { +static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { notify_regions_in_use(); static char *kwlist[] = {"name", NULL}; @@ -809,7 +809,7 @@ static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { _Py_MakeImmutable(self->name); // FIXME: Implicit freezing should take care of this instead if (!_Py_IsImmutable(self->name)) { - Region_add_object(self, self->name); + PyRegion_add_object(self, self->name); } } @@ -824,20 +824,20 @@ static int Region_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { return -1; // Propagate memory allocation failure } _Py_MakeImmutable((PyObject*)Py_TYPE(self->dict)); - Region_add_object(self, self->dict); + PyRegion_add_object(self, self->dict); return 0; } -static int PyRegionObject_traverse(PyRegionObject *self, visitproc visit, void *arg) { +static int PyRegion_traverse(PyRegionObject *self, visitproc visit, void *arg) { Py_VISIT(self->name); Py_VISIT(self->dict); return 0; } // is_open method (returns True if the region is open, otherwise False) -static PyObject *Region_is_open(PyRegionObject *self, PyObject *args) { - if (RegionMetadata_is_open(self->metadata)) { +static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *args) { + if (regionmetadata_is_open(self->metadata)) { Py_RETURN_TRUE; // Return True if the region is open } else { Py_RETURN_FALSE; // Return False if the region is closed @@ -845,24 +845,24 @@ static PyObject *Region_is_open(PyRegionObject *self, PyObject *args) { } // Open method (sets the region to "open") -static PyObject *Region_open(PyRegionObject *self, PyObject *args) { - RegionMetadata_open(self->metadata); +static PyObject *PyRegion_open(PyRegionObject *self, PyObject *args) { + regionmetadata_open(self->metadata); Py_RETURN_NONE; // Return None (standard for methods with no return value) } // Close method (sets the region to "closed") -static PyObject *Region_close(PyRegionObject *self, PyObject *args) { - RegionMetadata_close(self->metadata); // Mark as closed +static PyObject *PyRegion_close(PyRegionObject *self, PyObject *args) { + regionmetadata_close(self->metadata); // Mark as closed Py_RETURN_NONE; // Return None (standard for methods with no return value) } // Adds args object to self region -static PyObject *Region_add_object(PyRegionObject *self, PyObject *args) { +static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args) { if (!args) { Py_RETURN_NONE; } - regionmetadata* md = Region_get_metadata(self); + regionmetadata* md = PyRegion_get_metadata(self); if (args->ob_region == _Py_DEFAULT_REGION) { args->ob_region = (Py_uintptr_t) md; Py_RETURN_NONE; @@ -873,12 +873,12 @@ static PyObject *Region_add_object(PyRegionObject *self, PyObject *args) { } // Remove args object to self region -static PyObject *Region_remove_object(PyRegionObject *self, PyObject *args) { +static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { if (!args) { Py_RETURN_NONE; } - regionmetadata* md = Region_get_metadata(self); + regionmetadata* md = PyRegion_get_metadata(self); if (args->ob_region == (Py_uintptr_t) md) { args->ob_region = _Py_DEFAULT_REGION; Py_RETURN_NONE; @@ -889,15 +889,15 @@ static PyObject *Region_remove_object(PyRegionObject *self, PyObject *args) { } // Return True if args object is member of self region -static PyObject *Region_owns_object(PyRegionObject *self, PyObject *args) { - if ((Py_uintptr_t) Region_get_metadata(self) == args->ob_region) { +static PyObject *PyRegion_owns_object(PyRegionObject *self, PyObject *args) { + if ((Py_uintptr_t) PyRegion_get_metadata(self) == args->ob_region) { Py_RETURN_TRUE; } else { Py_RETURN_FALSE; } } -static PyObject *Region_repr(PyRegionObject *self) { +static PyObject *PyRegion_repr(PyRegionObject *self) { regionmetadata* data = self->metadata; // FIXME: deprecated flag, but config.parse_debug seems to not work? #pragma GCC diagnostic push @@ -916,54 +916,54 @@ static PyObject *Region_repr(PyRegionObject *self) { } // Define the RegionType with methods -static PyMethodDef Region_methods[] = { - {"open", (PyCFunction)Region_open, METH_NOARGS, "Open the region."}, - {"close", (PyCFunction)Region_close, METH_NOARGS, "Close the region."}, - {"is_open", (PyCFunction)Region_is_open, METH_NOARGS, "Check if the region is open."}, - {"add_object", (PyCFunction)Region_add_object, METH_O, "Add object to the region."}, - {"remove_object", (PyCFunction)Region_remove_object, METH_O, "Remove object from the region."}, - {"owns_object", (PyCFunction)Region_owns_object, METH_O, "Check if object is owned by the region."}, +static PyMethodDef PyRegion_methods[] = { + {"open", (PyCFunction)PyRegion_open, METH_NOARGS, "Open the region."}, + {"close", (PyCFunction)PyRegion_close, METH_NOARGS, "Close the region."}, + {"is_open", (PyCFunction)PyRegion_is_open, METH_NOARGS, "Check if the region is open."}, + {"add_object", (PyCFunction)PyRegion_add_object, METH_O, "Add object to the region."}, + {"remove_object", (PyCFunction)PyRegion_remove_object, METH_O, "Remove object from the region."}, + {"owns_object", (PyCFunction)PyRegion_owns_object, METH_O, "Check if object is owned by the region."}, {NULL} // Sentinel }; PyTypeObject PyRegion_Type = { PyVarObject_HEAD_INIT(NULL, 0) - "Region", /* tp_name */ - sizeof(PyRegionObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)Region_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)Region_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ + "Region", /* tp_name */ + sizeof(PyRegionObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyRegion_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + (reprfunc)PyRegion_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "TODO =^.^=", /* tp_doc */ - (traverseproc)PyRegionObject_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Region_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(PyRegionObject, dict), /* tp_dictoffset */ - (initproc)Region_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + (traverseproc)PyRegion_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyRegion_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(PyRegionObject, dict), /* tp_dictoffset */ + (initproc)PyRegion_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ }; From 1ae4863fcc2cc5480f6c319bee30a9c46b0141d2 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Fri, 29 Nov 2024 11:22:42 +0000 Subject: [PATCH 12/68] Pyrona: Add documentation to `_Py_MakeImmutable` --- Include/internal/pycore_regions.h | 9 +++++++++ Objects/regions.c | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 8909e896a8d6fb..5399dae2816c26 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -14,6 +14,15 @@ extern "C" { #define Py_CHECKWRITE(op) ((op) && _PyObject_CAST(op)->ob_region != _Py_IMMUTABLE) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} +/* This makes the given objects and all object reachable from the given + * object immutable. This will also move the objects into the immutable + * region. + * + * The argument is borrowed, meaning that it expects the calling context + * to handle the reference count. + * + * The function will return `Py_None` by default. + */ PyObject* _Py_MakeImmutable(PyObject* obj); #define Py_MakeImmutable(op) _Py_MakeImmutable(_PyObject_CAST(op)) diff --git a/Objects/regions.c b/Objects/regions.c index c8969f2805d78c..20469972b01266 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -677,7 +677,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) stack_free(frontier); - return obj; + Py_RETURN_NONE; } bool is_bridge_object(PyObject *op) { @@ -801,7 +801,7 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { return -1; if (self->name) { Py_XINCREF(self->name); - // Freeze the name and it's type. Short strings in python are inturned + // Freeze the name and it's type. Short strings in Python are interned // by default. This means that `id("AB") == id("AB")`. We therefore // need to either clone the name object or freeze it to share it // across regions. Freezing should be safe, since `+=` and other From 8924b0beb8be2f21fcbbddd9efd2b652eef33b1c Mon Sep 17 00:00:00 2001 From: xFrednet Date: Fri, 29 Nov 2024 12:02:15 +0000 Subject: [PATCH 13/68] Pyrona: Use more macros for cleaner code --- Include/object.h | 7 +++++++ Objects/regions.c | 45 ++++++++++++++++++++++----------------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/Include/object.h b/Include/object.h index 1617956902c3c2..ab5c514a99a694 100644 --- a/Include/object.h +++ b/Include/object.h @@ -314,6 +314,13 @@ static inline void Py_SET_REGION(PyObject *ob, Py_uintptr_t region) { # define Py_SET_REGION(ob, region) Py_SET_REGION(_PyObject_CAST(ob), (region)) #endif +static inline Py_uintptr_t Py_GET_REGION(PyObject *ob) { + return ob->ob_region; +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_GET_REGION(ob) Py_GET_REGION(_PyObject_CAST(ob)) +#endif + /* Type objects contain a string containing the type name (to help somewhat diff --git a/Objects/regions.c b/Objects/regions.c index 20469972b01266..197653df8bbb06 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -10,8 +10,8 @@ #include "pycore_object.h" #include "pycore_regions.h" -typedef struct PyRegionObject PyRegionObject; typedef struct regionmetadata regionmetadata; +typedef struct PyRegionObject PyRegionObject; static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); @@ -196,15 +196,15 @@ typedef struct _gc_runtime_state GCState; #define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head))) static int -is_immutable_region(regionmetadata* r) +is_immutable_region(Py_uintptr_t r) { - return ((Py_uintptr_t) r) == _Py_IMMUTABLE; + return r == _Py_IMMUTABLE; } static int -is_default_region(regionmetadata* r) +is_default_region(Py_uintptr_t r) { - return ((Py_uintptr_t) r) == _Py_DEFAULT_REGION; + return r == _Py_DEFAULT_REGION; } /* A traversal callback for _Py_CheckRegionInvariant. @@ -215,19 +215,19 @@ static int visit_invariant_check(PyObject *tgt, void *parent) { PyObject *src_op = _PyObject_CAST(parent); - regionmetadata* src_region = (regionmetadata*) src_op->ob_region; - regionmetadata* tgt_region = (regionmetadata*) tgt->ob_region; + regionmetadata* src_region = (regionmetadata*) Py_GET_REGION(src_op); + regionmetadata* tgt_region = (regionmetadata*) Py_GET_REGION(tgt); // Internal references are always allowed if (src_region == tgt_region) return 0; // Anything is allowed to point to immutable - if (is_immutable_region(tgt_region)) + if (is_immutable_region((Py_uintptr_t)tgt_region)) return 0; // Borrowed references are unrestricted - if (is_default_region(src_region)) + if (is_default_region((Py_uintptr_t)src_region)) return 0; // Since tgt is not immutable, src also may not be as immutable may not point to mutable - if (is_immutable_region(src_region)) { + if (is_immutable_region((Py_uintptr_t)src_region)) { set_failed_edge(src_op, tgt, "Destination is not immutable"); return 0; } @@ -299,7 +299,7 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) for (; gc != containers; gc = GC_NEXT(gc)) { PyObject *op = FROM_GC(gc); // Local can point to anything. No invariant needed - if (op->ob_region == _Py_DEFAULT_REGION) + if (is_default_region(Py_GET_REGION(op))) continue; // Functions are complex. // Removing from invariant initially. @@ -320,7 +320,6 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) // Also need to visit the type of the object // As this isn't covered by the traverse. - // TODO: this might be covered by tp_traverse? PyObject* type_op = PyObject_Type(op); visit_invariant_check(type_op, op); Py_DECREF(type_op); @@ -681,8 +680,8 @@ PyObject* _Py_MakeImmutable(PyObject* obj) } bool is_bridge_object(PyObject *op) { - Py_uintptr_t region = op->ob_region; - if (region == _Py_IMMUTABLE || region == _Py_DEFAULT_REGION) { + Py_uintptr_t region = Py_GET_REGION(op); + if (is_default_region(region) || is_immutable_region(region)) { return 0; } @@ -777,7 +776,7 @@ static void PyRegion_dealloc(PyRegionObject *self) { // We need to clear the ownership, since this dictionary might be // returned to an object pool rather than freed. This would result // in an error if the dictionary has the previous region. - PyRegion_remove_object(self, (PyObject*)self->dict); + PyRegion_remove_object(self, _PyObject_CAST(self->dict)); Py_DECREF(self->dict); self->dict = NULL; } @@ -814,8 +813,8 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { } // Make the region an owner of the bridge object - self->ob_base.ob_region = (Py_uintptr_t) self->metadata; - _Py_MakeImmutable((PyObject*)Py_TYPE(self)); + Py_SET_REGION(self, (Py_uintptr_t) self->metadata); + _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); // FIXME: Usually this is created on the fly. We need to do it manually to // set the region and freeze the type @@ -823,7 +822,7 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { if (self->dict == NULL) { return -1; // Propagate memory allocation failure } - _Py_MakeImmutable((PyObject*)Py_TYPE(self->dict)); + _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self->dict))); PyRegion_add_object(self, self->dict); return 0; @@ -863,8 +862,8 @@ static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args) { } regionmetadata* md = PyRegion_get_metadata(self); - if (args->ob_region == _Py_DEFAULT_REGION) { - args->ob_region = (Py_uintptr_t) md; + if (is_default_region(Py_GET_REGION(args))) { + Py_SET_REGION(args, (Py_uintptr_t) md); Py_RETURN_NONE; } else { PyErr_SetString(PyExc_RuntimeError, "Object already had an owner or was immutable!"); @@ -879,8 +878,8 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { } regionmetadata* md = PyRegion_get_metadata(self); - if (args->ob_region == (Py_uintptr_t) md) { - args->ob_region = _Py_DEFAULT_REGION; + if (Py_GET_REGION(args) == (Py_uintptr_t) md) { + Py_SET_REGION(args, _Py_DEFAULT_REGION); Py_RETURN_NONE; } else { PyErr_SetString(PyExc_RuntimeError, "Object not a member of region!"); @@ -890,7 +889,7 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { // Return True if args object is member of self region static PyObject *PyRegion_owns_object(PyRegionObject *self, PyObject *args) { - if ((Py_uintptr_t) PyRegion_get_metadata(self) == args->ob_region) { + if ((Py_uintptr_t) PyRegion_get_metadata(self) == Py_GET_REGION(args)) { Py_RETURN_TRUE; } else { Py_RETURN_FALSE; From b3221480ad54588a841026eb363c1c6aa7a136ad Mon Sep 17 00:00:00 2001 From: xFrednet Date: Fri, 29 Nov 2024 17:23:37 +0000 Subject: [PATCH 14/68] Pyrona: Address Matajoh's comments --- Include/object.h | 6 +++ Objects/regions.c | 111 ++++++++++++++++++++++------------------------ 2 files changed, 58 insertions(+), 59 deletions(-) diff --git a/Include/object.h b/Include/object.h index ab5c514a99a694..9b68365fcf5988 100644 --- a/Include/object.h +++ b/Include/object.h @@ -275,6 +275,12 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) } #define _Py_IsImmutable(op) _Py_IsImmutable(_PyObject_CAST(op)) +static inline Py_ALWAYS_INLINE int _Py_IsLocal(PyObject *op) +{ + return op->ob_region == _Py_DEFAULT_REGION; +} +#define _Py_IsLocal(op) _Py_IsLocal(_PyObject_CAST(op)) + static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { // This immortal check is for code that is unaware of immortal objects. diff --git a/Objects/regions.c b/Objects/regions.c index 197653df8bbb06..93cf61ba428395 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -195,57 +195,48 @@ typedef struct _gc_runtime_state GCState; #define GC_PREV _PyGCHead_PREV #define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head))) -static int -is_immutable_region(Py_uintptr_t r) -{ - return r == _Py_IMMUTABLE; -} - -static int -is_default_region(Py_uintptr_t r) -{ - return r == _Py_DEFAULT_REGION; -} +#define IS_IMMUTABLE_REGION(r) ((Py_uintptr_t)r == _Py_IMMUTABLE) +#define IS_DEFAULT_REGION(r) ((Py_uintptr_t)r == _Py_DEFAULT_REGION) /* A traversal callback for _Py_CheckRegionInvariant. - - op is the target of the reference we are checking, and - - parent is the source of the reference we are checking. + - tgt is the target of the reference we are checking, and + - src(_void) is the source of the reference we are checking. */ static int -visit_invariant_check(PyObject *tgt, void *parent) +visit_invariant_check(PyObject *tgt, void *src_void) { - PyObject *src_op = _PyObject_CAST(parent); - regionmetadata* src_region = (regionmetadata*) Py_GET_REGION(src_op); + PyObject *src = _PyObject_CAST(src_void); + regionmetadata* src_region = (regionmetadata*) Py_GET_REGION(src); regionmetadata* tgt_region = (regionmetadata*) Py_GET_REGION(tgt); // Internal references are always allowed if (src_region == tgt_region) return 0; // Anything is allowed to point to immutable - if (is_immutable_region((Py_uintptr_t)tgt_region)) + if (IS_IMMUTABLE_REGION(tgt_region)) return 0; // Borrowed references are unrestricted - if (is_default_region((Py_uintptr_t)src_region)) + if (IS_DEFAULT_REGION(src_region)) return 0; // Since tgt is not immutable, src also may not be as immutable may not point to mutable - if (is_immutable_region((Py_uintptr_t)src_region)) { - set_failed_edge(src_op, tgt, "Destination is not immutable"); + if (IS_IMMUTABLE_REGION(src_region)) { + set_failed_edge(src, tgt, "Reference from immutable object to mutable target"); return 0; } // Cross-region references must be to a bridge if (!is_bridge_object(tgt)) { - set_failed_edge(src_op, tgt, "Destination is not in the same region"); + set_failed_edge(src, tgt, "Reference from object in one region into another region"); return 0; } // Check if region is already added to captured list if (tgt_region->next != NULL) { // Bridge object was already captured - set_failed_edge(src_op, tgt, "Bridge object not externally unique"); + set_failed_edge(src, tgt, "Reference to bridge is not externally unique"); return 0; } // Forbid cycles in the region topology if (regionmetadata_has_ancestor((regionmetadata*)src_region, (regionmetadata*)tgt_region)) { - set_failed_edge(src_op, tgt, "Region cycle detected"); + set_failed_edge(src, tgt, "Regions create a cycle with subreagions"); return 0; } @@ -299,7 +290,7 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) for (; gc != containers; gc = GC_NEXT(gc)) { PyObject *op = FROM_GC(gc); // Local can point to anything. No invariant needed - if (is_default_region(Py_GET_REGION(op))) + if (_Py_IsLocal(op)) continue; // Functions are complex. // Removing from invariant initially. @@ -585,7 +576,7 @@ int _makeimmutable_visit(PyObject* obj, void* frontier) PyObject* _Py_MakeImmutable(PyObject* obj) { if (!obj) { - return NULL; + Py_RETURN_NONE; } // We have started using regions, so notify to potentially enable checks. @@ -681,15 +672,16 @@ PyObject* _Py_MakeImmutable(PyObject* obj) bool is_bridge_object(PyObject *op) { Py_uintptr_t region = Py_GET_REGION(op); - if (is_default_region(region) || is_immutable_region(region)) { - return 0; + if (IS_DEFAULT_REGION(region) || IS_IMMUTABLE_REGION(region)) { + return false; } - if ((Py_uintptr_t)((regionmetadata*)region)->bridge == (Py_uintptr_t)op) { - return 1; - } else { - return 0; - } + // It's not yet clear how immutability will interact with region objects. + // It's likely that the object will remain in the object topology but + // will use the properties of a bridge object. This therefore checks if + // the object is equal to the regions bridge object rather than checking + // that the type is `PyRegionObject` + return ((Py_uintptr_t)((regionmetadata*)region)->bridge == (Py_uintptr_t)op); } __attribute__((unused)) @@ -742,22 +734,18 @@ static void regionmetadata_unparent(regionmetadata* data) { } __attribute__((unused)) -static PyObject* regionmetadata_is_root(regionmetadata* data) { - if (regionmetadata_has_parent(data)) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } +static int regionmetadata_is_root(regionmetadata* data) { + return regionmetadata_has_parent(data); } static int regionmetadata_has_ancestor(regionmetadata* data, regionmetadata* other) { do { if (data == other) { - return 1; + return true; } data = regionmetadata_get_parent(data); } while (data); - return 0; + return false; } static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj) { @@ -805,11 +793,9 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { // need to either clone the name object or freeze it to share it // across regions. Freezing should be safe, since `+=` and other // operators return new strings and keep the old one intact - _Py_MakeImmutable(self->name); + // // FIXME: Implicit freezing should take care of this instead - if (!_Py_IsImmutable(self->name)) { - PyRegion_add_object(self, self->name); - } + _Py_MakeImmutable(self->name); } // Make the region an owner of the bridge object @@ -822,6 +808,8 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { if (self->dict == NULL) { return -1; // Propagate memory allocation failure } + // TODO: Once phase 2 is done, we might be able to do this statically + // at compile time. _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self->dict))); PyRegion_add_object(self, self->dict); @@ -862,7 +850,7 @@ static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args) { } regionmetadata* md = PyRegion_get_metadata(self); - if (is_default_region(Py_GET_REGION(args))) { + if (_Py_IsLocal(args)) { Py_SET_REGION(args, (Py_uintptr_t) md); Py_RETURN_NONE; } else { @@ -898,20 +886,23 @@ static PyObject *PyRegion_owns_object(PyRegionObject *self, PyObject *args) { static PyObject *PyRegion_repr(PyRegionObject *self) { regionmetadata* data = self->metadata; - // FIXME: deprecated flag, but config.parse_debug seems to not work? -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - if (Py_DebugFlag) { -#pragma GCC diagnostic pop - // Debug mode: include detailed representation - return PyUnicode_FromFormat( - "Region(lrc=%d, osc=%d, name=%S, is_open=%d)", data->lrc, data->osc, self->name ? self->name : Py_None, data->is_open - ); - } else { - // Normal mode: simple representation - return PyUnicode_FromFormat("Region(name=%S, is_open=%d)", self->name ? self->name : Py_None, data->is_open); - } - Py_RETURN_NONE; +#ifdef NDEBUG + // Debug mode: include detailed representation + return PyUnicode_FromFormat( + "Region(lrc=%d, osc=%d, name=%S, is_open=%d)", + data->lrc, + data->osc, + self->name ? self->name : Py_None, + data->is_open + ); +#else + // Normal mode: simple representation + return PyUnicode_FromFormat( + "Region(name=%S, is_open=%d)", + self->name ? self->name : Py_None, + data->is_open + ); +#endif } // Define the RegionType with methods @@ -919,6 +910,8 @@ static PyMethodDef PyRegion_methods[] = { {"open", (PyCFunction)PyRegion_open, METH_NOARGS, "Open the region."}, {"close", (PyCFunction)PyRegion_close, METH_NOARGS, "Close the region."}, {"is_open", (PyCFunction)PyRegion_is_open, METH_NOARGS, "Check if the region is open."}, + // Temporary methods for testing. These will be removed or at least renamed once + // the write barrier is done. {"add_object", (PyCFunction)PyRegion_add_object, METH_O, "Add object to the region."}, {"remove_object", (PyCFunction)PyRegion_remove_object, METH_O, "Remove object from the region."}, {"owns_object", (PyCFunction)PyRegion_owns_object, METH_O, "Check if object is owned by the region."}, From f4d59b3a75b836a3fdf734a91c6b2ca015144e35 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 6 Dec 2024 11:30:56 +0000 Subject: [PATCH 15/68] Add a minimal CI (#13) * Add a minimal CI * Apply static to internal functions. * Regened files and added a missing entry. * Change error to fix tests. * Manually allowing exception. --- .github/workflows/build_min.yml | 613 +++++++++++++++++++++++++++ Doc/data/stable_abi.dat | 1 + Lib/test/test_capi/test_abstract.py | 8 +- Lib/test/test_descr.py | 4 +- Lib/test/test_stable_abi_ctypes.py | 1 + Misc/stable_abi.toml | 3 + Objects/exceptions.c | 4 +- Objects/regions.c | 20 +- PC/python3dll.c | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 2 + 10 files changed, 640 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/build_min.yml diff --git a/.github/workflows/build_min.yml b/.github/workflows/build_min.yml new file mode 100644 index 00000000000000..c59e9fb3e7bbc9 --- /dev/null +++ b/.github/workflows/build_min.yml @@ -0,0 +1,613 @@ +name: TestsMin + +# gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because +# it prevents to mark a job as mandatory. A PR cannot be merged if a job is +# mandatory but not scheduled because of "paths-ignore". +on: + workflow_dispatch: + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable + cancel-in-progress: true + +jobs: + check_source: + name: 'Check for source changes' + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} + run_tests: ${{ steps.check.outputs.run_tests }} + run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} + config_hash: ${{ steps.config_hash.outputs.hash }} + steps: + - uses: actions/checkout@v4 + - name: Check for source changes + id: check + run: | + if [ -z "$GITHUB_BASE_REF" ]; then + echo "run_tests=true" >> $GITHUB_OUTPUT + else + git fetch origin $GITHUB_BASE_REF --depth=1 + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more + # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), + # but it requires to download more commits (this job uses + # "git fetch --depth=1"). + # + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git + # 2.26, but Git 2.28 is stricter and fails with "no merge base". + # + # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on + # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF + # into the PR branch anyway. + # + # https://github.com/python/core-workflow/issues/373 + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true + fi + + # Check if we should run hypothesis tests + GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + echo $GIT_BRANCH + if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then + echo "Branch too old for hypothesis tests" + echo "run_hypothesis=false" >> $GITHUB_OUTPUT + else + echo "Run hypothesis tests" + echo "run_hypothesis=true" >> $GITHUB_OUTPUT + fi + - name: Compute hash for config cache key + id: config_hash + run: | + echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT + - name: Get a list of the changed documentation-related files + if: github.event_name == 'pull_request' + id: changed-docs-files + uses: Ana06/get-changed-files@v2.2.0 + with: + filter: | + Doc/** + Misc/** + .github/workflows/reusable-docs.yml + format: csv # works for paths with spaces + - name: Check for docs changes + if: >- + github.event_name == 'pull_request' + && steps.changed-docs-files.outputs.added_modified_renamed != '' + id: docs-changes + run: | + echo "run-docs=true" >> "${GITHUB_OUTPUT}" + + # check-docs: + # name: Docs + # needs: check_source + # if: fromJSON(needs.check_source.outputs.run-docs) + # uses: ./.github/workflows/reusable-docs.yml + + # Pyrona is changing the ABI drop this test for now. + # + # check_abi: + # name: 'Check if the ABI has changed' + # runs-on: ubuntu-22.04 + # needs: check_source + # if: needs.check_source.outputs.run_tests == 'true' + # steps: + # - uses: actions/checkout@v3 + # - uses: actions/setup-python@v4 + # - name: Install dependencies + # run: | + # sudo ./.github/workflows/posix-deps-apt.sh + # sudo apt-get install -yq abigail-tools + # - name: Build CPython + # env: + # CFLAGS: -g3 -O0 + # run: | + # # Build Python with the libpython dynamic library + # ./configure --enable-shared + # make -j4 + # - name: Check for changes in the ABI + # id: check + # run: | + # if ! make check-abidump; then + # echo "Generated ABI file is not up to date." + # echo "Please add the release manager of this branch as a reviewer of this PR." + # echo "" + # echo "The up to date ABI file should be attached to this build as an artifact." + # echo "" + # echo "To learn more about this check: https://devguide.python.org/setup/#regenerate-the-abi-dump" + # echo "" + # exit 1 + # fi + # - name: Generate updated ABI files + # if: ${{ failure() && steps.check.conclusion == 'failure' }} + # run: | + # make regen-abidump + # - uses: actions/upload-artifact@v3 + # name: Publish updated ABI files + # if: ${{ failure() && steps.check.conclusion == 'failure' }} + # with: + # name: abi-data + # path: ./Doc/data/*.abi + + check_generated_files: + name: 'Check if generated files are up to date' + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Add ccache to PATH + run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Check Autoconf and aclocal versions + run: | + grep "Generated by GNU Autoconf 2.71" configure + grep "aclocal 1.16.4" aclocal.m4 + grep -q "runstatedir" configure + grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 + - name: Configure CPython + run: | + # Build Python with the libpython dynamic library + ./configure --config-cache --with-pydebug --enable-shared + - name: Regenerate autoconf files with container image + run: make regen-configure + - name: Build CPython + run: | + # Deepfreeze will usually cause global objects to be added or removed, + # so we run it before regen-global-objects gets rum (in regen-all). + make regen-deepfreeze + make -j4 regen-all + make regen-stdlib-module-names + - name: Check for changes + run: | + git add -u + changes=$(git status --porcelain) + # Check for changes in regenerated files + if test -n "$changes"; then + echo "Generated files not up to date." + echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)" + echo "configure files must be regenerated with a specific version of autoconf." + echo "$changes" + echo "" + git diff --staged || true + exit 1 + fi + - name: Check exported libpython symbols + run: make smelly + - name: Check limited ABI symbols + run: make check-limited-abi + - name: Check for unsupported C global variables + if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME + run: make check-c-globals + + # These were all broken before we started. + + # build_win32: + # name: 'Windows (x86)' + # runs-on: windows-latest + # timeout-minutes: 60 + # needs: check_source + # if: needs.check_source.outputs.run_tests == 'true' + # env: + # IncludeUwp: 'true' + # steps: + # - uses: actions/checkout@v4 + # - name: Build CPython + # run: .\PCbuild\build.bat -e -d -p Win32 + # - name: Display build info + # run: .\python.bat -m test.pythoninfo + # - name: Tests + # run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + + # build_win_amd64: + # name: 'Windows (x64)' + # runs-on: windows-latest + # timeout-minutes: 60 + # needs: check_source + # if: needs.check_source.outputs.run_tests == 'true' + # env: + # IncludeUwp: 'true' + # steps: + # - uses: actions/checkout@v4 + # - name: Register MSVC problem matcher + # run: echo "::add-matcher::.github/problem-matchers/msvc.json" + # - name: Build CPython + # run: .\PCbuild\build.bat -e -d -p x64 + # - name: Display build info + # run: .\python.bat -m test.pythoninfo + # - name: Tests + # run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + + # build_win_arm64: + # name: 'Windows (arm64)' + # runs-on: windows-latest + # timeout-minutes: 60 + # needs: check_source + # if: needs.check_source.outputs.run_tests == 'true' + # env: + # IncludeUwp: 'true' + # steps: + # - uses: actions/checkout@v4 + # - name: Register MSVC problem matcher + # run: echo "::add-matcher::.github/problem-matchers/msvc.json" + # - name: Build CPython + # run: .\PCbuild\build.bat -e -d -p arm64 + + # build_macos: + # name: 'macOS' + # runs-on: macos-latest + # timeout-minutes: 60 + # needs: check_source + # if: needs.check_source.outputs.run_tests == 'true' + # env: + # HOMEBREW_NO_ANALYTICS: 1 + # HOMEBREW_NO_AUTO_UPDATE: 1 + # HOMEBREW_NO_INSTALL_CLEANUP: 1 + # PYTHONSTRICTEXTENSIONBUILD: 1 + # steps: + # - uses: actions/checkout@v4 + # - name: Restore config.cache + # uses: actions/cache@v3 + # with: + # path: config.cache + # key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + # - name: Install Homebrew dependencies + # run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + # - name: Configure CPython + # run: | + # GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ + # GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ + # ./configure \ + # --config-cache \ + # --with-pydebug \ + # --prefix=/opt/python-dev \ + # --with-openssl="$(brew --prefix openssl@3.0)" + # - name: Build CPython + # run: make -j4 + # - name: Display build info + # run: make pythoninfo + # - name: Tests + # run: make buildbottest TESTOPTS="-j4 -uall,-cpu" + + build_ubuntu: + name: 'Ubuntu' + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + env: + OPENSSL_VER: 3.0.11 + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v4 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --with-openssl=$OPENSSL_DIR + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + - name: Tests + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + + # Removing as not changing this for now. + # + # build_ubuntu_ssltests: + # name: 'Ubuntu SSL tests with OpenSSL' + # runs-on: ubuntu-20.04 + # timeout-minutes: 60 + # needs: check_source + # if: needs.check_source.outputs.run_tests == 'true' + # strategy: + # fail-fast: false + # matrix: + # openssl_ver: [1.1.1w, 3.0.11, 3.1.3] + # env: + # OPENSSL_VER: ${{ matrix.openssl_ver }} + # MULTISSL_DIR: ${{ github.workspace }}/multissl + # OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} + # LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib + # steps: + # - uses: actions/checkout@v4 + # - name: Restore config.cache + # uses: actions/cache@v3 + # with: + # path: config.cache + # key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + # - name: Register gcc problem matcher + # run: echo "::add-matcher::.github/problem-matchers/gcc.json" + # - name: Install Dependencies + # run: sudo ./.github/workflows/posix-deps-apt.sh + # - name: Configure OpenSSL env vars + # run: | + # echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + # echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + # echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + # - name: 'Restore OpenSSL build' + # id: cache-openssl + # uses: actions/cache@v3 + # with: + # path: ./multissl/openssl/${{ env.OPENSSL_VER }} + # key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + # - name: Install OpenSSL + # if: steps.cache-openssl.outputs.cache-hit != 'true' + # run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + # - name: Add ccache to PATH + # run: | + # echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + # - name: Configure ccache action + # uses: hendrikmuhs/ccache-action@v1.2 + # - name: Configure CPython + # run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR + # - name: Build CPython + # run: make -j4 + # - name: Display build info + # run: make pythoninfo + # - name: SSL tests + # run: ./python Lib/test/ssltests.py + + test_hypothesis: + name: "Hypothesis tests on Ubuntu" + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' + env: + OPENSSL_VER: 3.0.11 + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v4 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --with-openssl=$OPENSSL_DIR + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: "Create hypothesis venv" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + VENV_LOC=$(realpath -m .)/hypovenv + VENV_PYTHON=$VENV_LOC/bin/python + echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV + echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV + ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt + - name: 'Restore Hypothesis database' + id: cache-hypothesis-database + uses: actions/cache@v3 + with: + path: ./hypothesis + key: hypothesis-database-${{ github.head_ref || github.run_id }} + restore-keys: | + - hypothesis-database- + - name: "Run tests" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + # Most of the excluded tests are slow test suites with no property tests + # + # (GH-104097) test_sysconfig is skipped because it has tests that are + # failing when executed from inside a virtual environment. + ${{ env.VENV_PYTHON }} -m test \ + -W \ + -o \ + -j4 \ + -x test_asyncio \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures \ + -x test_socket \ + -x test_subprocess \ + -x test_signal \ + -x test_sysconfig + - uses: actions/upload-artifact@v3 + if: always() + with: + name: hypothesis-example-db + path: .hypothesis/examples/ + + + build_asan: + name: 'Address sanitizer' + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + env: + OPENSSL_VER: 3.0.11 + PYTHONSTRICTEXTENSIONBUILD: 1 + ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 + steps: + - uses: actions/checkout@v4 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Set up GCC-10 for ASAN + uses: egor-tensin/setup-gcc@v1 + with: + version: 10 + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Configure CPython + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + + all-required-green: # This job does nothing and is only used for the branch protection + name: All required checks pass + if: always() + + needs: + - check_source # Transitive dependency, needed to access `run_tests` value + - check_generated_files + # - build_win32 + # - build_win_amd64 + # - build_win_arm64 + # - build_macos + - build_ubuntu + # - build_ubuntu_ssltests + - test_hypothesis + - build_asan + + runs-on: ubuntu-latest + + steps: + - name: Check whether the needed jobs succeeded or failed + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe + with: + allowed-failures: >- + test_hypothesis, + allowed-skips: >- + ${{ + needs.check_source.outputs.run_tests != 'true' + && ' + check_generated_files, + build_ubuntu, + build_asan, + ' + || '' + }} + ${{ + !fromJSON(needs.check_source.outputs.run_hypothesis) + && ' + test_hypothesis, + ' + || '' + }} + jobs: ${{ toJSON(needs) }} diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index f112d268129fd1..a5a86646ef1b78 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -238,6 +238,7 @@ var,PyExc_ModuleNotFoundError,3.6,, var,PyExc_NameError,3.2,, var,PyExc_NotADirectoryError,3.7,, var,PyExc_NotImplementedError,3.2,, +var,PyExc_NotWriteableError,4.0,, var,PyExc_OSError,3.2,, var,PyExc_OverflowError,3.2,, var,PyExc_PendingDeprecationWarning,3.2,, diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index e1ec3a17294465..116edd8d4d2fd6 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -116,7 +116,7 @@ def test_object_setattr(self): self.assertRaises(RuntimeError, xsetattr, obj, 'evil', NULL) self.assertRaises(RuntimeError, xsetattr, obj, 'evil', 'good') - self.assertRaises(NotWriteableError, xsetattr, 42, 'a', 5) + self.assertRaises(AttributeError, xsetattr, 42, 'a', 5) self.assertRaises(TypeError, xsetattr, obj, 1, 5) # CRASHES xsetattr(obj, NULL, 5) # CRASHES xsetattr(NULL, 'a', 5) @@ -136,7 +136,7 @@ def test_object_setattrstring(self): self.assertRaises(RuntimeError, setattrstring, obj, b'evil', NULL) self.assertRaises(RuntimeError, setattrstring, obj, b'evil', 'good') - self.assertRaises(NotWriteableError, setattrstring, 42, b'a', 5) + self.assertRaises(AttributeError, setattrstring, 42, b'a', 5) self.assertRaises(TypeError, setattrstring, obj, 1, 5) self.assertRaises(UnicodeDecodeError, setattrstring, obj, b'\xff', 5) # CRASHES setattrstring(obj, NULL, 5) @@ -153,7 +153,7 @@ def test_object_delattr(self): xdelattr(obj, '\U0001f40d') self.assertFalse(hasattr(obj, '\U0001f40d')) - self.assertRaises(NotWriteableError, xdelattr, 42, 'numerator') + self.assertRaises(AttributeError, xdelattr, 42, 'numerator') self.assertRaises(RuntimeError, xdelattr, obj, 'evil') self.assertRaises(TypeError, xdelattr, obj, 1) # CRASHES xdelattr(obj, NULL) @@ -170,7 +170,7 @@ def test_object_delattrstring(self): delattrstring(obj, '\U0001f40d'.encode()) self.assertFalse(hasattr(obj, '\U0001f40d')) - self.assertRaises(NotWriteableError, delattrstring, 42, b'numerator') + self.assertRaises(AttributeError, delattrstring, 42, b'numerator') self.assertRaises(RuntimeError, delattrstring, obj, b'evil') self.assertRaises(UnicodeDecodeError, delattrstring, obj, b'\xff') # CRASHES delattrstring(obj, NULL) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 07299a650a1112..3cbfa8428513fe 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1076,7 +1076,7 @@ class SubType(types.ModuleType): class MyInt(int): __slots__ = () - with self.assertRaises(NotWriteableError): + with self.assertRaises(TypeError): (1).__class__ = MyInt class MyFloat(float): @@ -1091,7 +1091,7 @@ class MyComplex(complex): class MyStr(str): __slots__ = () - with self.assertRaises(NotWriteableError): + with self.assertRaises(TypeError): "a".__class__ = MyStr class MyBytes(bytes): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 8cad71c7c34545..95eb05a1e6e39a 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -264,6 +264,7 @@ def test_windows_feature_macros(self): "PyExc_NameError", "PyExc_NotADirectoryError", "PyExc_NotImplementedError", + "PyExc_NotWriteableError", "PyExc_OSError", "PyExc_OverflowError", "PyExc_PendingDeprecationWarning", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 48299e9b35ff97..59488aaf0d5d6b 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2406,3 +2406,6 @@ added = '3.12' [const.Py_TPFLAGS_ITEMS_AT_END] added = '3.12' + +[data.PyExc_NotWriteableError] + added = '4.0' diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 5008811212703a..9dbab8669527e4 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3430,7 +3430,9 @@ PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; */ SimpleExtendsException(PyExc_Exception, BufferError, "Buffer error."); - +/* + * NotWriteableError extends Exception + */ SimpleExtendsException(PyExc_Exception, NotWriteableError, "Object is not writeable."); diff --git a/Objects/regions.c b/Objects/regions.c index 93cf61ba428395..3b19f3afd9b4f4 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -53,7 +53,7 @@ typedef struct stack_s { node* head; } stack; -stack* stack_new(void){ +static stack* stack_new(void){ stack* s = (stack*)malloc(sizeof(stack)); if(s == NULL){ return NULL; @@ -64,7 +64,7 @@ stack* stack_new(void){ return s; } -bool stack_push(stack* s, PyObject* object){ +static bool stack_push(stack* s, PyObject* object){ node* n = (node*)malloc(sizeof(node)); if(n == NULL){ Py_DECREF(object); @@ -78,7 +78,7 @@ bool stack_push(stack* s, PyObject* object){ return false; } -PyObject* stack_pop(stack* s){ +static PyObject* stack_pop(stack* s){ if(s->head == NULL){ return NULL; } @@ -91,7 +91,7 @@ PyObject* stack_pop(stack* s){ return object; } -void stack_free(stack* s){ +static void stack_free(stack* s){ while(s->head != NULL){ PyObject* op = stack_pop(s); Py_DECREF(op); @@ -100,18 +100,18 @@ void stack_free(stack* s){ free(s); } -bool stack_empty(stack* s){ +static bool stack_empty(stack* s){ return s->head == NULL; } -void stack_print(stack* s){ +static void stack_print(stack* s){ node* n = s->head; while(n != NULL){ n = n->next; } } -bool is_c_wrapper(PyObject* obj){ +static bool is_c_wrapper(PyObject* obj){ return PyCFunction_Check(obj) || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) || Py_IS_TYPE(obj, &PyWrapperDescr_Type); } @@ -339,7 +339,7 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) } \ } while(0) -PyObject* make_global_immutable(PyObject* globals, PyObject* name) +static PyObject* make_global_immutable(PyObject* globals, PyObject* name) { PyObject* value = PyDict_GetItem(globals, name); // value.rc = x @@ -364,7 +364,7 @@ PyObject* make_global_immutable(PyObject* globals, PyObject* name) * just those, and prevent those keys from being updated in the global dictionary * from this point onwards. */ -PyObject* walk_function(PyObject* op, stack* frontier) +static PyObject* walk_function(PyObject* op, stack* frontier) { PyObject* builtins; PyObject* globals; @@ -561,7 +561,7 @@ PyObject* walk_function(PyObject* op, stack* frontier) } \ } while(0) -int _makeimmutable_visit(PyObject* obj, void* frontier) +static int _makeimmutable_visit(PyObject* obj, void* frontier) { if(!_Py_IsImmutable(obj)){ if(stack_push((stack*)frontier, obj)){ diff --git a/PC/python3dll.c b/PC/python3dll.c index 1f4d006b8ad856..1dd359fe04a7cf 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -817,6 +817,7 @@ EXPORT_DATA(PyExc_ModuleNotFoundError) EXPORT_DATA(PyExc_NameError) EXPORT_DATA(PyExc_NotADirectoryError) EXPORT_DATA(PyExc_NotImplementedError) +EXPORT_DATA(PyExc_NotWriteableError) EXPORT_DATA(PyExc_OSError) EXPORT_DATA(PyExc_OverflowError) EXPORT_DATA(PyExc_PendingDeprecationWarning) @@ -844,7 +845,6 @@ EXPORT_DATA(PyExc_UnicodeTranslateError) EXPORT_DATA(PyExc_UnicodeWarning) EXPORT_DATA(PyExc_UserWarning) EXPORT_DATA(PyExc_ValueError) -EXPORT_DATA(PyExc_NotWriteableError) EXPORT_DATA(PyExc_Warning) EXPORT_DATA(PyExc_WindowsError) EXPORT_DATA(PyExc_ZeroDivisionError) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 6a7c14ebb220a8..a898d3c5a9e556 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -306,6 +306,8 @@ Modules/timemodule.c init_timezone YEAR - Objects/bytearrayobject.c - _PyByteArray_empty_string - Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - +Objects/exceptions.c - _PyExc_NotWriteableError - +Objects/exceptions.c - PyExc_NotWriteableError - Objects/genobject.c - ASYNC_GEN_IGNORED_EXIT_MSG - Objects/genobject.c - NON_INIT_CORO_MSG - Objects/longobject.c - _PyLong_DigitValue - From 1ebd426777caa1f1d44fe980df24003d04edb462 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 6 Dec 2024 11:50:02 +0000 Subject: [PATCH 16/68] Add missing statics --- Objects/regions.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index 3b19f3afd9b4f4..8f35894ca9c09a 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -137,7 +137,7 @@ regionmetadata* captured = CAPTURED_SENTINEL; /** * Enable the region check. */ -void notify_regions_in_use(void) +static void notify_regions_in_use(void) { // Do not re-enable, if we have detected a fault. if (!error_occurred) @@ -162,7 +162,7 @@ PyObject* _Py_EnableInvariant(void) * Set the global variables for a failure. * This allows the interpreter to inspect what has failed. */ -void set_failed_edge(PyObject* src, PyObject* tgt, const char* msg) +static void set_failed_edge(PyObject* src, PyObject* tgt, const char* msg) { Py_DecRef(error_src); Py_IncRef(src); @@ -247,7 +247,7 @@ visit_invariant_check(PyObject *tgt, void *src_void) return 0; } -void invariant_reset_captured_list(void) { +static void invariant_reset_captured_list(void) { // Reset the captured list while (captured != CAPTURED_SENTINEL) { regionmetadata* m = captured; @@ -670,7 +670,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) Py_RETURN_NONE; } -bool is_bridge_object(PyObject *op) { +static bool is_bridge_object(PyObject *op) { Py_uintptr_t region = Py_GET_REGION(op); if (IS_DEFAULT_REGION(region) || IS_IMMUTABLE_REGION(region)) { return false; From 25e4a1ad82d5ea8e2b362bd225838389c4ab8aac Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 6 Dec 2024 12:48:51 +0000 Subject: [PATCH 17/68] Fix hard coded number in test. --- Lib/test/test_doctest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index bca4915e0fa673..330fed2d5862db 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -707,7 +707,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 830 < len(tests) < 860 # approximate number of objects with docstrings + >>> 830 < len(tests) < 900 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests From 8f0167d675a3d48550d2f38590adc0723073b24d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 6 Dec 2024 12:56:12 +0000 Subject: [PATCH 18/68] Add missing statics --- Objects/regions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/regions.c b/Objects/regions.c index 8f35894ca9c09a..ad9d435ca88a9b 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -36,7 +36,7 @@ struct regionmetadata { regionmetadata* next; }; -bool is_bridge_object(PyObject *op); +static bool is_bridge_object(PyObject *op); static int regionmetadata_has_ancestor(regionmetadata* data, regionmetadata* other); static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj); From c5918e58019d317a25c007b195582cca18049c70 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 6 Dec 2024 13:36:37 +0000 Subject: [PATCH 19/68] White list. --- Tools/c-analyzer/cpython/ignored.tsv | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a898d3c5a9e556..546b105dfa2b87 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -714,3 +714,19 @@ Modules/expat/xmlrole.c - error - ## other Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - + +## Region Type Info this is constant +## Why do we have three of these? Surely it should just be in one file? +## Probably a bug in the analysis as they are listed as extern in two of the files. +Objects/object.c - PyRegion_Type - +Objects/regions.c - PyRegion_Type - +Python/bltinmodule.c - PyRegion_Type - + +## Regions Debug Info for Invariant +## Not to remain global, and should become localised to an interpreter +Objects/regions.c - do_region_check - +Objects/regions.c - error_src - +Objects/regions.c - error_tgt - +Objects/regions.c - error_occurred - +Objects/regions.c - captured - + From 9357239835ca090771733974a7cd46eb0c4cb5bd Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 2 Dec 2024 15:33:01 +0100 Subject: [PATCH 20/68] Pyrona: Add mask and visited flag to `ob_region` field --- Include/internal/pycore_regions.h | 2 +- Include/object.h | 21 ++++++++++----------- Objects/regions.c | 24 ++++++++++++++++-------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 5399dae2816c26..40f3cb5baf6e87 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -11,7 +11,7 @@ extern "C" { #include "object.h" -#define Py_CHECKWRITE(op) ((op) && _PyObject_CAST(op)->ob_region != _Py_IMMUTABLE) +#define Py_CHECKWRITE(op) ((op) && Py_REGION(ob) != _Py_IMMUTABLE) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} /* This makes the given objects and all object reachable from the given diff --git a/Include/object.h b/Include/object.h index 9b68365fcf5988..6b3c9c45af6c93 100644 --- a/Include/object.h +++ b/Include/object.h @@ -194,7 +194,9 @@ struct _object { PyTypeObject *ob_type; // VeronaPy: Field used for tracking which region this objects is stored in. - // Bottom bits stolen for distinguishing types of region ptr. + // Stolen bottom bits: + // 1. Indicates the region type. A set flag indicates the immutable region. + // 2. This flag is used for object traversal to indicate that it was visited. Py_uintptr_t ob_region; }; @@ -231,8 +233,12 @@ static inline PyTypeObject* Py_TYPE(PyObject *ob) { # define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) #endif +// This is the mask off all used bits to indicate the region. +// this should be used when the region pointer was requested. +// Macros for the individual flags are defined in regions.c. +#define Py_REGION_MASK (~((Py_uintptr_t)0x2)) static inline Py_uintptr_t Py_REGION(PyObject *ob) { - return ob->ob_region; + return (ob->ob_region & Py_REGION_MASK); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_REGION(ob) Py_REGION(_PyObject_CAST(ob)) @@ -271,13 +277,13 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) { - return op->ob_region == _Py_IMMUTABLE; + return Py_REGION(op) == _Py_IMMUTABLE; } #define _Py_IsImmutable(op) _Py_IsImmutable(_PyObject_CAST(op)) static inline Py_ALWAYS_INLINE int _Py_IsLocal(PyObject *op) { - return op->ob_region == _Py_DEFAULT_REGION; + return Py_REGION(op) == _Py_DEFAULT_REGION; } #define _Py_IsLocal(op) _Py_IsLocal(_PyObject_CAST(op)) @@ -320,13 +326,6 @@ static inline void Py_SET_REGION(PyObject *ob, Py_uintptr_t region) { # define Py_SET_REGION(ob, region) Py_SET_REGION(_PyObject_CAST(ob), (region)) #endif -static inline Py_uintptr_t Py_GET_REGION(PyObject *ob) { - return ob->ob_region; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_GET_REGION(ob) Py_GET_REGION(_PyObject_CAST(ob)) -#endif - /* Type objects contain a string containing the type name (to help somewhat diff --git a/Objects/regions.c b/Objects/regions.c index ad9d435ca88a9b..ec41a0a17c15de 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -10,6 +10,15 @@ #include "pycore_object.h" #include "pycore_regions.h" +#define Py_REGION_VISITED_FLAG ((Py_uintptr_t)0x2) +static inline Py_uintptr_t Py_REGION_WITH_FLAGS(PyObject *ob) { + return ob->ob_region; +} +#define Py_REGION_WITH_FLAGS(ob) Py_REGION_WITH_FLAGS(_PyObject_CAST(ob)) +#define REGION_SET_FLAG(ob, flag) (Py_REGION_WITH_FLAGS(ob) | flag) +#define REGION_GET_FLAG(ob, flag) (Py_REGION_WITH_FLAGS(ob) & flag) +#define REGION_CLEAR_FLAG(ob, flag) (Py_SET_REGION(ob, Py_REGION_WITH_FLAGS(ob) & (~flag))) + typedef struct regionmetadata regionmetadata; typedef struct PyRegionObject PyRegionObject; @@ -206,8 +215,8 @@ static int visit_invariant_check(PyObject *tgt, void *src_void) { PyObject *src = _PyObject_CAST(src_void); - regionmetadata* src_region = (regionmetadata*) Py_GET_REGION(src); - regionmetadata* tgt_region = (regionmetadata*) Py_GET_REGION(tgt); + regionmetadata* src_region = (regionmetadata*) Py_REGION(src); + regionmetadata* tgt_region = (regionmetadata*) Py_REGION(tgt); // Internal references are always allowed if (src_region == tgt_region) return 0; @@ -583,7 +592,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) notify_regions_in_use(); if(_Py_IsImmutable(obj) && _Py_IsImmutable(Py_TYPE(obj))){ - return obj; + Py_RETURN_NONE; } stack* frontier = stack_new(); @@ -607,9 +616,8 @@ PyObject* _Py_MakeImmutable(PyObject* obj) if(_Py_IsImmutable(item)){ // Direct access like this is not recommended, but will be removed in the future as // this is just for debugging purposes. - if(type->ob_base.ob_base.ob_region != _Py_IMMUTABLE){ + if((type->ob_base.ob_base.ob_region & Py_REGION_MASK) != _Py_IMMUTABLE){ // Why do we need to handle the type here, surely what ever made this immutable already did that? - // Log so we can investigate. } goto handle_type; } @@ -671,7 +679,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) } static bool is_bridge_object(PyObject *op) { - Py_uintptr_t region = Py_GET_REGION(op); + Py_uintptr_t region = Py_REGION(op); if (IS_DEFAULT_REGION(region) || IS_IMMUTABLE_REGION(region)) { return false; } @@ -866,7 +874,7 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { } regionmetadata* md = PyRegion_get_metadata(self); - if (Py_GET_REGION(args) == (Py_uintptr_t) md) { + if (Py_REGION(args) == (Py_uintptr_t) md) { Py_SET_REGION(args, _Py_DEFAULT_REGION); Py_RETURN_NONE; } else { @@ -877,7 +885,7 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { // Return True if args object is member of self region static PyObject *PyRegion_owns_object(PyRegionObject *self, PyObject *args) { - if ((Py_uintptr_t) PyRegion_get_metadata(self) == Py_GET_REGION(args)) { + if ((Py_uintptr_t) PyRegion_get_metadata(self) == Py_REGION(args)) { Py_RETURN_TRUE; } else { Py_RETURN_FALSE; From d197cddb25edf587d9ab8ddfb41ddfc0e3cfbec6 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 2 Dec 2024 18:35:02 +0100 Subject: [PATCH 21/68] Pyrona: Add `RegionError` and `add_to_region` for the write barrier --- Include/cpython/pyerrors.h | 6 + Include/internal/pycore_regions.h | 2 +- Include/object.h | 14 +- Include/pyerrors.h | 7 +- Lib/test/test_veronapy.py | 79 +++--- Objects/exceptions.c | 53 ++++ Objects/regions.c | 413 ++++++++++++++++++++++++++---- PC/python3dll.c | 1 + 8 files changed, 476 insertions(+), 99 deletions(-) diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 9890d1149ba7fc..c3a255fa3db824 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -82,6 +82,12 @@ typedef struct { PyObject *name; } PyAttributeErrorObject; +typedef struct { + PyException_HEAD + PyObject *source; + PyObject *target; +} PyRegionErrorObject; + /* Compatibility typedefs */ typedef PyOSErrorObject PyEnvironmentErrorObject; #ifdef MS_WINDOWS diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 40f3cb5baf6e87..9f871b7cefeeb1 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -11,7 +11,7 @@ extern "C" { #include "object.h" -#define Py_CHECKWRITE(op) ((op) && Py_REGION(ob) != _Py_IMMUTABLE) +#define Py_CHECKWRITE(op) ((op) && Py_REGION(op) != _Py_IMMUTABLE) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} /* This makes the given objects and all object reachable from the given diff --git a/Include/object.h b/Include/object.h index 6b3c9c45af6c93..e374dce4145e23 100644 --- a/Include/object.h +++ b/Include/object.h @@ -163,6 +163,9 @@ check by comparing the reference count field to the immortality reference count. #define PyObject_VAR_HEAD PyVarObject ob_base; #define Py_INVALID_SIZE (Py_ssize_t)-1 +typedef Py_uintptr_t Py_region_ptr; +typedef Py_uintptr_t Py_region_ptr_with_flags; + /* Nothing is actually declared to be a PyObject, but every pointer to * a Python object can be cast to a PyObject*. This is inheritance built * by hand. Similarly every pointer to a variable-size Python object can, @@ -233,7 +236,7 @@ static inline PyTypeObject* Py_TYPE(PyObject *ob) { # define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) #endif -// This is the mask off all used bits to indicate the region. +// This is the mask of all used bits to indicate the region. // this should be used when the region pointer was requested. // Macros for the individual flags are defined in regions.c. #define Py_REGION_MASK (~((Py_uintptr_t)0x2)) @@ -320,12 +323,19 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { #endif static inline void Py_SET_REGION(PyObject *ob, Py_uintptr_t region) { - ob->ob_region = region; + // Retain the old flags + ob->ob_region = (region & Py_REGION_MASK) | (ob->ob_region & (~Py_REGION_MASK)); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_REGION(ob, region) Py_SET_REGION(_PyObject_CAST(ob), (region)) #endif +static inline void Py_SET_REGION_WITH_FLAGS(PyObject *ob, Py_uintptr_t region) { + ob->ob_region = region; +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_REGION_WITH_FLAGS(ob, region) Py_SET_REGION_WITH_FLAGS(_PyObject_CAST(ob), (region)) +#endif /* Type objects contain a string containing the type name (to help somewhat diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 9dd230d3adcd60..659de5d49904b0 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -121,7 +121,6 @@ PyAPI_DATA(PyObject *) PyExc_UnicodeDecodeError; PyAPI_DATA(PyObject *) PyExc_UnicodeTranslateError; PyAPI_DATA(PyObject *) PyExc_ValueError; PyAPI_DATA(PyObject *) PyExc_ZeroDivisionError; -PyAPI_DATA(PyObject *) PyExc_NotWriteableError; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_DATA(PyObject *) PyExc_BlockingIOError; @@ -141,6 +140,12 @@ PyAPI_DATA(PyObject *) PyExc_ProcessLookupError; PyAPI_DATA(PyObject *) PyExc_TimeoutError; #endif +/* Pyrona Exceptions */ +PyAPI_DATA(PyObject *) PyExc_NotWriteableError; +// FIXME(xFrednet): We probably want finer error granualrity +// to destinqush the kind of error and if the system is in a +// valid state after the execption. +PyAPI_DATA(PyObject *) PyExc_RegionError; /* Compatibility aliases */ PyAPI_DATA(PyObject *) PyExc_EnvironmentError; diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index deeff7ec925f9c..94bbe7e1a160c7 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -423,18 +423,35 @@ def test_add_ownership2(self): r1.add_object(a) self.assertFalse(r2.owns_object(a)) - def test_should_fail_add_ownership_twice_1(self): + def test_add_object_is_deep(self): + # Create linked objects (a) -> (b) a = self.A() + b = self.A() + c = self.A() + a.b = b + b.c = c + + # Create a region and take ownership of a r = Region() r.add_object(a) - self.assertRaises(RuntimeError, r.add_object, a) + + # Check that b was also moved into the region + self.assertTrue(r.owns_object(a)) + self.assertTrue(r.owns_object(b)) + self.assertTrue(r.owns_object(c)) def test_should_fail_add_ownership_twice_2(self): a = self.A() - r = Region() - r.add_object(a) - r2 = Region() - self.assertRaises(RuntimeError, r2.add_object, a) + r1 = Region("r1") + r1.add_object(a) + r2 = Region("r2") + try: + r2.add_object(a) + except RegionError as e: + self.assertEqual(e.source, r2) + self.assertEqual(e.target, a) + else: + self.fail("Should not reach here -- a can't be owned by two objects") def test_init_with_name(self): r1 = Region() @@ -457,59 +474,33 @@ def test_init_same_name(self): # Check that we reach the end of the test self.assertTrue(True) -class TestRegionInvariance(unittest.TestCase): - class A: - pass - - def setUp(self): - # This freezes A and super and meta types of A namely `type` and `object` - makeimmutable(self.A) - enableinvariant() - - def test_invalid_point_to_local(self): - # Create linked objects (a) -> (b) - a = self.A() - b = self.A() - a.b = b - - # Create a region and take ownership of a - r = Region() - # FIXME: Once the write barrier is implemented, this assert will fail. - # The code above should work without any errors. - self.assertRaises(RuntimeError, r.add_object, a) - - # Check that the errors are on the appropriate objects - self.assertFalse(r.owns_object(b)) - self.assertTrue(r.owns_object(a)) - self.assertEqual(invariant_failure_src(), a) - self.assertEqual(invariant_failure_tgt(), b) - def test_allow_bridge_object_ref(self): # Create linked objects (a) -> (b) a = self.A() - b = Region() + b = Region("Child") a.b = b # Create a region and take ownership of a - r = Region() + r = Region("Parent") r.add_object(a) self.assertFalse(r.owns_object(b)) self.assertTrue(r.owns_object(a)) def test_should_fail_external_uniqueness(self): a = self.A() - r = Region() - a.f = r - a.g = r - r2 = Region() + r1 = Region("r1") + # Two refs from the local region are allowed + a.f = r1 + a.g = r1 + r2 = Region("r2") try: r2.add_object(a) - except RuntimeError: - # Check that the errors are on the appropriate objects - self.assertEqual(invariant_failure_src(), a) - self.assertEqual(invariant_failure_tgt(), r) + except RegionError as e: + # Check that the error is on the appropriate objects + self.assertEqual(e.source, a) + self.assertEqual(e.target, r1) else: - self.fail("Should not reach here -- external uniqueness validated but not caught by invariant checker") + self.fail("Should not reach here -- a can't be owned by two objects") # This test will make the Python environment unusable. # Should perhaps forbid making the frame immutable. diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 9dbab8669527e4..7f8663146ef2f4 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3430,11 +3430,63 @@ PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; */ SimpleExtendsException(PyExc_Exception, BufferError, "Buffer error."); +/* Pyrona Exceptions */ + /* * NotWriteableError extends Exception */ SimpleExtendsException(PyExc_Exception, NotWriteableError, "Object is not writeable."); +static int RegionError_init(PyRegionErrorObject *self, PyObject *args, PyObject *kwds) { + PyObject *source = NULL; + PyObject *target = NULL; + if (!PyArg_ParseTuple(args, "|OO", &source, &target)) { + return -1; + } + Py_XSETREF(self->source, Py_XNewRef(source)); + Py_XSETREF(self->target, Py_XNewRef(target)); + return 0; +} + +static int +RegionError_clear(PyRegionErrorObject *self) +{ + Py_CLEAR(self->source); + Py_CLEAR(self->target); + return BaseException_clear((PyBaseExceptionObject *)self); +} + +static void +RegionError_dealloc(PyRegionErrorObject *self) +{ + _PyObject_GC_UNTRACK(self); + RegionError_clear(self); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int +RegionError_traverse(PyRegionErrorObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->source); + Py_VISIT(self->target); + return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); +} + +static PyMemberDef RegionError_members[] = { + {"source", T_OBJECT, offsetof(PyRegionErrorObject, source), 0, PyDoc_STR("source")}, + {"target", T_OBJECT, offsetof(PyRegionErrorObject, target), 0, PyDoc_STR("target")}, + {NULL} /* Sentinel */ +}; + +static PyMethodDef RegionError_methods[] = { + {NULL} /* Sentinel */ +}; + +ComplexExtendsException(PyExc_Exception, RegionError, + RegionError, 0, + RegionError_methods, RegionError_members, + 0, BaseException_str, + "A reference violates the rules of ownership"); /* Warning category docstrings */ @@ -3622,6 +3674,7 @@ static struct static_exception static_exceptions[] = { ITEM(ValueError), ITEM(NotWriteableError), ITEM(Warning), + ITEM(RegionError), // Level 4: ArithmeticError(Exception) subclasses ITEM(FloatingPointError), diff --git a/Objects/regions.c b/Objects/regions.c index ec41a0a17c15de..fba011088beb95 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -9,15 +9,16 @@ #include "pycore_interp.h" #include "pycore_object.h" #include "pycore_regions.h" +#include "pycore_pyerrors.h" #define Py_REGION_VISITED_FLAG ((Py_uintptr_t)0x2) static inline Py_uintptr_t Py_REGION_WITH_FLAGS(PyObject *ob) { return ob->ob_region; } #define Py_REGION_WITH_FLAGS(ob) Py_REGION_WITH_FLAGS(_PyObject_CAST(ob)) -#define REGION_SET_FLAG(ob, flag) (Py_REGION_WITH_FLAGS(ob) | flag) +#define REGION_SET_FLAG(ob, flag) (Py_SET_REGION_WITH_FLAGS(ob, Py_REGION_WITH_FLAGS(ob) | flag)) #define REGION_GET_FLAG(ob, flag) (Py_REGION_WITH_FLAGS(ob) & flag) -#define REGION_CLEAR_FLAG(ob, flag) (Py_SET_REGION(ob, Py_REGION_WITH_FLAGS(ob) & (~flag))) +#define REGION_CLEAR_FLAG(ob, flag) (Py_SET_REGION_WITH_FLAGS(ob, Py_REGION_WITH_FLAGS(ob) & (~flag))) typedef struct regionmetadata regionmetadata; typedef struct PyRegionObject PyRegionObject; @@ -25,6 +26,50 @@ typedef struct PyRegionObject PyRegionObject; static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); +/** + * Global status for performing the region check. + */ +bool invariant_do_region_check = false; + +// The src object for an edge that invalidated the invariant. +PyObject* invariant_error_src = Py_None; + +// The tgt object for an edge that invalidated the invariant. +PyObject* invariant_error_tgt = Py_None; + +// Once an error has occurred this is used to surpress further checking +bool invariant_error_occurred = false; + +/* This uses the given arguments to create and throw a `RegionError` + */ +void throw_region_error(PyObject* src, PyObject* tgt, + const char *format_str, PyObject *obj) +{ + // Don't stomp existing exception + PyThreadState *tstate = _PyThreadState_GET(); + assert(tstate && "_PyThreadState_GET documentation says it's not safe, when?"); + if (_PyErr_Occurred(tstate)) { + return; + } + + // This disables the invariance check, as it could otherwise emit a runtime + // error before the emitted `RegionError` could be handled. + invariant_do_region_check = false; + invariant_error_occurred = true; + + // Create the error, this sets the error value in `tstate` + PyErr_Format(PyExc_RegionError, format_str, obj); + + // Set source and target fields + PyRegionErrorObject* exc = _Py_CAST(PyRegionErrorObject*, + PyErr_GetRaisedException()); + Py_XINCREF(src); + exc->source = src; + Py_XINCREF(tgt); + exc->target = tgt; + PyErr_SetRaisedException(_PyObject_CAST(exc)); +} + struct PyRegionObject { PyObject_HEAD regionmetadata* metadata; @@ -124,20 +169,6 @@ static bool is_c_wrapper(PyObject* obj){ return PyCFunction_Check(obj) || Py_IS_TYPE(obj, &_PyMethodWrapper_Type) || Py_IS_TYPE(obj, &PyWrapperDescr_Type); } -/** - * Global status for performing the region check. - */ -bool do_region_check = false; - -// The src object for an edge that invalidated the invariant. -PyObject* error_src = Py_None; - -// The tgt object for an edge that invalidated the invariant. -PyObject* error_tgt = Py_None; - -// Once an error has occurred this is used to surpress further checking -bool error_occurred = false; - // Start of a linked list of bridge objects used to check for external uniqueness // Bridge objects appear in this list if they are captured #define CAPTURED_SENTINEL ((regionmetadata*) 0xc0defefe) @@ -149,21 +180,21 @@ regionmetadata* captured = CAPTURED_SENTINEL; static void notify_regions_in_use(void) { // Do not re-enable, if we have detected a fault. - if (!error_occurred) - do_region_check = true; + if (!invariant_error_occurred) + invariant_do_region_check = true; } PyObject* _Py_EnableInvariant(void) { // Disable failure as program has explicitly requested invariant to be checked again. - error_occurred = false; + invariant_error_occurred = false; // Re-enable region check - do_region_check = true; + invariant_do_region_check = true; // Reset the error state - Py_DecRef(error_src); - error_src = Py_None; - Py_DecRef(error_tgt); - error_tgt = Py_None; + Py_DecRef(invariant_error_src); + invariant_error_src = Py_None; + Py_DecRef(invariant_error_tgt); + invariant_error_tgt = Py_None; return Py_None; } @@ -171,29 +202,37 @@ PyObject* _Py_EnableInvariant(void) * Set the global variables for a failure. * This allows the interpreter to inspect what has failed. */ -static void set_failed_edge(PyObject* src, PyObject* tgt, const char* msg) +static void emit_invariant_error(PyObject* src, PyObject* tgt, const char* msg) { - Py_DecRef(error_src); + Py_DecRef(invariant_error_src); Py_IncRef(src); - error_src = src; - Py_DecRef(error_tgt); + invariant_error_src = src; + Py_DecRef(invariant_error_tgt); Py_IncRef(tgt); - error_tgt = tgt; + invariant_error_tgt = tgt; + + /* Don't stomp existing exception */ + PyThreadState *tstate = _PyThreadState_GET(); + assert(tstate && "_PyThreadState_GET documentation says it's not safe, when?"); + if (_PyErr_Occurred(tstate)) { + return; + } + PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p -> %p: %s\n", src, tgt, msg); // We have discovered a failure. // Disable region check, until the program switches it back on. - do_region_check = false; - error_occurred = true; + invariant_do_region_check = false; + invariant_error_occurred = true; } PyObject* _Py_InvariantSrcFailure(void) { - return Py_NewRef(error_src); + return Py_NewRef(invariant_error_src); } PyObject* _Py_InvariantTgtFailure(void) { - return Py_NewRef(error_tgt); + return Py_NewRef(invariant_error_tgt); } @@ -228,24 +267,24 @@ visit_invariant_check(PyObject *tgt, void *src_void) return 0; // Since tgt is not immutable, src also may not be as immutable may not point to mutable if (IS_IMMUTABLE_REGION(src_region)) { - set_failed_edge(src, tgt, "Reference from immutable object to mutable target"); + emit_invariant_error(src, tgt, "Reference from immutable object to mutable target"); return 0; } // Cross-region references must be to a bridge if (!is_bridge_object(tgt)) { - set_failed_edge(src, tgt, "Reference from object in one region into another region"); + emit_invariant_error(src, tgt, "Reference from object in one region into another region"); return 0; } // Check if region is already added to captured list if (tgt_region->next != NULL) { // Bridge object was already captured - set_failed_edge(src, tgt, "Reference to bridge is not externally unique"); + emit_invariant_error(src, tgt, "Reference to bridge is not externally unique"); return 0; } // Forbid cycles in the region topology if (regionmetadata_has_ancestor((regionmetadata*)src_region, (regionmetadata*)tgt_region)) { - set_failed_edge(src, tgt, "Regions create a cycle with subreagions"); + emit_invariant_error(src, tgt, "Regions create a cycle with subregions"); return 0; } @@ -282,7 +321,7 @@ static void invariant_reset_captured_list(void) { int _Py_CheckRegionInvariant(PyThreadState *tstate) { // Check if we should perform the region invariant check - if(!do_region_check){ + if(!invariant_do_region_check){ return 0; } @@ -328,7 +367,7 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) // write too much. // TODO: The first error might not be the most useful. // So might not need to build all error edges as a structure. - if (error_occurred) { + if (invariant_error_occurred) { invariant_reset_captured_list(); return 1; } @@ -373,7 +412,7 @@ static PyObject* make_global_immutable(PyObject* globals, PyObject* name) * just those, and prevent those keys from being updated in the global dictionary * from this point onwards. */ -static PyObject* walk_function(PyObject* op, stack* frontier) +static PyObject* make_function_immutable(PyObject* op, stack* frontier) { PyObject* builtins; PyObject* globals; @@ -591,6 +630,9 @@ PyObject* _Py_MakeImmutable(PyObject* obj) // We have started using regions, so notify to potentially enable checks. notify_regions_in_use(); + // Some built-in objects are direclty created immutable. However, their types + // might be created in a mutable state. This therefore requres an additional + // check to see if the type is also immutable. if(_Py_IsImmutable(obj) && _Py_IsImmutable(Py_TYPE(obj))){ Py_RETURN_NONE; } @@ -616,7 +658,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) if(_Py_IsImmutable(item)){ // Direct access like this is not recommended, but will be removed in the future as // this is just for debugging purposes. - if((type->ob_base.ob_base.ob_region & Py_REGION_MASK) != _Py_IMMUTABLE){ + if (Py_REGION(&type->ob_base.ob_base) != _Py_IMMUTABLE) { // Why do we need to handle the type here, surely what ever made this immutable already did that? } goto handle_type; @@ -630,7 +672,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) } if(PyFunction_Check(item)){ - _Py_MAKEIMMUTABLE_CALL(walk_function, item, frontier); + _Py_MAKEIMMUTABLE_CALL(make_function_immutable, item, frontier); goto handle_type; } @@ -678,6 +720,282 @@ PyObject* _Py_MakeImmutable(PyObject* obj) Py_RETURN_NONE; } +typedef enum region_error_id { + /* Adding this object to a region or creating this reference would + * create a reference that points to a contained(non-bridge object) + * inside another region. + */ + ERR_CONTAINED_OBJ_REF, + /* Adding this object to a region or creating this reference would + * create a cycle in the region topology. + */ + ERR_CYCLE_CREATION, + /* Adding this object to a region or creating this reference would + * isn't possible as the referenced bridge object already has a parent + * region. + */ + ERR_SHARED_CUSTODY, + /* Functions can reference to global variables. That's why they need + * special handling, as can be seen in `_Py_MakeImmutable`. + * For now an error is emitted to see when this comes up and if + * `make_function_immutable` can be reused. + */ + ERR_WIP_FUNCTIONS, +} region_error_id; + +/* An error that occurred in `add_to_region`. The struct contains all + * informaiton needed to construct an error message or handle the error + * differently. + */ +typedef struct regionerror { + /* The source of the reference that created the region error. + * + * A weak reference, can be made into a strong reference with `Py_INCREF` + */ + PyObject* src; + /* The target of the reference that created the region error. + * + * A weak reference, can be made into a strong reference with `Py_INCREF` + */ + PyObject* tgt; + /* This ID indicates what kind of error occurred. + */ + region_error_id id; +} regionerror; + +/* Used by `_add_to_region_visit` to handle errors. The first argument is + * the error information. The second argument is supplementary data + * passed along by `add_to_region`. + */ +typedef int (*handle_add_to_region_error)(regionerror *, void *); + +/* This takes the region error and emits it as a `RegionError` to the + * user. This function will always return `false` to stop the propagation + * from `add_to_region` + * + * This function borrows both arguments. The memory has to be managed + * the caller. + */ +static int emit_region_error(regionerror *error, void*) { + const char* msg = NULL; + + switch (error->id) + { + case ERR_CONTAINED_OBJ_REF: + msg = "References to objects in other regions are forbidden"; + break; + case ERR_CYCLE_CREATION: + msg = "Regions are not allowed to create cycles"; + break; + case ERR_SHARED_CUSTODY: + msg = "Regions can only have one parent at a time"; + break; + case ERR_WIP_FUNCTIONS: + msg = "WIP: Functions in regions are not supported yet"; + break; + default: + assert(false && "unreachable?"); + break; + } + throw_region_error(error->src, error->tgt, msg, NULL); + + // We never want to continue once an error has been emitted. + return -1; +} + +typedef struct addtoregionvisitinfo { + stack* pending; + // The source object of the reference. This is used to create + // better error message + PyObject* src; + handle_add_to_region_error handle_error; + void* handle_error_data; +} addtoregionvisitinfo; + +static int _add_to_region_visit(PyObject* target, void* info_void) +{ + addtoregionvisitinfo *info = _Py_CAST(addtoregionvisitinfo *, info_void); + + // Region objects are allowed to reference immutable objects. Immutable + // objects are only allowed to reference other immutable objects and cowns. + // we therefore don't need to traverse them. + if (_Py_IsImmutable(target)) { + return 0; + } + + // If the object is already in our region, we don't need to traverse it + if (Py_REGION(target) == Py_REGION(info->src)) { + return 0; + } + + // C wrappers can propergate through the entire system and draw + // in a lot of unwanted objects. Since c wrappers don't have mutable + // data, we just make it immutable and have the immutability impl + // handle it. We then have an edge from our region to an immutable + // object which is again valid. + if(is_c_wrapper(target)) { + _Py_MakeImmutable(target); + return 0; + } + + // We push it onto the stack to be added to the region and traversed. + // The actual addition of the object is done in `add_to_region`. We keep + // it in the local region, to indicate to `add_to_region` that the object + // should actually be processed. + if (IS_DEFAULT_REGION(Py_REGION(target))) { + // The actual region update and write checks are done in the + // main body of `add_to_region` + if (stack_push(info->pending, target)) { + PyErr_NoMemory(); + return -1; + } + return 0; + } + + // At this point, we know that target is in another region. + // If target is in a different region, it has to be a bridge object. + // References to contained objects are forbidden. + if (!is_bridge_object(target)) { + regionerror err = {.src = info->src, .tgt = target, + .id = ERR_CONTAINED_OBJ_REF }; + return ((info->handle_error)(&err, info->handle_error_data)); + } + + // The target is a bridge object from another region. We now need to + // if it already has a parent. + regionmetadata *target_region = _Py_CAST(regionmetadata *, Py_REGION(target)); + if (target_region->parent != NULL) { + regionerror err = {.src = info->src, .tgt = target, + .id = ERR_SHARED_CUSTODY}; + return ((info->handle_error)(&err, info->handle_error_data)); + } + + // Make sure that the new subregion relation won't create a cycle + regionmetadata* region = _Py_CAST(regionmetadata*, Py_REGION(info->src)); + if (regionmetadata_has_ancestor(target_region, region)) { + regionerror err = {.src = info->src, .tgt = target, + .id = ERR_CYCLE_CREATION}; + return ((info->handle_error)(&err, info->handle_error_data)); + } + + // From the previous checks we know that `target` is the bridge object + // of a free region. Thus we can make it a sub region and allow the + // reference. + target_region->parent = region; + + return 0; +} + +/* This adds the given object and transitive objects to the given region. + * Errors will be passed to the given `handle_error` function along with + * the `handle_error_data`. + * + * The given region may contain flags. Objects from the local regions will + * have their region replaced with the `flagged_region` value. This will + * lose flags set on the local region. + */ +static PyObject *add_to_region(PyObject *obj, Py_uintptr_t flagged_region) +{ + if (!obj) { + Py_RETURN_NONE; + } + + // Make sure there are no pending exceptions that would be overwritten + // by us. + PyThreadState *tstate = _PyThreadState_GET(); + if (_PyErr_Occurred(tstate)) { + return NULL; + } + + // Make sure we check against the actual region and not the region + // plus magic flags + Py_uintptr_t region = flagged_region & Py_REGION_MASK; + + // The current implementation assumes region is a valid pointer. This + // restriction can be lifted if needed + assert(!IS_DEFAULT_REGION(region) || !IS_IMMUTABLE_REGION(region)); + regionmetadata *region_data = _Py_CAST(regionmetadata *, region); + + // Early return if the object is already in the region or immutable + if (Py_REGION(obj) == region || _Py_IsImmutable(obj)) { + Py_RETURN_NONE; + } + + addtoregionvisitinfo info = { + .pending = stack_new(), + // `src` is reassigned each iteration + .src = _PyObject_CAST(region_data->bridge), + .handle_error = emit_region_error, + .handle_error_data = NULL, + }; + if (info.pending == NULL) { + return PyErr_NoMemory(); + } + + // The visit call is used to correctly add the object or + // add it to the pending stack, for further processing. + if (_add_to_region_visit(obj, &info)) { + stack_free(info.pending); + return NULL; + } + + while (!stack_empty(info.pending)) { + PyObject *item = stack_pop(info.pending); + + // The item was previously in the local region but has already been + // added to the region by a previous iteration. We therefore only need + // to adjust the LRC + if (Py_REGION(item) == region) { + // -1 for the refernce we just followed + region_data->lrc -= 1; + continue; + } + + if (IS_DEFAULT_REGION(Py_REGION(item))) { + // Add reference to the object, + // minus one for the reference we just followed + region_data->lrc += item->ob_refcnt - 1; + Py_SET_REGION(item, region); + + // Add `info.src` for better error messages + info.src = item; + + if (PyFunction_Check(item)) { + // FIXME: This is a temporary error. It should be replaced by + // proper handling of moving the function into the region + regionerror err = {.src = _PyObject_CAST(region_data->bridge), + .tgt = item, .id = ERR_WIP_FUNCTIONS }; + emit_region_error(&err, NULL); + } else { + PyTypeObject *type = Py_TYPE(item); + traverseproc traverse = type->tp_traverse; + if (traverse != NULL) { + if (traverse(item, (visitproc)_add_to_region_visit, &info)) { + stack_free(info.pending); + return NULL; + } + } + } + + PyObject* type_op = _PyObject_CAST(Py_TYPE(item)); + if (Py_REGION(type_op) != region && !_Py_IsImmutable(type_op)) { + if (stack_push(info.pending, type_op)) + { + stack_free(info.pending); + return PyErr_NoMemory(); + } + } + + continue; + } + } + + stack_free(info.pending); + + Py_RETURN_NONE; +} + + static bool is_bridge_object(PyObject *op) { Py_uintptr_t region = Py_REGION(op); if (IS_DEFAULT_REGION(region) || IS_IMMUTABLE_REGION(region)) { @@ -857,14 +1175,7 @@ static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args) { Py_RETURN_NONE; } - regionmetadata* md = PyRegion_get_metadata(self); - if (_Py_IsLocal(args)) { - Py_SET_REGION(args, (Py_uintptr_t) md); - Py_RETURN_NONE; - } else { - PyErr_SetString(PyExc_RuntimeError, "Object already had an owner or was immutable!"); - return NULL; - } + return add_to_region(args, Py_REGION(self)); } // Remove args object to self region @@ -885,7 +1196,7 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { // Return True if args object is member of self region static PyObject *PyRegion_owns_object(PyRegionObject *self, PyObject *args) { - if ((Py_uintptr_t) PyRegion_get_metadata(self) == Py_REGION(args)) { + if (Py_REGION(self) == Py_REGION(args)) { Py_RETURN_TRUE; } else { Py_RETURN_FALSE; diff --git a/PC/python3dll.c b/PC/python3dll.c index 1dd359fe04a7cf..c204f04ff381b0 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -825,6 +825,7 @@ EXPORT_DATA(PyExc_PermissionError) EXPORT_DATA(PyExc_ProcessLookupError) EXPORT_DATA(PyExc_RecursionError) EXPORT_DATA(PyExc_ReferenceError) +EXPORT_DATA(PyExc_RegionError) EXPORT_DATA(PyExc_ResourceWarning) EXPORT_DATA(PyExc_RuntimeError) EXPORT_DATA(PyExc_RuntimeWarning) From 43a1497fa316a946a234446ccb3312dc002eeb91 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Fri, 6 Dec 2024 17:40:28 +0100 Subject: [PATCH 22/68] WIP Adding internally created __dict__'s to the region of their owners --- Include/internal/pycore_dict.h | 2 +- Include/internal/pycore_regions.h | 10 ++++ Lib/test/test_veronapy.py | 22 ++++++++ Objects/dictobject.c | 24 +++++++-- Objects/object.c | 4 +- Objects/regions.c | 84 ++++++++++++++++++++++++++----- 6 files changed, 127 insertions(+), 19 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 8f044874121741..02847bd63e9860 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -63,7 +63,7 @@ extern PyObject *_PyDict_SetKeyImmutable(PyDictObject *mp, PyObject *key); /* Consumes references to key and value */ extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); +extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value, PyObject* owner); extern PyObject *_PyDict_Pop_KnownHash(PyObject *, PyObject *, Py_hash_t, PyObject *); diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 5399dae2816c26..3912f1c8f4b336 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -38,6 +38,16 @@ PyObject* _Py_EnableInvariant(void); PyObject* _Py_ResetInvariant(void); #define Py_ResetInvariant() _Py_ResetInvariant() +// Invariant placeholder +bool _Pyrona_AddReference(PyObject* target, PyObject* new_ref); +#define Pyrona_ADDREFERENCE(a, b) _Pyrona_AddReference(a, b) +// Helper macros to count the number of arguments +#define _COUNT_ARGS(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, N, ...) N +#define COUNT_ARGS(...) _COUNT_ARGS(__VA_ARGS__, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) + +bool _Pyrona_AddReferences(PyObject* target, int new_refc, ...); +#define Pyrona_ADDREFERENCES(a, ...) _Pyrona_AddReferences(a, COUNT_ARGS(__VA_ARGS__), __VA_ARGS__) + #ifdef NDEBUG #define _Py_VPYDBG(fmt, ...) #define _Py_VPYDBGPRINT(fmt, ...) diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index deeff7ec925f9c..3bbf40f8863c05 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -396,6 +396,8 @@ class A: def setUp(self): # This freezes A and super and meta types of A namely `type` and `object` makeimmutable(self.A) + # FIXME: remove this line when the write barrier works + makeimmutable(type({})) enableinvariant() def test_default_ownership(self): @@ -457,6 +459,24 @@ def test_init_same_name(self): # Check that we reach the end of the test self.assertTrue(True) + def test_region__dict__(self): + r = Region() + r.f = self.A() + # The above line will fail unless the region has gotten a dict + self.assertTrue(True) + + def test_object__dict__(self): + r = Region() + a = self.A() + b = self.A() + r.add_object(b) + r.f = a + a.f = b + d = a.__dict__ + self.assertTrue(r.owns_object(d)) + self.assertTrue(r.owns_object(a)) + self.assertTrue(r.owns_object(b)) + class TestRegionInvariance(unittest.TestCase): class A: pass @@ -464,6 +484,8 @@ class A: def setUp(self): # This freezes A and super and meta types of A namely `type` and `object` makeimmutable(self.A) + # FIXME: remove this line when the write barrier works + makeimmutable(type({})) enableinvariant() def test_invalid_point_to_local(self): diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7867ad70431b6..4fea81fe9c5319 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5505,6 +5505,7 @@ _PyObject_InitializeDict(PyObject *obj) return -1; } PyObject **dictptr = _PyObject_ComputedDictPointer(obj); + Pyrona_ADDREFERENCE(obj, dict); *dictptr = dict; return 0; } @@ -5772,6 +5773,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) _Py_SetImmutable(dict); } else { + Pyrona_ADDREFERENCE(obj, dict); dorv_ptr->dict = dict; } } @@ -5785,6 +5787,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) _Py_SetImmutable(dict); } else { + Pyrona_ADDREFERENCE(obj, dict); dorv_ptr->dict = dict; } } @@ -5812,6 +5815,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) _Py_SetImmutable(dict); } else { + Pyrona_ADDREFERENCE(obj, dict); *dictptr = dict; } } @@ -5821,7 +5825,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, - PyObject *key, PyObject *value) + PyObject *key, PyObject *value, PyObject* owner) { PyObject *dict; int res; @@ -5835,6 +5839,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, if (dict == NULL) { dictkeys_incref(cached); dict = new_dict_with_shared_keys(interp, cached); + Pyrona_ADDREFERENCE(owner, dict); if (dict == NULL) return -1; *dictptr = dict; @@ -5843,7 +5848,12 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, res = PyDict_DelItem(dict, key); } else { - res = PyDict_SetItem(dict, key, value); + if (Pyrona_ADDREFERENCES(dict, key, value)) { + res = PyDict_SetItem(dict, key, value); + } else { + // Error is set inside ADDREFERENCE + return -1; + } } } else { dict = *dictptr; @@ -5851,12 +5861,18 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, dict = PyDict_New(); if (dict == NULL) return -1; + Pyrona_ADDREFERENCE(owner, dict); *dictptr = dict; } if (value == NULL) { res = PyDict_DelItem(dict, key); } else { - res = PyDict_SetItem(dict, key, value); + if (Pyrona_ADDREFERENCES(dict, key, value)) { + res = PyDict_SetItem(dict, key, value); + } else { + // Error is set inside ADDREFERENCE + return -1; + } } } ASSERT_CONSISTENT(dict); @@ -6067,4 +6083,4 @@ _PyDict_IsKeyImmutable(PyObject* op, PyObject* key) PyDictKeyEntry *ep = DK_ENTRIES(mp->ma_keys) + ix; return _PyDictEntry_IsImmutable(ep); } -} \ No newline at end of file +} diff --git a/Objects/object.c b/Objects/object.c index 09b663f1f58e6f..101c8e0514ecb6 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1267,6 +1267,7 @@ _PyObject_GetDictPtr(PyObject *obj) PyErr_Clear(); return NULL; } + Pyrona_ADDREFERENCE(obj, dict); dorv_ptr->dict = dict; } return &dorv_ptr->dict; @@ -1466,6 +1467,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, res = NULL; goto done; } + Pyrona_ADDREFERENCE(obj, dict); dorv_ptr->dict = dict; } } @@ -1596,7 +1598,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto done; } else { - res = _PyObjectDict_SetItem(tp, dictptr, name, value); + res = _PyObjectDict_SetItem(tp, dictptr, name, value, obj); } } else { diff --git a/Objects/regions.c b/Objects/regions.c index 93cf61ba428395..0ab9140bac5e13 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -15,6 +15,8 @@ typedef struct PyRegionObject PyRegionObject; static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); +static const char *get_region_name(PyObject* obj); +#define Py_REGION_DATA(ob) (_Py_CAST(regionmetadata*, Py_REGION(ob))) struct PyRegionObject { PyObject_HEAD @@ -170,7 +172,13 @@ void set_failed_edge(PyObject* src, PyObject* tgt, const char* msg) Py_DecRef(error_tgt); Py_IncRef(tgt); error_tgt = tgt; - PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p -> %p: %s\n", src, tgt, msg); + const char *src_region_name = get_region_name(src); + const char *tgt_region_name = get_region_name(tgt); + PyObject *tgt_type_repr = PyObject_Repr(PyObject_Type(tgt)); + const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; + PyObject *src_type_repr = PyObject_Repr(PyObject_Type(src)); + const char *src_desc = src_type_repr ? PyUnicode_AsUTF8(src_type_repr) : "<>"; + PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s): %s\n", src, src_desc, src_region_name, tgt, tgt_desc, tgt_region_name, msg); // We have discovered a failure. // Disable region check, until the program switches it back on. do_region_check = false; @@ -802,17 +810,6 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { Py_SET_REGION(self, (Py_uintptr_t) self->metadata); _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); - // FIXME: Usually this is created on the fly. We need to do it manually to - // set the region and freeze the type - self->dict = PyDict_New(); - if (self->dict == NULL) { - return -1; // Propagate memory allocation failure - } - // TODO: Once phase 2 is done, we might be able to do this statically - // at compile time. - _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self->dict))); - PyRegion_add_object(self, self->dict); - return 0; } @@ -853,8 +850,14 @@ static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args) { if (_Py_IsLocal(args)) { Py_SET_REGION(args, (Py_uintptr_t) md); Py_RETURN_NONE; + } else if (_Py_IsImmutable(args)) { + // Nothing to do! Let this pass for now. + Py_RETURN_NONE; } else { - PyErr_SetString(PyExc_RuntimeError, "Object already had an owner or was immutable!"); + const char *arg_region_name = get_region_name(args); + PyObject *src_type_repr = PyObject_Repr(PyObject_Type(args)); + const char *src_desc = src_type_repr ? PyUnicode_AsUTF8(src_type_repr) : "<>"; + PyErr_Format(PyExc_RuntimeError, "Object %s already had an owner (%s)!", src_desc, arg_region_name); return NULL; } } @@ -959,3 +962,58 @@ PyTypeObject PyRegion_Type = { 0, /* tp_alloc */ PyType_GenericNew, /* tp_new */ }; + +static const char *get_region_name(PyObject* obj) { + if (_Py_IsLocal(obj)) { + return "Default"; + } else if (_Py_IsImmutable(obj)) { + return "Immutable"; + } else { + const regionmetadata *md = Py_REGION_DATA(obj); + return md->bridge->name + ? PyUnicode_AsUTF8(md->bridge->name) + : ""; + } +} + +// TODO replace with write barrier code +bool _Pyrona_AddReference(PyObject *tgt, PyObject *new_ref) { + if (_Py_IsImmutable(new_ref)) { + // Nothing to do -- adding a ref to an immutable is always permitted + return true; + } + + if (Py_REGION(tgt) == Py_REGION(new_ref)) { + // Nothing to do -- intra-region references are always permitted + return true; + } + + if (_Py_IsLocal(new_ref)) { + // Slurp emphemerally owned object into the region of the target object + fprintf(stderr, "Added %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); + PyRegion_add_object(((regionmetadata*) tgt->ob_region)->bridge, new_ref); + return true; + } + + const char *new_ref_region_name = get_region_name(new_ref); + const char *tgt_region_name = get_region_name(tgt); + PyObject *tgt_type_repr = PyObject_Repr(PyObject_Type(tgt)); + const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; + PyObject *new_ref_type_repr = PyObject_Repr(PyObject_Type(new_ref)); + const char *new_ref_desc = new_ref_type_repr ? PyUnicode_AsUTF8(new_ref_type_repr) : "<>"; + PyErr_Format(PyExc_RuntimeError, "WBError: Invalid edge %p (%s in %s) -> %p (%s in %s)\n", tgt, tgt_desc, tgt_region_name, new_ref, new_ref_desc, new_ref_region_name); + return true; // Illegal reference +} + +bool _Pyrona_AddReferences(PyObject *tgt, int new_refc, ...) { + va_list args; + va_start(args, new_refc); + + for (int i = 0; i < new_refc; i++) { + int res = _Pyrona_AddReference(tgt, va_arg(args, PyObject*)); + if (!res) return false; + } + + va_end(args); + return true; +} From f3bccd377ab2426bd14c58829aa8ee38b4ef6048 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Fri, 6 Dec 2024 11:49:53 +0100 Subject: [PATCH 23/68] Pyrona: Add `Py_region_ptr_with_tags_t` --- Doc/conf.py | 1 + Doc/data/stable_abi.dat | 1 + Include/internal/pycore_object.h | 12 ++--- Include/internal/pycore_regions.h | 2 +- Include/object.h | 70 ++++++++++++++++------------ Lib/_compat_pickle.py | 1 + Lib/test/exception_hierarchy.txt | 1 + Lib/test/test_stable_abi_ctypes.py | 1 + Misc/stable_abi.toml | 2 + Objects/object.c | 2 +- Objects/regions.c | 46 ++++++++---------- Python/errors.c | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 10 ++-- 13 files changed, 81 insertions(+), 70 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index dbd75012988442..f6ff1c7f511c3d 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -200,6 +200,7 @@ ('c:data', 'PyExc_ProcessLookupError'), ('c:data', 'PyExc_RecursionError'), ('c:data', 'PyExc_ReferenceError'), + ('c:data', 'PyExc_RegionError'), ('c:data', 'PyExc_RuntimeError'), ('c:data', 'PyExc_StopAsyncIteration'), ('c:data', 'PyExc_StopIteration'), diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index a5a86646ef1b78..8cb401b6204983 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -246,6 +246,7 @@ var,PyExc_PermissionError,3.7,, var,PyExc_ProcessLookupError,3.7,, var,PyExc_RecursionError,3.7,, var,PyExc_ReferenceError,3.2,, +var,PyExc_RegionError,4.0,, var,PyExc_ResourceWarning,3.7,, var,PyExc_RuntimeError,3.2,, var,PyExc_RuntimeWarning,3.2,, diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index e1a0e2af059231..a5270a09d191ef 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -24,11 +24,11 @@ extern "C" { are not supported pre-C++20. Thus, keeping an internal copy here is the most backwards compatible solution */ #define _PyObject_HEAD_INIT(type) \ - { \ - _PyObject_EXTRA_INIT \ - .ob_refcnt = _Py_IMMORTAL_REFCNT, \ - .ob_type = (type), \ - .ob_region = _Py_DEFAULT_REGION \ + { \ + _PyObject_EXTRA_INIT \ + .ob_refcnt = _Py_IMMORTAL_REFCNT, \ + .ob_type = (type), \ + .ob_region = (Py_region_ptr_with_tags_t){_Py_DEFAULT_REGION} \ }, #define _PyVarObject_HEAD_INIT(type, size) \ { \ @@ -96,7 +96,7 @@ static inline void _Py_ClearImmortal(PyObject *op) static inline void _Py_SetImmutable(PyObject *op) { if(op) { - op->ob_region = _Py_IMMUTABLE; + Py_SET_REGION(op, _Py_IMMUTABLE); // TODO once reference counting across regions is fully working // we no longer need to make all immutable objects immortal op->ob_refcnt = _Py_IMMORTAL_REFCNT; diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 9f871b7cefeeb1..c95d4e6cf2ef7e 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -11,7 +11,7 @@ extern "C" { #include "object.h" -#define Py_CHECKWRITE(op) ((op) && Py_REGION(op) != _Py_IMMUTABLE) +#define Py_CHECKWRITE(op) ((op) && !_Py_IsImmutable(op)) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} /* This makes the given objects and all object reachable from the given diff --git a/Include/object.h b/Include/object.h index e374dce4145e23..c8588c214dc71b 100644 --- a/Include/object.h +++ b/Include/object.h @@ -125,26 +125,43 @@ check by comparing the reference count field to the immortality reference count. #define _Py_IMMORTAL_REFCNT (UINT_MAX >> 2) #endif -#define _Py_DEFAULT_REGION ((Py_uintptr_t)0) -#define _Py_IMMUTABLE ((Py_uintptr_t)1) +// This is only a typedef of `Py_uintptr_t` opposed to a custom typedef +// to allow comparisons and make casts easier. +typedef Py_uintptr_t Py_region_ptr_t; +typedef struct { Py_uintptr_t value; } Py_region_ptr_with_tags_t; + +// This is the mask of all used bits to indicate the region. +// this should be used when the region pointer was requested. +// Macros for the individual flags are defined in regions.c. +#define Py_REGION_MASK (~((Py_region_ptr_t)0x2)) +static inline Py_region_ptr_t Py_region_ptr(Py_region_ptr_with_tags_t tagged_region) { + return (Py_region_ptr_t)(tagged_region.value & Py_REGION_MASK); +} + +static inline Py_region_ptr_with_tags_t Py_region_ptr_with_tags(Py_region_ptr_t region) { + return (Py_region_ptr_with_tags_t) { region }; +} + +#define _Py_DEFAULT_REGION ((Py_region_ptr_t)0) +#define _Py_IMMUTABLE ((Py_region_ptr_t)1) // Make all internal uses of PyObject_HEAD_INIT immortal while preserving the // C-API expectation that the refcnt will be set to 1. #ifdef Py_BUILD_CORE -#define PyObject_HEAD_INIT(type) \ - { \ - _PyObject_EXTRA_INIT \ - { _Py_IMMORTAL_REFCNT }, \ - (type), \ - _Py_DEFAULT_REGION \ +#define PyObject_HEAD_INIT(type) \ + { \ + _PyObject_EXTRA_INIT \ + { _Py_IMMORTAL_REFCNT }, \ + (type), \ + (Py_region_ptr_with_tags_t){_Py_DEFAULT_REGION} \ }, #else -#define PyObject_HEAD_INIT(type) \ - { \ - _PyObject_EXTRA_INIT \ - { 1 }, \ - (type), \ - _Py_DEFAULT_REGION \ +#define PyObject_HEAD_INIT(type) \ + { \ + _PyObject_EXTRA_INIT \ + { 1 }, \ + (type), \ + (Py_region_ptr_with_tags_t){_Py_DEFAULT_REGION} \ }, #endif /* Py_BUILD_CORE */ @@ -163,9 +180,6 @@ check by comparing the reference count field to the immortality reference count. #define PyObject_VAR_HEAD PyVarObject ob_base; #define Py_INVALID_SIZE (Py_ssize_t)-1 -typedef Py_uintptr_t Py_region_ptr; -typedef Py_uintptr_t Py_region_ptr_with_flags; - /* Nothing is actually declared to be a PyObject, but every pointer to * a Python object can be cast to a PyObject*. This is inheritance built * by hand. Similarly every pointer to a variable-size Python object can, @@ -200,7 +214,7 @@ struct _object { // Stolen bottom bits: // 1. Indicates the region type. A set flag indicates the immutable region. // 2. This flag is used for object traversal to indicate that it was visited. - Py_uintptr_t ob_region; + Py_region_ptr_with_tags_t ob_region; }; /* Cast argument to PyObject* type. */ @@ -236,12 +250,9 @@ static inline PyTypeObject* Py_TYPE(PyObject *ob) { # define Py_TYPE(ob) Py_TYPE(_PyObject_CAST(ob)) #endif -// This is the mask of all used bits to indicate the region. -// this should be used when the region pointer was requested. -// Macros for the individual flags are defined in regions.c. -#define Py_REGION_MASK (~((Py_uintptr_t)0x2)) -static inline Py_uintptr_t Py_REGION(PyObject *ob) { - return (ob->ob_region & Py_REGION_MASK); + +static inline Py_region_ptr_t Py_REGION(PyObject *ob) { + return Py_region_ptr(ob->ob_region); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_REGION(ob) Py_REGION(_PyObject_CAST(ob)) @@ -322,19 +333,18 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) #endif -static inline void Py_SET_REGION(PyObject *ob, Py_uintptr_t region) { - // Retain the old flags - ob->ob_region = (region & Py_REGION_MASK) | (ob->ob_region & (~Py_REGION_MASK)); +static inline void Py_SET_REGION(PyObject *ob, Py_region_ptr_t region) { + ob->ob_region = Py_region_ptr_with_tags(region & Py_REGION_MASK); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_REGION(ob, region) Py_SET_REGION(_PyObject_CAST(ob), (region)) +# define Py_SET_REGION(ob, region) Py_SET_REGION(_PyObject_CAST(ob), _Py_CAST(Py_region_ptr_t, (region))) #endif -static inline void Py_SET_REGION_WITH_FLAGS(PyObject *ob, Py_uintptr_t region) { +static inline void Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t region) { ob->ob_region = region; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_REGION_WITH_FLAGS(ob, region) Py_SET_REGION_WITH_FLAGS(_PyObject_CAST(ob), (region)) +# define Py_SET_TAGGED_REGION(ob, region) Py_SET_REGION_WITH_FLAGS(_PyObject_CAST(ob), (region)) #endif /* diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index e034427ecea908..6bac5bff3256d3 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -138,6 +138,7 @@ "UserWarning", "ValueError", "NotWriteableError", + "RegionError", "Warning", "ZeroDivisionError", ) diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index c987419663409e..6b1284c838afbf 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -38,6 +38,7 @@ BaseException │ ├── ProcessLookupError │ └── TimeoutError ├── ReferenceError + ├── RegionError ├── RuntimeError │ ├── NotImplementedError │ └── RecursionError diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 95eb05a1e6e39a..cf0c367388e31a 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -272,6 +272,7 @@ def test_windows_feature_macros(self): "PyExc_ProcessLookupError", "PyExc_RecursionError", "PyExc_ReferenceError", + "PyExc_RegionError", "PyExc_ResourceWarning", "PyExc_RuntimeError", "PyExc_RuntimeWarning", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 59488aaf0d5d6b..c3cda5eb83c394 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2409,3 +2409,5 @@ [data.PyExc_NotWriteableError] added = '4.0' +[data.PyExc_RegionError] + added = '4.0' diff --git a/Objects/object.c b/Objects/object.c index 09b663f1f58e6f..2cca817676a234 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1918,7 +1918,7 @@ PyObject _Py_NoneStruct = { _PyObject_EXTRA_INIT { _Py_IMMORTAL_REFCNT }, &_PyNone_Type, - _Py_IMMUTABLE + (Py_region_ptr_with_tags_t){_Py_IMMUTABLE} }; /* NotImplemented is an object that can be used to signal that an diff --git a/Objects/regions.c b/Objects/regions.c index fba011088beb95..a8865d07504c39 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -11,14 +11,14 @@ #include "pycore_regions.h" #include "pycore_pyerrors.h" -#define Py_REGION_VISITED_FLAG ((Py_uintptr_t)0x2) -static inline Py_uintptr_t Py_REGION_WITH_FLAGS(PyObject *ob) { +#define Py_REGION_VISITED_FLAG ((Py_region_ptr_t)0x2) +static inline Py_region_ptr_with_tags_t Py_TAGGED_REGION(PyObject *ob) { return ob->ob_region; } -#define Py_REGION_WITH_FLAGS(ob) Py_REGION_WITH_FLAGS(_PyObject_CAST(ob)) -#define REGION_SET_FLAG(ob, flag) (Py_SET_REGION_WITH_FLAGS(ob, Py_REGION_WITH_FLAGS(ob) | flag)) -#define REGION_GET_FLAG(ob, flag) (Py_REGION_WITH_FLAGS(ob) & flag) -#define REGION_CLEAR_FLAG(ob, flag) (Py_SET_REGION_WITH_FLAGS(ob, Py_REGION_WITH_FLAGS(ob) & (~flag))) +#define Py_TAGGED_REGION(ob) Py_TAGGED_REGION(_PyObject_CAST(ob)) +#define REGION_SET_TAG(ob, tag) (Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(Py_TAGGED_REGION(ob).value | tag))) +#define REGION_GET_TAG(ob, tag) (Py_region_ptr_with_tags(Py_TAGGED_REGION(ob).value & tag)) +#define REGION_CLEAR_TAG(ob, tag) (Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(Py_TAGGED_REGION(ob).value & (~tag)))) typedef struct regionmetadata regionmetadata; typedef struct PyRegionObject PyRegionObject; @@ -42,8 +42,9 @@ bool invariant_error_occurred = false; /* This uses the given arguments to create and throw a `RegionError` */ -void throw_region_error(PyObject* src, PyObject* tgt, - const char *format_str, PyObject *obj) +static void throw_region_error( + PyObject* src, PyObject* tgt, + const char *format_str, PyObject *obj) { // Don't stomp existing exception PyThreadState *tstate = _PyThreadState_GET(); @@ -158,6 +159,7 @@ static bool stack_empty(stack* s){ return s->head == NULL; } +__attribute__((unused)) static void stack_print(stack* s){ node* n = s->head; while(n != NULL){ @@ -243,8 +245,8 @@ typedef struct _gc_runtime_state GCState; #define GC_PREV _PyGCHead_PREV #define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head))) -#define IS_IMMUTABLE_REGION(r) ((Py_uintptr_t)r == _Py_IMMUTABLE) -#define IS_DEFAULT_REGION(r) ((Py_uintptr_t)r == _Py_DEFAULT_REGION) +#define IS_IMMUTABLE_REGION(r) ((Py_region_ptr_t)r == _Py_IMMUTABLE) +#define IS_DEFAULT_REGION(r) ((Py_region_ptr_t)r == _Py_DEFAULT_REGION) /* A traversal callback for _Py_CheckRegionInvariant. - tgt is the target of the reference we are checking, and @@ -776,7 +778,7 @@ typedef int (*handle_add_to_region_error)(regionerror *, void *); * This function borrows both arguments. The memory has to be managed * the caller. */ -static int emit_region_error(regionerror *error, void*) { +static int emit_region_error(regionerror *error, void* ignored) { const char* msg = NULL; switch (error->id) @@ -887,14 +889,8 @@ static int _add_to_region_visit(PyObject* target, void* info_void) } /* This adds the given object and transitive objects to the given region. - * Errors will be passed to the given `handle_error` function along with - * the `handle_error_data`. - * - * The given region may contain flags. Objects from the local regions will - * have their region replaced with the `flagged_region` value. This will - * lose flags set on the local region. */ -static PyObject *add_to_region(PyObject *obj, Py_uintptr_t flagged_region) +static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) { if (!obj) { Py_RETURN_NONE; @@ -907,10 +903,6 @@ static PyObject *add_to_region(PyObject *obj, Py_uintptr_t flagged_region) return NULL; } - // Make sure we check against the actual region and not the region - // plus magic flags - Py_uintptr_t region = flagged_region & Py_REGION_MASK; - // The current implementation assumes region is a valid pointer. This // restriction can be lifted if needed assert(!IS_DEFAULT_REGION(region) || !IS_IMMUTABLE_REGION(region)); @@ -956,7 +948,7 @@ static PyObject *add_to_region(PyObject *obj, Py_uintptr_t flagged_region) // minus one for the reference we just followed region_data->lrc += item->ob_refcnt - 1; Py_SET_REGION(item, region); - + // Add `info.src` for better error messages info.src = item; @@ -997,7 +989,7 @@ static PyObject *add_to_region(PyObject *obj, Py_uintptr_t flagged_region) static bool is_bridge_object(PyObject *op) { - Py_uintptr_t region = Py_REGION(op); + Py_region_ptr_t region = Py_REGION(op); if (IS_DEFAULT_REGION(region) || IS_IMMUTABLE_REGION(region)) { return false; } @@ -1007,7 +999,7 @@ static bool is_bridge_object(PyObject *op) { // will use the properties of a bridge object. This therefore checks if // the object is equal to the regions bridge object rather than checking // that the type is `PyRegionObject` - return ((Py_uintptr_t)((regionmetadata*)region)->bridge == (Py_uintptr_t)op); + return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); } __attribute__((unused)) @@ -1125,7 +1117,7 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { } // Make the region an owner of the bridge object - Py_SET_REGION(self, (Py_uintptr_t) self->metadata); + Py_SET_REGION(self, self->metadata); _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); // FIXME: Usually this is created on the fly. We need to do it manually to @@ -1185,7 +1177,7 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { } regionmetadata* md = PyRegion_get_metadata(self); - if (Py_REGION(args) == (Py_uintptr_t) md) { + if (Py_REGION(args) == (Py_region_ptr_t) md) { Py_SET_REGION(args, _Py_DEFAULT_REGION); Py_RETURN_NONE; } else { diff --git a/Python/errors.c b/Python/errors.c index 698faec546526b..088d29204523cd 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1957,7 +1957,7 @@ _PyErr_WriteToImmutable(const char* filename, int lineno, PyObject* obj) PyThreadState *tstate = _PyThreadState_GET(); if (!_PyErr_Occurred(tstate)) { string = PyUnicode_FromFormat("object of type %s is immutable (in region %" PRIuPTR ") at %s:%d", - obj->ob_type->tp_name, obj->ob_region, filename, lineno); + obj->ob_type->tp_name, Py_REGION(obj), filename, lineno); if (string != NULL) { _PyErr_SetObject(tstate, PyExc_NotWriteableError, string); Py_DECREF(string); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 546b105dfa2b87..10e0453bde2cce 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -308,6 +308,8 @@ Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - Objects/exceptions.c - _PyExc_NotWriteableError - Objects/exceptions.c - PyExc_NotWriteableError - +Objects/exceptions.c - _PyExc_RegionError - +Objects/exceptions.c - PyExc_RegionError - Objects/genobject.c - ASYNC_GEN_IGNORED_EXIT_MSG - Objects/genobject.c - NON_INIT_CORO_MSG - Objects/longobject.c - _PyLong_DigitValue - @@ -724,9 +726,9 @@ Python/bltinmodule.c - PyRegion_Type - ## Regions Debug Info for Invariant ## Not to remain global, and should become localised to an interpreter -Objects/regions.c - do_region_check - -Objects/regions.c - error_src - -Objects/regions.c - error_tgt - -Objects/regions.c - error_occurred - +Objects/regions.c - invariant_do_region_check - +Objects/regions.c - invariant_error_src - +Objects/regions.c - invariant_error_tgt - +Objects/regions.c - invariant_error_occurred - Objects/regions.c - captured - From c0a644c98a2e2cd3379c6739c617872122339774 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Sun, 8 Dec 2024 13:29:58 +0100 Subject: [PATCH 24/68] Pyrona: Minor cleanup around regions --- Include/object.h | 2 +- Objects/regions.c | 126 +++++++++++++++++++++++++--------------------- 2 files changed, 70 insertions(+), 58 deletions(-) diff --git a/Include/object.h b/Include/object.h index c8588c214dc71b..3d23c400c92726 100644 --- a/Include/object.h +++ b/Include/object.h @@ -344,7 +344,7 @@ static inline void Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t ob->ob_region = region; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_TAGGED_REGION(ob, region) Py_SET_REGION_WITH_FLAGS(_PyObject_CAST(ob), (region)) +# define Py_SET_TAGGED_REGION(ob, region) Py_SET_TAGGED_REGION(_PyObject_CAST(ob), (region)) #endif /* diff --git a/Objects/regions.c b/Objects/regions.c index a8865d07504c39..80a47e08aca77c 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -17,8 +17,9 @@ static inline Py_region_ptr_with_tags_t Py_TAGGED_REGION(PyObject *ob) { } #define Py_TAGGED_REGION(ob) Py_TAGGED_REGION(_PyObject_CAST(ob)) #define REGION_SET_TAG(ob, tag) (Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(Py_TAGGED_REGION(ob).value | tag))) -#define REGION_GET_TAG(ob, tag) (Py_region_ptr_with_tags(Py_TAGGED_REGION(ob).value & tag)) +#define REGION_GET_TAG(ob, tag) (Py_TAGGED_REGION(ob).value & tag) #define REGION_CLEAR_TAG(ob, tag) (Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(Py_TAGGED_REGION(ob).value & (~tag)))) +#define Py_REGION_DATA(ob) (_Py_CAST(regionmetadata*, Py_REGION(ob))) typedef struct regionmetadata regionmetadata; typedef struct PyRegionObject PyRegionObject; @@ -122,6 +123,8 @@ static stack* stack_new(void){ static bool stack_push(stack* s, PyObject* object){ node* n = (node*)malloc(sizeof(node)); if(n == NULL){ + // FIXME: This DECREF should only be used by MakeImmutable, since + // `add_to_region` and other functions only use weak refs. Py_DECREF(object); // Should we also free the stack? return true; @@ -256,8 +259,8 @@ static int visit_invariant_check(PyObject *tgt, void *src_void) { PyObject *src = _PyObject_CAST(src_void); - regionmetadata* src_region = (regionmetadata*) Py_REGION(src); - regionmetadata* tgt_region = (regionmetadata*) Py_REGION(tgt); + regionmetadata* src_region = Py_REGION_DATA(src); + regionmetadata* tgt_region = Py_REGION_DATA(tgt); // Internal references are always allowed if (src_region == tgt_region) return 0; @@ -285,7 +288,7 @@ visit_invariant_check(PyObject *tgt, void *src_void) return 0; } // Forbid cycles in the region topology - if (regionmetadata_has_ancestor((regionmetadata*)src_region, (regionmetadata*)tgt_region)) { + if (regionmetadata_has_ancestor(src_region, tgt_region)) { emit_invariant_error(src, tgt, "Regions create a cycle with subregions"); return 0; } @@ -825,21 +828,38 @@ static int _add_to_region_visit(PyObject* target, void* info_void) return 0; } - // If the object is already in our region, we don't need to traverse it - if (Py_REGION(target) == Py_REGION(info->src)) { - return 0; - } - // C wrappers can propergate through the entire system and draw // in a lot of unwanted objects. Since c wrappers don't have mutable // data, we just make it immutable and have the immutability impl // handle it. We then have an edge from our region to an immutable // object which is again valid. - if(is_c_wrapper(target)) { + if (is_c_wrapper(target)) { _Py_MakeImmutable(target); return 0; } + if (_Py_IsLocal(target)) { + // Add reference to the object, + // minus one for the reference we just followed + Py_REGION_DATA(info->src)->lrc += target->ob_refcnt - 1; + Py_SET_REGION(target, Py_REGION(info->src)); + + if (stack_push(info->pending, target)) { + PyErr_NoMemory(); + return -1; + } + return 0; + } + + // The item was previously in the local region but has already been + // added to the region by a previous iteration. We therefore only need + // to adjust the LRC + if (Py_REGION(target) == Py_REGION(info->src)) { + // -1 for the refernce we just followed + Py_REGION_DATA(target)->lrc -= 1; + return 0; + } + // We push it onto the stack to be added to the region and traversed. // The actual addition of the object is done in `add_to_region`. We keep // it in the local region, to indicate to `add_to_region` that the object @@ -865,7 +885,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // The target is a bridge object from another region. We now need to // if it already has a parent. - regionmetadata *target_region = _Py_CAST(regionmetadata *, Py_REGION(target)); + regionmetadata *target_region = Py_REGION_DATA(target); if (target_region->parent != NULL) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_SHARED_CUSTODY}; @@ -873,8 +893,8 @@ static int _add_to_region_visit(PyObject* target, void* info_void) } // Make sure that the new subregion relation won't create a cycle - regionmetadata* region = _Py_CAST(regionmetadata*, Py_REGION(info->src)); - if (regionmetadata_has_ancestor(target_region, region)) { + regionmetadata* region = Py_REGION_DATA(info->src); + if (regionmetadata_has_ancestor(region, target_region)) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_CYCLE_CREATION}; return ((info->handle_error)(&err, info->handle_error_data)); @@ -888,6 +908,33 @@ static int _add_to_region_visit(PyObject* target, void* info_void) return 0; } +// This function visits all outgoing reference from `item` including the +// type. It will return `false` if the operation failed. +static int visit_object(PyObject *item, visitproc visit, void* info) { + if (PyFunction_Check(item)) { + // FIXME: This is a temporary error. It should be replaced by + // proper handling of moving the function into the region + regionerror err = {.src = NULL, + .tgt = item, .id = ERR_WIP_FUNCTIONS }; + emit_region_error(&err, NULL); + return false; + } else { + PyTypeObject *type = Py_TYPE(item); + traverseproc traverse = type->tp_traverse; + if (traverse != NULL) { + if (traverse(item, visit, info)) { + return false; + } + } + } + + // Visit the type manually, since it's not included in the normal + // `tp_treverse`. + PyObject* type_ob = _PyObject_CAST(Py_TYPE(item)); + // Visit will return 0 if everything was okayw + return ((visit)(type_ob, info) == 0); +} + /* This adds the given object and transitive objects to the given region. */ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) @@ -934,51 +981,12 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) while (!stack_empty(info.pending)) { PyObject *item = stack_pop(info.pending); - // The item was previously in the local region but has already been - // added to the region by a previous iteration. We therefore only need - // to adjust the LRC - if (Py_REGION(item) == region) { - // -1 for the refernce we just followed - region_data->lrc -= 1; - continue; - } + // Add `info.src` for better error messages + info.src = item; - if (IS_DEFAULT_REGION(Py_REGION(item))) { - // Add reference to the object, - // minus one for the reference we just followed - region_data->lrc += item->ob_refcnt - 1; - Py_SET_REGION(item, region); - - // Add `info.src` for better error messages - info.src = item; - - if (PyFunction_Check(item)) { - // FIXME: This is a temporary error. It should be replaced by - // proper handling of moving the function into the region - regionerror err = {.src = _PyObject_CAST(region_data->bridge), - .tgt = item, .id = ERR_WIP_FUNCTIONS }; - emit_region_error(&err, NULL); - } else { - PyTypeObject *type = Py_TYPE(item); - traverseproc traverse = type->tp_traverse; - if (traverse != NULL) { - if (traverse(item, (visitproc)_add_to_region_visit, &info)) { - stack_free(info.pending); - return NULL; - } - } - } - - PyObject* type_op = _PyObject_CAST(Py_TYPE(item)); - if (Py_REGION(type_op) != region && !_Py_IsImmutable(type_op)) { - if (stack_push(info.pending, type_op)) - { - stack_free(info.pending); - return PyErr_NoMemory(); - } - } - - continue; + if (!visit_object(item, (visitproc)_add_to_region_visit, &info)) { + stack_free(info.pending); + return NULL; } } @@ -1099,6 +1107,10 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"name", NULL}; self->metadata = (regionmetadata*)calloc(1, sizeof(regionmetadata)); + if (!self->metadata) { + PyErr_NoMemory(); + return -1; + } self->metadata->bridge = self; self->name = NULL; From 2cfe81ff318bf9833edcf756ae919b206760b421 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Tue, 10 Dec 2024 11:26:12 +0100 Subject: [PATCH 25/68] DEFAULT => LOCAL --- Include/internal/pycore_object.h | 6 +++--- Include/object.h | 13 +++++++++---- Objects/regions.c | 12 ++++++------ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index a5270a09d191ef..7553446ac9d74c 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -13,7 +13,7 @@ extern "C" { #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_runtime.h" // _PyRuntime -#include "pycore_regions.h" // _Py_DEFAULT_REGION +#include "pycore_regions.h" // _Py_LOCAL_REGION /* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid designated initializer conflicts in C++20. If we use the deinition in @@ -28,7 +28,7 @@ extern "C" { _PyObject_EXTRA_INIT \ .ob_refcnt = _Py_IMMORTAL_REFCNT, \ .ob_type = (type), \ - .ob_region = (Py_region_ptr_with_tags_t){_Py_DEFAULT_REGION} \ + .ob_region = (Py_region_ptr_with_tags_t){_Py_LOCAL_REGION} \ }, #define _PyVarObject_HEAD_INIT(type, size) \ { \ @@ -177,7 +177,7 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - Py_SET_REGION(op, _Py_DEFAULT_REGION); + Py_SET_REGION(op, _Py_LOCAL_REGION); if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) { Py_INCREF(typeobj); } diff --git a/Include/object.h b/Include/object.h index 3d23c400c92726..b656dfbcc699b1 100644 --- a/Include/object.h +++ b/Include/object.h @@ -142,7 +142,7 @@ static inline Py_region_ptr_with_tags_t Py_region_ptr_with_tags(Py_region_ptr_t return (Py_region_ptr_with_tags_t) { region }; } -#define _Py_DEFAULT_REGION ((Py_region_ptr_t)0) +#define _Py_LOCAL_REGION ((Py_region_ptr_t)0) #define _Py_IMMUTABLE ((Py_region_ptr_t)1) // Make all internal uses of PyObject_HEAD_INIT immortal while preserving the @@ -153,7 +153,7 @@ static inline Py_region_ptr_with_tags_t Py_region_ptr_with_tags(Py_region_ptr_t _PyObject_EXTRA_INIT \ { _Py_IMMORTAL_REFCNT }, \ (type), \ - (Py_region_ptr_with_tags_t){_Py_DEFAULT_REGION} \ + (Py_region_ptr_with_tags_t){_Py_LOCAL_REGION} \ }, #else #define PyObject_HEAD_INIT(type) \ @@ -161,7 +161,7 @@ static inline Py_region_ptr_with_tags_t Py_region_ptr_with_tags(Py_region_ptr_t _PyObject_EXTRA_INIT \ { 1 }, \ (type), \ - (Py_region_ptr_with_tags_t){_Py_DEFAULT_REGION} \ + (Py_region_ptr_with_tags_t){_Py_LOCAL_REGION} \ }, #endif /* Py_BUILD_CORE */ @@ -297,10 +297,15 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) static inline Py_ALWAYS_INLINE int _Py_IsLocal(PyObject *op) { - return Py_REGION(op) == _Py_DEFAULT_REGION; + return Py_REGION(op) == _Py_LOCAL_REGION; } #define _Py_IsLocal(op) _Py_IsLocal(_PyObject_CAST(op)) +static inline Py_ALWAYS_INLINE int _Py_IsCown(PyObject *op) +{ + return 0; // TODO: implement this when cowns are added +} +#define _Py_IsCown(op) _Py_IsCown(_PyObject_CAST(op)) static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { // This immortal check is for code that is unaware of immortal objects. diff --git a/Objects/regions.c b/Objects/regions.c index 3e11e10e7fafe9..7c16bab0ca980c 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -257,7 +257,7 @@ typedef struct _gc_runtime_state GCState; #define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head))) #define IS_IMMUTABLE_REGION(r) ((Py_region_ptr_t)r == _Py_IMMUTABLE) -#define IS_DEFAULT_REGION(r) ((Py_region_ptr_t)r == _Py_DEFAULT_REGION) +#define IS_LOCAL_REGION(r) ((Py_region_ptr_t)r == _Py_LOCAL_REGION) /* A traversal callback for _Py_CheckRegionInvariant. - tgt is the target of the reference we are checking, and @@ -276,7 +276,7 @@ visit_invariant_check(PyObject *tgt, void *src_void) if (IS_IMMUTABLE_REGION(tgt_region)) return 0; // Borrowed references are unrestricted - if (IS_DEFAULT_REGION(src_region)) + if (IS_LOCAL_REGION(src_region)) return 0; // Since tgt is not immutable, src also may not be as immutable may not point to mutable if (IS_IMMUTABLE_REGION(src_region)) { @@ -872,7 +872,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // The actual addition of the object is done in `add_to_region`. We keep // it in the local region, to indicate to `add_to_region` that the object // should actually be processed. - if (IS_DEFAULT_REGION(Py_REGION(target))) { + if (IS_LOCAL_REGION(Py_REGION(target))) { // The actual region update and write checks are done in the // main body of `add_to_region` if (stack_push(info->pending, target)) { @@ -960,7 +960,7 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) // The current implementation assumes region is a valid pointer. This // restriction can be lifted if needed - assert(!IS_DEFAULT_REGION(region) || !IS_IMMUTABLE_REGION(region)); + assert(!IS_LOCAL_REGION(region) || !IS_IMMUTABLE_REGION(region)); regionmetadata *region_data = _Py_CAST(regionmetadata *, region); // Early return if the object is already in the region or immutable @@ -1006,7 +1006,7 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) static bool is_bridge_object(PyObject *op) { Py_region_ptr_t region = Py_REGION(op); - if (IS_DEFAULT_REGION(region) || IS_IMMUTABLE_REGION(region)) { + if (IS_LOCAL_REGION(region) || IS_IMMUTABLE_REGION(region)) { return false; } @@ -1187,7 +1187,7 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { regionmetadata* md = PyRegion_get_metadata(self); if (Py_REGION(args) == (Py_region_ptr_t) md) { - Py_SET_REGION(args, _Py_DEFAULT_REGION); + Py_SET_REGION(args, _Py_LOCAL_REGION); Py_RETURN_NONE; } else { PyErr_SetString(PyExc_RuntimeError, "Object not a member of region!"); From 8b66b17c60ab93dcf10e0e49e3717839120025b4 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Tue, 10 Dec 2024 11:40:30 +0100 Subject: [PATCH 26/68] Better alignment with Fig. 6 from paper --- Objects/regions.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index 7c16bab0ca980c..753b5ddece28ac 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -943,8 +943,7 @@ static int visit_object(PyObject *item, visitproc visit, void* info) { return ((visit)(type_ob, info) == 0); } -/* This adds the given object and transitive objects to the given region. - */ +// Add the transitive closure of objets in the local region reachable from obj to region static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) { if (!obj) { @@ -1295,19 +1294,21 @@ static const char *get_region_name(PyObject* obj) { // TODO replace with write barrier code bool _Pyrona_AddReference(PyObject *tgt, PyObject *new_ref) { - if (_Py_IsImmutable(new_ref)) { - // Nothing to do -- adding a ref to an immutable is always permitted + if (Py_REGION(tgt) == Py_REGION(new_ref)) { + // Nothing to do -- intra-region references are always permitted return true; } - if (Py_REGION(tgt) == Py_REGION(new_ref)) { - // Nothing to do -- intra-region references are always permitted + if (_Py_IsImmutable(new_ref) || _Py_IsCown(new_ref)) { + // Nothing to do -- adding a ref to an immutable or a cown is always permitted return true; } if (_Py_IsLocal(new_ref)) { // Slurp emphemerally owned object into the region of the target object +#ifdef NDEBUG fprintf(stderr, "Added %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); +#endif add_to_region(new_ref, Py_REGION(tgt)); return true; } @@ -1322,6 +1323,7 @@ bool _Pyrona_AddReference(PyObject *tgt, PyObject *new_ref) { return true; // Illegal reference } +// Convenience function for moving multiple references into tgt at once bool _Pyrona_AddReferences(PyObject *tgt, int new_refc, ...) { va_list args; va_start(args, new_refc); From 40b27590d12a6e517438488566a886658efbe250 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Tue, 10 Dec 2024 14:52:13 +0100 Subject: [PATCH 27/68] Addressed comments from Matt P --- Include/internal/pycore_regions.h | 1 + Objects/dictobject.c | 1 + Objects/regions.c | 61 +++++++++++++++---------------- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 2ff2120db1acdf..3206881639ba34 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -57,6 +57,7 @@ bool _Pyrona_AddReferences(PyObject* target, int new_refc, ...); #endif int _Py_CheckRegionInvariant(PyThreadState *tstate); +void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg); #ifdef __cplusplus } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 4fea81fe9c5319..d8dca68121541a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5867,6 +5867,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, if (value == NULL) { res = PyDict_DelItem(dict, key); } else { + // TODO: remove this once we merge Matt P's changeset to dictionary object if (Pyrona_ADDREFERENCES(dict, key, value)) { res = PyDict_SetItem(dict, key, value); } else { diff --git a/Objects/regions.c b/Objects/regions.c index 753b5ddece28ac..8b6772b2d5bd70 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -43,8 +43,7 @@ PyObject* invariant_error_tgt = Py_None; // Once an error has occurred this is used to surpress further checking bool invariant_error_occurred = false; -/* This uses the given arguments to create and throw a `RegionError` - */ +// This uses the given arguments to create and throw a `RegionError` static void throw_region_error( PyObject* src, PyObject* tgt, const char *format_str, PyObject *obj) @@ -77,7 +76,6 @@ static void throw_region_error( struct PyRegionObject { PyObject_HEAD regionmetadata* metadata; - PyObject *name; // Optional string field for "name" PyObject *dict; }; @@ -87,6 +85,7 @@ struct regionmetadata { int is_open; regionmetadata* parent; PyRegionObject* bridge; + PyObject *name; // Optional string field for "name" // TODO: Currently only used for invariant checking. If it's not used for other things // it might make sense to make this conditional in debug builds (or something) // @@ -224,14 +223,9 @@ static void emit_invariant_error(PyObject* src, PyObject* tgt, const char* msg) if (_PyErr_Occurred(tstate)) { return; } - const char *src_region_name = get_region_name(src); - const char *tgt_region_name = get_region_name(tgt); - PyObject *tgt_type_repr = PyObject_Repr(PyObject_Type(tgt)); - const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; - PyObject *src_type_repr = PyObject_Repr(PyObject_Type(src)); - const char *src_desc = src_type_repr ? PyUnicode_AsUTF8(src_type_repr) : "<>"; - PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s): %s\n", src, src_desc, src_region_name, tgt, tgt_desc, tgt_region_name, msg); + _PyErr_Region(src, tgt, msg); + // We have discovered a failure. // Disable region check, until the program switches it back on. invariant_do_region_check = false; @@ -1088,8 +1082,8 @@ static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj) { static void PyRegion_dealloc(PyRegionObject *self) { // Name is immutable and not in our region. - Py_XDECREF(self->name); - self->name = NULL; + Py_XDECREF(self->metadata->name); + self->metadata->name = NULL; self->metadata->bridge = NULL; // The dictionary can be NULL if the Region constructor crashed @@ -1119,12 +1113,11 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { return -1; } self->metadata->bridge = self; - self->name = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|U", kwlist, &self->name)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|U", kwlist, &self->metadata->name)) return -1; - if (self->name) { - Py_XINCREF(self->name); + if (self->metadata->name) { + Py_XINCREF(self->metadata->name); // Freeze the name and it's type. Short strings in Python are interned // by default. This means that `id("AB") == id("AB")`. We therefore // need to either clone the name object or freeze it to share it @@ -1132,7 +1125,7 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { // operators return new strings and keep the old one intact // // FIXME: Implicit freezing should take care of this instead - _Py_MakeImmutable(self->name); + _Py_MakeImmutable(self->metadata->name); } // Make the region an owner of the bridge object @@ -1143,7 +1136,7 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { } static int PyRegion_traverse(PyRegionObject *self, visitproc visit, void *arg) { - Py_VISIT(self->name); + Py_VISIT(self->metadata->name); Py_VISIT(self->dict); return 0; } @@ -1211,14 +1204,14 @@ static PyObject *PyRegion_repr(PyRegionObject *self) { "Region(lrc=%d, osc=%d, name=%S, is_open=%d)", data->lrc, data->osc, - self->name ? self->name : Py_None, + self->metadata->name ? self->metadata->name : Py_None, data->is_open ); #else // Normal mode: simple representation return PyUnicode_FromFormat( "Region(name=%S, is_open=%d)", - self->name ? self->name : Py_None, + self->metadata->name ? self->metadata->name : Py_None, data->is_open ); #endif @@ -1279,6 +1272,16 @@ PyTypeObject PyRegion_Type = { PyType_GenericNew, /* tp_new */ }; +void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg) { + const char *new_ref_region_name = get_region_name(new_ref); + const char *tgt_region_name = get_region_name(tgt); + PyObject *tgt_type_repr = PyObject_Repr(PyObject_Type(tgt)); + const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; + PyObject *new_ref_type_repr = PyObject_Repr(PyObject_Type(new_ref)); + const char *new_ref_desc = new_ref_type_repr ? PyUnicode_AsUTF8(new_ref_type_repr) : "<>"; + PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", tgt, tgt_desc, tgt_region_name, new_ref, new_ref_desc, new_ref_region_name, msg); +} + static const char *get_region_name(PyObject* obj) { if (_Py_IsLocal(obj)) { return "Default"; @@ -1286,8 +1289,8 @@ static const char *get_region_name(PyObject* obj) { return "Immutable"; } else { const regionmetadata *md = Py_REGION_DATA(obj); - return md->bridge->name - ? PyUnicode_AsUTF8(md->bridge->name) + return md->name + ? PyUnicode_AsUTF8(md->name) : ""; } } @@ -1306,21 +1309,15 @@ bool _Pyrona_AddReference(PyObject *tgt, PyObject *new_ref) { if (_Py_IsLocal(new_ref)) { // Slurp emphemerally owned object into the region of the target object -#ifdef NDEBUG - fprintf(stderr, "Added %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); +#ifndef NDEBUG + _Py_VPYDBG("Added %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); #endif add_to_region(new_ref, Py_REGION(tgt)); return true; } - const char *new_ref_region_name = get_region_name(new_ref); - const char *tgt_region_name = get_region_name(tgt); - PyObject *tgt_type_repr = PyObject_Repr(PyObject_Type(tgt)); - const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; - PyObject *new_ref_type_repr = PyObject_Repr(PyObject_Type(new_ref)); - const char *new_ref_desc = new_ref_type_repr ? PyUnicode_AsUTF8(new_ref_type_repr) : "<>"; - PyErr_Format(PyExc_RuntimeError, "WBError: Invalid edge %p (%s in %s) -> %p (%s in %s)\n", tgt, tgt_desc, tgt_region_name, new_ref, new_ref_desc, new_ref_region_name); - return true; // Illegal reference + _PyErr_Region(tgt, new_ref, "(in WB/add_ref)"); + return false; // Illegal reference } // Convenience function for moving multiple references into tgt at once From a7b9819ee552e948cc40670a7033441c33d2578e Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Tue, 10 Dec 2024 16:07:18 +0100 Subject: [PATCH 28/68] Made _PyErr_Region module internal --- Include/internal/pycore_regions.h | 1 - Objects/regions.c | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 3206881639ba34..2ff2120db1acdf 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -57,7 +57,6 @@ bool _Pyrona_AddReferences(PyObject* target, int new_refc, ...); #endif int _Py_CheckRegionInvariant(PyThreadState *tstate); -void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg); #ifdef __cplusplus } diff --git a/Objects/regions.c b/Objects/regions.c index 8b6772b2d5bd70..1e503e0d7c7313 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -27,6 +27,7 @@ typedef struct PyRegionObject PyRegionObject; static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); static const char *get_region_name(PyObject* obj); +static void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg); #define Py_REGION_DATA(ob) (_Py_CAST(regionmetadata*, Py_REGION(ob))) /** From 7c3ec2817527df572d9ffc85cd5ab49126efc524 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 12 Dec 2024 11:24:54 +0000 Subject: [PATCH 29/68] Fix CI Github Actions has migrated to 24.04, but this is breaking some parts of CI. Pin the version to 22.04, which is the working version. --- .github/workflows/build_min.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_min.yml b/.github/workflows/build_min.yml index c59e9fb3e7bbc9..0385280d8aa00a 100644 --- a/.github/workflows/build_min.yml +++ b/.github/workflows/build_min.yml @@ -134,7 +134,7 @@ jobs: check_generated_files: name: 'Check if generated files are up to date' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' From 6320b95545357717fba757e67c25f30491883182 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 12 Dec 2024 13:11:48 +0000 Subject: [PATCH 30/68] Make _Py_NewReference reset the region to local (#20) * Add pooling failure test. * Make _Py_NewReference reset the region to local Some pooling code does not call _PyObject_Init, so but instead just calls _Py_NewReference. This is problematic for us as it means the region is not reset on reallocating from a Pool. It appears from inspection that _Py_NewReferenceNoTotal is not used in special cases that should keep the same region. --- Include/internal/pycore_object.h | 1 - Lib/test/test_veronapy.py | 13 +++++++++++++ Objects/object.c | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7553446ac9d74c..b15a32cc4bb85b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -177,7 +177,6 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj) { assert(op != NULL); Py_SET_TYPE(op, typeobj); - Py_SET_REGION(op, _Py_LOCAL_REGION); if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) { Py_INCREF(typeobj); } diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index f255f6fb040dd9..57aee01763dd45 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -547,5 +547,18 @@ def test_global_dict_mutation(self): self.assertTrue(isimmutable(f1)) self.assertRaises(NotWriteableError, f1) +class TestPoolAllocation(unittest.TestCase): + # If pooling does not reset region between allocations, + # then the second call to f will result in `a` being owned by + # the first region that no has been deallocated. This + # will result in a UAF that ASAN can detect. + def test_pool_allocation(self): + def f(): + r = Region() + a = {} + r.add_object(a) + f() + f() + if __name__ == '__main__': unittest.main() diff --git a/Objects/object.c b/Objects/object.c index 71886f85a618b7..53cdb16e1604c3 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2233,6 +2233,7 @@ _Py_NewReference(PyObject *op) reftotal_increment(_PyInterpreterState_GET()); #endif new_reference(op); + Py_SET_REGION(op, _Py_LOCAL_REGION); } void From eaa4ff9edbb10d8f42bb479134c32cb9f3a7d44d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 13 Dec 2024 14:08:09 +0000 Subject: [PATCH 31/68] Generic Attribute Issue (#23) * Add test for Generic Attribute * Allow an additional exception in case the origin has been made immutable. --- Lib/test/test_veronapy.py | 10 ++++++++++ Objects/genericaliasobject.c | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index 57aee01763dd45..15348caab00edc 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -560,5 +560,15 @@ def f(): f() f() +class TestGenericAliasBug(unittest.TestCase): + # The code inside generic alias attempts to set + # __orig_class__ on the empty tuple, which is not + # allowed. The make immutable means this can fail + # NotWriteableError rather than the TypeError or + # AttributeError that would be raised otherwise. + def test_generic_alias_bug(self): + c = makeimmutable(()) + tuple[int]() + if __name__ == '__main__': unittest.main() diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 117b4e8dfb960a..a0be5fadced065 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -596,7 +596,8 @@ set_orig_class(PyObject *obj, PyObject *self) if (obj != NULL) { if (PyObject_SetAttr(obj, &_Py_ID(__orig_class__), self) < 0) { if (!PyErr_ExceptionMatches(PyExc_AttributeError) && - !PyErr_ExceptionMatches(PyExc_TypeError)) + !PyErr_ExceptionMatches(PyExc_TypeError) && + !PyErr_ExceptionMatches(PyExc_NotWriteableError)) { Py_DECREF(obj); return NULL; From 6945ae8626b88bda5c2e3b0a1a77f4159ba56c32 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 18 Dec 2024 11:08:59 +0000 Subject: [PATCH 32/68] Make all the core library globally allocated objects immutable (#25) The core library bakes a bunch of objects in, such as types and small integers. This makes all the things that are baked in Immutable using the immutable region. --- Include/internal/pycore_object.h | 2 +- Include/object.h | 2 +- Lib/test/test_capi/test_abstract.py | 10 +++++----- Lib/test/test_descr.py | 10 +++++----- Lib/test/test_type_aliases.py | 2 +- Lib/test/test_type_annotations.py | 4 ++-- Lib/typing.py | 4 ++-- Lib/unittest/suite.py | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b15a32cc4bb85b..f34c2161e0922a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -28,7 +28,7 @@ extern "C" { _PyObject_EXTRA_INIT \ .ob_refcnt = _Py_IMMORTAL_REFCNT, \ .ob_type = (type), \ - .ob_region = (Py_region_ptr_with_tags_t){_Py_LOCAL_REGION} \ + .ob_region = (Py_region_ptr_with_tags_t){_Py_IMMUTABLE} \ }, #define _PyVarObject_HEAD_INIT(type, size) \ { \ diff --git a/Include/object.h b/Include/object.h index b656dfbcc699b1..0dae8ee1958d90 100644 --- a/Include/object.h +++ b/Include/object.h @@ -153,7 +153,7 @@ static inline Py_region_ptr_with_tags_t Py_region_ptr_with_tags(Py_region_ptr_t _PyObject_EXTRA_INIT \ { _Py_IMMORTAL_REFCNT }, \ (type), \ - (Py_region_ptr_with_tags_t){_Py_LOCAL_REGION} \ + (Py_region_ptr_with_tags_t){_Py_IMMUTABLE} \ }, #else #define PyObject_HEAD_INIT(type) \ diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 116edd8d4d2fd6..9d68b53614df6b 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -116,7 +116,7 @@ def test_object_setattr(self): self.assertRaises(RuntimeError, xsetattr, obj, 'evil', NULL) self.assertRaises(RuntimeError, xsetattr, obj, 'evil', 'good') - self.assertRaises(AttributeError, xsetattr, 42, 'a', 5) + self.assertRaises(NotWriteableError, xsetattr, 42, 'a', 5) self.assertRaises(TypeError, xsetattr, obj, 1, 5) # CRASHES xsetattr(obj, NULL, 5) # CRASHES xsetattr(NULL, 'a', 5) @@ -136,7 +136,7 @@ def test_object_setattrstring(self): self.assertRaises(RuntimeError, setattrstring, obj, b'evil', NULL) self.assertRaises(RuntimeError, setattrstring, obj, b'evil', 'good') - self.assertRaises(AttributeError, setattrstring, 42, b'a', 5) + self.assertRaises(NotWriteableError, setattrstring, 42, b'a', 5) self.assertRaises(TypeError, setattrstring, obj, 1, 5) self.assertRaises(UnicodeDecodeError, setattrstring, obj, b'\xff', 5) # CRASHES setattrstring(obj, NULL, 5) @@ -153,7 +153,7 @@ def test_object_delattr(self): xdelattr(obj, '\U0001f40d') self.assertFalse(hasattr(obj, '\U0001f40d')) - self.assertRaises(AttributeError, xdelattr, 42, 'numerator') + self.assertRaises(NotWriteableError, xdelattr, 42, 'numerator') self.assertRaises(RuntimeError, xdelattr, obj, 'evil') self.assertRaises(TypeError, xdelattr, obj, 1) # CRASHES xdelattr(obj, NULL) @@ -170,7 +170,7 @@ def test_object_delattrstring(self): delattrstring(obj, '\U0001f40d'.encode()) self.assertFalse(hasattr(obj, '\U0001f40d')) - self.assertRaises(AttributeError, delattrstring, 42, b'numerator') + self.assertRaises(NotWriteableError, delattrstring, 42, b'numerator') self.assertRaises(RuntimeError, delattrstring, obj, b'evil') self.assertRaises(UnicodeDecodeError, delattrstring, obj, b'\xff') # CRASHES delattrstring(obj, NULL) @@ -297,7 +297,7 @@ def test_object_setitem(self): self.assertRaises(SystemError, setitem, {}, 'a', NULL) self.assertRaises(IndexError, setitem, [], 1, 5) self.assertRaises(TypeError, setitem, [], 'a', 5) - self.assertRaises(TypeError, setitem, (), 1, 5) + self.assertRaises(NotWriteableError, setitem, (), 1, 5) self.assertRaises(SystemError, setitem, NULL, 'a', 5) def test_mapping_setitemstring(self): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 3cbfa8428513fe..14f3699a5e6f78 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1076,7 +1076,7 @@ class SubType(types.ModuleType): class MyInt(int): __slots__ = () - with self.assertRaises(TypeError): + with self.assertRaises(NotWriteableError): (1).__class__ = MyInt class MyFloat(float): @@ -1091,17 +1091,17 @@ class MyComplex(complex): class MyStr(str): __slots__ = () - with self.assertRaises(TypeError): + with self.assertRaises(NotWriteableError): "a".__class__ = MyStr class MyBytes(bytes): __slots__ = () - with self.assertRaises(TypeError): + with self.assertRaises((TypeError, NotWriteableError)): b"a".__class__ = MyBytes class MyTuple(tuple): __slots__ = () - with self.assertRaises(TypeError): + with self.assertRaises(NotWriteableError): ().__class__ = MyTuple class MyFrozenSet(frozenset): @@ -4082,7 +4082,7 @@ class D(C): try: list.__bases__ = (dict,) - except TypeError: + except NotWriteableError: pass else: self.fail("shouldn't be able to assign to list.__bases__") diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 8f0a998e1f3dc1..f98a8e48942c1c 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -232,7 +232,7 @@ def test_errors(self): class TypeAliasTypeTest(unittest.TestCase): def test_immutable(self): - with self.assertRaises(TypeError): + with self.assertRaises(NotWriteableError): TypeAliasType.whatever = "not allowed" def test_no_subclassing(self): diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3dbb35afcb620f..9efa21f04ee37f 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -32,9 +32,9 @@ def test_annotations_getset_raises(self): # builtin types don't have __annotations__ (yet!) with self.assertRaises(AttributeError): print(float.__annotations__) - with self.assertRaises(TypeError): + with self.assertRaises(NotWriteableError): float.__annotations__ = {} - with self.assertRaises(TypeError): + with self.assertRaises(NotWriteableError): del float.__annotations__ # double delete diff --git a/Lib/typing.py b/Lib/typing.py index 9e2adbe2214a8a..2d6acf216e7ebe 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2388,7 +2388,7 @@ def no_type_check(arg): no_type_check(obj) try: arg.__no_type_check__ = True - except TypeError: # built-in classes + except NotWriteableError: # built-in classes pass return arg @@ -2507,7 +2507,7 @@ class Other(Leaf): # Error reported by type checker """ try: f.__final__ = True - except (AttributeError, TypeError): + except (AttributeError, TypeError, NotWriteableError): # Skip the attribute silently if it is not writable. # AttributeError happens if the object has __slots__ or a # read-only property, TypeError if it's a builtin class. diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py index 6f45b6fe5f6039..0c8623e1091bf3 100644 --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -152,7 +152,7 @@ def _handleClassSetUp(self, test, result): failed = False try: currentClass._classSetupFailed = False - except TypeError: + except NotWriteableError: # test may actually be a function # so its class will be a builtin-type pass From fafff599506ab493d67dcd550c5b63653047c2be Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Wed, 18 Dec 2024 17:38:48 +0100 Subject: [PATCH 33/68] Align write barrier with paper --- Objects/regions.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index 1e503e0d7c7313..ccf2b4c53a5885 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -1296,29 +1296,27 @@ static const char *get_region_name(PyObject* obj) { } } -// TODO replace with write barrier code -bool _Pyrona_AddReference(PyObject *tgt, PyObject *new_ref) { - if (Py_REGION(tgt) == Py_REGION(new_ref)) { +// x.f = y ==> _Pyrona_AddReference(src=x, tgt=y) +bool _Pyrona_AddReference(PyObject *src, PyObject *tgt) { + if (Py_REGION(src) == Py_REGION(tgt)) { // Nothing to do -- intra-region references are always permitted return true; } - if (_Py_IsImmutable(new_ref) || _Py_IsCown(new_ref)) { + if (_Py_IsImmutable(tgt) || _Py_IsCown(tgt)) { // Nothing to do -- adding a ref to an immutable or a cown is always permitted return true; } - if (_Py_IsLocal(new_ref)) { + if (_Py_IsLocal(src)) { // Slurp emphemerally owned object into the region of the target object -#ifndef NDEBUG - _Py_VPYDBG("Added %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); -#endif - add_to_region(new_ref, Py_REGION(tgt)); + // _Py_VPYDBG("Added borrowed ref %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); return true; } - _PyErr_Region(tgt, new_ref, "(in WB/add_ref)"); - return false; // Illegal reference + // Try slurp emphemerally owned object into the region of the target object + // _Py_VPYDBG("Added owning ref %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); + add_to_region(tgt, Py_REGION(src)); } // Convenience function for moving multiple references into tgt at once From 7e5339e625a006db4c170b17e927fca5c946a952 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Thu, 19 Dec 2024 12:39:28 +0100 Subject: [PATCH 34/68] adding missing return --- Objects/regions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/regions.c b/Objects/regions.c index ccf2b4c53a5885..1d4c916145bee4 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -1316,7 +1316,7 @@ bool _Pyrona_AddReference(PyObject *src, PyObject *tgt) { // Try slurp emphemerally owned object into the region of the target object // _Py_VPYDBG("Added owning ref %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); - add_to_region(tgt, Py_REGION(src)); + return add_to_region(tgt, Py_REGION(src)); } // Convenience function for moving multiple references into tgt at once From 478a1293785cf350477888090169ede0f6c7ed84 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Thu, 19 Dec 2024 12:41:06 +0100 Subject: [PATCH 35/68] Update Objects/regions.c --- Objects/regions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/regions.c b/Objects/regions.c index 1d4c916145bee4..9b714209c62d5f 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -1316,7 +1316,7 @@ bool _Pyrona_AddReference(PyObject *src, PyObject *tgt) { // Try slurp emphemerally owned object into the region of the target object // _Py_VPYDBG("Added owning ref %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); - return add_to_region(tgt, Py_REGION(src)); + return add_to_region(tgt, Py_REGION(src)) == Py_None; } // Convenience function for moving multiple references into tgt at once From bc8b7f6afa8f4a8988780036a15b184daba15a17 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Sun, 8 Dec 2024 15:36:49 +0100 Subject: [PATCH 36/68] Pyrona: Add reference count to `regionmetadata` --- Include/internal/pycore_regions.h | 12 ++++++ Include/object.h | 14 ------- Objects/object.c | 1 + Objects/regions.c | 66 +++++++++++++++++++++++++------ 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 2ff2120db1acdf..2a8ebaff903012 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -14,6 +14,18 @@ extern "C" { #define Py_CHECKWRITE(op) ((op) && !_Py_IsImmutable(op)) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} +void _Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t region); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_TAGGED_REGION(ob, region) _Py_SET_TAGGED_REGION(_PyObject_CAST(ob), (region)) +#endif + +static inline void Py_SET_REGION(PyObject *ob, Py_region_ptr_t region) { + _Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(region & Py_REGION_MASK)); +} +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_SET_REGION(ob, region) Py_SET_REGION(_PyObject_CAST(ob), _Py_CAST(Py_region_ptr_t, (region))) +#endif + /* This makes the given objects and all object reachable from the given * object immutable. This will also move the objects into the immutable * region. diff --git a/Include/object.h b/Include/object.h index 0dae8ee1958d90..584ba5d212bef1 100644 --- a/Include/object.h +++ b/Include/object.h @@ -338,20 +338,6 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) #endif -static inline void Py_SET_REGION(PyObject *ob, Py_region_ptr_t region) { - ob->ob_region = Py_region_ptr_with_tags(region & Py_REGION_MASK); -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_REGION(ob, region) Py_SET_REGION(_PyObject_CAST(ob), _Py_CAST(Py_region_ptr_t, (region))) -#endif - -static inline void Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t region) { - ob->ob_region = region; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_TAGGED_REGION(ob, region) Py_SET_TAGGED_REGION(_PyObject_CAST(ob), (region)) -#endif - /* Type objects contain a string containing the type name (to help somewhat in debugging), the allocation parameters (see PyObject_New() and diff --git a/Objects/object.c b/Objects/object.c index 53cdb16e1604c3..2e52ba2f0df6d5 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2636,6 +2636,7 @@ _Py_Dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); destructor dealloc = type->tp_dealloc; + Py_SET_REGION(op, _Py_DEFAULT_REGION); #ifdef Py_DEBUG PyThreadState *tstate = _PyThreadState_GET(); PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; diff --git a/Objects/regions.c b/Objects/regions.c index 9b714209c62d5f..26a81f553bf19b 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -81,8 +81,12 @@ struct PyRegionObject { }; struct regionmetadata { - int lrc; // Integer field for "local reference count" - int osc; // Integer field for "open subregion count" + // The number of references coming in from the local region. + PY_UINT32_T lrc; + // The number of open subregions. + PY_UINT32_T osc; + // The number of objects inside this region. + PY_UINT32_T rc; int is_open; regionmetadata* parent; PyRegionObject* bridge; @@ -97,6 +101,24 @@ struct regionmetadata { static bool is_bridge_object(PyObject *op); static int regionmetadata_has_ancestor(regionmetadata* data, regionmetadata* other); static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj); +static void regionmetadata_inc_rc(regionmetadata* self); +static void regionmetadata_dec_rc(regionmetadata* self); + +void _Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t region) { + if (!(_Py_IsLocal(ob) || _Py_IsImmutable(ob))) { + regionmetadata* old_region = Py_REGION_DATA(ob); + old_region->rc -= 1; + + regionmetadata_dec_rc(old_region); + } + + ob->ob_region = region; + + if (!_Py_IsLocal(ob) && !_Py_IsImmutable(ob)) { + regionmetadata* new_region = Py_REGION_DATA(ob); + regionmetadata_inc_rc(new_region); + } +} /** * Simple implementation of stack for tracing during make immutable. @@ -1012,6 +1034,21 @@ static bool is_bridge_object(PyObject *op) { return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); } +void regionmetadata_inc_rc(regionmetadata* self) +{ + self->rc += 1; +} + +void regionmetadata_dec_rc(regionmetadata* self) +{ + self->rc -= 1; + if (self->rc == 0) { + Py_XDECREF(self->metadata->name); + self->metadata->name = NULL; + free(self); + } +} + __attribute__((unused)) static void regionmetadata_inc_lrc(regionmetadata* data) { data->lrc += 1; @@ -1083,9 +1120,13 @@ static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj) { static void PyRegion_dealloc(PyRegionObject *self) { // Name is immutable and not in our region. - Py_XDECREF(self->metadata->name); - self->metadata->name = NULL; + self->metadata->bridge = NULL; + regionmetadata_dec_rc(self->metadata); + self->metadata = NULL; + + // The region should be cleared by pythons general deallocator. + assert(Py_REGION(self) == _Py_DEFAULT_REGION); // The dictionary can be NULL if the Region constructor crashed if (self->dict) { @@ -1098,9 +1139,6 @@ static void PyRegion_dealloc(PyRegionObject *self) { } PyObject_GC_UnTrack((PyObject *)self); - - // The lifetimes are joined for now - free(self->metadata); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1113,8 +1151,18 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { PyErr_NoMemory(); return -1; } + + // Make sure the internal reference is also counted. + regionmetadata_inc_rc(self->metadata); + self->metadata->bridge = self; + // Make the region an owner of the bridge object + Py_SET_REGION(self, self->metadata); + + // Freeze the region type to share it with other regions + _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|U", kwlist, &self->metadata->name)) return -1; if (self->metadata->name) { @@ -1129,10 +1177,6 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { _Py_MakeImmutable(self->metadata->name); } - // Make the region an owner of the bridge object - Py_SET_REGION(self, self->metadata); - _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); - return 0; } From 9f820102c39d87e0fbce582ba9ab65e0bc6d4075 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 9 Dec 2024 14:08:42 +0100 Subject: [PATCH 37/68] Pyrona: Allow O(1) merging into local --- Include/cpython/listobject.h | 4 +- Include/internal/pycore_regions.h | 18 +- Include/object.h | 28 -- Include/regions.h | 19 + Lib/test/test_veronapy.py | 2 +- Objects/dictobject.c | 8 +- Objects/object.c | 10 +- Objects/regions.c | 553 ++++++++++++++++++++---------- Objects/setobject.c | 3 +- Objects/sliceobject.c | 3 +- Python/bltinmodule.c | 5 +- Python/errors.c | 4 +- Python/instrumentation.c | 6 +- 13 files changed, 442 insertions(+), 221 deletions(-) create mode 100644 Include/regions.h diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index a6a453fc1cb2a5..0d305a5b01ca85 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -2,6 +2,8 @@ # error "this header file must not be included directly" #endif +#include "regions.h" // Py_IsImmutable + typedef struct { PyObject_VAR_HEAD /* Vector of pointers to list elements. list[0] is ob_item[0], etc. */ @@ -40,7 +42,7 @@ static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) { static inline void PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { - if(_Py_IsImmutable(op)){ // _Py_CHECKWRITE(op) is not available + if(Py_IsImmutable(op)){ // _Py_CHECKWRITE(op) is not available // TODO this should be replaced with a _PyObject_ASSERT_MSG // when veronpy implementation is complete _PyObject_ASSERT_FAILED_MSG(op, "cannot modify immutable object"); diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 2a8ebaff903012..a3c6d53a9fe6d0 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -10,22 +10,34 @@ extern "C" { #endif #include "object.h" +#include "regions.h" -#define Py_CHECKWRITE(op) ((op) && !_Py_IsImmutable(op)) +#define Py_CHECKWRITE(op) ((op) && !Py_IsImmutable(op)) #define Py_REQUIREWRITE(op, msg) {if (Py_CHECKWRITE(op)) { _PyObject_ASSERT_FAILED_MSG(op, msg); }} +Py_region_ptr_t _Py_REGION(PyObject *ob); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_REGION(ob) _Py_REGION(_PyObject_CAST(ob)) +#endif + void _Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t region); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_TAGGED_REGION(ob, region) _Py_SET_TAGGED_REGION(_PyObject_CAST(ob), (region)) #endif -static inline void Py_SET_REGION(PyObject *ob, Py_region_ptr_t region) { +static inline void _Py_SET_REGION(PyObject *ob, Py_region_ptr_t region) { _Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(region & Py_REGION_MASK)); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_REGION(ob, region) Py_SET_REGION(_PyObject_CAST(ob), _Py_CAST(Py_region_ptr_t, (region))) +# define Py_SET_REGION(ob, region) (_Py_SET_REGION(_PyObject_CAST(ob), _Py_CAST(Py_region_ptr_t, (region)))) #endif +static inline Py_ALWAYS_INLINE int _Py_IsCown(PyObject *op) +{ + return 0; // TODO: implement this when cowns are added +} +#define _Py_IsCown(op) _Py_IsCown(_PyObject_CAST(op)) + /* This makes the given objects and all object reachable from the given * object immutable. This will also move the objects into the immutable * region. diff --git a/Include/object.h b/Include/object.h index 584ba5d212bef1..e1f067733356b4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -211,9 +211,6 @@ struct _object { PyTypeObject *ob_type; // VeronaPy: Field used for tracking which region this objects is stored in. - // Stolen bottom bits: - // 1. Indicates the region type. A set flag indicates the immutable region. - // 2. This flag is used for object traversal to indicate that it was visited. Py_region_ptr_with_tags_t ob_region; }; @@ -251,13 +248,6 @@ static inline PyTypeObject* Py_TYPE(PyObject *ob) { #endif -static inline Py_region_ptr_t Py_REGION(PyObject *ob) { - return Py_region_ptr(ob->ob_region); -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_REGION(ob) Py_REGION(_PyObject_CAST(ob)) -#endif - PyAPI_DATA(PyTypeObject) PyLong_Type; PyAPI_DATA(PyTypeObject) PyBool_Type; @@ -289,24 +279,6 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { # define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), (type)) #endif -static inline Py_ALWAYS_INLINE int _Py_IsImmutable(PyObject *op) -{ - return Py_REGION(op) == _Py_IMMUTABLE; -} -#define _Py_IsImmutable(op) _Py_IsImmutable(_PyObject_CAST(op)) - -static inline Py_ALWAYS_INLINE int _Py_IsLocal(PyObject *op) -{ - return Py_REGION(op) == _Py_LOCAL_REGION; -} -#define _Py_IsLocal(op) _Py_IsLocal(_PyObject_CAST(op)) - -static inline Py_ALWAYS_INLINE int _Py_IsCown(PyObject *op) -{ - return 0; // TODO: implement this when cowns are added -} -#define _Py_IsCown(op) _Py_IsCown(_PyObject_CAST(op)) - static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { // This immortal check is for code that is unaware of immortal objects. // The runtime tracks these objects and we should avoid as much diff --git a/Include/regions.h b/Include/regions.h new file mode 100644 index 00000000000000..f9f1eaadc5a271 --- /dev/null +++ b/Include/regions.h @@ -0,0 +1,19 @@ +#ifndef Py_REGIONS_H +#define Py_REGIONS_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "object.h" + +PyAPI_FUNC(int) _Py_IsImmutable(PyObject *op); +#define Py_IsImmutable(op) _Py_IsImmutable(_PyObject_CAST(op)) + + +PyAPI_FUNC(int) _Py_IsLocal(PyObject *op); +#define Py_IsLocal(op) _Py_IsLocal(_PyObject_CAST(op)) + +#ifdef __cplusplus +} +#endif +#endif // !Py_REGIONS_H diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index 15348caab00edc..a7bfaac0dc2174 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -426,7 +426,7 @@ def test_add_ownership2(self): self.assertFalse(r2.owns_object(a)) def test_add_object_is_deep(self): - # Create linked objects (a) -> (b) + # Create linked objects (a) -> (b) -> (c) a = self.A() b = self.A() c = self.A() diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d8dca68121541a..c1084ca1a3ca89 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -119,9 +119,11 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_dict.h" // PyDictKeysObject #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_regions.h" // _PyObject_GC_TRACK() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() #include "stringlib/eq.h" // unicode_eq() +#include "regions.h" // Py_IsImmutable() #include @@ -5769,7 +5771,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = make_dict_from_instance_attributes( interp, CACHED_KEYS(tp), values); if (dict != NULL) { - if (_Py_IsImmutable(obj)) { + if (Py_IsImmutable(obj)) { _Py_SetImmutable(dict); } else { @@ -5783,7 +5785,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - if (_Py_IsImmutable(obj)) { + if (Py_IsImmutable(obj)) { _Py_SetImmutable(dict); } else { @@ -5811,7 +5813,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) else { dict = PyDict_New(); } - if (_Py_IsImmutable(obj)) { + if (Py_IsImmutable(obj)) { _Py_SetImmutable(dict); } else { diff --git a/Objects/object.c b/Objects/object.c index 2e52ba2f0df6d5..b201df79b6837b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2023,7 +2023,8 @@ PyTypeObject _PyNotImplemented_Type = { PyObject _Py_NotImplementedStruct = { _PyObject_EXTRA_INIT { _Py_IMMORTAL_REFCNT }, - &_PyNotImplemented_Type + &_PyNotImplemented_Type, + (Py_region_ptr_with_tags_t) {_Py_IMMUTABLE} }; @@ -2233,7 +2234,10 @@ _Py_NewReference(PyObject *op) reftotal_increment(_PyInterpreterState_GET()); #endif new_reference(op); - Py_SET_REGION(op, _Py_LOCAL_REGION); + // This uses an assignment opposed to `Py_SET_REGION` since that + // function expects the previous value to be a valid object but newly + // created objects never had this value initilized. + op->ob_region = Py_region_ptr_with_tags(_Py_LOCAL_REGION); } void @@ -2636,7 +2640,7 @@ _Py_Dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); destructor dealloc = type->tp_dealloc; - Py_SET_REGION(op, _Py_DEFAULT_REGION); + Py_SET_REGION(op, _Py_LOCAL_REGION); #ifdef Py_DEBUG PyThreadState *tstate = _PyThreadState_GET(); PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; diff --git a/Objects/regions.c b/Objects/regions.c index 26a81f553bf19b..04217fe76a0a41 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -11,24 +11,37 @@ #include "pycore_regions.h" #include "pycore_pyerrors.h" -#define Py_REGION_VISITED_FLAG ((Py_region_ptr_t)0x2) +// This tag indicates that the `regionmetadata` object has been merged +// with another region. The `parent` pointer points to the region it was +// merged with. +// +// This tag is only used for the parent pointer in `regionmetadata`. +#define Py_METADATA_MERGE_TAG ((Py_region_ptr_t)0x2) static inline Py_region_ptr_with_tags_t Py_TAGGED_REGION(PyObject *ob) { return ob->ob_region; } #define Py_TAGGED_REGION(ob) Py_TAGGED_REGION(_PyObject_CAST(ob)) -#define REGION_SET_TAG(ob, tag) (Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(Py_TAGGED_REGION(ob).value | tag))) -#define REGION_GET_TAG(ob, tag) (Py_TAGGED_REGION(ob).value & tag) -#define REGION_CLEAR_TAG(ob, tag) (Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(Py_TAGGED_REGION(ob).value & (~tag)))) -#define Py_REGION_DATA(ob) (_Py_CAST(regionmetadata*, Py_REGION(ob))) +#define REGION_PRT_HAS_TAG(ptr, tag) ((ptr).value & tag) +#define REGION_PTR_SET_TAG(ptr, tag) (ptr = Py_region_ptr_with_tags((ptr).value | tag)) +#define REGION_PTR_CLEAR_TAG(ptr, tag) (ptr = Py_region_ptr_with_tags((ptr).value & (~tag))) + +#define REGION_DATA_CAST(r) (_Py_CAST(regionmetadata*, (r))) +#define REGION_PTR_CAST(r) (_Py_CAST(Py_region_ptr_t, (r))) +#define Py_REGION_DATA(ob) (REGION_DATA_CAST(Py_REGION(ob))) +#define Py_REGION_FIELD(ob) (ob->ob_region) + +#define IS_IMMUTABLE_REGION(r) (REGION_PTR_CAST(r) == _Py_IMMUTABLE) +#define IS_LOCAL_REGION(r) (REGION_PTR_CAST(r) == _Py_LOCAL_REGION) +#define HAS_METADATA(r) (!IS_LOCAL_REGION(r) && !IS_IMMUTABLE_REGION(r)) typedef struct regionmetadata regionmetadata; typedef struct PyRegionObject PyRegionObject; +static regionmetadata* regionmetadata_get_parent(regionmetadata* self); static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); static const char *get_region_name(PyObject* obj); static void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg); -#define Py_REGION_DATA(ob) (_Py_CAST(regionmetadata*, Py_REGION(ob))) /** * Global status for performing the region check. @@ -73,6 +86,9 @@ static void throw_region_error( exc->target = tgt; PyErr_SetRaisedException(_PyObject_CAST(exc)); } +#define throw_region_error(src, tgt, format_str, format_arg) \ + throw_region_error(_PyObject_CAST(src), _PyObject_CAST(tgt), \ + format_str, format_arg) struct PyRegionObject { PyObject_HEAD @@ -82,13 +98,19 @@ struct PyRegionObject { struct regionmetadata { // The number of references coming in from the local region. - PY_UINT32_T lrc; + Py_ssize_t lrc; // The number of open subregions. - PY_UINT32_T osc; - // The number of objects inside this region. - PY_UINT32_T rc; - int is_open; - regionmetadata* parent; + Py_ssize_t osc; + // The number of references to this object + Py_ssize_t rc; + bool is_open; + // This field might either point to the parent region or another region + // that this one was merged into. The `Py_METADATA_MERGE_TAG` tag is used + // to indicate this points to a merged region. + Py_region_ptr_with_tags_t parent; + // A weak reference to the bridge object. The bridge object has increased the + // rc of this metadata object. If this was a strong reference it could create + // a cycle. PyRegionObject* bridge; PyObject *name; // Optional string field for "name" // TODO: Currently only used for invariant checking. If it's not used for other things @@ -98,26 +120,298 @@ struct regionmetadata { regionmetadata* next; }; -static bool is_bridge_object(PyObject *op); -static int regionmetadata_has_ancestor(regionmetadata* data, regionmetadata* other); -static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj); -static void regionmetadata_inc_rc(regionmetadata* self); -static void regionmetadata_dec_rc(regionmetadata* self); +static Py_region_ptr_t regionmetadata_get_merge_tree_root(Py_region_ptr_t self) +{ + // Test for local and immutable region + if (!HAS_METADATA(self)) { + return self; + } -void _Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t region) { - if (!(_Py_IsLocal(ob) || _Py_IsImmutable(ob))) { - regionmetadata* old_region = Py_REGION_DATA(ob); - old_region->rc -= 1; + // Return self if it wasn't merged with another region + regionmetadata* self_data = REGION_DATA_CAST(self); + if (!REGION_PRT_HAS_TAG(self_data->parent, Py_METADATA_MERGE_TAG)) { + return self; + } - regionmetadata_dec_rc(old_region); + // FIXME: It can happen that there are several layers in this union-find + // structure. It would be efficient to directly update the parent pointers + // for deeper nodes. + return regionmetadata_get_merge_tree_root(Py_region_ptr(self_data->parent)); +} +#define regionmetadata_get_merge_tree_root(self) \ + regionmetadata_get_merge_tree_root(REGION_PTR_CAST(self)) + +static void regionmetadata_open(regionmetadata* self) { + assert(HAS_METADATA(self)); + self->is_open = true; +} + +static bool regionmetadata_is_open(Py_region_ptr_t self) { + if (!HAS_METADATA(self)) { + return REGION_DATA_CAST(self)->is_open; } - ob->ob_region = region; + // The immutable and local region are open by default and can't be closed. + return true; +} +#define regionmetadata_is_open(self) \ + regionmetadata_is_open(REGION_PTR_CAST(self)) + +static void regionmetadata_inc_osc(Py_region_ptr_t self_ptr) +{ + if (!HAS_METADATA(self_ptr)) { + return; + } + + regionmetadata* self = REGION_DATA_CAST(self_ptr); + self->osc += 1; + regionmetadata_open(self); +} +#define regionmetadata_inc_osc(self) \ + (regionmetadata_inc_osc(REGION_PTR_CAST(self))) + +static void regionmetadata_dec_osc(Py_region_ptr_t self_ptr) +{ + if (!HAS_METADATA(self_ptr)) { + return; + } + + REGION_DATA_CAST(self_ptr)->osc -= 1; +} +#define regionmetadata_dec_osc(self) \ + (regionmetadata_dec_osc(REGION_PTR_CAST(self))) + +static void regionmetadata_inc_rc(Py_region_ptr_t self) +{ + if (HAS_METADATA(self)) { + REGION_DATA_CAST(self)->rc += 1; + } +} +#define regionmetadata_inc_rc(self) \ + (regionmetadata_inc_rc(REGION_PTR_CAST(self))) + +static void regionmetadata_dec_rc(Py_region_ptr_t self_ptr) +{ + if (!HAS_METADATA(self_ptr)) { + return; + } + + // Update RC + regionmetadata* self = REGION_DATA_CAST(self_ptr); + self->rc -= 1; + if (self->rc != 0) { + return; + } + + // Sort out the funeral by informing everyone about the future freeing + Py_CLEAR(self->name); + + if (regionmetadata_is_open(self)) { + regionmetadata_dec_osc(regionmetadata_get_parent(self)); + } + + // This access the parent directly to update the rc. + // It also doesn't matter if the parent pointer is a + // merge or subregion relation, since both cases have + // increased the rc. + regionmetadata_dec_rc(Py_region_ptr(self->parent)); + + free(self); +} +#define regionmetadata_dec_rc(self) \ + (regionmetadata_dec_rc(REGION_PTR_CAST(self))) + +static void regionmetadata_set_parent(regionmetadata* self, regionmetadata* parent) { + // Just a sanity check, since these cases should never happen + assert(HAS_METADATA(self) && "Can't set the parent on the immutable and local region"); + assert(REGION_PTR_CAST(self) == regionmetadata_get_merge_tree_root(self) && "Sanity Check"); + assert(REGION_PTR_CAST(parent) == regionmetadata_get_merge_tree_root(parent) && "Sanity Check"); + + Py_region_ptr_t old_parent = Py_region_ptr(self->parent); + Py_region_ptr_t new_parent = REGION_PTR_CAST(parent); + self->parent = Py_region_ptr_with_tags(new_parent); + + // Update RCs + regionmetadata_inc_rc(new_parent); + if (regionmetadata_is_open(self)) { + regionmetadata_inc_osc(new_parent); + regionmetadata_dec_osc(old_parent); + } + regionmetadata_dec_rc(old_parent); +} + +static regionmetadata* regionmetadata_get_parent(regionmetadata* self) { + assert(REGION_PTR_CAST(self) == regionmetadata_get_merge_tree_root(self) && "Sanity check"); + if (!HAS_METADATA(self)) { + // The local and immutable regions never have a parent + return NULL; + } + + Py_region_ptr_t parent_field = Py_region_ptr(self->parent); + Py_region_ptr_t parent_root = regionmetadata_get_merge_tree_root(parent_field); + + // If the parent was merged with another region we want to update the + // pointer to point at the root. + if (parent_field != parent_root) { + // set_parent ensures that the RC's are correctly updated + regionmetadata_set_parent(self, REGION_DATA_CAST(parent_root)); + } + + return REGION_DATA_CAST(parent_root); +} +#define regionmetadata_get_parent(self) \ + regionmetadata_get_parent(REGION_DATA_CAST(self)) + +static bool regionmetadata_has_parent(regionmetadata* self) { + return regionmetadata_get_parent(self) != NULL; +} + +static bool regionmetadata_has_ancestor(regionmetadata* self, regionmetadata* other) { + // The immutable or local region can never be a parent + if (!HAS_METADATA(other)) { + return false; + } + + while (self) { + if (self == other) { + return true; + } + self = regionmetadata_get_parent(self); + } + return false; +} + + +// This implementation merges `self` into `other`. Merging is not allowed +// to break external uniqueness. It's therefore not allowed if both regions +// to have a parent. Except cases, where one region has the other region as +// it's parent. +// +// This function expects `self` to be a valid object. +__attribute__((unused)) +static PyObject* regionmetadata_merge(regionmetadata* self, Py_region_ptr_t other) { + assert(HAS_METADATA(self) && "The immutable and local region can't be merged into another region"); + assert(REGION_PTR_CAST(self) == regionmetadata_get_merge_tree_root(self) && "Sanity Check"); + + // If `other` is the parent of `self` we can merge it. We unset the the + // parent which will also update the rc and other counts. + regionmetadata* self_parent = regionmetadata_get_parent(self); + if (REGION_PTR_CAST(self_parent) == other) { + assert(HAS_METADATA(self_parent) && "The immutable and local region can never have children"); + + regionmetadata_set_parent(self, NULL); + self_parent = NULL; + } + + // If only `self` has a parent we can make `other` the child and + // remove the parent from `self`. The merged region will then again + // have the correct parent. + regionmetadata* other_parent = regionmetadata_get_parent(self); + if (self_parent && HAS_METADATA(other) && other_parent == NULL) { + // Make sure we don't create any cycles + if (regionmetadata_has_ancestor(self_parent, REGION_DATA_CAST(other))) { + throw_region_error(self->bridge, REGION_DATA_CAST(other)->bridge, + "Merging these regions would create a cycle", NULL); + return NULL; + } + + regionmetadata_set_parent(REGION_DATA_CAST(other), self_parent); + regionmetadata_set_parent(self, NULL); + self_parent = NULL; + } + + // If `self` still has a parent we can't merge it into `other` + if (self_parent != NULL) { + PyObject* other_node = NULL; + if (HAS_METADATA(other)) { + other_node = _PyObject_CAST(REGION_DATA_CAST(other)->bridge); + } + throw_region_error(self->bridge, other_node, + "Unable to merge regions", NULL); + return NULL; + } - if (!_Py_IsLocal(ob) && !_Py_IsImmutable(ob)) { - regionmetadata* new_region = Py_REGION_DATA(ob); - regionmetadata_inc_rc(new_region); + regionmetadata_inc_rc(other); + + // Move LRC and OSC into the root. + if (HAS_METADATA(other)) { + // Move information into the merge root + regionmetadata* other_data = REGION_DATA_CAST(other); + other_data->lrc += self->lrc; + other_data->osc += self->osc; + other_data->is_open |= self->is_open; + // remove information from self + self->lrc = 0; + self->osc = 0; + self->is_open = false; } + + self->parent = Py_region_ptr_with_tags(other); + REGION_PTR_SET_TAG(self->parent, Py_METADATA_MERGE_TAG); + // No decref, since this is a weak reference. Otherwise we would get + // a cycle between the `regionmetadata` as a non GC'ed object and the bridge. + self->bridge = NULL; + Py_RETURN_NONE; +} +#define regionmetadata_merge(self, other) \ + (regionmetadata_merge(self, REGION_PTR_CAST(other))); + +static bool is_bridge_object(PyObject *op) { + Py_region_ptr_t region = Py_REGION(op); + // The local and immutable region (represented as NULL) never have a bridge object. + if (!HAS_METADATA(region)) { + return false; + } + + // It's not yet clear how immutability will interact with region objects. + // It's likely that the object will remain in the object topology but + // will use the properties of a bridge object. This therefore checks if + // the object is equal to the regions bridge object rather than checking + // that the type is `PyRegionObject` + return _PyObject_CAST(REGION_DATA_CAST(region)->bridge) == op; +} + +int _Py_IsLocal(PyObject *op) { + return IS_LOCAL_REGION(Py_REGION(op)); +} + +int _Py_IsImmutable(PyObject *op) +{ + return IS_IMMUTABLE_REGION(Py_REGION(op)); +} + +Py_region_ptr_t _Py_REGION(PyObject *ob) { + if (!ob) { + return REGION_PTR_CAST(NULL); + } + + Py_region_ptr_t field_value = Py_region_ptr(Py_REGION_FIELD(ob)); + if (!HAS_METADATA(field_value)) { + return field_value; + } + + Py_region_ptr_t region = regionmetadata_get_merge_tree_root(field_value); + // Update the region if we're not pointing to the root of the merge tree. + // This can allow freeing of non root regions and speedup future lookups. + if (region != field_value) { + // We keep the tags, since the owning region stays the same. + Py_region_ptr_t tags = Py_region_ptr(Py_REGION_FIELD(ob)) & (~Py_REGION_MASK); + _Py_SET_TAGGED_REGION(ob, Py_region_ptr_with_tags(region | tags)); + } + + return region; +} + +void _Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t region) { + // Here we access the field directly, since we want to update the RC of the + // regions we're actually holding and not the root of the merge tree. + Py_region_ptr_t old_region = Py_region_ptr(Py_REGION_FIELD(ob)); + + ob->ob_region = region; + + // Update the RC of the region + regionmetadata_inc_rc(Py_region_ptr(region)); + regionmetadata_dec_rc(old_region); } /** @@ -273,9 +567,6 @@ typedef struct _gc_runtime_state GCState; #define GC_PREV _PyGCHead_PREV #define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head))) -#define IS_IMMUTABLE_REGION(r) ((Py_region_ptr_t)r == _Py_IMMUTABLE) -#define IS_LOCAL_REGION(r) ((Py_region_ptr_t)r == _Py_LOCAL_REGION) - /* A traversal callback for _Py_CheckRegionInvariant. - tgt is the target of the reference we are checking, and - src(_void) is the source of the reference we are checking. @@ -284,19 +575,21 @@ static int visit_invariant_check(PyObject *tgt, void *src_void) { PyObject *src = _PyObject_CAST(src_void); - regionmetadata* src_region = Py_REGION_DATA(src); - regionmetadata* tgt_region = Py_REGION_DATA(tgt); + + Py_region_ptr_t src_region_ptr = Py_REGION(src); + Py_region_ptr_t tgt_region_ptr = Py_REGION(tgt); // Internal references are always allowed - if (src_region == tgt_region) + if (src_region_ptr == tgt_region_ptr) return 0; + // Anything is allowed to point to immutable - if (IS_IMMUTABLE_REGION(tgt_region)) + if (Py_IsImmutable(tgt)) return 0; // Borrowed references are unrestricted - if (IS_LOCAL_REGION(src_region)) + if (Py_IsLocal(src)) return 0; // Since tgt is not immutable, src also may not be as immutable may not point to mutable - if (IS_IMMUTABLE_REGION(src_region)) { + if (Py_IsImmutable(src)) { emit_invariant_error(src, tgt, "Reference from immutable object to mutable target"); return 0; } @@ -306,6 +599,8 @@ visit_invariant_check(PyObject *tgt, void *src_void) emit_invariant_error(src, tgt, "Reference from object in one region into another region"); return 0; } + + regionmetadata* tgt_region = REGION_DATA_CAST(tgt_region_ptr); // Check if region is already added to captured list if (tgt_region->next != NULL) { // Bridge object was already captured @@ -313,7 +608,7 @@ visit_invariant_check(PyObject *tgt, void *src_void) return 0; } // Forbid cycles in the region topology - if (regionmetadata_has_ancestor(src_region, tgt_region)) { + if (regionmetadata_has_ancestor(REGION_DATA_CAST(src_region_ptr), tgt_region)) { emit_invariant_error(src, tgt, "Regions create a cycle with subregions"); return 0; } @@ -368,7 +663,7 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) for (; gc != containers; gc = GC_NEXT(gc)) { PyObject *op = FROM_GC(gc); // Local can point to anything. No invariant needed - if (_Py_IsLocal(op)) + if (Py_IsLocal(op)) continue; // Functions are complex. // Removing from invariant initially. @@ -409,7 +704,7 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate) } #define _Py_VISIT_FUNC_ATTR(attr, frontier) do { \ - if(attr != NULL && !_Py_IsImmutable(attr)){ \ + if(attr != NULL && !Py_IsImmutable(attr)){ \ Py_INCREF(attr); \ if(stack_push(frontier, attr)){ \ return PyErr_NoMemory(); \ @@ -423,7 +718,7 @@ static PyObject* make_global_immutable(PyObject* globals, PyObject* name) _PyDict_SetKeyImmutable((PyDictObject*)globals, name); - if(!_Py_IsImmutable(value)){ + if(!Py_IsImmutable(value)){ Py_INCREF(value); return value; }else{ @@ -532,7 +827,7 @@ static PyObject* make_function_immutable(PyObject* op, stack* frontier) _PyDict_SetKeyImmutable((PyDictObject*)builtins, name); PyObject* value = PyDict_GetItem(builtins, name); // value.rc = x - if(!_Py_IsImmutable(value)){ + if(!Py_IsImmutable(value)){ _Py_SetImmutable(value); } }else if(PyDict_Contains(module_dict, name)){ @@ -540,7 +835,7 @@ static PyObject* make_function_immutable(PyObject* op, stack* frontier) _PyDict_SetKeyImmutable((PyDictObject*)module_dict, name); - if(!_Py_IsImmutable(value)){ + if(!Py_IsImmutable(value)){ Py_INCREF(value); // value.rc = x + 1 if(stack_push(frontier, value)){ stack_free(f_stack); @@ -557,7 +852,7 @@ static PyObject* make_function_immutable(PyObject* op, stack* frontier) size = PySequence_Fast_GET_SIZE(f_code->co_consts); for(Py_ssize_t i = 0; i < size; i++){ PyObject* value = PySequence_Fast_GET_ITEM(f_code->co_consts, i); // value.rc = x - if(!_Py_IsImmutable(value)){ + if(!Py_IsImmutable(value)){ Py_INCREF(value); // value.rc = x + 1 if(PyCode_Check(value)){ @@ -641,7 +936,7 @@ static PyObject* make_function_immutable(PyObject* op, stack* frontier) static int _makeimmutable_visit(PyObject* obj, void* frontier) { - if(!_Py_IsImmutable(obj)){ + if(!Py_IsImmutable(obj)){ if(stack_push((stack*)frontier, obj)){ PyErr_NoMemory(); return -1; @@ -663,7 +958,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) // Some built-in objects are direclty created immutable. However, their types // might be created in a mutable state. This therefore requres an additional // check to see if the type is also immutable. - if(_Py_IsImmutable(obj) && _Py_IsImmutable(Py_TYPE(obj))){ + if(Py_IsImmutable(obj) && Py_IsImmutable(Py_TYPE(obj))){ Py_RETURN_NONE; } @@ -685,7 +980,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) PyObject* type_op = NULL; - if(_Py_IsImmutable(item)){ + if(Py_IsImmutable(item)){ // Direct access like this is not recommended, but will be removed in the future as // this is just for debugging purposes. if (Py_REGION(&type->ob_base.ob_base) != _Py_IMMUTABLE) { @@ -727,7 +1022,7 @@ PyObject* _Py_MakeImmutable(PyObject* obj) handle_type: type_op = PyObject_Type(item); // type_op.rc = x + 1 - if (!_Py_IsImmutable(type_op)){ + if (!Py_IsImmutable(type_op)){ // Previously this included a check for is_leaf_type, but if (stack_push(frontier, type_op)) { @@ -849,7 +1144,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // Region objects are allowed to reference immutable objects. Immutable // objects are only allowed to reference other immutable objects and cowns. // we therefore don't need to traverse them. - if (_Py_IsImmutable(target)) { + if (Py_IsImmutable(target)) { return 0; } @@ -863,11 +1158,12 @@ static int _add_to_region_visit(PyObject* target, void* info_void) return 0; } - if (_Py_IsLocal(target)) { + regionmetadata* source_region = Py_REGION_DATA(info->src); + if (Py_IsLocal(target)) { // Add reference to the object, // minus one for the reference we just followed - Py_REGION_DATA(info->src)->lrc += target->ob_refcnt - 1; - Py_SET_REGION(target, Py_REGION(info->src)); + source_region->lrc += target->ob_refcnt - 1; + Py_SET_REGION(target, source_region); if (stack_push(info->pending, target)) { PyErr_NoMemory(); @@ -876,12 +1172,12 @@ static int _add_to_region_visit(PyObject* target, void* info_void) return 0; } - // The item was previously in the local region but has already been + // The target was previously in the local region but has already been // added to the region by a previous iteration. We therefore only need // to adjust the LRC - if (Py_REGION(target) == Py_REGION(info->src)) { + if (Py_REGION_DATA(target) == source_region) { // -1 for the refernce we just followed - Py_REGION_DATA(target)->lrc -= 1; + source_region->lrc -= 1; return 0; } @@ -889,7 +1185,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // The actual addition of the object is done in `add_to_region`. We keep // it in the local region, to indicate to `add_to_region` that the object // should actually be processed. - if (IS_LOCAL_REGION(Py_REGION(target))) { + if (Py_IsLocal(target)) { // The actual region update and write checks are done in the // main body of `add_to_region` if (stack_push(info->pending, target)) { @@ -911,7 +1207,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // The target is a bridge object from another region. We now need to // if it already has a parent. regionmetadata *target_region = Py_REGION_DATA(target); - if (target_region->parent != NULL) { + if (regionmetadata_has_parent(target_region)) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_SHARED_CUSTODY}; return ((info->handle_error)(&err, info->handle_error_data)); @@ -928,7 +1224,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // From the previous checks we know that `target` is the bridge object // of a free region. Thus we can make it a sub region and allow the // reference. - target_region->parent = region; + regionmetadata_set_parent(target_region, region); return 0; } @@ -976,11 +1272,11 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) // The current implementation assumes region is a valid pointer. This // restriction can be lifted if needed - assert(!IS_LOCAL_REGION(region) || !IS_IMMUTABLE_REGION(region)); + assert(HAS_METADATA(region)); regionmetadata *region_data = _Py_CAST(regionmetadata *, region); // Early return if the object is already in the region or immutable - if (Py_REGION(obj) == region || _Py_IsImmutable(obj)) { + if (Py_REGION(obj) == region || Py_IsImmutable(obj)) { Py_RETURN_NONE; } @@ -1019,121 +1315,27 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) Py_RETURN_NONE; } - -static bool is_bridge_object(PyObject *op) { - Py_region_ptr_t region = Py_REGION(op); - if (IS_LOCAL_REGION(region) || IS_IMMUTABLE_REGION(region)) { - return false; - } - - // It's not yet clear how immutability will interact with region objects. - // It's likely that the object will remain in the object topology but - // will use the properties of a bridge object. This therefore checks if - // the object is equal to the regions bridge object rather than checking - // that the type is `PyRegionObject` - return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); -} - -void regionmetadata_inc_rc(regionmetadata* self) -{ - self->rc += 1; -} - -void regionmetadata_dec_rc(regionmetadata* self) -{ - self->rc -= 1; - if (self->rc == 0) { - Py_XDECREF(self->metadata->name); - self->metadata->name = NULL; - free(self); - } -} - -__attribute__((unused)) -static void regionmetadata_inc_lrc(regionmetadata* data) { - data->lrc += 1; -} - -__attribute__((unused)) -static void regionmetadata_dec_lrc(regionmetadata* data) { - data->lrc -= 1; -} - -__attribute__((unused)) -static void regionmetadata_inc_osc(regionmetadata* data) { - data->osc += 1; -} - -__attribute__((unused)) -static void regionmetadata_dec_osc(regionmetadata* data) { - data->osc -= 1; -} - -static void regionmetadata_open(regionmetadata* data) { - data->is_open = 1; -} - -static void regionmetadata_close(regionmetadata* data) { - data->is_open = 0; -} - -static bool regionmetadata_is_open(regionmetadata* data) { - return data->is_open == 0; -} - -static void regionmetadata_set_parent(regionmetadata* data, regionmetadata* parent) { - data->parent = parent; -} - -static bool regionmetadata_has_parent(regionmetadata* data) { - return data->parent != NULL; -} - -static regionmetadata* regionmetadata_get_parent(regionmetadata* data) { - return data->parent; -} - -__attribute__((unused)) -static void regionmetadata_unparent(regionmetadata* data) { - regionmetadata_set_parent(data, NULL); -} - -__attribute__((unused)) -static int regionmetadata_is_root(regionmetadata* data) { - return regionmetadata_has_parent(data); -} - -static int regionmetadata_has_ancestor(regionmetadata* data, regionmetadata* other) { - do { - if (data == other) { - return true; - } - data = regionmetadata_get_parent(data); - } while (data); - return false; -} - -static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj) { - return obj->metadata; -} - - static void PyRegion_dealloc(PyRegionObject *self) { // Name is immutable and not in our region. - self->metadata->bridge = NULL; - regionmetadata_dec_rc(self->metadata); - self->metadata = NULL; + // The object region has already been reset. + // We now need to update the RC of our metadata field. + if (self->metadata) { + regionmetadata* data = self->metadata; + self->metadata = NULL; + data->bridge = NULL; + regionmetadata_dec_rc(data); + } // The region should be cleared by pythons general deallocator. - assert(Py_REGION(self) == _Py_DEFAULT_REGION); + assert(Py_REGION(self) == _Py_LOCAL_REGION); // The dictionary can be NULL if the Region constructor crashed if (self->dict) { // We need to clear the ownership, since this dictionary might be // returned to an object pool rather than freed. This would result // in an error if the dictionary has the previous region. - PyRegion_remove_object(self, _PyObject_CAST(self->dict)); + Py_SET_REGION(self->dict, _Py_LOCAL_REGION); Py_DECREF(self->dict); self->dict = NULL; } @@ -1188,6 +1390,7 @@ static int PyRegion_traverse(PyRegionObject *self, visitproc visit, void *arg) { // is_open method (returns True if the region is open, otherwise False) static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *args) { + // FIXME: What is the behavior of a `PyRegionObject` that has been merged into another region? if (regionmetadata_is_open(self->metadata)) { Py_RETURN_TRUE; // Return True if the region is open } else { @@ -1197,14 +1400,16 @@ static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *args) { // Open method (sets the region to "open") static PyObject *PyRegion_open(PyRegionObject *self, PyObject *args) { - regionmetadata_open(self->metadata); + // `Py_REGION()` will fetch the root region of the merge tree. + // this might be different from the region in `self->metadata`. + regionmetadata_open(Py_REGION_DATA(self)); Py_RETURN_NONE; // Return None (standard for methods with no return value) } -// Close method (sets the region to "closed") -static PyObject *PyRegion_close(PyRegionObject *self, PyObject *args) { - regionmetadata_close(self->metadata); // Mark as closed - Py_RETURN_NONE; // Return None (standard for methods with no return value) +// try_close method (Attempts to close the region) +static PyObject *PyRegion_try_close(PyRegionObject *self, PyObject *args) { + // Not implemented for now. Return NULL to get an error + return NULL; } // Adds args object to self region @@ -1222,7 +1427,7 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { Py_RETURN_NONE; } - regionmetadata* md = PyRegion_get_metadata(self); + regionmetadata* md = Py_REGION_DATA(self); if (Py_REGION(args) == (Py_region_ptr_t) md) { Py_SET_REGION(args, _Py_LOCAL_REGION); Py_RETURN_NONE; @@ -1242,21 +1447,21 @@ static PyObject *PyRegion_owns_object(PyRegionObject *self, PyObject *args) { } static PyObject *PyRegion_repr(PyRegionObject *self) { - regionmetadata* data = self->metadata; + regionmetadata* data = Py_REGION_DATA(self); #ifdef NDEBUG // Debug mode: include detailed representation return PyUnicode_FromFormat( "Region(lrc=%d, osc=%d, name=%S, is_open=%d)", data->lrc, data->osc, - self->metadata->name ? self->metadata->name : Py_None, + data->name ? data->name : Py_None, data->is_open ); #else // Normal mode: simple representation return PyUnicode_FromFormat( "Region(name=%S, is_open=%d)", - self->metadata->name ? self->metadata->name : Py_None, + data->name ? data->name : Py_None, data->is_open ); #endif @@ -1265,7 +1470,7 @@ static PyObject *PyRegion_repr(PyRegionObject *self) { // Define the RegionType with methods static PyMethodDef PyRegion_methods[] = { {"open", (PyCFunction)PyRegion_open, METH_NOARGS, "Open the region."}, - {"close", (PyCFunction)PyRegion_close, METH_NOARGS, "Close the region."}, + {"try_close", (PyCFunction)PyRegion_try_close, METH_NOARGS, "Attempt to close the region."}, {"is_open", (PyCFunction)PyRegion_is_open, METH_NOARGS, "Check if the region is open."}, // Temporary methods for testing. These will be removed or at least renamed once // the write barrier is done. @@ -1330,7 +1535,7 @@ void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg) { static const char *get_region_name(PyObject* obj) { if (_Py_IsLocal(obj)) { return "Default"; - } else if (_Py_IsImmutable(obj)) { + } else if (Py_IsImmutable(obj)) { return "Immutable"; } else { const regionmetadata *md = Py_REGION_DATA(obj); @@ -1347,7 +1552,7 @@ bool _Pyrona_AddReference(PyObject *src, PyObject *tgt) { return true; } - if (_Py_IsImmutable(tgt) || _Py_IsCown(tgt)) { + if (Py_IsImmutable(tgt) || _Py_IsCown(tgt)) { // Nothing to do -- adding a ref to an immutable or a cown is always permitted return true; } diff --git a/Objects/setobject.c b/Objects/setobject.c index a53ac7d389e668..74dc478b34b9cf 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2588,5 +2588,6 @@ static PyTypeObject _PySetDummy_Type = { static PyObject _dummy_struct = { _PyObject_EXTRA_INIT { _Py_IMMORTAL_REFCNT }, - &_PySetDummy_Type + &_PySetDummy_Type, + (Py_region_ptr_with_tags_t) {_Py_IMMUTABLE} }; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index e6776ac92b669c..a424b089f42d72 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -100,7 +100,8 @@ PyTypeObject PyEllipsis_Type = { PyObject _Py_EllipsisObject = { _PyObject_EXTRA_INIT { _Py_IMMORTAL_REFCNT }, - &PyEllipsis_Type + &PyEllipsis_Type, + (Py_region_ptr_with_tags_t) {_Py_IMMUTABLE} }; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 338bb088463a01..d66a9559816450 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -11,8 +11,9 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ceval.h" // _PyEval_Vector() -#include "pycore_regions.h" // _Py_IMMUTABLE +#include "pycore_regions.h" // _Py_IMMUTABLE, PY_REGION() #include "pycore_dict.h" // _PyDict_SetGlobalImmutable() +#include "regions.h" // Py_IsImmutable() #include "clinic/bltinmodule.c.h" @@ -2755,7 +2756,7 @@ builtin_isimmutable(PyObject *module, PyObject *obj) _Py_VPYDBG("isimmutable("); _Py_VPYDBGPRINT(obj); _Py_VPYDBG(") region: %lu\n", Py_REGION(obj)); - return PyBool_FromLong(_Py_IsImmutable(obj)); + return PyBool_FromLong(Py_IsImmutable(obj)); } diff --git a/Python/errors.c b/Python/errors.c index 088d29204523cd..27ff884c4b934e 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1956,8 +1956,8 @@ _PyErr_WriteToImmutable(const char* filename, int lineno, PyObject* obj) PyObject* string; PyThreadState *tstate = _PyThreadState_GET(); if (!_PyErr_Occurred(tstate)) { - string = PyUnicode_FromFormat("object of type %s is immutable (in region %" PRIuPTR ") at %s:%d", - obj->ob_type->tp_name, Py_REGION(obj), filename, lineno); + string = PyUnicode_FromFormat("object of type %s is immutable at %s:%d", + obj->ob_type->tp_name, filename, lineno); if (string != NULL) { _PyErr_SetObject(tstate, PyExc_NotWriteableError, string); Py_DECREF(string); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index a6ff7a8a98506c..86ac2e90b628fd 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -19,13 +19,15 @@ PyObject _PyInstrumentation_DISABLE = { .ob_refcnt = _Py_IMMORTAL_REFCNT, - .ob_type = &PyBaseObject_Type + .ob_type = &PyBaseObject_Type, + .ob_region = (Py_region_ptr_with_tags_t){_Py_IMMUTABLE} }; PyObject _PyInstrumentation_MISSING = { .ob_refcnt = _Py_IMMORTAL_REFCNT, - .ob_type = &PyBaseObject_Type + .ob_type = &PyBaseObject_Type, + .ob_region = (Py_region_ptr_with_tags_t){_Py_IMMUTABLE} }; static const int8_t EVENT_FOR_OPCODE[256] = { From 62c444380f56ee0c47b23443a851b92422166651 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Thu, 12 Dec 2024 11:41:14 +0100 Subject: [PATCH 38/68] Add cowns Addresses #30 --- Include/internal/pycore_regions.h | 13 +- Include/object.h | 8 +- Include/regions.h | 4 +- Lib/test/test_using.py | 97 ++++++++ Lib/using.py | 47 ++++ Makefile.pre.in | 1 + Objects/cown.c | 314 +++++++++++++++++++++++++ Objects/object.c | 3 + Objects/regions.c | 152 +++++++++--- PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/bltinmodule.c | 2 + Python/stdlib_module_names.h | 1 + Tools/c-analyzer/cpython/ignored.tsv | 5 +- 16 files changed, 617 insertions(+), 38 deletions(-) create mode 100644 Lib/test/test_using.py create mode 100644 Lib/using.py create mode 100644 Objects/cown.c diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index a3c6d53a9fe6d0..6b363abd49423d 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -32,12 +32,6 @@ static inline void _Py_SET_REGION(PyObject *ob, Py_region_ptr_t region) { # define Py_SET_REGION(ob, region) (_Py_SET_REGION(_PyObject_CAST(ob), _Py_CAST(Py_region_ptr_t, (region)))) #endif -static inline Py_ALWAYS_INLINE int _Py_IsCown(PyObject *op) -{ - return 0; // TODO: implement this when cowns are added -} -#define _Py_IsCown(op) _Py_IsCown(_PyObject_CAST(op)) - /* This makes the given objects and all object reachable from the given * object immutable. This will also move the objects into the immutable * region. @@ -65,6 +59,7 @@ PyObject* _Py_ResetInvariant(void); // Invariant placeholder bool _Pyrona_AddReference(PyObject* target, PyObject* new_ref); #define Pyrona_ADDREFERENCE(a, b) _Pyrona_AddReference(a, b) +#define Pyrona_REMOVEREFERENCE(a, b) // TODO // Helper macros to count the number of arguments #define _COUNT_ARGS(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, N, ...) N #define COUNT_ARGS(...) _COUNT_ARGS(__VA_ARGS__, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) @@ -81,6 +76,12 @@ bool _Pyrona_AddReferences(PyObject* target, int new_refc, ...); #endif int _Py_CheckRegionInvariant(PyThreadState *tstate); +// Set a cown as parent of a region +void _PyRegion_set_cown_parent(PyObject* region, PyObject* cown); +// Check whether a region is closed +int _PyRegion_is_closed(PyObject* region); +int _PyCown_release(PyObject *self); +int _PyCown_is_released(PyObject *self); #ifdef __cplusplus } diff --git a/Include/object.h b/Include/object.h index e1f067733356b4..e6909cdd8ff3b1 100644 --- a/Include/object.h +++ b/Include/object.h @@ -142,8 +142,12 @@ static inline Py_region_ptr_with_tags_t Py_region_ptr_with_tags(Py_region_ptr_t return (Py_region_ptr_with_tags_t) { region }; } +int _Py_is_bridge_object(PyObject *op); +#define Py_is_bridge_object(op) (_Py_is_bridge_object(_PyObject_CAST(op))) + #define _Py_LOCAL_REGION ((Py_region_ptr_t)0) -#define _Py_IMMUTABLE ((Py_region_ptr_t)1) +#define _Py_IMMUTABLE ((Py_region_ptr_t)1) +#define _Py_COWN ((Py_region_ptr_t)4) // Make all internal uses of PyObject_HEAD_INIT immortal while preserving the // C-API expectation that the refcnt will be set to 1. @@ -279,6 +283,8 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { # define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), (type)) #endif +void _Py_notify_regions_in_use(void); + static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { // This immortal check is for code that is unaware of immortal objects. // The runtime tracks these objects and we should avoid as much diff --git a/Include/regions.h b/Include/regions.h index f9f1eaadc5a271..52a7ec3e9d15a5 100644 --- a/Include/regions.h +++ b/Include/regions.h @@ -9,10 +9,12 @@ extern "C" { PyAPI_FUNC(int) _Py_IsImmutable(PyObject *op); #define Py_IsImmutable(op) _Py_IsImmutable(_PyObject_CAST(op)) - PyAPI_FUNC(int) _Py_IsLocal(PyObject *op); #define Py_IsLocal(op) _Py_IsLocal(_PyObject_CAST(op)) +PyAPI_FUNC(int) _Py_IsCown(PyObject *op); +#define Py_IsCown(op) _Py_IsCown(_PyObject_CAST(op)) + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py new file mode 100644 index 00000000000000..b827cefc729bfc --- /dev/null +++ b/Lib/test/test_using.py @@ -0,0 +1,97 @@ +import unittest +from using import * + +# Initial test cases for using and cowns +# Note: no concurrency test yet +class UsingTest(unittest.TestCase): + obj = None + + def setUp(self): + makeimmutable(self.obj) + + def test_cown(self): + def invalid_assignment1(c): + c.value = 42 + def invalid_assignment2(c): + c.f = 42 + def invalid_assignment3(c): + c["g"] = 42 + + c = Cown() + self.assertRaises(AttributeError, invalid_assignment1, c) + self.assertRaises(AttributeError, invalid_assignment2, c) + self.assertRaises(TypeError, invalid_assignment3, c) + # Cannot access unacquired cown + self.assertRaises(RegionError, lambda _ : c.get(), c) + self.assertRaises(RegionError, lambda _ : c.set(Region()), c) + + def test_cown_aquired_access(self): + c = Cown() + @using(c) + def _(): + c.set(self.obj) + @using(c) + def _(): + self.assertEqual(c.get(), self.obj) + + # Returns the state of a cown as a string + # Hacky but want to avoid adding methods to cowns just for testing + def hacky_state_check(self, cown, expected_state): + s = repr(cown) + return expected_state in s + + def test_release(self): + r = Region() + c = Cown(r) + self.assertFalse(r.is_open()) + self.assertTrue(self.hacky_state_check(c, "released")) + + def test_early_release_cown(self): + c = Cown() + @using(c) + def _(): + self.assertTrue(self.hacky_state_check(c, "acquired")) + c.set(c) + self.assertTrue(self.hacky_state_check(c, "released")) + self.assertTrue(self.hacky_state_check(c, "released")) + + def test_early_release_closed_region(self): + c = Cown() + self.assertTrue(self.hacky_state_check(c, "released")) + @using(c) + def _(): + self.assertTrue(self.hacky_state_check(c, "acquired")) + r = Region() + self.assertFalse(r.is_open()) + c.set(r) + self.assertTrue(self.hacky_state_check(c, "released")) + self.assertTrue(self.hacky_state_check(c, "released")) + + def test_early_release_immutable(self): + c = Cown() + @using(c) + def _(): + self.assertTrue(self.hacky_state_check(c, "acquired")) + c.set(self.obj) + self.assertTrue(self.hacky_state_check(c, "released")) + self.assertTrue(self.hacky_state_check(c, "released")) + + def test_pending_release(self): + r = Region() + r.open() + self.assertTrue(r.is_open()) + c = Cown(r) + self.assertTrue(self.hacky_state_check(c, "pending-release")) + r.close() + self.assertFalse(r.is_open()) + self.assertTrue(self.hacky_state_check(c, "released")) + + def _test_acquire(self): + c = Cown(Region()) + @using(c) + def _(): + self.assertTrue(self.hacky_state_check(c, "acquired")) + c.get().close() + self.assertTrue(self.hacky_state_check(c, "released")) + self.assertTrue(c.get().is_closed()) + self.assertTrue(self.hacky_state_check(c, "released")) diff --git a/Lib/using.py b/Lib/using.py new file mode 100644 index 00000000000000..30ae93a3bbb1cf --- /dev/null +++ b/Lib/using.py @@ -0,0 +1,47 @@ +from contextlib import contextmanager + +# This library defines a decorator "@using" that uses blocking semantics. +# A function decorated by a @using will be called as a result of its +# definition. +# +# Example: +# +# @using(c1, c2) +# def _(): +# print(f"c1 and c2 are now acquired") +# +# Assuming c1 and c2 are cowns, the system will block on acquiring them, +# then call the function _ and release c1 and c2 when the function +# terminates. If c1 or c2 are updated with a closed region, a cown or an +# immutable object, c1 or c2 will be released immediately. + + +def using(*args): + @contextmanager + def CS(cowns, *args): + for c in cowns: + c.acquire() + + try: + # Yield control to the code inside the 'with' block + yield args + finally: + for c in cowns: + c.release() + + def argument_check(cowns, args): + for a in args: + # Append cowns to the list of things that must be acquired + if isinstance(a, Cown): + cowns.append(a) + else: + raise Exception("Using only works on cowns, " + "but was passed " + repr(a)) + + def decorator(func): + cowns = [] + argument_check(cowns, args) + + with CS(cowns, *args): + return func() + return decorator diff --git a/Makefile.pre.in b/Makefile.pre.in index 0b80d67452011f..572b1813af676a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -452,6 +452,7 @@ OBJECT_OBJS= \ Objects/classobject.o \ Objects/codeobject.o \ Objects/complexobject.o \ + Objects/cown.o \ Objects/descrobject.o \ Objects/enumobject.o \ Objects/exceptions.o \ diff --git a/Objects/cown.c b/Objects/cown.c new file mode 100644 index 00000000000000..0f0438b6fcaf26 --- /dev/null +++ b/Objects/cown.c @@ -0,0 +1,314 @@ +#include "Python.h" +#include +#include +#include +#include +#include +#include +#include +#include "methodobject.h" +#include "modsupport.h" +#include "object.h" +#include "pycore_ast.h" +#include "pycore_dict.h" +#include "pycore_interp.h" +#include "pycore_object.h" +#include "pycore_regions.h" +#include "pycore_pyerrors.h" +#include "pycore_atomic.h" +#include "pyerrors.h" +#include "pystate.h" + +typedef enum { + Cown_RELEASED = 0, + Cown_ACQUIRED = 1, + Cown_PENDING_RELEASE = 2, +} CownState; + +typedef struct PyCownObject { + PyObject_HEAD + _Py_atomic_int state; + size_t owning_thread; + sem_t semaphore; + PyObject* value; +} PyCownObject; + +static PyObject *PyCown_set_unchecked(PyCownObject *self, PyObject *arg); +static PyObject *PyCown_set(PyCownObject *self, PyObject *arg); +static PyObject *PyCown_get(PyCownObject *self); +static PyObject *PyCown_acquire(PyCownObject *self); + +#define POSIX_FAIL_GUARD(exp) \ + if ((exp)) { \ + fprintf(stderr, "Unsuccessful return from %s", #exp); \ + abort(); \ + } + +static void PyCown_dealloc(PyCownObject *self) { + POSIX_FAIL_GUARD(sem_destroy(&self->semaphore)); + + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack((PyObject *)self); + Py_TRASHCAN_BEGIN(self, PyCown_dealloc) + Py_CLEAR(self->value); + PyObject_GC_Del(self); + Py_DECREF(tp); + Py_TRASHCAN_END +} + +static int PyCown_init(PyCownObject *self, PyObject *args, PyObject *kwds) { + // TODO: should not be needed in the future + _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); + _Py_notify_regions_in_use(); + + POSIX_FAIL_GUARD(sem_init(&self->semaphore, 0, 0)); + Py_SET_REGION(self, _Py_COWN); + + static char *kwlist[] = {"value", NULL}; + PyObject *value = NULL; + + // See if we got a value as a keyword argument + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &value)) { + return -1; // Return -1 on failure + } + + if (value) { + PyCown_set_unchecked(self, value); + + } else { + _Py_atomic_store(&self->state, Cown_RELEASED); + self->value = Py_None; + } + return 0; +} + +static int PyCown_traverse(PyCownObject *self, visitproc visit, void *arg) { + Py_VISIT(self->value); + return 0; +} + +#define STATE(op) op->state._value + +#define BAIL_IF_OWNED(o, msg) \ + do { \ + /* Note: we must hold the GIL at this point -- note for future threading implementation. */ \ + if (o->owning_thread != 0) { \ + PyErr_Format(PyExc_RegionError, "%s: %S -- %zd", msg, o, o->owning_thread); \ + return NULL; \ + } \ + } while(0); + +#define BAIL_UNLESS_OWNED(o, msg) \ + do { \ + /* Note: we must hold the GIL at this point -- note for future threading implementation. */ \ + PyThreadState *tstate = PyThreadState_Get(); \ + if (o->owning_thread != tstate->thread_id) { \ + PyErr_Format(PyExc_RegionError, "%s: %S", msg, o); \ + return NULL; \ + } \ + } while(0); + +#define BAIL_UNLESS_IN_STATE(o, expected_state, msg) \ + do { \ + /* Note: we must hold the GIL at this point -- note for future threading implementation. */ \ + if (STATE(o) != expected_state) { \ + PyErr_Format(PyExc_RegionError, "%s: %S", msg, o); \ + return NULL; \ + } \ + } while(0); + +#define BAIL_UNLESS_ACQUIRED(o, msg) \ + BAIL_UNLESS_OWNED(o, msg) \ + BAIL_UNLESS_IN_STATE(o, Cown_ACQUIRED, msg) + +static PyObject *PyCown_acquire(PyCownObject *self) { + int expected = Cown_RELEASED; + + // TODO: eventually replace this with something from pycore_atomic (nothing there now) + while (!atomic_compare_exchange_strong(&self->state._value, &expected, Cown_ACQUIRED)) { + sem_wait(&self->semaphore); + expected = Cown_RELEASED; + } + + // Note: we must hold the GIL at this point -- note for future + // threading implementation. + PyThreadState *tstate = PyThreadState_Get(); + self->owning_thread = tstate->thread_id; + + Py_RETURN_NONE; +} + +static PyObject *PyCown_release(PyCownObject *self) { + if (STATE(self) == Cown_RELEASED) { + BAIL_IF_OWNED(self, "BUG: Released cown had owning thread: %p"); + Py_RETURN_NONE; + } + + BAIL_UNLESS_OWNED(self, "Thread attempted to release a cown it did not own"); + + self->owning_thread = 0; + _Py_atomic_store(&self->state, Cown_RELEASED); + sem_post(&self->semaphore); + + Py_RETURN_NONE; +} + +int _PyCown_release(PyObject *self) { + PyObject* res = PyCown_release((PyCownObject *)self); + return res == Py_None ? 0 : -1; +} + +int _PyCown_is_released(PyObject *self) { + PyCownObject *cown = (PyCownObject *)self; + return STATE(cown) == Cown_RELEASED; +} + +static PyObject *PyCown_get(PyCownObject *self) { + BAIL_UNLESS_ACQUIRED(self, "Attempt to get value of unacquired cown"); + + if (self->value) { + return Py_NewRef(self->value); + } else { + Py_RETURN_NONE; + } +} + +// Needed to test for region object +extern PyTypeObject PyRegion_Type; +extern PyTypeObject PyCown_Type; + +static PyObject *PyCown_set_unchecked(PyCownObject *self, PyObject *arg) { + // Cowns are cells that hold a reference to a bridge object, + // (or another cown or immutable object) + const bool is_region_object = + Py_IS_TYPE(arg, &PyRegion_Type) && _Py_is_bridge_object(arg); + if (is_region_object || + arg->ob_type == &PyCown_Type || + _Py_IsImmutable(arg)) { + + PyObject* old = self->value; + Py_XINCREF(arg); + self->value = arg; + + // Tell the region that it is owned by a cown, + // to enable it to release the cown on close + if (is_region_object) { + _PyRegion_set_cown_parent(arg, _PyObject_CAST(self)); + if (_PyRegion_is_closed(arg)) { + PyCown_release(self); + } else { + _Py_atomic_store(&self->state, Cown_PENDING_RELEASE); + // TODO: do we need an owning thread for this? + PyThreadState *tstate = PyThreadState_Get(); + self->owning_thread = tstate->thread_id; + } + } else { + // We can release this cown immediately + // _Py_atomic_store(&self->state, Cown_RELEASED); + // self->owning_thread = 0; + PyCown_release(self); + } + + return old ? old : Py_None; + } else { + // Invalid cown content + PyErr_SetString(PyExc_RuntimeError, + "Cowns can only store bridge objects, immutable objects or other cowns!"); + return NULL; + } +} + +static PyObject *PyCown_set(PyCownObject *self, PyObject *arg) { + BAIL_UNLESS_ACQUIRED(self, "Attempt to set value of unacquired cown"); + return PyCown_set_unchecked(self, arg); +} + +static int PyCown_clear(PyCownObject *self) { + Py_CLEAR(self->value); + return 0; +} + +static PyObject *PyCown_repr(PyCownObject *self) { +#ifdef PYDEBUG + if (STATE(self) == Cown_ACQUIRED) { + return PyUnicode_FromFormat( + "Cown(status=acquired by thread %zd,value=%S)", + PyThreadState_Get()->thread_id, + PyObject_Repr(self->value) + ); + } else { + return PyUnicode_FromFormat( + "Cown(status=%s,value=%S)", + STATE(self) == Cown_RELEASED + ? "released" + : "pending-release", + PyObject_Repr(self->value) + ); + } +#else + if (STATE(self) == Cown_ACQUIRED) { + return PyUnicode_FromFormat( + "Cown(status=acquired by thread %zd)", + PyThreadState_Get()->thread_id + ); + } else { + return PyUnicode_FromFormat( + "Cown(status=%s)", + STATE(self) == Cown_RELEASED + ? "released" + : "pending-release" + ); + } +#endif +} + +// Define the CownType with methods +static PyMethodDef PyCown_methods[] = { + {"acquire", (PyCFunction)PyCown_acquire, METH_NOARGS, "Acquire the cown."}, + {"release", (PyCFunction)PyCown_release, METH_NOARGS, "Release the cown."}, + {"get", (PyCFunction)PyCown_get, METH_NOARGS, "Get contents of acquired cown."}, + {"set", (PyCFunction)PyCown_set, METH_O, "Set contents of acquired cown."}, + {NULL} // Sentinel +}; + + +PyTypeObject PyCown_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "Cown", /* tp_name */ + sizeof(PyCownObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyCown_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + (reprfunc)PyCown_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)PyCown_traverse, /* tp_traverse */ + (inquiry)PyCown_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyCown_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PyCown_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; diff --git a/Objects/object.c b/Objects/object.c index b201df79b6837b..b9ce043f6e4140 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2045,6 +2045,7 @@ extern PyTypeObject _PyLineIterator; extern PyTypeObject _PyPositionsIterator; extern PyTypeObject _PyLegacyEventHandler_Type; extern PyTypeObject PyRegion_Type; +extern PyTypeObject PyCown_Type; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and @@ -2168,6 +2169,8 @@ static PyTypeObject* static_types[] = { // Pyrona Region: &PyRegion_Type, + // Pyrona Cown: + &PyCown_Type, }; diff --git a/Objects/regions.c b/Objects/regions.c index 04217fe76a0a41..1020bf4febcfdf 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -5,11 +5,13 @@ #include #include #include "object.h" +#include "regions.h" #include "pycore_dict.h" #include "pycore_interp.h" #include "pycore_object.h" #include "pycore_regions.h" #include "pycore_pyerrors.h" +#include "pyerrors.h" // This tag indicates that the `regionmetadata` object has been merged // with another region. The `parent` pointer points to the region it was @@ -32,7 +34,8 @@ static inline Py_region_ptr_with_tags_t Py_TAGGED_REGION(PyObject *ob) { #define IS_IMMUTABLE_REGION(r) (REGION_PTR_CAST(r) == _Py_IMMUTABLE) #define IS_LOCAL_REGION(r) (REGION_PTR_CAST(r) == _Py_LOCAL_REGION) -#define HAS_METADATA(r) (!IS_LOCAL_REGION(r) && !IS_IMMUTABLE_REGION(r)) +#define IS_COWN_REGION(r) (REGION_PTR_CAST(r) == _Py_COWN) +#define HAS_METADATA(r) (!IS_LOCAL_REGION(r) && !IS_IMMUTABLE_REGION(r) && !IS_COWN_REGION(r)) typedef struct regionmetadata regionmetadata; typedef struct PyRegionObject PyRegionObject; @@ -118,6 +121,7 @@ struct regionmetadata { // // Intrinsic list for invariant checking regionmetadata* next; + PyObject* cown; // To be able to release a cown; to be integrated with parent }; static Py_region_ptr_t regionmetadata_get_merge_tree_root(Py_region_ptr_t self) @@ -147,7 +151,7 @@ static void regionmetadata_open(regionmetadata* self) { } static bool regionmetadata_is_open(Py_region_ptr_t self) { - if (!HAS_METADATA(self)) { + if (HAS_METADATA(self)) { return REGION_DATA_CAST(self)->is_open; } @@ -379,6 +383,10 @@ int _Py_IsImmutable(PyObject *op) { return IS_IMMUTABLE_REGION(Py_REGION(op)); } +int _Py_IsCown(PyObject *op) +{ + return Py_REGION(op) == _Py_COWN; +} Py_region_ptr_t _Py_REGION(PyObject *ob) { if (!ob) { @@ -500,7 +508,7 @@ regionmetadata* captured = CAPTURED_SENTINEL; /** * Enable the region check. */ -static void notify_regions_in_use(void) +void _Py_notify_regions_in_use(void) { // Do not re-enable, if we have detected a fault. if (!invariant_error_occurred) @@ -595,11 +603,17 @@ visit_invariant_check(PyObject *tgt, void *src_void) } // Cross-region references must be to a bridge - if (!is_bridge_object(tgt)) { + if (!_Py_is_bridge_object(tgt)) { emit_invariant_error(src, tgt, "Reference from object in one region into another region"); return 0; } + regionmetadata* src_region = REGION_DATA_CAST(src_region_ptr); + // Region objects may be stored in cowns + if (IS_COWN_REGION(src_region)) { + return 0; + } + regionmetadata* tgt_region = REGION_DATA_CAST(tgt_region_ptr); // Check if region is already added to captured list if (tgt_region->next != NULL) { @@ -608,7 +622,7 @@ visit_invariant_check(PyObject *tgt, void *src_void) return 0; } // Forbid cycles in the region topology - if (regionmetadata_has_ancestor(REGION_DATA_CAST(src_region_ptr), tgt_region)) { + if (regionmetadata_has_ancestor(src_region, tgt_region)) { emit_invariant_error(src, tgt, "Regions create a cycle with subregions"); return 0; } @@ -948,12 +962,12 @@ static int _makeimmutable_visit(PyObject* obj, void* frontier) PyObject* _Py_MakeImmutable(PyObject* obj) { - if (!obj) { + if (!obj || _Py_IsCown(obj)) { Py_RETURN_NONE; } // We have started using regions, so notify to potentially enable checks. - notify_regions_in_use(); + _Py_notify_regions_in_use(); // Some built-in objects are direclty created immutable. However, their types // might be created in a mutable state. This therefore requres an additional @@ -1198,7 +1212,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // At this point, we know that target is in another region. // If target is in a different region, it has to be a bridge object. // References to contained objects are forbidden. - if (!is_bridge_object(target)) { + if (!_Py_is_bridge_object(target)) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_CONTAINED_OBJ_REF }; return ((info->handle_error)(&err, info->handle_error_data)); @@ -1256,10 +1270,10 @@ static int visit_object(PyObject *item, visitproc visit, void* info) { return ((visit)(type_ob, info) == 0); } -// Add the transitive closure of objets in the local region reachable from obj to region +// Add the transitive closure of objects in the local region reachable from obj to region static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) { - if (!obj) { + if (!obj || _Py_IsCown(obj)) { Py_RETURN_NONE; } @@ -1315,6 +1329,48 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) Py_RETURN_NONE; } +int _Py_is_bridge_object(PyObject *op) { + Py_region_ptr_t region = Py_REGION(op); + if (IS_LOCAL_REGION(region) || IS_IMMUTABLE_REGION(region)) { + return false; + } + + // It's not yet clear how immutability will interact with region objects. + // It's likely that the object will remain in the object topology but + // will use the properties of a bridge object. This therefore checks if + // the object is equal to the regions bridge object rather than checking + // that the type is `PyRegionObject` + return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); +} + +__attribute__((unused)) +static void regionmetadata_inc_lrc(regionmetadata* data) { + data->lrc += 1; +} + +__attribute__((unused)) +static void regionmetadata_dec_lrc(regionmetadata* data) { + data->lrc -= 1; +} + +static void regionmetadata_close(regionmetadata* data) { + data->is_open = 0; +} + +__attribute__((unused)) +static void regionmetadata_unparent(regionmetadata* data) { + regionmetadata_set_parent(data, NULL); +} + +__attribute__((unused)) +static int regionmetadata_is_root(regionmetadata* data) { + return regionmetadata_has_parent(data); +} + +static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj) { + return obj->metadata; +} + static void PyRegion_dealloc(PyRegionObject *self) { // Name is immutable and not in our region. @@ -1327,25 +1383,31 @@ static void PyRegion_dealloc(PyRegionObject *self) { regionmetadata_dec_rc(data); } - // The region should be cleared by pythons general deallocator. - assert(Py_REGION(self) == _Py_LOCAL_REGION); - - // The dictionary can be NULL if the Region constructor crashed + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(_PyObject_CAST(self)); + Py_TRASHCAN_BEGIN(self, PyRegion_dealloc); if (self->dict) { // We need to clear the ownership, since this dictionary might be // returned to an object pool rather than freed. This would result // in an error if the dictionary has the previous region. + // TODO: revisit in #16 Py_SET_REGION(self->dict, _Py_LOCAL_REGION); - Py_DECREF(self->dict); - self->dict = NULL; + Py_CLEAR(self->dict); } - PyObject_GC_UnTrack((PyObject *)self); - Py_TYPE(self)->tp_free((PyObject *)self); + PyObject_GC_Del(self); + Py_DECREF(tp); + Py_TRASHCAN_END } static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { - notify_regions_in_use(); + // TODO: should not be needed in the future + _Py_notify_regions_in_use(); + _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); + +#ifndef NDEBUG + fprintf(stderr, "Region created (%p)\n", self); +#endif static char *kwlist[] = {"name", NULL}; self->metadata = (regionmetadata*)calloc(1, sizeof(regionmetadata)); @@ -1388,6 +1450,12 @@ static int PyRegion_traverse(PyRegionObject *self, visitproc visit, void *arg) { return 0; } +static int PyRegion_clear(PyRegionObject *self) { + Py_CLEAR(self->metadata->name); + Py_CLEAR(self->dict); + return 0; +} + // is_open method (returns True if the region is open, otherwise False) static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *args) { // FIXME: What is the behavior of a `PyRegionObject` that has been merged into another region? @@ -1406,10 +1474,28 @@ static PyObject *PyRegion_open(PyRegionObject *self, PyObject *args) { Py_RETURN_NONE; // Return None (standard for methods with no return value) } -// try_close method (Attempts to close the region) -static PyObject *PyRegion_try_close(PyRegionObject *self, PyObject *args) { - // Not implemented for now. Return NULL to get an error - return NULL; +int _PyRegion_is_closed(PyObject* self) { + return PyRegion_is_open((PyRegionObject *)self, NULL) == Py_False; +} + +// Close method (attempts to set the region to "closed") +// TODO: integrate with #19 and associated PRs +static PyObject *PyRegion_close(PyRegionObject *self, PyObject *args) { + regionmetadata* const md = REGION_DATA_CAST(Py_REGION(self)); + if (regionmetadata_is_open(md)) { + regionmetadata_close(md); // Mark as closed + + // Check if in a cown -- if so, release cown + if (md->cown) { + if (_PyCown_release(md->cown) != 0) { + // Propagate error from release + return NULL; + } + } + Py_RETURN_NONE; // Return None (standard for methods with no return value) + } else { + Py_RETURN_NONE; // Double close is OK + } } // Adds args object to self region @@ -1451,18 +1537,18 @@ static PyObject *PyRegion_repr(PyRegionObject *self) { #ifdef NDEBUG // Debug mode: include detailed representation return PyUnicode_FromFormat( - "Region(lrc=%d, osc=%d, name=%S, is_open=%d)", + "Region(lrc=%d, osc=%d, name=%S, is_open=%s)", data->lrc, data->osc, data->name ? data->name : Py_None, - data->is_open + data->is_open ? "yes" : "no" ); #else // Normal mode: simple representation return PyUnicode_FromFormat( - "Region(name=%S, is_open=%d)", + "Region(name=%S, is_open=%s)", data->name ? data->name : Py_None, - data->is_open + data->is_open ? "yes" : "no" ); #endif } @@ -1470,7 +1556,7 @@ static PyObject *PyRegion_repr(PyRegionObject *self) { // Define the RegionType with methods static PyMethodDef PyRegion_methods[] = { {"open", (PyCFunction)PyRegion_open, METH_NOARGS, "Open the region."}, - {"try_close", (PyCFunction)PyRegion_try_close, METH_NOARGS, "Attempt to close the region."}, + {"close", (PyCFunction)PyRegion_close, METH_NOARGS, "Attempt to close the region."}, {"is_open", (PyCFunction)PyRegion_is_open, METH_NOARGS, "Check if the region is open."}, // Temporary methods for testing. These will be removed or at least renamed once // the write barrier is done. @@ -1504,7 +1590,7 @@ PyTypeObject PyRegion_Type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "TODO =^.^=", /* tp_doc */ (traverseproc)PyRegion_traverse, /* tp_traverse */ - 0, /* tp_clear */ + (inquiry)PyRegion_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ @@ -1537,6 +1623,8 @@ static const char *get_region_name(PyObject* obj) { return "Default"; } else if (Py_IsImmutable(obj)) { return "Immutable"; + } else if (_Py_IsCown(obj)) { + return "Cown"; } else { const regionmetadata *md = Py_REGION_DATA(obj); return md->name @@ -1581,3 +1669,9 @@ bool _Pyrona_AddReferences(PyObject *tgt, int new_refc, ...) { va_end(args); return true; } + +void _PyRegion_set_cown_parent(PyObject* region, PyObject* cown) { + regionmetadata* md = PyRegion_get_metadata((PyRegionObject*) region); + Py_XINCREF(cown); + Py_XSETREF(md->cown, cown); +} diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 31b94b81f5e889..68dd51666731f7 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -131,6 +131,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 3366289ccd05ef..0994cdbf0d1ccd 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -103,6 +103,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 48d882a803a998..2e5e879a47e819 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -455,6 +455,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4e4ba7b0b63330..f51fa339810746 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -995,6 +995,9 @@ Objects + + Objects + Objects diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d66a9559816450..89d51a9d7d3bab 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3164,6 +3164,7 @@ static struct PyModuleDef builtinsmodule = { }; extern PyTypeObject PyRegion_Type; +extern PyTypeObject PyCown_Type; PyObject * _PyBuiltin_Init(PyInterpreterState *interp) @@ -3226,6 +3227,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("type", &PyType_Type); SETBUILTIN("zip", &PyZip_Type); SETBUILTIN("Region", &PyRegion_Type); + SETBUILTIN("Cown", &PyCown_Type); debug = PyBool_FromLong(config->optimization_level == 0); if (PyDict_SetItemString(dict, "__debug__", debug) < 0) { diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index ed4a0ac2dd32de..7b9d6c46bdb3e0 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -284,6 +284,7 @@ static const char* _Py_stdlib_module_names[] = { "unicodedata", "unittest", "urllib", +"using", "uu", "uuid", "venv", diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 10e0453bde2cce..d563d8313e3590 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -719,10 +719,13 @@ Modules/_sqlite/module.c - _sqlite3module - ## Region Type Info this is constant ## Why do we have three of these? Surely it should just be in one file? -## Probably a bug in the analysis as they are listed as extern in two of the files. Objects/object.c - PyRegion_Type - Objects/regions.c - PyRegion_Type - +Objects/cown.c - PyRegion_Type - Python/bltinmodule.c - PyRegion_Type - +Objects/object.c - PyCown_Type - +Objects/cown.c - PyCown_Type - +Python/bltinmodule.c - PyCown_Type - ## Regions Debug Info for Invariant ## Not to remain global, and should become localised to an interpreter From 257fa452ae4998ddbc358855eacf63cd56a632ff Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Fri, 20 Dec 2024 12:39:42 +0100 Subject: [PATCH 39/68] Addressed comments by @xFrednet --- Objects/cown.c | 25 +++++++++++++---------- Objects/regions.c | 51 +++++++++-------------------------------------- 2 files changed, 23 insertions(+), 53 deletions(-) diff --git a/Objects/cown.c b/Objects/cown.c index 0f0438b6fcaf26..b62df2ccb47f52 100644 --- a/Objects/cown.c +++ b/Objects/cown.c @@ -35,8 +35,8 @@ typedef struct PyCownObject { static PyObject *PyCown_set_unchecked(PyCownObject *self, PyObject *arg); static PyObject *PyCown_set(PyCownObject *self, PyObject *arg); -static PyObject *PyCown_get(PyCownObject *self); -static PyObject *PyCown_acquire(PyCownObject *self); +static PyObject *PyCown_get(PyCownObject *self, PyObject *ignored); +static PyObject *PyCown_acquire(PyCownObject *self, PyObject *ignored); #define POSIX_FAIL_GUARD(exp) \ if ((exp)) { \ @@ -121,7 +121,9 @@ static int PyCown_traverse(PyCownObject *self, visitproc visit, void *arg) { BAIL_UNLESS_OWNED(o, msg) \ BAIL_UNLESS_IN_STATE(o, Cown_ACQUIRED, msg) -static PyObject *PyCown_acquire(PyCownObject *self) { +// The ignored argument is required for this function's type to be +// compatible with PyCFunction +static PyObject *PyCown_acquire(PyCownObject *self, PyObject *ignored) { int expected = Cown_RELEASED; // TODO: eventually replace this with something from pycore_atomic (nothing there now) @@ -138,7 +140,9 @@ static PyObject *PyCown_acquire(PyCownObject *self) { Py_RETURN_NONE; } -static PyObject *PyCown_release(PyCownObject *self) { +// The ignored argument is required for this function's type to be +// compatible with PyCFunction +static PyObject *PyCown_release(PyCownObject *self, PyObject *ignored) { if (STATE(self) == Cown_RELEASED) { BAIL_IF_OWNED(self, "BUG: Released cown had owning thread: %p"); Py_RETURN_NONE; @@ -154,7 +158,7 @@ static PyObject *PyCown_release(PyCownObject *self) { } int _PyCown_release(PyObject *self) { - PyObject* res = PyCown_release((PyCownObject *)self); + PyObject* res = PyCown_release((PyCownObject *)self, NULL); return res == Py_None ? 0 : -1; } @@ -163,7 +167,9 @@ int _PyCown_is_released(PyObject *self) { return STATE(cown) == Cown_RELEASED; } -static PyObject *PyCown_get(PyCownObject *self) { +// The ignored argument is required for this function's type to be +// compatible with PyCFunction +static PyObject *PyCown_get(PyCownObject *self, PyObject *ignored) { BAIL_UNLESS_ACQUIRED(self, "Attempt to get value of unacquired cown"); if (self->value) { @@ -195,18 +201,15 @@ static PyObject *PyCown_set_unchecked(PyCownObject *self, PyObject *arg) { if (is_region_object) { _PyRegion_set_cown_parent(arg, _PyObject_CAST(self)); if (_PyRegion_is_closed(arg)) { - PyCown_release(self); + PyCown_release(self, NULL); } else { _Py_atomic_store(&self->state, Cown_PENDING_RELEASE); - // TODO: do we need an owning thread for this? PyThreadState *tstate = PyThreadState_Get(); self->owning_thread = tstate->thread_id; } } else { // We can release this cown immediately - // _Py_atomic_store(&self->state, Cown_RELEASED); - // self->owning_thread = 0; - PyCown_release(self); + PyCown_release(self, NULL); } return old ? old : Py_None; diff --git a/Objects/regions.c b/Objects/regions.c index 1020bf4febcfdf..152fa6813a4ab2 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -360,21 +360,6 @@ static PyObject* regionmetadata_merge(regionmetadata* self, Py_region_ptr_t othe #define regionmetadata_merge(self, other) \ (regionmetadata_merge(self, REGION_PTR_CAST(other))); -static bool is_bridge_object(PyObject *op) { - Py_region_ptr_t region = Py_REGION(op); - // The local and immutable region (represented as NULL) never have a bridge object. - if (!HAS_METADATA(region)) { - return false; - } - - // It's not yet clear how immutability will interact with region objects. - // It's likely that the object will remain in the object topology but - // will use the properties of a bridge object. This therefore checks if - // the object is equal to the regions bridge object rather than checking - // that the type is `PyRegionObject` - return _PyObject_CAST(REGION_DATA_CAST(region)->bridge) == op; -} - int _Py_IsLocal(PyObject *op) { return IS_LOCAL_REGION(Py_REGION(op)); } @@ -1343,30 +1328,10 @@ int _Py_is_bridge_object(PyObject *op) { return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); } -__attribute__((unused)) -static void regionmetadata_inc_lrc(regionmetadata* data) { - data->lrc += 1; -} - -__attribute__((unused)) -static void regionmetadata_dec_lrc(regionmetadata* data) { - data->lrc -= 1; -} - static void regionmetadata_close(regionmetadata* data) { data->is_open = 0; } -__attribute__((unused)) -static void regionmetadata_unparent(regionmetadata* data) { - regionmetadata_set_parent(data, NULL); -} - -__attribute__((unused)) -static int regionmetadata_is_root(regionmetadata* data) { - return regionmetadata_has_parent(data); -} - static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj) { return obj->metadata; } @@ -1405,10 +1370,6 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { _Py_notify_regions_in_use(); _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); -#ifndef NDEBUG - fprintf(stderr, "Region created (%p)\n", self); -#endif - static char *kwlist[] = {"name", NULL}; self->metadata = (regionmetadata*)calloc(1, sizeof(regionmetadata)); if (!self->metadata) { @@ -1457,7 +1418,9 @@ static int PyRegion_clear(PyRegionObject *self) { } // is_open method (returns True if the region is open, otherwise False) -static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *args) { +// The ignored argument is required for this function's type to be +// compatible with PyCFunction +static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *ignored) { // FIXME: What is the behavior of a `PyRegionObject` that has been merged into another region? if (regionmetadata_is_open(self->metadata)) { Py_RETURN_TRUE; // Return True if the region is open @@ -1467,7 +1430,9 @@ static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *args) { } // Open method (sets the region to "open") -static PyObject *PyRegion_open(PyRegionObject *self, PyObject *args) { +// The ignored argument is required for this function's type to be +// compatible with PyCFunction +static PyObject *PyRegion_open(PyRegionObject *self, PyObject *ignored) { // `Py_REGION()` will fetch the root region of the merge tree. // this might be different from the region in `self->metadata`. regionmetadata_open(Py_REGION_DATA(self)); @@ -1480,7 +1445,9 @@ int _PyRegion_is_closed(PyObject* self) { // Close method (attempts to set the region to "closed") // TODO: integrate with #19 and associated PRs -static PyObject *PyRegion_close(PyRegionObject *self, PyObject *args) { +// The ignored argument is required for this function's type to be +// compatible with PyCFunction +static PyObject *PyRegion_close(PyRegionObject *self, PyObject *ignored) { regionmetadata* const md = REGION_DATA_CAST(Py_REGION(self)); if (regionmetadata_is_open(md)) { regionmetadata_close(md); // Mark as closed From 9d5264580def70e601c323bfe89c790e2c5f0070 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Sat, 21 Dec 2024 00:27:52 +0100 Subject: [PATCH 40/68] Found these missing cases when I was taking try_close for a spin 1. Invariant did not permit regions to reference cowns 2. Add to region did not handle references to cowns 3. Asking if an object was a bridge object crashed on cowns --- Lib/test/test_using.py | 14 +++++++++++--- Objects/regions.c | 11 ++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index b827cefc729bfc..5b22f14c1b7b77 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -86,12 +86,20 @@ def test_pending_release(self): self.assertFalse(r.is_open()) self.assertTrue(self.hacky_state_check(c, "released")) - def _test_acquire(self): + def test_acquire(self): c = Cown(Region()) + self.assertTrue(self.hacky_state_check(c, "released")) @using(c) def _(): + r = c.get() + r.open() self.assertTrue(self.hacky_state_check(c, "acquired")) - c.get().close() + r.close() self.assertTrue(self.hacky_state_check(c, "released")) - self.assertTrue(c.get().is_closed()) + self.assertFalse(r.is_open()) self.assertTrue(self.hacky_state_check(c, "released")) + + def test_region_cown_ptr(self): + r = Region() + r.f = Cown() + self.assertTrue(True) diff --git a/Objects/regions.c b/Objects/regions.c index 152fa6813a4ab2..ccc79ee1489588 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -581,6 +581,9 @@ visit_invariant_check(PyObject *tgt, void *src_void) // Borrowed references are unrestricted if (Py_IsLocal(src)) return 0; + // References to cowns are unrestricted + if (Py_IsCown(tgt)) + return 0; // Since tgt is not immutable, src also may not be as immutable may not point to mutable if (Py_IsImmutable(src)) { emit_invariant_error(src, tgt, "Reference from immutable object to mutable target"); @@ -1147,6 +1150,12 @@ static int _add_to_region_visit(PyObject* target, void* info_void) return 0; } + // References to cowns are unrestricted; cowns are opaque so + // do not need travsersing. + if (Py_IsCown(target)) { + return 0; + } + // C wrappers can propergate through the entire system and draw // in a lot of unwanted objects. Since c wrappers don't have mutable // data, we just make it immutable and have the immutability impl @@ -1316,7 +1325,7 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) int _Py_is_bridge_object(PyObject *op) { Py_region_ptr_t region = Py_REGION(op); - if (IS_LOCAL_REGION(region) || IS_IMMUTABLE_REGION(region)) { + if (IS_LOCAL_REGION(region) || IS_IMMUTABLE_REGION(region) || IS_COWN_REGION(region)) { return false; } From 67a5ef9bdd70600f9e64cd97643de61f2c9a2f82 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 17 Dec 2024 15:37:42 +0100 Subject: [PATCH 41/68] Pyrona: Simplify `_add_to_to_region_visit` --- Objects/regions.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index ccc79ee1489588..512e73d3ffc442 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -1103,7 +1103,7 @@ typedef int (*handle_add_to_region_error)(regionerror *, void *); * This function borrows both arguments. The memory has to be managed * the caller. */ -static int emit_region_error(regionerror *error, void* ignored) { +static int emit_region_error(regionerror *error) { const char* msg = NULL; switch (error->id) @@ -1135,8 +1135,6 @@ typedef struct addtoregionvisitinfo { // The source object of the reference. This is used to create // better error message PyObject* src; - handle_add_to_region_error handle_error; - void* handle_error_data; } addtoregionvisitinfo; static int _add_to_region_visit(PyObject* target, void* info_void) @@ -1209,7 +1207,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) if (!_Py_is_bridge_object(target)) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_CONTAINED_OBJ_REF }; - return ((info->handle_error)(&err, info->handle_error_data)); + return emit_region_error(&err); } // The target is a bridge object from another region. We now need to @@ -1218,7 +1216,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) if (regionmetadata_has_parent(target_region)) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_SHARED_CUSTODY}; - return ((info->handle_error)(&err, info->handle_error_data)); + return emit_region_error(&err); } // Make sure that the new subregion relation won't create a cycle @@ -1226,7 +1224,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) if (regionmetadata_has_ancestor(region, target_region)) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_CYCLE_CREATION}; - return ((info->handle_error)(&err, info->handle_error_data)); + return emit_region_error(&err); } // From the previous checks we know that `target` is the bridge object @@ -1245,7 +1243,7 @@ static int visit_object(PyObject *item, visitproc visit, void* info) { // proper handling of moving the function into the region regionerror err = {.src = NULL, .tgt = item, .id = ERR_WIP_FUNCTIONS }; - emit_region_error(&err, NULL); + emit_region_error(&err); return false; } else { PyTypeObject *type = Py_TYPE(item); @@ -1292,8 +1290,6 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) .pending = stack_new(), // `src` is reassigned each iteration .src = _PyObject_CAST(region_data->bridge), - .handle_error = emit_region_error, - .handle_error_data = NULL, }; if (info.pending == NULL) { return PyErr_NoMemory(); From ead419482d0c632e5a390879657c51c0c7fdd946 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Wed, 18 Dec 2024 14:03:15 +0100 Subject: [PATCH 42/68] Pyrona: Attempting to closing the first regions (`try_close`) --- Lib/test/test_veronapy.py | 19 +++++ Objects/regions.c | 175 +++++++++++++++++++++++++++++++++----- 2 files changed, 172 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index a7bfaac0dc2174..6f24c7f8be5860 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -522,6 +522,25 @@ def test_should_fail_external_uniqueness(self): else: self.fail("Should not reach here -- a can't be owned by two objects") +class TestTryCloseRegion(unittest.TestCase): + class A: + pass + + def setUp(self): + # This freezes A and super and meta types of A namely `type` and `object` + makeimmutable(self.A) + # FIXME: remove this line when the write barrier works + makeimmutable(type({})) + + def test_try_close_fail(self): + a = self.A() + r1 = Region("r1") + + # This creates a reference into the region + r1.a = a + self.assertFalse(r1.try_close()) + + # This test will make the Python environment unusable. # Should perhaps forbid making the frame immutable. # class TestStackCapture(unittest.TestCase): diff --git a/Objects/regions.c b/Objects/regions.c index 512e73d3ffc442..a3bfb8e332ca29 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -145,18 +145,27 @@ static Py_region_ptr_t regionmetadata_get_merge_tree_root(Py_region_ptr_t self) #define regionmetadata_get_merge_tree_root(self) \ regionmetadata_get_merge_tree_root(REGION_PTR_CAST(self)) +static void regionmetadata_inc_osc(Py_region_ptr_t self_ptr); static void regionmetadata_open(regionmetadata* self) { assert(HAS_METADATA(self)); + if (self->is_open) { + return; + } self->is_open = true; + regionmetadata_inc_osc(REGION_PTR_CAST(regionmetadata_get_parent(self))); +} + +static void regionmetadata_close(regionmetadata* data) { + data->is_open = 0; } static bool regionmetadata_is_open(Py_region_ptr_t self) { - if (HAS_METADATA(self)) { - return REGION_DATA_CAST(self)->is_open; + if (!HAS_METADATA(self)) { + // The immutable and local region are open by default and can't be closed. + return true; } - // The immutable and local region are open by default and can't be closed. - return true; + return REGION_DATA_CAST(self)->is_open; } #define regionmetadata_is_open(self) \ regionmetadata_is_open(REGION_PTR_CAST(self)) @@ -292,7 +301,6 @@ static bool regionmetadata_has_ancestor(regionmetadata* self, regionmetadata* ot // it's parent. // // This function expects `self` to be a valid object. -__attribute__((unused)) static PyObject* regionmetadata_merge(regionmetadata* self, Py_region_ptr_t other) { assert(HAS_METADATA(self) && "The immutable and local region can't be merged into another region"); assert(REGION_PTR_CAST(self) == regionmetadata_get_merge_tree_root(self) && "Sanity Check"); @@ -300,7 +308,7 @@ static PyObject* regionmetadata_merge(regionmetadata* self, Py_region_ptr_t othe // If `other` is the parent of `self` we can merge it. We unset the the // parent which will also update the rc and other counts. regionmetadata* self_parent = regionmetadata_get_parent(self); - if (REGION_PTR_CAST(self_parent) == other) { + if (self_parent && REGION_PTR_CAST(self_parent) == other) { assert(HAS_METADATA(self_parent) && "The immutable and local region can never have children"); regionmetadata_set_parent(self, NULL); @@ -1132,6 +1140,8 @@ static int emit_region_error(regionerror *error) { typedef struct addtoregionvisitinfo { stack* pending; + // An optional stack to collect newly added subregions + stack* new_sub_regions; // The source object of the reference. This is used to create // better error message PyObject* src; @@ -1168,7 +1178,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) if (Py_IsLocal(target)) { // Add reference to the object, // minus one for the reference we just followed - source_region->lrc += target->ob_refcnt - 1; + source_region->lrc += Py_REFCNT(target) - 1; Py_SET_REGION(target, source_region); if (stack_push(info->pending, target)) { @@ -1230,7 +1240,15 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // From the previous checks we know that `target` is the bridge object // of a free region. Thus we can make it a sub region and allow the // reference. + // + // `set_parent` will also ensure that the `osc` counter is updated. regionmetadata_set_parent(target_region, region); + if (info->new_sub_regions) { + if (stack_push(info->new_sub_regions, target)) { + PyErr_NoMemory(); + return -1; + } + } return 0; } @@ -1279,7 +1297,7 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) // The current implementation assumes region is a valid pointer. This // restriction can be lifted if needed assert(HAS_METADATA(region)); - regionmetadata *region_data = _Py_CAST(regionmetadata *, region); + regionmetadata *region_data = REGION_DATA_CAST(region); // Early return if the object is already in the region or immutable if (Py_REGION(obj) == region || Py_IsImmutable(obj)) { @@ -1288,6 +1306,7 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) addtoregionvisitinfo info = { .pending = stack_new(), + .new_sub_regions = NULL, // `src` is reassigned each iteration .src = _PyObject_CAST(region_data->bridge), }; @@ -1316,6 +1335,10 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) stack_free(info.pending); + if (region_data->lrc > 0 || region_data->osc > 0) { + regionmetadata_open(region_data); + } + Py_RETURN_NONE; } @@ -1333,12 +1356,116 @@ int _Py_is_bridge_object(PyObject *op) { return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); } -static void regionmetadata_close(regionmetadata* data) { - data->is_open = 0; -} +static PyObject *try_close(PyRegionObject *root_bridge) { + + addtoregionvisitinfo info = { + .pending = stack_new(), + .new_sub_regions = stack_new(), + // `src` is reassigned each iteration + .src = NULL, + }; + if (!info.pending || !info.new_sub_regions) { + goto fail; + } + + if (stack_push(info.new_sub_regions, _PyObject_CAST(root_bridge))) { + PyErr_NoMemory(); + goto fail; + } + + // The *LRC* and *is_open* status is currently not updated when references + // to the bridge are created. This means that the bridge might have multiple + // unknown references. + // + // Using the LRC is an over approximation, since internal cycles from objects + // to the region object will also increase the RC thereby preventing it from + // closing. + // + // The root region is expected to have two references, one from the owning + // reference and one from the `self` argument + if (Py_REFCNT(root_bridge) > 2) { + regionmetadata_open(Py_REGION_DATA(root_bridge)); + } + + while (!stack_empty(info.new_sub_regions)) { + PyObject *bridge = stack_pop(info.new_sub_regions); + assert(Py_is_bridge_object(bridge)); + regionmetadata* old_data = Py_REGION_DATA(bridge); + + // If it's closed there is nothing we need to do. + if (!regionmetadata_is_open(old_data)) { + continue; + } + regionmetadata* parent = regionmetadata_get_parent(old_data); + PyObject* region_name = old_data->name; + Py_XINCREF(region_name); + regionmetadata_set_parent(old_data, NULL); + regionmetadata_merge(old_data, _Py_LOCAL_REGION); + + // Create the new `regionmetadata*` + regionmetadata* new_data = (regionmetadata*)calloc(1, sizeof(regionmetadata)); + if (!new_data) { + PyErr_NoMemory(); + goto fail; + } + _Py_CAST(PyRegionObject *, bridge)->metadata = new_data; + regionmetadata_inc_rc(new_data); + new_data->bridge = _Py_CAST(PyRegionObject*, bridge); + new_data->name = region_name; + regionmetadata_set_parent(new_data, parent); + + // -1 for the reference we just followed + new_data->lrc += Py_REFCNT(bridge) - 1; + Py_SET_REGION(bridge, new_data); + + if (stack_push(info.pending, bridge)) { + goto fail; + } + + // Re-add everything to the current region + while (!stack_empty(info.pending)) { + PyObject *item = stack_pop(info.pending); + + // Add `info.src` for better error messages + info.src = item; + + if (!visit_object(item, (visitproc)_add_to_region_visit, &info)) { + goto fail; + } + } + + // Update the open status and make sure the parent knows + if (new_data->lrc != 0 || new_data->osc != 0) { + regionmetadata_open(new_data); + } -static regionmetadata* PyRegion_get_metadata(PyRegionObject* obj) { - return obj->metadata; + // The LRC will never decrease after this point. If the region is open + // due to the LRC it will remain open and the close fails. + // + // FIXME: The root_bride has a higher allowed LRC + if (new_data->lrc != 0 && bridge != _PyObject_CAST(root_bridge)) { + break; + } + } + + regionmetadata *root_data = Py_REGION_DATA(root_bridge); + Py_ssize_t lrc_limit = regionmetadata_has_parent(root_data) ? 1 : 2; + if (root_data->lrc <= lrc_limit && root_data->osc == 0) { + root_data->is_open = false; + } + + stack_free(info.pending); + stack_free(info.new_sub_regions); + return PyBool_FromLong(_Py_CAST(long, !regionmetadata_is_open(root_data))); + +fail: + if (info.pending) { + stack_free(info.pending); + } + if (info.new_sub_regions) { + stack_free(info.new_sub_regions); + } + return NULL; } static void PyRegion_dealloc(PyRegionObject *self) { @@ -1427,11 +1554,8 @@ static int PyRegion_clear(PyRegionObject *self) { // compatible with PyCFunction static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *ignored) { // FIXME: What is the behavior of a `PyRegionObject` that has been merged into another region? - if (regionmetadata_is_open(self->metadata)) { - Py_RETURN_TRUE; // Return True if the region is open - } else { - Py_RETURN_FALSE; // Return False if the region is closed - } + assert(Py_is_bridge_object(_PyObject_CAST(self)) && "FIXME: When does this happend and what should it do?"); + return PyBool_FromLong(_Py_CAST(long, regionmetadata_is_open(self->metadata))); } // Open method (sets the region to "open") @@ -1470,6 +1594,12 @@ static PyObject *PyRegion_close(PyRegionObject *self, PyObject *ignored) { } } +// try_close method (Attempts to close the region) +static PyObject *PyRegion_try_close(PyRegionObject *self, PyObject *args) { + assert(Py_is_bridge_object(self) && "self is not a bridge object"); + return try_close(self); +} + // Adds args object to self region static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args) { if (!args) { @@ -1587,7 +1717,7 @@ void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg) { const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; PyObject *new_ref_type_repr = PyObject_Repr(PyObject_Type(new_ref)); const char *new_ref_desc = new_ref_type_repr ? PyUnicode_AsUTF8(new_ref_type_repr) : "<>"; - PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", tgt, tgt_desc, tgt_region_name, new_ref, new_ref_desc, new_ref_region_name, msg); + // PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", tgt, tgt_desc, tgt_region_name, new_ref, new_ref_desc, new_ref_region_name, msg); } static const char *get_region_name(PyObject* obj) { @@ -1642,8 +1772,9 @@ bool _Pyrona_AddReferences(PyObject *tgt, int new_refc, ...) { return true; } -void _PyRegion_set_cown_parent(PyObject* region, PyObject* cown) { - regionmetadata* md = PyRegion_get_metadata((PyRegionObject*) region); +void _PyRegion_set_cown_parent(PyObject* bridge, PyObject* cown) { + assert(Py_is_bridge_object(bridge)); + regionmetadata* data = Py_REGION_DATA(bridge); Py_XINCREF(cown); - Py_XSETREF(md->cown, cown); + Py_XSETREF(data->cown, cown); } From a26d20284ac463c53009f1290b64c1d2ece8d9c9 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Fri, 20 Dec 2024 13:51:04 +0100 Subject: [PATCH 43/68] Pyrona: `try_close` tests and bug fixes --- Lib/test/test_using.py | 8 +- Lib/test/test_veronapy.py | 258 ++++++++++++++++++++++++++++++++++++-- Objects/cown.c | 2 +- Objects/regions.c | 253 ++++++++++++++++++++++++++----------- 4 files changed, 437 insertions(+), 84 deletions(-) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index 5b22f14c1b7b77..671d69d1d6008e 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -81,9 +81,9 @@ def test_pending_release(self): r.open() self.assertTrue(r.is_open()) c = Cown(r) + r = None self.assertTrue(self.hacky_state_check(c, "pending-release")) - r.close() - self.assertFalse(r.is_open()) + c.get().close() self.assertTrue(self.hacky_state_check(c, "released")) def test_acquire(self): @@ -94,9 +94,9 @@ def _(): r = c.get() r.open() self.assertTrue(self.hacky_state_check(c, "acquired")) - r.close() + r = None + c.get().close() self.assertTrue(self.hacky_state_check(c, "released")) - self.assertFalse(r.is_open()) self.assertTrue(self.hacky_state_check(c, "released")) def test_region_cown_ptr(self): diff --git a/Lib/test/test_veronapy.py b/Lib/test/test_veronapy.py index 6f24c7f8be5860..f64e4abdcd92c7 100644 --- a/Lib/test/test_veronapy.py +++ b/Lib/test/test_veronapy.py @@ -396,8 +396,6 @@ class A: def setUp(self): # This freezes A and super and meta types of A namely `type` and `object` makeimmutable(self.A) - # FIXME: remove this line when the write barrier works - makeimmutable(type({})) enableinvariant() def test_default_ownership(self): @@ -529,16 +527,258 @@ class A: def setUp(self): # This freezes A and super and meta types of A namely `type` and `object` makeimmutable(self.A) - # FIXME: remove this line when the write barrier works - makeimmutable(type({})) - def test_try_close_fail(self): - a = self.A() + def test_new_region_is_closed(self): + r1 = Region("r1") + self.assertFalse(r1.is_open()) + self.assertTrue(r1.try_close()) + # Check it remained closed after the `try_close` call + self.assertFalse(r1.is_open()) + + def test_try_close_with_bridge_ref(self): + r1 = Region("r1") + + # Create a local reference + r1_ref = r1 + + # The region is still marked as closed since the write barrier + # doesn't catch the new `r1_ref` reference + self.assertFalse(r1.is_open(), "Should fail once WB on the Frame is in place") + + # Closing the region fails due to `r1_ref` + self.assertFalse(r1.try_close()) + # The open status was now updated + self.assertTrue(r1.is_open()) + + # Remove the local reference + r1_ref = None + + # Closing the region should now succeed + self.assertTrue(r1.try_close()) + self.assertFalse(r1.is_open()) + + def test_try_close_with_bridge_ref_owned_by_region(self): + r1 = Region("r1") + r1.r2 = Region("r2") + + # Create a local reference + r2_ref = r1.r2 + + # The region is still marked as closed since the write barrier + # doesn't catch the new `r1_ref` reference + self.assertFalse(r1.r2.is_open(), "Should fail once WB on the Frame is in place") + + # Closing the region fails due to `r1_ref` + self.assertFalse(r1.r2.try_close()) + # The open status was now updated + self.assertTrue(r1.r2.is_open()) + + # Remove the local reference + r2_ref = None + + # Closing the region should now succeed + self.assertTrue(r1.r2.try_close()) + self.assertFalse(r1.r2.is_open()) + + def test_try_close_with_bridge_ref_owned_by_cown(self): + r1 = Region("r1") + r1.a = self.A() + + # Create a local reference + r1_ref = r1 + + # r1 should be open here + self.assertTrue(r1.is_open()) + + # Create a new cown which isn't released yet + c = Cown(r1) + + # Closing the region fails due to `r1_ref` + self.assertFalse(c.get().try_close()) + + # Remove local references + r1 = None + r1_ref = None + + print("Checkout 3") + # Closing the region should now succeed + self.assertTrue(c.get().try_close()) + + print("Checkout 5") + # Check that the cown has been released + self.assertRaises(RegionError, lambda _ : c.get(), c) + + def test_try_close_with_contained_ref(self): + r1 = Region("r1") + r1.a = self.A() + + # The region is now open, since we added something to it + # (This is temporary, while the write barrier is not sufficient) + self.assertTrue(r1.is_open()) + + # Create a local reference + a = r1.a + + # Closing the region fails due to `a` + self.assertFalse(r1.try_close()) + self.assertTrue(r1.is_open()) + + # Remove the local reference + a = None + + # Closing the region should now succeed + self.assertTrue(r1.try_close()) + self.assertFalse(r1.is_open()) + + def test_try_close_sub_region_contained_ref(self): r1 = Region("r1") - - # This creates a reference into the region - r1.a = a + r1.a = self.A() + r1.a.r2 = Region("r2") + r1.a.r2.b = self.A() + + # The regions are now open, since we added something to them + self.assertTrue(r1.is_open()) + self.assertTrue(r1.a.r2.is_open()) + + # Create a local reference + b = r1.a.r2.b + + # Closing the regions fails due to `b` self.assertFalse(r1.try_close()) + self.assertTrue(r1.is_open()) + + # Remove the local reference + b = None + + # Closing the regions succeed now + self.assertTrue(r1.try_close()) + self.assertFalse(r1.is_open()) + + def test_try_close_sub_sub_region_contained_ref(self): + r1 = Region("region") + r1.r2 = Region("sub-region") + r1.r2.r3 = Region("sub-sub-region") + r1.r2.r3.a = self.A() + + # Create a local reference to a contained object + a = r1.r2.r3.a + + # The region is now open + self.assertTrue(r1.is_open()) + + # Closing the regions fails due to `a` + self.assertFalse(r1.try_close()) + self.assertTrue(r1.is_open()) + + # Kill the local reference + a = None + + # Closing the regions succeed now + self.assertTrue(r1.try_close()) + self.assertFalse(r1.is_open()) + + def test_try_close_sub_sub_region_bridge_ref(self): + r1 = Region("region") + r1.r2 = Region("sub-region") + r1.r2.r3 = Region("sub-sub-region") + r1.r2.r3.a = self.A() + + # The region is now open + self.assertTrue(r1.is_open()) + + # Closing r3 should succeeed and propagate to r1 + self.assertTrue(r1.r2.r3.try_close()) + self.assertFalse(r1.is_open()) + + # Create a local reference to a bridge + r3 = r1.r2.r3 + + # Manually open r2, while the WB is missing on attributes + r1.r2.a = self.A() + self.assertTrue(r1.r2.is_open()) + + # r3 is still marked as closed due to the missing WB on the frame + # This test should become irrelevant once the WB is in place and + # the open status is correctly tracked. + self.assertFalse(r1.r2.r3.is_open(), "Should fail once WB on the Frame is in place") + + # Closing the regions fails due to `r3` + self.assertFalse(r1.try_close()) + self.assertTrue(r1.is_open()) + + # Kill the local reference + r3 = None + + # Closing the regions succeed now + self.assertTrue(r1.try_close()) + self.assertFalse(r1.is_open()) + + def test_try_close_with_contained_cycle(self): + r1 = Region("r1") + r1.a = self.A() + r1.a.self = r1.a + r1.a.region = r1 + + # Create a local reference + a = r1.a + + # The region is now open + self.assertTrue(r1.is_open()) + + # Closing the regions fails due to `a` + self.assertFalse(r1.try_close()) + self.assertTrue(r1.is_open()) + + # Remove the local reference + a = None + + # Closing the regions succeed now + self.assertTrue(r1.try_close()) + self.assertFalse(r1.is_open()) + + def test_try_close_banish_unreachable_contained(self): + r1 = Region("r1") + r1.a = self.A() + + # Create a small tree we can later detach + b = self.A() + b.c = self.A() + r1.add_object(b) # TODO: Remove once the write barrier on objects works + r1.a.b = b + + # Check that r1 owns the objects now + self.assertTrue(r1.owns_object(b)) + self.assertTrue(r1.owns_object(b.c)) + + # Make `b` and `c` unreachable from the bridge + r1.a.b = None + + # `b` and `c` should remain members of r1 + self.assertTrue(r1.owns_object(b)) + self.assertTrue(r1.owns_object(b.c)) + + # Closing the regions should succeed but kickout `b` and `c` + self.assertTrue(r1.try_close()) + self.assertFalse(r1.is_open()) + self.assertFalse(r1.owns_object(b)) + self.assertFalse(r1.owns_object(b.c)) + + # A new region could now take ownership of `b` and `c` + r2 = Region("r2") + r2.c = b.c + self.assertTrue(r2.owns_object(b.c)) + + # Closing the regions fails since the local `b` points to `c` + self.assertFalse(r2.try_close()) + self.assertTrue(r2.is_open()) + + # Remove the local reference + b = None + + # Closing the regions succeed now + self.assertTrue(r2.try_close()) + self.assertFalse(r2.is_open()) + # This test will make the Python environment unusable. diff --git a/Objects/cown.c b/Objects/cown.c index b62df2ccb47f52..c498fb3e95e08d 100644 --- a/Objects/cown.c +++ b/Objects/cown.c @@ -170,7 +170,7 @@ int _PyCown_is_released(PyObject *self) { // The ignored argument is required for this function's type to be // compatible with PyCFunction static PyObject *PyCown_get(PyCownObject *self, PyObject *ignored) { - BAIL_UNLESS_ACQUIRED(self, "Attempt to get value of unacquired cown"); + BAIL_UNLESS_OWNED(self, "Attempt to get value of unacquired cown"); if (self->value) { return Py_NewRef(self->value); diff --git a/Objects/regions.c b/Objects/regions.c index a3bfb8e332ca29..fdb9f89b36cc85 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -107,6 +107,10 @@ struct regionmetadata { // The number of references to this object Py_ssize_t rc; bool is_open; + // Indicates if the LRC value can be trusted or not. + // + // FIXME: Only a single bit is needed, this can be integrated into another field + bool is_dirty; // This field might either point to the parent region or another region // that this one was merged into. The `Py_METADATA_MERGE_TAG` tag is used // to indicate this points to a merged region. @@ -145,7 +149,39 @@ static Py_region_ptr_t regionmetadata_get_merge_tree_root(Py_region_ptr_t self) #define regionmetadata_get_merge_tree_root(self) \ regionmetadata_get_merge_tree_root(REGION_PTR_CAST(self)) +__attribute__((unused)) +static void regionmetadata_mark_as_dirty(Py_region_ptr_t self_ptr) { + if (!HAS_METADATA(self_ptr)) { + return; + } + + REGION_DATA_CAST(self_ptr)->is_dirty = true; +} +# define regionmetadata_mark_as_dirty(data) \ + (regionmetadata_mark_as_dirty(REGION_PTR_CAST(data))) + +static void regionmetadata_mark_as_not_dirty(Py_region_ptr_t self_ptr) { + if (!HAS_METADATA(self_ptr)) { + return; + } + + REGION_DATA_CAST(self_ptr)->is_dirty = false; +} +# define regionmetadata_mark_as_not_dirty(data) \ + (regionmetadata_mark_as_not_dirty(REGION_PTR_CAST(data))) + +static bool regionmetadata_is_dirty(Py_region_ptr_t self_ptr) { + if (!HAS_METADATA(self_ptr)) { + return false; + } + + return REGION_DATA_CAST(self_ptr)->is_dirty; +} +# define regionmetadata_is_dirty(data) \ + (regionmetadata_is_dirty(REGION_PTR_CAST(data))) + static void regionmetadata_inc_osc(Py_region_ptr_t self_ptr); +static int regionmetadata_dec_osc(Py_region_ptr_t self_ptr); static void regionmetadata_open(regionmetadata* self) { assert(HAS_METADATA(self)); if (self->is_open) { @@ -155,8 +191,31 @@ static void regionmetadata_open(regionmetadata* self) { regionmetadata_inc_osc(REGION_PTR_CAST(regionmetadata_get_parent(self))); } -static void regionmetadata_close(regionmetadata* data) { - data->is_open = 0; +static int regionmetadata_close(regionmetadata* self) { + // The LRC might be 1 or 2, if the owning references is a local and the + // bridge object was used as an argument. + assert(self->lrc <= 2 && "Attempting to close a region with an LRC > 2"); + assert(self->osc == 0 && "Attempting to close a region with an OSC != 0"); + if (!self->is_open) { + return 0; + } + + self->is_open = false; + + Py_region_ptr_t parent = REGION_PTR_CAST(regionmetadata_get_parent(self)); + if (HAS_METADATA(parent)) { + // Cowns and parents are mutually exclusive this can therefore return directly + return regionmetadata_dec_osc(parent); + } + + // Check if in a cown -- if so, release cown + if (self->cown) { + // Propagate error from release + return _PyCown_release(self->cown); + } + + // Everything is a-okay + return 0; } static bool regionmetadata_is_open(Py_region_ptr_t self) { @@ -183,13 +242,21 @@ static void regionmetadata_inc_osc(Py_region_ptr_t self_ptr) #define regionmetadata_inc_osc(self) \ (regionmetadata_inc_osc(REGION_PTR_CAST(self))) -static void regionmetadata_dec_osc(Py_region_ptr_t self_ptr) +static int regionmetadata_dec_osc(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { - return; + return 0; + } + + regionmetadata* self = REGION_DATA_CAST(self_ptr); + self->osc -= 1; + + // Check if the OSC decrease has closed this region as well. + if (self->osc == 0 && self->lrc == 0 && !regionmetadata_is_dirty(self)) { + return regionmetadata_close(self); } - REGION_DATA_CAST(self_ptr)->osc -= 1; + return 0; } #define regionmetadata_dec_osc(self) \ (regionmetadata_dec_osc(REGION_PTR_CAST(self))) @@ -203,33 +270,38 @@ static void regionmetadata_inc_rc(Py_region_ptr_t self) #define regionmetadata_inc_rc(self) \ (regionmetadata_inc_rc(REGION_PTR_CAST(self))) -static void regionmetadata_dec_rc(Py_region_ptr_t self_ptr) +static int regionmetadata_dec_rc(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { - return; + return 0; } // Update RC regionmetadata* self = REGION_DATA_CAST(self_ptr); self->rc -= 1; if (self->rc != 0) { - return; + return 0; } // Sort out the funeral by informing everyone about the future freeing Py_CLEAR(self->name); + // Buffer the results since we don't want to leak any memory if this fails. + // OSC decreases in this function should also be safe. + int result = 0; if (regionmetadata_is_open(self)) { - regionmetadata_dec_osc(regionmetadata_get_parent(self)); + result |= regionmetadata_dec_osc(regionmetadata_get_parent(self)); } // This access the parent directly to update the rc. // It also doesn't matter if the parent pointer is a // merge or subregion relation, since both cases have // increased the rc. - regionmetadata_dec_rc(Py_region_ptr(self->parent)); + result |= regionmetadata_dec_rc(Py_region_ptr(self->parent)); free(self); + + return result; } #define regionmetadata_dec_rc(self) \ (regionmetadata_dec_rc(REGION_PTR_CAST(self))) @@ -345,19 +417,21 @@ static PyObject* regionmetadata_merge(regionmetadata* self, Py_region_ptr_t othe regionmetadata_inc_rc(other); - // Move LRC and OSC into the root. + // Merge state into the root. if (HAS_METADATA(other)) { - // Move information into the merge root regionmetadata* other_data = REGION_DATA_CAST(other); other_data->lrc += self->lrc; other_data->osc += self->osc; other_data->is_open |= self->is_open; - // remove information from self - self->lrc = 0; - self->osc = 0; - self->is_open = false; + other_data->is_dirty |= self->is_dirty; } + // remove information from self + self->lrc = 0; + self->osc = 0; + self->is_open = false; + self->is_dirty = false; + self->parent = Py_region_ptr_with_tags(other); REGION_PTR_SET_TAG(self->parent, Py_METADATA_MERGE_TAG); // No decref, since this is a weak reference. Otherwise we would get @@ -1304,6 +1378,9 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) Py_RETURN_NONE; } + // Mark the region as open, since we're adding stuff to it. + regionmetadata_open(region_data); + addtoregionvisitinfo info = { .pending = stack_new(), .new_sub_regions = NULL, @@ -1335,10 +1412,6 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) stack_free(info.pending); - if (region_data->lrc > 0 || region_data->osc > 0) { - regionmetadata_open(region_data); - } - Py_RETURN_NONE; } @@ -1356,8 +1429,7 @@ int _Py_is_bridge_object(PyObject *op) { return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); } -static PyObject *try_close(PyRegionObject *root_bridge) { - +static int try_close(PyRegionObject *root_bridge) { addtoregionvisitinfo info = { .pending = stack_new(), .new_sub_regions = stack_new(), @@ -1373,18 +1445,14 @@ static PyObject *try_close(PyRegionObject *root_bridge) { goto fail; } - // The *LRC* and *is_open* status is currently not updated when references - // to the bridge are created. This means that the bridge might have multiple - // unknown references. - // - // Using the LRC is an over approximation, since internal cycles from objects - // to the region object will also increase the RC thereby preventing it from - // closing. - // - // The root region is expected to have two references, one from the owning - // reference and one from the `self` argument - if (Py_REFCNT(root_bridge) > 2) { - regionmetadata_open(Py_REGION_DATA(root_bridge)); + // The root region can have to have two local references, one from the + // owning reference and one from the `self` argument + Py_ssize_t root_region_lrc_limit; + regionmetadata *root_data = Py_REGION_DATA(root_bridge); + if (regionmetadata_has_parent(root_data) || root_data->cown) { + root_region_lrc_limit = 1; + } else { + root_region_lrc_limit = 2; } while (!stack_empty(info.new_sub_regions)) { @@ -1392,15 +1460,32 @@ static PyObject *try_close(PyRegionObject *root_bridge) { assert(Py_is_bridge_object(bridge)); regionmetadata* old_data = Py_REGION_DATA(bridge); + // One from the owning reference + Py_ssize_t rc_limit = 1; + Py_ssize_t lrc_limit = 0; + + // The root bridge has different limits since it's currently used + // as an argument for this method. + if (bridge == _PyObject_CAST(root_bridge)) { + rc_limit += 1; + lrc_limit = root_region_lrc_limit; + } + + // The *LRC* and *is_open* status is currently not updated when references + // to the bridge are created. This means that the bridge might have multiple + // unknown references. + // + // Using the object RC is an over approximation, since internal cycles from + // objects to the bridge object will also increase the RC thereby tricking + // this check into opening it again. + if (Py_REFCNT(bridge) > rc_limit) { + regionmetadata_open(Py_REGION_DATA(bridge)); + } + // If it's closed there is nothing we need to do. if (!regionmetadata_is_open(old_data)) { continue; } - regionmetadata* parent = regionmetadata_get_parent(old_data); - PyObject* region_name = old_data->name; - Py_XINCREF(region_name); - regionmetadata_set_parent(old_data, NULL); - regionmetadata_merge(old_data, _Py_LOCAL_REGION); // Create the new `regionmetadata*` regionmetadata* new_data = (regionmetadata*)calloc(1, sizeof(regionmetadata)); @@ -1411,12 +1496,26 @@ static PyObject *try_close(PyRegionObject *root_bridge) { _Py_CAST(PyRegionObject *, bridge)->metadata = new_data; regionmetadata_inc_rc(new_data); new_data->bridge = _Py_CAST(PyRegionObject*, bridge); - new_data->name = region_name; - regionmetadata_set_parent(new_data, parent); + Py_XSETREF(new_data->name, old_data->name); + regionmetadata_open(new_data); + regionmetadata_set_parent(new_data, regionmetadata_get_parent(old_data)); + new_data->cown = old_data->cown; + old_data->cown = NULL; + + // Merge the old region data into local. This has to be done after the + // created of the `new_data` to prevent the parent from closing + // premeturely when the old data gets detached from it. + regionmetadata_set_parent(old_data, NULL); + regionmetadata_merge(old_data, _Py_LOCAL_REGION); - // -1 for the reference we just followed - new_data->lrc += Py_REFCNT(bridge) - 1; Py_SET_REGION(bridge, new_data); + new_data->lrc += Py_REFCNT(bridge); + // Only subtract 1 from the LRC if the reference comes from a parent. + // Owning references from the local region should still count towards + // the LRC. + if (regionmetadata_has_parent(new_data) || new_data->cown) { + new_data->lrc -= 1; + } if (stack_push(info.pending, bridge)) { goto fail; @@ -1434,29 +1533,33 @@ static PyObject *try_close(PyRegionObject *root_bridge) { } } - // Update the open status and make sure the parent knows - if (new_data->lrc != 0 || new_data->osc != 0) { - regionmetadata_open(new_data); - } + // Mark the region as clean + regionmetadata_mark_as_not_dirty(new_data); // The LRC will never decrease after this point. If the region is open // due to the LRC it will remain open and the close fails. - // - // FIXME: The root_bride has a higher allowed LRC - if (new_data->lrc != 0 && bridge != _PyObject_CAST(root_bridge)) { + if (new_data->lrc > lrc_limit) { break; } + + // Update the open status and make sure the parent knows + if (new_data->osc == 0) { + if (regionmetadata_close(new_data) != 0) { + goto fail; + } + } } - regionmetadata *root_data = Py_REGION_DATA(root_bridge); - Py_ssize_t lrc_limit = regionmetadata_has_parent(root_data) ? 1 : 2; - if (root_data->lrc <= lrc_limit && root_data->osc == 0) { - root_data->is_open = false; + root_data = Py_REGION_DATA(root_bridge); + if (root_data->lrc <= root_region_lrc_limit && root_data->osc == 0) { + if (regionmetadata_close(root_data) != 0) { + goto fail; + } } stack_free(info.pending); stack_free(info.new_sub_regions); - return PyBool_FromLong(_Py_CAST(long, !regionmetadata_is_open(root_data))); + return 0; fail: if (info.pending) { @@ -1465,7 +1568,7 @@ static PyObject *try_close(PyRegionObject *root_bridge) { if (info.new_sub_regions) { stack_free(info.new_sub_regions); } - return NULL; + return -1; } static void PyRegion_dealloc(PyRegionObject *self) { @@ -1578,26 +1681,35 @@ int _PyRegion_is_closed(PyObject* self) { // compatible with PyCFunction static PyObject *PyRegion_close(PyRegionObject *self, PyObject *ignored) { regionmetadata* const md = REGION_DATA_CAST(Py_REGION(self)); - if (regionmetadata_is_open(md)) { - regionmetadata_close(md); // Mark as closed - - // Check if in a cown -- if so, release cown - if (md->cown) { - if (_PyCown_release(md->cown) != 0) { - // Propagate error from release - return NULL; - } - } - Py_RETURN_NONE; // Return None (standard for methods with no return value) - } else { + if (!regionmetadata_is_open(md)) { Py_RETURN_NONE; // Double close is OK } + + // Attempt to close the region + if (try_close(self) != 0) { + return NULL; + } + + // Check if the region is now closed + if (regionmetadata_is_open(Py_REGION(self))) { + PyErr_Format(PyExc_RegionError, "Attempting to close the region failed"); + return NULL; + } + + // Return None (standard for methods with no return value) + Py_RETURN_NONE; } // try_close method (Attempts to close the region) static PyObject *PyRegion_try_close(PyRegionObject *self, PyObject *args) { assert(Py_is_bridge_object(self) && "self is not a bridge object"); - return try_close(self); + // Propagate potentual errors + if (try_close(self) != 0) { + return NULL; + } + + // Check if the region was closed + return PyBool_FromLong(_Py_CAST(long, !regionmetadata_is_open(Py_REGION(self)))); } // Adds args object to self region @@ -1660,6 +1772,7 @@ static PyMethodDef PyRegion_methods[] = { {"open", (PyCFunction)PyRegion_open, METH_NOARGS, "Open the region."}, {"close", (PyCFunction)PyRegion_close, METH_NOARGS, "Attempt to close the region."}, {"is_open", (PyCFunction)PyRegion_is_open, METH_NOARGS, "Check if the region is open."}, + {"try_close", (PyCFunction)PyRegion_try_close, METH_NOARGS, "Attempt to close the region."}, // Temporary methods for testing. These will be removed or at least renamed once // the write barrier is done. {"add_object", (PyCFunction)PyRegion_add_object, METH_O, "Add object to the region."}, @@ -1717,7 +1830,7 @@ void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg) { const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; PyObject *new_ref_type_repr = PyObject_Repr(PyObject_Type(new_ref)); const char *new_ref_desc = new_ref_type_repr ? PyUnicode_AsUTF8(new_ref_type_repr) : "<>"; - // PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", tgt, tgt_desc, tgt_region_name, new_ref, new_ref_desc, new_ref_region_name, msg); + PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", tgt, tgt_desc, tgt_region_name, new_ref, new_ref_desc, new_ref_region_name, msg); } static const char *get_region_name(PyObject* obj) { From 1cd1408386d13a66a4ab3f9dd06754c115af7f03 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 29 Nov 2024 15:55:03 +0000 Subject: [PATCH 44/68] Add a flag for region awareness of a type. This will be used to specify that a type correctly performs region topology updates. --- Include/cpython/object.h | 2 +- Include/object.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ae7f780a93182a..413e44f28ec955 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -176,7 +176,7 @@ struct _typeobject { PyBufferProcs *tp_as_buffer; /* Flags to define presence of optional/expanded features */ - unsigned long tp_flags; + uint64_t tp_flags; // Made flags 64 bit to support region flags. const char *tp_doc; /* Documentation string */ diff --git a/Include/object.h b/Include/object.h index e6909cdd8ff3b1..51c981ab7dd0b9 100644 --- a/Include/object.h +++ b/Include/object.h @@ -581,6 +581,10 @@ given type object has a specified feature. #define Py_TPFLAGS_BASE_EXC_SUBCLASS (1UL << 30) #define Py_TPFLAGS_TYPE_SUBCLASS (1UL << 31) +/* Used to indicate that a type is aware of the region model, and + can be trusted to correctly modify the region topology.*/ +#define Py_TPFLAGS_REGION_AWARE (1UL << 32) + #define Py_TPFLAGS_DEFAULT ( \ Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \ 0) From e764b13070ec686e1d112547b71a3c583849e557 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Wed, 8 Jan 2025 15:34:17 +0100 Subject: [PATCH 45/68] Address MJP's comments (and add several comments) --- Objects/regions.c | 71 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index fdb9f89b36cc85..b1451e59f6647e 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -149,7 +149,6 @@ static Py_region_ptr_t regionmetadata_get_merge_tree_root(Py_region_ptr_t self) #define regionmetadata_get_merge_tree_root(self) \ regionmetadata_get_merge_tree_root(REGION_PTR_CAST(self)) -__attribute__((unused)) static void regionmetadata_mark_as_dirty(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { return; @@ -191,6 +190,12 @@ static void regionmetadata_open(regionmetadata* self) { regionmetadata_inc_osc(REGION_PTR_CAST(regionmetadata_get_parent(self))); } +/// This function marks the region as closed and propagartes the status to +/// the parent region or owning cown. +/// +/// It returns `0` if the close was successful. It should only fails, if the +/// system is in an inconsistent state and this close attempted to release a +/// cown which is currently not owned by the current thread. static int regionmetadata_close(regionmetadata* self) { // The LRC might be 1 or 2, if the owning references is a local and the // bridge object was used as an argument. @@ -242,6 +247,11 @@ static void regionmetadata_inc_osc(Py_region_ptr_t self_ptr) #define regionmetadata_inc_osc(self) \ (regionmetadata_inc_osc(REGION_PTR_CAST(self))) +/// Decrements the OSC of the region. This might close the region if the LRC +/// and ORC both hit zero and the region is not marked as dirty. +/// +/// Returns `0` on success. An error might come from closing the region +/// see `regionmetadata_close` for potential errors. static int regionmetadata_dec_osc(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { @@ -1221,6 +1231,13 @@ typedef struct addtoregionvisitinfo { PyObject* src; } addtoregionvisitinfo; +/// Adds the `target` object to the region of the `src` object stored +/// in the `addtoregionvisitinfo*` instance provided via `info_void`. +/// +/// This function can fail: +/// - If no memory is available to push nodes on the stacks of +/// `addtoregionvisitinfo`. +/// - If it's not possible to add the object to the region. static int _add_to_region_visit(PyObject* target, void* info_void) { addtoregionvisitinfo *info = _Py_CAST(addtoregionvisitinfo *, info_void); @@ -1328,7 +1345,11 @@ static int _add_to_region_visit(PyObject* target, void* info_void) } // This function visits all outgoing reference from `item` including the -// type. It will return `false` if the operation failed. +// type. +// +// It will return `false` if the given `visit` function fails. +// (Or if it's called on a function, this is a limitation of the current +// implementation which should be lifted soon-ish) static int visit_object(PyObject *item, visitproc visit, void* info) { if (PyFunction_Check(item)) { // FIXME: This is a temporary error. It should be replaced by @@ -1429,6 +1450,19 @@ int _Py_is_bridge_object(PyObject *op) { return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); } +/// This function attempts to close a region. It does this, by first merging +/// it into the local region and then reconstructing the region from the +/// given bridge object. All reachable objects will be added to the region, +/// similar to how `add_to_region` works. +/// +/// This function will also attempt to close open subregions, as that's +/// needed to close the given region. Closed subregions will remain closed +/// if possible. +/// +/// This function returns `-1` if any errors occurred. This can be due to +/// memory problems, region errors or problems with releasing cowns not owned +/// by the current thread. `0` only indicates that the function didn't error. +/// `regionmetadata_is_open()` should be used to check the region status. static int try_close(PyRegionObject *root_bridge) { addtoregionvisitinfo info = { .pending = stack_new(), @@ -1493,9 +1527,20 @@ static int try_close(PyRegionObject *root_bridge) { PyErr_NoMemory(); goto fail; } - _Py_CAST(PyRegionObject *, bridge)->metadata = new_data; + + PyRegionObject* bridge_obj = _Py_CAST(PyRegionObject *, bridge); + // Increase the RC for the reference given to the `metadata` field of the + // `PyRegionObject` object. + // + // The RC of the old value is directly decreased. The RC of `old_data` + // will remain `>= 1` until the region field of the bridge object is + // updated by `Py_SET_REGION(bridge, new_data);` This ensures that + // `old_data` stays valid while all the data is transferred to `new_data` + regionmetadata_dec_rc(bridge_obj->metadata); + bridge_obj->metadata = new_data; regionmetadata_inc_rc(new_data); - new_data->bridge = _Py_CAST(PyRegionObject*, bridge); + + new_data->bridge = bridge_obj; Py_XSETREF(new_data->name, old_data->name); regionmetadata_open(new_data); regionmetadata_set_parent(new_data, regionmetadata_get_parent(old_data)); @@ -1507,7 +1552,11 @@ static int try_close(PyRegionObject *root_bridge) { // premeturely when the old data gets detached from it. regionmetadata_set_parent(old_data, NULL); regionmetadata_merge(old_data, _Py_LOCAL_REGION); + old_data = NULL; + // This region update also triggers an RC decrease on `old_data`. + // afterwards it might be deallocated. This has to happen after + // all data has been transferred. Py_SET_REGION(bridge, new_data); new_data->lrc += Py_REFCNT(bridge); // Only subtract 1 from the LRC if the reference comes from a parent. @@ -1518,6 +1567,9 @@ static int try_close(PyRegionObject *root_bridge) { } if (stack_push(info.pending, bridge)) { + // No more memory, make sure the region is marked as dirty thereby + // preventing it from being closed in an inconsitent state. + regionmetadata_mark_as_dirty(Py_REGION_DATA(root_bridge)); goto fail; } @@ -1529,6 +1581,11 @@ static int try_close(PyRegionObject *root_bridge) { info.src = item; if (!visit_object(item, (visitproc)_add_to_region_visit, &info)) { + // The system is out of memory, or an object couldn't be added + // to the region. + // + // Either way, this means that the LRC of the region can't be trusted. + regionmetadata_mark_as_dirty(Py_REGION_DATA(root_bridge)); goto fail; } } @@ -1545,6 +1602,9 @@ static int try_close(PyRegionObject *root_bridge) { // Update the open status and make sure the parent knows if (new_data->osc == 0) { if (regionmetadata_close(new_data) != 0) { + // See `regionmetadata_close` for when this can fail. + // In either case, this region has just been cleaned and should + // be in a consistent state. goto fail; } } @@ -1553,6 +1613,9 @@ static int try_close(PyRegionObject *root_bridge) { root_data = Py_REGION_DATA(root_bridge); if (root_data->lrc <= root_region_lrc_limit && root_data->osc == 0) { if (regionmetadata_close(root_data) != 0) { + // See `regionmetadata_close` for when this can fail. + // In either case, this region has just been cleaned and should + // be in a consistent state. goto fail; } } From eb88bba301341cb57c7cb85c52b4d3eaf28a0818 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Thu, 16 Jan 2025 08:23:04 +0100 Subject: [PATCH 46/68] Fix crash bug when invalid values are used to initiate a cown --- Lib/test/test_using.py | 4 ++++ Objects/cown.c | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index 671d69d1d6008e..987bc64a4679a5 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -103,3 +103,7 @@ def test_region_cown_ptr(self): r = Region() r.f = Cown() self.assertTrue(True) + + def test_invalid_cown_init(self): + # Create cown with invalid init value + self.assertRaises(RegionError, Cown, [42]) diff --git a/Objects/cown.c b/Objects/cown.c index c498fb3e95e08d..8a126bd8d3629c 100644 --- a/Objects/cown.c +++ b/Objects/cown.c @@ -73,7 +73,9 @@ static int PyCown_init(PyCownObject *self, PyObject *args, PyObject *kwds) { } if (value) { - PyCown_set_unchecked(self, value); + PyObject* result = PyCown_set_unchecked(self, value); + // Propagate errors from set_unchecked + if (result == NULL) return -1; } else { _Py_atomic_store(&self->state, Cown_RELEASED); @@ -215,7 +217,7 @@ static PyObject *PyCown_set_unchecked(PyCownObject *self, PyObject *arg) { return old ? old : Py_None; } else { // Invalid cown content - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_RegionError, "Cowns can only store bridge objects, immutable objects or other cowns!"); return NULL; } From 01a25863242db1b7f3932310d0042c0702c6fdb6 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Mon, 20 Jan 2025 21:45:28 +0000 Subject: [PATCH 47/68] Add WriteBarrier to dictobject (#34) This implements the addreference and remove reference logic for the dictionary. Adds a Py_CLEAR_OBJECT_FIELD to remove the reference in the correct point. --- Include/internal/pycore_regions.h | 42 ++++++++++++-- Lib/test/test_regions_dictobject.py | 80 +++++++++++++++++++++++++++ Objects/dictobject.c | 86 ++++++++++++++++++++++------- Objects/object.c | 4 +- Objects/regions.c | 73 ++++++++++++++++++++---- 5 files changed, 247 insertions(+), 38 deletions(-) create mode 100644 Lib/test/test_regions_dictobject.py diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 6b363abd49423d..aeb06bf517afc8 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -57,15 +57,21 @@ PyObject* _Py_ResetInvariant(void); #define Py_ResetInvariant() _Py_ResetInvariant() // Invariant placeholder -bool _Pyrona_AddReference(PyObject* target, PyObject* new_ref); -#define Pyrona_ADDREFERENCE(a, b) _Pyrona_AddReference(a, b) -#define Pyrona_REMOVEREFERENCE(a, b) // TODO +bool _Py_RegionAddReference(PyObject* src, PyObject* new_tgt); +#define Py_REGIONADDREFERENCE(a, b) _Py_RegionAddReference(_PyObject_CAST(a), b) + +void _Py_RegionAddLocalReference(PyObject* new_tgt); +#define Py_REGIONADDLOCALREFERENCE(b) _Py_RegionAddLocalReference(b) + // Helper macros to count the number of arguments #define _COUNT_ARGS(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, N, ...) N #define COUNT_ARGS(...) _COUNT_ARGS(__VA_ARGS__, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1) -bool _Pyrona_AddReferences(PyObject* target, int new_refc, ...); -#define Pyrona_ADDREFERENCES(a, ...) _Pyrona_AddReferences(a, COUNT_ARGS(__VA_ARGS__), __VA_ARGS__) +bool _Py_RegionAddReferences(PyObject* src, int new_tgtc, ...); +#define Py_REGIONADDREFERENCES(a, ...) _Py_RegionAddReferences(_PyObject_CAST(a), COUNT_ARGS(__VA_ARGS__), __VA_ARGS__) + +void _Py_RegionRemoveReference(PyObject* src, PyObject* old_tgt); +#define Py_REGIONREMOVEREFERENCE(a, b) _Py_RegionRemoveReference(_PyObject_CAST(a), b) #ifdef NDEBUG #define _Py_VPYDBG(fmt, ...) @@ -83,6 +89,32 @@ int _PyRegion_is_closed(PyObject* region); int _PyCown_release(PyObject *self); int _PyCown_is_released(PyObject *self); + +#ifdef _Py_TYPEOF +#define Py_CLEAR_OBJECT_FIELD(op, field) \ + do { \ + _Py_TYPEOF(op)* _tmp_field_ptr = &(field); \ + _Py_TYPEOF(op) _tmp_old_field = (*_tmp_field_ptr); \ + if (_tmp_old_field != NULL) { \ + *_tmp_field_ptr = _Py_NULL; \ + Py_REGIONREMOVEREFERENCE(op, _tmp_old_field); \ + Py_DECREF(_tmp_old_field); \ + } \ + } while (0) +#else +#define Py_CLEAR_OBJECT_FIELD(op, field) \ + do { \ + PyObject **_tmp_field_ptr = _Py_CAST(PyObject**, &(op)); \ + PyObject *_tmp_old_field = (*_tmp_field_ptr); \ + if (_tmp_old_field != NULL) { \ + PyObject *_null_ptr = _Py_NULL; \ + memcpy(_tmp_field_ptr, &_null_ptr, sizeof(PyObject*)); \ + Py_REGIONREMOVEREFERENCE(op, _tmp_old_field); \ + Py_DECREF(_tmp_old_field); \ + } \ + } while (0) +#endif + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_regions_dictobject.py b/Lib/test/test_regions_dictobject.py new file mode 100644 index 00000000000000..d2d4164b34a670 --- /dev/null +++ b/Lib/test/test_regions_dictobject.py @@ -0,0 +1,80 @@ +import unittest + +class TestRegionsDictObject(unittest.TestCase): + def setUp(self): + enableinvariant() + + def test_dict_insert_empty_dict(self): + # Create Region with Empty dictionary + r = Region() + d = {} + r.body = d + n = {} + # Add local object to region + d["foo"] = n + self.assertTrue(r.owns_object(n)) + + def test_dict_insert_nonempty_dict(self): + # Create Region with Nonempty dictionary + r = Region() + d = {} + d["bar"] = 1 + r.body = d + # Add local object to region + n = {} + d["foo"] = n + self.assertTrue(r.owns_object(n)) + + def test_dict_update_dict(self): + # Create Region with Nonempty dictionary + r = Region() + d = {} + n1 = {} + d["foo"] = n1 + r.body = d + # Update dictionary to contain a local object + n2 = {} + d["foo"] = n2 + self.assertTrue(r.owns_object(n2)) + + def test_dict_clear(self): + # Create Region with Nonempty dictionary + r = Region() + d = {} + n = {} + d["foo"] = n + r.body = d + # Clear dictionary + d.clear() + # As LRC is not checked by the invariant, this test cannot + # check anything useful yet. + + def test_dict_copy(self): + r = Region() + d = {} + r.body = d + r2 = Region() + d["foo"] = r2 + d.copy() + + def test_dict_setdefault(self): + r = Region("outer") + d = {} + r.body = d + r2 = Region("inner") + d["foo"] = r2 + d.setdefault("foo", r2) + self.assertRaises(RegionError, d.setdefault, "bar", r2) + + def test_dict_update(self): + # Create a region containing two dictionaries + r = Region() + d = {} + r.body = d + d2 = {} + r.body2 = d2 + # Add a contained region to the first dictionary + d["reg"] = Region() + # Update the second dictionary to contain the elements of the first + self.assertRaises(RegionError, d2.update, d) + self.assertRaises(RegionError, d2.update, d) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index c1084ca1a3ca89..5556795b1d919b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -122,6 +122,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_regions.h" // _PyObject_GC_TRACK() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_regions.h" // Py_ADDREGIONREFERENCE(), ... (region) #include "stringlib/eq.h" // unicode_eq() #include "regions.h" // Py_IsImmutable() @@ -666,6 +667,10 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) { + // TODO: This feels like it should remove the references in the regions + // but keys is not a Python object, so it's not clear how to do that. + // mjp: Leaving as a TODO for now. + assert(keys != Py_EMPTY_KEYS); if (DK_IS_UNICODE(keys)) { PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); @@ -788,8 +793,12 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) } +/* The target represents the dictionary that this object will become part of. + If target is NULL, the object is not part of a freshly allocated dictionary, so should + be considered as part of te local region. +*/ static PyDictKeysObject * -clone_combined_dict_keys(PyDictObject *orig) +clone_combined_dict_keys(PyDictObject *orig, PyObject* target) { assert(PyDict_Check(orig)); assert(Py_TYPE(orig)->tp_iter == (getiterfunc)dict_iter); @@ -830,6 +839,14 @@ clone_combined_dict_keys(PyDictObject *orig) if (value != NULL) { Py_INCREF(value); Py_INCREF(*pkey); + if (target != NULL) { + if (!Py_REGIONADDREFERENCES(target, *pkey, value)) + return NULL; + } + else { + Py_REGIONADDLOCALREFERENCE(*pkey); + Py_REGIONADDLOCALREFERENCE(value); + } } pvalue += offs; pkey += offs; @@ -1258,6 +1275,9 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, MAINTAIN_TRACKING(mp, key, value); if (ix == DKIX_EMPTY) { + if (!Py_REGIONADDREFERENCES((PyObject*)mp, key, value)) { + goto Fail; + } uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, value); /* Insert into new slot. */ @@ -1303,6 +1323,9 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } if (old_value != value) { + if (!Py_REGIONADDREFERENCE((PyObject*)mp, value)) { + goto Fail; + } if(DK_IS_UNICODE(mp->ma_keys)){ PyDictUnicodeEntry *ep; ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; @@ -1336,6 +1359,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, else { _PyDictEntry_SetValue(DK_ENTRIES(mp->ma_keys) + ix, value); } + Py_REGIONREMOVEREFERENCE((PyObject*)mp, old_value); } mp->ma_version_tag = new_version; } @@ -1365,6 +1389,9 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, return -1; } + if (!Py_REGIONADDREFERENCES((PyObject*)mp, key, value)) + return -1; + uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, value); @@ -2015,6 +2042,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, assert(ix < SHARED_KEYS_MAX_SIZE); /* Update order */ delete_index_from_values(mp->ma_values, ix); + Py_REGIONREMOVEREFERENCE(mp, old_value); ASSERT_CONSISTENT(mp); } else { @@ -2181,7 +2209,7 @@ PyDict_Clear(PyObject *op) if (oldvalues != NULL) { n = oldkeys->dk_nentries; for (i = 0; i < n; i++) - Py_CLEAR(oldvalues->values[i]); + Py_CLEAR_OBJECT_FIELD(op, oldvalues->values[i]); free_values(oldvalues); dictkeys_decref(interp, oldkeys); } @@ -2459,6 +2487,8 @@ dict_dealloc(PyDictObject *mp) Py_TRASHCAN_BEGIN(mp, dict_dealloc) if (values != NULL) { for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) { + PyObject *value = values->values[i]; + Py_REGIONREMOVEREFERENCE(mp, value); Py_XDECREF(values->values[i]); } free_values(values); @@ -2939,7 +2969,7 @@ dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_CLONED, mp, b, NULL); - PyDictKeysObject *keys = clone_combined_dict_keys(other); + PyDictKeysObject *keys = clone_combined_dict_keys(other, a); // Need to say what owns the keys? if (keys == NULL) { return -1; } @@ -3142,6 +3172,13 @@ PyDict_Copy(PyObject *o) dictkeys_incref(mp->ma_keys); for (size_t i = 0; i < size; i++) { PyObject *value = mp->ma_values->values[i]; + if (!Py_REGIONADDREFERENCE(split_copy, value)) + { + // TODO: is this safe to dealloc the split_copy? + // is it in a valid enough state to be deallocated? + Py_DECREF(split_copy); + return NULL; + } split_copy->ma_values->values[i] = Py_XNewRef(value); } if (_PyObject_GC_IS_TRACKED(mp)) @@ -3167,7 +3204,7 @@ PyDict_Copy(PyObject *o) operations and copied after that. In cases like this, we defer to PyDict_Merge, which produces a compacted copy. */ - PyDictKeysObject *keys = clone_combined_dict_keys(mp); + PyDictKeysObject *keys = clone_combined_dict_keys(mp, NULL); if (keys == NULL) { return NULL; } @@ -3427,6 +3464,8 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; if (ix == DKIX_EMPTY) { + if (!Py_REGIONADDREFERENCE(mp, defaultobj)) + return NULL; uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, defaultobj); mp->ma_keys->dk_version = 0; @@ -3467,6 +3506,8 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) assert(mp->ma_keys->dk_usable >= 0); } else if (value == NULL) { + if (!Py_REGIONADDREFERENCE(mp, defaultobj)) + return NULL; uint64_t new_version = _PyDict_NotifyEvent( interp, PyDict_EVENT_ADDED, mp, key, defaultobj); value = defaultobj; @@ -5507,7 +5548,7 @@ _PyObject_InitializeDict(PyObject *obj) return -1; } PyObject **dictptr = _PyObject_ComputedDictPointer(obj); - Pyrona_ADDREFERENCE(obj, dict); + Py_REGIONADDREFERENCE(obj, dict); *dictptr = dict; return 0; } @@ -5553,11 +5594,20 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, assert(values != NULL); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - if(!Py_CHECKWRITE(obj)){ + if (!Py_CHECKWRITE(obj)){ PyErr_WriteToImmutable(obj); return -1; } + //TODO: PYRONA: The addition of the key is complex here. + // The keys PyDictKeysObject, might already have the key. Note that + // the keys PyDictKeysObject is not a PyObject. So it is unclear where + // this edge is created. + // The keys is coming from ht_cached_keys on the type object. + // This is also interesting from a race condition perspective. + // Can this be shared, should it be treated immutably when the type is? + // mjp: Leaving for a future PR. + Py_ssize_t ix = DKIX_EMPTY; if (PyUnicode_CheckExact(name)) { ix = insert_into_dictkeys(keys, name); @@ -5589,6 +5639,9 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, return PyDict_SetItem(dict, name, value); } } + if (!Py_REGIONADDREFERENCE(obj, value)) { + return -1; + } PyObject *old_value = values->values[ix]; values->values[ix] = Py_XNewRef(value); if (old_value == NULL) { @@ -5604,6 +5657,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, if (value == NULL) { delete_index_from_values(values, ix); } + Py_REGIONREMOVEREFERENCE(obj, old_value); Py_DECREF(old_value); } return 0; @@ -5775,7 +5829,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) _Py_SetImmutable(dict); } else { - Pyrona_ADDREFERENCE(obj, dict); + Py_REGIONADDREFERENCE(obj, dict); dorv_ptr->dict = dict; } } @@ -5789,7 +5843,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) _Py_SetImmutable(dict); } else { - Pyrona_ADDREFERENCE(obj, dict); + Py_REGIONADDREFERENCE(obj, dict); dorv_ptr->dict = dict; } } @@ -5817,7 +5871,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) _Py_SetImmutable(dict); } else { - Pyrona_ADDREFERENCE(obj, dict); + Py_REGIONADDREFERENCE(obj, dict); *dictptr = dict; } } @@ -5841,7 +5895,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, if (dict == NULL) { dictkeys_incref(cached); dict = new_dict_with_shared_keys(interp, cached); - Pyrona_ADDREFERENCE(owner, dict); + Py_REGIONADDREFERENCE(owner, dict); if (dict == NULL) return -1; *dictptr = dict; @@ -5850,7 +5904,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, res = PyDict_DelItem(dict, key); } else { - if (Pyrona_ADDREFERENCES(dict, key, value)) { + if (Py_REGIONADDREFERENCES(dict, key, value)) { res = PyDict_SetItem(dict, key, value); } else { // Error is set inside ADDREFERENCE @@ -5863,19 +5917,13 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, dict = PyDict_New(); if (dict == NULL) return -1; - Pyrona_ADDREFERENCE(owner, dict); + Py_REGIONADDREFERENCE(owner, dict); *dictptr = dict; } if (value == NULL) { res = PyDict_DelItem(dict, key); } else { - // TODO: remove this once we merge Matt P's changeset to dictionary object - if (Pyrona_ADDREFERENCES(dict, key, value)) { - res = PyDict_SetItem(dict, key, value); - } else { - // Error is set inside ADDREFERENCE - return -1; - } + res = PyDict_SetItem(dict, key, value); } } ASSERT_CONSISTENT(dict); diff --git a/Objects/object.c b/Objects/object.c index b9ce043f6e4140..a9813a999fdb62 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1267,7 +1267,7 @@ _PyObject_GetDictPtr(PyObject *obj) PyErr_Clear(); return NULL; } - Pyrona_ADDREFERENCE(obj, dict); + Py_REGIONADDREFERENCE(obj, dict); dorv_ptr->dict = dict; } return &dorv_ptr->dict; @@ -1467,7 +1467,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, res = NULL; goto done; } - Pyrona_ADDREFERENCE(obj, dict); + Py_REGIONADDREFERENCE(obj, dict); dorv_ptr->dict = dict; } } diff --git a/Objects/regions.c b/Objects/regions.c index b1451e59f6647e..7f138973965bd3 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -44,7 +44,7 @@ static regionmetadata* regionmetadata_get_parent(regionmetadata* self); static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); static const char *get_region_name(PyObject* obj); -static void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg); +static void _PyErr_Region(PyObject *src, PyObject *tgt, const char *msg); /** * Global status for performing the region check. @@ -1886,14 +1886,14 @@ PyTypeObject PyRegion_Type = { PyType_GenericNew, /* tp_new */ }; -void _PyErr_Region(PyObject *tgt, PyObject *new_ref, const char *msg) { - const char *new_ref_region_name = get_region_name(new_ref); +void _PyErr_Region(PyObject *src, PyObject *tgt, const char *msg) { const char *tgt_region_name = get_region_name(tgt); + const char *src_region_name = get_region_name(src); + PyObject *src_type_repr = PyObject_Repr(PyObject_Type(src)); + const char *src_desc = src_type_repr ? PyUnicode_AsUTF8(src_type_repr) : "<>"; PyObject *tgt_type_repr = PyObject_Repr(PyObject_Type(tgt)); const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; - PyObject *new_ref_type_repr = PyObject_Repr(PyObject_Type(new_ref)); - const char *new_ref_desc = new_ref_type_repr ? PyUnicode_AsUTF8(new_ref_type_repr) : "<>"; - PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", tgt, tgt_desc, tgt_region_name, new_ref, new_ref_desc, new_ref_region_name, msg); + PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", src, src_desc, src_region_name, tgt, tgt_desc, tgt_region_name, msg); } static const char *get_region_name(PyObject* obj) { @@ -1912,7 +1912,7 @@ static const char *get_region_name(PyObject* obj) { } // x.f = y ==> _Pyrona_AddReference(src=x, tgt=y) -bool _Pyrona_AddReference(PyObject *src, PyObject *tgt) { +bool _Py_RegionAddReference(PyObject *src, PyObject *tgt) { if (Py_REGION(src) == Py_REGION(tgt)) { // Nothing to do -- intra-region references are always permitted return true; @@ -1924,8 +1924,9 @@ bool _Pyrona_AddReference(PyObject *src, PyObject *tgt) { } if (_Py_IsLocal(src)) { - // Slurp emphemerally owned object into the region of the target object + // Record the borrowed reference in the LRC of the target region // _Py_VPYDBG("Added borrowed ref %p --> %p (owner: '%s')\n", tgt, new_ref, get_region_name(tgt)); + Py_REGION_DATA(tgt)->lrc += 1; return true; } @@ -1934,13 +1935,25 @@ bool _Pyrona_AddReference(PyObject *src, PyObject *tgt) { return add_to_region(tgt, Py_REGION(src)) == Py_None; } +// Used to add a reference from a local object that might not have been created yet +// to tgt. +void _Py_RegionAddLocalReference(PyObject *tgt) { + // Only need to increment the LRC of the target region + // if it is not local, immutable, or a cown. + if (_Py_IsLocal(tgt) || Py_IsImmutable(tgt) || _Py_IsCown(tgt)) { + return; + } + + Py_REGION_DATA(tgt)->lrc += 1; +} + // Convenience function for moving multiple references into tgt at once -bool _Pyrona_AddReferences(PyObject *tgt, int new_refc, ...) { +bool _Py_RegionAddReferences(PyObject *src, int tgtc, ...) { va_list args; - va_start(args, new_refc); + va_start(args, tgtc); - for (int i = 0; i < new_refc; i++) { - int res = _Pyrona_AddReference(tgt, va_arg(args, PyObject*)); + for (int i = 0; i < tgtc; i++) { + int res = _Py_RegionAddReference(src, va_arg(args, PyObject*)); if (!res) return false; } @@ -1954,3 +1967,39 @@ void _PyRegion_set_cown_parent(PyObject* bridge, PyObject* cown) { Py_XINCREF(cown); Py_XSETREF(data->cown, cown); } + +void _Py_RegionRemoveReference(PyObject *src, PyObject *tgt) { + if (Py_REGION(src) == Py_REGION(tgt)) { + // Nothing to do -- intra-region references have no accounting. + return; + } + + // If the target is local, then so must the source be. So this should + // be covered by the previous check. + assert(!_Py_IsLocal(tgt)); + + if (_Py_IsImmutable(tgt) || _Py_IsCown(tgt)) { + // Nothing to do -- removing a ref to an immutable or a cown has no additional accounting. + return; + } + + regionmetadata* tgt_md = Py_REGION_DATA(tgt); + if (_Py_IsLocal(src)) { + // Dec LRC of the previously referenced region + // TODO should this decrement be a function, if it hits zero, + // then a region could become unreachable. + tgt_md->lrc -= 1; + return; + } + + // This must be a parent reference, so we need to remove the parent reference. + regionmetadata* src_md = Py_REGION_DATA(src); + regionmetadata* tgt_parent_md = REGION_DATA_CAST(Py_region_ptr(tgt_md->parent)); + if (tgt_parent_md != src_md) { + // TODO: Could `dirty` mean this isn't an error? + _PyErr_Region(src, tgt, "(in WB/remove_ref)"); + } + + // Unparent the region. + regionmetadata_set_parent(tgt_md, NULL); +} From 140e27f1de11a9466180eb543e6eca1d780eef5a Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 21 Jan 2025 14:43:02 +0000 Subject: [PATCH 48/68] Make region invariant a compile option (#41) This makes the region invariant off by default, and requires configure to be passed: --with-region-invariant This then enables the invariant to run on every byte code instruction. --- .github/workflows/build_min.yml | 4 +++- Python/ceval_gil.c | 2 ++ Python/ceval_macros.h | 10 ++++++++++ configure | 27 +++++++++++++++++++++++++++ configure.ac | 15 +++++++++++++++ pyconfig.h.in | 3 +++ 6 files changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_min.yml b/.github/workflows/build_min.yml index 0385280d8aa00a..a7b8d9449823e6 100644 --- a/.github/workflows/build_min.yml +++ b/.github/workflows/build_min.yml @@ -337,6 +337,7 @@ jobs: ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ + --with-region-invariant \ --with-openssl=$OPENSSL_DIR - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} @@ -460,6 +461,7 @@ jobs: ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ + --with-region-invariant \ --with-openssl=$OPENSSL_DIR - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} @@ -561,7 +563,7 @@ jobs: - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc --with-region-invariant - name: Build CPython run: make -j4 - name: Display build info diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 4003553a7aae9c..6f448dcf75b0c5 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1057,10 +1057,12 @@ _Py_HandlePending(PyThreadState *tstate) struct _ceval_runtime_state *ceval = &runtime->ceval; struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; +#ifdef Py_REGION_INVARIANT /* Check the region invariant if required. */ if (_Py_CheckRegionInvariant(tstate) != 0) { return -1; } +#endif /* Pending signals */ if (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 3cf4b8cf6b1322..3029cd340a0d42 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -88,6 +88,7 @@ #endif +#ifdef Py_REGION_INVARIANT /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ { \ @@ -97,6 +98,15 @@ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } +#else +/* Do interpreter dispatch accounting for tracing and instrumentation */ +#define DISPATCH() \ + { \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ + } +#endif #define DISPATCH_SAME_OPARG() \ { \ diff --git a/configure b/configure index b6f90bcd8c7300..af5d2e2b75fd02 100755 --- a/configure +++ b/configure @@ -1078,6 +1078,7 @@ enable_shared with_static_libpython enable_profiling with_pydebug +with_region_invariant with_trace_refs enable_pystats with_assertions @@ -1850,6 +1851,8 @@ Optional Packages: do not build libpythonMAJOR.MINOR.a and do not install python.o (default is yes) --with-pydebug build with Py_DEBUG defined (default is no) + --with-region-invariant enable region invariant for debugging purpose + (default is no) --with-trace-refs enable tracing references for debugging purpose (default is no) --with-assertions build with C assertions enabled (default is no) @@ -8075,6 +8078,30 @@ printf "%s\n" "no" >&6; } fi +# Check for --with-region-invariant +# --with-region-invariant +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-region-invariant" >&5 +printf %s "checking for --with-region-invariant... " >&6; } + +# Check whether --with-region-invariant was given. +if test ${with_region_invariant+y} +then : + withval=$with_region_invariant; +else $as_nop + with_region_invariant=yes + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_region_invariant" >&5 +printf "%s\n" "$with_region_invariant" >&6; } + +if test "$with_region_invariant" = "yes" +then + +printf "%s\n" "#define Py_REGION_INVARIANT 1" >>confdefs.h + +fi + # Check for --with-trace-refs # --with-trace-refs { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-trace-refs" >&5 diff --git a/configure.ac b/configure.ac index ba768aea930714..a476ba833e6ab3 100644 --- a/configure.ac +++ b/configure.ac @@ -1689,6 +1689,21 @@ else AC_MSG_RESULT([no]); Py_DEBUG='false' fi], [AC_MSG_RESULT([no])]) +# Check for --with-region-invariant +# --with-region-invariant +AC_MSG_CHECKING([for --with-region-invariant]) +AC_ARG_WITH([region-invariant], + [AS_HELP_STRING([--with-region-invariant], [enable region invariant for debugging purpose (default is no)])], + [], [with_region_invariant=yes] +) +AC_MSG_RESULT([$with_region_invariant]) + +if test "$with_region_invariant" = "yes" +then + AC_DEFINE([Py_REGION_INVARIANT], [1], + [Define if you want to enable region invariant for debugging purpose]) +fi + # Check for --with-trace-refs # --with-trace-refs AC_MSG_CHECKING([for --with-trace-refs]) diff --git a/pyconfig.h.in b/pyconfig.h.in index ada9dccfef1084..2dd38014570b5c 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1615,6 +1615,9 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM +/* Define if you want to enable region invariant for debugging purpose */ +#undef Py_REGION_INVARIANT + /* Define if you want to enable internal statistics gathering. */ #undef Py_STATS From 68a9b705ffbb06f49227265a1333518f7e19c65f Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Tue, 21 Jan 2025 15:44:37 +0100 Subject: [PATCH 49/68] Permitting key parts of Cowns to run without GIL (#38) * Permitting key parts of Cowns to run without GIL * Cleaned up last check in test self.fail() is performed outside of the acquired cown --- Lib/test/test_using.py | 59 ++++++++++++++++++++++++++++++++++++++++++ Objects/cown.c | 13 +++++++--- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index 987bc64a4679a5..ef005b34c4a2a6 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -107,3 +107,62 @@ def test_region_cown_ptr(self): def test_invalid_cown_init(self): # Create cown with invalid init value self.assertRaises(RegionError, Cown, [42]) + + def test_threads(self): + from threading import Thread + from using import using + + + class Counter(object): + def __init__(self, value): + self.value = value + + def inc(self): + self.value += 1 + + def dec(self): + self.value -= 1 + + def __repr__(self): + return "Counter(" + str(self.value) + ")" + + + # Freezes the **class** -- not needed explicitly later + makeimmutable(Counter) + + def ThreadSafeValue(value): + r = Region("counter region") + r.value = value + c = Cown(r) + # Dropping value, r and closing not needed explicitly later + del value + del r + c.get().close() + return c + + def work(c): + for _ in range(0, 100): + c.inc() + + def work_in_parallel(c): + @using(c) + def _(): + work(c.get().value) + + + c = ThreadSafeValue(Counter(0)) + + t1 = Thread(target=work_in_parallel, args=(c,)) + t2 = Thread(target=work_in_parallel, args=(c,)) + t1.start() + t2.start() + t1.join() + t2.join() + + result = 0 + @using(c) + def _(): + nonlocal result + result = c.get().value.value + if result != 200: + self.fail() diff --git a/Objects/cown.c b/Objects/cown.c index 8a126bd8d3629c..b20caab71b84a6 100644 --- a/Objects/cown.c +++ b/Objects/cown.c @@ -94,8 +94,9 @@ static int PyCown_traverse(PyCownObject *self, visitproc visit, void *arg) { #define BAIL_IF_OWNED(o, msg) \ do { \ /* Note: we must hold the GIL at this point -- note for future threading implementation. */ \ - if (o->owning_thread != 0) { \ - PyErr_Format(PyExc_RegionError, "%s: %S -- %zd", msg, o, o->owning_thread); \ + size_t tid = o->owning_thread; \ + if (tid != 0) { \ + PyErr_Format(PyExc_RegionError, "%s: %S -- %zd", msg, o, tid); \ return NULL; \ } \ } while(0); @@ -126,18 +127,20 @@ static int PyCown_traverse(PyCownObject *self, visitproc visit, void *arg) { // The ignored argument is required for this function's type to be // compatible with PyCFunction static PyObject *PyCown_acquire(PyCownObject *self, PyObject *ignored) { + PyThreadState *tstate = PyThreadState_Get(); + Py_BEGIN_ALLOW_THREADS int expected = Cown_RELEASED; // TODO: eventually replace this with something from pycore_atomic (nothing there now) while (!atomic_compare_exchange_strong(&self->state._value, &expected, Cown_ACQUIRED)) { - sem_wait(&self->semaphore); expected = Cown_RELEASED; + sem_wait(&self->semaphore); } // Note: we must hold the GIL at this point -- note for future // threading implementation. - PyThreadState *tstate = PyThreadState_Get(); self->owning_thread = tstate->thread_id; + Py_END_ALLOW_THREADS Py_RETURN_NONE; } @@ -152,9 +155,11 @@ static PyObject *PyCown_release(PyCownObject *self, PyObject *ignored) { BAIL_UNLESS_OWNED(self, "Thread attempted to release a cown it did not own"); + Py_BEGIN_ALLOW_THREADS self->owning_thread = 0; _Py_atomic_store(&self->state, Cown_RELEASED); sem_post(&self->semaphore); + Py_END_ALLOW_THREADS Py_RETURN_NONE; } From 2c3fbf5e654271e893ad44b95b1e3abb4f7e8fef Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Fri, 17 Jan 2025 23:41:48 +0100 Subject: [PATCH 50/68] Check thread arguments Adds a check to thread constructors ensuring that only cowns, immutables, and externally unique regions can be passed in as thread arguments. --- Lib/test/test_using.py | 12 ++++++++++++ Lib/threading.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index ef005b34c4a2a6..eddf136697aacd 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -166,3 +166,15 @@ def _(): result = c.get().value.value if result != 200: self.fail() + + def test_thread_creation(self): + from threading import Thread as T + + class Mutable: pass + self.assertRaises(RuntimeError, T, kwargs = { 'target' : print, 'args' : (Mutable(),) }) + self.assertRaises(RuntimeError, T, kwargs = { 'target' : print, 'kwargs' : {'a' : Mutable()} }) + self.assertRaises(RuntimeError, T, kwargs = { 'target' : print, 'args' : (Mutable(),), 'kwargs' : {'a' : Mutable()} }) + + T(target=print, args=(42, Cown(), Region())) + T(target=print, kwargs={'imm' : 42, 'cown' : Cown(), 'region' : Region()}) + self.assertTrue(True) # To make sure we got here correctly diff --git a/Lib/threading.py b/Lib/threading.py index df273870fa4273..2d77998bfd4fb8 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -895,6 +895,22 @@ class is implemented. except AttributeError: pass + def movable(o): + return isinstance(o, Region) and not o.is_open() + + for k, v in kwargs.items(): + if not (isimmutable(v) or isinstance(v, Cown) or movable(v)): + raise RuntimeError(f'thread was passed {k} : {type(v)} -- ' + 'only immutable objects, cowns and free ' + 'regions may be passed to a thread') + for a in args: + if not (isimmutable(a) or isinstance(a, Cown) or movable(a)): + from sys import getrefcount as rc + print(a, rc(a)) + raise RuntimeError(f'thread was passed {type(a)} -- ' + 'only immutable objects, cowns and free ' + 'regions may be passed to a thread') + self._target = target self._name = name self._args = args From 16afce86c5b026b56d306ed30a990ae234dd91c6 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Sat, 18 Jan 2025 11:32:49 +0100 Subject: [PATCH 51/68] Made pyrona checks conditional on region use --- Lib/threading.py | 32 +++++++++++++++++--------------- Python/bltinmodule.c | 9 +++++++++ Python/clinic/bltinmodule.c.h | 20 ++++++++++++++++++++ 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py index 2d77998bfd4fb8..4ec925806143a7 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -895,21 +895,23 @@ class is implemented. except AttributeError: pass - def movable(o): - return isinstance(o, Region) and not o.is_open() - - for k, v in kwargs.items(): - if not (isimmutable(v) or isinstance(v, Cown) or movable(v)): - raise RuntimeError(f'thread was passed {k} : {type(v)} -- ' - 'only immutable objects, cowns and free ' - 'regions may be passed to a thread') - for a in args: - if not (isimmutable(a) or isinstance(a, Cown) or movable(a)): - from sys import getrefcount as rc - print(a, rc(a)) - raise RuntimeError(f'thread was passed {type(a)} -- ' - 'only immutable objects, cowns and free ' - 'regions may be passed to a thread') + # Only check when a program uses pyrona + if is_pyrona_program(): + def movable(o): + return isinstance(o, Region) and not o.is_open() + + for k, v in kwargs.items(): + if not (isimmutable(v) or isinstance(v, Cown) or movable(v)): + raise RuntimeError(f'thread was passed {k} : {type(v)} -- ' + 'only immutable objects, cowns and free ' + 'regions may be passed to a thread') + for a in args: + if not (isimmutable(a) or isinstance(a, Cown) or movable(a)): + from sys import getrefcount as rc + print(a, rc(a)) + raise RuntimeError(f'thread was passed {type(a)} -- ' + 'only immutable objects, cowns and free ' + 'regions may be passed to a thread') self._target = target self._name = name diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 89d51a9d7d3bab..cdd22a125a88f2 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2776,6 +2776,14 @@ builtin_makeimmutable(PyObject *module, PyObject *obj) return Py_MakeImmutable(obj); } +extern bool invariant_do_region_check; + +static PyObject * +builtin_is_pyrona_program_impl(PyObject *module) +{ + return invariant_do_region_check ? Py_True : Py_False; +} + /*[clinic input] invariant_failure_src as builtin_invariantsrcfailure @@ -3118,6 +3126,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_MAKEIMMUTABLE_METHODDEF BUILTIN_INVARIANTSRCFAILURE_METHODDEF BUILTIN_INVARIANTTGTFAILURE_METHODDEF + BUILTIN_IS_PYRONA_PROGRAM_METHODDEF BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 34d29e1b5c81a2..72a2d4485f0b46 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1481,4 +1481,24 @@ builtin_enableinvariant(PyObject *module, PyObject *Py_UNUSED(ignored)) { return builtin_enableinvariant_impl(module); } + +PyDoc_STRVAR(builtin_is_pyrona_program__doc__, +"is_pyrona_program($module, /)\n" +"--\n" +"\n" +"Returns True if the program has used regions or cowns."); + +static PyObject * +builtin_is_pyrona_program_impl(PyObject *module); + +static PyObject * +builtin_is_pyrona_program(PyObject *module, PyObject *Py_UNUSED(ignored)) +/*[clinic end generated code: output=4e665122542dfd24 input=bec4cf1797c848d4]*/ +{ + return builtin_is_pyrona_program_impl(module); +} + +#define BUILTIN_IS_PYRONA_PROGRAM_METHODDEF \ + {"is_pyrona_program", (PyCFunction)builtin_is_pyrona_program, METH_NOARGS, builtin_is_pyrona_program__doc__}, + /*[clinic end generated code: output=3a883fa08bbd248e input=a9049054013a1b77]*/ From 05260f0a3ee0a76dcf0815b039d06439fcd278e8 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Sun, 19 Jan 2025 09:18:53 +0100 Subject: [PATCH 52/68] Clinic --- Python/bltinmodule.c | 9 --------- Python/clinic/bltinmodule.c.h | 20 -------------------- 2 files changed, 29 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index cdd22a125a88f2..89d51a9d7d3bab 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2776,14 +2776,6 @@ builtin_makeimmutable(PyObject *module, PyObject *obj) return Py_MakeImmutable(obj); } -extern bool invariant_do_region_check; - -static PyObject * -builtin_is_pyrona_program_impl(PyObject *module) -{ - return invariant_do_region_check ? Py_True : Py_False; -} - /*[clinic input] invariant_failure_src as builtin_invariantsrcfailure @@ -3126,7 +3118,6 @@ static PyMethodDef builtin_methods[] = { BUILTIN_MAKEIMMUTABLE_METHODDEF BUILTIN_INVARIANTSRCFAILURE_METHODDEF BUILTIN_INVARIANTTGTFAILURE_METHODDEF - BUILTIN_IS_PYRONA_PROGRAM_METHODDEF BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 72a2d4485f0b46..34d29e1b5c81a2 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1481,24 +1481,4 @@ builtin_enableinvariant(PyObject *module, PyObject *Py_UNUSED(ignored)) { return builtin_enableinvariant_impl(module); } - -PyDoc_STRVAR(builtin_is_pyrona_program__doc__, -"is_pyrona_program($module, /)\n" -"--\n" -"\n" -"Returns True if the program has used regions or cowns."); - -static PyObject * -builtin_is_pyrona_program_impl(PyObject *module); - -static PyObject * -builtin_is_pyrona_program(PyObject *module, PyObject *Py_UNUSED(ignored)) -/*[clinic end generated code: output=4e665122542dfd24 input=bec4cf1797c848d4]*/ -{ - return builtin_is_pyrona_program_impl(module); -} - -#define BUILTIN_IS_PYRONA_PROGRAM_METHODDEF \ - {"is_pyrona_program", (PyCFunction)builtin_is_pyrona_program, METH_NOARGS, builtin_is_pyrona_program__doc__}, - /*[clinic end generated code: output=3a883fa08bbd248e input=a9049054013a1b77]*/ From dce65bfab695cf91ea17a523ae48398cf04853b1 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Sun, 19 Jan 2025 09:35:37 +0100 Subject: [PATCH 53/68] Clinic fix --- Include/regions.h | 2 ++ Objects/regions.c | 3 +++ Python/bltinmodule.c | 14 ++++++++++++++ Python/clinic/bltinmodule.c.h | 20 +++++++++++++++++++- 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/Include/regions.h b/Include/regions.h index 52a7ec3e9d15a5..1cfc49ce0984b6 100644 --- a/Include/regions.h +++ b/Include/regions.h @@ -15,6 +15,8 @@ PyAPI_FUNC(int) _Py_IsLocal(PyObject *op); PyAPI_FUNC(int) _Py_IsCown(PyObject *op); #define Py_IsCown(op) _Py_IsCown(_PyObject_CAST(op)) +int Py_is_invariant_enabled(void); + #ifdef __cplusplus } #endif diff --git a/Objects/regions.c b/Objects/regions.c index 7f138973965bd3..f4747cd20349f4 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -50,6 +50,9 @@ static void _PyErr_Region(PyObject *src, PyObject *tgt, const char *msg); * Global status for performing the region check. */ bool invariant_do_region_check = false; +int Py_is_invariant_enabled(void) { + return invariant_do_region_check; +} // The src object for an edge that invalidated the invariant. PyObject* invariant_error_src = Py_None; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 89d51a9d7d3bab..65fbb57c8808e6 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2815,6 +2815,19 @@ builtin_enableinvariant_impl(PyObject *module) return Py_EnableInvariant(); } +/*[clinic input] +is_pyrona_program as builtin_is_pyrona_program + +Returns True if the program has used regions or cowns. +[clinic start generated code]*/ + +static PyObject * +builtin_is_pyrona_program_impl(PyObject *module) +/*[clinic end generated code: output=2b6729469da00221 input=d36655a3dc34a881]*/ +{ + return Py_is_invariant_enabled() ? Py_True : Py_False; +} + typedef struct { PyObject_HEAD Py_ssize_t tuplesize; @@ -3116,6 +3129,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISSUBCLASS_METHODDEF BUILTIN_ISIMMUTABLE_METHODDEF BUILTIN_MAKEIMMUTABLE_METHODDEF + BUILTIN_IS_PYRONA_PROGRAM_METHODDEF BUILTIN_INVARIANTSRCFAILURE_METHODDEF BUILTIN_INVARIANTTGTFAILURE_METHODDEF BUILTIN_ITER_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 34d29e1b5c81a2..1771189974cfbf 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1481,4 +1481,22 @@ builtin_enableinvariant(PyObject *module, PyObject *Py_UNUSED(ignored)) { return builtin_enableinvariant_impl(module); } -/*[clinic end generated code: output=3a883fa08bbd248e input=a9049054013a1b77]*/ + +PyDoc_STRVAR(builtin_is_pyrona_program__doc__, +"is_pyrona_program($module, /)\n" +"--\n" +"\n" +"Returns True if the program has used regions or cowns."); + +#define BUILTIN_IS_PYRONA_PROGRAM_METHODDEF \ + {"is_pyrona_program", (PyCFunction)builtin_is_pyrona_program, METH_NOARGS, builtin_is_pyrona_program__doc__}, + +static PyObject * +builtin_is_pyrona_program_impl(PyObject *module); + +static PyObject * +builtin_is_pyrona_program(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return builtin_is_pyrona_program_impl(module); +} +/*[clinic end generated code: output=c1f628b019efa501 input=a9049054013a1b77]*/ From f1cc65e0609f4ffc503fc8c98638c9c6a6dd04a1 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Mon, 20 Jan 2025 12:35:47 +0100 Subject: [PATCH 54/68] Improved threading check, added technical debt todos --- Lib/threading.py | 50 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/Lib/threading.py b/Lib/threading.py index 4ec925806143a7..7619da1ed5c071 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -896,22 +896,44 @@ class is implemented. pass # Only check when a program uses pyrona + from sys import getrefcount as rc + # TODO: improve this check for final version of phase 3 + # - Revisit the rc checks + # - Consider throwing a different kind of error (e.g. RegionError) + # - Improve error messages if is_pyrona_program(): - def movable(o): - return isinstance(o, Region) and not o.is_open() - - for k, v in kwargs.items(): - if not (isimmutable(v) or isinstance(v, Cown) or movable(v)): - raise RuntimeError(f'thread was passed {k} : {type(v)} -- ' - 'only immutable objects, cowns and free ' - 'regions may be passed to a thread') + def ok_share(o): + if isimmutable(o): + return True + if isinstance(o, Cown): + return True + return False + def ok_move(o): + if isinstance(o, Region): + if rc(o) != 4: + # rc = 4 because: + # 1. ref to o in rc + # 2. ref to o on this frame + # 3. ref to o on the calling frame + # 4. ref to o from kwargs dictionary or args tuple/list + raise RuntimeError("Region passed to thread was not moved into thread") + if o.is_open(): + raise RuntimeError("Region passed to thread was open") + return True + return False + + for k in kwargs: + # rc(args) == 6 because we need to know that the args list is moved into the thread too + # TODO: Why 6??? + v = kwargs[k] + if not (ok_share(v) or (ok_move(v) and rc(kwargs) == 6)): + raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") + for a in args: - if not (isimmutable(a) or isinstance(a, Cown) or movable(a)): - from sys import getrefcount as rc - print(a, rc(a)) - raise RuntimeError(f'thread was passed {type(a)} -- ' - 'only immutable objects, cowns and free ' - 'regions may be passed to a thread') + # rc(args) == 6 because we need to know that the args list is moved into the thread too + # TODO: Why 6??? + if not (ok_share(a) or (ok_move(a) and rc(args) == 6)): + raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") self._target = target self._name = name From c7ce3d8185e4b588f1fb98fb37d12d7b2293d6be Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Mon, 20 Jan 2025 12:39:22 +0100 Subject: [PATCH 55/68] Added TODO for builtin_is_pyrona_program_impl --- Python/bltinmodule.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 65fbb57c8808e6..36232e780c198e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2825,6 +2825,12 @@ static PyObject * builtin_is_pyrona_program_impl(PyObject *module) /*[clinic end generated code: output=2b6729469da00221 input=d36655a3dc34a881]*/ { + // TODO: This is an abuse of notions. We need to revisit + // the definition of when a program is a Pyrona program + // at some later point. The reason for having the definition + // conflated with the invariant being enabled is to only + // perform Pyrona checks (see threading.py) when a program + // must adhere to these checks for correctness. return Py_is_invariant_enabled() ? Py_True : Py_False; } From 1c949a032602decb2609dc2fccfdd9062c63d5c5 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Mon, 20 Jan 2025 14:16:56 +0100 Subject: [PATCH 56/68] Stupid trailing whitespace --- Python/bltinmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 36232e780c198e..f9dc986736d61e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2825,10 +2825,10 @@ static PyObject * builtin_is_pyrona_program_impl(PyObject *module) /*[clinic end generated code: output=2b6729469da00221 input=d36655a3dc34a881]*/ { - // TODO: This is an abuse of notions. We need to revisit + // TODO: This is an abuse of notions. We need to revisit // the definition of when a program is a Pyrona program // at some later point. The reason for having the definition - // conflated with the invariant being enabled is to only + // conflated with the invariant being enabled is to only // perform Pyrona checks (see threading.py) when a program // must adhere to these checks for correctness. return Py_is_invariant_enabled() ? Py_True : Py_False; From 7ff62389a57d61680c871ddb1137207d336ff3a4 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Mon, 27 Jan 2025 20:44:35 +0100 Subject: [PATCH 57/68] Made separate PyronaThread constructor in using.py --- Lib/test/test_using.py | 2 +- Lib/threading.py | 40 ------------------------------- Lib/using.py | 53 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index eddf136697aacd..086df63dde88df 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -168,7 +168,7 @@ def _(): self.fail() def test_thread_creation(self): - from threading import Thread as T + from threading import PyronaThread as T class Mutable: pass self.assertRaises(RuntimeError, T, kwargs = { 'target' : print, 'args' : (Mutable(),) }) diff --git a/Lib/threading.py b/Lib/threading.py index 7619da1ed5c071..df273870fa4273 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -895,46 +895,6 @@ class is implemented. except AttributeError: pass - # Only check when a program uses pyrona - from sys import getrefcount as rc - # TODO: improve this check for final version of phase 3 - # - Revisit the rc checks - # - Consider throwing a different kind of error (e.g. RegionError) - # - Improve error messages - if is_pyrona_program(): - def ok_share(o): - if isimmutable(o): - return True - if isinstance(o, Cown): - return True - return False - def ok_move(o): - if isinstance(o, Region): - if rc(o) != 4: - # rc = 4 because: - # 1. ref to o in rc - # 2. ref to o on this frame - # 3. ref to o on the calling frame - # 4. ref to o from kwargs dictionary or args tuple/list - raise RuntimeError("Region passed to thread was not moved into thread") - if o.is_open(): - raise RuntimeError("Region passed to thread was open") - return True - return False - - for k in kwargs: - # rc(args) == 6 because we need to know that the args list is moved into the thread too - # TODO: Why 6??? - v = kwargs[k] - if not (ok_share(v) or (ok_move(v) and rc(kwargs) == 6)): - raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") - - for a in args: - # rc(args) == 6 because we need to know that the args list is moved into the thread too - # TODO: Why 6??? - if not (ok_share(a) or (ok_move(a) and rc(args) == 6)): - raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") - self._target = target self._name = name self._args = args diff --git a/Lib/using.py b/Lib/using.py index 30ae93a3bbb1cf..af8f5d76dac7d7 100644 --- a/Lib/using.py +++ b/Lib/using.py @@ -45,3 +45,56 @@ def decorator(func): with CS(cowns, *args): return func() return decorator + +def PyronaThread(group=None, target=None, name=None, + args=(), kwargs=None, *, daemon=None): + # Only check when a program uses pyrona + from sys import getrefcount as rc + from threading import Thread + # TODO: improve this check for final version of phase 3 + # - Revisit the rc checks + # - Consider throwing a different kind of error (e.g. RegionError) + # - Improve error messages + def ok_share(o): + if isimmutable(o): + return True + if isinstance(o, Cown): + return True + return False + def ok_move(o): + if isinstance(o, Region): + if rc(o) != 4: + # rc = 4 because: + # 1. ref to o in rc + # 2. ref to o on this frame + # 3. ref to o on the calling frame + # 4. ref to o from kwargs dictionary or args tuple/list + raise RuntimeError("Region passed to thread was not moved into thread") + if o.is_open(): + raise RuntimeError("Region passed to thread was open") + return True + return False + + if kwargs is None: + for a in args: + # rc(args) == 3 because we need to know that the args list is moved into the thread too + # rc = 3 because: + # 1. ref to args in rc + # 2. ref to args on this frame + # 3. ref to args on the calling frame + if not (ok_share(a) or (ok_move(a) and rc(args) == 3)): + raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") + return Thread(group, target, name, args, daemon) + else: + for k in kwargs: + # rc(args) == 3 because we need to know that keyword dict is moved into the thread too + # rc = 3 because: + # 1. ref to kwargs in rc + # 2. ref to kwargs on this frame + # 3. ref to kwargs on the calling frame + v = kwargs[k] + if not (ok_share(v) or (ok_move(v) and rc(kwargs) == 3)): + raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") + return Thread(group, target, name, kwargs, daemon) + + From 250fa9769c5d066cf5a05fa7b268fa2b5b4c459c Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Mon, 27 Jan 2025 20:50:18 +0100 Subject: [PATCH 58/68] Added comment suggested by @mjp41 and fixed trailing space bug --- Lib/test/test_using.py | 2 +- Lib/using.py | 5 +++-- Objects/regions.c | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index 086df63dde88df..e9fbbb8b7c4235 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -166,7 +166,7 @@ def _(): result = c.get().value.value if result != 200: self.fail() - + def test_thread_creation(self): from threading import PyronaThread as T diff --git a/Lib/using.py b/Lib/using.py index af8f5d76dac7d7..e0501368bd900b 100644 --- a/Lib/using.py +++ b/Lib/using.py @@ -46,6 +46,9 @@ def decorator(func): return func() return decorator +# TODO: this creates a normal Python thread and ensures that all its +# arguments are moved to the new thread. Eventually we should revisit +# this behaviour as we go multiple interpreters / multicore. def PyronaThread(group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None): # Only check when a program uses pyrona @@ -96,5 +99,3 @@ def ok_move(o): if not (ok_share(v) or (ok_move(v) and rc(kwargs) == 3)): raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") return Thread(group, target, name, kwargs, daemon) - - diff --git a/Objects/regions.c b/Objects/regions.c index f4747cd20349f4..6cf0ad06a6814e 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -50,6 +50,9 @@ static void _PyErr_Region(PyObject *src, PyObject *tgt, const char *msg); * Global status for performing the region check. */ bool invariant_do_region_check = false; +/** + * TODO: revisit the definition of this builting function + */ int Py_is_invariant_enabled(void) { return invariant_do_region_check; } From 627bdeac5e0207d20e9c831565cb368cc5677ea5 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Tue, 28 Jan 2025 18:41:01 +0100 Subject: [PATCH 59/68] Fixed import bug --- Lib/test/test_using.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index e9fbbb8b7c4235..f6285a9a7a68ed 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -168,7 +168,7 @@ def _(): self.fail() def test_thread_creation(self): - from threading import PyronaThread as T + from using import PyronaThread as T class Mutable: pass self.assertRaises(RuntimeError, T, kwargs = { 'target' : print, 'args' : (Mutable(),) }) From e39f73f1738ae28bded1740a417023f5d77d24de Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Sat, 1 Feb 2025 11:39:11 +0100 Subject: [PATCH 60/68] Refactoring of checking logic Co-authored-by: Matthew Parkinson --- Lib/using.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/Lib/using.py b/Lib/using.py index e0501368bd900b..be77a381a88d66 100644 --- a/Lib/using.py +++ b/Lib/using.py @@ -78,24 +78,19 @@ def ok_move(o): return True return False + def check(a, args): + # rc(args) == 3 because we need to know that the args list is moved into the thread too + # rc = 3 because: + # 1. ref to args in rc + # 2. ref to args on this frame + # 3. ref to args on the calling framedef check(a, args): + if not ok_share(a) or (ok_move(a) and rc(args) == 3): + raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") if kwargs is None: for a in args: - # rc(args) == 3 because we need to know that the args list is moved into the thread too - # rc = 3 because: - # 1. ref to args in rc - # 2. ref to args on this frame - # 3. ref to args on the calling frame - if not (ok_share(a) or (ok_move(a) and rc(args) == 3)): - raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") + check(a, args) return Thread(group, target, name, args, daemon) else: for k in kwargs: - # rc(args) == 3 because we need to know that keyword dict is moved into the thread too - # rc = 3 because: - # 1. ref to kwargs in rc - # 2. ref to kwargs on this frame - # 3. ref to kwargs on the calling frame - v = kwargs[k] - if not (ok_share(v) or (ok_move(v) and rc(kwargs) == 3)): - raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") + check(k, kwargs) return Thread(group, target, name, kwargs, daemon) From d408a0f5f4366589643490120bab1be7109743f2 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Sat, 1 Feb 2025 12:43:09 +0100 Subject: [PATCH 61/68] Improve test and update RC due to refactored logic --- Lib/test/test_using.py | 18 ++++++++++++++---- Lib/using.py | 26 ++++++++++++++++---------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index f6285a9a7a68ed..8a1fd9a74a7b5b 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -171,10 +171,20 @@ def test_thread_creation(self): from using import PyronaThread as T class Mutable: pass - self.assertRaises(RuntimeError, T, kwargs = { 'target' : print, 'args' : (Mutable(),) }) - self.assertRaises(RuntimeError, T, kwargs = { 'target' : print, 'kwargs' : {'a' : Mutable()} }) - self.assertRaises(RuntimeError, T, kwargs = { 'target' : print, 'args' : (Mutable(),), 'kwargs' : {'a' : Mutable()} }) + self.assertRaises(RuntimeError, lambda x: T(target=print, args=(Mutable(),)), None) + self.assertRaises(RuntimeError, lambda x: T(target=print, kwargs={'a' : Mutable()}), None) + self.assertRaises(RuntimeError, lambda x: T(target=print, args=(Mutable(),), kwargs={'a' : Mutable()}), None) + self.assertRaises(RuntimeError, lambda x: T(target=print, args=(Mutable(), 42)), None) + self.assertRaises(RuntimeError, lambda x: T(target=print, args=(Mutable(), Cown())), None) + self.assertRaises(RuntimeError, lambda x: T(target=print, args=(Mutable(), Region())), None) - T(target=print, args=(42, Cown(), Region())) T(target=print, kwargs={'imm' : 42, 'cown' : Cown(), 'region' : Region()}) + T(target=print, kwargs={'a': 42}) + T(target=print, kwargs={'a': Cown()}) + T(target=print, kwargs={'a': Region()}) + + T(target=print, args=(42, Cown(), Region())) + T(target=print, args=(42,)) + T(target=print, args=(Cown(),)) + T(target=print, args=(Region(),)) self.assertTrue(True) # To make sure we got here correctly diff --git a/Lib/using.py b/Lib/using.py index be77a381a88d66..aafc4e30b1e343 100644 --- a/Lib/using.py +++ b/Lib/using.py @@ -49,6 +49,7 @@ def decorator(func): # TODO: this creates a normal Python thread and ensures that all its # arguments are moved to the new thread. Eventually we should revisit # this behaviour as we go multiple interpreters / multicore. +# TODO: require RC to be one less when move is upstreamed def PyronaThread(group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None): # Only check when a program uses pyrona @@ -66,12 +67,13 @@ def ok_share(o): return False def ok_move(o): if isinstance(o, Region): - if rc(o) != 4: + if rc(o) != 5: # rc = 4 because: # 1. ref to o in rc - # 2. ref to o on this frame - # 3. ref to o on the calling frame - # 4. ref to o from kwargs dictionary or args tuple/list + # 2. ref to o on this frame (ok_move) + # 3. ref to o on the calling frame (check) + # 4. ref to o from iteration over kwargs dictionary or args tuple/list + # 5. ref to o from kwargs dictionary or args tuple/list raise RuntimeError("Region passed to thread was not moved into thread") if o.is_open(): raise RuntimeError("Region passed to thread was open") @@ -79,18 +81,22 @@ def ok_move(o): return False def check(a, args): - # rc(args) == 3 because we need to know that the args list is moved into the thread too - # rc = 3 because: + # rc(args) == 4 because we need to know that the args list is moved into the thread too + # rc = 4 because: # 1. ref to args in rc # 2. ref to args on this frame # 3. ref to args on the calling framedef check(a, args): - if not ok_share(a) or (ok_move(a) and rc(args) == 3): + # 4. ref from frame calling PyronaThread -- FIXME: not valid; revisit after #45 + if not (ok_share(a) or (ok_move(a) and rc(args) == 4)): raise RuntimeError("Thread was passed an object which was neither immutable, a cown, or a unique region") + if kwargs is None: for a in args: check(a, args) - return Thread(group, target, name, args, daemon) + return Thread(group, target, name, args, daemon) else: for k in kwargs: - check(k, kwargs) - return Thread(group, target, name, kwargs, daemon) + # Important to get matching RCs in both paths + v = kwargs[k] + check(v, kwargs) + return Thread(group, target, name, kwargs, daemon) From c9ef79d83b9d7e6274140c7cfac27debe4d364f3 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Sat, 1 Feb 2025 12:45:56 +0100 Subject: [PATCH 62/68] Removed is_pyrona_program builtin --- Python/bltinmodule.c | 20 -------------------- Python/clinic/bltinmodule.c.h | 20 +------------------- 2 files changed, 1 insertion(+), 39 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f9dc986736d61e..89d51a9d7d3bab 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2815,25 +2815,6 @@ builtin_enableinvariant_impl(PyObject *module) return Py_EnableInvariant(); } -/*[clinic input] -is_pyrona_program as builtin_is_pyrona_program - -Returns True if the program has used regions or cowns. -[clinic start generated code]*/ - -static PyObject * -builtin_is_pyrona_program_impl(PyObject *module) -/*[clinic end generated code: output=2b6729469da00221 input=d36655a3dc34a881]*/ -{ - // TODO: This is an abuse of notions. We need to revisit - // the definition of when a program is a Pyrona program - // at some later point. The reason for having the definition - // conflated with the invariant being enabled is to only - // perform Pyrona checks (see threading.py) when a program - // must adhere to these checks for correctness. - return Py_is_invariant_enabled() ? Py_True : Py_False; -} - typedef struct { PyObject_HEAD Py_ssize_t tuplesize; @@ -3135,7 +3116,6 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISSUBCLASS_METHODDEF BUILTIN_ISIMMUTABLE_METHODDEF BUILTIN_MAKEIMMUTABLE_METHODDEF - BUILTIN_IS_PYRONA_PROGRAM_METHODDEF BUILTIN_INVARIANTSRCFAILURE_METHODDEF BUILTIN_INVARIANTTGTFAILURE_METHODDEF BUILTIN_ITER_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 1771189974cfbf..34d29e1b5c81a2 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1481,22 +1481,4 @@ builtin_enableinvariant(PyObject *module, PyObject *Py_UNUSED(ignored)) { return builtin_enableinvariant_impl(module); } - -PyDoc_STRVAR(builtin_is_pyrona_program__doc__, -"is_pyrona_program($module, /)\n" -"--\n" -"\n" -"Returns True if the program has used regions or cowns."); - -#define BUILTIN_IS_PYRONA_PROGRAM_METHODDEF \ - {"is_pyrona_program", (PyCFunction)builtin_is_pyrona_program, METH_NOARGS, builtin_is_pyrona_program__doc__}, - -static PyObject * -builtin_is_pyrona_program_impl(PyObject *module); - -static PyObject * -builtin_is_pyrona_program(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return builtin_is_pyrona_program_impl(module); -} -/*[clinic end generated code: output=c1f628b019efa501 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3a883fa08bbd248e input=a9049054013a1b77]*/ From 94af8f41a5ab44084f9c2fda51493c5e5166b31b Mon Sep 17 00:00:00 2001 From: xFrednet Date: Fri, 31 Jan 2025 16:54:36 +0100 Subject: [PATCH 63/68] Update: actions/upload-artifact@v3 -> v4 --- .github/workflows/build.yml | 4 ++-- .github/workflows/build_min.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af674de20c94ef..aafe40804c9b59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,7 +140,7 @@ jobs: if: ${{ failure() && steps.check.conclusion == 'failure' }} run: | make regen-abidump - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 name: Publish updated ABI files if: ${{ failure() && steps.check.conclusion == 'failure' }} with: @@ -520,7 +520,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: hypothesis-example-db diff --git a/.github/workflows/build_min.yml b/.github/workflows/build_min.yml index a7b8d9449823e6..e50d4ed8ea3ef2 100644 --- a/.github/workflows/build_min.yml +++ b/.github/workflows/build_min.yml @@ -125,7 +125,7 @@ jobs: # if: ${{ failure() && steps.check.conclusion == 'failure' }} # run: | # make regen-abidump - # - uses: actions/upload-artifact@v3 + # - uses: actions/upload-artifact@v4 # name: Publish updated ABI files # if: ${{ failure() && steps.check.conclusion == 'failure' }} # with: @@ -511,7 +511,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: hypothesis-example-db From da00da65c2dfa203a3adec9d3aa939ee83113706 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Mon, 24 Feb 2025 17:49:37 +0100 Subject: [PATCH 64/68] Cown release (#44) * Cown release A cown will now be released: - When it is in pending-release state and its containing region closes - When a closed region, other cown or immutable object it stored in it - At the end of a @using if the cown's contained region can be closed * TODO => TODO Pyrona: Co-authored-by: Fridtjof Stoldt * Remove GIL operations in release (since this operation cannot be contended) --------- Co-authored-by: Fridtjof Stoldt --- Include/internal/pycore_regions.h | 6 ++++- Lib/test/test_using.py | 2 +- Objects/cown.c | 42 ++++++++++++++++++++++--------- Objects/regions.c | 21 +++++++++++++--- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index aeb06bf517afc8..6a2c808ec2987a 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -85,9 +85,13 @@ int _Py_CheckRegionInvariant(PyThreadState *tstate); // Set a cown as parent of a region void _PyRegion_set_cown_parent(PyObject* region, PyObject* cown); // Check whether a region is closed -int _PyRegion_is_closed(PyObject* region); int _PyCown_release(PyObject *self); int _PyCown_is_released(PyObject *self); +int _PyCown_is_pending_release(PyObject *self); +PyObject *_PyCown_close_region(PyObject* ob); +#define PyCown_close_region(op) _PyCown_close_region(_PyObject_CAST(op)) +int _PyRegion_is_closed(PyObject* op); +#define PyRegion_is_closed(op) _PyRegion_is_closed(_PyObject_CAST(op)) #ifdef _Py_TYPEOF diff --git a/Lib/test/test_using.py b/Lib/test/test_using.py index 8a1fd9a74a7b5b..f123c2d23521fc 100644 --- a/Lib/test/test_using.py +++ b/Lib/test/test_using.py @@ -96,7 +96,7 @@ def _(): self.assertTrue(self.hacky_state_check(c, "acquired")) r = None c.get().close() - self.assertTrue(self.hacky_state_check(c, "released")) + self.assertTrue(self.hacky_state_check(c, "acquired")) self.assertTrue(self.hacky_state_check(c, "released")) def test_region_cown_ptr(self): diff --git a/Objects/cown.c b/Objects/cown.c index b20caab71b84a6..384fdaba33fb57 100644 --- a/Objects/cown.c +++ b/Objects/cown.c @@ -19,6 +19,10 @@ #include "pyerrors.h" #include "pystate.h" +// Needed to test for region object +extern PyTypeObject PyRegion_Type; +extern PyTypeObject PyCown_Type; + typedef enum { Cown_RELEASED = 0, Cown_ACQUIRED = 1, @@ -57,7 +61,7 @@ static void PyCown_dealloc(PyCownObject *self) { } static int PyCown_init(PyCownObject *self, PyObject *args, PyObject *kwds) { - // TODO: should not be needed in the future + // TODO: Pyrona: should not be needed in the future _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); _Py_notify_regions_in_use(); @@ -128,10 +132,12 @@ static int PyCown_traverse(PyCownObject *self, visitproc visit, void *arg) { // compatible with PyCFunction static PyObject *PyCown_acquire(PyCownObject *self, PyObject *ignored) { PyThreadState *tstate = PyThreadState_Get(); + + // TODO: Pyrona: releasing the GIL will eventually not be necessary here Py_BEGIN_ALLOW_THREADS int expected = Cown_RELEASED; - // TODO: eventually replace this with something from pycore_atomic (nothing there now) + // TODO: Pyrona: eventually replace this with something from pycore_atomic (nothing there now) while (!atomic_compare_exchange_strong(&self->state._value, &expected, Cown_ACQUIRED)) { expected = Cown_RELEASED; sem_wait(&self->semaphore); @@ -155,11 +161,16 @@ static PyObject *PyCown_release(PyCownObject *self, PyObject *ignored) { BAIL_UNLESS_OWNED(self, "Thread attempted to release a cown it did not own"); - Py_BEGIN_ALLOW_THREADS + if (self->value && Py_TYPE(self->value) == &PyRegion_Type) { + if (PyCown_close_region(self->value) == NULL) { + // Close region failed -- propagate its error + return NULL; + } + } + self->owning_thread = 0; _Py_atomic_store(&self->state, Cown_RELEASED); sem_post(&self->semaphore); - Py_END_ALLOW_THREADS Py_RETURN_NONE; } @@ -174,6 +185,13 @@ int _PyCown_is_released(PyObject *self) { return STATE(cown) == Cown_RELEASED; } +int _PyCown_is_pending_release(PyObject *self) { + assert(Py_TYPE(self) == &PyCown_Type && "Is pending release called on non-cown!"); + + PyCownObject *cown = _Py_CAST(PyCownObject *, self); + return STATE(cown) == Cown_PENDING_RELEASE; +} + // The ignored argument is required for this function's type to be // compatible with PyCFunction static PyObject *PyCown_get(PyCownObject *self, PyObject *ignored) { @@ -186,16 +204,12 @@ static PyObject *PyCown_get(PyCownObject *self, PyObject *ignored) { } } -// Needed to test for region object -extern PyTypeObject PyRegion_Type; -extern PyTypeObject PyCown_Type; - static PyObject *PyCown_set_unchecked(PyCownObject *self, PyObject *arg) { // Cowns are cells that hold a reference to a bridge object, // (or another cown or immutable object) - const bool is_region_object = + const bool arg_is_region_object = Py_IS_TYPE(arg, &PyRegion_Type) && _Py_is_bridge_object(arg); - if (is_region_object || + if (arg_is_region_object || arg->ob_type == &PyCown_Type || _Py_IsImmutable(arg)) { @@ -205,10 +219,14 @@ static PyObject *PyCown_set_unchecked(PyCownObject *self, PyObject *arg) { // Tell the region that it is owned by a cown, // to enable it to release the cown on close - if (is_region_object) { + if (arg_is_region_object) { _PyRegion_set_cown_parent(arg, _PyObject_CAST(self)); + // TODO: Pyrona: should not run try close here unless dirty at the end of phase 3 + // if (_PyCown_close_region(arg) == Py_None) { if (_PyRegion_is_closed(arg)) { - PyCown_release(self, NULL); + if (PyCown_release(self, NULL) == NULL) { + PyErr_Clear(); + } } else { _Py_atomic_store(&self->state, Cown_PENDING_RELEASE); PyThreadState *tstate = PyThreadState_Get(); diff --git a/Objects/regions.c b/Objects/regions.c index 6cf0ad06a6814e..ab7e602e211663 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -219,8 +219,8 @@ static int regionmetadata_close(regionmetadata* self) { return regionmetadata_dec_osc(parent); } - // Check if in a cown -- if so, release cown - if (self->cown) { + // Check if in a cown which is waiting for the region to close -- if so, release cown + if (self->cown && _PyCown_is_pending_release(self->cown)) { // Propagate error from release return _PyCown_release(self->cown); } @@ -1749,13 +1749,16 @@ int _PyRegion_is_closed(PyObject* self) { // The ignored argument is required for this function's type to be // compatible with PyCFunction static PyObject *PyRegion_close(PyRegionObject *self, PyObject *ignored) { - regionmetadata* const md = REGION_DATA_CAST(Py_REGION(self)); - if (!regionmetadata_is_open(md)) { + if (PyRegion_is_closed(self)) { Py_RETURN_NONE; // Double close is OK } // Attempt to close the region if (try_close(self) != 0) { + if (!PyErr_Occurred()) { + // try_close did not run out of memory but failed to close the region + PyErr_Format(PyExc_RegionError, "Attempting to close the region failed"); + } return NULL; } @@ -2009,3 +2012,13 @@ void _Py_RegionRemoveReference(PyObject *src, PyObject *tgt) { // Unparent the region. regionmetadata_set_parent(tgt_md, NULL); } + +PyObject *_PyCown_close_region(PyObject* ob) { + if (Py_TYPE(ob) == &PyRegion_Type) { + // Attempt to close the region + return PyRegion_close(_Py_CAST(PyRegionObject*, ob), NULL); + } else { + PyErr_SetString(PyExc_RegionError, "Attempted to close a region through a non-bridge object"); + return NULL; + } +} From 349a598c44c1ebe2f183e90acacfb447ab60fc84 Mon Sep 17 00:00:00 2001 From: Tobias Wrigstad Date: Mon, 24 Feb 2025 17:50:16 +0100 Subject: [PATCH 65/68] Added region check to tuple object (#46) As far as I can see in the source code, tuples are always created in the local region (effectively), and get their arguments before they are assigned anywhere. So the only check added to set_item should always succeed. Leaving optimising this case for future work. --- Objects/tupleobject.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 45e09573d6d0d2..e043054197ae05 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -124,6 +124,14 @@ PyTuple_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) return -1; } + // TODO: Pyrona: Possibly optimise this case as tuples + // should always be in local when they are assigned. + if (!Py_REGIONADDREFERENCE(op, newitem)){ + Py_XDECREF(newitem); + // Error set by region add test + return -1; + } + if (i < 0 || i >= Py_SIZE(op)) { Py_XDECREF(newitem); PyErr_SetString(PyExc_IndexError, From 4b950cff472cbd011bde4fd1d201cb6c16d871ac Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 7 Apr 2025 13:16:59 +0200 Subject: [PATCH 66/68] Rename `regionmetadata` -> `regiondata` --- Objects/regions.c | 282 +++++++++++++++++++++++----------------------- 1 file changed, 141 insertions(+), 141 deletions(-) diff --git a/Objects/regions.c b/Objects/regions.c index ab7e602e211663..79140ddab91d16 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -13,11 +13,11 @@ #include "pycore_pyerrors.h" #include "pyerrors.h" -// This tag indicates that the `regionmetadata` object has been merged +// This tag indicates that the `regiondata` object has been merged // with another region. The `parent` pointer points to the region it was // merged with. // -// This tag is only used for the parent pointer in `regionmetadata`. +// This tag is only used for the parent pointer in `regiondata`. #define Py_METADATA_MERGE_TAG ((Py_region_ptr_t)0x2) static inline Py_region_ptr_with_tags_t Py_TAGGED_REGION(PyObject *ob) { return ob->ob_region; @@ -27,7 +27,7 @@ static inline Py_region_ptr_with_tags_t Py_TAGGED_REGION(PyObject *ob) { #define REGION_PTR_SET_TAG(ptr, tag) (ptr = Py_region_ptr_with_tags((ptr).value | tag)) #define REGION_PTR_CLEAR_TAG(ptr, tag) (ptr = Py_region_ptr_with_tags((ptr).value & (~tag))) -#define REGION_DATA_CAST(r) (_Py_CAST(regionmetadata*, (r))) +#define REGION_DATA_CAST(r) (_Py_CAST(regiondata*, (r))) #define REGION_PTR_CAST(r) (_Py_CAST(Py_region_ptr_t, (r))) #define Py_REGION_DATA(ob) (REGION_DATA_CAST(Py_REGION(ob))) #define Py_REGION_FIELD(ob) (ob->ob_region) @@ -37,10 +37,10 @@ static inline Py_region_ptr_with_tags_t Py_TAGGED_REGION(PyObject *ob) { #define IS_COWN_REGION(r) (REGION_PTR_CAST(r) == _Py_COWN) #define HAS_METADATA(r) (!IS_LOCAL_REGION(r) && !IS_IMMUTABLE_REGION(r) && !IS_COWN_REGION(r)) -typedef struct regionmetadata regionmetadata; +typedef struct regiondata regiondata; typedef struct PyRegionObject PyRegionObject; -static regionmetadata* regionmetadata_get_parent(regionmetadata* self); +static regiondata* regiondata_get_parent(regiondata* self); static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); static const char *get_region_name(PyObject* obj); @@ -101,11 +101,11 @@ static void throw_region_error( struct PyRegionObject { PyObject_HEAD - regionmetadata* metadata; + regiondata* metadata; PyObject *dict; }; -struct regionmetadata { +struct regiondata { // The number of references coming in from the local region. Py_ssize_t lrc; // The number of open subregions. @@ -130,11 +130,11 @@ struct regionmetadata { // it might make sense to make this conditional in debug builds (or something) // // Intrinsic list for invariant checking - regionmetadata* next; + regiondata* next; PyObject* cown; // To be able to release a cown; to be integrated with parent }; -static Py_region_ptr_t regionmetadata_get_merge_tree_root(Py_region_ptr_t self) +static Py_region_ptr_t regiondata_get_merge_tree_root(Py_region_ptr_t self) { // Test for local and immutable region if (!HAS_METADATA(self)) { @@ -142,7 +142,7 @@ static Py_region_ptr_t regionmetadata_get_merge_tree_root(Py_region_ptr_t self) } // Return self if it wasn't merged with another region - regionmetadata* self_data = REGION_DATA_CAST(self); + regiondata* self_data = REGION_DATA_CAST(self); if (!REGION_PRT_HAS_TAG(self_data->parent, Py_METADATA_MERGE_TAG)) { return self; } @@ -150,50 +150,50 @@ static Py_region_ptr_t regionmetadata_get_merge_tree_root(Py_region_ptr_t self) // FIXME: It can happen that there are several layers in this union-find // structure. It would be efficient to directly update the parent pointers // for deeper nodes. - return regionmetadata_get_merge_tree_root(Py_region_ptr(self_data->parent)); + return regiondata_get_merge_tree_root(Py_region_ptr(self_data->parent)); } -#define regionmetadata_get_merge_tree_root(self) \ - regionmetadata_get_merge_tree_root(REGION_PTR_CAST(self)) +#define regiondata_get_merge_tree_root(self) \ + regiondata_get_merge_tree_root(REGION_PTR_CAST(self)) -static void regionmetadata_mark_as_dirty(Py_region_ptr_t self_ptr) { +static void regiondata_mark_as_dirty(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { return; } REGION_DATA_CAST(self_ptr)->is_dirty = true; } -# define regionmetadata_mark_as_dirty(data) \ - (regionmetadata_mark_as_dirty(REGION_PTR_CAST(data))) +# define regiondata_mark_as_dirty(data) \ + (regiondata_mark_as_dirty(REGION_PTR_CAST(data))) -static void regionmetadata_mark_as_not_dirty(Py_region_ptr_t self_ptr) { +static void regiondata_mark_as_not_dirty(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { return; } REGION_DATA_CAST(self_ptr)->is_dirty = false; } -# define regionmetadata_mark_as_not_dirty(data) \ - (regionmetadata_mark_as_not_dirty(REGION_PTR_CAST(data))) +# define regiondata_mark_as_not_dirty(data) \ + (regiondata_mark_as_not_dirty(REGION_PTR_CAST(data))) -static bool regionmetadata_is_dirty(Py_region_ptr_t self_ptr) { +static bool regiondata_is_dirty(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { return false; } return REGION_DATA_CAST(self_ptr)->is_dirty; } -# define regionmetadata_is_dirty(data) \ - (regionmetadata_is_dirty(REGION_PTR_CAST(data))) +# define regiondata_is_dirty(data) \ + (regiondata_is_dirty(REGION_PTR_CAST(data))) -static void regionmetadata_inc_osc(Py_region_ptr_t self_ptr); -static int regionmetadata_dec_osc(Py_region_ptr_t self_ptr); -static void regionmetadata_open(regionmetadata* self) { +static void regiondata_inc_osc(Py_region_ptr_t self_ptr); +static int regiondata_dec_osc(Py_region_ptr_t self_ptr); +static void regiondata_open(regiondata* self) { assert(HAS_METADATA(self)); if (self->is_open) { return; } self->is_open = true; - regionmetadata_inc_osc(REGION_PTR_CAST(regionmetadata_get_parent(self))); + regiondata_inc_osc(REGION_PTR_CAST(regiondata_get_parent(self))); } /// This function marks the region as closed and propagartes the status to @@ -202,7 +202,7 @@ static void regionmetadata_open(regionmetadata* self) { /// It returns `0` if the close was successful. It should only fails, if the /// system is in an inconsistent state and this close attempted to release a /// cown which is currently not owned by the current thread. -static int regionmetadata_close(regionmetadata* self) { +static int regiondata_close(regiondata* self) { // The LRC might be 1 or 2, if the owning references is a local and the // bridge object was used as an argument. assert(self->lrc <= 2 && "Attempting to close a region with an LRC > 2"); @@ -213,10 +213,10 @@ static int regionmetadata_close(regionmetadata* self) { self->is_open = false; - Py_region_ptr_t parent = REGION_PTR_CAST(regionmetadata_get_parent(self)); + Py_region_ptr_t parent = REGION_PTR_CAST(regiondata_get_parent(self)); if (HAS_METADATA(parent)) { // Cowns and parents are mutually exclusive this can therefore return directly - return regionmetadata_dec_osc(parent); + return regiondata_dec_osc(parent); } // Check if in a cown which is waiting for the region to close -- if so, release cown @@ -229,7 +229,7 @@ static int regionmetadata_close(regionmetadata* self) { return 0; } -static bool regionmetadata_is_open(Py_region_ptr_t self) { +static bool regiondata_is_open(Py_region_ptr_t self) { if (!HAS_METADATA(self)) { // The immutable and local region are open by default and can't be closed. return true; @@ -237,63 +237,63 @@ static bool regionmetadata_is_open(Py_region_ptr_t self) { return REGION_DATA_CAST(self)->is_open; } -#define regionmetadata_is_open(self) \ - regionmetadata_is_open(REGION_PTR_CAST(self)) +#define regiondata_is_open(self) \ + regiondata_is_open(REGION_PTR_CAST(self)) -static void regionmetadata_inc_osc(Py_region_ptr_t self_ptr) +static void regiondata_inc_osc(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { return; } - regionmetadata* self = REGION_DATA_CAST(self_ptr); + regiondata* self = REGION_DATA_CAST(self_ptr); self->osc += 1; - regionmetadata_open(self); + regiondata_open(self); } -#define regionmetadata_inc_osc(self) \ - (regionmetadata_inc_osc(REGION_PTR_CAST(self))) +#define regiondata_inc_osc(self) \ + (regiondata_inc_osc(REGION_PTR_CAST(self))) /// Decrements the OSC of the region. This might close the region if the LRC /// and ORC both hit zero and the region is not marked as dirty. /// /// Returns `0` on success. An error might come from closing the region -/// see `regionmetadata_close` for potential errors. -static int regionmetadata_dec_osc(Py_region_ptr_t self_ptr) +/// see `regiondata_close` for potential errors. +static int regiondata_dec_osc(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { return 0; } - regionmetadata* self = REGION_DATA_CAST(self_ptr); + regiondata* self = REGION_DATA_CAST(self_ptr); self->osc -= 1; // Check if the OSC decrease has closed this region as well. - if (self->osc == 0 && self->lrc == 0 && !regionmetadata_is_dirty(self)) { - return regionmetadata_close(self); + if (self->osc == 0 && self->lrc == 0 && !regiondata_is_dirty(self)) { + return regiondata_close(self); } return 0; } -#define regionmetadata_dec_osc(self) \ - (regionmetadata_dec_osc(REGION_PTR_CAST(self))) +#define regiondata_dec_osc(self) \ + (regiondata_dec_osc(REGION_PTR_CAST(self))) -static void regionmetadata_inc_rc(Py_region_ptr_t self) +static void regiondata_inc_rc(Py_region_ptr_t self) { if (HAS_METADATA(self)) { REGION_DATA_CAST(self)->rc += 1; } } -#define regionmetadata_inc_rc(self) \ - (regionmetadata_inc_rc(REGION_PTR_CAST(self))) +#define regiondata_inc_rc(self) \ + (regiondata_inc_rc(REGION_PTR_CAST(self))) -static int regionmetadata_dec_rc(Py_region_ptr_t self_ptr) +static int regiondata_dec_rc(Py_region_ptr_t self_ptr) { if (!HAS_METADATA(self_ptr)) { return 0; } // Update RC - regionmetadata* self = REGION_DATA_CAST(self_ptr); + regiondata* self = REGION_DATA_CAST(self_ptr); self->rc -= 1; if (self->rc != 0) { return 0; @@ -305,69 +305,69 @@ static int regionmetadata_dec_rc(Py_region_ptr_t self_ptr) // Buffer the results since we don't want to leak any memory if this fails. // OSC decreases in this function should also be safe. int result = 0; - if (regionmetadata_is_open(self)) { - result |= regionmetadata_dec_osc(regionmetadata_get_parent(self)); + if (regiondata_is_open(self)) { + result |= regiondata_dec_osc(regiondata_get_parent(self)); } // This access the parent directly to update the rc. // It also doesn't matter if the parent pointer is a // merge or subregion relation, since both cases have // increased the rc. - result |= regionmetadata_dec_rc(Py_region_ptr(self->parent)); + result |= regiondata_dec_rc(Py_region_ptr(self->parent)); free(self); return result; } -#define regionmetadata_dec_rc(self) \ - (regionmetadata_dec_rc(REGION_PTR_CAST(self))) +#define regiondata_dec_rc(self) \ + (regiondata_dec_rc(REGION_PTR_CAST(self))) -static void regionmetadata_set_parent(regionmetadata* self, regionmetadata* parent) { +static void regiondata_set_parent(regiondata* self, regiondata* parent) { // Just a sanity check, since these cases should never happen assert(HAS_METADATA(self) && "Can't set the parent on the immutable and local region"); - assert(REGION_PTR_CAST(self) == regionmetadata_get_merge_tree_root(self) && "Sanity Check"); - assert(REGION_PTR_CAST(parent) == regionmetadata_get_merge_tree_root(parent) && "Sanity Check"); + assert(REGION_PTR_CAST(self) == regiondata_get_merge_tree_root(self) && "Sanity Check"); + assert(REGION_PTR_CAST(parent) == regiondata_get_merge_tree_root(parent) && "Sanity Check"); Py_region_ptr_t old_parent = Py_region_ptr(self->parent); Py_region_ptr_t new_parent = REGION_PTR_CAST(parent); self->parent = Py_region_ptr_with_tags(new_parent); // Update RCs - regionmetadata_inc_rc(new_parent); - if (regionmetadata_is_open(self)) { - regionmetadata_inc_osc(new_parent); - regionmetadata_dec_osc(old_parent); + regiondata_inc_rc(new_parent); + if (regiondata_is_open(self)) { + regiondata_inc_osc(new_parent); + regiondata_dec_osc(old_parent); } - regionmetadata_dec_rc(old_parent); + regiondata_dec_rc(old_parent); } -static regionmetadata* regionmetadata_get_parent(regionmetadata* self) { - assert(REGION_PTR_CAST(self) == regionmetadata_get_merge_tree_root(self) && "Sanity check"); +static regiondata* regiondata_get_parent(regiondata* self) { + assert(REGION_PTR_CAST(self) == regiondata_get_merge_tree_root(self) && "Sanity check"); if (!HAS_METADATA(self)) { // The local and immutable regions never have a parent return NULL; } Py_region_ptr_t parent_field = Py_region_ptr(self->parent); - Py_region_ptr_t parent_root = regionmetadata_get_merge_tree_root(parent_field); + Py_region_ptr_t parent_root = regiondata_get_merge_tree_root(parent_field); // If the parent was merged with another region we want to update the // pointer to point at the root. if (parent_field != parent_root) { // set_parent ensures that the RC's are correctly updated - regionmetadata_set_parent(self, REGION_DATA_CAST(parent_root)); + regiondata_set_parent(self, REGION_DATA_CAST(parent_root)); } return REGION_DATA_CAST(parent_root); } -#define regionmetadata_get_parent(self) \ - regionmetadata_get_parent(REGION_DATA_CAST(self)) +#define regiondata_get_parent(self) \ + regiondata_get_parent(REGION_DATA_CAST(self)) -static bool regionmetadata_has_parent(regionmetadata* self) { - return regionmetadata_get_parent(self) != NULL; +static bool regiondata_has_parent(regiondata* self) { + return regiondata_get_parent(self) != NULL; } -static bool regionmetadata_has_ancestor(regionmetadata* self, regionmetadata* other) { +static bool regiondata_has_ancestor(regiondata* self, regiondata* other) { // The immutable or local region can never be a parent if (!HAS_METADATA(other)) { return false; @@ -377,7 +377,7 @@ static bool regionmetadata_has_ancestor(regionmetadata* self, regionmetadata* ot if (self == other) { return true; } - self = regionmetadata_get_parent(self); + self = regiondata_get_parent(self); } return false; } @@ -389,34 +389,34 @@ static bool regionmetadata_has_ancestor(regionmetadata* self, regionmetadata* ot // it's parent. // // This function expects `self` to be a valid object. -static PyObject* regionmetadata_merge(regionmetadata* self, Py_region_ptr_t other) { +static PyObject* regiondata_merge(regiondata* self, Py_region_ptr_t other) { assert(HAS_METADATA(self) && "The immutable and local region can't be merged into another region"); - assert(REGION_PTR_CAST(self) == regionmetadata_get_merge_tree_root(self) && "Sanity Check"); + assert(REGION_PTR_CAST(self) == regiondata_get_merge_tree_root(self) && "Sanity Check"); // If `other` is the parent of `self` we can merge it. We unset the the // parent which will also update the rc and other counts. - regionmetadata* self_parent = regionmetadata_get_parent(self); + regiondata* self_parent = regiondata_get_parent(self); if (self_parent && REGION_PTR_CAST(self_parent) == other) { assert(HAS_METADATA(self_parent) && "The immutable and local region can never have children"); - regionmetadata_set_parent(self, NULL); + regiondata_set_parent(self, NULL); self_parent = NULL; } // If only `self` has a parent we can make `other` the child and // remove the parent from `self`. The merged region will then again // have the correct parent. - regionmetadata* other_parent = regionmetadata_get_parent(self); + regiondata* other_parent = regiondata_get_parent(self); if (self_parent && HAS_METADATA(other) && other_parent == NULL) { // Make sure we don't create any cycles - if (regionmetadata_has_ancestor(self_parent, REGION_DATA_CAST(other))) { + if (regiondata_has_ancestor(self_parent, REGION_DATA_CAST(other))) { throw_region_error(self->bridge, REGION_DATA_CAST(other)->bridge, "Merging these regions would create a cycle", NULL); return NULL; } - regionmetadata_set_parent(REGION_DATA_CAST(other), self_parent); - regionmetadata_set_parent(self, NULL); + regiondata_set_parent(REGION_DATA_CAST(other), self_parent); + regiondata_set_parent(self, NULL); self_parent = NULL; } @@ -431,11 +431,11 @@ static PyObject* regionmetadata_merge(regionmetadata* self, Py_region_ptr_t othe return NULL; } - regionmetadata_inc_rc(other); + regiondata_inc_rc(other); // Merge state into the root. if (HAS_METADATA(other)) { - regionmetadata* other_data = REGION_DATA_CAST(other); + regiondata* other_data = REGION_DATA_CAST(other); other_data->lrc += self->lrc; other_data->osc += self->osc; other_data->is_open |= self->is_open; @@ -451,12 +451,12 @@ static PyObject* regionmetadata_merge(regionmetadata* self, Py_region_ptr_t othe self->parent = Py_region_ptr_with_tags(other); REGION_PTR_SET_TAG(self->parent, Py_METADATA_MERGE_TAG); // No decref, since this is a weak reference. Otherwise we would get - // a cycle between the `regionmetadata` as a non GC'ed object and the bridge. + // a cycle between the `regiondata` as a non GC'ed object and the bridge. self->bridge = NULL; Py_RETURN_NONE; } -#define regionmetadata_merge(self, other) \ - (regionmetadata_merge(self, REGION_PTR_CAST(other))); +#define regiondata_merge(self, other) \ + (regiondata_merge(self, REGION_PTR_CAST(other))); int _Py_IsLocal(PyObject *op) { return IS_LOCAL_REGION(Py_REGION(op)); @@ -481,7 +481,7 @@ Py_region_ptr_t _Py_REGION(PyObject *ob) { return field_value; } - Py_region_ptr_t region = regionmetadata_get_merge_tree_root(field_value); + Py_region_ptr_t region = regiondata_get_merge_tree_root(field_value); // Update the region if we're not pointing to the root of the merge tree. // This can allow freeing of non root regions and speedup future lookups. if (region != field_value) { @@ -501,8 +501,8 @@ void _Py_SET_TAGGED_REGION(PyObject *ob, Py_region_ptr_with_tags_t region) { ob->ob_region = region; // Update the RC of the region - regionmetadata_inc_rc(Py_region_ptr(region)); - regionmetadata_dec_rc(old_region); + regiondata_inc_rc(Py_region_ptr(region)); + regiondata_dec_rc(old_region); } /** @@ -585,8 +585,8 @@ static bool is_c_wrapper(PyObject* obj){ // Start of a linked list of bridge objects used to check for external uniqueness // Bridge objects appear in this list if they are captured -#define CAPTURED_SENTINEL ((regionmetadata*) 0xc0defefe) -regionmetadata* captured = CAPTURED_SENTINEL; +#define CAPTURED_SENTINEL ((regiondata*) 0xc0defefe) +regiondata* captured = CAPTURED_SENTINEL; /** * Enable the region check. @@ -694,13 +694,13 @@ visit_invariant_check(PyObject *tgt, void *src_void) return 0; } - regionmetadata* src_region = REGION_DATA_CAST(src_region_ptr); + regiondata* src_region = REGION_DATA_CAST(src_region_ptr); // Region objects may be stored in cowns if (IS_COWN_REGION(src_region)) { return 0; } - regionmetadata* tgt_region = REGION_DATA_CAST(tgt_region_ptr); + regiondata* tgt_region = REGION_DATA_CAST(tgt_region_ptr); // Check if region is already added to captured list if (tgt_region->next != NULL) { // Bridge object was already captured @@ -708,7 +708,7 @@ visit_invariant_check(PyObject *tgt, void *src_void) return 0; } // Forbid cycles in the region topology - if (regionmetadata_has_ancestor(src_region, tgt_region)) { + if (regiondata_has_ancestor(src_region, tgt_region)) { emit_invariant_error(src, tgt, "Regions create a cycle with subregions"); return 0; } @@ -723,7 +723,7 @@ visit_invariant_check(PyObject *tgt, void *src_void) static void invariant_reset_captured_list(void) { // Reset the captured list while (captured != CAPTURED_SENTINEL) { - regionmetadata* m = captured; + regiondata* m = captured; captured = m->next; m->next = NULL; } @@ -1271,7 +1271,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) return 0; } - regionmetadata* source_region = Py_REGION_DATA(info->src); + regiondata* source_region = Py_REGION_DATA(info->src); if (Py_IsLocal(target)) { // Add reference to the object, // minus one for the reference we just followed @@ -1319,16 +1319,16 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // The target is a bridge object from another region. We now need to // if it already has a parent. - regionmetadata *target_region = Py_REGION_DATA(target); - if (regionmetadata_has_parent(target_region)) { + regiondata *target_region = Py_REGION_DATA(target); + if (regiondata_has_parent(target_region)) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_SHARED_CUSTODY}; return emit_region_error(&err); } // Make sure that the new subregion relation won't create a cycle - regionmetadata* region = Py_REGION_DATA(info->src); - if (regionmetadata_has_ancestor(region, target_region)) { + regiondata* region = Py_REGION_DATA(info->src); + if (regiondata_has_ancestor(region, target_region)) { regionerror err = {.src = info->src, .tgt = target, .id = ERR_CYCLE_CREATION}; return emit_region_error(&err); @@ -1339,7 +1339,7 @@ static int _add_to_region_visit(PyObject* target, void* info_void) // reference. // // `set_parent` will also ensure that the `osc` counter is updated. - regionmetadata_set_parent(target_region, region); + regiondata_set_parent(target_region, region); if (info->new_sub_regions) { if (stack_push(info->new_sub_regions, target)) { PyErr_NoMemory(); @@ -1398,7 +1398,7 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) // The current implementation assumes region is a valid pointer. This // restriction can be lifted if needed assert(HAS_METADATA(region)); - regionmetadata *region_data = REGION_DATA_CAST(region); + regiondata *region_data = REGION_DATA_CAST(region); // Early return if the object is already in the region or immutable if (Py_REGION(obj) == region || Py_IsImmutable(obj)) { @@ -1406,7 +1406,7 @@ static PyObject *add_to_region(PyObject *obj, Py_region_ptr_t region) } // Mark the region as open, since we're adding stuff to it. - regionmetadata_open(region_data); + regiondata_open(region_data); addtoregionvisitinfo info = { .pending = stack_new(), @@ -1453,7 +1453,7 @@ int _Py_is_bridge_object(PyObject *op) { // will use the properties of a bridge object. This therefore checks if // the object is equal to the regions bridge object rather than checking // that the type is `PyRegionObject` - return ((Py_region_ptr_t)((regionmetadata*)region)->bridge == (Py_region_ptr_t)op); + return ((Py_region_ptr_t)((regiondata*)region)->bridge == (Py_region_ptr_t)op); } /// This function attempts to close a region. It does this, by first merging @@ -1468,7 +1468,7 @@ int _Py_is_bridge_object(PyObject *op) { /// This function returns `-1` if any errors occurred. This can be due to /// memory problems, region errors or problems with releasing cowns not owned /// by the current thread. `0` only indicates that the function didn't error. -/// `regionmetadata_is_open()` should be used to check the region status. +/// `regiondata_is_open()` should be used to check the region status. static int try_close(PyRegionObject *root_bridge) { addtoregionvisitinfo info = { .pending = stack_new(), @@ -1488,8 +1488,8 @@ static int try_close(PyRegionObject *root_bridge) { // The root region can have to have two local references, one from the // owning reference and one from the `self` argument Py_ssize_t root_region_lrc_limit; - regionmetadata *root_data = Py_REGION_DATA(root_bridge); - if (regionmetadata_has_parent(root_data) || root_data->cown) { + regiondata *root_data = Py_REGION_DATA(root_bridge); + if (regiondata_has_parent(root_data) || root_data->cown) { root_region_lrc_limit = 1; } else { root_region_lrc_limit = 2; @@ -1498,7 +1498,7 @@ static int try_close(PyRegionObject *root_bridge) { while (!stack_empty(info.new_sub_regions)) { PyObject *bridge = stack_pop(info.new_sub_regions); assert(Py_is_bridge_object(bridge)); - regionmetadata* old_data = Py_REGION_DATA(bridge); + regiondata* old_data = Py_REGION_DATA(bridge); // One from the owning reference Py_ssize_t rc_limit = 1; @@ -1519,16 +1519,16 @@ static int try_close(PyRegionObject *root_bridge) { // objects to the bridge object will also increase the RC thereby tricking // this check into opening it again. if (Py_REFCNT(bridge) > rc_limit) { - regionmetadata_open(Py_REGION_DATA(bridge)); + regiondata_open(Py_REGION_DATA(bridge)); } // If it's closed there is nothing we need to do. - if (!regionmetadata_is_open(old_data)) { + if (!regiondata_is_open(old_data)) { continue; } - // Create the new `regionmetadata*` - regionmetadata* new_data = (regionmetadata*)calloc(1, sizeof(regionmetadata)); + // Create the new `regiondata*` + regiondata* new_data = (regiondata*)calloc(1, sizeof(regiondata)); if (!new_data) { PyErr_NoMemory(); goto fail; @@ -1542,22 +1542,22 @@ static int try_close(PyRegionObject *root_bridge) { // will remain `>= 1` until the region field of the bridge object is // updated by `Py_SET_REGION(bridge, new_data);` This ensures that // `old_data` stays valid while all the data is transferred to `new_data` - regionmetadata_dec_rc(bridge_obj->metadata); + regiondata_dec_rc(bridge_obj->metadata); bridge_obj->metadata = new_data; - regionmetadata_inc_rc(new_data); + regiondata_inc_rc(new_data); new_data->bridge = bridge_obj; Py_XSETREF(new_data->name, old_data->name); - regionmetadata_open(new_data); - regionmetadata_set_parent(new_data, regionmetadata_get_parent(old_data)); + regiondata_open(new_data); + regiondata_set_parent(new_data, regiondata_get_parent(old_data)); new_data->cown = old_data->cown; old_data->cown = NULL; // Merge the old region data into local. This has to be done after the // created of the `new_data` to prevent the parent from closing // premeturely when the old data gets detached from it. - regionmetadata_set_parent(old_data, NULL); - regionmetadata_merge(old_data, _Py_LOCAL_REGION); + regiondata_set_parent(old_data, NULL); + regiondata_merge(old_data, _Py_LOCAL_REGION); old_data = NULL; // This region update also triggers an RC decrease on `old_data`. @@ -1568,14 +1568,14 @@ static int try_close(PyRegionObject *root_bridge) { // Only subtract 1 from the LRC if the reference comes from a parent. // Owning references from the local region should still count towards // the LRC. - if (regionmetadata_has_parent(new_data) || new_data->cown) { + if (regiondata_has_parent(new_data) || new_data->cown) { new_data->lrc -= 1; } if (stack_push(info.pending, bridge)) { // No more memory, make sure the region is marked as dirty thereby // preventing it from being closed in an inconsitent state. - regionmetadata_mark_as_dirty(Py_REGION_DATA(root_bridge)); + regiondata_mark_as_dirty(Py_REGION_DATA(root_bridge)); goto fail; } @@ -1591,13 +1591,13 @@ static int try_close(PyRegionObject *root_bridge) { // to the region. // // Either way, this means that the LRC of the region can't be trusted. - regionmetadata_mark_as_dirty(Py_REGION_DATA(root_bridge)); + regiondata_mark_as_dirty(Py_REGION_DATA(root_bridge)); goto fail; } } // Mark the region as clean - regionmetadata_mark_as_not_dirty(new_data); + regiondata_mark_as_not_dirty(new_data); // The LRC will never decrease after this point. If the region is open // due to the LRC it will remain open and the close fails. @@ -1607,8 +1607,8 @@ static int try_close(PyRegionObject *root_bridge) { // Update the open status and make sure the parent knows if (new_data->osc == 0) { - if (regionmetadata_close(new_data) != 0) { - // See `regionmetadata_close` for when this can fail. + if (regiondata_close(new_data) != 0) { + // See `regiondata_close` for when this can fail. // In either case, this region has just been cleaned and should // be in a consistent state. goto fail; @@ -1618,8 +1618,8 @@ static int try_close(PyRegionObject *root_bridge) { root_data = Py_REGION_DATA(root_bridge); if (root_data->lrc <= root_region_lrc_limit && root_data->osc == 0) { - if (regionmetadata_close(root_data) != 0) { - // See `regionmetadata_close` for when this can fail. + if (regiondata_close(root_data) != 0) { + // See `regiondata_close` for when this can fail. // In either case, this region has just been cleaned and should // be in a consistent state. goto fail; @@ -1646,10 +1646,10 @@ static void PyRegion_dealloc(PyRegionObject *self) { // The object region has already been reset. // We now need to update the RC of our metadata field. if (self->metadata) { - regionmetadata* data = self->metadata; + regiondata* data = self->metadata; self->metadata = NULL; data->bridge = NULL; - regionmetadata_dec_rc(data); + regiondata_dec_rc(data); } PyTypeObject *tp = Py_TYPE(self); @@ -1675,14 +1675,14 @@ static int PyRegion_init(PyRegionObject *self, PyObject *args, PyObject *kwds) { _Py_MakeImmutable(_PyObject_CAST(Py_TYPE(self))); static char *kwlist[] = {"name", NULL}; - self->metadata = (regionmetadata*)calloc(1, sizeof(regionmetadata)); + self->metadata = (regiondata*)calloc(1, sizeof(regiondata)); if (!self->metadata) { PyErr_NoMemory(); return -1; } // Make sure the internal reference is also counted. - regionmetadata_inc_rc(self->metadata); + regiondata_inc_rc(self->metadata); self->metadata->bridge = self; @@ -1727,7 +1727,7 @@ static int PyRegion_clear(PyRegionObject *self) { static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *ignored) { // FIXME: What is the behavior of a `PyRegionObject` that has been merged into another region? assert(Py_is_bridge_object(_PyObject_CAST(self)) && "FIXME: When does this happend and what should it do?"); - return PyBool_FromLong(_Py_CAST(long, regionmetadata_is_open(self->metadata))); + return PyBool_FromLong(_Py_CAST(long, regiondata_is_open(self->metadata))); } // Open method (sets the region to "open") @@ -1736,7 +1736,7 @@ static PyObject *PyRegion_is_open(PyRegionObject *self, PyObject *ignored) { static PyObject *PyRegion_open(PyRegionObject *self, PyObject *ignored) { // `Py_REGION()` will fetch the root region of the merge tree. // this might be different from the region in `self->metadata`. - regionmetadata_open(Py_REGION_DATA(self)); + regiondata_open(Py_REGION_DATA(self)); Py_RETURN_NONE; // Return None (standard for methods with no return value) } @@ -1763,7 +1763,7 @@ static PyObject *PyRegion_close(PyRegionObject *self, PyObject *ignored) { } // Check if the region is now closed - if (regionmetadata_is_open(Py_REGION(self))) { + if (regiondata_is_open(Py_REGION(self))) { PyErr_Format(PyExc_RegionError, "Attempting to close the region failed"); return NULL; } @@ -1781,7 +1781,7 @@ static PyObject *PyRegion_try_close(PyRegionObject *self, PyObject *args) { } // Check if the region was closed - return PyBool_FromLong(_Py_CAST(long, !regionmetadata_is_open(Py_REGION(self)))); + return PyBool_FromLong(_Py_CAST(long, !regiondata_is_open(Py_REGION(self)))); } // Adds args object to self region @@ -1799,7 +1799,7 @@ static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args) { Py_RETURN_NONE; } - regionmetadata* md = Py_REGION_DATA(self); + regiondata* md = Py_REGION_DATA(self); if (Py_REGION(args) == (Py_region_ptr_t) md) { Py_SET_REGION(args, _Py_LOCAL_REGION); Py_RETURN_NONE; @@ -1819,7 +1819,7 @@ static PyObject *PyRegion_owns_object(PyRegionObject *self, PyObject *args) { } static PyObject *PyRegion_repr(PyRegionObject *self) { - regionmetadata* data = Py_REGION_DATA(self); + regiondata* data = Py_REGION_DATA(self); #ifdef NDEBUG // Debug mode: include detailed representation return PyUnicode_FromFormat( @@ -1913,7 +1913,7 @@ static const char *get_region_name(PyObject* obj) { } else if (_Py_IsCown(obj)) { return "Cown"; } else { - const regionmetadata *md = Py_REGION_DATA(obj); + const regiondata *md = Py_REGION_DATA(obj); return md->name ? PyUnicode_AsUTF8(md->name) : ""; @@ -1972,7 +1972,7 @@ bool _Py_RegionAddReferences(PyObject *src, int tgtc, ...) { void _PyRegion_set_cown_parent(PyObject* bridge, PyObject* cown) { assert(Py_is_bridge_object(bridge)); - regionmetadata* data = Py_REGION_DATA(bridge); + regiondata* data = Py_REGION_DATA(bridge); Py_XINCREF(cown); Py_XSETREF(data->cown, cown); } @@ -1992,7 +1992,7 @@ void _Py_RegionRemoveReference(PyObject *src, PyObject *tgt) { return; } - regionmetadata* tgt_md = Py_REGION_DATA(tgt); + regiondata* tgt_md = Py_REGION_DATA(tgt); if (_Py_IsLocal(src)) { // Dec LRC of the previously referenced region // TODO should this decrement be a function, if it hits zero, @@ -2002,15 +2002,15 @@ void _Py_RegionRemoveReference(PyObject *src, PyObject *tgt) { } // This must be a parent reference, so we need to remove the parent reference. - regionmetadata* src_md = Py_REGION_DATA(src); - regionmetadata* tgt_parent_md = REGION_DATA_CAST(Py_region_ptr(tgt_md->parent)); + regiondata* src_md = Py_REGION_DATA(src); + regiondata* tgt_parent_md = REGION_DATA_CAST(Py_region_ptr(tgt_md->parent)); if (tgt_parent_md != src_md) { // TODO: Could `dirty` mean this isn't an error? _PyErr_Region(src, tgt, "(in WB/remove_ref)"); } // Unparent the region. - regionmetadata_set_parent(tgt_md, NULL); + regiondata_set_parent(tgt_md, NULL); } PyObject *_PyCown_close_region(PyObject* ob) { From 0cd4fca4db3e286e06d6847cdeb562b258901b30 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Tue, 8 Apr 2025 11:30:37 +0200 Subject: [PATCH 67/68] Pyrona: Remove global invariant objects --- Include/internal/pycore_regions.h | 6 --- Objects/regions.c | 66 ++++++++----------------------- Python/bltinmodule.c | 28 ------------- Python/clinic/bltinmodule.c.h | 38 +----------------- 4 files changed, 17 insertions(+), 121 deletions(-) diff --git a/Include/internal/pycore_regions.h b/Include/internal/pycore_regions.h index 6a2c808ec2987a..2996b95d81b578 100644 --- a/Include/internal/pycore_regions.h +++ b/Include/internal/pycore_regions.h @@ -44,12 +44,6 @@ static inline void _Py_SET_REGION(PyObject *ob, Py_region_ptr_t region) { PyObject* _Py_MakeImmutable(PyObject* obj); #define Py_MakeImmutable(op) _Py_MakeImmutable(_PyObject_CAST(op)) -PyObject* _Py_InvariantSrcFailure(void); -#define Py_InvariantSrcFailure() _Py_InvariantSrcFailure() - -PyObject* _Py_InvariantTgtFailure(void); -#define Py_InvariantTgtFailure() _Py_InvariantTgtFailure() - PyObject* _Py_EnableInvariant(void); #define Py_EnableInvariant() _Py_EnableInvariant() diff --git a/Objects/regions.c b/Objects/regions.c index 79140ddab91d16..999164e3121472 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -44,7 +44,6 @@ static regiondata* regiondata_get_parent(regiondata* self); static PyObject *PyRegion_add_object(PyRegionObject *self, PyObject *args); static PyObject *PyRegion_remove_object(PyRegionObject *self, PyObject *args); static const char *get_region_name(PyObject* obj); -static void _PyErr_Region(PyObject *src, PyObject *tgt, const char *msg); /** * Global status for performing the region check. @@ -57,12 +56,6 @@ int Py_is_invariant_enabled(void) { return invariant_do_region_check; } -// The src object for an edge that invalidated the invariant. -PyObject* invariant_error_src = Py_None; - -// The tgt object for an edge that invalidated the invariant. -PyObject* invariant_error_tgt = Py_None; - // Once an error has occurred this is used to surpress further checking bool invariant_error_occurred = false; @@ -604,11 +597,6 @@ PyObject* _Py_EnableInvariant(void) invariant_error_occurred = false; // Re-enable region check invariant_do_region_check = true; - // Reset the error state - Py_DecRef(invariant_error_src); - invariant_error_src = Py_None; - Py_DecRef(invariant_error_tgt); - invariant_error_tgt = Py_None; return Py_None; } @@ -618,39 +606,27 @@ PyObject* _Py_EnableInvariant(void) */ static void emit_invariant_error(PyObject* src, PyObject* tgt, const char* msg) { - Py_DecRef(invariant_error_src); - Py_IncRef(src); - invariant_error_src = src; - Py_DecRef(invariant_error_tgt); - Py_IncRef(tgt); - invariant_error_tgt = tgt; - - /* Don't stomp existing exception */ - PyThreadState *tstate = _PyThreadState_GET(); - assert(tstate && "_PyThreadState_GET documentation says it's not safe, when?"); - if (_PyErr_Occurred(tstate)) { + const char *tgt_region_name = get_region_name(tgt); + const char *src_region_name = get_region_name(src); + PyObject *src_type_repr = PyObject_Repr(PyObject_Type(src)); + const char *src_desc = src_type_repr ? PyUnicode_AsUTF8(src_type_repr) : "<>"; + PyObject *tgt_type_repr = PyObject_Repr(PyObject_Type(tgt)); + const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; + PyObject* formatted = PyUnicode_FromFormat( + "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", + src, src_desc, src_region_name, tgt, tgt_desc, tgt_region_name, msg); + + // If the formatting failes, we have bigger problems + if (!formatted) { return; } - _PyErr_Region(src, tgt, msg); + const char* formatted_str = PyUnicode_AsUTF8(formatted); + throw_region_error(src, tgt, formatted_str, Py_None); - // We have discovered a failure. - // Disable region check, until the program switches it back on. - invariant_do_region_check = false; - invariant_error_occurred = true; + Py_DECREF(formatted); } -PyObject* _Py_InvariantSrcFailure(void) -{ - return Py_NewRef(invariant_error_src); -} - -PyObject* _Py_InvariantTgtFailure(void) -{ - return Py_NewRef(invariant_error_tgt); -} - - // Lifted from gcmodule.c typedef struct _gc_runtime_state GCState; #define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) @@ -1895,16 +1871,6 @@ PyTypeObject PyRegion_Type = { PyType_GenericNew, /* tp_new */ }; -void _PyErr_Region(PyObject *src, PyObject *tgt, const char *msg) { - const char *tgt_region_name = get_region_name(tgt); - const char *src_region_name = get_region_name(src); - PyObject *src_type_repr = PyObject_Repr(PyObject_Type(src)); - const char *src_desc = src_type_repr ? PyUnicode_AsUTF8(src_type_repr) : "<>"; - PyObject *tgt_type_repr = PyObject_Repr(PyObject_Type(tgt)); - const char *tgt_desc = tgt_type_repr ? PyUnicode_AsUTF8(tgt_type_repr) : "<>"; - PyErr_Format(PyExc_RuntimeError, "Error: Invalid edge %p (%s in %s) -> %p (%s in %s) %s\n", src, src_desc, src_region_name, tgt, tgt_desc, tgt_region_name, msg); -} - static const char *get_region_name(PyObject* obj) { if (_Py_IsLocal(obj)) { return "Default"; @@ -2006,7 +1972,7 @@ void _Py_RegionRemoveReference(PyObject *src, PyObject *tgt) { regiondata* tgt_parent_md = REGION_DATA_CAST(Py_region_ptr(tgt_md->parent)); if (tgt_parent_md != src_md) { // TODO: Could `dirty` mean this isn't an error? - _PyErr_Region(src, tgt, "(in WB/remove_ref)"); + throw_region_error(src, tgt, "(in WB/remove_ref)", Py_None); } // Unparent the region. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 89d51a9d7d3bab..bfbd9cfaa5d53c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2776,32 +2776,6 @@ builtin_makeimmutable(PyObject *module, PyObject *obj) return Py_MakeImmutable(obj); } -/*[clinic input] -invariant_failure_src as builtin_invariantsrcfailure - -Find the source of an invariant failure. -[clinic start generated code]*/ - -static PyObject * -builtin_invariantsrcfailure_impl(PyObject *module) -/*[clinic end generated code: output=8830901cbbefe8ba input=0266aae8308be0a4]*/ -{ - return Py_InvariantSrcFailure(); -} - -/*[clinic input] -invariant_failure_tgt as builtin_invarianttgtfailure - -Find the target of an invariant failure. -[clinic start generated code]*/ - -static PyObject * -builtin_invarianttgtfailure_impl(PyObject *module) -/*[clinic end generated code: output=f7c9cd7cb737bd13 input=9c79a563d1eb52f9]*/ -{ - return Py_InvariantTgtFailure(); -} - /*[clinic input] enableinvariant as builtin_enableinvariant @@ -3116,8 +3090,6 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISSUBCLASS_METHODDEF BUILTIN_ISIMMUTABLE_METHODDEF BUILTIN_MAKEIMMUTABLE_METHODDEF - BUILTIN_INVARIANTSRCFAILURE_METHODDEF - BUILTIN_INVARIANTTGTFAILURE_METHODDEF BUILTIN_ITER_METHODDEF BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 34d29e1b5c81a2..8f9edc96f78e5a 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1428,42 +1428,6 @@ PyDoc_STRVAR(builtin_makeimmutable__doc__, #define BUILTIN_MAKEIMMUTABLE_METHODDEF \ {"makeimmutable", (PyCFunction)builtin_makeimmutable, METH_O, builtin_makeimmutable__doc__}, -PyDoc_STRVAR(builtin_invariantsrcfailure__doc__, -"invariant_failure_src($module, /)\n" -"--\n" -"\n" -"Find the source of an invariant failure."); - -#define BUILTIN_INVARIANTSRCFAILURE_METHODDEF \ - {"invariant_failure_src", (PyCFunction)builtin_invariantsrcfailure, METH_NOARGS, builtin_invariantsrcfailure__doc__}, - -static PyObject * -builtin_invariantsrcfailure_impl(PyObject *module); - -static PyObject * -builtin_invariantsrcfailure(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return builtin_invariantsrcfailure_impl(module); -} - -PyDoc_STRVAR(builtin_invarianttgtfailure__doc__, -"invariant_failure_tgt($module, /)\n" -"--\n" -"\n" -"Find the target of an invariant failure."); - -#define BUILTIN_INVARIANTTGTFAILURE_METHODDEF \ - {"invariant_failure_tgt", (PyCFunction)builtin_invarianttgtfailure, METH_NOARGS, builtin_invarianttgtfailure__doc__}, - -static PyObject * -builtin_invarianttgtfailure_impl(PyObject *module); - -static PyObject * -builtin_invarianttgtfailure(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return builtin_invarianttgtfailure_impl(module); -} - PyDoc_STRVAR(builtin_enableinvariant__doc__, "enableinvariant($module, /)\n" "--\n" @@ -1481,4 +1445,4 @@ builtin_enableinvariant(PyObject *module, PyObject *Py_UNUSED(ignored)) { return builtin_enableinvariant_impl(module); } -/*[clinic end generated code: output=3a883fa08bbd248e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2dcfa0885e1e7a45 input=a9049054013a1b77]*/ From 5b0724d9497a15c9b1cbb2b1a312a0d01eaffbf9 Mon Sep 17 00:00:00 2001 From: xFrednet Date: Mon, 12 May 2025 14:58:59 +0200 Subject: [PATCH 68/68] Pyrona: Add and remove reference fixes in `PyObjectDict` --- Objects/dictobject.c | 13 ++++++------- Objects/regions.c | 4 ++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 5556795b1d919b..b0e7db70e3b6b2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5895,21 +5895,18 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, if (dict == NULL) { dictkeys_incref(cached); dict = new_dict_with_shared_keys(interp, cached); - Py_REGIONADDREFERENCE(owner, dict); if (dict == NULL) return -1; + Py_REGIONADDREFERENCE(owner, dict); *dictptr = dict; } if (value == NULL) { + // Pyrona: Remove reference is called by `DelItem` res = PyDict_DelItem(dict, key); } else { - if (Py_REGIONADDREFERENCES(dict, key, value)) { - res = PyDict_SetItem(dict, key, value); - } else { - // Error is set inside ADDREFERENCE - return -1; - } + // Pyrona: Add and remove reference is called by `SetItem` + res = PyDict_SetItem(dict, key, value); } } else { dict = *dictptr; @@ -5921,8 +5918,10 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, *dictptr = dict; } if (value == NULL) { + // Pyrona: Remove reference is called by `DelItem` res = PyDict_DelItem(dict, key); } else { + // Pyrona: Add and remove reference is called by `SetItem` res = PyDict_SetItem(dict, key, value); } } diff --git a/Objects/regions.c b/Objects/regions.c index ab7e602e211663..8735116ece35ed 100644 --- a/Objects/regions.c +++ b/Objects/regions.c @@ -2004,6 +2004,10 @@ void _Py_RegionRemoveReference(PyObject *src, PyObject *tgt) { // This must be a parent reference, so we need to remove the parent reference. regionmetadata* src_md = Py_REGION_DATA(src); regionmetadata* tgt_parent_md = REGION_DATA_CAST(Py_region_ptr(tgt_md->parent)); + // FIXME(Pyrona): We might want to allow the `tgt_parent_md` to be NULL. + // This would prevent an exception if someone calls remove reference twice. + // We could also make this a dynamic check, which is lenient by default but + // can be turned strict by a flag. if (tgt_parent_md != src_md) { // TODO: Could `dirty` mean this isn't an error? _PyErr_Region(src, tgt, "(in WB/remove_ref)");