From 1ac565b5e2e5e74b5e28f8098bd8d3c4c9103ac0 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 19 Jan 2025 13:55:53 -0500 Subject: [PATCH 1/5] Don't expose async_generator_wrapped_value in sys.monitoring --- Include/internal/pycore_genobject.h | 12 ++++++++++++ Objects/genobject.c | 13 ------------- Python/bytecodes.c | 9 ++++++++- Python/generated_cases.c.h | 11 ++++++++++- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index f6d7e6d367177b..c1a0b604395dad 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -67,6 +67,18 @@ extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; +extern PyTypeObject _PyAsyncGenWrappedValue_Type; + +typedef struct _PyAsyncGenWrappedValue { + PyObject_HEAD + PyObject *agw_val; +} _PyAsyncGenWrappedValue; + +#define _PyAsyncGenWrappedValue_CheckExact(o) \ + Py_IS_TYPE(o, &_PyAsyncGenWrappedValue_Type) +#define _PyAsyncGenWrappedValue_CAST(op) \ + (assert(_PyAsyncGenWrappedValue_CheckExact(op)), \ + _Py_CAST(_PyAsyncGenWrappedValue*, (op))) #ifdef __cplusplus } diff --git a/Objects/genobject.c b/Objects/genobject.c index b32140766c4a38..b65c2bc8549b22 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1441,19 +1441,6 @@ typedef struct PyAsyncGenAThrow { } PyAsyncGenAThrow; -typedef struct _PyAsyncGenWrappedValue { - PyObject_HEAD - PyObject *agw_val; -} _PyAsyncGenWrappedValue; - - -#define _PyAsyncGenWrappedValue_CheckExact(o) \ - Py_IS_TYPE(o, &_PyAsyncGenWrappedValue_Type) -#define _PyAsyncGenWrappedValue_CAST(op) \ - (assert(_PyAsyncGenWrappedValue_CheckExact(op)), \ - _Py_CAST(_PyAsyncGenWrappedValue*, (op))) - - static int async_gen_traverse(PyObject *self, visitproc visit, void *arg) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c0ef767a9dd68b..69a33493f63319 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1312,9 +1312,16 @@ dummy_func( } tier1 op(_YIELD_VALUE_EVENT, (val -- val)) { + PyObject *yielded = PyStackRef_AsPyObjectBorrow(val); + if (_PyAsyncGenWrappedValue_CheckExact(yielded)) + { + /* gh-129013: Async generators have a special wrapper that they + yield. Don't expose that to the user. */ + yielded = _PyAsyncGenWrappedValue_CAST(yielded)->agw_val; + } int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + frame, this_instr, yielded); if (err) { ERROR_NO_POP(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dc90f75f2645e1..5790a4ffe399e7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5070,10 +5070,19 @@ // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; + PyObject *yielded = PyStackRef_AsPyObjectBorrow(val); + if (_PyAsyncGenWrappedValue_CheckExact(yielded)) + { + /* gh-129013: Async generators have a special wrapper that they + yield. Don't expose that to the user. */ + _PyFrame_SetStackPointer(frame, stack_pointer); + yielded = _PyAsyncGenWrappedValue_CAST(yielded)->agw_val; + stack_pointer = _PyFrame_GetStackPointer(frame); + } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + frame, this_instr, yielded); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { goto error; From 0e471787c6e6174df604d8ec16fc8e1b6f27a77a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 19 Jan 2025 14:06:14 -0500 Subject: [PATCH 2/5] Add a test. --- Lib/test/test_monitoring.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 364381e7dce00a..e1b5c021471c00 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2083,6 +2083,29 @@ def callback(code, instruction_offset): sys.monitoring.restart_events() sys.monitoring.set_events(0, 0) + def test_yield_async_generator(self): + # gh-129013: Async generators have a special type that they + # use to yield values. This type shouldn't be exposed by PY_YIELD + # events. + asyncio = test.support.import_helper.import_module("asyncio") + + async def gen(): + yield 42 + + async def main(): + async for _ in gen(): + pass + + def handle_yield(code, offset, value): + self.assertEqual(value, 42) + + sys.monitoring.use_tool_id(0, "test") + sys.monitoring.register_callback(0, sys.monitoring.events.PY_YIELD, handle_yield) + sys.monitoring.set_events(0, sys.monitoring.events.PY_YIELD) + + asyncio.run(main()) + sys.monitoring.set_events(0, 0) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): @@ -2329,3 +2352,6 @@ def test_enter_scope_two_events(self): finally: sys.monitoring.set_events(TEST_TOOL, 0) + +if __name__ == "__main__": + unittest.main() From d36fbef7dc4a9c6f3404d2c373de5a84f8f7a470 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 19 Jan 2025 14:15:42 -0500 Subject: [PATCH 3/5] Add blurb. --- .../2025-01-19-14-15-36.gh-issue-129013.28rL6Z.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-19-14-15-36.gh-issue-129013.28rL6Z.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-19-14-15-36.gh-issue-129013.28rL6Z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-19-14-15-36.gh-issue-129013.28rL6Z.rst new file mode 100644 index 00000000000000..de435d75f41c78 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-19-14-15-36.gh-issue-129013.28rL6Z.rst @@ -0,0 +1,2 @@ +Fix odd/unusable value when using :monitoring-event:`PY_YIELD` with an +:term:`asynchronous generator`. From 3b3ba5124078929c97b11fb6505a3c283e76bf02 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 19 Jan 2025 14:40:29 -0500 Subject: [PATCH 4/5] Skip test if socket doesn't work. --- Lib/test/test_monitoring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index e1b5c021471c00..1968f1354352c0 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2083,6 +2083,7 @@ def callback(code, instruction_offset): sys.monitoring.restart_events() sys.monitoring.set_events(0, 0) + @test.support.requires_working_socket() # For asyncio def test_yield_async_generator(self): # gh-129013: Async generators have a special type that they # use to yield values. This type shouldn't be exposed by PY_YIELD From 0b609597fec9101ce9e636de234a4931418d5a4d Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 3 Apr 2025 06:33:13 -0400 Subject: [PATCH 5/5] Fix extra extern. --- Include/internal/pycore_genobject.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index c1a0b604395dad..5aae60ffc3d887 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -67,7 +67,6 @@ extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; -extern PyTypeObject _PyAsyncGenWrappedValue_Type; typedef struct _PyAsyncGenWrappedValue { PyObject_HEAD