From 6fb789bfc1a010b30bca26bf028f7ff7b44d1feb Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Mon, 30 Mar 2026 15:53:01 -0700 Subject: [PATCH 01/26] Shrink the inline storage in connect_awaitable This diff adapts an idea originally due to @lewissbaker, originally shared at https://godbolt.org/z/zGG9fsPrz. Changes to Lewis's original include: * never invoke `coro.destroy()` since it's a no-op anyway * hard-code the integration with `connect_awaitable` since that's all we're using it for anyway * integrate with stdexec's support for various ways of making senders awaitable * MOAR `std::unreachable()` * support `unhandled_stopped()` --- .../stdexec/__detail/__connect_awaitable.hpp | 244 ++++++++++-------- 1 file changed, 136 insertions(+), 108 deletions(-) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 2fa75cda8..f62afeb49 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -38,13 +38,7 @@ namespace STDEXEC // __connect_await namespace __connect_await { - STDEXEC_PRAGMA_OPTIMIZE_BEGIN() - -# if STDEXEC_MSVC() - static constexpr std::size_t __storage_size = 256; -# else - static constexpr std::size_t __storage_size = 8 * sizeof(void*); -# endif + static constexpr std::size_t __storage_size = 5 * sizeof(void*); static constexpr std::size_t __storage_align = __STDCPP_DEFAULT_NEW_ALIGNMENT__; // clang-format off @@ -80,32 +74,33 @@ namespace STDEXEC __with_await_transform() = default; }; - struct __awaiter_base + struct __final_awaiter { static constexpr auto await_ready() noexcept -> bool { return false; } + template + static constexpr void await_suspend(__std::coroutine_handle<_Promise> __h) noexcept + { + try + { + __h.promise().__opstate_.__on_resume(); + } + catch (...) + { + __std::unreachable(); + } + } + [[noreturn]] - inline void await_resume() noexcept + static void await_resume() noexcept { __std::unreachable(); } }; - inline void __destroy_coro(__std::coroutine_handle<> __coro) noexcept - { -# if STDEXEC_MSVC() - // MSVCBUG https://developercommunity.visualstudio.com/t/Double-destroy-of-a-local-in-coroutine-d/10456428 - // Reassign __coro before calling destroy to make the mutation - // observable and to hopefully ensure that the compiler does not eliminate it. - std::exchange(__coro, {}).destroy(); -# else - __coro.destroy(); -# endif - } - template struct __opstate; @@ -114,79 +109,44 @@ namespace STDEXEC { using __opstate_t = __opstate<_Awaitable, _Receiver>; - struct __task - { - using promise_type = __promise; - - constexpr explicit __task(__std::coroutine_handle<__promise> __coro) noexcept - : __coro_(__coro) - {} - - STDEXEC_IMMOVABLE(__task); - - ~__task() - { - __connect_await::__destroy_coro(__coro_); - } - - __std::coroutine_handle<__promise> __coro_{}; - }; - - struct __final_awaiter : __awaiter_base - { - void await_suspend(__std::coroutine_handle<>) noexcept - { - using __awaitable_t = __result_of<__get_awaitable, _Awaitable, __promise&>; - using __awaiter_t = __awaiter_of_t<__awaitable_t>; - using __result_t = decltype(__declval<__awaiter_t>().await_resume()); - - if (__opstate_.__eptr_) - { - STDEXEC::set_error(static_cast<_Receiver&&>(__opstate_.__rcvr_), - std::move(__opstate_.__eptr_)); - } - else if constexpr (__same_as<__result_t, void>) - { - STDEXEC_ASSERT(__opstate_.__result_.has_value()); - STDEXEC::set_value(static_cast<_Receiver&&>(__opstate_.__rcvr_)); - } - else - { - STDEXEC_ASSERT(__opstate_.__result_.has_value()); - STDEXEC::set_value(static_cast<_Receiver&&>(__opstate_.__rcvr_), - static_cast<__result_t&&>(*__opstate_.__result_)); - } - // This coroutine is never resumed; its work is done. - } - - __opstate<_Awaitable, _Receiver>& __opstate_; - }; - constexpr explicit(!STDEXEC_EDG()) __promise(__opstate_t& __opstate) noexcept : __opstate_(__opstate) {} -# if !STDEXEC_GCC() || STDEXEC_GCC_VERSION >= 12'00 + ~__promise() + { + // never invoked + __std::unreachable(); + } + static constexpr auto operator new([[maybe_unused]] std::size_t __bytes, __opstate_t& __opstate) noexcept -> void* { - STDEXEC_ASSERT(__bytes <= sizeof(__opstate.__storage_)); + STDEXEC_ASSERT(__bytes == __storage_size); return __opstate.__storage_; } - static constexpr void operator delete([[maybe_unused]] void* __ptr) noexcept + static constexpr void operator delete(void*, std::size_t) noexcept { - // no-op + // never invoked + __std::unreachable(); } -# endif - constexpr auto get_return_object() noexcept -> __task + constexpr auto get_return_object() noexcept -> __std::coroutine_handle<__promise> { - return __task{__std::coroutine_handle<__promise>::from_promise(*this)}; + try + { + return __std::coroutine_handle<__promise>::from_promise(*this); + } + catch (...) + { + __std::unreachable(); + } } [[noreturn]] - static auto get_return_object_on_allocation_failure() noexcept -> __task + static auto + get_return_object_on_allocation_failure() noexcept -> __std::coroutine_handle<__promise> { __std::unreachable(); } @@ -196,26 +156,27 @@ namespace STDEXEC return {}; } - void unhandled_exception() noexcept + [[noreturn]] + static void unhandled_exception() noexcept { - __opstate_.__eptr_ = std::current_exception(); + __std::unreachable(); } constexpr auto unhandled_stopped() noexcept -> __std::coroutine_handle<> { - STDEXEC::set_stopped(static_cast<_Receiver&&>(__opstate_.__rcvr_)); + __opstate_.__on_stopped(); // Returning noop_coroutine here causes the __connect_awaitable // coroutine to never resume past the point where it co_await's // the awaitable. return __std::noop_coroutine(); } - constexpr auto final_suspend() noexcept -> __final_awaiter + static constexpr auto final_suspend() noexcept -> __final_awaiter { - return __final_awaiter{{}, __opstate_}; + return __final_awaiter{}; } - static void return_void() noexcept + static constexpr void return_void() noexcept { // no-op } @@ -228,67 +189,134 @@ namespace STDEXEC __opstate<_Awaitable, _Receiver>& __opstate_; }; + } // namespace __connect_await +} + +template +struct std::coroutine_traits< + STDEXEC::__std::coroutine_handle>, + STDEXEC::__connect_await::__opstate<_Awaitable, _Receiver>&> +{ + using promise_type = STDEXEC::__connect_await::__promise<_Awaitable, _Receiver>; +}; +namespace STDEXEC +{ + namespace __connect_await + { template struct __opstate { constexpr explicit __opstate(_Awaitable&& __awaitable, _Receiver&& __rcvr) noexcept(__is_nothrow) : __rcvr_(static_cast<_Receiver&&>(__rcvr)) - , __task_(__co_impl(*this)) + , __coro(__co_impl(*this)) , __awaitable1_(static_cast<_Awaitable&&>(__awaitable)) - , __awaitable2_( - __get_awaitable(static_cast<_Awaitable&&>(__awaitable1_), __task_.__coro_.promise())) + , __awaitable2_(__get_awaitable(static_cast<_Awaitable&&>(__awaitable1_), __coro.promise())) , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable2_))) {} void start() & noexcept { - __task_.__coro_.resume(); + try + { + if (!__awaiter_.await_ready()) + { + using __suspend_result_t = decltype(__awaiter_.await_suspend(__coro)); + + // suspended + if constexpr (std::is_void_v<__suspend_result_t>) + { + // void-returning await_suspend means "always suspend" + __awaiter_.await_suspend(__coro); + return; + } + else if constexpr (std::same_as) + { + if (__awaiter_.await_suspend(__coro)) + { + // returning true from a bool-returning await_suspend means suspend + return; + } + else + { + // returning false means immediately resume + } + } + else + { + static_assert(__std::convertible_to<__suspend_result_t, __std::coroutine_handle<>>); + auto __resume_target = __awaiter_.await_suspend(__coro); + __resume_target.resume(); + return; + } + } + + // immediate resumption + __on_resume(); + } + catch (...) + { + if constexpr (!noexcept(__awaiter_.await_ready()) + || !noexcept(__awaiter_.await_suspend(__coro))) + { + STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr_), std::current_exception()); + } + } } private: using __promise_t = __promise<_Awaitable, _Receiver>; - using __task_t = __promise_t::__task; using __awaitable_t = __result_of<__get_awaitable, _Awaitable, __promise_t&>; using __awaiter_t = __awaiter_of_t<__awaitable_t>; - using __result_t = decltype(__declval<__awaiter_t>().await_resume()); friend __promise_t; + friend __final_awaiter; static constexpr bool __is_nothrow = __nothrow_move_constructible<_Awaitable> && __noexcept_of<__get_awaitable, _Awaitable, __promise_t&> && __noexcept_of<__get_awaiter, __awaitable_t>; - static constexpr std::size_t __storage_size = __connect_await::__storage_size - + sizeof(__manual_lifetime<__result_t>) - - __same_as<__result_t, void>; + static auto __co_impl(__opstate&) noexcept -> __std::coroutine_handle<__promise_t> + { + co_return; + } - static auto __co_impl(__opstate& __op) noexcept -> __task_t + constexpr void __on_resume() noexcept { - using __op_awaiter_t = decltype(__op.__awaiter_); - if constexpr (__same_as) + try { - co_await static_cast<__op_awaiter_t&&>(__op.__awaiter_); - __op.__result_.emplace(); + if constexpr (std::is_void_v) + { + __awaiter_.await_resume(); + STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_)); + } + else + { + STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), __awaiter_.await_resume()); + } } - else + catch (...) { - __op.__result_.emplace(co_await static_cast<__op_awaiter_t&&>(__op.__awaiter_)); + if constexpr (!noexcept(__awaiter_.await_resume())) + { + STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr_), std::current_exception()); + } } } + constexpr void __on_stopped() noexcept + { + STDEXEC::set_stopped(static_cast<_Receiver&&>(__rcvr_)); + } + alignas(__storage_align) std::byte __storage_[__storage_size]; - _Receiver __rcvr_; - __promise_t::__task __task_; - _Awaitable __awaitable1_; - __awaitable_t __awaitable2_; - __awaiter_t __awaiter_; - std::exception_ptr __eptr_{}; - __optional<__result_t> __result_{}; + _Receiver __rcvr_; + __std::coroutine_handle<__promise_t> __coro; + _Awaitable __awaitable1_; + __awaitable_t __awaitable2_; + __awaiter_t __awaiter_; }; - - STDEXEC_PRAGMA_OPTIMIZE_END() } // namespace __connect_await struct __connect_awaitable_t From db2f2b8c703cf4dc389992d0eecba31fbd226045 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Mon, 30 Mar 2026 16:03:31 -0700 Subject: [PATCH 02/26] Credit Lewis in the source, too --- include/stdexec/__detail/__connect_awaitable.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index f62afeb49..ef4052885 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -122,6 +122,9 @@ namespace STDEXEC static constexpr auto operator new([[maybe_unused]] std::size_t __bytes, __opstate_t& __opstate) noexcept -> void* { + // the first implementation of storing the coroutine frame inline in __opstate using the + // technique in this file is due to Lewis Baker , and was first + // shared at https://godbolt.org/z/zGG9fsPrz STDEXEC_ASSERT(__bytes == __storage_size); return __opstate.__storage_; } From bffef39cf0f5ad1c8a7478ce744b68d752c3e4e6 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Mon, 30 Mar 2026 16:12:32 -0700 Subject: [PATCH 03/26] Code review feedback * `s/try/STDEXEC_TRY/g` * `s/catch (...)/STDEXEC_CATCH_ALL/g` * `s/__coro/&_/g` --- .../stdexec/__detail/__connect_awaitable.hpp | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index ef4052885..d27054079 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -84,11 +84,11 @@ namespace STDEXEC template static constexpr void await_suspend(__std::coroutine_handle<_Promise> __h) noexcept { - try + STDEXEC_TRY { __h.promise().__opstate_.__on_resume(); } - catch (...) + STDEXEC_CATCH_ALL { __std::unreachable(); } @@ -137,11 +137,11 @@ namespace STDEXEC constexpr auto get_return_object() noexcept -> __std::coroutine_handle<__promise> { - try + STDEXEC_TRY { return __std::coroutine_handle<__promise>::from_promise(*this); } - catch (...) + STDEXEC_CATCH_ALL { __std::unreachable(); } @@ -213,30 +213,31 @@ namespace STDEXEC constexpr explicit __opstate(_Awaitable&& __awaitable, _Receiver&& __rcvr) noexcept(__is_nothrow) : __rcvr_(static_cast<_Receiver&&>(__rcvr)) - , __coro(__co_impl(*this)) + , __coro_(__co_impl(*this)) , __awaitable1_(static_cast<_Awaitable&&>(__awaitable)) - , __awaitable2_(__get_awaitable(static_cast<_Awaitable&&>(__awaitable1_), __coro.promise())) + , __awaitable2_( + __get_awaitable(static_cast<_Awaitable&&>(__awaitable1_), __coro_.promise())) , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable2_))) {} void start() & noexcept { - try + STDEXEC_TRY { if (!__awaiter_.await_ready()) { - using __suspend_result_t = decltype(__awaiter_.await_suspend(__coro)); + using __suspend_result_t = decltype(__awaiter_.await_suspend(__coro_)); // suspended if constexpr (std::is_void_v<__suspend_result_t>) { // void-returning await_suspend means "always suspend" - __awaiter_.await_suspend(__coro); + __awaiter_.await_suspend(__coro_); return; } else if constexpr (std::same_as) { - if (__awaiter_.await_suspend(__coro)) + if (__awaiter_.await_suspend(__coro_)) { // returning true from a bool-returning await_suspend means suspend return; @@ -249,7 +250,7 @@ namespace STDEXEC else { static_assert(__std::convertible_to<__suspend_result_t, __std::coroutine_handle<>>); - auto __resume_target = __awaiter_.await_suspend(__coro); + auto __resume_target = __awaiter_.await_suspend(__coro_); __resume_target.resume(); return; } @@ -258,10 +259,10 @@ namespace STDEXEC // immediate resumption __on_resume(); } - catch (...) + STDEXEC_CATCH_ALL { if constexpr (!noexcept(__awaiter_.await_ready()) - || !noexcept(__awaiter_.await_suspend(__coro))) + || !noexcept(__awaiter_.await_suspend(__coro_))) { STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr_), std::current_exception()); } @@ -287,7 +288,7 @@ namespace STDEXEC constexpr void __on_resume() noexcept { - try + STDEXEC_TRY { if constexpr (std::is_void_v) { @@ -299,7 +300,7 @@ namespace STDEXEC STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), __awaiter_.await_resume()); } } - catch (...) + STDEXEC_CATCH_ALL { if constexpr (!noexcept(__awaiter_.await_resume())) { @@ -315,7 +316,7 @@ namespace STDEXEC alignas(__storage_align) std::byte __storage_[__storage_size]; _Receiver __rcvr_; - __std::coroutine_handle<__promise_t> __coro; + __std::coroutine_handle<__promise_t> __coro_; _Awaitable __awaitable1_; __awaitable_t __awaitable2_; __awaiter_t __awaiter_; From 38f28173a3d3214b67adc4ea3022263bf59dbc02 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Tue, 31 Mar 2026 09:48:17 -0700 Subject: [PATCH 04/26] Address self-review comments * Fix up the description of what's done in `unhandled_stop` * Take @ericniebler's suggestion and make it explicitly UB for the `coroutine_handle` returned from `await_suspend` to throw from `resume()`. --- include/stdexec/__detail/__connect_awaitable.hpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index d27054079..7954444e7 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -169,8 +169,7 @@ namespace STDEXEC { __opstate_.__on_stopped(); // Returning noop_coroutine here causes the __connect_awaitable - // coroutine to never resume past the point where it co_await's - // the awaitable. + // coroutine to never resume past its initial_suspend point return __std::noop_coroutine(); } @@ -251,7 +250,17 @@ namespace STDEXEC { static_assert(__std::convertible_to<__suspend_result_t, __std::coroutine_handle<>>); auto __resume_target = __awaiter_.await_suspend(__coro_); - __resume_target.resume(); + STDEXEC_TRY + { + __resume_target.resume(); + } + STDEXEC_CATCH_ALL + { + STDEXEC_ASSERT(false + && "about to deliberately commit UB in response to a misbehaving " + "awaitable"); + __std::unreachable(); + } return; } } From 27bdd56c8732a02af696ed6a41922d693d9335f3 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Tue, 31 Mar 2026 11:06:46 -0700 Subject: [PATCH 05/26] Defer construction of the awaitable state Move `__awaitable1_` and `__awaiter_` into a helper type defer construction thereof to `start()`; this saves us from storing the `coroutine_handle` in the opstate. --- .../stdexec/__detail/__connect_awaitable.hpp | 84 +++++++++++++------ 1 file changed, 58 insertions(+), 26 deletions(-) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 7954444e7..424f55530 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -210,33 +210,33 @@ namespace STDEXEC struct __opstate { constexpr explicit __opstate(_Awaitable&& __awaitable, _Receiver&& __rcvr) - noexcept(__is_nothrow) + noexcept(__nothrow_move_constructible<_Awaitable>) : __rcvr_(static_cast<_Receiver&&>(__rcvr)) - , __coro_(__co_impl(*this)) - , __awaitable1_(static_cast<_Awaitable&&>(__awaitable)) - , __awaitable2_( - __get_awaitable(static_cast<_Awaitable&&>(__awaitable1_), __coro_.promise())) - , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable2_))) + , __source_awaitable_(static_cast<_Awaitable&&>(__awaitable)) {} void start() & noexcept { + auto __coro = __co_impl(*this); + STDEXEC_TRY { - if (!__awaiter_.await_ready()) + __awaiter_.emplace(__source_awaitable_, __coro); + + if (!__awaiter_->await_ready()) { - using __suspend_result_t = decltype(__awaiter_.await_suspend(__coro_)); + using __suspend_result_t = decltype(__awaiter_->await_suspend(__coro)); // suspended if constexpr (std::is_void_v<__suspend_result_t>) { // void-returning await_suspend means "always suspend" - __awaiter_.await_suspend(__coro_); + __awaiter_->await_suspend(__coro); return; } else if constexpr (std::same_as) { - if (__awaiter_.await_suspend(__coro_)) + if (__awaiter_->await_suspend(__coro)) { // returning true from a bool-returning await_suspend means suspend return; @@ -249,7 +249,7 @@ namespace STDEXEC else { static_assert(__std::convertible_to<__suspend_result_t, __std::coroutine_handle<>>); - auto __resume_target = __awaiter_.await_suspend(__coro_); + auto __resume_target = __awaiter_->await_suspend(__coro); STDEXEC_TRY { __resume_target.resume(); @@ -270,8 +270,11 @@ namespace STDEXEC } STDEXEC_CATCH_ALL { - if constexpr (!noexcept(__awaiter_.await_ready()) - || !noexcept(__awaiter_.await_suspend(__coro_))) + if constexpr (!__nothrow_constructible_from<__awaitable_state, + _Awaitable&, + __std::coroutine_handle<__promise_t>> + || !noexcept(__awaiter_->await_ready()) + || !noexcept(__awaiter_->await_suspend(__coro))) { STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr_), std::current_exception()); } @@ -286,10 +289,6 @@ namespace STDEXEC friend __promise_t; friend __final_awaiter; - static constexpr bool __is_nothrow = __nothrow_move_constructible<_Awaitable> - && __noexcept_of<__get_awaitable, _Awaitable, __promise_t&> - && __noexcept_of<__get_awaiter, __awaitable_t>; - static auto __co_impl(__opstate&) noexcept -> __std::coroutine_handle<__promise_t> { co_return; @@ -299,19 +298,19 @@ namespace STDEXEC { STDEXEC_TRY { - if constexpr (std::is_void_v) + if constexpr (std::is_void_vawait_resume())>) { - __awaiter_.await_resume(); + __awaiter_->await_resume(); STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_)); } else { - STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), __awaiter_.await_resume()); + STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), __awaiter_->await_resume()); } } STDEXEC_CATCH_ALL { - if constexpr (!noexcept(__awaiter_.await_resume())) + if constexpr (!noexcept(__awaiter_->await_resume())) { STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr_), std::current_exception()); } @@ -323,12 +322,45 @@ namespace STDEXEC STDEXEC::set_stopped(static_cast<_Receiver&&>(__rcvr_)); } + struct __awaitable_state + { + explicit __awaitable_state(_Awaitable& __source, + __std::coroutine_handle<__promise_t> __coro) + noexcept(__is_nothrow) + : __awaitable_(__get_awaitable(static_cast<_Awaitable&&>(__source), __coro.promise())) + , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable_))) + {} + + constexpr auto await_ready() noexcept(noexcept(__awaiter_.await_ready())) -> bool + { + return __awaiter_.await_ready(); + } + + template + constexpr auto await_suspend(__std::coroutine_handle<_P> __h) + noexcept(noexcept(__awaiter_.await_suspend(__h))) + { + return __awaiter_.await_suspend(__h); + } + + constexpr decltype(auto) await_resume() noexcept(noexcept(__awaiter_.await_resume())) + { + return __awaiter_.await_resume(); + } + + private: + static constexpr bool __is_nothrow = + __noexcept_of<__get_awaitable, _Awaitable, __promise_t&> + && __noexcept_of<__get_awaiter, __awaitable_t>; + + __awaitable_t __awaitable_; + __awaiter_t __awaiter_; + }; + alignas(__storage_align) std::byte __storage_[__storage_size]; - _Receiver __rcvr_; - __std::coroutine_handle<__promise_t> __coro_; - _Awaitable __awaitable1_; - __awaitable_t __awaitable2_; - __awaiter_t __awaiter_; + _Receiver __rcvr_; + _Awaitable __source_awaitable_; + __optional<__awaitable_state> __awaiter_; }; } // namespace __connect_await From 231c707656ded6be692c021619a1129caaa28614 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Tue, 31 Mar 2026 11:58:19 -0700 Subject: [PATCH 06/26] Shrink storage to four pointers, maybe fix GCC We're only storing two function pointers in the coroutine frame now, so we've saved another pointer's worth of storage. The compiler seems to be reserving a pointer's width for the promise even though it's now an empty type; I wonder if that's Clang-specific since that's the only compiler I have easy access to. I've also rearranged the declaration order of the members of `__awaitable_state` in hopes of fixing the GCC 11 build break. --- .../stdexec/__detail/__connect_awaitable.hpp | 51 ++++++++++++------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 424f55530..174e852c0 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -38,7 +38,7 @@ namespace STDEXEC // __connect_await namespace __connect_await { - static constexpr std::size_t __storage_size = 5 * sizeof(void*); + static constexpr std::size_t __storage_size = 4 * sizeof(void*); static constexpr std::size_t __storage_align = __STDCPP_DEFAULT_NEW_ALIGNMENT__; // clang-format off @@ -86,7 +86,7 @@ namespace STDEXEC { STDEXEC_TRY { - __h.promise().__opstate_.__on_resume(); + __h.promise().__get_opstate().__on_resume(); } STDEXEC_CATCH_ALL { @@ -109,9 +109,14 @@ namespace STDEXEC { using __opstate_t = __opstate<_Awaitable, _Receiver>; - constexpr explicit(!STDEXEC_EDG()) __promise(__opstate_t& __opstate) noexcept - : __opstate_(__opstate) - {} + static constexpr std::ptrdiff_t __promise_offset = sizeof(void*) * 2; + + explicit(!STDEXEC_EDG()) __promise([[maybe_unused]] + __opstate_t& __opstate) noexcept + { + STDEXEC_ASSERT(__promise_offset + == reinterpret_cast(this) - __opstate.__storage_); + } ~__promise() { @@ -167,7 +172,7 @@ namespace STDEXEC constexpr auto unhandled_stopped() noexcept -> __std::coroutine_handle<> { - __opstate_.__on_stopped(); + __get_opstate().__on_stopped(); // Returning noop_coroutine here causes the __connect_awaitable // coroutine to never resume past its initial_suspend point return __std::noop_coroutine(); @@ -186,10 +191,20 @@ namespace STDEXEC [[nodiscard]] constexpr auto get_env() const noexcept -> env_of_t<_Receiver> { - return STDEXEC::get_env(__opstate_.__rcvr_); + return STDEXEC::get_env(__get_opstate().__rcvr_); } - __opstate<_Awaitable, _Receiver>& __opstate_; + __opstate<_Awaitable, _Receiver>& __get_opstate() noexcept + { + return *reinterpret_cast<__opstate<_Awaitable, _Receiver>*>( + reinterpret_cast(this) - __promise_offset); + } + + __opstate<_Awaitable, _Receiver> const & __get_opstate() const noexcept + { + return *reinterpret_cast<__opstate<_Awaitable, _Receiver> const *>( + reinterpret_cast(this) - __promise_offset); + } }; } // namespace __connect_await } @@ -322,8 +337,15 @@ namespace STDEXEC STDEXEC::set_stopped(static_cast<_Receiver&&>(__rcvr_)); } - struct __awaitable_state + class __awaitable_state { + static constexpr bool __is_nothrow = + __noexcept_of<__get_awaitable, _Awaitable, __promise_t&> + && __noexcept_of<__get_awaiter, __awaitable_t>; + + __awaitable_t __awaitable_; + __awaiter_t __awaiter_; + public: explicit __awaitable_state(_Awaitable& __source, __std::coroutine_handle<__promise_t> __coro) noexcept(__is_nothrow) @@ -343,18 +365,11 @@ namespace STDEXEC return __awaiter_.await_suspend(__h); } - constexpr decltype(auto) await_resume() noexcept(noexcept(__awaiter_.await_resume())) + constexpr auto await_resume() noexcept(noexcept(__awaiter_.await_resume())) + -> decltype(__awaiter_.await_resume()) { return __awaiter_.await_resume(); } - - private: - static constexpr bool __is_nothrow = - __noexcept_of<__get_awaitable, _Awaitable, __promise_t&> - && __noexcept_of<__get_awaiter, __awaitable_t>; - - __awaitable_t __awaitable_; - __awaiter_t __awaiter_; }; alignas(__storage_align) std::byte __storage_[__storage_size]; From 2991cd7ebd3e2f34ce15e72df292ac27de4294d2 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Tue, 31 Mar 2026 12:08:06 -0700 Subject: [PATCH 07/26] Eliminate padding in __opstate The last pointer's width of bytes in `__opstate<...>::__storage_` only stores a single byte at the beginning for the coroutine state, which is written on entry to the coroutine and then never read again. This diff under-allocates storage by one byte so we can store a `bool` in the last byte that indicates whether the operation was ever started or not. Having moved the "did we start" knowledge out of the `__awaitable_state` and into the new `bool`, we can use a `union` to implement our own manual lifetime for the awaitables and save the padding that was otherwise in the `__optional<__awaitable_state>`. --- .../stdexec/__detail/__connect_awaitable.hpp | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 174e852c0..8cadbd986 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -38,7 +38,7 @@ namespace STDEXEC // __connect_await namespace __connect_await { - static constexpr std::size_t __storage_size = 4 * sizeof(void*); + static constexpr std::size_t __storage_size = 4 * sizeof(void*) - 1; static constexpr std::size_t __storage_align = __STDCPP_DEFAULT_NEW_ALIGNMENT__; // clang-format off @@ -130,7 +130,7 @@ namespace STDEXEC // the first implementation of storing the coroutine frame inline in __opstate using the // technique in this file is due to Lewis Baker , and was first // shared at https://godbolt.org/z/zGG9fsPrz - STDEXEC_ASSERT(__bytes == __storage_size); + STDEXEC_ASSERT(__bytes == __storage_size + 1); return __opstate.__storage_; } @@ -230,28 +230,39 @@ namespace STDEXEC , __source_awaitable_(static_cast<_Awaitable&&>(__awaitable)) {} + __opstate(__opstate&&) = delete; + + ~__opstate() + { + if (__started_) + { + std::destroy_at(&__awaiter_); + } + } + void start() & noexcept { auto __coro = __co_impl(*this); + __started_ = true; STDEXEC_TRY { - __awaiter_.emplace(__source_awaitable_, __coro); + std::construct_at(&__awaiter_, __source_awaitable_, __coro); - if (!__awaiter_->await_ready()) + if (!__awaiter_.await_ready()) { - using __suspend_result_t = decltype(__awaiter_->await_suspend(__coro)); + using __suspend_result_t = decltype(__awaiter_.await_suspend(__coro)); // suspended if constexpr (std::is_void_v<__suspend_result_t>) { // void-returning await_suspend means "always suspend" - __awaiter_->await_suspend(__coro); + __awaiter_.await_suspend(__coro); return; } else if constexpr (std::same_as) { - if (__awaiter_->await_suspend(__coro)) + if (__awaiter_.await_suspend(__coro)) { // returning true from a bool-returning await_suspend means suspend return; @@ -264,7 +275,7 @@ namespace STDEXEC else { static_assert(__std::convertible_to<__suspend_result_t, __std::coroutine_handle<>>); - auto __resume_target = __awaiter_->await_suspend(__coro); + auto __resume_target = __awaiter_.await_suspend(__coro); STDEXEC_TRY { __resume_target.resume(); @@ -288,8 +299,8 @@ namespace STDEXEC if constexpr (!__nothrow_constructible_from<__awaitable_state, _Awaitable&, __std::coroutine_handle<__promise_t>> - || !noexcept(__awaiter_->await_ready()) - || !noexcept(__awaiter_->await_suspend(__coro))) + || !noexcept(__awaiter_.await_ready()) + || !noexcept(__awaiter_.await_suspend(__coro))) { STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr_), std::current_exception()); } @@ -313,19 +324,19 @@ namespace STDEXEC { STDEXEC_TRY { - if constexpr (std::is_void_vawait_resume())>) + if constexpr (std::is_void_v) { - __awaiter_->await_resume(); + __awaiter_.await_resume(); STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_)); } else { - STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), __awaiter_->await_resume()); + STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), __awaiter_.await_resume()); } } STDEXEC_CATCH_ALL { - if constexpr (!noexcept(__awaiter_->await_resume())) + if constexpr (!noexcept(__awaiter_.await_resume())) { STDEXEC::set_error(static_cast<_Receiver&&>(__rcvr_), std::current_exception()); } @@ -373,9 +384,14 @@ namespace STDEXEC }; alignas(__storage_align) std::byte __storage_[__storage_size]; - _Receiver __rcvr_; - _Awaitable __source_awaitable_; - __optional<__awaitable_state> __awaiter_; + [[no_unique_address]] + bool __started_{false}; + _Receiver __rcvr_; + _Awaitable __source_awaitable_; + union + { + __awaitable_state __awaiter_; + }; }; } // namespace __connect_await From 33985c5ffe099c8fcd009d367d9a16a4055556a5 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Tue, 31 Mar 2026 13:41:30 -0700 Subject: [PATCH 08/26] Conditionally store the awaitable and awaiter Not all awaitables need to store the source object, the result of __get_awaitable, _and_ the result of __get_awaiter, so only store what's needed. This diff adds enough complexity that I should add explicit unit tests covering all the variations of connecting an awaitable to a receiver. --- .../stdexec/__detail/__connect_awaitable.hpp | 271 ++++++++++++++---- 1 file changed, 223 insertions(+), 48 deletions(-) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 8cadbd986..64d268d16 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -74,6 +74,222 @@ namespace STDEXEC __with_await_transform() = default; }; + template + struct __awaitable_wrapper + { + constexpr auto& __awaiter() noexcept + { + return static_cast<_Derived*>(this)->__awaiter_; + } + + constexpr auto await_ready() noexcept(noexcept(__awaiter().await_ready())) -> bool + { + return __awaiter().await_ready(); + } + + template + constexpr auto await_suspend(__std::coroutine_handle<_Promise> __h) + noexcept(noexcept(__awaiter().await_suspend(__h))) + { + return __awaiter().await_suspend(__h); + } + + constexpr decltype(auto) await_resume() noexcept(noexcept(__awaiter().await_resume())) + { + return __awaiter().await_resume(); + } + }; + + template + concept __has_distinct_awaitable = __has_as_awaitable_member<_Tp, _Promise>; + + template + concept __has_distinct_awaiter = requires(_Awaitable&& __awaitable) { + { static_cast<_Awaitable&&>(__awaitable).operator co_await() }; + } || requires(_Awaitable&& __awaitable) { + { operator co_await(static_cast<_Awaitable&&>(__awaitable)) }; + }; + + template + struct __awaitable_state : __awaitable_wrapper<__awaitable_state<_Awaitable, _Promise>> + { + using __awaitable_t = __result_of<__get_awaitable, _Awaitable, _Promise&>; + using __awaiter_t = __awaiter_of_t<__awaitable_t>; + + static constexpr bool __is_nothrow = __noexcept_of<__get_awaitable, _Awaitable, _Promise&> + && __noexcept_of<__get_awaiter, __awaitable_t>; + + struct __state : __awaitable_wrapper<__state> + { + __state(_Awaitable&& __source, __std::coroutine_handle<_Promise> __coro) + noexcept(__is_nothrow) + : __awaitable_(__get_awaitable(static_cast<_Awaitable&&>(__source), __coro.promise())) + , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable_))) + {} + + __awaitable_t __awaitable_; + __awaiter_t __awaiter_; + }; + + _Awaitable __source_awaitable_; + union + { + __state __awaiter_; + }; + + template + requires(!std::same_as, __awaitable_state>) + __awaitable_state(_A&& __awaitable) noexcept(__nothrow_constructible_from<_Awaitable, _A>) + : __source_awaitable_(static_cast<_A&&>(__awaitable)) + {} + + ~__awaitable_state() {} + + constexpr void construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) + { + std::construct_at(&__awaiter_, static_cast<_Awaitable&&>(__source_awaitable_), __coro); + } + + constexpr void destroy() noexcept + { + std::destroy_at(&__awaiter_); + } + }; + + template + requires __awaitable<_Awaitable, _Promise> + && (!__has_distinct_awaitable<_Awaitable, _Promise>) + && __has_distinct_awaiter<_Awaitable> + struct __awaitable_state<_Awaitable, _Promise> + : __awaitable_wrapper<__awaitable_state<_Awaitable, _Promise>> + { + // _Awaitable has a distinct awaiter, but no distinct as_awaitable() + // so we don't need separate storage for it + using __awaiter_t = __awaiter_of_t<_Awaitable&&>; + + static constexpr bool __is_nothrow = __noexcept_of<__get_awaiter, _Awaitable&&>; + + struct __state : __awaitable_wrapper<__state> + { + __state(_Awaitable&& __source, __std::coroutine_handle<_Promise> __coro) + noexcept(__is_nothrow) + : __awaiter_(__get_awaiter(static_cast<_Awaitable&&>(__source))) + { + [[maybe_unused]] + auto&& __awaitable = __get_awaitable(static_cast<_Awaitable&&>(__source), + __coro.promise()); + + STDEXEC_ASSERT(std::addressof(__awaitable) == std::addressof(__source)); + } + + __awaiter_t __awaiter_; + }; + + _Awaitable __source_awaitable_; + union + { + __state __awaiter_; + }; + + template + requires(!std::same_as, __awaitable_state>) + __awaitable_state(_A&& __awaitable) noexcept(__nothrow_constructible_from<_Awaitable, _A>) + : __source_awaitable_(static_cast<_A&&>(__awaitable)) + {} + + ~__awaitable_state() {} + + constexpr void construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) + { + std::construct_at(&__awaiter_, static_cast<_Awaitable&&>(__source_awaitable_), __coro); + } + + constexpr void destroy() noexcept + { + std::destroy_at(&__awaiter_); + } + }; + + template + requires __awaitable<_Awaitable, _Promise> // + && __has_distinct_awaitable<_Awaitable, _Promise> + && (!__has_distinct_awaiter<_Awaitable>) + struct __awaitable_state<_Awaitable, _Promise> + : __awaitable_wrapper<__awaitable_state<_Awaitable, _Promise>> + { + // _Awaitable has a distinct awaitable, but no distinct awaiter + // so we don't need separate storage for it + using __awaiter_t = __result_of<__get_awaitable, _Awaitable, _Promise&>; + + static constexpr bool __is_nothrow = __noexcept_of<__get_awaitable, _Awaitable, _Promise&>; + + struct __state : __awaitable_wrapper<__state> + { + __state(_Awaitable&& __source, __std::coroutine_handle<_Promise> __coro) + noexcept(__is_nothrow) + : __awaiter_(__get_awaitable(static_cast<_Awaitable&&>(__source), __coro.promise())) + { + [[maybe_unused]] + auto&& __awaiter = __get_awaiter(static_cast<__awaiter_t&&>(__awaiter_)); + STDEXEC_ASSERT(std::addressof(__awaiter) == std::addressof(__awaiter_)); + } + + __awaiter_t __awaiter_; + }; + + _Awaitable __source_awaitable_; + union + { + __state __awaiter_; + }; + + template + requires(!std::same_as, __awaitable_state>) + __awaitable_state(_A&& __awaitable) noexcept(__nothrow_constructible_from<_Awaitable, _A>) + : __source_awaitable_(static_cast<_A&&>(__awaitable)) + {} + + ~__awaitable_state() {} + + constexpr void construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) + { + std::construct_at(&__awaiter_, static_cast<_Awaitable&&>(__source_awaitable_), __coro); + } + + constexpr void destroy() noexcept + { + std::destroy_at(&__awaiter_); + } + }; + + template + requires __awaitable<_Awaitable, _Promise> + && (!__has_distinct_awaitable<_Awaitable, _Promise>) + && (!__has_distinct_awaiter<_Awaitable>) + struct __awaitable_state<_Awaitable, _Promise> + : __awaitable_wrapper<__awaitable_state<_Awaitable, _Promise>> + { + // _Awaitable has neither a distinct awaiter, nor a distinct awaitable + // so we don't need separate storage for either + _Awaitable __awaiter_; + + template + requires(!std::same_as, __awaitable_state>) + __awaitable_state(_A&& __awaitable) noexcept(__nothrow_constructible_from<_Awaitable, _A>) + : __awaiter_(static_cast<_A&&>(__awaitable)) + {} + + static constexpr void construct(__std::coroutine_handle<_Promise>) noexcept + { + // no-op + } + + static constexpr void destroy() noexcept + { + // no-op + } + }; + struct __final_awaiter { static constexpr auto await_ready() noexcept -> bool @@ -227,7 +443,7 @@ namespace STDEXEC constexpr explicit __opstate(_Awaitable&& __awaitable, _Receiver&& __rcvr) noexcept(__nothrow_move_constructible<_Awaitable>) : __rcvr_(static_cast<_Receiver&&>(__rcvr)) - , __source_awaitable_(static_cast<_Awaitable&&>(__awaitable)) + , __awaiter_(static_cast<_Awaitable&&>(__awaitable)) {} __opstate(__opstate&&) = delete; @@ -236,7 +452,7 @@ namespace STDEXEC { if (__started_) { - std::destroy_at(&__awaiter_); + __awaiter_.destroy(); } } @@ -247,7 +463,7 @@ namespace STDEXEC STDEXEC_TRY { - std::construct_at(&__awaiter_, __source_awaitable_, __coro); + __awaiter_.construct(__coro); if (!__awaiter_.await_ready()) { @@ -296,9 +512,7 @@ namespace STDEXEC } STDEXEC_CATCH_ALL { - if constexpr (!__nothrow_constructible_from<__awaitable_state, - _Awaitable&, - __std::coroutine_handle<__promise_t>> + if constexpr (!noexcept(__awaiter_.construct(__coro)) || !noexcept(__awaiter_.await_ready()) || !noexcept(__awaiter_.await_suspend(__coro))) { @@ -348,50 +562,11 @@ namespace STDEXEC STDEXEC::set_stopped(static_cast<_Receiver&&>(__rcvr_)); } - class __awaitable_state - { - static constexpr bool __is_nothrow = - __noexcept_of<__get_awaitable, _Awaitable, __promise_t&> - && __noexcept_of<__get_awaiter, __awaitable_t>; - - __awaitable_t __awaitable_; - __awaiter_t __awaiter_; - public: - explicit __awaitable_state(_Awaitable& __source, - __std::coroutine_handle<__promise_t> __coro) - noexcept(__is_nothrow) - : __awaitable_(__get_awaitable(static_cast<_Awaitable&&>(__source), __coro.promise())) - , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable_))) - {} - - constexpr auto await_ready() noexcept(noexcept(__awaiter_.await_ready())) -> bool - { - return __awaiter_.await_ready(); - } - - template - constexpr auto await_suspend(__std::coroutine_handle<_P> __h) - noexcept(noexcept(__awaiter_.await_suspend(__h))) - { - return __awaiter_.await_suspend(__h); - } - - constexpr auto await_resume() noexcept(noexcept(__awaiter_.await_resume())) - -> decltype(__awaiter_.await_resume()) - { - return __awaiter_.await_resume(); - } - }; - alignas(__storage_align) std::byte __storage_[__storage_size]; [[no_unique_address]] - bool __started_{false}; - _Receiver __rcvr_; - _Awaitable __source_awaitable_; - union - { - __awaitable_state __awaiter_; - }; + bool __started_{false}; + _Receiver __rcvr_; + __awaitable_state<_Awaitable, __promise_t> __awaiter_; }; } // namespace __connect_await From 1ef38a586628de3ed0651e513c5ee48dc38318f6 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Tue, 31 Mar 2026 18:47:52 -0700 Subject: [PATCH 09/26] Add tests for connect_awaitable This only exercises the various non-exception control flows through `connect_awaitable`; more tests are needed to confirm we're doing exception handling correctly. These tests found a bug, that's also fixed here. --- .../stdexec/__detail/__connect_awaitable.hpp | 21 +- test/CMakeLists.txt | 1 + .../cpos/test_cpo_connect_awaitable.cpp | 490 ++++++++++++++++++ 3 files changed, 508 insertions(+), 4 deletions(-) create mode 100644 test/stdexec/cpos/test_cpo_connect_awaitable.cpp diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 64d268d16..0f60fd4d6 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -127,13 +127,17 @@ namespace STDEXEC , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable_))) {} + [[no_unique_addres]] __awaitable_t __awaitable_; - __awaiter_t __awaiter_; + [[no_unique_addres]] + __awaiter_t __awaiter_; }; + [[no_unique_addres]] _Awaitable __source_awaitable_; union { + [[no_unique_addres]] __state __awaiter_; }; @@ -182,12 +186,15 @@ namespace STDEXEC STDEXEC_ASSERT(std::addressof(__awaitable) == std::addressof(__source)); } + [[no_unique_addres]] __awaiter_t __awaiter_; }; + [[no_unique_addres]] _Awaitable __source_awaitable_; union { + [[no_unique_addres]] __state __awaiter_; }; @@ -213,7 +220,7 @@ namespace STDEXEC template requires __awaitable<_Awaitable, _Promise> // && __has_distinct_awaitable<_Awaitable, _Promise> - && (!__has_distinct_awaiter<_Awaitable>) + && (!__has_distinct_awaiter<__result_of<__get_awaitable, _Awaitable, _Promise&>>) struct __awaitable_state<_Awaitable, _Promise> : __awaitable_wrapper<__awaitable_state<_Awaitable, _Promise>> { @@ -234,12 +241,15 @@ namespace STDEXEC STDEXEC_ASSERT(std::addressof(__awaiter) == std::addressof(__awaiter_)); } + [[no_unique_addres]] __awaiter_t __awaiter_; }; + [[no_unique_addres]] _Awaitable __source_awaitable_; union { + [[no_unique_addres]] __state __awaiter_; }; @@ -271,6 +281,7 @@ namespace STDEXEC { // _Awaitable has neither a distinct awaiter, nor a distinct awaitable // so we don't need separate storage for either + [[no_unique_addres]] _Awaitable __awaiter_; template @@ -564,8 +575,10 @@ namespace STDEXEC alignas(__storage_align) std::byte __storage_[__storage_size]; [[no_unique_address]] - bool __started_{false}; - _Receiver __rcvr_; + bool __started_{false}; + [[no_unique_addres]] + _Receiver __rcvr_; + [[no_unique_addres]] __awaitable_state<_Awaitable, __promise_t> __awaiter_; }; } // namespace __connect_await diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 24d8dbe11..3566d85eb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,6 +20,7 @@ set(stdexec_test_sources stdexec/cpos/test_cpo_receiver.cpp stdexec/cpos/test_cpo_start.cpp stdexec/cpos/test_cpo_connect.cpp + stdexec/cpos/test_cpo_connect_awaitable.cpp stdexec/cpos/test_cpo_schedule.cpp stdexec/cpos/test_cpo_upon_error.cpp stdexec/cpos/test_cpo_upon_stopped.cpp diff --git a/test/stdexec/cpos/test_cpo_connect_awaitable.cpp b/test/stdexec/cpos/test_cpo_connect_awaitable.cpp new file mode 100644 index 000000000..36b3a4383 --- /dev/null +++ b/test/stdexec/cpos/test_cpo_connect_awaitable.cpp @@ -0,0 +1,490 @@ +/* + * Copyright (c) 2026 Ian Petersen + * Copyright (c) 2026 NVIDIA Corporation + * + * Licensed under the Apache License Version 2.0 with LLVM Exceptions + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://llvm.org/LICENSE.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test_common/receivers.hpp" +#include "test_common/type_helpers.hpp" +#include +#include + +#include + +namespace ex = STDEXEC; + +namespace +{ + template + struct ready_awaitable + { + public: + explicit constexpr ready_awaitable(T&& t) noexcept + : t_(std::move(t)) + {} + + explicit constexpr ready_awaitable(T const & t) + : t_(t) + {} + + static constexpr bool await_ready() noexcept + { + return true; + } + + static void await_suspend(std::coroutine_handle<>) noexcept + { + FAIL_CHECK("this awaitable should never suspend"); + } + + constexpr T await_resume() noexcept + { + return std::move(t_); + } + + ready_awaitable& base() noexcept + { + return *this; + } + + private: + T t_; + }; + + template <> + struct ready_awaitable + { + static constexpr bool await_ready() noexcept + { + return true; + } + + static void await_suspend(std::coroutine_handle<>) noexcept + { + FAIL_CHECK("this awaitable should never suspend"); + } + + static constexpr void await_resume() noexcept {} + + ready_awaitable& base() noexcept + { + return *this; + } + }; + + template + struct awaitable_ref + { + Awaitable* awaitable_; + + explicit constexpr awaitable_ref(Awaitable& awaitable) noexcept + : awaitable_(&awaitable) + {} + + constexpr auto await_ready() const noexcept(noexcept(awaitable_->await_ready())) + requires requires(Awaitable& a) { + { a.await_ready() }; + } + { + return awaitable_->await_ready(); + } + + template + constexpr auto await_suspend(std::coroutine_handle coro) const + noexcept(noexcept(awaitable_->await_suspend(coro))) + requires requires(Awaitable& a) { + { a.await_suspend(coro) }; + } + { + return awaitable_->await_suspend(coro); + } + + constexpr decltype(auto) await_resume() const noexcept(noexcept(awaitable_->await_resume())) + requires requires(Awaitable& a) { + { a.await_resume() }; + } + { + return awaitable_->await_resume(); + } + + template + requires requires(Awaitable& a, Promise& p) { + { a.as_awaitable(p) } -> ex::__awaitable; + } + constexpr auto as_awaitable(Promise& p) const noexcept(noexcept(awaitable_->as_awaitable(p))) + { + return awaitable_->as_awaitable(p); + } + + constexpr auto operator co_await() const noexcept(noexcept(awaitable_->operator co_await())) + requires requires(Awaitable& a) { + { a.operator co_await() }; + } + { + return awaitable_->operator co_await(); + } + }; + + template + awaitable_ref(Awaitable&) -> awaitable_ref; + + template + requires requires(Awaitable& a) { + { operator co_await(a) }; + } + constexpr auto operator co_await(awaitable_ref ref) + noexcept(noexcept(operator co_await(*ref.awaitable_))) + { + return operator co_await(*ref.awaitable_); + } + + template + struct suspending_awaitable : ready_awaitable + { + using ready_awaitable::ready_awaitable; + + suspending_awaitable(suspending_awaitable&&) = delete; + + ~suspending_awaitable() = default; + + constexpr void resume_parent() const noexcept + { + parent_.resume(); + } + + static constexpr bool await_ready() noexcept + { + return false; + } + + constexpr void await_suspend(std::coroutine_handle<> coro) noexcept + { + parent_ = coro; + } + + suspending_awaitable& base() noexcept + { + return *this; + } + + private: + std::coroutine_handle<> parent_; + }; + + template + struct conditionally_suspending_awaitable : suspending_awaitable + { + template + requires(sizeof...(U) == 0 && std::same_as) + || (sizeof...(U) == 1 && !std::same_as) + explicit constexpr conditionally_suspending_awaitable(bool suspend, U&&... u) noexcept + : suspending_awaitable(std::forward(u)...) + , suspend_(suspend) + {} + + constexpr bool await_suspend(std::coroutine_handle<> coro) noexcept + { + if (suspend_) + { + suspending_awaitable::await_suspend(coro); + } + + return suspend_; + } + + conditionally_suspending_awaitable& base() noexcept + { + return *this; + } + + private: + bool suspend_; + }; + + template + struct symmetrically_suspending_awaitable : conditionally_suspending_awaitable + { + using conditionally_suspending_awaitable::conditionally_suspending_awaitable; + + constexpr std::coroutine_handle<> await_suspend(std::coroutine_handle<> coro) noexcept + { + if (conditionally_suspending_awaitable::await_suspend(coro)) + { + return std::noop_coroutine(); + } + else + { + return coro; + } + } + + symmetrically_suspending_awaitable& base() noexcept + { + return *this; + } + }; + + template + struct with_as_awaitable + { + template + requires std::constructible_from + explicit(sizeof...(T) == 1) with_as_awaitable(T&&... t) + noexcept(std::is_nothrow_constructible_v) + : awaitable_(std::forward(t)...) + {} + + template + awaitable_ref as_awaitable(Promise&) noexcept + { + return awaitable_ref(awaitable_); + } + + auto& base() noexcept + { + return awaitable_.base(); + } + + private: + Awaitable awaitable_; + }; + + template + struct with_member_co_await + { + template + requires std::constructible_from + explicit(sizeof...(T) == 1) with_member_co_await(T&&... t) + noexcept(std::is_nothrow_constructible_v) + : awaitable_(std::forward(t)...) + {} + + constexpr awaitable_ref operator co_await() noexcept + { + return awaitable_ref(awaitable_); + } + + auto& base() noexcept + { + return awaitable_.base(); + } + + private: + Awaitable awaitable_; + }; + + template + struct with_friend_co_await + { + template + requires std::constructible_from + explicit(sizeof...(T) == 1) with_friend_co_await(T&&... t) + noexcept(std::is_nothrow_constructible_v) + : awaitable_(std::forward(t)...) + {} + + auto& base() noexcept + { + return awaitable_.base(); + } + + private: + Awaitable awaitable_; + + template + requires std::same_as, with_friend_co_await> + friend constexpr awaitable_ref operator co_await(Self&& self) noexcept + { + return awaitable_ref(self.awaitable_); + } + }; + + TEST_CASE("can connect and start a ready_awaitable", "[cpo][cpo_connect_awaitable]") + { + auto test = [](auto awaitable) noexcept + { + auto op = ex::connect(std::move(awaitable), expect_value_receiver{42}); + op.start(); + }; + + test(ready_awaitable{42}); + test(with_as_awaitable>{42}); + test(with_member_co_await>{42}); + test(with_friend_co_await>{42}); + test(with_as_awaitable>>{42}); + test(with_as_awaitable>>{42}); + } + + TEST_CASE("can connect and start a ready_awaitable", "[cpo][cpo_connect_awaitable]") + { + auto test = [](auto awaitable) noexcept + { + auto op = ex::connect(std::move(awaitable), expect_void_receiver{}); + op.start(); + }; + + test(ready_awaitable{}); + test(with_as_awaitable>{}); + test(with_member_co_await>{}); + test(with_friend_co_await>{}); + test(with_as_awaitable>>{}); + test(with_as_awaitable>>{}); + } + + TEST_CASE("can connect and start a suspending_awaitable", "[cpo][cpo_connect_awaitable]") + { + auto test = [](auto awaitable, auto... values) noexcept + { + auto op = ex::connect(awaitable_ref(awaitable), expect_value_receiver{std::move(values)...}); + op.start(); + awaitable.base().resume_parent(); + }; + + test(suspending_awaitable{42}, 42); + test(with_as_awaitable>{42}, 42); + test(with_member_co_await>{42}, 42); + test(with_friend_co_await>{42}, 42); + test(with_as_awaitable>>{42}, 42); + test(with_as_awaitable>>{42}, 42); + + test(suspending_awaitable{}); + test(with_as_awaitable>{}); + test(with_member_co_await>{}); + test(with_friend_co_await>{}); + test(with_as_awaitable>>{}); + test(with_as_awaitable>>{}); + } + + TEST_CASE("can connect and start a conditionally_suspending_awaitable", + "[cpo][cpo_connect_awaitable]") + { + { + auto test = [](auto awaitable, auto... values) noexcept + { + auto op = ex::connect(awaitable_ref(awaitable), + expect_value_receiver{std::move(values)...}); + op.start(); + awaitable.base().resume_parent(); + }; + + test(conditionally_suspending_awaitable(true, 42), 42); + test(with_as_awaitable>(true, 42), 42); + test(with_member_co_await>(true, 42), 42); + test(with_friend_co_await>(true, 42), 42); + test(with_as_awaitable>>(true, + 42), + 42); + test(with_as_awaitable>>(true, + 42), + 42); + + test(conditionally_suspending_awaitable(true)); + test(with_as_awaitable>(true)); + test(with_member_co_await>(true)); + test(with_friend_co_await>(true)); + test(with_as_awaitable>>(true)); + test(with_as_awaitable>>(true)); + } + + { + auto test = [](auto awaitable, auto... values) noexcept + { + auto op = ex::connect(awaitable_ref(awaitable), + expect_value_receiver{std::move(values)...}); + op.start(); + }; + + test(conditionally_suspending_awaitable(false, 42), 42); + test(with_as_awaitable>(false, 42), 42); + test(with_member_co_await>(false, 42), 42); + test(with_friend_co_await>(false, 42), 42); + test(with_as_awaitable>>(false, + 42), + 42); + test(with_as_awaitable>>(false, + 42), + 42); + + test(conditionally_suspending_awaitable(false)); + test(with_as_awaitable>(false)); + test(with_member_co_await>(false)); + test(with_friend_co_await>(false)); + test( + with_as_awaitable>>(false)); + test( + with_as_awaitable>>(false)); + } + } + + TEST_CASE("can connect and start a symmetrically_suspending_awaitable", + "[cpo][cpo_connect_awaitable]") + { + { + auto test = [](auto awaitable, auto... values) noexcept + { + auto op = ex::connect(awaitable_ref(awaitable), + expect_value_receiver{std::move(values)...}); + op.start(); + awaitable.base().resume_parent(); + }; + + test(symmetrically_suspending_awaitable(true, 42), 42); + test(with_as_awaitable>(true, 42), 42); + test(with_member_co_await>(true, 42), 42); + test(with_friend_co_await>(true, 42), 42); + test(with_as_awaitable>>(true, + 42), + 42); + test(with_as_awaitable>>(true, + 42), + 42); + + test(symmetrically_suspending_awaitable(true)); + test(with_as_awaitable>(true)); + test(with_member_co_await>(true)); + test(with_friend_co_await>(true)); + test(with_as_awaitable>>(true)); + test(with_as_awaitable>>(true)); + } + + { + auto test = [](auto awaitable, auto... values) noexcept + { + auto op = ex::connect(awaitable_ref(awaitable), + expect_value_receiver{std::move(values)...}); + op.start(); + }; + + test(symmetrically_suspending_awaitable(false, 42), 42); + test(with_as_awaitable>(false, 42), 42); + test(with_member_co_await>(false, 42), 42); + test(with_friend_co_await>(false, 42), 42); + test(with_as_awaitable>>(false, + 42), + 42); + test(with_as_awaitable>>(false, + 42), + 42); + + test(symmetrically_suspending_awaitable(false)); + test(with_as_awaitable>(false)); + test(with_member_co_await>(false)); + test(with_friend_co_await>(false)); + test( + with_as_awaitable>>(false)); + test( + with_as_awaitable>>(false)); + } + } +} // namespace From 1689f62aa39e0e954037b5f618d767320ebd37c9 Mon Sep 17 00:00:00 2001 From: Ian Petersen Date: Tue, 31 Mar 2026 20:13:03 -0700 Subject: [PATCH 10/26] Test exception handling in connect_awaitable --- .../stdexec/__detail/__connect_awaitable.hpp | 40 +-- .../cpos/test_cpo_connect_awaitable.cpp | 229 +++++++++++++++++- 2 files changed, 247 insertions(+), 22 deletions(-) diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 0f60fd4d6..25775bc22 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -127,23 +127,24 @@ namespace STDEXEC , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable_))) {} - [[no_unique_addres]] + [[no_unique_address]] __awaitable_t __awaitable_; - [[no_unique_addres]] + [[no_unique_address]] __awaiter_t __awaiter_; }; - [[no_unique_addres]] + [[no_unique_address]] _Awaitable __source_awaitable_; union { - [[no_unique_addres]] + [[no_unique_address]] __state __awaiter_; }; template requires(!std::same_as, __awaitable_state>) - __awaitable_state(_A&& __awaitable) noexcept(__nothrow_constructible_from<_Awaitable, _A>) + explicit __awaitable_state(_A&& __awaitable) + noexcept(__nothrow_constructible_from<_Awaitable, _A>) : __source_awaitable_(static_cast<_A&&>(__awaitable)) {} @@ -186,21 +187,22 @@ namespace STDEXEC STDEXEC_ASSERT(std::addressof(__awaitable) == std::addressof(__source)); } - [[no_unique_addres]] + [[no_unique_address]] __awaiter_t __awaiter_; }; - [[no_unique_addres]] + [[no_unique_address]] _Awaitable __source_awaitable_; union { - [[no_unique_addres]] + [[no_unique_address]] __state __awaiter_; }; template requires(!std::same_as, __awaitable_state>) - __awaitable_state(_A&& __awaitable) noexcept(__nothrow_constructible_from<_Awaitable, _A>) + explicit __awaitable_state(_A&& __awaitable) + noexcept(__nothrow_constructible_from<_Awaitable, _A>) : __source_awaitable_(static_cast<_A&&>(__awaitable)) {} @@ -241,21 +243,22 @@ namespace STDEXEC STDEXEC_ASSERT(std::addressof(__awaiter) == std::addressof(__awaiter_)); } - [[no_unique_addres]] + [[no_unique_address]] __awaiter_t __awaiter_; }; - [[no_unique_addres]] + [[no_unique_address]] _Awaitable __source_awaitable_; union { - [[no_unique_addres]] + [[no_unique_address]] __state __awaiter_; }; template requires(!std::same_as, __awaitable_state>) - __awaitable_state(_A&& __awaitable) noexcept(__nothrow_constructible_from<_Awaitable, _A>) + explicit __awaitable_state(_A&& __awaitable) + noexcept(__nothrow_constructible_from<_Awaitable, _A>) : __source_awaitable_(static_cast<_A&&>(__awaitable)) {} @@ -281,12 +284,13 @@ namespace STDEXEC { // _Awaitable has neither a distinct awaiter, nor a distinct awaitable // so we don't need separate storage for either - [[no_unique_addres]] + [[no_unique_address]] _Awaitable __awaiter_; template requires(!std::same_as, __awaitable_state>) - __awaitable_state(_A&& __awaitable) noexcept(__nothrow_constructible_from<_Awaitable, _A>) + explicit __awaitable_state(_A&& __awaitable) + noexcept(__nothrow_constructible_from<_Awaitable, _A>) : __awaiter_(static_cast<_A&&>(__awaitable)) {} @@ -470,11 +474,11 @@ namespace STDEXEC void start() & noexcept { auto __coro = __co_impl(*this); - __started_ = true; STDEXEC_TRY { __awaiter_.construct(__coro); + __started_ = true; if (!__awaiter_.await_ready()) { @@ -576,9 +580,9 @@ namespace STDEXEC alignas(__storage_align) std::byte __storage_[__storage_size]; [[no_unique_address]] bool __started_{false}; - [[no_unique_addres]] + [[no_unique_address]] _Receiver __rcvr_; - [[no_unique_addres]] + [[neo_unique_addres]] __awaitable_state<_Awaitable, __promise_t> __awaiter_; }; } // namespace __connect_await diff --git a/test/stdexec/cpos/test_cpo_connect_awaitable.cpp b/test/stdexec/cpos/test_cpo_connect_awaitable.cpp index 36b3a4383..5604c286d 100644 --- a/test/stdexec/cpos/test_cpo_connect_awaitable.cpp +++ b/test/stdexec/cpos/test_cpo_connect_awaitable.cpp @@ -20,6 +20,7 @@ #include #include +#include #include namespace ex = STDEXEC; @@ -140,13 +141,13 @@ namespace awaitable_ref(Awaitable&) -> awaitable_ref; template - requires requires(Awaitable& a) { - { operator co_await(a) }; + requires requires(Awaitable&& a) { + { operator co_await(std::move(a)) }; } constexpr auto operator co_await(awaitable_ref ref) - noexcept(noexcept(operator co_await(*ref.awaitable_))) + noexcept(noexcept(operator co_await(std::move(*ref.awaitable_)))) { - return operator co_await(*ref.awaitable_); + return operator co_await(std::move(*ref.awaitable_)); } template @@ -487,4 +488,224 @@ namespace with_as_awaitable>>(false)); } } + + TEST_CASE("exceptions thrown from await_ready are reported to set_error", + "[cpo][cpo_connect_awaitable]") + { + struct throw_on_ready : ready_awaitable + { + static bool await_ready() + { + throw std::runtime_error("not ready!"); + } + }; + + auto op = ex::connect(throw_on_ready{}, expect_error_receiver{}); + op.start(); + } + + TEST_CASE("exceptions thrown from void-returning await_suspend are reported to set_error", + "[cpo][cpo_connect_awaitable]") + { + struct throw_on_suspend : suspending_awaitable + { + static void await_suspend(std::coroutine_handle<>) + { + throw std::runtime_error("do not suspend!"); + } + } awaiter; + + auto op = ex::connect(awaitable_ref{awaiter}, expect_error_receiver{}); + op.start(); + } + + TEST_CASE("exceptions thrown from bool-returning await_suspend are reported to set_error", + "[cpo][cpo_connect_awaitable]") + { + struct throw_on_suspend : suspending_awaitable + { + static bool await_suspend(std::coroutine_handle<>) + { + throw std::runtime_error("do not suspend!"); + } + } awaiter; + + auto op = ex::connect(awaitable_ref{awaiter}, expect_error_receiver{}); + op.start(); + } + + TEST_CASE("exceptions thrown from handle-returning await_suspend are reported to set_error", + "[cpo][cpo_connect_awaitable]") + { + struct throw_on_suspend : suspending_awaitable + { + static std::coroutine_handle<> await_suspend(std::coroutine_handle<>) + { + throw std::runtime_error("do not suspend!"); + } + } awaiter; + + auto op = ex::connect(awaitable_ref{awaiter}, expect_error_receiver{}); + op.start(); + } + + TEST_CASE("exceptions thrown from immediately-invoked await_resume are reported to set_error", + "[cpo][cpo_connect_awaitable]") + { + { + struct throw_on_void_resume : ready_awaitable + { + static void await_resume() + { + throw std::runtime_error("no result for you!"); + } + }; + + auto op = ex::connect(throw_on_void_resume{}, expect_error_receiver{}); + op.start(); + } + { + struct throw_on_int_resume : ready_awaitable + { + static int await_resume() + { + throw std::runtime_error("no result for you!"); + } + }; + + auto op = ex::connect(throw_on_int_resume{}, expect_error_receiver{}); + op.start(); + } + } + + TEST_CASE("exceptions thrown from deferred-invoked await_resume are reported to set_error", + "[cpo][cpo_connect_awaitable]") + { + { + { + struct throw_on_void_resume : suspending_awaitable + { + static void await_resume() + { + throw std::runtime_error("no result for you!"); + } + } awaitable; + + auto op = ex::connect(awaitable_ref{awaitable}, expect_error_receiver{}); + op.start(); + awaitable.resume_parent(); + } + { + struct throw_on_int_resume : suspending_awaitable + { + static int await_resume() + { + throw std::runtime_error("no result for you!"); + } + } awaitable; + + auto op = ex::connect(awaitable_ref{awaitable}, expect_error_receiver{}); + op.start(); + awaitable.resume_parent(); + } + } + } + + template