From f21f2bd3a2c5da822b125bf20d1555a43ed938bb Mon Sep 17 00:00:00 2001 From: Eric Niebler Date: Thu, 26 Mar 2026 19:33:29 -0700 Subject: [PATCH 1/4] change `__connect_awaitable` to await the awaitable with no dynamic allocations --- examples/hello_coro.cpp | 7 +- include/stdexec/__detail/__awaitable.hpp | 30 +- include/stdexec/__detail/__config.hpp | 9 + .../stdexec/__detail/__connect_awaitable.hpp | 274 ++++++++++++------ .../stdexec/__detail/__manual_lifetime.hpp | 71 ++++- 5 files changed, 274 insertions(+), 117 deletions(-) diff --git a/examples/hello_coro.cpp b/examples/hello_coro.cpp index 457695308..5139b299c 100644 --- a/examples/hello_coro.cpp +++ b/examples/hello_coro.cpp @@ -19,12 +19,11 @@ #include #if !STDEXEC_NO_STDCPP_COROUTINES() && !STDEXEC_NVHPC() -# include using namespace stdexec; template -auto async_answer(S1 s1, S2 s2) -> exec::task +auto async_answer(S1 s1, S2 s2) -> stdexec::task { // Senders are implicitly awaitable (in this coroutine type): co_await static_cast(s2); @@ -32,13 +31,13 @@ auto async_answer(S1 s1, S2 s2) -> exec::task } template -auto async_answer2(S1 s1, S2 s2) -> exec::task> +auto async_answer2(S1 s1, S2 s2) -> stdexec::task> { co_return co_await stopped_as_optional(async_answer(s1, s2)); } // tasks have an associated stop token -auto async_stop_token() -> exec::task> +auto async_stop_token() -> stdexec::task> { co_return co_await stopped_as_optional(get_stop_token()); } diff --git a/include/stdexec/__detail/__awaitable.hpp b/include/stdexec/__detail/__awaitable.hpp index 268f10862..02c3a8ace 100644 --- a/include/stdexec/__detail/__awaitable.hpp +++ b/include/stdexec/__detail/__awaitable.hpp @@ -15,6 +15,7 @@ */ #pragma once +#include "../functional.hpp" #include "__concepts.hpp" #include "__config.hpp" #include "__meta.hpp" @@ -39,20 +40,20 @@ namespace STDEXEC __promise.await_transform(static_cast<_Awaitable &&>(__awaitable)); }; - template - constexpr auto __get_awaitable(_Awaitable &&__awaitable, __ignore = {}) -> decltype(auto) - { - return static_cast<_Awaitable &&>(__awaitable); - } + inline constexpr auto __get_awaitable = __first_callable{ + [] _Awaitable>(_Awaitable &&__awaitable, + _Promise &__promise) + -> decltype(auto) + { return __promise.await_transform(static_cast<_Awaitable &&>(__awaitable)); }, + [](_Awaitable &&__awaitable, __ignore = {}) -> decltype(auto) + { return static_cast<_Awaitable &&>(__awaitable); }}; - template _Awaitable> - constexpr auto __get_awaitable(_Awaitable &&__awaitable, _Promise &__promise) -> decltype(auto) - { - return __promise.await_transform(static_cast<_Awaitable &&>(__awaitable)); - } + template + using __awaitable_of_t = decltype(STDEXEC::__get_awaitable(__declval<_Awaitable>(), + __declval<_Promise &>())); - template - constexpr auto __get_awaiter(_Awaitable &&__awaitable) -> decltype(auto) + inline constexpr auto __get_awaiter = + [](_Awaitable &&__awaitable) -> decltype(auto) { if constexpr (requires { __declval<_Awaitable>().operator co_await(); }) { @@ -66,7 +67,10 @@ namespace STDEXEC { return static_cast<_Awaitable &&>(__awaitable); } - } + }; + + template + using __awaiter_of_t = decltype(STDEXEC::__get_awaiter(__declval<_Awaitable>())); template concept __awaitable = requires(_Awaitable &&__awaitable, _Promise &...__promise) { diff --git a/include/stdexec/__detail/__config.hpp b/include/stdexec/__detail/__config.hpp index 9ea098967..6eba12998 100644 --- a/include/stdexec/__detail/__config.hpp +++ b/include/stdexec/__detail/__config.hpp @@ -401,6 +401,15 @@ namespace STDEXEC::__std # define STDEXEC_PRAGMA_IGNORE_MSVC(...) #endif +#if STDEXEC_MSVC() +# define STDEXEC_PRAGMA_OPTIMIZE_BEGIN() STDEXEC_PRAGMA(optimize("", on)) +# define STDEXEC_PRAGMA_OPTIMIZE_END() STDEXEC_PRAGMA(optimize("", off)) +#else +# define STDEXEC_PRAGMA_OPTIMIZE_BEGIN() STDEXEC_PRAGMA(GCC push_options) \ + STDEXEC_PRAGMA(GCC optimize("O3")) +# define STDEXEC_PRAGMA_OPTIMIZE_END() STDEXEC_PRAGMA(GCC pop_options) +#endif + #if !STDEXEC_MSVC() && defined(__has_builtin) # define STDEXEC_HAS_BUILTIN __has_builtin #else diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 7a783a46d..9092aa182 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -17,11 +17,14 @@ #include "__execution_fwd.hpp" +#include "../coroutine.hpp" +#include "../functional.hpp" #include "__awaitable.hpp" #include "__completion_signatures.hpp" #include "__concepts.hpp" #include "__config.hpp" #include "__env.hpp" +#include "__manual_lifetime.hpp" #include "__receivers.hpp" #include @@ -37,6 +40,11 @@ namespace STDEXEC // __connect_await namespace __connect_await { + STDEXEC_PRAGMA_OPTIMIZE_BEGIN() + + static constexpr std::size_t __storage_size = 256; + static constexpr std::size_t __storage_align = __STDCPP_DEFAULT_NEW_ALIGNMENT__; + // clang-format off template concept __has_as_awaitable_member = requires(_Tp&& __t, _Promise& __promise) { @@ -80,45 +88,61 @@ namespace STDEXEC } }; - struct __promise_base + struct __awaiter_base { - constexpr auto initial_suspend() noexcept -> __std::suspend_always + static constexpr auto await_ready() noexcept -> bool { - return {}; + return false; } [[noreturn]] - auto final_suspend() noexcept -> __std::suspend_always + void await_resume() noexcept { - std::terminate(); + __std::unreachable(); } + }; - [[noreturn]] - void unhandled_exception() noexcept + // Turn a nothrow nullary-callable into an awaitable that simply invokes the callable + // in await_suspend. + template + struct __set_value_awaiter; + + template + struct __set_value_awaiter<_Receiver> : __awaiter_base + { + constexpr void await_suspend(__std::coroutine_handle<>) noexcept { - std::terminate(); + STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_)); + // This coroutine is never resumed; its work is done. } - [[noreturn]] - void return_void() noexcept + _Receiver& __rcvr_; + }; + + template + struct __set_value_awaiter<_Receiver, _Arg> : __awaiter_base + { + constexpr void await_suspend(__std::coroutine_handle<>) noexcept { - std::terminate(); + STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), static_cast<_Arg&&>(__arg_)); + // This coroutine is never resumed; its work is done. } + + _Receiver& __rcvr_; + _Arg&& __arg_; }; - struct __operation_base + struct __task_base { - __std::coroutine_handle<> __coro_; - - constexpr explicit __operation_base(__std::coroutine_handle<> __hcoro) noexcept - : __coro_(__hcoro) + constexpr explicit __task_base(__std::coroutine_handle<> __coro) noexcept + : __coro_(__coro) {} - constexpr __operation_base(__operation_base&& __other) noexcept + constexpr __task_base(__task_base&& __other) noexcept : __coro_(std::exchange(__other.__coro_, {})) {} - constexpr ~__operation_base() + constexpr ~__task_base() { if (__coro_) { @@ -136,135 +160,195 @@ namespace STDEXEC } } - void start() & noexcept - { - __coro_.resume(); - } + __std::coroutine_handle<> __coro_; }; template struct __promise; template - struct __operation : __operation_base + struct __task : __task_base { using promise_type = __promise<_Receiver>; - using __operation_base::__operation_base; + using __task_base::__task_base; + }; + + struct __promise_base + { + static constexpr auto initial_suspend() noexcept -> __std::suspend_always + { + return {}; + } + + void unhandled_exception() noexcept + { + __eptr_ = std::current_exception(); + } + + [[noreturn]] + static void return_void() noexcept + { + __std::unreachable(); + } + + std::exception_ptr __eptr_; }; template - struct __promise + struct __opstate_base + { + _Receiver __rcvr_; + alignas(__storage_align) std::byte __storage_[__storage_size]; + }; + + template + struct STDEXEC_ATTRIBUTE(empty_bases) __promise : __promise_base , __with_await_transform<__promise<_Receiver>> { - constexpr explicit(!STDEXEC_EDG()) __promise(auto&, _Receiver& __rcvr) noexcept - : __rcvr_(__rcvr) + constexpr explicit(!STDEXEC_EDG()) __promise(__opstate_base<_Receiver>& __opstate) noexcept + : __opstate_(__opstate) {} + static constexpr auto operator new([[maybe_unused]] std::size_t __bytes, + __opstate_base<_Receiver>& __opstate) noexcept -> void* + { + STDEXEC_ASSERT(__bytes <= __storage_size); + return __opstate.__storage_; + } + + static constexpr void operator delete([[maybe_unused]] void* __ptr) noexcept + { + // no-op + } + + struct __set_error_awaiter : __awaiter_base + { + void await_suspend(__std::coroutine_handle<>) noexcept + { + STDEXEC::set_error(static_cast<_Receiver&&>(__promise_.__opstate_.__rcvr_), + std::move(__promise_.__eptr_)); + // This coroutine is never resumed; its work is done. + } + + __promise& __promise_; + }; + + constexpr auto final_suspend() noexcept -> __set_error_awaiter + { + STDEXEC_ASSERT(__eptr_); + return __set_error_awaiter{{}, *this}; + } + constexpr auto unhandled_stopped() noexcept -> __std::coroutine_handle<> { - STDEXEC::set_stopped(static_cast<_Receiver&&>(__rcvr_)); + STDEXEC::set_stopped(static_cast<_Receiver&&>(__opstate_.__rcvr_)); // Returning noop_coroutine here causes the __connect_awaitable // coroutine to never resume past the point where it co_await's // the awaitable. return __std::noop_coroutine(); } - constexpr auto get_return_object() noexcept -> __operation<_Receiver> + constexpr auto get_return_object() noexcept -> __task<_Receiver> { - return __operation<_Receiver>{__std::coroutine_handle<__promise>::from_promise(*this)}; + return __task<_Receiver>{__std::coroutine_handle<__promise>::from_promise(*this)}; + } + + [[noreturn]] + static constexpr auto get_return_object_on_allocation_failure() noexcept -> __task<_Receiver> + { + __std::unreachable(); } - // Pass through the get_env receiver query constexpr auto get_env() const noexcept -> env_of_t<_Receiver> { - return STDEXEC::get_env(__rcvr_); + return STDEXEC::get_env(__opstate_.__rcvr_); } - _Receiver& __rcvr_; + __opstate_base<_Receiver>& __opstate_; }; - } // namespace __connect_await - struct __connect_awaitable_t - { - private: - template - static constexpr auto __co_call(_Fun __fun, _Ts&&... __as) noexcept + template + struct __opstate : __opstate_base<_Receiver> { - auto __fn = [&, __fun]() noexcept + explicit __opstate(_Awaitable&& __awaitable, _Receiver&& __rcvr) + noexcept(__nothrow_move_constructible<_Awaitable>) + : __opstate_base<_Receiver>{static_cast<_Receiver&&>(__rcvr)} + , __awaitable1_(static_cast<_Awaitable&&>(__awaitable)) + , __task_(__opstate::__co_impl(*this)) { - __fun(static_cast<_Ts&&>(__as)...); - }; + auto __coro = STDEXEC::__coroutine_handle_cast<__promise<_Receiver>>(__task_.__coro_); - struct __awaiter - { - decltype(__fn) __fn_; + __awaitable2_.__construct_from(STDEXEC::__get_awaitable, + static_cast<_Awaitable&&>(__awaitable1_), + __coro.promise()); - static constexpr auto await_ready() noexcept -> bool - { - return false; - } - - constexpr void await_suspend(__std::coroutine_handle<>) noexcept - { - __fn_(); - } + __awaiter_.__construct_from(STDEXEC::__get_awaiter, + static_cast<__awaitable_t&&>(__awaitable2_.__get())); + } - [[noreturn]] - void await_resume() noexcept - { - std::terminate(); - } - }; + STDEXEC_IMMOVABLE(__opstate); - return __awaiter{__fn}; - } + ~__opstate() + { + __awaiter_.__destroy(); + __awaitable2_.__destroy(); + } - template -# if STDEXEC_GCC() && (STDEXEC_GCC_VERSION >= 12'00) - __attribute__((__used__)) -# endif - static auto __co_impl(_Awaitable __awaitable, _Receiver __rcvr) // - -> __connect_await::__operation<_Receiver> - { - using __result_t = __await_result_t<_Awaitable, __connect_await::__promise<_Receiver>>; - std::exception_ptr __eptr; - STDEXEC_TRY + void start() & noexcept { - if constexpr (__std::same_as<__result_t, void>) - co_await (co_await static_cast<_Awaitable&&>(__awaitable), - __co_call(set_value, static_cast<_Receiver&&>(__rcvr))); - else - co_await __co_call(set_value, - static_cast<_Receiver&&>(__rcvr), - co_await static_cast<_Awaitable&&>(__awaitable)); + __task_.__coro_.resume(); } - STDEXEC_CATCH_ALL + + private: + using __awaitable_t = __awaitable_of_t<_Awaitable, __promise<_Receiver>>; + using __awaiter_t = __awaiter_of_t<__awaitable_t>; + + static auto __co_impl(__opstate& __op) noexcept -> __task<_Receiver> { - __eptr = std::current_exception(); + auto&& __awaiter = __op.__awaiter_.__get(); + using __result_t = decltype(__declval<__awaiter_t>().await_resume()); + + if constexpr (__same_as<__result_t, void>) + { + using __set_value_t = __set_value_awaiter<_Receiver>; + co_await (co_await static_cast<__awaiter_t&&>(__awaiter), + __set_value_t{{}, __op.__rcvr_}); + } + else + { + using __set_value_t = __set_value_awaiter<_Receiver, __result_t>; + co_await __set_value_t{{}, __op.__rcvr_, co_await static_cast<__awaiter_t&&>(__awaiter)}; + } } - co_await __co_call(set_error, - static_cast<_Receiver&&>(__rcvr), - static_cast(__eptr)); - } - template - using __completions_t = completion_signatures< - __single_value_sig_t<__await_result_t<_Awaitable, __connect_await::__promise<_Receiver>>>, - set_error_t(std::exception_ptr), - set_stopped_t()>; + _Awaitable __awaitable1_; + __manual_lifetime<__awaitable_t> __awaitable2_; + __manual_lifetime<__awaiter_t> __awaiter_; + __task<_Receiver> __task_; + }; - public: + STDEXEC_PRAGMA_OPTIMIZE_END() + } // namespace __connect_await + + struct __connect_awaitable_t + { template > _Awaitable> - requires receiver_of<_Receiver, __completions_t<_Receiver, _Awaitable>> auto operator()(_Awaitable&& __awaitable, _Receiver __rcvr) const - -> __connect_await::__operation<_Receiver> + noexcept(__nothrow_move_constructible<_Awaitable>) { - return __co_impl(static_cast<_Awaitable&&>(__awaitable), static_cast<_Receiver&&>(__rcvr)); + using __result_t = __await_result_t<_Awaitable, __connect_await::__promise<_Receiver>>; + using __completions_t = completion_signatures<__single_value_sig_t<__result_t>, + set_error_t(std::exception_ptr), + set_stopped_t()>; + static_assert(receiver_of<_Receiver, __completions_t>); + return __connect_await::__opstate(static_cast<_Awaitable&&>(__awaitable), + static_cast<_Receiver&&>(__rcvr)); } }; #else + namespace __connect_await { template @@ -278,7 +362,9 @@ namespace STDEXEC struct __connect_awaitable_t {}; + #endif + inline constexpr __connect_awaitable_t __connect_awaitable{}; } // namespace STDEXEC diff --git a/include/stdexec/__detail/__manual_lifetime.hpp b/include/stdexec/__detail/__manual_lifetime.hpp index 4c3e632a9..a805490b0 100644 --- a/include/stdexec/__detail/__manual_lifetime.hpp +++ b/include/stdexec/__detail/__manual_lifetime.hpp @@ -20,6 +20,7 @@ #include #include +#include namespace STDEXEC { @@ -67,38 +68,96 @@ namespace STDEXEC } //! End the lifetime of the contained `_Ty`. - //! Precondition: The lifetime has started. + //! \pre The lifetime has started. constexpr void __destroy() noexcept { std::destroy_at(&__get()); } //! Get access to the `_Ty`. - //! Precondition: The lifetime has started. + //! \pre The lifetime has started. constexpr auto __get() & noexcept -> _Ty& { return *reinterpret_cast<_Ty*>(__buffer_); } //! Get access to the `_Ty`. - //! Precondition: The lifetime has started. + //! \pre The lifetime has started. constexpr auto __get() && noexcept -> _Ty&& { return static_cast<_Ty&&>(*reinterpret_cast<_Ty*>(__buffer_)); } //! Get access to the `_Ty`. - //! Precondition: The lifetime has started. + //! \pre The lifetime has started. constexpr auto __get() const & noexcept -> _Ty const & { return *reinterpret_cast<_Ty const *>(__buffer_); } - //! Move semantics aren't supported. - //! If you want to move the `_Ty`, use `std::move(ml.__get())`. constexpr auto __get() const && noexcept -> _Ty const && = delete; + constexpr auto operator->() noexcept -> _Ty* + { + return reinterpret_cast<_Ty*>(__buffer_); + } + + constexpr auto operator->() const noexcept -> _Ty const * + { + return reinterpret_cast<_Ty const *>(__buffer_); + } + private: alignas(_Ty) unsigned char __buffer_[sizeof(_Ty)]{}; }; + + template + requires std::is_reference_v<_Reference> + class __manual_lifetime<_Reference> + { + public: + //! Constructor does nothing: It's on you to call `__construct(...)` or `__construct_from(...)` + //! if you want the `_Reference`'s lifetime to begin. + constexpr __manual_lifetime() noexcept = default; + + //! Destructor does nothing: It's on you to call `__destroy()` if you mean to. + constexpr ~__manual_lifetime() = default; + + constexpr __manual_lifetime(__manual_lifetime const &) = delete; + constexpr auto operator=(__manual_lifetime const &) -> __manual_lifetime& = delete; + + constexpr __manual_lifetime(__manual_lifetime&&) = delete; + constexpr auto operator=(__manual_lifetime&&) -> __manual_lifetime& = delete; + + constexpr auto __construct(_Reference __ref) noexcept -> _Reference + { + __ptr_ = std::addressof(__ref); + return static_cast<_Reference>(*__ptr_); + } + + template + constexpr auto __construct_from(_Func&& func, _Args&&... __args) + noexcept(__nothrow_callable<_Func, _Args...>) -> _Reference + { + decltype(auto) __result = static_cast<_Func&&>(func)(static_cast<_Args&&>(__args)...); + static_assert(std::is_reference_v, "Result must be a reference"); + __ptr_ = std::addressof(__result); + return static_cast<_Reference>(*__ptr_); + } + + constexpr void __destroy() noexcept {} + + constexpr auto __get() const noexcept -> _Reference + { + return static_cast<_Reference>(*__ptr_); + } + + constexpr auto operator->() const noexcept -> std::add_pointer_t<_Reference> + { + return __ptr_; + } + + private: + std::add_pointer_t<_Reference> __ptr_{nullptr}; + }; } // namespace STDEXEC From 3c6f9d02fc9267388ec8395289f5616aa5c3fd74 Mon Sep 17 00:00:00 2001 From: Eric Niebler Date: Sat, 28 Mar 2026 14:19:58 -0700 Subject: [PATCH 2/4] bring down coroutine frame size --- include/exec/any_sender_of.hpp | 10 +- include/exec/at_coroutine_exit.hpp | 2 +- include/exec/on_coro_disposition.hpp | 2 +- include/stdexec/__detail/__connect.hpp | 3 +- .../stdexec/__detail/__connect_awaitable.hpp | 318 ++++++++---------- .../stdexec/__detail/__manual_lifetime.hpp | 50 +-- include/stdexec/__detail/__optional.hpp | 109 +++--- .../__detail/__with_awaitable_senders.hpp | 148 +++----- include/stdexec/coroutine.hpp | 97 +++++- 9 files changed, 341 insertions(+), 398 deletions(-) diff --git a/include/exec/any_sender_of.hpp b/include/exec/any_sender_of.hpp index 05b5fb794..9a109ba5b 100644 --- a/include/exec/any_sender_of.hpp +++ b/include/exec/any_sender_of.hpp @@ -371,9 +371,8 @@ namespace experimental::execution void __reset() noexcept { - (*__vtable_)(__any::__delete, this); - __object_pointer_ = nullptr; - __vtable_ = __default_storage_vtable(static_cast<__vtable_t*>(nullptr)); + auto* __default_vtable = __default_storage_vtable((__vtable_t*) nullptr); + (*std::exchange(__vtable_, __default_vtable))(__any::__delete, this); } [[nodiscard]] @@ -561,9 +560,8 @@ namespace experimental::execution void __reset() noexcept { - (*__vtable_)(__any::__delete, this); - __object_pointer_ = nullptr; - __vtable_ = __default_storage_vtable(static_cast<__vtable_t*>(nullptr)); + auto* __default_vtable = __default_storage_vtable((__vtable_t*) nullptr); + (*std::exchange(__vtable_, __default_vtable))(__any::__delete, this); } auto __get_vtable() const noexcept -> _Vtable const * diff --git a/include/exec/at_coroutine_exit.hpp b/include/exec/at_coroutine_exit.hpp index fb3e70d14..f585fc2c8 100644 --- a/include/exec/at_coroutine_exit.hpp +++ b/include/exec/at_coroutine_exit.hpp @@ -181,7 +181,7 @@ namespace experimental::execution { auto __cont = __h.promise().continuation(); auto __coro = __h.promise().__is_stopped_ ? __cont.unhandled_stopped() : __cont.handle(); - return STDEXEC_DESTROY_AND_CONTINUE(__h, __coro); + return STDEXEC_CORO_DESTROY_AND_CONTINUE(__h, __coro); } void await_resume() const noexcept {} diff --git a/include/exec/on_coro_disposition.hpp b/include/exec/on_coro_disposition.hpp index 2141fbe31..a20f9336d 100644 --- a/include/exec/on_coro_disposition.hpp +++ b/include/exec/on_coro_disposition.hpp @@ -122,7 +122,7 @@ namespace experimental::execution { auto __cont = __h.promise().continuation(); auto __coro = __h.promise().__is_stopped_ ? __cont.unhandled_stopped() : __cont.handle(); - return STDEXEC_DESTROY_AND_CONTINUE(__h, __coro); + return STDEXEC_CORO_DESTROY_AND_CONTINUE(__h, __coro); } void await_resume() const noexcept {} diff --git a/include/stdexec/__detail/__connect.hpp b/include/stdexec/__detail/__connect.hpp index 4b09205fc..4ffd5fb90 100644 --- a/include/stdexec/__detail/__connect.hpp +++ b/include/stdexec/__detail/__connect.hpp @@ -43,7 +43,8 @@ namespace STDEXEC }; template - concept __with_co_await = __awaitable<_Sender, __connect_await::__promise<_Receiver>>; + concept __with_co_await = + __awaitable<_Sender, __connect_awaitable_t::__promise_t<_Sender, _Receiver>>; template concept __with_legacy_tag_invoke = __tag_invocable; diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 9092aa182..685a58e8a 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -18,13 +18,11 @@ #include "__execution_fwd.hpp" #include "../coroutine.hpp" -#include "../functional.hpp" #include "__awaitable.hpp" -#include "__completion_signatures.hpp" #include "__concepts.hpp" #include "__config.hpp" #include "__env.hpp" -#include "__manual_lifetime.hpp" +#include "__optional.hpp" #include "__receivers.hpp" #include @@ -42,12 +40,13 @@ namespace STDEXEC { STDEXEC_PRAGMA_OPTIMIZE_BEGIN() - static constexpr std::size_t __storage_size = 256; + static constexpr std::size_t __storage_size = 8 * sizeof(void*); static constexpr std::size_t __storage_align = __STDCPP_DEFAULT_NEW_ALIGNMENT__; // clang-format off template - concept __has_as_awaitable_member = requires(_Tp&& __t, _Promise& __promise) { + concept __has_as_awaitable_member = requires(_Tp&& __t, _Promise& __promise) + { static_cast<_Tp&&>(__t).as_awaitable(__promise); }; // clang-format on @@ -63,29 +62,18 @@ namespace STDEXEC return static_cast<_Ty&&>(__value); } - template - requires __has_as_awaitable_member<_Ty, _Promise&> + template <__has_as_awaitable_member<_Promise&> _Ty> STDEXEC_ATTRIBUTE(nodiscard, host, device) - auto await_transform(_Ty&& __value) - noexcept(noexcept(__declval<_Ty>().as_awaitable(__declval<_Promise&>()))) - -> decltype(__declval<_Ty>().as_awaitable(__declval<_Promise&>())) + constexpr auto await_transform(_Ty&& __value) // + noexcept(noexcept(__declval<_Ty>().as_awaitable(__declval<_Promise&>()))) // + -> decltype(auto) { return static_cast<_Ty&&>(__value).as_awaitable(static_cast<_Promise&>(*this)); } - template - requires __has_as_awaitable_member<_Ty, _Promise&> - || __tag_invocable - [[deprecated("The use of tag_invoke to customize the behavior of await_transform is " - "deprecated. Please provide an as_awaitable member function instead.")]] - STDEXEC_ATTRIBUTE(nodiscard, host, device) auto await_transform(_Ty&& __value) - noexcept(__nothrow_tag_invocable) - -> __tag_invoke_result_t - { - return __tag_invoke(as_awaitable, - static_cast<_Ty&&>(__value), - static_cast<_Promise&>(*this)); - } + private: + friend _Promise; + __with_await_transform() = default; }; struct __awaiter_base @@ -96,124 +84,82 @@ namespace STDEXEC } [[noreturn]] - void await_resume() noexcept + inline void await_resume() noexcept { __std::unreachable(); } }; - // Turn a nothrow nullary-callable into an awaitable that simply invokes the callable - // in await_suspend. - template - struct __set_value_awaiter; - - template - struct __set_value_awaiter<_Receiver> : __awaiter_base - { - constexpr void await_suspend(__std::coroutine_handle<>) noexcept - { - STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_)); - // This coroutine is never resumed; its work is done. - } - - _Receiver& __rcvr_; - }; - - template - struct __set_value_awaiter<_Receiver, _Arg> : __awaiter_base + inline void __destroy_coro(__std::coroutine_handle<> __coro) noexcept { - constexpr void await_suspend(__std::coroutine_handle<>) noexcept - { - STDEXEC::set_value(static_cast<_Receiver&&>(__rcvr_), static_cast<_Arg&&>(__arg_)); - // This coroutine is never resumed; its work is done. - } - - _Receiver& __rcvr_; - _Arg&& __arg_; - }; - - struct __task_base - { - constexpr explicit __task_base(__std::coroutine_handle<> __coro) noexcept - : __coro_(__coro) - {} - - constexpr __task_base(__task_base&& __other) noexcept - : __coro_(std::exchange(__other.__coro_, {})) - {} - - constexpr ~__task_base() - { - if (__coro_) - { # if STDEXEC_MSVC() - // MSVCBUG https://developercommunity.visualstudio.com/t/Double-destroy-of-a-local-in-coroutine-d/10456428 - - // Reassign __coro_ before calling destroy to make the mutation - // observable and to hopefully ensure that the compiler does not eliminate it. - auto __coro = __coro_; - __coro_ = {}; - __coro.destroy(); + // MSVCBUG https://developercommunity.visualstudio.com/t/Double-destroy-of-a-local-in-coroutine-d/10456428 + // Reassign __coro_ before calling destroy to make the mutation + // observable and to hopefully ensure that the compiler does not eliminate it. + std::exchange(__coro, {}).destroy(); # else - __coro_.destroy(); + __coro.destroy(); # endif - } - } - - __std::coroutine_handle<> __coro_; - }; + } - template - struct __promise; + template + struct __opstate; - template - struct __task : __task_base + template + struct __promise : __with_await_transform<__promise<_Awaitable, _Receiver>> { - using promise_type = __promise<_Receiver>; - using __task_base::__task_base; - }; + using __opstate_t = __opstate<_Awaitable, _Receiver>; - struct __promise_base - { - static constexpr auto initial_suspend() noexcept -> __std::suspend_always + struct __task : __immovable { - return {}; - } + using promise_type = __promise; - void unhandled_exception() noexcept - { - __eptr_ = std::current_exception(); - } + ~__task() + { + __connect_await::__destroy_coro(__coro_); + } - [[noreturn]] - static void return_void() noexcept - { - __std::unreachable(); - } + __std::coroutine_handle<__promise> __coro_{}; + }; - std::exception_ptr __eptr_; - }; + struct __final_awaiter : __awaiter_base + { + void await_suspend(__std::coroutine_handle<>) noexcept + { + using __awaitable_t = __result_of<__get_awaitable, _Awaitable, __promise&>; + using __awaiter_t = __awaiter_of_t<__awaitable_t>; + using __result_t = decltype(__declval<__awaiter_t>().await_resume()); + + if (__opstate_.__eptr_) + { + STDEXEC::set_error(static_cast<_Receiver&&>(__opstate_.__rcvr_), + std::move(__opstate_.__eptr_)); + } + else if constexpr (__same_as<__result_t, void>) + { + STDEXEC_ASSERT(__opstate_.__result_.has_value()); + STDEXEC::set_value(static_cast<_Receiver&&>(__opstate_.__rcvr_)); + } + else + { + STDEXEC_ASSERT(__opstate_.__result_.has_value()); + STDEXEC::set_value(static_cast<_Receiver&&>(__opstate_.__rcvr_), + static_cast<__result_t&&>(*__opstate_.__result_)); + } + // This coroutine is never resumed; its work is done. + } - template - struct __opstate_base - { - _Receiver __rcvr_; - alignas(__storage_align) std::byte __storage_[__storage_size]; - }; + __opstate<_Awaitable, _Receiver>& __opstate_; + }; - template - struct STDEXEC_ATTRIBUTE(empty_bases) __promise - : __promise_base - , __with_await_transform<__promise<_Receiver>> - { - constexpr explicit(!STDEXEC_EDG()) __promise(__opstate_base<_Receiver>& __opstate) noexcept + constexpr explicit(!STDEXEC_EDG()) __promise(__opstate_t& __opstate) noexcept : __opstate_(__opstate) {} - static constexpr auto operator new([[maybe_unused]] std::size_t __bytes, - __opstate_base<_Receiver>& __opstate) noexcept -> void* + static constexpr auto + operator new([[maybe_unused]] std::size_t __bytes, __opstate_t& __opstate) noexcept -> void* { - STDEXEC_ASSERT(__bytes <= __storage_size); + STDEXEC_ASSERT(__bytes <= sizeof(__opstate.__storage_)); return __opstate.__storage_; } @@ -222,22 +168,25 @@ namespace STDEXEC // no-op } - struct __set_error_awaiter : __awaiter_base + constexpr auto get_return_object() noexcept -> __task { - void await_suspend(__std::coroutine_handle<>) noexcept - { - STDEXEC::set_error(static_cast<_Receiver&&>(__promise_.__opstate_.__rcvr_), - std::move(__promise_.__eptr_)); - // This coroutine is never resumed; its work is done. - } + return __task{{}, __std::coroutine_handle<__promise>::from_promise(*this)}; + } - __promise& __promise_; - }; + [[noreturn]] + static auto get_return_object_on_allocation_failure() noexcept -> __task + { + __std::unreachable(); + } - constexpr auto final_suspend() noexcept -> __set_error_awaiter + static constexpr auto initial_suspend() noexcept -> __std::suspend_always { - STDEXEC_ASSERT(__eptr_); - return __set_error_awaiter{{}, *this}; + return {}; + } + + void unhandled_exception() noexcept + { + __opstate_.__eptr_ = std::current_exception(); } constexpr auto unhandled_stopped() noexcept -> __std::coroutine_handle<> @@ -249,51 +198,37 @@ namespace STDEXEC return __std::noop_coroutine(); } - constexpr auto get_return_object() noexcept -> __task<_Receiver> + constexpr auto final_suspend() noexcept -> __final_awaiter { - return __task<_Receiver>{__std::coroutine_handle<__promise>::from_promise(*this)}; + return __final_awaiter{{}, __opstate_}; } - [[noreturn]] - static constexpr auto get_return_object_on_allocation_failure() noexcept -> __task<_Receiver> + static void return_void() noexcept { - __std::unreachable(); + // no-op } + [[nodiscard]] constexpr auto get_env() const noexcept -> env_of_t<_Receiver> { return STDEXEC::get_env(__opstate_.__rcvr_); } - __opstate_base<_Receiver>& __opstate_; + __opstate<_Awaitable, _Receiver>& __opstate_; }; template - struct __opstate : __opstate_base<_Receiver> + struct __opstate { - explicit __opstate(_Awaitable&& __awaitable, _Receiver&& __rcvr) - noexcept(__nothrow_move_constructible<_Awaitable>) - : __opstate_base<_Receiver>{static_cast<_Receiver&&>(__rcvr)} + constexpr explicit __opstate(_Awaitable&& __awaitable, _Receiver&& __rcvr) + noexcept(__is_nothrow) + : __rcvr_(static_cast<_Receiver&&>(__rcvr)) + , __task_(__co_impl(*this)) , __awaitable1_(static_cast<_Awaitable&&>(__awaitable)) - , __task_(__opstate::__co_impl(*this)) - { - auto __coro = STDEXEC::__coroutine_handle_cast<__promise<_Receiver>>(__task_.__coro_); - - __awaitable2_.__construct_from(STDEXEC::__get_awaitable, - static_cast<_Awaitable&&>(__awaitable1_), - __coro.promise()); - - __awaiter_.__construct_from(STDEXEC::__get_awaiter, - static_cast<__awaitable_t&&>(__awaitable2_.__get())); - } - - STDEXEC_IMMOVABLE(__opstate); - - ~__opstate() - { - __awaiter_.__destroy(); - __awaitable2_.__destroy(); - } + , __awaitable2_( + __get_awaitable(static_cast<_Awaitable&&>(__awaitable1_), __task_.__coro_.promise())) + , __awaiter_(__get_awaiter(static_cast<__awaitable_t&&>(__awaitable2_))) + {} void start() & noexcept { @@ -301,31 +236,44 @@ namespace STDEXEC } private: - using __awaitable_t = __awaitable_of_t<_Awaitable, __promise<_Receiver>>; + using __promise_t = __promise<_Awaitable, _Receiver>; + using __task_t = __promise_t::__task; + using __awaitable_t = __result_of<__get_awaitable, _Awaitable, __promise_t&>; using __awaiter_t = __awaiter_of_t<__awaitable_t>; + using __result_t = decltype(__declval<__awaiter_t>().await_resume()); - static auto __co_impl(__opstate& __op) noexcept -> __task<_Receiver> - { - auto&& __awaiter = __op.__awaiter_.__get(); - using __result_t = decltype(__declval<__awaiter_t>().await_resume()); + friend __promise_t; - if constexpr (__same_as<__result_t, void>) + static constexpr bool __is_nothrow = __nothrow_move_constructible<_Awaitable> + && __noexcept_of<__get_awaitable, _Awaitable, __promise_t&> + && __noexcept_of<__get_awaiter, __awaitable_t>; + + static constexpr std::size_t __storage_size = __connect_await::__storage_size + + sizeof(__manual_lifetime<__result_t>) + - __same_as<__result_t, void>; + + static auto __co_impl(__opstate& __op) noexcept -> __task_t + { + using __awaiter_t = decltype(__op.__awaiter_); + if constexpr (__same_as) { - using __set_value_t = __set_value_awaiter<_Receiver>; - co_await (co_await static_cast<__awaiter_t&&>(__awaiter), - __set_value_t{{}, __op.__rcvr_}); + co_await static_cast<__awaiter_t&&>(__op.__awaiter_); + __op.__result_.emplace(); } else { - using __set_value_t = __set_value_awaiter<_Receiver, __result_t>; - co_await __set_value_t{{}, __op.__rcvr_, co_await static_cast<__awaiter_t&&>(__awaiter)}; + __op.__result_.emplace(co_await static_cast<__awaiter_t&&>(__op.__awaiter_)); } } - _Awaitable __awaitable1_; - __manual_lifetime<__awaitable_t> __awaitable2_; - __manual_lifetime<__awaiter_t> __awaiter_; - __task<_Receiver> __task_; + alignas(__storage_align) std::byte __storage_[__storage_size]; + _Receiver __rcvr_; + __promise_t::__task __task_; + _Awaitable __awaitable1_; + __awaitable_t __awaitable2_; + __awaiter_t __awaiter_; + std::exception_ptr __eptr_{}; + __optional<__result_t> __result_{}; }; STDEXEC_PRAGMA_OPTIMIZE_END() @@ -333,17 +281,19 @@ namespace STDEXEC struct __connect_awaitable_t { - template > _Awaitable> - auto operator()(_Awaitable&& __awaitable, _Receiver __rcvr) const - noexcept(__nothrow_move_constructible<_Awaitable>) + template + using __opstate_t = __connect_await::__opstate<_Awaitable, _Receiver>; + + template + using __promise_t = __connect_await::__promise<_Awaitable, _Receiver>; + + template + requires __awaitable<_Awaitable, __promise_t<_Awaitable, _Receiver>> + auto operator()(_Awaitable&& __awaitable, _Receiver __rcvr) const noexcept( + __nothrow_constructible_from<__opstate_t<_Awaitable, _Receiver>, _Awaitable, _Receiver>) { - using __result_t = __await_result_t<_Awaitable, __connect_await::__promise<_Receiver>>; - using __completions_t = completion_signatures<__single_value_sig_t<__result_t>, - set_error_t(std::exception_ptr), - set_stopped_t()>; - static_assert(receiver_of<_Receiver, __completions_t>); - return __connect_await::__opstate(static_cast<_Awaitable&&>(__awaitable), - static_cast<_Receiver&&>(__rcvr)); + return __opstate_t<_Awaitable, _Receiver>(static_cast<_Awaitable&&>(__awaitable), + static_cast<_Receiver&&>(__rcvr)); } }; diff --git a/include/stdexec/__detail/__manual_lifetime.hpp b/include/stdexec/__detail/__manual_lifetime.hpp index a805490b0..1d90e6460 100644 --- a/include/stdexec/__detail/__manual_lifetime.hpp +++ b/include/stdexec/__detail/__manual_lifetime.hpp @@ -17,6 +17,7 @@ #pragma once #include "__concepts.hpp" +#include "__utility.hpp" #include #include @@ -24,11 +25,10 @@ namespace STDEXEC { - //! Holds storage for a `_Ty`, but allows clients to `__construct(...)`, `__destry()`, //! and `__get()` the `_Ty` without regard for usual lifetime rules. template - class __manual_lifetime + class __manual_lifetime : __immovable { public: //! Constructor does nothing: It's on you to call `__construct(...)` or `__construct_from(...)` @@ -38,12 +38,6 @@ namespace STDEXEC //! Destructor does nothing: It's on you to call `__destroy()` if you mean to. constexpr ~__manual_lifetime() = default; - constexpr __manual_lifetime(__manual_lifetime const &) = delete; - constexpr auto operator=(__manual_lifetime const &) -> __manual_lifetime& = delete; - - constexpr __manual_lifetime(__manual_lifetime&&) = delete; - constexpr auto operator=(__manual_lifetime&&) -> __manual_lifetime& = delete; - //! Construct the `_Ty` in place. //! There are no safeties guarding against the case that there's already one there. template @@ -64,7 +58,7 @@ namespace STDEXEC // Use placement new instead of std::construct_at in case the function returns an immovable // type. return *std::launder(::new (static_cast(__buffer_)) - _Ty{(static_cast<_Func&&>(func))(static_cast<_Args&&>(__args)...)}); + _Ty{static_cast<_Func&&>(func)(static_cast<_Args&&>(__args)...)}); } //! End the lifetime of the contained `_Ty`. @@ -113,22 +107,9 @@ namespace STDEXEC template requires std::is_reference_v<_Reference> - class __manual_lifetime<_Reference> + class __manual_lifetime<_Reference> : __immovable { public: - //! Constructor does nothing: It's on you to call `__construct(...)` or `__construct_from(...)` - //! if you want the `_Reference`'s lifetime to begin. - constexpr __manual_lifetime() noexcept = default; - - //! Destructor does nothing: It's on you to call `__destroy()` if you mean to. - constexpr ~__manual_lifetime() = default; - - constexpr __manual_lifetime(__manual_lifetime const &) = delete; - constexpr auto operator=(__manual_lifetime const &) -> __manual_lifetime& = delete; - - constexpr __manual_lifetime(__manual_lifetime&&) = delete; - constexpr auto operator=(__manual_lifetime&&) -> __manual_lifetime& = delete; - constexpr auto __construct(_Reference __ref) noexcept -> _Reference { __ptr_ = std::addressof(__ref); @@ -160,4 +141,27 @@ namespace STDEXEC private: std::add_pointer_t<_Reference> __ptr_{nullptr}; }; + + template <> + struct __manual_lifetime : __immovable + { + template + constexpr void __construct(_Args&&...) noexcept + {} + + template + constexpr void __construct_from(_Func&& __func, _Args&&... __args) noexcept + { + (void) static_cast<_Func&&>(__func)(static_cast<_Args&&>(__args)...); + } + + constexpr void __destroy() noexcept {} + + constexpr auto __get() const noexcept -> void {} + + constexpr auto operator->() const noexcept -> void* + { + return nullptr; + } + }; } // namespace STDEXEC diff --git a/include/stdexec/__detail/__optional.hpp b/include/stdexec/__detail/__optional.hpp index bdf528ef1..371c4780b 100644 --- a/include/stdexec/__detail/__optional.hpp +++ b/include/stdexec/__detail/__optional.hpp @@ -19,6 +19,7 @@ // include these after __execution_fwd.hpp #include "__concepts.hpp" +#include "__manual_lifetime.hpp" #include "__scope.hpp" #include @@ -55,13 +56,6 @@ namespace STDEXEC { static_assert(__std::destructible<_Tp>); - union - { - _Tp __value_; - }; - - bool __has_value_ = false; - constexpr __optional() noexcept {} constexpr __optional(__nullopt_t) noexcept {} @@ -87,7 +81,7 @@ namespace STDEXEC { if (__has_value_) { - std::destroy_at(std::addressof(__value_)); + __value_.__destroy(); } } @@ -110,10 +104,10 @@ namespace STDEXEC emplace(_Us&&... __us) noexcept(__nothrow_constructible_from<_Tp, _Us...>) -> _Tp& { reset(); - auto __sg = __mk_has_value_guard(__has_value_); - auto* __p = std::construct_at(std::addressof(__value_), static_cast<_Us&&>(__us)...); + auto __sg = __mk_has_value_guard(__has_value_); + _Tp& __t = __value_.__construct(static_cast<_Us&&>(__us)...); __sg.__dismiss(); - return *std::launder(__p); + return __t; } template @@ -122,11 +116,11 @@ namespace STDEXEC -> _Tp& { reset(); - auto __sg = __mk_has_value_guard(__has_value_); - auto* __p = ::new (static_cast(std::addressof(__value_))) - _Tp(static_cast<_Fn&&>(__f)(static_cast<_Args&&>(__args)...)); + auto __sg = __mk_has_value_guard(__has_value_); + _Tp& __t = __value_.__construct_from(static_cast<_Fn&&>(__f), + static_cast<_Args&&>(__args)...); __sg.__dismiss(); - return *std::launder(__p); + return __t; } constexpr auto value() & -> _Tp& @@ -135,7 +129,7 @@ namespace STDEXEC { STDEXEC_THROW(__bad_optional_access()); } - return __value_; + return __value_.__get(); } constexpr auto value() const & -> _Tp const & @@ -144,7 +138,7 @@ namespace STDEXEC { STDEXEC_THROW(__bad_optional_access()); } - return __value_; + return __value_.__get(); } constexpr auto value() && -> _Tp&& @@ -153,37 +147,37 @@ namespace STDEXEC { STDEXEC_THROW(__bad_optional_access()); } - return static_cast<_Tp&&>(static_cast<_Tp&>(__value_)); + return std::move(__value_).__get(); } constexpr auto operator*() & noexcept -> _Tp& { STDEXEC_ASSERT(__has_value_); - return __value_; + return __value_.__get(); } constexpr auto operator*() const & noexcept -> _Tp const & { STDEXEC_ASSERT(__has_value_); - return __value_; + return __value_.__get(); } constexpr auto operator*() && noexcept -> _Tp&& { STDEXEC_ASSERT(__has_value_); - return static_cast<_Tp&&>(static_cast<_Tp&>(__value_)); + return std::move(__value_).__get(); } constexpr auto operator->() & noexcept -> std::add_pointer_t<_Tp> { STDEXEC_ASSERT(__has_value_); - return std::addressof(static_cast<_Tp&>(__value_)); + return std::addressof(__value_.__get()); } constexpr auto operator->() const & noexcept -> std::add_pointer_t<_Tp const> { STDEXEC_ASSERT(__has_value_); - return std::addressof(static_cast<_Tp const &>(__value_)); + return std::addressof(__value_.__get()); } [[nodiscard]] @@ -196,85 +190,62 @@ namespace STDEXEC { if (__has_value_) { - std::destroy_at(std::addressof(__value_)); + __value_.__destroy(); __has_value_ = false; } } + + private: + __manual_lifetime<_Tp> __value_; + bool __has_value_ = false; }; - // __optional - template - struct __optional<_Tp&> + // __optional + template <> + struct __optional { - _Tp* __value_ = nullptr; - __optional() noexcept = default; __optional(__nullopt_t) noexcept {} - template <__not_decays_to<__optional> _Up> - requires __std::constructible_from<_Tp&, _Up> - constexpr __optional(_Up&& __val) noexcept(__nothrow_constructible_from<_Tp&, _Up>) - { - emplace(static_cast<_Up&&>(__val)); - } - - template <__not_decays_to<__optional> _Up> - requires __std::constructible_from<_Tp&, _Up> - constexpr __optional(std::in_place_t, _Up&& __val) - noexcept(__nothrow_constructible_from<_Tp&, _Up>) - { - emplace(static_cast<_Up&&>(__val)); - } - - template - requires __std::constructible_from<_Tp&, _Up> - constexpr auto emplace(_Up&& __us) noexcept(__nothrow_constructible_from<_Tp&, _Up>) -> _Tp& - { - __value_ = std::addressof(static_cast<_Up&&>(__us)); - return *__value_; - } - - template - requires __std::same_as<_Tp&, __call_result_t<_Fn, _Args...>> - auto __emplace_from(_Fn&& __f, _Args&&... __args) noexcept(__nothrow_callable<_Fn, _Args...>) - -> _Tp& + template + constexpr auto emplace(_Us&&...) noexcept -> void { - __value_ = std::addressof(static_cast<_Fn&&>(__f)(static_cast<_Args&&>(__args)...)); - return *__value_; + __has_value_ = true; } - constexpr auto value() const -> _Tp& + constexpr auto value() const -> void { - if (__value_ == nullptr) + if (!__has_value_) { STDEXEC_THROW(__bad_optional_access()); } - return *__value_; } - constexpr auto operator*() const -> _Tp& + constexpr auto operator*() const -> void { - STDEXEC_ASSERT(__value_ != nullptr); - return *__value_; + STDEXEC_ASSERT(__has_value_); } - constexpr auto operator->() const -> _Tp* + constexpr auto operator->() const -> void* { - STDEXEC_ASSERT(__value_ != nullptr); - return __value_; + STDEXEC_ASSERT(__has_value_); + return nullptr; } [[nodiscard]] constexpr auto has_value() const noexcept -> bool { - return __value_ != nullptr; + return __has_value_; } constexpr void reset() noexcept { - __value_ = nullptr; + __has_value_ = false; } + + private: + bool __has_value_ = false; }; } // namespace __opt diff --git a/include/stdexec/__detail/__with_awaitable_senders.hpp b/include/stdexec/__detail/__with_awaitable_senders.hpp index a8b580fdf..0b4831d97 100644 --- a/include/stdexec/__detail/__with_awaitable_senders.hpp +++ b/include/stdexec/__detail/__with_awaitable_senders.hpp @@ -17,142 +17,70 @@ #include "__execution_fwd.hpp" -#include "../coroutine.hpp" // IWYU pragma: keep for __std::coroutine_handle +#include "../coroutine.hpp" // IWYU pragma: keep for __coroutine_handle #include "__as_awaitable.hpp" #include "__concepts.hpp" -#include - namespace STDEXEC { #if !STDEXEC_NO_STDCPP_COROUTINES() - template - class __coroutine_handle; + template + struct with_awaitable_senders; - template <> - class __coroutine_handle : __std::coroutine_handle<> + namespace __detail { - public: - constexpr __coroutine_handle() = default; - - template - constexpr __coroutine_handle(__std::coroutine_handle<_Promise> __coro) noexcept - : __std::coroutine_handle<>(__coro) + struct __with_awaitable_senders { - if constexpr (requires(_Promise& __promise) { __promise.unhandled_stopped(); }) + template + constexpr void set_continuation(__std::coroutine_handle<_OtherPromise> __hcoro) noexcept { - __stopped_callback_ = [](void* __address) noexcept -> __std::coroutine_handle<> - { - // This causes the rest of the coroutine (the part after the co_await - // of the sender) to be skipped and invokes the calling coroutine's - // stopped handler. - return __std::coroutine_handle<_Promise>::from_address(__address) - .promise() - .unhandled_stopped(); - }; + static_assert(!__same_as<_OtherPromise, void>); + __continuation_ = __hcoro; } - // If _Promise doesn't implement unhandled_stopped(), then if a "stopped" unwind - // reaches this point, it's considered an unhandled exception and terminate() - // is called. - } - - [[nodiscard]] - constexpr auto handle() const noexcept -> __std::coroutine_handle<> - { - return *this; - } - - [[nodiscard]] - constexpr auto unhandled_stopped() const noexcept -> __std::coroutine_handle<> - { - return __stopped_callback_(address()); - } - - private: - using __stopped_callback_t = __std::coroutine_handle<> (*)(void*) noexcept; - - __stopped_callback_t __stopped_callback_ = [](void*) noexcept -> __std::coroutine_handle<> - { - std::terminate(); - }; - }; - - template - class __coroutine_handle : public __coroutine_handle<> - { - public: - constexpr __coroutine_handle() = default; - - constexpr __coroutine_handle(__std::coroutine_handle<_Promise> __coro) noexcept - : __coroutine_handle<>{__coro} - {} - - [[nodiscard]] - static constexpr auto from_promise(_Promise& __promise) noexcept -> __coroutine_handle - { - return __coroutine_handle(__std::coroutine_handle<_Promise>::from_promise(__promise)); - } - - [[nodiscard]] - constexpr auto promise() const noexcept -> _Promise& - { - return __std::coroutine_handle<_Promise>::from_address(address()).promise(); - } - [[nodiscard]] - constexpr auto handle() const noexcept -> __std::coroutine_handle<_Promise> - { - return __std::coroutine_handle<_Promise>::from_address(address()); - } - - [[nodiscard]] - constexpr operator __std::coroutine_handle<_Promise>() const noexcept - { - return handle(); - } - }; + constexpr void set_continuation(__coroutine_handle<> __continuation) noexcept + { + __continuation_ = __continuation; + } - struct __with_awaitable_senders_base - { - template - constexpr void set_continuation(__std::coroutine_handle<_OtherPromise> __hcoro) noexcept - { - static_assert(!__same_as<_OtherPromise, void>); - __continuation_ = __hcoro; - } + [[nodiscard]] + constexpr auto continuation() const noexcept -> __coroutine_handle<> + { + return __continuation_; + } - constexpr void set_continuation(__coroutine_handle<> __continuation) noexcept - { - __continuation_ = __continuation; - } + [[nodiscard]] + constexpr auto unhandled_stopped() noexcept -> __std::coroutine_handle<> + { + return __continuation_.unhandled_stopped(); + } - [[nodiscard]] - constexpr auto continuation() const noexcept -> __coroutine_handle<> - { - return __continuation_; - } + private: + template + friend struct STDEXEC::with_awaitable_senders; - [[nodiscard]] - constexpr auto unhandled_stopped() noexcept -> __std::coroutine_handle<> - { - return __continuation_.unhandled_stopped(); - } + __with_awaitable_senders() = default; - private: - __coroutine_handle<> __continuation_{}; - }; + __coroutine_handle<> __continuation_{}; + }; + } // namespace __detail template - struct with_awaitable_senders : __with_awaitable_senders_base + struct with_awaitable_senders : __detail::__with_awaitable_senders { template [[nodiscard]] - constexpr auto - await_transform(_Value&& __val) -> __call_result_t + constexpr auto await_transform(_Value&& __val) + noexcept(__nothrow_callable) // + -> __call_result_t { static_assert(__std::derived_from<_Promise, with_awaitable_senders>); return as_awaitable(static_cast<_Value&&>(__val), static_cast<_Promise&>(*this)); } + + private: + friend _Promise; + with_awaitable_senders() = default; }; #endif } // namespace STDEXEC diff --git a/include/stdexec/coroutine.hpp b/include/stdexec/coroutine.hpp index da6b14e97..115d211ac 100644 --- a/include/stdexec/coroutine.hpp +++ b/include/stdexec/coroutine.hpp @@ -19,6 +19,8 @@ #include "__detail/__concepts.hpp" #include "__detail/__config.hpp" +#include + #if !STDEXEC_NO_STDCPP_COROUTINES() namespace STDEXEC @@ -30,6 +32,94 @@ namespace STDEXEC return __std::coroutine_handle<_Tp>::from_address(__h.address()); } + // A coroutine handle that also supports unhandled_stopped() for propagating stop + // signals through co_awaits of senders. + template + class __coroutine_handle; + + template <> + class __coroutine_handle : __std::coroutine_handle<> + { + public: + constexpr __coroutine_handle() = default; + + template + constexpr __coroutine_handle(__std::coroutine_handle<_Promise> __coro) noexcept + : __std::coroutine_handle<>(__coro) + { + if constexpr (requires(_Promise& __promise) { __promise.unhandled_stopped(); }) + { + __stopped_callback_ = [](void* __address) noexcept -> __std::coroutine_handle<> + { + // This causes the rest of the coroutine (the part after the co_await + // of the sender) to be skipped and invokes the calling coroutine's + // stopped handler. + return __std::coroutine_handle<_Promise>::from_address(__address) + .promise() + .unhandled_stopped(); + }; + } + // If _Promise doesn't implement unhandled_stopped(), then if a "stopped" unwind + // reaches this point, it's considered an unhandled exception and terminate() + // is called. + } + + [[nodiscard]] + constexpr auto handle() const noexcept -> __std::coroutine_handle<> + { + return *this; + } + + [[nodiscard]] + constexpr auto unhandled_stopped() const noexcept -> __std::coroutine_handle<> + { + return __stopped_callback_(address()); + } + + private: + using __stopped_callback_t = __std::coroutine_handle<> (*)(void*) noexcept; + + __stopped_callback_t __stopped_callback_ = [](void*) noexcept -> __std::coroutine_handle<> + { + std::terminate(); + }; + }; + + template + class __coroutine_handle : public __coroutine_handle<> + { + public: + constexpr __coroutine_handle() = default; + + constexpr __coroutine_handle(__std::coroutine_handle<_Promise> __coro) noexcept + : __coroutine_handle<>{__coro} + {} + + [[nodiscard]] + static constexpr auto from_promise(_Promise& __promise) noexcept -> __coroutine_handle + { + return __coroutine_handle(__std::coroutine_handle<_Promise>::from_promise(__promise)); + } + + [[nodiscard]] + constexpr auto promise() const noexcept -> _Promise& + { + return __std::coroutine_handle<_Promise>::from_address(address()).promise(); + } + + [[nodiscard]] + constexpr auto handle() const noexcept -> __std::coroutine_handle<_Promise> + { + return __std::coroutine_handle<_Promise>::from_address(address()); + } + + [[nodiscard]] + constexpr operator __std::coroutine_handle<_Promise>() const noexcept + { + return handle(); + } + }; + # if STDEXEC_MSVC() && STDEXEC_MSVC_VERSION <= 19'39 // MSVCBUG https://developercommunity.visualstudio.com/t/destroy-coroutine-from-final_suspend-r/10096047 @@ -140,10 +230,11 @@ namespace STDEXEC } } // namespace __destroy_and_continue_msvc -# define STDEXEC_DESTROY_AND_CONTINUE(__destroy, __continue) \ - (::STDEXEC::__destroy_and_continue_msvc::__impl(__destroy, __continue)) +# define STDEXEC_CORO_DESTROY_AND_CONTINUE(__destroy, __continue) \ + (::STDEXEC::__destroy_and_continue_msvc::__impl(__destroy, __continue)) # else -# define STDEXEC_DESTROY_AND_CONTINUE(__destroy, __continue) (__destroy.destroy(), __continue) +# define STDEXEC_CORO_DESTROY_AND_CONTINUE(__destroy, __continue) \ + (__destroy.destroy(), __continue) # endif } // namespace STDEXEC From 8eada519a1c0ecff6c60ff38928a0c64d5216cf8 Mon Sep 17 00:00:00 2001 From: Eric Niebler Date: Sat, 28 Mar 2026 15:33:40 -0700 Subject: [PATCH 3/4] fix no-coroutines build --- include/stdexec/__detail/__connect.hpp | 2 +- include/stdexec/__detail/__connect_awaitable.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/stdexec/__detail/__connect.hpp b/include/stdexec/__detail/__connect.hpp index 4ffd5fb90..3d97ffa8d 100644 --- a/include/stdexec/__detail/__connect.hpp +++ b/include/stdexec/__detail/__connect.hpp @@ -44,7 +44,7 @@ namespace STDEXEC template concept __with_co_await = - __awaitable<_Sender, __connect_awaitable_t::__promise_t<_Sender, _Receiver>>; + __awaitable<_Sender, __connect_await::__promise<_Sender, _Receiver>>; template concept __with_legacy_tag_invoke = __tag_invocable; diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index 685a58e8a..34a1e7491 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -94,7 +94,7 @@ namespace STDEXEC { # if STDEXEC_MSVC() // MSVCBUG https://developercommunity.visualstudio.com/t/Double-destroy-of-a-local-in-coroutine-d/10456428 - // Reassign __coro_ before calling destroy to make the mutation + // Reassign __coro before calling destroy to make the mutation // observable and to hopefully ensure that the compiler does not eliminate it. std::exchange(__coro, {}).destroy(); # else @@ -301,7 +301,7 @@ namespace STDEXEC namespace __connect_await { - template + template struct __promise {}; From 0fd3088dd9b1601592fa3dc6c576c60a3c29e628 Mon Sep 17 00:00:00 2001 From: Eric Niebler Date: Sat, 28 Mar 2026 15:38:48 -0700 Subject: [PATCH 4/4] formatting --- include/stdexec/__detail/__connect.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/stdexec/__detail/__connect.hpp b/include/stdexec/__detail/__connect.hpp index 3d97ffa8d..7f8eb853e 100644 --- a/include/stdexec/__detail/__connect.hpp +++ b/include/stdexec/__detail/__connect.hpp @@ -43,8 +43,7 @@ namespace STDEXEC }; template - concept __with_co_await = - __awaitable<_Sender, __connect_await::__promise<_Sender, _Receiver>>; + concept __with_co_await = __awaitable<_Sender, __connect_await::__promise<_Sender, _Receiver>>; template concept __with_legacy_tag_invoke = __tag_invocable;