From cf032c54efd31a0914272cd7b5acae5b0e4f8e10 Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Fri, 1 May 2026 09:40:42 -0700 Subject: [PATCH 01/20] Initial addition of http-handlers for C++ mirrored from Rust --- crates/bindings-cpp/include/spacetimedb.h | 3 + .../include/spacetimedb/handler_context.h | 88 ++++++++ .../include/spacetimedb/http_client_impl.h | 2 +- .../include/spacetimedb/http_convert.h | 20 ++ .../include/spacetimedb/http_handler_macros.h | 57 +++++ .../include/spacetimedb/http_wire.h | 64 ++++++ .../include/spacetimedb/internal/Module.h | 7 + .../internal/autogen/MethodOrAny.g.h | 28 +++ .../internal/autogen/RawHttpHandlerDefV10.g.h | 26 +++ .../internal/autogen/RawHttpRouteDefV10.g.h | 31 +++ .../autogen/RawModuleDefV10Section.g.h | 4 +- .../internal/runtime_registration.h | 8 + .../spacetimedb/internal/tx_execution.h | 150 +++++++++++++ .../spacetimedb/internal/v10_builder.h | 13 ++ .../include/spacetimedb/procedure_context.h | 102 ++------- .../include/spacetimedb/reducer_context.h | 3 + .../bindings-cpp/include/spacetimedb/router.h | 186 ++++++++++++++++ .../bindings-cpp/src/abi/module_exports.cpp | 15 ++ crates/bindings-cpp/src/internal/Module.cpp | 66 ++++++ .../bindings-cpp/src/internal/v10_builder.cpp | 38 ++++ .../tests/compile/CMakeLists.module.txt | 48 ++++ crates/bindings-cpp/tests/compile/README.md | 64 ++++++ .../error_http_handler_immutable_ctx.cpp | 14 ++ .../error_http_handler_no_args.cpp | 12 + .../error_http_handler_no_connection_id.cpp | 14 ++ .../error_http_handler_no_db.cpp | 20 ++ .../error_http_handler_no_request_arg.cpp | 13 ++ .../error_http_handler_no_return_type.cpp | 12 + .../error_http_handler_no_sender.cpp | 14 ++ .../error_http_handler_wrong_ctx.cpp | 14 ++ ...or_http_handler_wrong_request_arg_type.cpp | 14 ++ .../error_http_handler_wrong_return_type.cpp | 9 + .../error_http_router_not_a_function.cpp | 5 + .../error_http_router_with_args.cpp | 19 ++ .../error_http_router_wrong_return_type.cpp | 7 + .../http-handlers/ok_http_handlers_basic.cpp | 30 +++ .../tests/compile/run-compile-tests.ps1 | 208 +++++++++++++++++ .../tests/compile/run-compile-tests.sh | 209 ++++++++++++++++++ 38 files changed, 1550 insertions(+), 87 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/handler_context.h create mode 100644 crates/bindings-cpp/include/spacetimedb/http_handler_macros.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpHandlerDefV10.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpRouteDefV10.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/tx_execution.h create mode 100644 crates/bindings-cpp/include/spacetimedb/router.h create mode 100644 crates/bindings-cpp/tests/compile/CMakeLists.module.txt create mode 100644 crates/bindings-cpp/tests/compile/README.md create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_args.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_not_a_function.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_wrong_return_type.cpp create mode 100644 crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp create mode 100644 crates/bindings-cpp/tests/compile/run-compile-tests.ps1 create mode 100644 crates/bindings-cpp/tests/compile/run-compile-tests.sh diff --git a/crates/bindings-cpp/include/spacetimedb.h b/crates/bindings-cpp/include/spacetimedb.h index a36d96074a8..0eaaf9936cc 100644 --- a/crates/bindings-cpp/include/spacetimedb.h +++ b/crates/bindings-cpp/include/spacetimedb.h @@ -126,6 +126,9 @@ // Procedure context and macros #include "spacetimedb/procedure_macros.h" +#include "spacetimedb/handler_context.h" +#include "spacetimedb/router.h" +#include "spacetimedb/http_handler_macros.h" // ============================================================================= // VIEW SYSTEM diff --git a/crates/bindings-cpp/include/spacetimedb/handler_context.h b/crates/bindings-cpp/include/spacetimedb/handler_context.h new file mode 100644 index 00000000000..03a10eba0b5 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/handler_context.h @@ -0,0 +1,88 @@ +#ifndef SPACETIMEDB_HANDLER_CONTEXT_H +#define SPACETIMEDB_HANDLER_CONTEXT_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { + +struct HandlerContext { + Timestamp timestamp; + HttpClient http; + +private: + mutable std::shared_ptr rng_instance; + mutable uint32_t counter_uuid_ = 0; + +public: + HandlerContext() = default; + explicit HandlerContext(Timestamp t) : timestamp(t) {} + + Identity identity() const { + std::array id_bytes; + ::identity(id_bytes.data()); + return Identity(id_bytes); + } + + StdbRng& rng() const { + if (!rng_instance) { + rng_instance = std::make_shared(timestamp); + } + return *rng_instance; + } + + Uuid new_uuid_v4() const { + std::array random_bytes; + rng().fill_bytes(random_bytes.data(), random_bytes.size()); + return Uuid::from_random_bytes_v4(random_bytes); + } + + Uuid new_uuid_v7() const { + std::array random_bytes; + rng().fill_bytes(random_bytes.data(), random_bytes.size()); + return Uuid::from_counter_v7(counter_uuid_, timestamp, random_bytes); + } + +#ifdef SPACETIMEDB_UNSTABLE_FEATURES + template + auto with_tx(Func&& body) -> decltype(body(std::declval())) { + auto make_reducer_ctx = [](Timestamp tx_timestamp) { + return ReducerContext( + Identity{}, + std::nullopt, + tx_timestamp, + AuthCtx::internal() + ); + }; + return Internal::with_tx(make_reducer_ctx, body); + } + + template + auto try_with_tx(Func&& body) -> decltype(body(std::declval())) { + auto make_reducer_ctx = [](Timestamp tx_timestamp) { + return ReducerContext( + Identity{}, + std::nullopt, + tx_timestamp, + AuthCtx::internal() + ); + }; + return Internal::try_with_tx(make_reducer_ctx, body); + } +#endif +}; + +} // namespace SpacetimeDB + +#endif // SPACETIMEDB_HANDLER_CONTEXT_H diff --git a/crates/bindings-cpp/include/spacetimedb/http_client_impl.h b/crates/bindings-cpp/include/spacetimedb/http_client_impl.h index ff3e0faf3df..374b60a406c 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_client_impl.h +++ b/crates/bindings-cpp/include/spacetimedb/http_client_impl.h @@ -7,7 +7,7 @@ #include "spacetimedb/http_convert.h" #include "spacetimedb/abi/abi.h" #include "spacetimedb/bsatn/bsatn.h" -#include "spacetimedb/internal/Module.h" +#include "spacetimedb/internal/runtime_registration.h" namespace SpacetimeDB { diff --git a/crates/bindings-cpp/include/spacetimedb/http_convert.h b/crates/bindings-cpp/include/spacetimedb/http_convert.h index b479cf84d03..75f2652de15 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_convert.h +++ b/crates/bindings-cpp/include/spacetimedb/http_convert.h @@ -268,6 +268,26 @@ inline HttpResponse from_wire(const wire::HttpResponse& response) { return result; } +inline wire::RequestAndBody to_wire_with_body(const HttpRequest& request) { + return wire::RequestAndBody{to_wire(request), request.body.bytes}; +} + +inline HttpRequest from_wire(const wire::RequestAndBody& request) { + HttpRequest result = from_wire(request.request); + result.body.bytes = request.body; + return result; +} + +inline wire::ResponseAndBody to_wire_with_body(const HttpResponse& response) { + return wire::ResponseAndBody{to_wire(response), response.body.bytes}; +} + +inline HttpResponse from_wire(const wire::ResponseAndBody& response) { + HttpResponse result = from_wire(response.response); + result.body.bytes = response.body; + return result; +} + } // namespace convert } // namespace SpacetimeDB diff --git a/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h b/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h new file mode 100644 index 00000000000..864b3ea4425 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h @@ -0,0 +1,57 @@ +#pragma once + +#include "spacetimedb/handler_context.h" +#include "spacetimedb/http.h" +#include "spacetimedb/internal/runtime_registration.h" +#include "spacetimedb/internal/template_utils.h" +#include "spacetimedb/internal/v10_builder.h" +#include "spacetimedb/macros.h" +#include "spacetimedb/router.h" + +namespace SpacetimeDB::Internal { + +template +inline void RegisterHttpHandlerMacro(const char* handler_name, Func func) { + using traits = function_traits; + static_assert(traits::arity == 2, "HTTP handlers must take exactly two arguments"); + using ContextType = typename traits::template arg_t<0>; + using RequestType = typename traits::template arg_t<1>; + using ReturnType = typename traits::result_type; + static_assert(std::is_same_v, "First parameter of HTTP handler must be HandlerContext"); + static_assert(std::is_same_v, "Second parameter of HTTP handler must be HttpRequest"); + static_assert(std::is_same_v, "HTTP handlers must return HttpResponse"); + + std::function handler = + [func](HandlerContext& ctx, HttpRequest request) -> HttpResponse { + return func(ctx, std::move(request)); + }; + RegisterHttpHandlerHandler(handler_name, reinterpret_cast(func), std::move(handler)); + getV10Builder().RegisterHttpHandlerDef(handler_name); +} + +} // namespace SpacetimeDB::Internal + +#define SPACETIMEDB_HTTP_HANDLER(handler_name, ctx_param, request_param) \ + SpacetimeDB::HttpResponse handler_name(ctx_param, request_param); \ + __attribute__((export_name("__preinit__60_http_handler_" #handler_name))) \ + extern "C" void CONCAT(_spacetimedb_preinit_register_http_handler_, handler_name)() { \ + ::SpacetimeDB::Internal::RegisterHttpHandlerMacro(#handler_name, handler_name); \ + } \ + SpacetimeDB::HttpResponse handler_name(ctx_param, request_param) + +#define SPACETIMEDB_HTTP_HANDLER_NAMED(handler_name, canonical_name, ctx_param, request_param) \ + SpacetimeDB::HttpResponse handler_name(ctx_param, request_param); \ + __attribute__((export_name("__preinit__60_http_handler_" #handler_name))) \ + extern "C" void CONCAT(_spacetimedb_preinit_register_http_handler_, handler_name)() { \ + ::SpacetimeDB::Internal::RegisterHttpHandlerMacro(#handler_name, handler_name); \ + SpacetimeDB::Module::RegisterExplicitFunctionName(#handler_name, canonical_name); \ + } \ + SpacetimeDB::HttpResponse handler_name(ctx_param, request_param) + +#define SPACETIMEDB_HTTP_ROUTER(router_name) \ + SpacetimeDB::Router router_name(); \ + __attribute__((export_name("__preinit__61_http_router_" #router_name))) \ + extern "C" void CONCAT(_spacetimedb_preinit_register_http_router_, router_name)() { \ + ::SpacetimeDB::Internal::getV10Builder().RegisterHttpRouter(router_name()); \ + } \ + SpacetimeDB::Router router_name() diff --git a/crates/bindings-cpp/include/spacetimedb/http_wire.h b/crates/bindings-cpp/include/spacetimedb/http_wire.h index c4631a749aa..51871ab385e 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_wire.h +++ b/crates/bindings-cpp/include/spacetimedb/http_wire.h @@ -59,6 +59,14 @@ struct HttpMethod { std::string extension; // Only valid when tag == Extension }; +inline bool operator==(const HttpMethod& lhs, const HttpMethod& rhs) { + return lhs.tag == rhs.tag && lhs.extension == rhs.extension; +} + +inline bool operator!=(const HttpMethod& lhs, const HttpMethod& rhs) { + return !(lhs == rhs); +} + /** * @brief Wire format for HTTP version * @@ -155,6 +163,16 @@ struct HttpResponse { uint16_t code; // Field 2: HTTP status code }; +struct RequestAndBody { + HttpRequest request; + std::vector body; +}; + +struct ResponseAndBody { + HttpResponse response; + std::vector body; +}; + } // namespace wire } // namespace SpacetimeDB @@ -169,6 +187,8 @@ template<> struct bsatn_traits; template<> struct bsatn_traits; template<> struct bsatn_traits; template<> struct bsatn_traits; +template<> struct bsatn_traits; +template<> struct bsatn_traits; // HttpMethod enum serialization template<> @@ -341,6 +361,50 @@ struct bsatn_traits { } }; +template<> +struct bsatn_traits { + static void serialize(Writer& writer, const wire::RequestAndBody& value) { + bsatn::serialize(writer, value.request); + bsatn::serialize(writer, value.body); + } + + static wire::RequestAndBody deserialize(Reader& reader) { + wire::RequestAndBody result; + result.request = bsatn::deserialize(reader); + result.body = bsatn::deserialize>(reader); + return result; + } + + static AlgebraicType algebraic_type() { + ProductTypeBuilder builder; + builder.with_field("request"); + builder.with_field>("body"); + return AlgebraicType::make_product(builder.build()); + } +}; + +template<> +struct bsatn_traits { + static void serialize(Writer& writer, const wire::ResponseAndBody& value) { + bsatn::serialize(writer, value.response); + bsatn::serialize(writer, value.body); + } + + static wire::ResponseAndBody deserialize(Reader& reader) { + wire::ResponseAndBody result; + result.response = bsatn::deserialize(reader); + result.body = bsatn::deserialize>(reader); + return result; + } + + static AlgebraicType algebraic_type() { + ProductTypeBuilder builder; + builder.with_field("response"); + builder.with_field>("body"); + return AlgebraicType::make_product(builder.build()); + } +}; + } // namespace SpacetimeDB::bsatn #endif // SPACETIMEDB_HTTP_WIRE_H diff --git a/crates/bindings-cpp/include/spacetimedb/internal/Module.h b/crates/bindings-cpp/include/spacetimedb/internal/Module.h index dd27c18dc3a..b669dffc36f 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/Module.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/Module.h @@ -75,6 +75,13 @@ class Module { BytesSource args_source, BytesSink result_sink ); + + static int16_t __call_http_handler__( + uint32_t id, + uint64_t timestamp_microseconds, + BytesSource request_source, + BytesSink result_sink + ); // Internal registration methods (inline to avoid linking issues) template diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h new file mode 100644 index 00000000000..f266d538c3d --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h @@ -0,0 +1,28 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" +#include "spacetimedb/http_wire.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(MethodOrAny_Method_Wrapper) { + SpacetimeDB::wire::HttpMethod value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; + +SPACETIMEDB_INTERNAL_TAGGED_ENUM(MethodOrAny, std::monostate, MethodOrAny_Method_Wrapper) +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpHandlerDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpHandlerDefV10.g.h new file mode 100644 index 00000000000..b6495235e7b --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpHandlerDefV10.g.h @@ -0,0 +1,26 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawHttpHandlerDefV10) { + std::string source_name; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, source_name); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(source_name) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpRouteDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpRouteDefV10.g.h new file mode 100644 index 00000000000..791882e6dff --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpRouteDefV10.g.h @@ -0,0 +1,31 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" +#include "MethodOrAny.g.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawHttpRouteDefV10) { + std::string handler_function; + SpacetimeDB::Internal::MethodOrAny method; + std::string path; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, handler_function); + ::SpacetimeDB::bsatn::serialize(writer, method); + ::SpacetimeDB::bsatn::serialize(writer, path); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(handler_function, method, path) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index 241466f467c..5ddbde5e11a 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -23,8 +23,10 @@ #include "RawLifeCycleReducerDefV10.g.h" #include "RawRowLevelSecurityDefV9.g.h" #include "ExplicitNames.g.h" +#include "RawHttpHandlerDefV10.g.h" +#include "RawHttpRouteDefV10.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h b/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h index 4d84cb975d4..ee00ff1d54a 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h @@ -14,6 +14,9 @@ struct ReducerContext; struct ViewContext; struct AnonymousViewContext; struct ProcedureContext; +struct HandlerContext; +struct HttpRequest; +struct HttpResponse; namespace Internal { @@ -26,9 +29,14 @@ void RegisterAnonymousViewHandler(const std::string& name, std::function(AnonymousViewContext&, BytesSource)> handler); void RegisterProcedureHandler(const std::string& name, std::function(ProcedureContext&, BytesSource)> handler); +void RegisterHttpHandlerHandler(const std::string& name, + const void* handler_symbol, + std::function handler); +std::string LookupHttpHandlerName(const void* handler_symbol); size_t GetViewHandlerCount(); size_t GetAnonymousViewHandlerCount(); size_t GetProcedureHandlerCount(); +size_t GetHttpHandlerCount(); std::vector ConsumeBytes(BytesSource source); void SetMultiplePrimaryKeyError(const std::string& table_name); void SetConstraintRegistrationError(const std::string& code, const std::string& details); diff --git a/crates/bindings-cpp/include/spacetimedb/internal/tx_execution.h b/crates/bindings-cpp/include/spacetimedb/internal/tx_execution.h new file mode 100644 index 00000000000..4f0f8db2d0c --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/tx_execution.h @@ -0,0 +1,150 @@ +#ifndef SPACETIMEDB_INTERNAL_TX_EXECUTION_H +#define SPACETIMEDB_INTERNAL_TX_EXECUTION_H + +#include +#include +#include +#include +#include + +namespace SpacetimeDB::Internal { + +#ifdef SPACETIMEDB_UNSTABLE_FEATURES + +template +struct is_outcome : std::false_type {}; + +template +struct is_outcome> : std::true_type {}; + +template +inline constexpr bool is_outcome_v = is_outcome>>::value; + +template +bool tx_result_should_commit(const T& result) { + using ResultType = std::remove_cv_t>; + // TODO(http-handlers-cpp): Consider tightening try_with_tx in a future breaking release + // so rollback-aware callbacks use Outcome (and possibly bool for compatibility) + // instead of silently treating arbitrary return types as commit-on-success. + if constexpr (std::is_same_v) { + return result; + } else if constexpr (is_outcome_v) { + return result.is_ok(); + } else { + return true; + } +} + +class TxAbortGuard { +public: + TxAbortGuard() = default; + TxAbortGuard(const TxAbortGuard&) = delete; + TxAbortGuard& operator=(const TxAbortGuard&) = delete; + + ~TxAbortGuard() { + if (!armed_) { + return; + } + Status status = FFI::procedure_abort_mut_tx(); + if (is_error(status)) { + LOG_PANIC("Failed to abort transaction"); + } + } + + void disarm() { + armed_ = false; + } + +private: + bool armed_ = true; +}; + +inline void commit_tx_or_panic() { + Status status = FFI::procedure_commit_mut_tx(); + if (is_error(status)) { + LOG_PANIC("Failed to commit transaction"); + } +} + +inline bool try_commit_tx() { + return is_ok(FFI::procedure_commit_mut_tx()); +} + +inline void abort_tx_or_panic() { + Status status = FFI::procedure_abort_mut_tx(); + if (is_error(status)) { + LOG_PANIC("Failed to abort transaction"); + } +} + +template +auto run_tx_once(MakeReducerContext&& make_reducer_ctx, Func& body) -> decltype(body(std::declval())) { + using ResultType = decltype(body(std::declval())); + + int64_t tx_timestamp = 0; + Status status = FFI::procedure_start_mut_tx(&tx_timestamp); + if (is_error(status)) { + LOG_PANIC("Failed to start transaction"); + } + + TxAbortGuard abort_guard; + ReducerContext reducer_ctx = make_reducer_ctx(Timestamp::from_micros_since_epoch(tx_timestamp)); + TxContext tx{reducer_ctx}; + + if constexpr (std::is_void_v) { + body(tx); + abort_guard.disarm(); + } else { + ResultType result = body(tx); + abort_guard.disarm(); + return result; + } +} + +template +auto with_tx(MakeReducerContext&& make_reducer_ctx, Func& body) -> decltype(body(std::declval())) { + using ResultType = decltype(body(std::declval())); + + if constexpr (std::is_void_v) { + run_tx_once(std::forward(make_reducer_ctx), body); + if (!try_commit_tx()) { + run_tx_once(std::forward(make_reducer_ctx), body); + commit_tx_or_panic(); + } + } else { + ResultType result = run_tx_once(std::forward(make_reducer_ctx), body); + if (!try_commit_tx()) { + result = run_tx_once(std::forward(make_reducer_ctx), body); + commit_tx_or_panic(); + } + return result; + } +} + +template +auto try_with_tx(MakeReducerContext&& make_reducer_ctx, Func& body) -> decltype(body(std::declval())) { + using ResultType = decltype(body(std::declval())); + + ResultType result = run_tx_once(std::forward(make_reducer_ctx), body); + if (!tx_result_should_commit(result)) { + abort_tx_or_panic(); + return result; + } + + if (!try_commit_tx()) { + result = run_tx_once(std::forward(make_reducer_ctx), body); + if (tx_result_should_commit(result)) { + commit_tx_or_panic(); + } else { + abort_tx_or_panic(); + } + } + + return result; +} + +#endif + +} // namespace SpacetimeDB::Internal + +#endif // SPACETIMEDB_INTERNAL_TX_EXECUTION_H diff --git a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h index 9de0f0a2312..f746093c574 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h @@ -28,6 +28,8 @@ #include "autogen/RawViewDefV10.g.h" #include "autogen/RawScheduleDefV10.g.h" #include "autogen/RawLifeCycleReducerDefV10.g.h" +#include "autogen/RawHttpHandlerDefV10.g.h" +#include "autogen/RawHttpRouteDefV10.g.h" #include "autogen/RawColumnDefaultValueV10.g.h" #include "autogen/RawRowLevelSecurityDefV9.g.h" #include "autogen/RawTypeDefV10.g.h" @@ -39,6 +41,8 @@ namespace SpacetimeDB { +class Router; + void fail_reducer(std::string message); namespace Internal { @@ -584,6 +588,10 @@ class V10Builder { UpsertProcedure(procedure_def); } + void RegisterHttpHandlerDef(const std::string& handler_name); + void RegisterHttpRoute(const RawHttpRouteDefV10& route); + void RegisterHttpRouter(const ::SpacetimeDB::Router& router); + void RegisterSchedule(const std::string& table_name, uint16_t scheduled_at_column, const std::string& reducer_name) { if (g_circular_ref_error) { std::fprintf(stderr, "ERROR: Skipping schedule registration for table '%s' because circular reference error is set\n", @@ -628,6 +636,8 @@ class V10Builder { const std::vector& GetReducers() const { return reducers_; } const std::optional& GetCaseConversionPolicy() const { return case_conversion_policy_; } const std::vector& GetExplicitNames() const { return explicit_names_; } + const std::vector& GetHttpHandlers() const { return http_handlers_; } + const std::vector& GetHttpRoutes() const { return http_routes_; } private: std::vector::iterator FindTable(const std::string& table_name) { @@ -638,6 +648,7 @@ class V10Builder { void UpsertReducer(const RawReducerDefV10& reducer); void UpsertProcedure(const RawProcedureDefV10& procedure); void UpsertView(const RawViewDefV10& view); + void UpsertHttpHandler(const RawHttpHandlerDefV10& handler); RawIndexDefV10 CreateBTreeIndex(const std::string& table_name, const std::string& source_name, const std::vector& columns, @@ -656,6 +667,8 @@ class V10Builder { std::vector reducers_; std::vector procedures_; std::vector views_; + std::vector http_handlers_; + std::vector http_routes_; std::vector schedules_; std::vector lifecycle_reducers_; std::vector row_level_security_; diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index f9107d70251..ea189f8ef12 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -6,6 +6,7 @@ #include // For Uuid #include // For TxContext #include // For transaction syscalls +#include #include // For StdbRng #ifdef SPACETIMEDB_UNSTABLE_FEATURES #include // For HttpClient @@ -196,46 +197,14 @@ struct ProcedureContext { */ template auto with_tx(Func&& body) -> decltype(body(std::declval())) { - using ResultType = decltype(body(std::declval())); - - // Start transaction - int64_t tx_timestamp; - Status status = ::procedure_start_mut_tx(&tx_timestamp); - if (is_error(status)) { - LOG_PANIC("Failed to start transaction"); - } - - // Create a ReducerContext for this transaction - // Note: connection_id converted to std::optional - ReducerContext reducer_ctx( - sender(), - std::optional(connection_id), - Timestamp::from_micros_since_epoch(tx_timestamp) - ); - - // Create transaction context wrapping the reducer context - TxContext tx{reducer_ctx}; - - // Execute callback - if constexpr (std::is_void_v) { - body(tx); - - // Commit transaction - status = ::procedure_commit_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to commit transaction"); - } - } else { - ResultType result = body(tx); - - // Commit transaction - status = ::procedure_commit_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to commit transaction"); - } - - return result; - } + auto make_reducer_ctx = [this](Timestamp tx_timestamp) { + return ReducerContext( + sender(), + std::optional(connection_id), + tx_timestamp + ); + }; + return Internal::with_tx(make_reducer_ctx, body); } /** @@ -260,51 +229,14 @@ struct ProcedureContext { */ template auto try_with_tx(Func&& body) -> decltype(body(std::declval())) { - using ResultType = decltype(body(std::declval())); - - // Start transaction - int64_t tx_timestamp; - Status status = ::procedure_start_mut_tx(&tx_timestamp); - if (is_error(status)) { - LOG_PANIC("Failed to start transaction"); - } - - // Create a ReducerContext for this transaction - ReducerContext reducer_ctx( - sender(), - std::optional(connection_id), - Timestamp::from_micros_since_epoch(tx_timestamp) - ); - - // Create transaction context wrapping the reducer context - TxContext tx{reducer_ctx}; - - // Execute callback - ResultType result = body(tx); - - // For bool results, use the value to decide commit/rollback - // For other types, always commit (caller can use LOG_PANIC to abort) - if constexpr (std::is_same_v) { - if (result) { - status = ::procedure_commit_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to commit transaction"); - } - } else { - status = ::procedure_abort_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to rollback transaction"); - } - } - } else { - // For non-bool returns, always commit - status = ::procedure_commit_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to commit transaction"); - } - } - - return result; + auto make_reducer_ctx = [this](Timestamp tx_timestamp) { + return ReducerContext( + sender(), + std::optional(connection_id), + tx_timestamp + ); + }; + return Internal::try_with_tx(make_reducer_ctx, body); } #endif }; diff --git a/crates/bindings-cpp/include/spacetimedb/reducer_context.h b/crates/bindings-cpp/include/spacetimedb/reducer_context.h index 8c8fba26e72..41865b14f3a 100644 --- a/crates/bindings-cpp/include/spacetimedb/reducer_context.h +++ b/crates/bindings-cpp/include/spacetimedb/reducer_context.h @@ -124,6 +124,9 @@ struct ReducerContext { ReducerContext(Identity s, std::optional cid, Timestamp ts) : sender_(s), connection_id(cid), timestamp(ts), sender_auth_(AuthCtx::from_connection_id_opt(cid, s)) {} + + ReducerContext(Identity s, std::optional cid, Timestamp ts, AuthCtx auth) + : sender_(s), connection_id(cid), timestamp(ts), sender_auth_(std::move(auth)) {} }; } // namespace SpacetimeDB diff --git a/crates/bindings-cpp/include/spacetimedb/router.h b/crates/bindings-cpp/include/spacetimedb/router.h new file mode 100644 index 00000000000..9f7bde5b34a --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/router.h @@ -0,0 +1,186 @@ +#ifndef SPACETIMEDB_ROUTER_H +#define SPACETIMEDB_ROUTER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { + +class Router { +public: + struct RouteSpec { + Internal::MethodOrAny method; + std::string path; + std::string handler_name; + }; + + Router() = default; + + template + Router get(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::get()), std::move(path), handler); + } + + template + Router head(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::head()), std::move(path), handler); + } + + template + Router options(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::options()), std::move(path), handler); + } + + template + Router put(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::put()), std::move(path), handler); + } + + template + Router delete_(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::del()), std::move(path), handler); + } + + template + Router post(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::post()), std::move(path), handler); + } + + template + Router patch(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::patch()), std::move(path), handler); + } + + template + Router any(std::string path, Func handler) const { + return add_route(make_any(), std::move(path), handler); + } + + Router nest(std::string path, const Router& sub_router) const { + assert_valid_path(path); + Router merged = *this; + for (const auto& route : routes_) { + if (route.path.starts_with(path)) { + fail_router_registration("Cannot nest router at `" + path + "`; existing routes overlap with nested path"); + } + } + for (const auto& route : sub_router.routes_) { + merged = merged.add_route(route.method, join_paths(path, route.path), route.handler_name); + } + return merged; + } + + Router merge(const Router& other) const { + Router merged = *this; + for (const auto& route : other.routes_) { + merged = merged.add_route(route.method, route.path, route.handler_name); + } + return merged; + } + + const std::vector& routes() const { + return routes_; + } + +private: + std::vector routes_; + + [[noreturn]] static void fail_router_registration(const std::string& message) { + std::fprintf(stderr, "Router registration failed: %s\n", message.c_str()); + std::abort(); + } + + static bool character_is_acceptable_for_route_path(char c) { + return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '~' || c == '/'; + } + + static void assert_valid_path(const std::string& path) { + if (!path.empty() && path[0] != '/') { + fail_router_registration("Route paths must start with `/`: " + path); + } + for (char c : path) { + if (!character_is_acceptable_for_route_path(c)) { + fail_router_registration("Route paths may contain only ASCII lowercase letters, digits and `-_~/`: " + path); + } + } + } + + static std::string join_paths(const std::string& prefix, const std::string& suffix) { + if (prefix == "/") { + return suffix; + } + if (suffix == "/") { + return prefix; + } + std::string trimmed_prefix = prefix; + while (!trimmed_prefix.empty() && trimmed_prefix.back() == '/') { + trimmed_prefix.pop_back(); + } + size_t start = 0; + while (start < suffix.size() && suffix[start] == '/') { + ++start; + } + return trimmed_prefix + "/" + suffix.substr(start); + } + + static bool routes_overlap(const RouteSpec& a, const RouteSpec& b) { + if (a.path != b.path) { + return false; + } + if (a.method.is<0>() || b.method.is<0>()) { + return true; + } + const auto& a_method = a.method.template get<1>().value; + const auto& b_method = b.method.template get<1>().value; + return a_method == b_method; + } + + static Internal::MethodOrAny make_any() { + Internal::MethodOrAny method; + method.set<0>(std::monostate{}); + return method; + } + + static Internal::MethodOrAny make_method(const HttpMethod& method) { + Internal::MethodOrAny result; + Internal::MethodOrAny_Method_Wrapper wrapper; + wrapper.value = convert::to_wire(method); + result.set<1>(wrapper); + return result; + } + + template + Router add_route(Internal::MethodOrAny method, std::string path, Func handler) const { + return add_route(std::move(method), std::move(path), resolve_handler_name(handler)); + } + + Router add_route(Internal::MethodOrAny method, std::string path, std::string handler_name) const { + assert_valid_path(path); + RouteSpec candidate{method, path, std::move(handler_name)}; + for (const auto& route : routes_) { + if (routes_overlap(route, candidate)) { + fail_router_registration("Route conflict for `" + candidate.path + "`"); + } + } + Router next = *this; + next.routes_.push_back(std::move(candidate)); + return next; + } + + template + static std::string resolve_handler_name(Func handler) { + const void* symbol = reinterpret_cast(handler); + return Internal::LookupHttpHandlerName(symbol); + } +}; + +} // namespace SpacetimeDB + +#endif // SPACETIMEDB_ROUTER_H diff --git a/crates/bindings-cpp/src/abi/module_exports.cpp b/crates/bindings-cpp/src/abi/module_exports.cpp index 6156e32025a..473dc9e2fb1 100644 --- a/crates/bindings-cpp/src/abi/module_exports.cpp +++ b/crates/bindings-cpp/src/abi/module_exports.cpp @@ -99,4 +99,19 @@ extern "C" { ); } + STDB_EXPORT(__call_http_handler__) + int16_t __call_http_handler__( + uint32_t id, + uint64_t timestamp_microseconds, + SpacetimeDB::BytesSource request_source, + SpacetimeDB::BytesSink result_sink + ) { + return SpacetimeDB::Internal::Module::__call_http_handler__( + id, + timestamp_microseconds, + request_source, + result_sink + ); + } + } // extern "C" diff --git a/crates/bindings-cpp/src/internal/Module.cpp b/crates/bindings-cpp/src/internal/Module.cpp index b0dcb1ceae3..ebb73313160 100644 --- a/crates/bindings-cpp/src/internal/Module.cpp +++ b/crates/bindings-cpp/src/internal/Module.cpp @@ -16,6 +16,11 @@ #include "spacetimedb/reducer_error.h" #include "spacetimedb/view_context.h" #include "spacetimedb/procedure_context.h" +#include "spacetimedb/handler_context.h" +#include "spacetimedb/http_convert.h" +#include "spacetimedb/http_wire.h" +#include +#include #include #include #include @@ -55,6 +60,13 @@ namespace Internal { std::function(ProcedureContext&, BytesSource)> handler; }; static std::vector g_procedure_handlers; + + struct HttpHandler { + std::string name; + const void* symbol; + std::function handler; + }; + static std::vector g_http_handlers; /** * @brief View result header for serializing view return values @@ -116,6 +128,23 @@ namespace Internal { std::function(ProcedureContext&, BytesSource)> handler) { g_procedure_handlers.push_back({name, handler}); } + + void RegisterHttpHandlerHandler(const std::string& name, + const void* handler_symbol, + std::function handler) { + g_http_handlers.push_back({name, handler_symbol, handler}); + } + + std::string LookupHttpHandlerName(const void* handler_symbol) { + auto it = std::find_if(g_http_handlers.begin(), g_http_handlers.end(), [&](const auto& existing) { + return existing.symbol == handler_symbol; + }); + if (it == g_http_handlers.end()) { + fprintf(stderr, "ERROR: HTTP handler must be registered before it is referenced by a router\n"); + std::abort(); + } + return it->name; + } // Get the number of registered view handlers size_t GetViewHandlerCount() { @@ -130,6 +159,10 @@ namespace Internal { size_t GetProcedureHandlerCount() { return g_procedure_handlers.size(); } + + size_t GetHttpHandlerCount() { + return g_http_handlers.size(); + } void SetTableIsEventFlag(const std::string& table_name, bool is_event) { getV10Builder().SetTableIsEventFlag(table_name, is_event); @@ -146,6 +179,7 @@ namespace Internal { g_view_handlers.clear(); // Clear view handlers g_view_anon_handlers.clear(); // Clear anonymous view handlers g_procedure_handlers.clear(); // Clear procedure handlers + g_http_handlers.clear(); // Clear http handlers g_multiple_primary_key_error = false; // Reset error flag g_multiple_primary_key_table_name = ""; // Reset error table name g_constraint_registration_error = false; @@ -630,6 +664,38 @@ int16_t Module::__call_procedure__( return 0; // Success (StatusCode::OK) } +int16_t Module::__call_http_handler__( + uint32_t id, + uint64_t timestamp_microseconds, + BytesSource request_source, + BytesSink result_sink +) { + if (id >= g_http_handlers.size()) { + fprintf(stderr, "ERROR: Invalid http handler ID %u (have %zu handlers)\n", + id, g_http_handlers.size()); + return -1; + } + + Timestamp timestamp = Timestamp::from_micros_since_epoch(static_cast(timestamp_microseconds)); + HandlerContext ctx(timestamp); + + std::vector request_bytes = ConsumeBytes(request_source); + bsatn::Reader request_reader(request_bytes.data(), request_bytes.size()); + wire::RequestAndBody wire_request = bsatn::deserialize(request_reader); + HttpRequest request = convert::from_wire(wire_request); + + HttpResponse response = g_http_handlers[id].handler(ctx, std::move(request)); + wire::ResponseAndBody wire_response = convert::to_wire_with_body(response); + + std::vector result_data; + { + bsatn::Writer writer(result_data); + bsatn::serialize(writer, wire_response); + } + WriteBytes(result_sink, result_data); + return 0; +} + void Module::SetCaseConversionPolicy(CaseConversionPolicy policy) { getV10Builder().SetCaseConversionPolicy(policy); } diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index 65f1895b035..eb22114e8b9 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -7,6 +7,7 @@ #include "spacetimedb/internal/autogen/RawScopedTypeNameV10.g.h" #include "spacetimedb/internal/autogen/FunctionVisibility.g.h" #include "spacetimedb/internal/autogen/ExplicitNames.g.h" +#include "spacetimedb/router.h" #include #include @@ -37,6 +38,8 @@ void V10Builder::Clear() { reducers_.clear(); procedures_.clear(); views_.clear(); + http_handlers_.clear(); + http_routes_.clear(); schedules_.clear(); lifecycle_reducers_.clear(); row_level_security_.clear(); @@ -150,6 +153,31 @@ void V10Builder::UpsertView(const RawViewDefV10& view) { } } +void V10Builder::UpsertHttpHandler(const RawHttpHandlerDefV10& handler) { + auto it = std::find_if(http_handlers_.begin(), http_handlers_.end(), [&](const auto& existing) { + return existing.source_name == handler.source_name; + }); + if (it == http_handlers_.end()) { + http_handlers_.push_back(handler); + } else { + *it = handler; + } +} + +void V10Builder::RegisterHttpHandlerDef(const std::string& handler_name) { + UpsertHttpHandler(RawHttpHandlerDefV10{handler_name}); +} + +void V10Builder::RegisterHttpRoute(const RawHttpRouteDefV10& route) { + http_routes_.push_back(route); +} + +void V10Builder::RegisterHttpRouter(const ::SpacetimeDB::Router& router) { + for (const auto& route : router.routes()) { + RegisterHttpRoute(RawHttpRouteDefV10{route.handler_name, route.method, route.path}); + } +} + RawIndexDefV10 V10Builder::CreateBTreeIndex(const std::string& table_name, const std::string& source_name, const std::vector& columns, @@ -256,6 +284,16 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { section_explicit_names.set<10>(ExplicitNames{explicit_names_}); v10_module.sections.push_back(std::move(section_explicit_names)); } + if (!http_handlers_.empty()) { + RawModuleDefV10Section section_http_handlers; + section_http_handlers.set<11>(http_handlers_); + v10_module.sections.push_back(std::move(section_http_handlers)); + } + if (!http_routes_.empty()) { + RawModuleDefV10Section section_http_routes; + section_http_routes.set<12>(http_routes_); + v10_module.sections.push_back(std::move(section_http_routes)); + } if (!row_level_security_.empty()) { RawModuleDefV10Section section_rls; section_rls.set<8>(row_level_security_); diff --git a/crates/bindings-cpp/tests/compile/CMakeLists.module.txt b/crates/bindings-cpp/tests/compile/CMakeLists.module.txt new file mode 100644 index 00000000000..5c264b8f0cb --- /dev/null +++ b/crates/bindings-cpp/tests/compile/CMakeLists.module.txt @@ -0,0 +1,48 @@ +cmake_minimum_required(VERSION 3.16) +project(module) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT DEFINED MODULE_SOURCE) + message(FATAL_ERROR "MODULE_SOURCE must be defined") +endif() + +if(NOT DEFINED OUTPUT_NAME) + set(OUTPUT_NAME "module") +endif() + +if(NOT DEFINED SPACETIMEDB_LIBRARY_DIR) + message(FATAL_ERROR "SPACETIMEDB_LIBRARY_DIR must be defined") +endif() + +if(NOT DEFINED SPACETIMEDB_INCLUDE_DIR) + message(FATAL_ERROR "SPACETIMEDB_INCLUDE_DIR must be defined") +endif() + +add_executable(${OUTPUT_NAME} ${MODULE_SOURCE}) +target_include_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_INCLUDE_DIR}) +target_link_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_LIBRARY_DIR}) +target_link_libraries(${OUTPUT_NAME} PRIVATE spacetimedb_cpp_library) + +if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + set(EXPORTED_FUNCS "['_malloc','_free','___describe_module__','___call_reducer__','___call_http_handler__']") + + target_link_options(${OUTPUT_NAME} PRIVATE + "SHELL:-sSTANDALONE_WASM=1" + "SHELL:-sWASM=1" + "SHELL:--no-entry" + "SHELL:-sEXPORTED_FUNCTIONS=${EXPORTED_FUNCS}" + "SHELL:-sERROR_ON_UNDEFINED_SYMBOLS=1" + "SHELL:-sFILESYSTEM=0" + "SHELL:-sDISABLE_EXCEPTION_CATCHING=1" + "SHELL:-sALLOW_MEMORY_GROWTH=0" + "SHELL:-sINITIAL_MEMORY=16MB" + "SHELL:-sSUPPORT_LONGJMP=0" + "SHELL:-sSUPPORT_ERRNO=0" + "SHELL:-std=c++20" + "SHELL:-O2" + ) + + set_target_properties(${OUTPUT_NAME} PROPERTIES OUTPUT_NAME "lib" SUFFIX ".wasm") +endif() diff --git a/crates/bindings-cpp/tests/compile/README.md b/crates/bindings-cpp/tests/compile/README.md new file mode 100644 index 00000000000..34c011b0931 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/README.md @@ -0,0 +1,64 @@ +# SpacetimeDB C++ Compile Tests + +Focused compile-surface regression tests for the C++ bindings. + +This harness is intended for: +- authoring-time success cases +- compile-fail regression cases +- API surface checks that should fail before publish/runtime + +## HTTP Handler Coverage + +The `http-handlers` suite mirrors the Rust coverage in +`crates/bindings/tests/ui/http_handlers.rs` as closely as the C++ macro surface allows. + +Covered cases: +- valid handler/router authoring +- no handler args +- immutable handler context +- wrong handler context type +- missing request arg +- wrong request arg type +- missing return +- wrong return type +- forbidden `HandlerContext::sender()` +- forbidden `HandlerContext::connection_id` +- forbidden `HandlerContext::db` +- router authored with args +- router wrong return type +- router misuse in a non-function position + +## Run + +From Git Bash or Linux-style shells: + +```bash +./crates/bindings-cpp/tests/compile/run-compile-tests.sh --suite http-handlers +``` + +From PowerShell at the repo root: + +```powershell +.\crates\bindings-cpp\tests\compile\run-compile-tests.ps1 -Suite http-handlers +``` + +Or from the compile test directory: + +```powershell +.\run-compile-tests.ps1 -Suite http-handlers +``` + +## Output + +Build artifacts and logs are written under: + +```text +crates/bindings-cpp/tests/compile/build/ +``` + +Each case gets: +- `build//configure.log` +- `build//build.log` + +The shared bindings library build is under: +- `build/library/` diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp new file mode 100644 index 00000000000..1832f28496a --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp @@ -0,0 +1,14 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_immutable_ctx, const HandlerContext& ctx, HttpRequest request) { + (void)ctx; + (void)request; + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_args.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_args.cpp new file mode 100644 index 00000000000..7391a55450f --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_args.cpp @@ -0,0 +1,12 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_no_args) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp new file mode 100644 index 00000000000..cbebae53852 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp @@ -0,0 +1,14 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_no_connection_id, HandlerContext ctx, HttpRequest request) { + (void)request; + auto conn_id = ctx.connection_id(); + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string(conn_id.to_hex_string()), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp new file mode 100644 index 00000000000..707b2183be0 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp @@ -0,0 +1,20 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +struct TestRow { + uint32_t value; +}; +SPACETIMEDB_STRUCT(TestRow, value) +SPACETIMEDB_TABLE(TestRow, test_row, Public) + +SPACETIMEDB_HTTP_HANDLER(handler_no_db, HandlerContext ctx, HttpRequest request) { + (void)request; + auto count = ctx.db[test_row].count(); + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string(std::to_string(count)), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp new file mode 100644 index 00000000000..ac2eebad2d2 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp @@ -0,0 +1,13 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_no_request_arg, HandlerContext ctx) { + (void)ctx; + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp new file mode 100644 index 00000000000..cbc5cd4f29f --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp @@ -0,0 +1,12 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +#if defined(__clang__) +#pragma clang diagnostic error "-Wreturn-type" +#endif + +SPACETIMEDB_HTTP_HANDLER(handler_no_return_type, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp new file mode 100644 index 00000000000..6d84d5e4af1 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp @@ -0,0 +1,14 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_no_sender, HandlerContext ctx, HttpRequest request) { + (void)request; + auto sender = ctx.sender(); + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string(sender.to_hex_string()), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp new file mode 100644 index 00000000000..f2637cfc53c --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp @@ -0,0 +1,14 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_wrong_ctx, ProcedureContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp new file mode 100644 index 00000000000..5c06d017dba --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp @@ -0,0 +1,14 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_wrong_request_arg_type, HandlerContext ctx, uint32_t request) { + (void)ctx; + (void)request; + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp new file mode 100644 index 00000000000..ca22aedef90 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp @@ -0,0 +1,9 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_wrong_return_type, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return 7u; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_not_a_function.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_not_a_function.cpp new file mode 100644 index 00000000000..a18cb84cd0f --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_not_a_function.cpp @@ -0,0 +1,5 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_ROUTER(register_http_routes) = Router(); diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp new file mode 100644 index 00000000000..173b0b7deca --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp @@ -0,0 +1,19 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(hello_handler, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} + +SPACETIMEDB_HTTP_ROUTER(register_http_routes, HandlerContext ctx) { + (void)ctx; + return Router().get("/hello", hello_handler); +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_wrong_return_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_wrong_return_type.cpp new file mode 100644 index 00000000000..fd20453bc4c --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_wrong_return_type.cpp @@ -0,0 +1,7 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_ROUTER(register_http_routes) { + return 7u; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp new file mode 100644 index 00000000000..2bcdd546426 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp @@ -0,0 +1,30 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(hello_handler, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} + +SPACETIMEDB_HTTP_ROUTER(register_http_routes) { + Router nested = Router() + .get("/nested", hello_handler); + + Router merged = Router() + .get("", hello_handler) + .head("/health", hello_handler); + + return Router() + .get("/hello", hello_handler) + .delete_("/delete", hello_handler) + .any("/", hello_handler) + .merge(merged) + .nest("/api", nested); +} diff --git a/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 b/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 new file mode 100644 index 00000000000..889c44787c4 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 @@ -0,0 +1,208 @@ +[CmdletBinding()] +param( + [ValidateSet("http-handlers")] + [string]$Suite = "http-handlers" +) + +$ErrorActionPreference = "Stop" + +function Find-Emcmake { + $candidates = @( + (Get-Command emcmake.bat -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Source -First 1), + (Get-Command emcmake -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Source -First 1) + ) | Where-Object { $_ } + + if ($candidates.Count -eq 0) { + throw "Unable to locate emcmake or emcmake.bat." + } + + return $candidates[0] +} + +function Invoke-LoggedCommand { + param( + [Parameter(Mandatory = $true)] + [string]$FilePath, + [Parameter(Mandatory = $true)] + [string[]]$Arguments, + [Parameter(Mandatory = $true)] + [string]$LogPath, + [string]$WorkingDirectory + ) + + if ($WorkingDirectory) { + Push-Location $WorkingDirectory + } + + try { + & $FilePath @Arguments *> $LogPath + return $LASTEXITCODE + } finally { + if ($WorkingDirectory) { + Pop-Location + } + } +} + +function New-CompileCase { + param( + [string]$Name, + [string]$RelativePath, + [ValidateSet("success", "failure")] + [string]$Expectation, + [string]$Marker = "" + ) + + return [pscustomobject]@{ + Name = $Name + RelativePath = $RelativePath + Expectation = $Expectation + Marker = $Marker + } +} + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$bindingsRoot = Split-Path -Parent (Split-Path -Parent $scriptDir) +$repoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $bindingsRoot)) +$includeDir = Join-Path $bindingsRoot "include" +$buildRoot = Join-Path $scriptDir "build" +$libraryBuildDir = Join-Path $buildRoot "library" +$libraryLogDir = Join-Path $buildRoot "logs" +$templatePath = Join-Path $scriptDir "CMakeLists.module.txt" +$emcmake = Find-Emcmake + +$cases = switch ($Suite) { + "http-handlers" { + @( + (New-CompileCase "ok_http_handlers_basic" "cases/http-handlers/ok_http_handlers_basic.cpp" "success") + (New-CompileCase "error_http_handler_no_args" "cases/http-handlers/error_http_handler_no_args.cpp" "failure" "too few arguments provided to function-like macro invocation") + (New-CompileCase "error_http_handler_immutable_ctx" "cases/http-handlers/error_http_handler_immutable_ctx.cpp" "failure" "First parameter of HTTP handler must be HandlerContext") + (New-CompileCase "error_http_handler_wrong_ctx" "cases/http-handlers/error_http_handler_wrong_ctx.cpp" "failure" "First parameter of HTTP handler must be HandlerContext") + (New-CompileCase "error_http_handler_no_request_arg" "cases/http-handlers/error_http_handler_no_request_arg.cpp" "failure" "too few arguments provided to function-like macro invocation") + (New-CompileCase "error_http_handler_wrong_request_arg_type" "cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp" "failure" "Second parameter of HTTP handler must be HttpRequest") + (New-CompileCase "error_http_handler_no_return_type" "cases/http-handlers/error_http_handler_no_return_type.cpp" "failure" "non-void function does not return a value") + (New-CompileCase "error_http_handler_wrong_return_type" "cases/http-handlers/error_http_handler_wrong_return_type.cpp" "failure" "no viable conversion from returned value of type 'unsigned int' to function return type 'SpacetimeDB::HttpResponse'") + (New-CompileCase "error_http_handler_no_sender" "cases/http-handlers/error_http_handler_no_sender.cpp" "failure" "no member named 'sender' in 'SpacetimeDB::HandlerContext'") + (New-CompileCase "error_http_handler_no_connection_id" "cases/http-handlers/error_http_handler_no_connection_id.cpp" "failure" "no member named 'connection_id' in 'SpacetimeDB::HandlerContext'") + (New-CompileCase "error_http_handler_no_db" "cases/http-handlers/error_http_handler_no_db.cpp" "failure" "no member named 'db' in 'SpacetimeDB::HandlerContext'") + (New-CompileCase "error_http_router_not_a_function" "cases/http-handlers/error_http_router_not_a_function.cpp" "failure" "illegal initializer") + (New-CompileCase "error_http_router_with_args" "cases/http-handlers/error_http_router_with_args.cpp" "failure" "too many arguments provided to function-like macro invocation") + (New-CompileCase "error_http_router_wrong_return_type" "cases/http-handlers/error_http_router_wrong_return_type.cpp" "failure" "no viable conversion from returned value of type 'unsigned int' to function return type 'SpacetimeDB::Router'") + ) + } +} + +New-Item -ItemType Directory -Force -Path $buildRoot | Out-Null +New-Item -ItemType Directory -Force -Path $libraryLogDir | Out-Null + +$libraryConfigureLog = Join-Path $libraryLogDir "library-configure.log" +$libraryBuildLog = Join-Path $libraryLogDir "library-build.log" + +Write-Host "Building bindings library..." +$configureExit = Invoke-LoggedCommand -FilePath $emcmake -Arguments @( + "cmake", + "-S", $bindingsRoot, + "-B", $libraryBuildDir +) -LogPath $libraryConfigureLog -WorkingDirectory $scriptDir + +if ($configureExit -ne 0) { + Write-Host "Library configure failed. See $libraryConfigureLog" + exit 1 +} + +$buildExit = Invoke-LoggedCommand -FilePath "cmake" -Arguments @( + "--build", $libraryBuildDir +) -LogPath $libraryBuildLog -WorkingDirectory $scriptDir + +if ($buildExit -ne 0) { + Write-Host "Library build failed. See $libraryBuildLog" + exit 1 +} + +$results = @() + +foreach ($case in $cases) { + $caseSource = Join-Path $scriptDir $case.RelativePath + $caseBuildDir = Join-Path $buildRoot $case.Name + $configureLog = Join-Path $caseBuildDir "configure.log" + $buildLog = Join-Path $caseBuildDir "build.log" + + if (Test-Path $caseBuildDir) { + Remove-Item $caseBuildDir -Recurse -Force + } + + New-Item -ItemType Directory -Force -Path $caseBuildDir | Out-Null + Copy-Item $templatePath (Join-Path $caseBuildDir "CMakeLists.txt") + + Write-Host "Running $($case.Name)..." + $configureExit = Invoke-LoggedCommand -FilePath $emcmake -Arguments @( + "cmake", + "-S", $caseBuildDir, + "-B", $caseBuildDir, + "-DMODULE_SOURCE=$caseSource", + "-DOUTPUT_NAME=$($case.Name)", + "-DSPACETIMEDB_LIBRARY_DIR=$libraryBuildDir", + "-DSPACETIMEDB_INCLUDE_DIR=$includeDir" + ) -LogPath $configureLog -WorkingDirectory $scriptDir + + $buildExit = 0 + if ($configureExit -eq 0) { + $buildExit = Invoke-LoggedCommand -FilePath "cmake" -Arguments @( + "--build", $caseBuildDir + ) -LogPath $buildLog -WorkingDirectory $scriptDir + } + + $combinedLog = "" + if (Test-Path $configureLog) { + $combinedLog += Get-Content $configureLog -Raw + } + if (Test-Path $buildLog) { + $combinedLog += "`n" + $combinedLog += Get-Content $buildLog -Raw + } + + $passed = $false + $detail = "" + if ($case.Expectation -eq "success") { + $passed = ($configureExit -eq 0 -and $buildExit -eq 0) + if (-not $passed) { + $detail = "Expected build success." + } + } else { + $failedBuild = ($configureExit -ne 0 -or $buildExit -ne 0) + $matchedMarker = ($case.Marker -and $combinedLog.Contains($case.Marker)) + $passed = ($failedBuild -and $matchedMarker) + if (-not $passed) { + if (-not $failedBuild) { + $detail = "Expected build failure." + } else { + $detail = "Expected marker not found: $($case.Marker)" + } + } + } + + if (-not $passed -and -not $detail) { + $detail = (($combinedLog -split "`r?`n" | Where-Object { $_.Trim() }) | Select-Object -First 8) -join " " + } + + $results += [pscustomobject]@{ + Case = $case.Name + Expectation = $case.Expectation + Result = if ($passed) { "PASS" } else { "FAIL" } + Detail = $detail + } +} + +$results | Format-Table -AutoSize + +if ($results.Result -contains "FAIL") { + Write-Host "" + Write-Host "Failures:" + $results | Where-Object Result -eq "FAIL" | ForEach-Object { + Write-Host "- $($_.Case): $($_.Detail)" + } + exit 1 +} + +Write-Host "" +Write-Host "All compile tests passed." diff --git a/crates/bindings-cpp/tests/compile/run-compile-tests.sh b/crates/bindings-cpp/tests/compile/run-compile-tests.sh new file mode 100644 index 00000000000..d7c634bc9d7 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/run-compile-tests.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BINDINGS_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +INCLUDE_DIR="$BINDINGS_ROOT/include" +BUILD_ROOT="$SCRIPT_DIR/build" +LIBRARY_BUILD_DIR="$BUILD_ROOT/library" +LIBRARY_LOG_DIR="$BUILD_ROOT/logs" +TEMPLATE_PATH="$SCRIPT_DIR/CMakeLists.module.txt" + +SUITE="http-handlers" + +while [[ $# -gt 0 ]]; do + case "$1" in + --suite) + SUITE="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +if [[ "$SUITE" != "http-handlers" ]]; then + echo "Unsupported suite: $SUITE" >&2 + exit 1 +fi + +if command -v emcmake >/dev/null 2>&1; then + EMCMAKE_CMD="emcmake" +elif command -v emcmake.bat >/dev/null 2>&1; then + EMCMAKE_CMD="emcmake.bat" +else + echo "Unable to locate emcmake or emcmake.bat" >&2 + exit 1 +fi + +mkdir -p "$BUILD_ROOT" "$LIBRARY_LOG_DIR" + +LIBRARY_CONFIGURE_LOG="$LIBRARY_LOG_DIR/library-configure.log" +LIBRARY_BUILD_LOG="$LIBRARY_LOG_DIR/library-build.log" + +echo "Building bindings library..." +if ! "$EMCMAKE_CMD" cmake -S "$BINDINGS_ROOT" -B "$LIBRARY_BUILD_DIR" >"$LIBRARY_CONFIGURE_LOG" 2>&1; then + echo "Library configure failed. See $LIBRARY_CONFIGURE_LOG" >&2 + exit 1 +fi + +if ! cmake --build "$LIBRARY_BUILD_DIR" >"$LIBRARY_BUILD_LOG" 2>&1; then + echo "Library build failed. See $LIBRARY_BUILD_LOG" >&2 + exit 1 +fi + +declare -a CASE_NAMES=( + "ok_http_handlers_basic" + "error_http_handler_no_args" + "error_http_handler_immutable_ctx" + "error_http_handler_wrong_ctx" + "error_http_handler_no_request_arg" + "error_http_handler_wrong_request_arg_type" + "error_http_handler_no_return_type" + "error_http_handler_wrong_return_type" + "error_http_handler_no_sender" + "error_http_handler_no_connection_id" + "error_http_handler_no_db" + "error_http_router_not_a_function" + "error_http_router_with_args" + "error_http_router_wrong_return_type" +) + +declare -A CASE_EXPECTATION +declare -A CASE_MARKER +declare -A CASE_SOURCE + +CASE_EXPECTATION["ok_http_handlers_basic"]="success" +CASE_SOURCE["ok_http_handlers_basic"]="$SCRIPT_DIR/cases/http-handlers/ok_http_handlers_basic.cpp" + +CASE_EXPECTATION["error_http_handler_no_args"]="failure" +CASE_MARKER["error_http_handler_no_args"]="too few arguments provided to function-like macro invocation" +CASE_SOURCE["error_http_handler_no_args"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_args.cpp" + +CASE_EXPECTATION["error_http_handler_immutable_ctx"]="failure" +CASE_MARKER["error_http_handler_immutable_ctx"]="First parameter of HTTP handler must be HandlerContext" +CASE_SOURCE["error_http_handler_immutable_ctx"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_immutable_ctx.cpp" + +CASE_EXPECTATION["error_http_handler_wrong_ctx"]="failure" +CASE_MARKER["error_http_handler_wrong_ctx"]="First parameter of HTTP handler must be HandlerContext" +CASE_SOURCE["error_http_handler_wrong_ctx"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_wrong_ctx.cpp" + +CASE_EXPECTATION["error_http_handler_no_request_arg"]="failure" +CASE_MARKER["error_http_handler_no_request_arg"]="too few arguments provided to function-like macro invocation" +CASE_SOURCE["error_http_handler_no_request_arg"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_request_arg.cpp" + +CASE_EXPECTATION["error_http_handler_wrong_request_arg_type"]="failure" +CASE_MARKER["error_http_handler_wrong_request_arg_type"]="Second parameter of HTTP handler must be HttpRequest" +CASE_SOURCE["error_http_handler_wrong_request_arg_type"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp" + +CASE_EXPECTATION["error_http_handler_no_return_type"]="failure" +CASE_MARKER["error_http_handler_no_return_type"]="non-void function does not return a value" +CASE_SOURCE["error_http_handler_no_return_type"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_return_type.cpp" + +CASE_EXPECTATION["error_http_handler_wrong_return_type"]="failure" +CASE_MARKER["error_http_handler_wrong_return_type"]="no viable conversion from returned value of type 'unsigned int' to function return type 'SpacetimeDB::HttpResponse'" +CASE_SOURCE["error_http_handler_wrong_return_type"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_wrong_return_type.cpp" + +CASE_EXPECTATION["error_http_handler_no_sender"]="failure" +CASE_MARKER["error_http_handler_no_sender"]="no member named 'sender' in 'SpacetimeDB::HandlerContext'" +CASE_SOURCE["error_http_handler_no_sender"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_sender.cpp" + +CASE_EXPECTATION["error_http_handler_no_connection_id"]="failure" +CASE_MARKER["error_http_handler_no_connection_id"]="no member named 'connection_id' in 'SpacetimeDB::HandlerContext'" +CASE_SOURCE["error_http_handler_no_connection_id"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_connection_id.cpp" + +CASE_EXPECTATION["error_http_handler_no_db"]="failure" +CASE_MARKER["error_http_handler_no_db"]="no member named 'db' in 'SpacetimeDB::HandlerContext'" +CASE_SOURCE["error_http_handler_no_db"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_db.cpp" + +CASE_EXPECTATION["error_http_router_not_a_function"]="failure" +CASE_MARKER["error_http_router_not_a_function"]="illegal initializer" +CASE_SOURCE["error_http_router_not_a_function"]="$SCRIPT_DIR/cases/http-handlers/error_http_router_not_a_function.cpp" + +CASE_EXPECTATION["error_http_router_with_args"]="failure" +CASE_MARKER["error_http_router_with_args"]="too many arguments provided to function-like macro invocation" +CASE_SOURCE["error_http_router_with_args"]="$SCRIPT_DIR/cases/http-handlers/error_http_router_with_args.cpp" + +CASE_EXPECTATION["error_http_router_wrong_return_type"]="failure" +CASE_MARKER["error_http_router_wrong_return_type"]="no viable conversion from returned value of type 'unsigned int' to function return type 'SpacetimeDB::Router'" +CASE_SOURCE["error_http_router_wrong_return_type"]="$SCRIPT_DIR/cases/http-handlers/error_http_router_wrong_return_type.cpp" + +FAILURES=0 + +for CASE_NAME in "${CASE_NAMES[@]}"; do + CASE_BUILD_DIR="$BUILD_ROOT/$CASE_NAME" + CONFIGURE_LOG="$CASE_BUILD_DIR/configure.log" + BUILD_LOG="$CASE_BUILD_DIR/build.log" + + rm -rf "$CASE_BUILD_DIR" + mkdir -p "$CASE_BUILD_DIR" + cp "$TEMPLATE_PATH" "$CASE_BUILD_DIR/CMakeLists.txt" + + echo "Running $CASE_NAME..." + + CONFIGURE_EXIT=0 + BUILD_EXIT=0 + + if "$EMCMAKE_CMD" cmake -S "$CASE_BUILD_DIR" -B "$CASE_BUILD_DIR" \ + -DMODULE_SOURCE="${CASE_SOURCE[$CASE_NAME]}" \ + -DOUTPUT_NAME="$CASE_NAME" \ + -DSPACETIMEDB_LIBRARY_DIR="$LIBRARY_BUILD_DIR" \ + -DSPACETIMEDB_INCLUDE_DIR="$INCLUDE_DIR" >"$CONFIGURE_LOG" 2>&1; then + CONFIGURE_EXIT=0 + else + CONFIGURE_EXIT=$? + fi + + if [[ $CONFIGURE_EXIT -eq 0 ]]; then + if cmake --build "$CASE_BUILD_DIR" >"$BUILD_LOG" 2>&1; then + BUILD_EXIT=0 + else + BUILD_EXIT=$? + fi + fi + + COMBINED_LOG="" + [[ -f "$CONFIGURE_LOG" ]] && COMBINED_LOG+="$(cat "$CONFIGURE_LOG")"$'\n' + [[ -f "$BUILD_LOG" ]] && COMBINED_LOG+="$(cat "$BUILD_LOG")" + + PASS=0 + DETAIL="" + + if [[ "${CASE_EXPECTATION[$CASE_NAME]}" == "success" ]]; then + if [[ $CONFIGURE_EXIT -eq 0 && $BUILD_EXIT -eq 0 ]]; then + PASS=1 + else + DETAIL="Expected build success." + fi + else + if [[ $CONFIGURE_EXIT -ne 0 || $BUILD_EXIT -ne 0 ]]; then + if [[ "$COMBINED_LOG" == *"${CASE_MARKER[$CASE_NAME]}"* ]]; then + PASS=1 + else + DETAIL="Expected marker not found: ${CASE_MARKER[$CASE_NAME]}" + fi + else + DETAIL="Expected build failure." + fi + fi + + if [[ $PASS -eq 1 ]]; then + printf '%-40s PASS\n' "$CASE_NAME" + else + printf '%-40s FAIL\n' "$CASE_NAME" + [[ -z "$DETAIL" ]] && DETAIL="$(printf '%s' "$COMBINED_LOG" | grep -v '^[[:space:]]*$' | head -n 8 | tr '\n' ' ')" + echo " $DETAIL" + FAILURES=1 + fi +done + +if [[ $FAILURES -ne 0 ]]; then + echo + echo "Compile test failures detected." + exit 1 +fi + +echo +echo "All compile tests passed." From 088c7ba070a137ef22dd684b06ecd689bb2ae2ab Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Mon, 11 May 2026 14:58:18 -0700 Subject: [PATCH 02/20] Matching to 5fab877 --- crates/bindings-cpp/CMakeLists.txt | 45 +-------------- .../include/spacetimedb/abi/abi.h | 31 ++++++++++ .../include/spacetimedb/http_convert.h | 26 +++------ .../include/spacetimedb/http_wire.h | 56 ------------------- .../include/spacetimedb/internal/Module.h | 4 +- .../bindings-cpp/src/abi/module_exports.cpp | 8 ++- crates/bindings-cpp/src/internal/Module.cpp | 17 +++--- crates/bindings-cpp/tests/unit/CMakeLists.txt | 34 +++++++++++ crates/bindings-cpp/tests/unit/README.md | 52 +++++++++++++++++ .../tests/unit/http_unit_tests.cpp | 56 +++++++++++++++++++ crates/bindings-cpp/tests/unit/main.cpp | 34 +++++++++++ .../tests/unit/run-unit-tests.ps1 | 52 +++++++++++++++++ .../bindings-cpp/tests/unit/run-unit-tests.sh | 55 ++++++++++++++++++ crates/bindings-cpp/tests/unit/test_harness.h | 49 ++++++++++++++++ 14 files changed, 392 insertions(+), 127 deletions(-) create mode 100644 crates/bindings-cpp/tests/unit/CMakeLists.txt create mode 100644 crates/bindings-cpp/tests/unit/README.md create mode 100644 crates/bindings-cpp/tests/unit/http_unit_tests.cpp create mode 100644 crates/bindings-cpp/tests/unit/main.cpp create mode 100644 crates/bindings-cpp/tests/unit/run-unit-tests.ps1 create mode 100644 crates/bindings-cpp/tests/unit/run-unit-tests.sh create mode 100644 crates/bindings-cpp/tests/unit/test_harness.h diff --git a/crates/bindings-cpp/CMakeLists.txt b/crates/bindings-cpp/CMakeLists.txt index 35a4e0a1701..4c00b43323d 100644 --- a/crates/bindings-cpp/CMakeLists.txt +++ b/crates/bindings-cpp/CMakeLists.txt @@ -60,46 +60,5 @@ if(PROJECT_IS_TOP_LEVEL) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) endif() -# ---- Tests ---- -# Default: ON only when building this project directly; OFF when used via FetchContent/add_subdirectory -if(CMAKE_VERSION VERSION_LESS 3.21) - # Fallback heuristic for older CMake - set(_is_top_level FALSE) - if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - set(_is_top_level TRUE) - endif() -else() - set(_is_top_level ${PROJECT_IS_TOP_LEVEL}) -endif() - -option(BUILD_TESTS "Build the test suite" ${_is_top_level}) - -if(BUILD_TESTS AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - enable_testing() - - # Add test executable - add_executable(test_bsatn tests/main.cpp tests/module_library_unit_tests.cpp) - - # Link against the module library - target_link_libraries(test_bsatn PRIVATE spacetimedb_cpp_library) - - # Set C++20 standard for tests - target_compile_features(test_bsatn PRIVATE cxx_std_20) - - # Add test to CTest - add_test(NAME bsatn_tests COMMAND test_bsatn) - - # Add verbose test variant - add_test(NAME bsatn_tests_verbose COMMAND test_bsatn -v) - - # Set test properties - set_tests_properties(bsatn_tests PROPERTIES - TIMEOUT 30 - LABELS "unit" - ) - - set_tests_properties(bsatn_tests_verbose PROPERTIES - TIMEOUT 30 - LABELS "unit;verbose" - ) -endif() +# Unit/compile/smoke test harnesses live under `tests/` as standalone runners +# rather than being built through the top-level library CMake target. diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 54cefc8e9c3..e1aa12ac2de 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -216,6 +216,37 @@ int16_t __call_reducer__( BytesSource args, BytesSink error); +STDB_EXPORT(__call_view__) +int16_t __call_view__( + uint32_t id, + uint64_t sender_0, uint64_t sender_1, uint64_t sender_2, uint64_t sender_3, + BytesSource args, + BytesSink result); + +STDB_EXPORT(__call_view_anon__) +int16_t __call_view_anon__( + uint32_t id, + BytesSource args, + BytesSink result); + +STDB_EXPORT(__call_procedure__) +int16_t __call_procedure__( + uint32_t id, + uint64_t sender_0, uint64_t sender_1, uint64_t sender_2, uint64_t sender_3, + uint64_t conn_id_0, uint64_t conn_id_1, + uint64_t timestamp_microseconds, + BytesSource args_source, + BytesSink result_sink); + +STDB_EXPORT(__call_http_handler__) +int16_t __call_http_handler__( + uint32_t id, + uint64_t timestamp_microseconds, + BytesSource request_source, + BytesSource request_body_source, + BytesSink response_sink, + BytesSink response_body_sink); + // ======================================================================== // WASI SHIMS // ======================================================================== diff --git a/crates/bindings-cpp/include/spacetimedb/http_convert.h b/crates/bindings-cpp/include/spacetimedb/http_convert.h index 75f2652de15..7c02a1fb021 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_convert.h +++ b/crates/bindings-cpp/include/spacetimedb/http_convert.h @@ -237,6 +237,12 @@ inline HttpRequest from_wire(const wire::HttpRequest& request) { return result; } +inline HttpRequest from_wire(const wire::HttpRequest& request, std::vector body) { + HttpRequest result = from_wire(request); + result.body.bytes = std::move(body); + return result; +} + // ==================== HttpResponse Conversions ==================== /** @@ -268,24 +274,8 @@ inline HttpResponse from_wire(const wire::HttpResponse& response) { return result; } -inline wire::RequestAndBody to_wire_with_body(const HttpRequest& request) { - return wire::RequestAndBody{to_wire(request), request.body.bytes}; -} - -inline HttpRequest from_wire(const wire::RequestAndBody& request) { - HttpRequest result = from_wire(request.request); - result.body.bytes = request.body; - return result; -} - -inline wire::ResponseAndBody to_wire_with_body(const HttpResponse& response) { - return wire::ResponseAndBody{to_wire(response), response.body.bytes}; -} - -inline HttpResponse from_wire(const wire::ResponseAndBody& response) { - HttpResponse result = from_wire(response.response); - result.body.bytes = response.body; - return result; +inline std::pair> to_wire_split(const HttpResponse& response) { + return {to_wire(response), response.body.bytes}; } } // namespace convert diff --git a/crates/bindings-cpp/include/spacetimedb/http_wire.h b/crates/bindings-cpp/include/spacetimedb/http_wire.h index 51871ab385e..ae473512a37 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_wire.h +++ b/crates/bindings-cpp/include/spacetimedb/http_wire.h @@ -163,16 +163,6 @@ struct HttpResponse { uint16_t code; // Field 2: HTTP status code }; -struct RequestAndBody { - HttpRequest request; - std::vector body; -}; - -struct ResponseAndBody { - HttpResponse response; - std::vector body; -}; - } // namespace wire } // namespace SpacetimeDB @@ -187,8 +177,6 @@ template<> struct bsatn_traits; template<> struct bsatn_traits; template<> struct bsatn_traits; template<> struct bsatn_traits; -template<> struct bsatn_traits; -template<> struct bsatn_traits; // HttpMethod enum serialization template<> @@ -361,50 +349,6 @@ struct bsatn_traits { } }; -template<> -struct bsatn_traits { - static void serialize(Writer& writer, const wire::RequestAndBody& value) { - bsatn::serialize(writer, value.request); - bsatn::serialize(writer, value.body); - } - - static wire::RequestAndBody deserialize(Reader& reader) { - wire::RequestAndBody result; - result.request = bsatn::deserialize(reader); - result.body = bsatn::deserialize>(reader); - return result; - } - - static AlgebraicType algebraic_type() { - ProductTypeBuilder builder; - builder.with_field("request"); - builder.with_field>("body"); - return AlgebraicType::make_product(builder.build()); - } -}; - -template<> -struct bsatn_traits { - static void serialize(Writer& writer, const wire::ResponseAndBody& value) { - bsatn::serialize(writer, value.response); - bsatn::serialize(writer, value.body); - } - - static wire::ResponseAndBody deserialize(Reader& reader) { - wire::ResponseAndBody result; - result.response = bsatn::deserialize(reader); - result.body = bsatn::deserialize>(reader); - return result; - } - - static AlgebraicType algebraic_type() { - ProductTypeBuilder builder; - builder.with_field("response"); - builder.with_field>("body"); - return AlgebraicType::make_product(builder.build()); - } -}; - } // namespace SpacetimeDB::bsatn #endif // SPACETIMEDB_HTTP_WIRE_H diff --git a/crates/bindings-cpp/include/spacetimedb/internal/Module.h b/crates/bindings-cpp/include/spacetimedb/internal/Module.h index b669dffc36f..7e02e858fb6 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/Module.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/Module.h @@ -80,7 +80,9 @@ class Module { uint32_t id, uint64_t timestamp_microseconds, BytesSource request_source, - BytesSink result_sink + BytesSource request_body_source, + BytesSink response_sink, + BytesSink response_body_sink ); // Internal registration methods (inline to avoid linking issues) diff --git a/crates/bindings-cpp/src/abi/module_exports.cpp b/crates/bindings-cpp/src/abi/module_exports.cpp index 473dc9e2fb1..a31d50be1fb 100644 --- a/crates/bindings-cpp/src/abi/module_exports.cpp +++ b/crates/bindings-cpp/src/abi/module_exports.cpp @@ -104,13 +104,17 @@ extern "C" { uint32_t id, uint64_t timestamp_microseconds, SpacetimeDB::BytesSource request_source, - SpacetimeDB::BytesSink result_sink + SpacetimeDB::BytesSource request_body_source, + SpacetimeDB::BytesSink response_sink, + SpacetimeDB::BytesSink response_body_sink ) { return SpacetimeDB::Internal::Module::__call_http_handler__( id, timestamp_microseconds, request_source, - result_sink + request_body_source, + response_sink, + response_body_sink ); } diff --git a/crates/bindings-cpp/src/internal/Module.cpp b/crates/bindings-cpp/src/internal/Module.cpp index ebb73313160..421c4968c5d 100644 --- a/crates/bindings-cpp/src/internal/Module.cpp +++ b/crates/bindings-cpp/src/internal/Module.cpp @@ -668,7 +668,9 @@ int16_t Module::__call_http_handler__( uint32_t id, uint64_t timestamp_microseconds, BytesSource request_source, - BytesSink result_sink + BytesSource request_body_source, + BytesSink response_sink, + BytesSink response_body_sink ) { if (id >= g_http_handlers.size()) { fprintf(stderr, "ERROR: Invalid http handler ID %u (have %zu handlers)\n", @@ -681,18 +683,19 @@ int16_t Module::__call_http_handler__( std::vector request_bytes = ConsumeBytes(request_source); bsatn::Reader request_reader(request_bytes.data(), request_bytes.size()); - wire::RequestAndBody wire_request = bsatn::deserialize(request_reader); - HttpRequest request = convert::from_wire(wire_request); + wire::HttpRequest wire_request = bsatn::deserialize(request_reader); + HttpRequest request = convert::from_wire(wire_request, ConsumeBytes(request_body_source)); HttpResponse response = g_http_handlers[id].handler(ctx, std::move(request)); - wire::ResponseAndBody wire_response = convert::to_wire_with_body(response); + auto [wire_response, response_body] = convert::to_wire_split(response); - std::vector result_data; + std::vector response_metadata; { - bsatn::Writer writer(result_data); + bsatn::Writer writer(response_metadata); bsatn::serialize(writer, wire_response); } - WriteBytes(result_sink, result_data); + WriteBytes(response_sink, response_metadata); + WriteBytes(response_body_sink, response_body); return 0; } diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt new file mode 100644 index 00000000000..7a058e88fb9 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.16) +project(bindings_cpp_unit_tests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + message(FATAL_ERROR "tests/unit is intended to be built with Emscripten via emcmake") +endif() + +add_executable(bindings_cpp_unit_tests + main.cpp + http_unit_tests.cpp +) + +target_include_directories(bindings_cpp_unit_tests PRIVATE + ../../include +) + +if(MSVC) + target_compile_options(bindings_cpp_unit_tests PRIVATE /W4) +else() + target_compile_options(bindings_cpp_unit_tests PRIVATE -Wall -Wextra) +endif() + +target_link_options(bindings_cpp_unit_tests PRIVATE + "SHELL:-sWASM=1" + "SHELL:-sENVIRONMENT=node" + "SHELL:-sEXIT_RUNTIME=1" + "SHELL:-sASSERTIONS=1" + "SHELL:-O2" +) + +set_target_properties(bindings_cpp_unit_tests PROPERTIES SUFFIX ".cjs") diff --git a/crates/bindings-cpp/tests/unit/README.md b/crates/bindings-cpp/tests/unit/README.md new file mode 100644 index 00000000000..558027cb113 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/README.md @@ -0,0 +1,52 @@ +# C++ Unit Tests + +Standalone unit-test harness for pure bindings/library behavior. + +This suite is the right home for: +- conversion helpers +- small pure-library regressions +- behavior that does not need wasm module compilation +- behavior that does not need a live SpacetimeDB server + +Current coverage includes the HTTP request/response split-body conversion checks that +mirror the Rust tests added next to `crates/bindings/src/http.rs`. + +This harness is intentionally separate from the top-level bindings CMake so that +small header-only/library tests do not need to build the full module ABI/export layer. + +It is built with Emscripten and run under Node, which matches the existing wasm-oriented +C++ test toolchain more closely than adding a separate native-MSVC path. + +The generated Node launcher uses a `.cjs` suffix so it is treated as CommonJS even though +the repo root sets `"type": "module"`. + +## Run + +Prerequisites: + +- `emcmake` on `PATH` +- `node` on `PATH` + +From PowerShell: + +```powershell +.\crates\bindings-cpp\tests\unit\run-unit-tests.ps1 +``` + +Verbose: + +```powershell +.\crates\bindings-cpp\tests\unit\run-unit-tests.ps1 -Detailed +``` + +From Git Bash: + +```bash +./crates/bindings-cpp/tests/unit/run-unit-tests.sh +``` + +Verbose: + +```bash +./crates/bindings-cpp/tests/unit/run-unit-tests.sh --verbose +``` diff --git a/crates/bindings-cpp/tests/unit/http_unit_tests.cpp b/crates/bindings-cpp/tests/unit/http_unit_tests.cpp new file mode 100644 index 00000000000..16d98db8f09 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/http_unit_tests.cpp @@ -0,0 +1,56 @@ +#include "test_harness.h" + +#include "spacetimedb/http_convert.h" + +#include +#include +#include + +using namespace SpacetimeDB; + +TEST_CASE(request_from_wire_preserves_metadata_and_body) { + wire::HttpRequest request; + request.method = wire::HttpMethod{wire::HttpMethod::Tag::Post, ""}; + request.headers.entries = { + wire::HttpHeaderPair{"content-type", std::vector{'a','p','p','l','i','c','a','t','i','o','n','/','o','c','t','e','t','-','s','t','r','e','a','m'}}, + wire::HttpHeaderPair{"x-echo", std::vector{'v','a','l','u','e'}}, + }; + request.timeout = std::nullopt; + request.uri = "https://example.invalid/upload?x=1"; + request.version = wire::HttpVersion{wire::HttpVersion::Tag::Http2}; + + HttpRequest converted = convert::from_wire(request, std::vector{'p','a','y','l','o','a','d'}); + + ASSERT_EQ(std::string("POST"), converted.method.value); + ASSERT_EQ(std::string("https://example.invalid/upload?x=1"), converted.uri); + ASSERT_EQ(HttpVersion::Http2, converted.version); + ASSERT_EQ(static_cast(2), converted.headers.size()); + ASSERT_EQ(std::string("content-type"), converted.headers[0].name); + ASSERT_EQ(std::vector({'a','p','p','l','i','c','a','t','i','o','n','/','o','c','t','e','t','-','s','t','r','e','a','m'}), converted.headers[0].value); + ASSERT_EQ(std::string("x-echo"), converted.headers[1].name); + ASSERT_EQ(std::vector({'v','a','l','u','e'}), converted.headers[1].value); + ASSERT_EQ(std::vector({'p','a','y','l','o','a','d'}), converted.body.bytes); +} + +TEST_CASE(response_into_wire_splits_metadata_and_body) { + HttpResponse response{ + 201, + HttpVersion::Http11, + { + HttpHeader{"content-type", "text/plain"}, + HttpHeader{"x-result", "ok"}, + }, + HttpBody::from_string("created"), + }; + + auto [response_meta, response_body] = convert::to_wire_split(response); + + ASSERT_EQ(static_cast(201), response_meta.code); + ASSERT_EQ(wire::HttpVersion::Tag::Http11, response_meta.version.tag); + ASSERT_EQ(static_cast(2), response_meta.headers.entries.size()); + ASSERT_EQ(std::string("content-type"), response_meta.headers.entries[0].name); + ASSERT_EQ(std::vector({'t','e','x','t','/','p','l','a','i','n'}), response_meta.headers.entries[0].value); + ASSERT_EQ(std::string("x-result"), response_meta.headers.entries[1].name); + ASSERT_EQ(std::vector({'o','k'}), response_meta.headers.entries[1].value); + ASSERT_EQ(std::vector({'c','r','e','a','t','e','d'}), response_body); +} diff --git a/crates/bindings-cpp/tests/unit/main.cpp b/crates/bindings-cpp/tests/unit/main.cpp new file mode 100644 index 00000000000..054370390b6 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/main.cpp @@ -0,0 +1,34 @@ +#include "test_harness.h" + +#include +#include + +int main(int argc, char** argv) { + bool verbose = argc > 1 && std::string(argv[1]) == "-v"; + int failures = 0; + + for (const auto& test : SpacetimeDB::UnitTests::all_tests()) { + try { + test.func(); + if (verbose) { + std::cout << "[PASS] " << test.name << '\n'; + } + } catch (const std::exception& ex) { + ++failures; + std::cerr << "[FAIL] " << test.name << ": " << ex.what() << '\n'; + } catch (...) { + ++failures; + std::cerr << "[FAIL] " << test.name << ": unknown exception\n"; + } + } + + if (!verbose) { + if (failures == 0) { + std::cout << "Passed " << SpacetimeDB::UnitTests::all_tests().size() << " unit tests\n"; + } else { + std::cerr << failures << " unit test(s) failed\n"; + } + } + + return failures == 0 ? 0 : 1; +} diff --git a/crates/bindings-cpp/tests/unit/run-unit-tests.ps1 b/crates/bindings-cpp/tests/unit/run-unit-tests.ps1 new file mode 100644 index 00000000000..39e023828d9 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/run-unit-tests.ps1 @@ -0,0 +1,52 @@ +[CmdletBinding()] +param( + [switch]$Detailed +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$buildDir = Join-Path $scriptDir 'build' +$launcherPath = Join-Path $buildDir 'bindings_cpp_unit_tests.cjs' + +$emcmake = Get-Command emcmake.bat -ErrorAction SilentlyContinue +if ($null -eq $emcmake) { + $emcmake = Get-Command emcmake -ErrorAction SilentlyContinue +} +if ($null -eq $emcmake) { + throw 'Unable to locate emcmake or emcmake.bat' +} + +$node = Get-Command node -ErrorAction SilentlyContinue +if ($null -eq $node) { + throw 'Unable to locate node' +} + +Write-Host '' +Write-Host '==> Configuring unit tests' -ForegroundColor Cyan +& $emcmake.Source cmake -S $scriptDir -B $buildDir +if ($LASTEXITCODE -ne 0) { + throw "cmake configure failed with exit code $LASTEXITCODE" +} + +Write-Host '' +Write-Host '==> Building unit tests' -ForegroundColor Cyan +cmake --build $buildDir --target bindings_cpp_unit_tests +if ($LASTEXITCODE -ne 0) { + throw "cmake build failed with exit code $LASTEXITCODE" +} + +Write-Host '' +Write-Host '==> Running unit tests' -ForegroundColor Cyan +if (-not (Test-Path $launcherPath)) { + throw "Could not find built bindings_cpp_unit_tests.cjs launcher at $launcherPath" +} +if ($Detailed) { + & $node.Source $launcherPath -v +} else { + & $node.Source $launcherPath +} +if ($LASTEXITCODE -ne 0) { + throw "unit tests failed with exit code $LASTEXITCODE" +} diff --git a/crates/bindings-cpp/tests/unit/run-unit-tests.sh b/crates/bindings-cpp/tests/unit/run-unit-tests.sh new file mode 100644 index 00000000000..7de9aba68b8 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/run-unit-tests.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build" +VERBOSE=0 + +if command -v emcmake >/dev/null 2>&1; then + EMCMAKE_CMD="emcmake" +elif command -v emcmake.bat >/dev/null 2>&1; then + EMCMAKE_CMD="emcmake.bat" +else + echo "Unable to locate emcmake or emcmake.bat" >&2 + exit 1 +fi + +if ! command -v node >/dev/null 2>&1; then + echo "Unable to locate node" >&2 + exit 1 +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + -v|--verbose) + VERBOSE=1 + shift + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +echo +echo "==> Configuring unit tests" +"$EMCMAKE_CMD" cmake -S "$SCRIPT_DIR" -B "$BUILD_DIR" + +echo +echo "==> Building unit tests" +cmake --build "$BUILD_DIR" --target bindings_cpp_unit_tests + +echo +echo "==> Running unit tests" +LAUNCHER="$BUILD_DIR/bindings_cpp_unit_tests.cjs" +if [[ ! -f "$LAUNCHER" ]]; then + echo "Could not find built bindings_cpp_unit_tests.cjs launcher" >&2 + exit 1 +fi + +if [[ $VERBOSE -eq 1 ]]; then + node "$LAUNCHER" -v +else + node "$LAUNCHER" +fi diff --git a/crates/bindings-cpp/tests/unit/test_harness.h b/crates/bindings-cpp/tests/unit/test_harness.h new file mode 100644 index 00000000000..c3e6dff4f64 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/test_harness.h @@ -0,0 +1,49 @@ +#ifndef SPACETIMEDB_TEST_HARNESS_H +#define SPACETIMEDB_TEST_HARNESS_H + +#include +#include +#include + +namespace SpacetimeDB::UnitTests { + +struct TestCase { + const char* name; + void (*func)(); +}; + +inline std::vector& all_tests() { + static std::vector tests; + return tests; +} + +struct TestRegistrar { + TestRegistrar(const char* name, void (*func)()) { + all_tests().push_back(TestCase{name, func}); + } +}; + +} // namespace SpacetimeDB::UnitTests + +#define TEST_CASE(name) \ + void name(); \ + static ::SpacetimeDB::UnitTests::TestRegistrar name##_registrar(#name, &name); \ + void name() + +#define ASSERT_TRUE(condition) \ + do { \ + if (!(condition)) { \ + throw std::runtime_error(std::string("Assertion failed: ") + #condition); \ + } \ + } while (0) + +#define ASSERT_EQ(expected, actual) \ + do { \ + auto expected_value = (expected); \ + auto actual_value = (actual); \ + if (!(expected_value == actual_value)) { \ + throw std::runtime_error(std::string("Assertion failed: ") + #expected " == " #actual); \ + } \ + } while (0) + +#endif // SPACETIMEDB_TEST_HARNESS_H From 272cd574ee2b683543249da397154bc8762a6cad Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Tue, 12 May 2026 15:55:00 -0700 Subject: [PATCH 03/20] Fix unstable flag, add smoketests and update docs --- crates/bindings-cpp/CMakeLists.txt | 1 + crates/bindings-cpp/include/spacetimedb.h | 2 + .../include/spacetimedb/handler_context.h | 4 + .../bindings-cpp/include/spacetimedb/http.h | 4 + .../include/spacetimedb/http_handler_macros.h | 4 + .../bindings-cpp/include/spacetimedb/router.h | 4 + .../tests/compile/CMakeLists.module.txt | 1 + .../tests/compile/run-compile-tests.ps1 | 21 +- .../type-isolation-test/CMakeLists.module.txt | 3 +- crates/bindings-cpp/tests/unit/CMakeLists.txt | 4 + crates/smoketests/src/lib.rs | 107 +++- .../tests/smoketests/http_routes.rs | 571 ++++++++++++++++-- .../00200-functions/00600-HTTP-handlers.md | 92 +++ 13 files changed, 761 insertions(+), 57 deletions(-) diff --git a/crates/bindings-cpp/CMakeLists.txt b/crates/bindings-cpp/CMakeLists.txt index 4c00b43323d..a66b50ad9e2 100644 --- a/crates/bindings-cpp/CMakeLists.txt +++ b/crates/bindings-cpp/CMakeLists.txt @@ -30,6 +30,7 @@ target_sources(spacetimedb_cpp_library PRIVATE ${LIBRARY_SOURCES}) # Require C++20 for consumers of this library without forcing global flags target_compile_features(spacetimedb_cpp_library PUBLIC cxx_std_20) +target_compile_definitions(spacetimedb_cpp_library PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) # Set include directories target_include_directories(spacetimedb_cpp_library diff --git a/crates/bindings-cpp/include/spacetimedb.h b/crates/bindings-cpp/include/spacetimedb.h index 0eaaf9936cc..4ed389d06fa 100644 --- a/crates/bindings-cpp/include/spacetimedb.h +++ b/crates/bindings-cpp/include/spacetimedb.h @@ -126,9 +126,11 @@ // Procedure context and macros #include "spacetimedb/procedure_macros.h" +#ifdef SPACETIMEDB_UNSTABLE_FEATURES #include "spacetimedb/handler_context.h" #include "spacetimedb/router.h" #include "spacetimedb/http_handler_macros.h" +#endif // ============================================================================= // VIEW SYSTEM diff --git a/crates/bindings-cpp/include/spacetimedb/handler_context.h b/crates/bindings-cpp/include/spacetimedb/handler_context.h index 03a10eba0b5..cb9b4abe84c 100644 --- a/crates/bindings-cpp/include/spacetimedb/handler_context.h +++ b/crates/bindings-cpp/include/spacetimedb/handler_context.h @@ -1,6 +1,10 @@ #ifndef SPACETIMEDB_HANDLER_CONTEXT_H #define SPACETIMEDB_HANDLER_CONTEXT_H +#ifndef SPACETIMEDB_UNSTABLE_FEATURES +#error "spacetimedb/handler_context.h requires SPACETIMEDB_UNSTABLE_FEATURES to be enabled" +#endif + #include #include #include diff --git a/crates/bindings-cpp/include/spacetimedb/http.h b/crates/bindings-cpp/include/spacetimedb/http.h index cb51acafa0a..f24c76a8128 100644 --- a/crates/bindings-cpp/include/spacetimedb/http.h +++ b/crates/bindings-cpp/include/spacetimedb/http.h @@ -3,6 +3,10 @@ #pragma once +#ifndef SPACETIMEDB_UNSTABLE_FEATURES +#error "spacetimedb/http.h requires SPACETIMEDB_UNSTABLE_FEATURES to be enabled" +#endif + #include #include #include diff --git a/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h b/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h index 864b3ea4425..df9a1fc3a18 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h +++ b/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h @@ -1,5 +1,9 @@ #pragma once +#ifndef SPACETIMEDB_UNSTABLE_FEATURES +#error "spacetimedb/http_handler_macros.h requires SPACETIMEDB_UNSTABLE_FEATURES to be enabled" +#endif + #include "spacetimedb/handler_context.h" #include "spacetimedb/http.h" #include "spacetimedb/internal/runtime_registration.h" diff --git a/crates/bindings-cpp/include/spacetimedb/router.h b/crates/bindings-cpp/include/spacetimedb/router.h index 9f7bde5b34a..8c3f3b5191c 100644 --- a/crates/bindings-cpp/include/spacetimedb/router.h +++ b/crates/bindings-cpp/include/spacetimedb/router.h @@ -1,6 +1,10 @@ #ifndef SPACETIMEDB_ROUTER_H #define SPACETIMEDB_ROUTER_H +#ifndef SPACETIMEDB_UNSTABLE_FEATURES +#error "spacetimedb/router.h requires SPACETIMEDB_UNSTABLE_FEATURES to be enabled" +#endif + #include #include #include diff --git a/crates/bindings-cpp/tests/compile/CMakeLists.module.txt b/crates/bindings-cpp/tests/compile/CMakeLists.module.txt index 5c264b8f0cb..63d00912c90 100644 --- a/crates/bindings-cpp/tests/compile/CMakeLists.module.txt +++ b/crates/bindings-cpp/tests/compile/CMakeLists.module.txt @@ -24,6 +24,7 @@ add_executable(${OUTPUT_NAME} ${MODULE_SOURCE}) target_include_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_INCLUDE_DIR}) target_link_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_LIBRARY_DIR}) target_link_libraries(${OUTPUT_NAME} PRIVATE spacetimedb_cpp_library) +target_compile_definitions(${OUTPUT_NAME} PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(EXPORTED_FUNCS "['_malloc','_free','___describe_module__','___call_reducer__','___call_http_handler__']") diff --git a/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 b/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 index 889c44787c4..2dd8a40eaa4 100644 --- a/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 +++ b/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 @@ -35,7 +35,12 @@ function Invoke-LoggedCommand { } try { - & $FilePath @Arguments *> $LogPath + $quotedParts = @($FilePath) + $Arguments | ForEach-Object { + '"' + ($_ -replace '"', '\"') + '"' + } + $commandLine = ($quotedParts -join ' ') + " > `"$LogPath`" 2>&1" + + cmd /c $commandLine | Out-Null return $LASTEXITCODE } finally { if ($WorkingDirectory) { @@ -61,6 +66,11 @@ function New-CompileCase { } } +function Convert-ToCMakePath { + param([Parameter(Mandatory = $true)][string]$Path) + return $Path.Replace('\', '/') +} + $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $bindingsRoot = Split-Path -Parent (Split-Path -Parent $scriptDir) $repoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $bindingsRoot)) @@ -126,6 +136,9 @@ foreach ($case in $cases) { $caseBuildDir = Join-Path $buildRoot $case.Name $configureLog = Join-Path $caseBuildDir "configure.log" $buildLog = Join-Path $caseBuildDir "build.log" + $caseSourceCMake = Convert-ToCMakePath $caseSource + $libraryBuildDirCMake = Convert-ToCMakePath $libraryBuildDir + $includeDirCMake = Convert-ToCMakePath $includeDir if (Test-Path $caseBuildDir) { Remove-Item $caseBuildDir -Recurse -Force @@ -139,10 +152,10 @@ foreach ($case in $cases) { "cmake", "-S", $caseBuildDir, "-B", $caseBuildDir, - "-DMODULE_SOURCE=$caseSource", + "-DMODULE_SOURCE=$caseSourceCMake", "-DOUTPUT_NAME=$($case.Name)", - "-DSPACETIMEDB_LIBRARY_DIR=$libraryBuildDir", - "-DSPACETIMEDB_INCLUDE_DIR=$includeDir" + "-DSPACETIMEDB_LIBRARY_DIR=$libraryBuildDirCMake", + "-DSPACETIMEDB_INCLUDE_DIR=$includeDirCMake" ) -LogPath $configureLog -WorkingDirectory $scriptDir $buildExit = 0 diff --git a/crates/bindings-cpp/tests/type-isolation-test/CMakeLists.module.txt b/crates/bindings-cpp/tests/type-isolation-test/CMakeLists.module.txt index 243fcc5b902..38db22d8b82 100644 --- a/crates/bindings-cpp/tests/type-isolation-test/CMakeLists.module.txt +++ b/crates/bindings-cpp/tests/type-isolation-test/CMakeLists.module.txt @@ -28,6 +28,7 @@ add_executable(${OUTPUT_NAME} ${MODULE_SOURCE}) # Include directories target_include_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_INCLUDE_DIR}) +target_compile_definitions(${OUTPUT_NAME} PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) # Link the pre-built library target_link_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_LIBRARY_DIR}) @@ -61,4 +62,4 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") # Name the output lib.wasm set_target_properties(${OUTPUT_NAME} PROPERTIES OUTPUT_NAME "lib" SUFFIX ".wasm") -endif() \ No newline at end of file +endif() diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt index 7a058e88fb9..0ced4e0194c 100644 --- a/crates/bindings-cpp/tests/unit/CMakeLists.txt +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -17,6 +17,10 @@ target_include_directories(bindings_cpp_unit_tests PRIVATE ../../include ) +target_compile_definitions(bindings_cpp_unit_tests PRIVATE + SPACETIMEDB_UNSTABLE_FEATURES +) + if(MSVC) target_compile_options(bindings_cpp_unit_tests PRIVATE /W4) else() diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 1ef17047d31..088b51611db 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -353,6 +353,63 @@ pub fn have_emscripten() -> bool { *HAVE_EMSCRIPTEN.get_or_init(|| which("emcc").is_ok() || which("emcc.bat").is_ok()) } +const CPP_SMOKETEST_CMAKELISTS: &str = r#"cmake_minimum_required(VERSION 3.16) +project(smoketest_cpp_module) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(SPACETIMEDB_CPP_LIBRARY_PATH "@SPACETIMEDB_CPP_LIBRARY_PATH@") + +add_executable(lib src/lib.cpp) + +target_include_directories(lib PRIVATE + ${SPACETIMEDB_CPP_LIBRARY_PATH}/include +) + +if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + target_compile_options(lib PRIVATE -fno-exceptions -O2 -g0) + target_compile_definitions(lib PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSPACETIMEDB_UNSTABLE_FEATURES") +endif() + +add_subdirectory(${SPACETIMEDB_CPP_LIBRARY_PATH} ${CMAKE_CURRENT_BINARY_DIR}/spacetimedb_cpp_library) +target_link_libraries(lib PRIVATE spacetimedb_cpp_library) + +if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + set(EXPORTED_FUNCS + "['_malloc','_free','___describe_module__','___call_reducer__','___call_procedure__','___call_http_handler__']" + ) + + target_link_options(lib PRIVATE + "SHELL:-sSTANDALONE_WASM=1" + "SHELL:-sWASM=1" + "SHELL:--no-entry" + "SHELL:-sEXPORTED_FUNCTIONS=${EXPORTED_FUNCS}" + "SHELL:-sERROR_ON_UNDEFINED_SYMBOLS=1" + "SHELL:-sFILESYSTEM=0" + "SHELL:-sDISABLE_EXCEPTION_CATCHING=1" + "SHELL:-sALLOW_MEMORY_GROWTH=0" + "SHELL:-sINITIAL_MEMORY=16MB" + "SHELL:-sSUPPORT_LONGJMP=0" + "SHELL:-sSUPPORT_ERRNO=0" + "SHELL:-std=c++20" + "SHELL:-O2" + "SHELL:-g0" + ) + + set_target_properties(lib PROPERTIES OUTPUT_NAME "lib" SUFFIX ".wasm") +endif() +"#; + +fn parse_identity_from_publish_output(publish_output: &str) -> Result { + let re = Regex::new(r"identity: ([0-9a-fA-F]+)").unwrap(); + re.captures(publish_output) + .and_then(|caps| caps.get(1)) + .map(|m| m.as_str().to_string()) + .context("Failed to parse database identity from publish output") +} + /// A smoketest instance that manages a SpacetimeDB server and module project. pub struct Smoketest { /// The SpacetimeDB server guard (stops server on drop). @@ -929,12 +986,50 @@ impl Smoketest { ])?; csharp::verify_csharp_module_restore(&module_path)?; - let re = Regex::new(r"identity: ([0-9a-fA-F]+)").unwrap(); - let identity = re - .captures(&publish_output) - .and_then(|caps| caps.get(1)) - .map(|m| m.as_str().to_string()) - .context("Failed to parse database identity from publish output")?; + let identity = parse_identity_from_publish_output(&publish_output)?; + self.database_identity = Some(identity.clone()); + + Ok(identity) + } + + /// Writes and publishes a C++ module from source. + /// + /// The module is created at `/`. + /// On success this updates `self.database_identity`. + pub fn publish_cpp_module_source( + &mut self, + project_dir_name: &str, + module_name: &str, + module_source: &str, + ) -> Result { + let module_path = self.project_dir.path().join(project_dir_name); + let src_dir = module_path.join("src"); + fs::create_dir_all(&src_dir).context("Failed to create C++ source directory")?; + + let bindings_cpp_path = workspace_root() + .join("crates/bindings-cpp") + .display() + .to_string() + .replace('\\', "/"); + let cmakelists = CPP_SMOKETEST_CMAKELISTS.replace("@SPACETIMEDB_CPP_LIBRARY_PATH@", &bindings_cpp_path); + + fs::write(module_path.join("CMakeLists.txt"), cmakelists) + .context("Failed to write C++ CMakeLists.txt")?; + fs::write(src_dir.join("lib.cpp"), module_source).context("Failed to write C++ module code")?; + + let module_path_str = module_path.to_str().context("Invalid C++ module path")?; + let publish_output = self.spacetime(&[ + "publish", + "--server", + &self.server_url, + "--module-path", + module_path_str, + "--yes", + "--clear-database", + module_name, + ])?; + + let identity = parse_identity_from_publish_output(&publish_output)?; self.database_identity = Some(identity.clone()); Ok(identity) diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/smoketests/http_routes.rs index 40abde11f8e..7c14f82ad2c 100644 --- a/crates/smoketests/tests/smoketests/http_routes.rs +++ b/crates/smoketests/tests/smoketests/http_routes.rs @@ -1,5 +1,5 @@ use regex::Regex; -use spacetimedb_smoketests::{workspace_root, Smoketest}; +use spacetimedb_smoketests::{require_emscripten, workspace_root, Smoketest}; use std::{fs, path::Path}; const MODULE_CODE: &str = r#" @@ -230,6 +230,363 @@ fn router() -> Router { } "#; +const CPP_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +struct Entry { + uint64_t id; + std::string value; +}; +SPACETIMEDB_STRUCT(Entry, id, value) +SPACETIMEDB_TABLE(Entry, entry, Public) + +namespace { + +std::string header_value_utf8(const HttpRequest& request, const std::string& header_name) { + for (const auto& header : request.headers) { + if (header.name == header_name) { + return std::string(header.value.begin(), header.value.end()); + } + } + return ""; +} + +HttpResponse text_response(uint16_t status_code, std::string body) { + return HttpResponse{ + status_code, + HttpVersion::Http11, + { HttpHeader{"content-type", "text/plain; charset=utf-8"} }, + HttpBody::from_string(body), + }; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(get_simple, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response(200, "ok"); +} + +SPACETIMEDB_HTTP_HANDLER(post_insert, HandlerContext ctx, HttpRequest request) { + (void)request; + ctx.with_tx([](TxContext& tx) { + uint64_t id = tx.db[entry].count(); + tx.db[entry].insert(Entry{ id, "posted" }); + }); + return text_response(200, "inserted"); +} + +SPACETIMEDB_HTTP_HANDLER(get_count, HandlerContext ctx, HttpRequest request) { + (void)request; + uint64_t count = ctx.with_tx([](TxContext& tx) -> uint64_t { + return tx.db[entry].count(); + }); + return text_response(200, std::to_string(count)); +} + +SPACETIMEDB_HTTP_HANDLER(any_handler, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response(200, "any"); +} + +SPACETIMEDB_HTTP_HANDLER(header_echo, HandlerContext ctx, HttpRequest request) { + (void)ctx; + return text_response(200, header_value_utf8(request, "x-echo")); +} + +SPACETIMEDB_HTTP_HANDLER(set_response_header, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return HttpResponse{ + 200, + HttpVersion::Http11, + { HttpHeader{"x-response", "set"} }, + HttpBody::from_string("header-set"), + }; +} + +SPACETIMEDB_HTTP_HANDLER(body_handler, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response(200, "non-empty"); +} + +SPACETIMEDB_HTTP_HANDLER(teapot, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response(418, "teapot"); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .get("/get", get_simple) + .post("/post", post_insert) + .get("/count", get_count) + .any("/any", any_handler) + .get("/header", header_echo) + .get("/set-header", set_response_header) + .get("/body", body_handler) + .get("/teapot", teapot); +} +"#; + +const CPP_EXAMPLE_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +struct Data { + uint64_t id; + std::vector body; +}; +SPACETIMEDB_STRUCT(Data, id, body) +SPACETIMEDB_TABLE(Data, data, Public) +FIELD_PrimaryKeyAutoInc(data, id) + +namespace { + +HttpResponse bytes_response(uint16_t status_code, std::vector body) { + return HttpResponse{ + status_code, + HttpVersion::Http11, + {}, + HttpBody{std::move(body)}, + }; +} + +HttpResponse text_response(uint16_t status_code, std::string body) { + return HttpResponse{ + status_code, + HttpVersion::Http11, + {}, + HttpBody::from_string(body), + }; +} + +std::string query_value(const std::string& uri, const std::string& key) { + std::string needle = "?" + key + "="; + size_t pos = uri.find(needle); + if (pos == std::string::npos) { + needle = "&" + key + "="; + pos = uri.find(needle); + } + if (pos == std::string::npos) { + return ""; + } + pos += needle.size(); + size_t end = uri.find('&', pos); + return uri.substr(pos, end == std::string::npos ? std::string::npos : end - pos); +} + +bool try_parse_u64(const std::string& text, uint64_t& value) { + if (text.empty()) { + return false; + } + uint64_t result = 0; + for (char c : text) { + if (c < '0' || c > '9') { + return false; + } + result = (result * 10) + static_cast(c - '0'); + } + value = result; + return true; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(insert, HandlerContext ctx, HttpRequest request) { + std::vector body = request.body.to_bytes(); + uint64_t id = ctx.with_tx([&](TxContext& tx) -> uint64_t { + return tx.db[data].insert(Data{0, body}).id; + }); + return text_response(200, std::to_string(id)); +} + +SPACETIMEDB_HTTP_HANDLER(retrieve, HandlerContext ctx, HttpRequest request) { + uint64_t id = 0; + if (!try_parse_u64(query_value(request.uri, "id"), id)) { + return text_response(500, "invalid id"); + } + + auto body = ctx.with_tx([&](TxContext& tx) -> std::optional> { + auto row = tx.db[data_id].find(id); + if (row.has_value()) { + return row->body; + } + return std::nullopt; + }); + + if (body.has_value()) { + return bytes_response(200, std::move(body.value())); + } + return bytes_response(404, {}); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router().post("/insert", insert).get("/retrieve", retrieve); +} +"#; + +const CPP_STRICT_ROOT_ROUTING_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +namespace { + +HttpResponse text_response(const std::string& body) { + return HttpResponse{200, HttpVersion::Http11, {}, HttpBody::from_string(body)}; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(empty_root, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response("empty"); +} + +SPACETIMEDB_HTTP_HANDLER(slash_root, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response("slash"); +} + +SPACETIMEDB_HTTP_HANDLER(foo, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response("foo"); +} + +SPACETIMEDB_HTTP_HANDLER(foo_slash, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response("foo-slash"); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .get("", empty_root) + .get("/", slash_root) + .get("/foo", foo) + .get("/foo/", foo_slash); +} +"#; + +const CPP_STRICT_NON_ROOT_ROUTING_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +namespace { + +HttpResponse text_response(const std::string& body) { + return HttpResponse{200, HttpVersion::Http11, {}, HttpBody::from_string(body)}; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(foo, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response("foo"); +} + +SPACETIMEDB_HTTP_HANDLER(foo_slash, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return text_response("foo-slash"); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .get("/foo", foo) + .get("/foo/", foo_slash); +} +"#; + +const CPP_FULL_URI_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(echo_uri, HandlerContext ctx, HttpRequest request) { + (void)ctx; + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string(request.uri), + }; +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router().get("/echo-uri", echo_uri); +} +"#; + +const CPP_HANDLE_REQUEST_BODY_MODULE_CODE: &str = r#"#include "spacetimedb.h" +#include + +using namespace SpacetimeDB; + +namespace { + +HttpResponse bytes_response(uint16_t status_code, std::vector body) { + return HttpResponse{status_code, HttpVersion::Http11, {}, HttpBody{std::move(body)}}; +} + +HttpResponse text_response(uint16_t status_code, const std::string& body) { + return HttpResponse{status_code, HttpVersion::Http11, {}, HttpBody::from_string(body)}; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(reverse_bytes, HandlerContext ctx, HttpRequest request) { + (void)ctx; + std::vector reversed = request.body.to_bytes(); + std::reverse(reversed.begin(), reversed.end()); + return bytes_response(200, std::move(reversed)); +} + +SPACETIMEDB_HTTP_HANDLER(reverse_words, HandlerContext ctx, HttpRequest request) { + (void)ctx; + const std::vector bytes = request.body.to_bytes(); + std::string body(bytes.begin(), bytes.end()); + if (body.find(static_cast(0x80)) != std::string::npos) { + return text_response(400, "request body must be valid UTF-8"); + } + + std::vector words; + size_t start = 0; + while (true) { + size_t pos = body.find(' ', start); + words.push_back(body.substr(start, pos == std::string::npos ? std::string::npos : pos - start)); + if (pos == std::string::npos) { + break; + } + start = pos + 1; + } + std::reverse(words.begin(), words.end()); + + std::string reversed; + for (size_t i = 0; i < words.size(); ++i) { + if (i != 0) { + reversed += " "; + } + reversed += words[i]; + } + + return text_response(200, reversed); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .post("/reverse-bytes", reverse_bytes) + .post("/reverse-words", reverse_words); +} +"#; + const NO_SUCH_ROUTE_BODY: &str = "Database has not registered a handler for this route"; fn extract_rust_code_blocks(doc_path: &Path) -> String { @@ -251,12 +608,46 @@ fn extract_rust_code_blocks(doc_path: &Path) -> String { blocks.join("\n\n") } -#[test] -fn http_routes_end_to_end() { - let test = Smoketest::builder().module_code(MODULE_CODE).build(); - let identity = test.database_identity.as_ref().expect("database identity missing"); +fn extract_cpp_code_blocks(doc_path: &Path) -> String { + let doc = fs::read_to_string(doc_path).unwrap_or_else(|e| panic!("failed to read {}: {e}", doc_path.display())); + let doc = doc.replace("\r\n", "\n"); - let base = format!("{}/v1/database/{}/route", test.server_url, identity); + let re = Regex::new(r"```(?:cpp|c\+\+)\n([\s\S]*?)\n```").expect("regex should compile"); + let blocks: Vec<_> = re + .captures_iter(&doc) + .map(|cap| cap.get(1).expect("capture group should exist").as_str().to_string()) + .collect(); + + assert!( + !blocks.is_empty(), + "expected at least one cpp code block in {}", + doc_path.display() + ); + + blocks.join("\n\n") +} + +fn rust_http_test(module_code: &str) -> (Smoketest, String) { + let test = Smoketest::builder().module_code(module_code).build(); + let identity = test + .database_identity + .as_ref() + .expect("database identity missing") + .clone(); + (test, identity) +} + +fn cpp_http_test(module_name: &str, db_name: &str, module_code: &str) -> (Smoketest, String) { + require_emscripten!(); + let mut test = Smoketest::builder().autopublish(false).build(); + let identity = test + .publish_cpp_module_source(module_name, db_name, module_code) + .unwrap(); + (test, identity) +} + +fn assert_http_routes_end_to_end(server_url: &str, identity: &str) { + let base = format!("{server_url}/v1/database/{identity}/route"); let client = reqwest::blocking::Client::new(); let resp = client.get(format!("{base}/get")).send().expect("get failed"); @@ -311,10 +702,7 @@ fn http_routes_end_to_end() { assert_eq!(resp.text().expect("missing route body"), NO_SUCH_ROUTE_BODY); let resp = client - .get(format!( - "{}/v1/database/{}/schema?version=10", - test.server_url, identity - )) + .get(format!("{server_url}/v1/database/{identity}/schema?version=10")) .header("authorization", "Bearer not-a-jwt") .send() .expect("schema request failed"); @@ -328,12 +716,8 @@ fn http_routes_end_to_end() { assert!(resp.status().is_success()); } -#[test] -fn http_routes_pr_example_round_trip() { - let test = Smoketest::builder().module_code(EXAMPLE_MODULE_CODE).build(); - let identity = test.database_identity.as_ref().expect("database identity missing"); - - let base = format!("{}/v1/database/{}/route", test.server_url, identity); +fn assert_http_routes_pr_example_round_trip(server_url: &str, identity: &str) { + let base = format!("{server_url}/v1/database/{identity}/route"); let client = reqwest::blocking::Client::new(); let payload = b"hello from the PR example".to_vec(); @@ -368,14 +752,8 @@ fn http_routes_pr_example_round_trip() { assert!(resp.status().is_server_error()); } -#[test] -fn http_routes_are_strict_for_non_root_paths() { - let test = Smoketest::builder() - .module_code(STRICT_NON_ROOT_ROUTING_MODULE_CODE) - .build(); - let identity = test.database_identity.as_ref().expect("database identity missing"); - - let base = format!("{}/v1/database/{}/route", test.server_url, identity); +fn assert_http_routes_are_strict_for_non_root_paths(server_url: &str, identity: &str) { + let base = format!("{server_url}/v1/database/{identity}/route"); let client = reqwest::blocking::Client::new(); let resp = client.get(format!("{base}/foo")).send().expect("foo failed"); @@ -398,14 +776,8 @@ fn http_routes_are_strict_for_non_root_paths() { assert_eq!(resp.text().expect("double slash foo body"), NO_SUCH_ROUTE_BODY); } -#[test] -fn http_routes_are_strict_for_root_paths() { - let test = Smoketest::builder() - .module_code(STRICT_ROOT_ROUTING_MODULE_CODE) - .build(); - let identity = test.database_identity.as_ref().expect("database identity missing"); - - let base = format!("{}/v1/database/{}/route", test.server_url, identity); +fn assert_http_routes_are_strict_for_root_paths(server_url: &str, identity: &str) { + let base = format!("{server_url}/v1/database/{identity}/route"); let client = reqwest::blocking::Client::new(); let resp = client.get(base.clone()).send().expect("empty root failed"); @@ -417,12 +789,8 @@ fn http_routes_are_strict_for_root_paths() { assert_eq!(resp.text().expect("slash root body"), "slash"); } -#[test] -fn http_handler_observes_full_external_uri() { - let test = Smoketest::builder().module_code(FULL_URI_MODULE_CODE).build(); - let identity = test.database_identity.as_ref().expect("database identity missing"); - - let base = format!("{}/v1/database/{}/route", test.server_url, identity); +fn assert_http_handler_observes_full_external_uri(server_url: &str, identity: &str) { + let base = format!("{server_url}/v1/database/{identity}/route"); let url = format!("{base}/echo-uri?alpha=beta"); let client = reqwest::blocking::Client::new(); @@ -431,14 +799,8 @@ fn http_handler_observes_full_external_uri() { assert_eq!(resp.text().expect("echo-uri body"), url); } -#[test] -fn handle_request_body() { - let test = Smoketest::builder() - .module_code(HANDLE_REQUEST_BODY_MODULE_CODE) - .build(); - let identity = test.database_identity.as_ref().expect("database identity missing"); - - let base = format!("{}/v1/database/{}/route", test.server_url, identity); +fn assert_handle_request_body(server_url: &str, identity: &str) { + let base = format!("{server_url}/v1/database/{identity}/route"); let client = reqwest::blocking::Client::new(); let resp = client @@ -502,6 +864,98 @@ fn handle_request_body() { ); } +#[test] +fn http_routes_end_to_end() { + let (test, identity) = rust_http_test(MODULE_CODE); + assert_http_routes_end_to_end(&test.server_url, &identity); +} + +#[test] +fn http_routes_pr_example_round_trip() { + let (test, identity) = rust_http_test(EXAMPLE_MODULE_CODE); + assert_http_routes_pr_example_round_trip(&test.server_url, &identity); +} + +#[test] +fn http_routes_are_strict_for_non_root_paths() { + let (test, identity) = rust_http_test(STRICT_NON_ROOT_ROUTING_MODULE_CODE); + assert_http_routes_are_strict_for_non_root_paths(&test.server_url, &identity); +} + +#[test] +fn http_routes_are_strict_for_root_paths() { + let (test, identity) = rust_http_test(STRICT_ROOT_ROUTING_MODULE_CODE); + assert_http_routes_are_strict_for_root_paths(&test.server_url, &identity); +} + +#[test] +fn http_handler_observes_full_external_uri() { + let (test, identity) = rust_http_test(FULL_URI_MODULE_CODE); + assert_http_handler_observes_full_external_uri(&test.server_url, &identity); +} + +#[test] +fn handle_request_body() { + let (test, identity) = rust_http_test(HANDLE_REQUEST_BODY_MODULE_CODE); + assert_handle_request_body(&test.server_url, &identity); +} + +#[test] +fn cpp_http_routes_end_to_end() { + let (test, identity) = cpp_http_test("http-routes-cpp-basic", "http-routes-cpp-basic", CPP_MODULE_CODE); + assert_http_routes_end_to_end(&test.server_url, &identity); +} + +#[test] +fn cpp_http_routes_pr_example_round_trip() { + let (test, identity) = cpp_http_test( + "http-routes-cpp-example", + "http-routes-cpp-example", + CPP_EXAMPLE_MODULE_CODE, + ); + assert_http_routes_pr_example_round_trip(&test.server_url, &identity); +} + +#[test] +fn cpp_http_routes_are_strict_for_non_root_paths() { + let (test, identity) = cpp_http_test( + "http-routes-cpp-strict-non-root", + "http-routes-cpp-strict-non-root", + CPP_STRICT_NON_ROOT_ROUTING_MODULE_CODE, + ); + assert_http_routes_are_strict_for_non_root_paths(&test.server_url, &identity); +} + +#[test] +fn cpp_http_routes_are_strict_for_root_paths() { + let (test, identity) = cpp_http_test( + "http-routes-cpp-strict-root", + "http-routes-cpp-strict-root", + CPP_STRICT_ROOT_ROUTING_MODULE_CODE, + ); + assert_http_routes_are_strict_for_root_paths(&test.server_url, &identity); +} + +#[test] +fn cpp_http_handler_observes_full_external_uri() { + let (test, identity) = cpp_http_test( + "http-routes-cpp-full-uri", + "http-routes-cpp-full-uri", + CPP_FULL_URI_MODULE_CODE, + ); + assert_http_handler_observes_full_external_uri(&test.server_url, &identity); +} + +#[test] +fn cpp_handle_request_body() { + let (test, identity) = cpp_http_test( + "http-routes-cpp-request-body", + "http-routes-cpp-request-body", + CPP_HANDLE_REQUEST_BODY_MODULE_CODE, + ); + assert_handle_request_body(&test.server_url, &identity); +} + /// Validates the Rust example from `docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md`. #[test] fn http_handlers_tutorial_say_hello_route_works() { @@ -518,3 +972,28 @@ fn http_handlers_tutorial_say_hello_route_works() { assert!(resp.status().is_success()); assert_eq!(resp.text().expect("say-hello body"), "Hello!"); } + +/// Validates the C++ example from `docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md`. +#[test] +fn cpp_http_handlers_tutorial_say_hello_route_works() { + require_emscripten!(); + + let module_code = extract_cpp_code_blocks( + &workspace_root().join("docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md"), + ); + let mut test = Smoketest::builder().autopublish(false).build(); + let identity = test + .publish_cpp_module_source( + "http-handlers-docs-cpp", + "http-handlers-docs-cpp", + &module_code, + ) + .unwrap(); + + let url = format!("{}/v1/database/{identity}/route/say-hello", test.server_url); + let client = reqwest::blocking::Client::new(); + + let resp = client.get(&url).send().expect("say-hello failed"); + assert!(resp.status().is_success()); + assert_eq!(resp.text().expect("say-hello body"), "Hello!"); +} diff --git a/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md b/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md index 67589fad0ba..657003f7d38 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md +++ b/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md @@ -16,6 +16,29 @@ External clients can make HTTP requests to routes nested under [`/v1/database/:n ## Defining HTTP Handlers + + +Define an HTTP handler with `spacetimedb.httpHandler`. + +The function must accept exactly two arguments: + +1. A `HandlerContext`. +2. A `Request`. + +The function must return a `SyncResponse`. + +```typescript +import { schema, SyncResponse } from "spacetimedb/server"; + +const spacetimedb = schema({}); +export default spacetimedb; + +export const say_hello = spacetimedb.httpHandler((_ctx, _req) => { + return new SyncResponse("Hello!"); +}); +``` + + Because HTTP handlers are unstable, Rust modules that define them must opt in to the `unstable` feature in their `Cargo.toml`: @@ -43,6 +66,37 @@ fn say_hello(_ctx: &mut HandlerContext, _req: Request) -> Response { } ``` + + + +Because HTTP handlers are unstable, C++ modules that define them must enable `SPACETIMEDB_UNSTABLE_FEATURES` when compiling. + +Define an HTTP handler with `SPACETIMEDB_HTTP_HANDLER`. + +The function must accept exactly two arguments: + +1. A `SpacetimeDB::HandlerContext`. +2. A `SpacetimeDB::HttpRequest`. + +The function must return a `SpacetimeDB::HttpResponse`. + +```cpp +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(say_hello, HandlerContext ctx, HttpRequest request) { + (void)ctx; + (void)request; + return HttpResponse{ + 200, + HttpVersion::Http11, + { HttpHeader{"content-type", "text/plain; charset=utf-8"} }, + HttpBody::from_string("Hello!"), + }; +} +``` + @@ -51,6 +105,26 @@ fn say_hello(_ctx: &mut HandlerContext, _req: Request) -> Response { Once you've [defined an HTTP handler](#defining-http-handlers), you must register it to a route in order to make it reachable for requests. + + +All routes exposed by your module are declared in a `Router`. Register the `Router` for your database by passing it to `spacetimedb.httpRouter`. + +```typescript +import { Router } from "spacetimedb/server"; + +export const router = spacetimedb.httpRouter( + new Router() + .get("/say-hello", say_hello) +); +``` + +Add routes within a router with the `get`, `head`, `options`, `put`, `delete`, `post`, `patch` and `any` methods, which register an HTTP handler for that HTTP method at a given path. + +Nest routers with `router.nest(prefix, subRouter)`, which causes `subRouter` to handle routing for all paths that start with `prefix`. + +Combine routers with `router.merge(otherRouter)`, which combines both routers. + + All routes exposed by your module are declared in a `spacetimedb::http::Router`. Register the `Router` for your database by returning it from a function annotated with `#[spacetimedb::http::router]`. @@ -71,6 +145,24 @@ Nest routers with `router.nest(prefix, sub_router)`, which causes `sub_router` t Combine routers with `router.merge(other_router)`, which combines both routers. + + + +All routes exposed by your module are declared in a `SpacetimeDB::Router`. Register the `Router` for your database by returning it from a function defined with `SPACETIMEDB_HTTP_ROUTER`. + +```cpp +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .get("/say-hello", say_hello); +} +``` + +Add routes within a router with the `get`, `head`, `options`, `put`, `delete_`, `post`, `patch` and `any` methods, which register an HTTP handler for that HTTP method at a given path. + +Nest routers with `router.nest(prefix, sub_router)`, which causes `sub_router` to handle routing for all paths that start with `prefix`. + +Combine routers with `router.merge(other_router)`, which combines both routers. + From e212729155b6eb3ccb765aa729e5a7851a6fd917 Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Wed, 13 May 2026 11:26:35 -0700 Subject: [PATCH 04/20] Mirror changes in TypeScript for smoketests --- .../tests/smoketests/http_routes.rs | 86 ++++++------------- 1 file changed, 28 insertions(+), 58 deletions(-) diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/smoketests/http_routes.rs index 7c14f82ad2c..3efcda47301 100644 --- a/crates/smoketests/tests/smoketests/http_routes.rs +++ b/crates/smoketests/tests/smoketests/http_routes.rs @@ -589,11 +589,11 @@ SPACETIMEDB_HTTP_ROUTER(router) { const NO_SUCH_ROUTE_BODY: &str = "Database has not registered a handler for this route"; -fn extract_rust_code_blocks(doc_path: &Path) -> String { +fn extract_code_blocks(doc_path: &Path, regex_src: &str, language_name: &str) -> String { let doc = fs::read_to_string(doc_path).unwrap_or_else(|e| panic!("failed to read {}: {e}", doc_path.display())); let doc = doc.replace("\r\n", "\n"); - let re = Regex::new(r"```rust\n([\s\S]*?)\n```").expect("regex should compile"); + let re = Regex::new(regex_src).expect("regex should compile"); let blocks: Vec<_> = re .captures_iter(&doc) .map(|cap| cap.get(1).expect("capture group should exist").as_str().to_string()) @@ -601,26 +601,8 @@ fn extract_rust_code_blocks(doc_path: &Path) -> String { assert!( !blocks.is_empty(), - "expected at least one rust code block in {}", - doc_path.display() - ); - - blocks.join("\n\n") -} - -fn extract_cpp_code_blocks(doc_path: &Path) -> String { - let doc = fs::read_to_string(doc_path).unwrap_or_else(|e| panic!("failed to read {}: {e}", doc_path.display())); - let doc = doc.replace("\r\n", "\n"); - - let re = Regex::new(r"```(?:cpp|c\+\+)\n([\s\S]*?)\n```").expect("regex should compile"); - let blocks: Vec<_> = re - .captures_iter(&doc) - .map(|cap| cap.get(1).expect("capture group should exist").as_str().to_string()) - .collect(); - - assert!( - !blocks.is_empty(), - "expected at least one cpp code block in {}", + "expected at least one {} code block in {}", + language_name, doc_path.display() ); @@ -637,17 +619,21 @@ fn rust_http_test(module_code: &str) -> (Smoketest, String) { (test, identity) } -fn cpp_http_test(module_name: &str, db_name: &str, module_code: &str) -> (Smoketest, String) { +fn cpp_http_test(name: &str, module_code: &str) -> (Smoketest, String) { require_emscripten!(); let mut test = Smoketest::builder().autopublish(false).build(); let identity = test - .publish_cpp_module_source(module_name, db_name, module_code) + .publish_cpp_module_source(name, name, module_code) .unwrap(); (test, identity) } +fn route_base(server_url: &str, identity: &str) -> String { + format!("{server_url}/v1/database/{identity}/route") +} + fn assert_http_routes_end_to_end(server_url: &str, identity: &str) { - let base = format!("{server_url}/v1/database/{identity}/route"); + let base = route_base(server_url, identity); let client = reqwest::blocking::Client::new(); let resp = client.get(format!("{base}/get")).send().expect("get failed"); @@ -717,7 +703,7 @@ fn assert_http_routes_end_to_end(server_url: &str, identity: &str) { } fn assert_http_routes_pr_example_round_trip(server_url: &str, identity: &str) { - let base = format!("{server_url}/v1/database/{identity}/route"); + let base = route_base(server_url, identity); let client = reqwest::blocking::Client::new(); let payload = b"hello from the PR example".to_vec(); @@ -753,7 +739,7 @@ fn assert_http_routes_pr_example_round_trip(server_url: &str, identity: &str) { } fn assert_http_routes_are_strict_for_non_root_paths(server_url: &str, identity: &str) { - let base = format!("{server_url}/v1/database/{identity}/route"); + let base = route_base(server_url, identity); let client = reqwest::blocking::Client::new(); let resp = client.get(format!("{base}/foo")).send().expect("foo failed"); @@ -777,7 +763,7 @@ fn assert_http_routes_are_strict_for_non_root_paths(server_url: &str, identity: } fn assert_http_routes_are_strict_for_root_paths(server_url: &str, identity: &str) { - let base = format!("{server_url}/v1/database/{identity}/route"); + let base = route_base(server_url, identity); let client = reqwest::blocking::Client::new(); let resp = client.get(base.clone()).send().expect("empty root failed"); @@ -790,7 +776,7 @@ fn assert_http_routes_are_strict_for_root_paths(server_url: &str, identity: &str } fn assert_http_handler_observes_full_external_uri(server_url: &str, identity: &str) { - let base = format!("{server_url}/v1/database/{identity}/route"); + let base = route_base(server_url, identity); let url = format!("{base}/echo-uri?alpha=beta"); let client = reqwest::blocking::Client::new(); @@ -800,7 +786,7 @@ fn assert_http_handler_observes_full_external_uri(server_url: &str, identity: &s } fn assert_handle_request_body(server_url: &str, identity: &str) { - let base = format!("{server_url}/v1/database/{identity}/route"); + let base = route_base(server_url, identity); let client = reqwest::blocking::Client::new(); let resp = client @@ -902,65 +888,47 @@ fn handle_request_body() { #[test] fn cpp_http_routes_end_to_end() { - let (test, identity) = cpp_http_test("http-routes-cpp-basic", "http-routes-cpp-basic", CPP_MODULE_CODE); + let (test, identity) = cpp_http_test("http-routes-cpp-basic", CPP_MODULE_CODE); assert_http_routes_end_to_end(&test.server_url, &identity); } #[test] fn cpp_http_routes_pr_example_round_trip() { - let (test, identity) = cpp_http_test( - "http-routes-cpp-example", - "http-routes-cpp-example", - CPP_EXAMPLE_MODULE_CODE, - ); + let (test, identity) = cpp_http_test("http-routes-cpp-example", CPP_EXAMPLE_MODULE_CODE); assert_http_routes_pr_example_round_trip(&test.server_url, &identity); } #[test] fn cpp_http_routes_are_strict_for_non_root_paths() { - let (test, identity) = cpp_http_test( - "http-routes-cpp-strict-non-root", - "http-routes-cpp-strict-non-root", - CPP_STRICT_NON_ROOT_ROUTING_MODULE_CODE, - ); + let (test, identity) = cpp_http_test("http-routes-cpp-strict-non-root", CPP_STRICT_NON_ROOT_ROUTING_MODULE_CODE); assert_http_routes_are_strict_for_non_root_paths(&test.server_url, &identity); } #[test] fn cpp_http_routes_are_strict_for_root_paths() { - let (test, identity) = cpp_http_test( - "http-routes-cpp-strict-root", - "http-routes-cpp-strict-root", - CPP_STRICT_ROOT_ROUTING_MODULE_CODE, - ); + let (test, identity) = cpp_http_test("http-routes-cpp-strict-root", CPP_STRICT_ROOT_ROUTING_MODULE_CODE); assert_http_routes_are_strict_for_root_paths(&test.server_url, &identity); } #[test] fn cpp_http_handler_observes_full_external_uri() { - let (test, identity) = cpp_http_test( - "http-routes-cpp-full-uri", - "http-routes-cpp-full-uri", - CPP_FULL_URI_MODULE_CODE, - ); + let (test, identity) = cpp_http_test("http-routes-cpp-full-uri", CPP_FULL_URI_MODULE_CODE); assert_http_handler_observes_full_external_uri(&test.server_url, &identity); } #[test] fn cpp_handle_request_body() { - let (test, identity) = cpp_http_test( - "http-routes-cpp-request-body", - "http-routes-cpp-request-body", - CPP_HANDLE_REQUEST_BODY_MODULE_CODE, - ); + let (test, identity) = cpp_http_test("http-routes-cpp-request-body", CPP_HANDLE_REQUEST_BODY_MODULE_CODE); assert_handle_request_body(&test.server_url, &identity); } /// Validates the Rust example from `docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md`. #[test] fn http_handlers_tutorial_say_hello_route_works() { - let module_code = extract_rust_code_blocks( + let module_code = extract_code_blocks( &workspace_root().join("docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md"), + r"```rust\n([\s\S]*?)\n```", + "rust", ); let test = Smoketest::builder().module_code(&module_code).build(); let identity = test.database_identity.as_ref().expect("database identity missing"); @@ -978,8 +946,10 @@ fn http_handlers_tutorial_say_hello_route_works() { fn cpp_http_handlers_tutorial_say_hello_route_works() { require_emscripten!(); - let module_code = extract_cpp_code_blocks( + let module_code = extract_code_blocks( &workspace_root().join("docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md"), + r"```(?:cpp|c\+\+)\n([\s\S]*?)\n```", + "cpp", ); let mut test = Smoketest::builder().autopublish(false).build(); let identity = test From 8600c3bc466e5f6d7cee3416453845eb16b7c2e9 Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Thu, 14 May 2026 14:37:26 -0700 Subject: [PATCH 05/20] Lint repair and update from missing autogen --- .../bindings-cpp/include/spacetimedb/http.h | 6 +- .../include/spacetimedb/http_convert.h | 5 ++ .../internal/autogen/HttpMethod.g.h | 75 +++++++++++++++++++ .../internal/autogen/MethodOrAny.g.h | 12 +-- .../bindings-cpp/include/spacetimedb/router.h | 62 +++++++++++++-- crates/smoketests/src/lib.rs | 3 +- .../tests/smoketests/http_routes.rs | 15 ++-- 7 files changed, 148 insertions(+), 30 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/HttpMethod.g.h diff --git a/crates/bindings-cpp/include/spacetimedb/http.h b/crates/bindings-cpp/include/spacetimedb/http.h index f24c76a8128..82a55beeb32 100644 --- a/crates/bindings-cpp/include/spacetimedb/http.h +++ b/crates/bindings-cpp/include/spacetimedb/http.h @@ -316,8 +316,10 @@ class HttpClient { } // namespace SpacetimeDB -// Include implementation after class definition to avoid circular dependencies -#ifdef SPACETIMEDB_UNSTABLE_FEATURES +// Include implementation dependencies after class definition to avoid circular dependencies +#if defined(SPACETIMEDB_UNSTABLE_FEATURES) && !defined(SPACETIMEDB_HTTP_CONVERT_H) +#include "spacetimedb/logger.h" +#include "spacetimedb/http_convert.h" #include "spacetimedb/http_client_impl.h" #endif diff --git a/crates/bindings-cpp/include/spacetimedb/http_convert.h b/crates/bindings-cpp/include/spacetimedb/http_convert.h index 7c02a1fb021..e514f4b864e 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_convert.h +++ b/crates/bindings-cpp/include/spacetimedb/http_convert.h @@ -281,4 +281,9 @@ inline std::pair> to_wire_split(const H } // namespace convert } // namespace SpacetimeDB +#ifdef SPACETIMEDB_UNSTABLE_FEATURES +#include "spacetimedb/logger.h" +#include "spacetimedb/http_client_impl.h" +#endif + #endif // SPACETIMEDB_HTTP_CONVERT_H diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/HttpMethod.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/HttpMethod.g.h new file mode 100644 index 00000000000..bb53d566b39 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/HttpMethod.g.h @@ -0,0 +1,75 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Head_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Post_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Put_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Delete_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Connect_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Options_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Trace_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Patch_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_TAGGED_ENUM(HttpMethod, std::monostate, HttpMethod_Head_Wrapper, HttpMethod_Post_Wrapper, HttpMethod_Put_Wrapper, HttpMethod_Delete_Wrapper, HttpMethod_Connect_Wrapper, HttpMethod_Options_Wrapper, HttpMethod_Trace_Wrapper, HttpMethod_Patch_Wrapper, std::string) +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h index f266d538c3d..347702f1a61 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h @@ -12,17 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "spacetimedb/http_wire.h" +#include "HttpMethod.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_PRODUCT_TYPE(MethodOrAny_Method_Wrapper) { - SpacetimeDB::wire::HttpMethod value; - void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { - ::SpacetimeDB::bsatn::serialize(writer, value); - } - SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) -}; - -SPACETIMEDB_INTERNAL_TAGGED_ENUM(MethodOrAny, std::monostate, MethodOrAny_Method_Wrapper) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(MethodOrAny, std::monostate, SpacetimeDB::Internal::HttpMethod) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/router.h b/crates/bindings-cpp/include/spacetimedb/router.h index 8c3f3b5191c..e3ee6ca8962 100644 --- a/crates/bindings-cpp/include/spacetimedb/router.h +++ b/crates/bindings-cpp/include/spacetimedb/router.h @@ -6,7 +6,6 @@ #endif #include -#include #include #include #include @@ -141,9 +140,34 @@ class Router { if (a.method.is<0>() || b.method.is<0>()) { return true; } - const auto& a_method = a.method.template get<1>().value; - const auto& b_method = b.method.template get<1>().value; - return a_method == b_method; + return method_key(a.method.template get<1>()) == method_key(b.method.template get<1>()); + } + + static std::string method_key(const Internal::HttpMethod& method) { + switch (method.get_tag()) { + case 0: + return "GET"; + case 1: + return "HEAD"; + case 2: + return "POST"; + case 3: + return "PUT"; + case 4: + return "DELETE"; + case 5: + return "CONNECT"; + case 6: + return "OPTIONS"; + case 7: + return "TRACE"; + case 8: + return "PATCH"; + case 9: + return method.template get<9>(); + default: + fail_router_registration("Unsupported internal HTTP method tag"); + } } static Internal::MethodOrAny make_any() { @@ -154,9 +178,33 @@ class Router { static Internal::MethodOrAny make_method(const HttpMethod& method) { Internal::MethodOrAny result; - Internal::MethodOrAny_Method_Wrapper wrapper; - wrapper.value = convert::to_wire(method); - result.set<1>(wrapper); + result.set<1>(to_internal_http_method(method)); + return result; + } + + static Internal::HttpMethod to_internal_http_method(const HttpMethod& method) { + Internal::HttpMethod result; + if (method.value == "GET") { + result.set<0>(std::monostate{}); + } else if (method.value == "HEAD") { + result.set<1>(Internal::HttpMethod_Head_Wrapper{}); + } else if (method.value == "POST") { + result.set<2>(Internal::HttpMethod_Post_Wrapper{}); + } else if (method.value == "PUT") { + result.set<3>(Internal::HttpMethod_Put_Wrapper{}); + } else if (method.value == "DELETE") { + result.set<4>(Internal::HttpMethod_Delete_Wrapper{}); + } else if (method.value == "CONNECT") { + result.set<5>(Internal::HttpMethod_Connect_Wrapper{}); + } else if (method.value == "OPTIONS") { + result.set<6>(Internal::HttpMethod_Options_Wrapper{}); + } else if (method.value == "TRACE") { + result.set<7>(Internal::HttpMethod_Trace_Wrapper{}); + } else if (method.value == "PATCH") { + result.set<8>(Internal::HttpMethod_Patch_Wrapper{}); + } else { + result.set<9>(method.value); + } return result; } diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 088b51611db..93746dfbc4e 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -1013,8 +1013,7 @@ impl Smoketest { .replace('\\', "/"); let cmakelists = CPP_SMOKETEST_CMAKELISTS.replace("@SPACETIMEDB_CPP_LIBRARY_PATH@", &bindings_cpp_path); - fs::write(module_path.join("CMakeLists.txt"), cmakelists) - .context("Failed to write C++ CMakeLists.txt")?; + fs::write(module_path.join("CMakeLists.txt"), cmakelists).context("Failed to write C++ CMakeLists.txt")?; fs::write(src_dir.join("lib.cpp"), module_source).context("Failed to write C++ module code")?; let module_path_str = module_path.to_str().context("Invalid C++ module path")?; diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/smoketests/http_routes.rs index 3efcda47301..1e5e7b5d270 100644 --- a/crates/smoketests/tests/smoketests/http_routes.rs +++ b/crates/smoketests/tests/smoketests/http_routes.rs @@ -622,9 +622,7 @@ fn rust_http_test(module_code: &str) -> (Smoketest, String) { fn cpp_http_test(name: &str, module_code: &str) -> (Smoketest, String) { require_emscripten!(); let mut test = Smoketest::builder().autopublish(false).build(); - let identity = test - .publish_cpp_module_source(name, name, module_code) - .unwrap(); + let identity = test.publish_cpp_module_source(name, name, module_code).unwrap(); (test, identity) } @@ -900,7 +898,10 @@ fn cpp_http_routes_pr_example_round_trip() { #[test] fn cpp_http_routes_are_strict_for_non_root_paths() { - let (test, identity) = cpp_http_test("http-routes-cpp-strict-non-root", CPP_STRICT_NON_ROOT_ROUTING_MODULE_CODE); + let (test, identity) = cpp_http_test( + "http-routes-cpp-strict-non-root", + CPP_STRICT_NON_ROOT_ROUTING_MODULE_CODE, + ); assert_http_routes_are_strict_for_non_root_paths(&test.server_url, &identity); } @@ -953,11 +954,7 @@ fn cpp_http_handlers_tutorial_say_hello_route_works() { ); let mut test = Smoketest::builder().autopublish(false).build(); let identity = test - .publish_cpp_module_source( - "http-handlers-docs-cpp", - "http-handlers-docs-cpp", - &module_code, - ) + .publish_cpp_module_source("http-handlers-docs-cpp", "http-handlers-docs-cpp", &module_code) .unwrap(); let url = format!("{}/v1/database/{identity}/route/say-hello", test.server_url); From 2c705f6a2e8feaf7234914a0195239ef7ddfab57 Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Fri, 15 May 2026 11:32:57 -0700 Subject: [PATCH 06/20] Clean up of overuse of (void) --- .../error_http_handler_immutable_ctx.cpp | 2 -- .../error_http_handler_no_connection_id.cpp | 1 - .../error_http_handler_no_db.cpp | 1 - .../error_http_handler_no_request_arg.cpp | 1 - .../error_http_handler_no_return_type.cpp | 2 -- .../error_http_handler_no_sender.cpp | 1 - .../error_http_handler_wrong_ctx.cpp | 2 -- ...or_http_handler_wrong_request_arg_type.cpp | 2 -- .../error_http_handler_wrong_return_type.cpp | 2 -- .../error_http_router_with_args.cpp | 3 -- .../http-handlers/ok_http_handlers_basic.cpp | 2 -- .../tests/smoketests/http_routes.rs | 28 ------------------- 12 files changed, 47 deletions(-) diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp index 1832f28496a..acb7d2f731a 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp @@ -3,8 +3,6 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(handler_immutable_ctx, const HandlerContext& ctx, HttpRequest request) { - (void)ctx; - (void)request; return HttpResponse{ 200, HttpVersion::Http11, diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp index cbebae53852..c775e6c3430 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp @@ -3,7 +3,6 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(handler_no_connection_id, HandlerContext ctx, HttpRequest request) { - (void)request; auto conn_id = ctx.connection_id(); return HttpResponse{ 200, diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp index 707b2183be0..4f650b55e71 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp @@ -9,7 +9,6 @@ SPACETIMEDB_STRUCT(TestRow, value) SPACETIMEDB_TABLE(TestRow, test_row, Public) SPACETIMEDB_HTTP_HANDLER(handler_no_db, HandlerContext ctx, HttpRequest request) { - (void)request; auto count = ctx.db[test_row].count(); return HttpResponse{ 200, diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp index ac2eebad2d2..4543a97ef8c 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp @@ -3,7 +3,6 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(handler_no_request_arg, HandlerContext ctx) { - (void)ctx; return HttpResponse{ 200, HttpVersion::Http11, diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp index cbc5cd4f29f..ea22473d38f 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp @@ -7,6 +7,4 @@ using namespace SpacetimeDB; #endif SPACETIMEDB_HTTP_HANDLER(handler_no_return_type, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; } diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp index 6d84d5e4af1..9f4ba51b5d0 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp @@ -3,7 +3,6 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(handler_no_sender, HandlerContext ctx, HttpRequest request) { - (void)request; auto sender = ctx.sender(); return HttpResponse{ 200, diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp index f2637cfc53c..01893c6d278 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp @@ -3,8 +3,6 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(handler_wrong_ctx, ProcedureContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return HttpResponse{ 200, HttpVersion::Http11, diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp index 5c06d017dba..6ee189776b6 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp @@ -3,8 +3,6 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(handler_wrong_request_arg_type, HandlerContext ctx, uint32_t request) { - (void)ctx; - (void)request; return HttpResponse{ 200, HttpVersion::Http11, diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp index ca22aedef90..b04c6bfaa3e 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp @@ -3,7 +3,5 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(handler_wrong_return_type, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return 7u; } diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp index 173b0b7deca..c4f7fb6a5d3 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp @@ -3,8 +3,6 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(hello_handler, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return HttpResponse{ 200, HttpVersion::Http11, @@ -14,6 +12,5 @@ SPACETIMEDB_HTTP_HANDLER(hello_handler, HandlerContext ctx, HttpRequest request) } SPACETIMEDB_HTTP_ROUTER(register_http_routes, HandlerContext ctx) { - (void)ctx; return Router().get("/hello", hello_handler); } diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp index 2bcdd546426..e482c0561df 100644 --- a/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp @@ -3,8 +3,6 @@ using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(hello_handler, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return HttpResponse{ 200, HttpVersion::Http11, diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/smoketests/http_routes.rs index 1e5e7b5d270..a7ce4ec4bf9 100644 --- a/crates/smoketests/tests/smoketests/http_routes.rs +++ b/crates/smoketests/tests/smoketests/http_routes.rs @@ -264,13 +264,10 @@ HttpResponse text_response(uint16_t status_code, std::string body) { } // namespace SPACETIMEDB_HTTP_HANDLER(get_simple, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response(200, "ok"); } SPACETIMEDB_HTTP_HANDLER(post_insert, HandlerContext ctx, HttpRequest request) { - (void)request; ctx.with_tx([](TxContext& tx) { uint64_t id = tx.db[entry].count(); tx.db[entry].insert(Entry{ id, "posted" }); @@ -279,7 +276,6 @@ SPACETIMEDB_HTTP_HANDLER(post_insert, HandlerContext ctx, HttpRequest request) { } SPACETIMEDB_HTTP_HANDLER(get_count, HandlerContext ctx, HttpRequest request) { - (void)request; uint64_t count = ctx.with_tx([](TxContext& tx) -> uint64_t { return tx.db[entry].count(); }); @@ -287,19 +283,14 @@ SPACETIMEDB_HTTP_HANDLER(get_count, HandlerContext ctx, HttpRequest request) { } SPACETIMEDB_HTTP_HANDLER(any_handler, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response(200, "any"); } SPACETIMEDB_HTTP_HANDLER(header_echo, HandlerContext ctx, HttpRequest request) { - (void)ctx; return text_response(200, header_value_utf8(request, "x-echo")); } SPACETIMEDB_HTTP_HANDLER(set_response_header, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return HttpResponse{ 200, HttpVersion::Http11, @@ -309,14 +300,10 @@ SPACETIMEDB_HTTP_HANDLER(set_response_header, HandlerContext ctx, HttpRequest re } SPACETIMEDB_HTTP_HANDLER(body_handler, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response(200, "non-empty"); } SPACETIMEDB_HTTP_HANDLER(teapot, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response(418, "teapot"); } @@ -443,26 +430,18 @@ HttpResponse text_response(const std::string& body) { } // namespace SPACETIMEDB_HTTP_HANDLER(empty_root, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response("empty"); } SPACETIMEDB_HTTP_HANDLER(slash_root, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response("slash"); } SPACETIMEDB_HTTP_HANDLER(foo, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response("foo"); } SPACETIMEDB_HTTP_HANDLER(foo_slash, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response("foo-slash"); } @@ -488,14 +467,10 @@ HttpResponse text_response(const std::string& body) { } // namespace SPACETIMEDB_HTTP_HANDLER(foo, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response("foo"); } SPACETIMEDB_HTTP_HANDLER(foo_slash, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return text_response("foo-slash"); } @@ -511,7 +486,6 @@ const CPP_FULL_URI_MODULE_CODE: &str = r#"#include "spacetimedb.h" using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(echo_uri, HandlerContext ctx, HttpRequest request) { - (void)ctx; return HttpResponse{ 200, HttpVersion::Http11, @@ -543,14 +517,12 @@ HttpResponse text_response(uint16_t status_code, const std::string& body) { } // namespace SPACETIMEDB_HTTP_HANDLER(reverse_bytes, HandlerContext ctx, HttpRequest request) { - (void)ctx; std::vector reversed = request.body.to_bytes(); std::reverse(reversed.begin(), reversed.end()); return bytes_response(200, std::move(reversed)); } SPACETIMEDB_HTTP_HANDLER(reverse_words, HandlerContext ctx, HttpRequest request) { - (void)ctx; const std::vector bytes = request.body.to_bytes(); std::string body(bytes.begin(), bytes.end()); if (body.find(static_cast(0x80)) != std::string::npos) { From 3e25d8ddece05e89f87cef41302751606fb1cca7 Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Fri, 15 May 2026 12:17:56 -0700 Subject: [PATCH 07/20] Docs cleanup --- .../00200-core-concepts/00200-functions/00600-HTTP-handlers.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md b/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md index 657003f7d38..185d53be561 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md +++ b/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md @@ -86,8 +86,6 @@ The function must return a `SpacetimeDB::HttpResponse`. using namespace SpacetimeDB; SPACETIMEDB_HTTP_HANDLER(say_hello, HandlerContext ctx, HttpRequest request) { - (void)ctx; - (void)request; return HttpResponse{ 200, HttpVersion::Http11, From ca949e54b9a6197f1777fa3733be30d50909d969 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Mon, 18 May 2026 12:12:22 -0700 Subject: [PATCH 08/20] Do not `msync` the entire offset index file on every transaction (#5018) # Description of Changes `msync` the modified pages instead of the entire file. # API and ABI breaking changes None # Expected complexity level and risk 1 # Testing Refactor --- crates/commitlog/src/index/indexfile.rs | 19 +++++++++++++++++-- crates/commitlog/src/lib.rs | 2 +- crates/commitlog/src/segment.rs | 9 +++------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/crates/commitlog/src/index/indexfile.rs b/crates/commitlog/src/index/indexfile.rs index 080fac97472..338c3e4552a 100644 --- a/crates/commitlog/src/index/indexfile.rs +++ b/crates/commitlog/src/index/indexfile.rs @@ -161,7 +161,7 @@ impl + From> IndexFileMut { /// Errors /// - `IndexError::InvalidInput`: Either Key or Value is 0 /// - `IndexError::OutOfMemory`: Append after index file is already full. - pub fn append(&mut self, key: Key, value: u64) -> Result<(), IndexError> { + pub fn append(&mut self, key: Key, value: u64) -> Result { let key = key.into(); let last_key = self.last_key()?; if last_key >= key { @@ -179,7 +179,7 @@ impl + From> IndexFileMut { self.inner[start..start + KEY_SIZE].copy_from_slice(&key_bytes); self.inner[start + KEY_SIZE..start + ENTRY_SIZE].copy_from_slice(&value_bytes); self.num_entries += 1; - Ok(()) + Ok(start) } /// Asynchronously flushes any pending changes to the index file @@ -190,6 +190,21 @@ impl + From> IndexFileMut { self.inner.flush_async() } + /// Asynchronously flushes the index entry starting at `offset` to the index file. + /// + /// On linux, the underlying `msync` is a documented no-op since the kernel already + /// tracks dirty pages and flushes them as needed. + /// + /// See https://man7.org/linux/man-pages/man2/msync.2.html for details. + /// + /// On macOS, it is not a documented no-op, and it explicitly states that `msync` + /// will only examine pages covered by the provided address range. Hence this should + /// be preferred over [`Self::async_flush`] when only flushing a single entry at a + /// time, since it may avoid examining pages across the whole mapping. + pub fn async_flush_entry(&self, offset: usize) -> io::Result<()> { + self.inner.flush_async_range(offset, ENTRY_SIZE) + } + /// Truncates the index file starting from the entry with a key greater than /// or equal to the given key. /// diff --git a/crates/commitlog/src/lib.rs b/crates/commitlog/src/lib.rs index abc8729c978..0f0753f0f36 100644 --- a/crates/commitlog/src/lib.rs +++ b/crates/commitlog/src/lib.rs @@ -109,7 +109,7 @@ impl Default for Options { impl Options { pub const DEFAULT_MAX_SEGMENT_SIZE: u64 = 1024 * 1024 * 1024; pub const DEFAULT_OFFSET_INDEX_INTERVAL_BYTES: NonZeroU64 = NonZeroU64::new(4096).expect("4096 > 0, qed"); - pub const DEFAULT_OFFSET_INDEX_REQUIRE_SEGMENT_FSYNC: bool = false; + pub const DEFAULT_OFFSET_INDEX_REQUIRE_SEGMENT_FSYNC: bool = true; pub const DEFAULT_PREALLOCATE_SEGMENTS: bool = false; pub const DEFAULT_WRITE_BUFFER_SIZE: usize = 8 * 1024; diff --git a/crates/commitlog/src/segment.rs b/crates/commitlog/src/segment.rs index 7913a3799e5..d4f85bf8fa0 100644 --- a/crates/commitlog/src/segment.rs +++ b/crates/commitlog/src/segment.rs @@ -356,9 +356,10 @@ impl OffsetIndexWriter { return Ok(()); } - self.head + let entry_offset = self + .head .append(self.candidate_min_tx_offset, self.candidate_byte_offset)?; - self.head.async_flush()?; + self.head.async_flush_entry(entry_offset)?; self.reset(); Ok(()) @@ -371,10 +372,6 @@ impl FileLike for OffsetIndexWriter { let _ = self.append_internal().map_err(|e| { warn!("failed to append to offset index: {e:?}"); }); - let _ = self - .head - .async_flush() - .map_err(|e| warn!("failed to flush offset index: {e:?}")); Ok(()) } From a9fd5e6da1e384ced68633aeb7275ff7dcbfc859 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Mon, 18 May 2026 14:56:15 -0700 Subject: [PATCH 09/20] Pipeline the websocket send path (#5051) # Description of Changes In keeping with the "pipeline everything" ethos, I've replaced `recv` with `recv_many` on `ClientConnectionReceiver` so that a client connection's websocket send path works on a batch of messages at a time, if possible. # API and ABI breaking changes None # Expected complexity level and risk 1.5 # Testing Refactor --- crates/client-api/src/routes/subscribe.rs | 27 ++-- crates/core/src/client/client_connection.rs | 134 +++++++++++++------- crates/testing/src/modules.rs | 3 +- 3 files changed, 103 insertions(+), 61 deletions(-) diff --git a/crates/client-api/src/routes/subscribe.rs b/crates/client-api/src/routes/subscribe.rs index e2131abfa31..ce7c127be98 100644 --- a/crates/client-api/src/routes/subscribe.rs +++ b/crates/client-api/src/routes/subscribe.rs @@ -1064,13 +1064,13 @@ enum UnorderedWsMessage { /// Abstraction over [`ClientConnectionReceiver`], so tests can use a plain /// [`mpsc::Receiver`]. trait Receiver { - fn recv(&mut self) -> impl Future> + Send; + fn recv_many(&mut self, buf: &mut Vec, max: usize) -> impl Future + Send; fn close(&mut self); } impl Receiver for ClientConnectionReceiver { - async fn recv(&mut self) -> Option { - ClientConnectionReceiver::recv(self).await + async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + ClientConnectionReceiver::recv_many(self, buf, max).await } fn close(&mut self) { @@ -1079,8 +1079,8 @@ impl Receiver for ClientConnectionReceiver { } impl Receiver for mpsc::Receiver { - async fn recv(&mut self) -> Option { - mpsc::Receiver::recv(self).await + async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + mpsc::Receiver::recv_many(self, buf, max).await } fn close(&mut self) { @@ -1148,6 +1148,8 @@ async fn ws_send_loop_inner( // The default frame size is 4KiB, hence we write in batches of 32KiB. const FRAME_BATCH_SIZE: usize = 8; let mut frames_batch = Vec::with_capacity(FRAME_BATCH_SIZE); + const MESSAGE_BATCH_SIZE: usize = ClientConnectionReceiver::DEFAULT_RECV_MANY_LIMIT; + let mut message_batch = Vec::new(); let (frames_tx, mut frames_rx) = mpsc::unbounded_channel(); let (encode_tx, encode_rx) = mpsc::unbounded_channel(); @@ -1262,12 +1264,15 @@ async fn ws_send_loop_inner( // Take on more work. // // Branch is disabled if we already sent a close frame. - Some(message) = messages.recv(), if !closed => { - encode_tx - .send(message.into()) - // `ws_encode_task` shouldn't terminate until - // `encode_tx` is dropped, except by panicking. - .expect("encode task panicked"); + n = messages.recv_many(&mut message_batch, MESSAGE_BATCH_SIZE), if !closed => { + log::trace!("encoding batch of {n} messages"); + for message in message_batch.drain(..n) { + encode_tx + .send(message.into()) + // `ws_encode_task` shouldn't terminate until + // `encode_tx` is dropped, except by panicking. + .expect("encode task panicked"); + } }, } diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 0a3fa198b01..8f19156d6bb 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -19,6 +19,7 @@ use bytes::Bytes; use bytestring::ByteString; use derive_more::From; use futures::prelude::*; +use log::warn; use prometheus::{Histogram, IntCounter, IntGauge}; use spacetimedb_auth::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2}; @@ -29,7 +30,7 @@ use spacetimedb_lib::Identity; use tokio::sync::mpsc::error::{SendError, TrySendError}; use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::AbortHandle; -use tracing::{trace, warn}; +use tracing::trace; #[derive(PartialEq, Eq, Clone, Copy, Hash, Debug)] pub enum Protocol { @@ -154,11 +155,13 @@ impl DurableOffsetSupply for Arc { pub struct ClientConnectionReceiver { confirmed_reads: bool, channel: MeteredReceiver, - current: Option, + pending: Vec, offset_supply: Box, } impl ClientConnectionReceiver { + pub const DEFAULT_RECV_MANY_LIMIT: usize = 4096; + fn new( confirmed_reads: bool, channel: MeteredReceiver, @@ -167,82 +170,115 @@ impl ClientConnectionReceiver { Self { confirmed_reads, channel, - current: None, + pending: Vec::new(), offset_supply: Box::new(offset_supply), } } - /// Receive the next message from this channel. - /// - /// If this method returns `None`, the channel is closed and no more messages - /// are in the internal buffers. No more messages can ever be received from - /// the channel. + #[cfg(test)] + pub(crate) async fn recv(&mut self) -> Option { + let mut buf = Vec::with_capacity(1); + (self.recv_many(&mut buf, 1).await != 0).then(|| buf.remove(0)) + } + + /// Receive multiple messages from this channel. /// /// Messages are returned immediately if: /// - /// - The (internal) [`ClientUpdate`] does not have a `tx_offset` + /// - The [`ClientUpdate`] does not have a `tx_offset` /// (such as for error messages). - /// - The client hasn't requested confirmed reads (i.e. - /// [`ClientConfig::confirmed_reads`] is `false`). + /// - The client hasn't requested confirmed reads + /// (i.e. [`ClientConfig::confirmed_reads`] is `false`). /// - The database is configured to not persist transactions. /// - /// Otherwise, the update's `tx_offset` is compared against the module's + /// Otherwise, the last `tx_offset` in the batch is compared against the module's /// durable offset. If the durable offset is behind the `tx_offset`, the /// method waits until it catches up before returning the message. /// /// If the database is shut down while waiting for the durable offset, - /// `None` is returned. In this case, no more messages can ever be received + /// 0 is returned. In this case, no more messages can ever be received /// from the channel. /// + /// For non-zero values of `max`, this method will never return `0` unless the + /// input channel has been closed and there are no pending messages, or if the + /// database goes away. This indicates that no further values can ever be received + /// from this `Receiver`. + /// /// # Cancel safety /// /// This method is cancel safe, as long as `self` is not dropped. /// - /// If `recv` is used in a [`tokio::select!`] statement, it may get + /// If `recv_many` is used in a [`tokio::select!`] statement, it may get /// cancelled while waiting for the durable offset to catch up. At this - /// point, it has already received a value from the underlying channel. - /// This value is stored internally, so calling `recv` again will not lose - /// data. - // - // TODO: Can we make a cancel-safe `recv_many` with confirmed reads semantics? - pub async fn recv(&mut self) -> Option { - let ClientUpdate { tx_offset, message } = match self.current.take() { - None => self.channel.recv().await?, - Some(update) => update, - }; + /// point, it has already received values from the underlying channel. + /// These values are stored internally, so calling `recv_many` again will + /// not lose data. + pub async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + // If there are no pending updates and the input channel has been closed, + // no more messages can be received from this receiver. + if max == 0 || (self.pending.is_empty() && self.channel.recv_many(&mut self.pending, max).await == 0) { + return 0; + } + + // If we don't have to wait for txns to be made durable, + // drain the pending updates. if !self.confirmed_reads { - return Some(message); + return self.drain_pending(buf, max); + } + + // If we do have to wait for txns to be made durable, + // but the next client update doesn't have a tx offset, + // there's no reason to wait - just send it. + if !self.pending_update_has_offset() { + return self.drain_pending(buf, 1); } - if let Some(tx_offset) = tx_offset { - match self.offset_supply.durable_offset() { - Ok(Some(mut durable)) => { - // Store the current update in case we get cancelled while - // waiting for the durable offset. - self.current = Some(ClientUpdate { - tx_offset: Some(tx_offset), - message, - }); - trace!("waiting for offset {tx_offset} to become durable"); - durable - .wait_for(tx_offset) - .await - .inspect_err(|_| { - warn!("database went away while waiting for durable offset"); - }) - .ok()?; - self.current.take().map(|update| update.message) + // Otherwise, grab the next offset that we should wait for. + let (n, wait_for_offset) = self.next_confirmed_reads_batch(max); + + match self.offset_supply.durable_offset() { + Ok(Some(mut durable)) => { + trace!("waiting for offset {wait_for_offset} to become durable"); + if durable.wait_for(wait_for_offset).await.is_err() { + warn!("database went away while waiting for durable offset"); + return 0; } - // Database shut down or crashed. - Err(NoSuchModule) => None, - // In-memory database. - Ok(None) => Some(message), + self.drain_pending(buf, n) } - } else { - Some(message) + // Database shut down or crashed. + Err(NoSuchModule) => 0, + // In-memory database. + Ok(None) => self.drain_pending(buf, max), } } + /// Compute the next batch of pending client updates that have a tx offset. + /// What is the size of the batch and what is the max offset? + fn next_confirmed_reads_batch(&self, max: usize) -> (usize, TxOffset) { + self.pending + .iter() + .take(max) + .map_while(|update| update.tx_offset) + .fold((0, 0), |(count, max_offset), tx_offset| { + (count + 1, max_offset.max(tx_offset)) + }) + } + + /// Drain the pending [`ClientUpdate`]s, up to `max, into `buf`. + fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + let n = self.pending.len().min(max); + buf.reserve(n); + buf.extend(self.pending.drain(..n).map(|u| u.message)); + n + } + + /// Does the next pending update have a tx offset? + /// + /// Assumes that [`Self::pending`] is not empty. + fn pending_update_has_offset(&self) -> bool { + self.pending.first().is_some_and(|update| update.tx_offset.is_some()) + } + /// Close the receiver without dropping it. /// /// This is used to notify the [`ClientConnectionSender`] that the receiver diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 750d9ca7606..7556b7f692e 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -101,7 +101,8 @@ impl ModuleHandle { } pub async fn recv_message(&mut self) -> Option { - self.receiver.recv().await + let mut buf = Vec::with_capacity(1); + (self.receiver.recv_many(&mut buf, 1).await != 0).then(|| buf.remove(0)) } pub async fn recv_reducer_update(&mut self, request_id: RequestId) -> anyhow::Result<()> { From 451fc78863c28433afb188aaf3334315d5c5f075 Mon Sep 17 00:00:00 2001 From: Jason Larabie Date: Mon, 18 May 2026 17:32:28 -0700 Subject: [PATCH 10/20] Fix view auto-migrate with canonical names (#4985) # Description of Changes Closes: https://github.com/clockworklabs/SpacetimeDB/issues/4842 Fixes the auto-migration for views where the exported name differs from the accessor/source name in the validation. The bug looked like the columns weren't matching and it turned out that the columns were keyed to the accessor name then attempted to match up against the canonical name. That caused the auto-migration to think the columns changed and triggered a remove/add for the view. # API and ABI breaking changes None intended # Expected complexity level and risk 2 - This is small and targeted but I'm not familiar with this part of the codebase # Testing - [x] Rebuilt the sample from the issue in TypeScript and confirmed the issue and the repair - [x] Built a mirror in Rust to confirm it was an issue for all languages - [x] Ran local tests --------- Co-authored-by: joshua-spacetime --- crates/schema/src/auto_migrate.rs | 57 +++++++++++++++++++++++++++ crates/schema/src/def/validate/v10.rs | 54 ++++++++++++++++++++++--- crates/schema/src/def/validate/v9.rs | 40 ++++++++++--------- 3 files changed, 127 insertions(+), 24 deletions(-) diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index 6d169a902cc..f2fba813fa8 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -1101,6 +1101,7 @@ mod tests { AlgebraicType, AlgebraicValue, ProductType, ScheduleAt, }; use spacetimedb_primitives::ColId; + use v10::{ExplicitNames, RawModuleDefV10Builder}; use v9::{RawModuleDefV9Builder, TableAccess}; use validate::tests::expect_identifier; @@ -1113,6 +1114,15 @@ mod tests { .expect("new_def should be a valid database definition") } + fn create_module_def_v10(build_module: impl Fn(&mut RawModuleDefV10Builder)) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + build_module(&mut builder); + builder + .finish() + .try_into() + .expect("new_def should be a valid database definition") + } + fn initial_module_def() -> ModuleDef { let mut builder = RawModuleDefV9Builder::new(); let schedule_at = builder.add_type::(); @@ -1982,6 +1992,53 @@ mod tests { } } + #[test] + fn migrate_view_with_explicit_name() { + fn module_def() -> ModuleDef { + create_module_def_v10(|builder| { + let return_type_ref = builder.add_algebraic_type( + [], + "Person", + AlgebraicType::product([("PersonId", AlgebraicType::U64)]), + true, + ); + builder.add_view( + "PersonAtLevel2", + 0, + true, + true, + ProductType::from([("Level", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(return_type_ref)), + ); + + let mut explicit = ExplicitNames::default(); + explicit.insert_function("PersonAtLevel2", "Level2Person"); + builder.add_explicit_names(explicit); + }) + } + + let old_def = module_def(); + let new_def = module_def(); + let level_2_person = expect_identifier("Level2Person"); + + let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::UpdateView(&level_2_person)), + "steps: {steps:?}" + ); + assert!( + !steps.contains(&AutoMigrateStep::AddView(&level_2_person)), + "steps: {steps:?}" + ); + assert!( + !steps.contains(&AutoMigrateStep::RemoveView(&level_2_person)), + "steps: {steps:?}" + ); + } + #[test] fn migrate_view_disconnect_clients() { struct TestCase { diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index c0ff11df333..7f2b4b99238 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -377,8 +377,13 @@ impl<'a> ModuleValidatorV10<'a> { }) })?; - let mut table_validator = - TableValidator::new(raw_table_name.clone(), product_type_ref, product_type, &mut self.core)?; + let mut table_validator = TableValidator::new( + raw_table_name.clone(), + product_type_ref, + product_type, + &mut self.core, + CoreValidator::resolve_table_ident, + )?; let table_ident = table_validator.table_ident.clone(); @@ -719,10 +724,12 @@ impl<'a> ModuleValidatorV10<'a> { }) })?; + let name = self.core.resolve_function_ident(accessor_name.clone())?; + let params_for_generate = self.core .params_for_generate(¶ms, |position, arg_name| TypeLocation::ViewArg { - view_name: accessor_name.clone(), + view_name: name.as_raw().clone(), position, arg_name, })?; @@ -736,13 +743,11 @@ impl<'a> ModuleValidatorV10<'a> { let return_type_for_generate = self.core.validate_for_type_use( || TypeLocation::ViewReturn { - view_name: accessor_name.clone(), + view_name: name.as_raw().clone(), }, &return_type_for_generate_input, ); - let name = self.core.resolve_function_ident(accessor_name.clone())?; - let mut view_validator = ViewValidator::new( accessor_name.clone(), product_type_ref, @@ -2170,4 +2175,41 @@ mod tests { assert_eq!(schedule.at_column, 1.into()); assert_eq!(schedule.function_kind, FunctionKind::Reducer); } + + #[test] + fn test_child_defs_use_explicit_view_name() { + use spacetimedb_lib::db::raw_def::v10::ExplicitNames; + + let id = |s: &str| Identifier::for_test(s); + + let mut builder = RawModuleDefV10Builder::new(); + let return_type_ref = builder.add_algebraic_type( + [], + "Person", + AlgebraicType::product([("PersonId", AlgebraicType::U64)]), + true, + ); + builder.add_view( + "PersonAtLevel2", + 0, + true, + true, + ProductType::from([("Level", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(return_type_ref)), + ); + + let mut explicit = ExplicitNames::default(); + explicit.insert_function("PersonAtLevel2", "Level2Person"); + builder.add_explicit_names(explicit); + + let def: ModuleDef = builder.finish().try_into().unwrap(); + let view = def + .view("Level2Person") + .expect("view should use explicit canonical name"); + + assert_eq!(view.name, id("Level2Person")); + assert_eq!(view.accessor_name, id("PersonAtLevel2")); + assert_eq!(view.return_columns[0].view_name, id("Level2Person")); + assert_eq!(view.param_columns[0].view_name, id("Level2Person")); + } } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 5daf738a4c0..289b4a991f5 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -202,8 +202,13 @@ impl ModuleValidatorV9<'_> { }) })?; - let mut table_in_progress = - TableValidator::new(raw_table_name.clone(), product_type_ref, product_type, &mut self.core)?; + let mut table_in_progress = TableValidator::new( + raw_table_name.clone(), + product_type_ref, + product_type, + &mut self.core, + CoreValidator::resolve_table_ident, + )?; let table_ident = table_in_progress.table_ident.clone(); @@ -929,6 +934,7 @@ impl CoreValidator<'_> { /// 2. Insert view names into the global namespace. pub(crate) struct ViewValidator<'a, 'b> { inner: TableValidator<'a, 'b>, + view_name: Identifier, params: &'a ProductType, params_for_generate: &'a [(Identifier, AlgebraicTypeUse)], } @@ -942,8 +948,12 @@ impl<'a, 'b> ViewValidator<'a, 'b> { params_for_generate: &'a [(Identifier, AlgebraicTypeUse)], module_validator: &'a mut CoreValidator<'b>, ) -> Result { + let view_name = module_validator.resolve_function_ident(raw_name.clone())?; Ok(Self { - inner: TableValidator::new(raw_name, product_type_ref, product_type, module_validator)?, + inner: TableValidator::new(raw_name, product_type_ref, product_type, module_validator, |_, _| { + Ok(view_name.clone()) + })?, + view_name, params, params_for_generate, }) @@ -968,25 +978,14 @@ impl<'a, 'b> ViewValidator<'a, 'b> { .unwrap_or_else(|| RawIdentifier::new(format!("param_{}", col_id))), ); - // This error will be created multiple times if the view name is invalid, - // but we sort and deduplicate the error stream afterwards, - // so it isn't a huge deal. - // - // This is necessary because we require `ErrorStream` to be nonempty. - // We need to put something in there if the view name is invalid. - let view_name = self - .inner - .module_validator - .resolve_identifier_with_case(self.inner.raw_name.clone()); - - let (name, view_name) = (name, view_name).combine_errors()?; + let name = name?; Ok(ViewParamDef { name, ty: column.algebraic_type.clone(), ty_for_generate: ty_for_generate.clone(), col_id, - view_name, + view_name: self.view_name.clone(), }) } @@ -1005,7 +1004,11 @@ impl<'a, 'b> ViewValidator<'a, 'b> { } } -/// A partially validated table. +/// A partially validated table-shaped definition. +/// +/// This is also used by [`ViewValidator`]. Tables and views do not resolve +/// their source names in the same namespace, so callers provide the +/// appropriate name resolver. pub(crate) struct TableValidator<'a, 'b> { pub(crate) module_validator: &'a mut CoreValidator<'b>, raw_name: RawIdentifier, @@ -1021,8 +1024,9 @@ impl<'a, 'b> TableValidator<'a, 'b> { product_type_ref: AlgebraicTypeRef, product_type: &'a ProductType, module_validator: &'a mut CoreValidator<'b>, + resolve_name: impl FnOnce(&CoreValidator<'b>, RawIdentifier) -> Result, ) -> Result { - let table_ident = module_validator.resolve_table_ident(raw_name.clone())?; + let table_ident = resolve_name(module_validator, raw_name.clone())?; Ok(Self { raw_name, product_type_ref, From 868f65f2707fd5b2c2cffd1940cc0bf787af09c2 Mon Sep 17 00:00:00 2001 From: Kilian Strunz <93079615+kistz@users.noreply.github.com> Date: Tue, 19 May 2026 10:01:13 +0200 Subject: [PATCH 11/20] Add vue `useProcedure` hook (#4999) # Description of Changes Tin. Just mechanically mirrors the `useReducer` hook just like it is for React Would be cool if this could get in #4998 # API and ABI breaking changes None. # Expected complexity level and risk 1 # Testing - [x] Works in my project --- crates/bindings-typescript/src/vue/index.ts | 1 + .../src/vue/useProcedure.ts | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 crates/bindings-typescript/src/vue/useProcedure.ts diff --git a/crates/bindings-typescript/src/vue/index.ts b/crates/bindings-typescript/src/vue/index.ts index b32fbdf4b80..3b3e64aba48 100644 --- a/crates/bindings-typescript/src/vue/index.ts +++ b/crates/bindings-typescript/src/vue/index.ts @@ -2,3 +2,4 @@ export * from './SpacetimeDBProvider.ts'; export { useSpacetimeDB } from './useSpacetimeDB.ts'; export { useTable } from './useTable.ts'; export { useReducer } from './useReducer.ts'; +export { useProcedure } from './useProcedure.ts'; diff --git a/crates/bindings-typescript/src/vue/useProcedure.ts b/crates/bindings-typescript/src/vue/useProcedure.ts new file mode 100644 index 00000000000..385b0c96ae8 --- /dev/null +++ b/crates/bindings-typescript/src/vue/useProcedure.ts @@ -0,0 +1,62 @@ +import { shallowRef, watch, onUnmounted } from 'vue'; +import { useSpacetimeDB } from './useSpacetimeDB'; +import type { UntypedProcedureDef } from '../sdk/procedures'; +import type { + ProcedureParamsType, + ProcedureReturnType, +} from '../sdk/type_utils'; + +export function useProcedure( + procedureDef: ProcedureDef +): ( + ...params: ProcedureParamsType +) => Promise> { + const conn = useSpacetimeDB(); + const procedureName = procedureDef.accessorName; + + const queueRef = shallowRef< + { + params: ProcedureParamsType; + resolve: (val: any) => void; + reject: (err: unknown) => void; + }[] + >([]); + + const stopWatch = watch( + () => conn.isActive, + () => { + const connection = conn.getConnection(); + if (!connection) return; + + const fn = (connection.procedures as any)[procedureName] as ( + ...p: ProcedureParamsType + ) => Promise>; + if (queueRef.value.length) { + const pending = queueRef.value.splice(0); + for (const item of pending) { + fn(...item.params).then(item.resolve, item.reject); + } + } + }, + { immediate: true } + ); + + onUnmounted(() => { + stopWatch(); + }); + + return (...params: ProcedureParamsType) => { + const connection = conn.getConnection(); + if (!connection) { + return new Promise>( + (resolve, reject) => { + queueRef.value.push({ params, resolve, reject }); + } + ); + } + const fn = (connection.procedures as any)[procedureName] as ( + ...p: ProcedureParamsType + ) => Promise>; + return fn(...params); + }; +} From 8641c17e4e9bbe45f4b37dac3acf3f713198e612 Mon Sep 17 00:00:00 2001 From: Zeke Foppa <196249+bfops@users.noreply.github.com> Date: Wed, 20 May 2026 11:57:20 -0700 Subject: [PATCH 12/20] Client binaries from DigitalOcean -> AWS (#5077) # Description of Changes Use an AWS bucket for client binaries instead of DigitalOcean. Note that this will effectively hamstring the DigitalOcean mirror for any old clients (i.e. ones before this change is released). But it's only a mirror, and they can "fix" it by upgrading to a version with this change. # API and ABI breaking changes None # Expected complexity level and risk 1 # Testing - [x] Manually ran the package job on this PR and confirmed that some of the resulting URLs are indeed downloadable from the AWS urls --------- Co-authored-by: Zeke Foppa --- .github/workflows/attach-artifacts.yml | 4 ++-- .github/workflows/benchmarks.yml | 9 ++++++--- .github/workflows/package.yml | 3 +-- crates/update/spacetime-install.ps1 | 2 +- crates/update/spacetime-install.sh | 2 +- crates/update/src/cli/install.rs | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/attach-artifacts.yml b/.github/workflows/attach-artifacts.yml index 23ad34343de..90a331387fe 100644 --- a/.github/workflows/attach-artifacts.yml +++ b/.github/workflows/attach-artifacts.yml @@ -17,10 +17,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Download artifacts from private base URL + - name: Download artifacts from AWS base URL env: RELEASE_TAG: ${{ github.event.inputs.release_tag }} - BASE_URL: ${{ secrets.ARTIFACT_BASE_URL }} + BASE_URL: https://${{ vars.AWS_BUCKET }}.s3.amazonaws.com/refs/tags run: | set -euo pipefail diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 2785a84d95a..1a934842bfb 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -2,7 +2,6 @@ on: push: branches: - master - - jgilles/fix-callgrind-again workflow_dispatch: inputs: @@ -24,8 +23,10 @@ jobs: benchmark: name: run criterion benchmarks runs-on: benchmarks-runner + # disable until we fix the benchmarks + if: false # filter for a comment containing 'benchmarks please' - if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} + #if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} env: PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }} steps: @@ -185,8 +186,10 @@ jobs: container: image: rust:1.93.0 options: --privileged + # disable until we fix the benchmarks + if: false # filter for a comment containing 'benchmarks please' - if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} + #if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} env: PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }} steps: diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 38632014506..1aa47cb0eae 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -142,12 +142,11 @@ jobs: run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT id: extract_branch - - name: Upload to DO Spaces + - name: Upload to AWS S3 uses: shallwefootball/s3-upload-action@master with: aws_key_id: ${{ secrets.AWS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} aws_bucket: ${{ vars.AWS_BUCKET }} source_dir: build - endpoint: https://nyc3.digitaloceanspaces.com destination_dir: ${{ steps.extract_branch.outputs.branch }} diff --git a/crates/update/spacetime-install.ps1 b/crates/update/spacetime-install.ps1 index c5f9ecf6c14..2ade395ca25 100644 --- a/crates/update/spacetime-install.ps1 +++ b/crates/update/spacetime-install.ps1 @@ -32,7 +32,7 @@ function Install { $AssetName = "spacetimedb-update-x86_64-pc-windows-msvc.exe" $DownloadUrl = "https://github.com/clockworklabs/SpacetimeDB/releases/latest/download/$AssetName" - $MirrorBase = "https://spacetimedb-client-binaries.nyc3.digitaloceanspaces.com" + $MirrorBase = "https://spacetimedb-client-binaries.s3.amazonaws.com" Write-Output "Downloading installer..." function UpdatePathIfNotExists { diff --git a/crates/update/spacetime-install.sh b/crates/update/spacetime-install.sh index 47d3bc60739..d171a0f5467 100644 --- a/crates/update/spacetime-install.sh +++ b/crates/update/spacetime-install.sh @@ -65,7 +65,7 @@ main() { # Define the latest SpacetimeDB download url local _asset_name="spacetimedb-update-$_host$_ext" local _url="$SPACETIME_DOWNLOAD_ROOT/$_asset_name" - local _mirror_base="https://spacetimedb-client-binaries.nyc3.digitaloceanspaces.com" + local _mirror_base="https://spacetimedb-client-binaries.s3.amazonaws.com" echo "Downloading installer..." local _ok=false if [ "$_downloader" = curl ]; then diff --git a/crates/update/src/cli/install.rs b/crates/update/src/cli/install.rs index 3a75ce84c53..c2cadb5bb3d 100644 --- a/crates/update/src/cli/install.rs +++ b/crates/update/src/cli/install.rs @@ -59,7 +59,7 @@ fn releases_url() -> String { .unwrap_or_else(|_| "https://api.github.com/repos/clockworklabs/SpacetimeDB/releases".to_owned()) } -const MIRROR_BASE_URL: &str = "https://spacetimedb-client-binaries.nyc3.digitaloceanspaces.com"; +const MIRROR_BASE_URL: &str = "https://spacetimedb-client-binaries.s3.amazonaws.com"; /// Fetch the latest version tag from the mirror. /// From 940667dd37a1a0742ed951a3ebfe7a2e602fab22 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Wed, 20 May 2026 13:47:14 -0700 Subject: [PATCH 13/20] Add commitlog knobs to server config (#5074) # Description of Changes Adds the commitlog knobs `max_segment_size`, `write_buffer_size`, and `preallocate_segments` to the server's `config.toml`. There are other seemingly more advanced knobs that I did not add at this time. This patch also increases the `DEFAULT_WRITE_BUFFER_SIZE` from `8KiB` to `128KiB` to optimize high throughput workloads like the keynote-2 benchmark. # API and ABI breaking changes None # Expected complexity level and risk 1 # Testing Manual --- crates/commitlog/src/lib.rs | 8 +- crates/core/src/db/persistence.rs | 73 ++++++++++++++++++- crates/core/src/db/relational_db.rs | 16 +++- crates/standalone/config.toml | 20 +++++ crates/standalone/src/lib.rs | 7 +- crates/standalone/src/subcommands/start.rs | 45 ++++++++++++ crates/testing/src/modules.rs | 1 + .../00200-standalone-config.md | 54 ++++++++++++++ .../00300-internals/00400-commitlog.md | 2 +- 9 files changed, 216 insertions(+), 10 deletions(-) diff --git a/crates/commitlog/src/lib.rs b/crates/commitlog/src/lib.rs index 0f0753f0f36..599c71c0136 100644 --- a/crates/commitlog/src/lib.rs +++ b/crates/commitlog/src/lib.rs @@ -70,7 +70,7 @@ pub struct Options { /// If `true`, require that the segment must be synced to disk before an /// index entry is added. /// - /// Setting this to `false` (the default) will update the index every + /// Setting this to `false` will update the index every /// `offset_index_interval_bytes`, even if the commitlog wasn't synced. /// This means that the index could contain non-existent entries in the /// event of a crash. @@ -80,7 +80,7 @@ pub struct Options { /// This means that the index could contain fewer index entries than /// strictly every `offset_index_interval_bytes`. /// - /// Default: false + /// Default: true #[cfg_attr( feature = "serde", serde(default = "Options::default_offset_index_require_segment_fsync") @@ -95,7 +95,7 @@ pub struct Options { /// Size in bytes of the memory buffer holding commit data before flushing /// to storage. /// - /// Default: 8KiB + /// Default: 128KiB #[cfg_attr(feature = "serde", serde(default = "Options::default_write_buffer_size"))] pub write_buffer_size: usize, } @@ -111,7 +111,7 @@ impl Options { pub const DEFAULT_OFFSET_INDEX_INTERVAL_BYTES: NonZeroU64 = NonZeroU64::new(4096).expect("4096 > 0, qed"); pub const DEFAULT_OFFSET_INDEX_REQUIRE_SEGMENT_FSYNC: bool = true; pub const DEFAULT_PREALLOCATE_SEGMENTS: bool = false; - pub const DEFAULT_WRITE_BUFFER_SIZE: usize = 8 * 1024; + pub const DEFAULT_WRITE_BUFFER_SIZE: usize = 128 * 1024; pub const DEFAULT: Self = Self { log_format_version: DEFAULT_LOG_FORMAT_VERSION, diff --git a/crates/core/src/db/persistence.rs b/crates/core/src/db/persistence.rs index 5b0daa5145c..d7c51b34e98 100644 --- a/crates/core/src/db/persistence.rs +++ b/crates/core/src/db/persistence.rs @@ -1,4 +1,8 @@ -use std::{io, sync::Arc}; +use std::{ + io, + num::{NonZeroU64, NonZeroUsize}, + sync::Arc, +}; use async_trait::async_trait; use spacetimedb_commitlog::SizeOnDisk; @@ -13,6 +17,59 @@ use super::{ snapshot::{self, SnapshotDatabaseState, SnapshotWorker}, }; +/// Local durability configuration exposed through server config. +#[derive(Clone, Copy, Debug, Default, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct DurabilityConfig { + #[serde(default)] + pub commitlog: CommitlogConfig, +} + +/// Commitlog configuration exposed through server config. +#[derive(Clone, Copy, Debug, Default, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct CommitlogConfig { + pub log_format_version: Option, + pub max_segment_size: Option, + #[serde(alias = "offset-interval-bytes")] + pub offset_index_interval_bytes: Option, + #[serde(alias = "offset-index-require-fsync")] + pub offset_index_require_segment_fsync: Option, + pub preallocate_segments: Option, + pub write_buffer_size: Option, +} + +impl DurabilityConfig { + fn into_options(self) -> spacetimedb_durability::local::Options { + let mut opts = spacetimedb_durability::local::Options::default(); + self.commitlog.apply_to(&mut opts.commitlog); + opts + } +} + +impl CommitlogConfig { + fn apply_to(self, opts: &mut spacetimedb_commitlog::Options) { + if let Some(log_format_version) = self.log_format_version { + opts.log_format_version = log_format_version; + } + if let Some(max_segment_size) = self.max_segment_size { + opts.max_segment_size = max_segment_size.get(); + } + if let Some(offset_index_interval_bytes) = self.offset_index_interval_bytes { + opts.offset_index_interval_bytes = offset_index_interval_bytes; + } + if let Some(offset_index_require_segment_fsync) = self.offset_index_require_segment_fsync { + opts.offset_index_require_segment_fsync = offset_index_require_segment_fsync; + } + if let Some(preallocate_segments) = self.preallocate_segments { + opts.preallocate_segments = preallocate_segments; + } + if let Some(write_buffer_size) = self.write_buffer_size { + opts.write_buffer_size = write_buffer_size.get(); + } + } +} + /// [spacetimedb_durability::Durability] impls with a [`Txdata`] transaction /// payload, suitable for use in the [`relational_db::RelationalDB`]. pub type Durability = dyn spacetimedb_durability::Durability; @@ -128,14 +185,21 @@ pub trait PersistenceProvider: Send + Sync { /// [compresses]: relational_db::snapshot_watching_commitlog_compressor pub struct LocalPersistenceProvider { data_dir: Arc, + durability: DurabilityConfig, } impl LocalPersistenceProvider { pub fn new(data_dir: impl Into>) -> Self { Self { data_dir: data_dir.into(), + durability: DurabilityConfig::default(), } } + + pub fn with_durability_config(mut self, durability: DurabilityConfig) -> Self { + self.durability = durability; + self + } } #[async_trait] @@ -149,7 +213,12 @@ impl PersistenceProvider for LocalPersistenceProvider { asyncify(move || relational_db::open_snapshot_repo(snapshot_dir, database_identity, replica_id)) .await .map(|repo| SnapshotWorker::new(repo, snapshot::Compression::Enabled))?; - let (durability, disk_size) = relational_db::local_durability(replica_dir, Some(&snapshot_worker)).await?; + let (durability, disk_size) = relational_db::local_durability_with_options( + replica_dir, + Some(&snapshot_worker), + self.durability.into_options(), + ) + .await?; tokio::spawn(relational_db::snapshot_watching_commitlog_compressor( snapshot_worker.subscribe(), diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 9f041c92ccb..a2b3a28db4c 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -1670,6 +1670,20 @@ pub type LocalDurability = Arc>; pub async fn local_durability( replica_dir: ReplicaDir, snapshot_worker: Option<&SnapshotWorker>, +) -> Result<(LocalDurability, DiskSizeFn), DBError> { + local_durability_with_options(replica_dir, snapshot_worker, <_>::default()).await +} + +/// Initialize local durability with explicit parameters. +/// +/// Also returned is a [`DiskSizeFn`] as required by [`RelationalDB::open`]. +/// +/// Note that this operation can be expensive, as it needs to traverse a suffix +/// of the commitlog. +pub async fn local_durability_with_options( + replica_dir: ReplicaDir, + snapshot_worker: Option<&SnapshotWorker>, + opts: durability::local::Options, ) -> Result<(LocalDurability, DiskSizeFn), DBError> { let rt = tokio::runtime::Handle::current(); let on_new_segment = snapshot_worker.map(|snapshot_worker| { @@ -1684,7 +1698,7 @@ pub async fn local_durability( durability::Local::open( replica_dir.clone(), rt, - <_>::default(), + opts, // Give the durability a handle to request a new snapshot run, // which it will send down whenever we rotate commitlog segments. on_new_segment, diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index 29a4bf3556e..9eeef5d3535 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -44,4 +44,24 @@ directives = [ # Apply a V8 heap limit in MiB. Set to 0 to use V8's default limit. # heap-limit-mb = 0 +[commitlog] +# The maximum supported commitlog format version, also used for writing. +# log-format-version = 1 + +# Maximum size in bytes for each commitlog segment. +# max-segment-size = 1073741824 + +# Number of bytes written to the commitlog after which an entry is added to the offset index. +# offset-index-interval-bytes = 4096 + +# Require that the commitlog segment is synced before adding an offset index entry. +# offset-index-require-segment-fsync = true + +# Preallocate disk space for commitlog segments up to max-segment-size. +# Has no effect unless commitlog fallocate support is enabled. +# preallocate-segments = false + +# Size in bytes of the memory buffer holding commit data before flushing to storage. +# write-buffer-size = 131072 + # vim: set nowritebackup: << otherwise triggers cargo-watch diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index de4b80ce78c..189bafa11bd 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -12,7 +12,7 @@ use http::StatusCode; use spacetimedb::client::ClientActorIndex; use spacetimedb::config::{CertificateAuthority, MetadataFile, V8Config, WasmConfig}; use spacetimedb::db; -use spacetimedb::db::persistence::LocalPersistenceProvider; +use spacetimedb::db::persistence::{DurabilityConfig, LocalPersistenceProvider}; use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; use spacetimedb::host::{DiskStorage, HostController, HostRuntimeConfig, MigratePlanResult, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; @@ -41,6 +41,7 @@ pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL} #[derive(Clone, Copy)] pub struct StandaloneOptions { pub db_config: db::Config, + pub durability: DurabilityConfig, pub websocket: WebSocketOptions, pub wasm: WasmConfig, pub v8: V8Config, @@ -76,7 +77,8 @@ impl StandaloneEnv { let energy_monitor = Arc::new(NullEnergyMonitor); let program_store = Arc::new(DiskStorage::new(data_dir.program_bytes().0).await?); - let persistence_provider = Arc::new(LocalPersistenceProvider::new(data_dir.clone())); + let persistence_provider = + Arc::new(LocalPersistenceProvider::new(data_dir.clone()).with_durability_config(config.durability)); let host_controller = HostController::new( data_dir, config.db_config, @@ -650,6 +652,7 @@ mod tests { storage: Storage::Memory, page_pool_max_size: None, }, + durability: DurabilityConfig::default(), websocket: WebSocketOptions::default(), wasm: WasmConfig::default(), v8: V8Config::default(), diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 50f6db19257..2373d11d26d 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -11,6 +11,7 @@ use axum::extract::DefaultBodyLimit; use clap::ArgAction::SetTrue; use clap::{Arg, ArgMatches}; use spacetimedb::config::{parse_config, CertificateAuthority}; +use spacetimedb::db::persistence::{CommitlogConfig, DurabilityConfig}; use spacetimedb::db::{self, Storage}; use spacetimedb::startup::{self, TracingOptions}; use spacetimedb::util::jobs::JobCores; @@ -99,6 +100,8 @@ struct ConfigFile { #[serde(flatten)] common: spacetimedb::config::ConfigFile, #[serde(default)] + commitlog: CommitlogConfig, + #[serde(default)] websocket: WebSocketOptions, } @@ -181,6 +184,9 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { let ctx = StandaloneEnv::init( StandaloneOptions { db_config, + durability: DurabilityConfig { + commitlog: config.commitlog, + }, websocket: config.websocket, wasm: config.common.wasm, v8: config.common.v8, @@ -525,6 +531,14 @@ mod tests { heap-gc-trigger-fraction = 0.6 heap-retire-fraction = 0.8 heap-limit-mb = 128 + + [commitlog] + log-format-version = 1 + max-segment-size = 1048576 + offset-index-interval-bytes = 8192 + offset-index-require-segment-fsync = false + preallocate-segments = true + write-buffer-size = 131072 "#; let config: ConfigFile = toml::from_str(toml).unwrap(); @@ -543,6 +557,21 @@ mod tests { assert_eq!(config.common.v8.heap_policy.heap_gc_trigger_fraction, 0.6); assert_eq!(config.common.v8.heap_policy.heap_retire_fraction, 0.8); assert_eq!(config.common.v8.heap_policy.heap_limit_bytes, 128 * 1024 * 1024); + assert_eq!(config.commitlog.log_format_version, Some(1)); + assert_eq!( + config.commitlog.max_segment_size.map(|val| val.get()), + Some(1024 * 1024) + ); + assert_eq!( + config.commitlog.offset_index_interval_bytes.map(|val| val.get()), + Some(8192) + ); + assert_eq!(config.commitlog.offset_index_require_segment_fsync, Some(false)); + assert_eq!(config.commitlog.preallocate_segments, Some(true)); + assert_eq!( + config.commitlog.write_buffer_size.map(|val| val.get()), + Some(128 * 1024) + ); assert_eq!( config.websocket, @@ -553,4 +582,20 @@ mod tests { } ); } + + #[test] + fn commitlog_options_accept_aliases() { + let toml = r#" + [commitlog] + offset-interval-bytes = 16384 + offset-index-require-fsync = true +"#; + + let config: ConfigFile = toml::from_str(toml).unwrap(); + assert_eq!( + config.commitlog.offset_index_interval_bytes.map(|val| val.get()), + Some(16 * 1024) + ); + assert_eq!(config.commitlog.offset_index_require_segment_fsync, Some(true)); + } } diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 7556b7f692e..21fea57fe96 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -245,6 +245,7 @@ impl CompiledModule { let env = spacetimedb_standalone::StandaloneEnv::init( spacetimedb_standalone::StandaloneOptions { db_config: config, + durability: Default::default(), websocket: WebSocketOptions::default(), wasm: Default::default(), v8: Default::default(), diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md index dce6afe8c5f..6863d642a98 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md @@ -18,6 +18,10 @@ On Linux and macOS, this directory is by default `~/.local/share/spacetime/data` - [`logs`](#logs) +- [`commitlog`](#commitlog) + +- [`websocket`](#websocket) + ### `certificate-authority` ```toml @@ -47,6 +51,56 @@ Can be one of `"error"`, `"warn"`, `"info"`, `"debug"`, `"trace"`, or `"off"`, c A list of filtering directives controlling what messages get logged, which overwrite the global [`logs.level`](#logslevel). See [`tracing documentation`](https://docs.rs/tracing-subscriber/0.3/tracing_subscriber/filter/struct.EnvFilter.html#directives) for syntax. Note that this is primarily intended as a debugging tool, and log message fields and targets are not considered stable. +### `commitlog` + +```toml +[commitlog] +log-format-version = 1 +max-segment-size = 1073741824 # 1GiB +offset-index-interval-bytes = 4096 +offset-index-require-segment-fsync = true +preallocate-segments = false +write-buffer-size = 131072 # 128KiB +``` + +The `commitlog` table configures local durability. These settings are advanced and may affect recovery behavior, disk usage, memory usage, and write throughput. Omitted fields use the server's built-in defaults. + +#### `commitlog.log-format-version` + +The maximum supported commitlog format version, also used for writing. + +::::caution +This setting should not normally be changed from the commitlog crate's default. A reason to change it could be to make the server accept an older, incompatible commitlog. +:::: + +#### `commitlog.max-segment-size` + +The maximum size in bytes to which commitlog segments should be allowed to grow. + +#### `commitlog.offset-index-interval-bytes` + +Number of bytes written to the commitlog after which an entry is added to the offset index. + +#### `commitlog.offset-index-require-segment-fsync` + +If `true`, require that the segment must be synced to disk before an index entry is added. + +Setting this to `false` will update the index every `offset-index-interval-bytes`, even if the commitlog was not synced. This means that the index could contain non-existent entries in the event of a crash. + +Setting this to `true` will update the index when the commitlog is synced, and `offset-index-interval-bytes` have been written. This means that the index could contain fewer index entries than strictly every `offset-index-interval-bytes`. + +::::note +The commitlog operates correctly under both settings, but the choice can have performance implications. +:::: + +#### `commitlog.preallocate-segments` + +If `true`, preallocate disk space for commitlog segments up to `commitlog.max-segment-size`. This has no effect unless commitlog fallocate support is enabled. + +#### `commitlog.write-buffer-size` + +Size in bytes of the memory buffer holding commit data before flushing to storage. + ### `websocket` ```toml diff --git a/docs/docs/00300-resources/00200-reference/00300-internals/00400-commitlog.md b/docs/docs/00300-resources/00200-reference/00300-internals/00400-commitlog.md index 01fc3063807..59ceb1add10 100644 --- a/docs/docs/00300-resources/00200-reference/00300-internals/00400-commitlog.md +++ b/docs/docs/00300-resources/00200-reference/00300-internals/00400-commitlog.md @@ -238,7 +238,7 @@ The offset index is controlled by two parameters: | Parameter | Default | Description | |-----------|---------|-------------| | `offset_index_interval_bytes` | 4,096 | An index entry is written whenever this many bytes have been flushed to the active segment. | -| `offset_index_require_segment_fsync` | false | If true, the segment must be synced to disk before an index entry is written. | +| `offset_index_require_segment_fsync` | true | If true, the segment must be synced to disk before an index entry is written. | ## Wire Format From f39360b666b961cd0be4436e5fe2f50371b50932 Mon Sep 17 00:00:00 2001 From: joshua-spacetime Date: Wed, 20 May 2026 14:40:04 -0700 Subject: [PATCH 14/20] Batch websocket responses using the v3 protocol (#5061) # Description of Changes In keeping with the "pipeline everything" approach of SpacetimeDB, this patch serializes multiple client updates in a single websocket message using the v3 protocol. # API and ABI breaking changes None. `spacetime subscribe` was updated to use the v3 websocket api, but it falls back to v1 if protocol negotiation fails. # Expected complexity level and risk 2 # Testing This patch updates `spacetime subscribe` to use the v3 websocket protocol by default in order to get adequate coverage via the smoketests. --- crates/cli/src/subcommands/subscribe.rs | 565 +++++++++++++++++--- crates/client-api/src/routes/subscribe.rs | 609 ++++++++++++++++------ crates/core/src/client/messages.rs | 38 +- crates/core/src/worker_metrics/mod.rs | 8 +- 4 files changed, 979 insertions(+), 241 deletions(-) diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index 5b22b4cd8a5..f8cbc61e0cf 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -1,20 +1,28 @@ use anyhow::Context; +use bytes::Bytes; use clap::{value_parser, Arg, ArgAction, ArgMatches}; use futures::{Sink, SinkExt, TryStream, TryStreamExt}; use http::header; use reqwest::Url; use serde_json::Value; -use spacetimedb_client_api_messages::websocket::v1 as ws_v1; +use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2, v3 as ws_v3}; use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::{DeserializeWrapper, SeedWrapper}; +use spacetimedb_lib::de::DeserializeSeed as BsatnDeserializeSeed; +use spacetimedb_lib::sats::WithTypespace; use spacetimedb_lib::ser::serde::SerializeWrapper; +use spacetimedb_lib::{bsatn, AlgebraicType}; +use std::collections::VecDeque; use std::io; use std::time::Duration; use thiserror::Error; use tokio::io::AsyncWriteExt; +use tokio::net::TcpStream; use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::handshake::client::Request as WsRequest; use tokio_tungstenite::tungstenite::{Error as WsError, Message as WsMessage}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; use crate::api::ClientApi; use crate::common_args; @@ -72,56 +80,68 @@ pub fn cli() -> clap::Command { .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) } -fn parse_msg_json(msg: &WsMessage) -> Option> { - let WsMessage::Text(msg) = msg else { return None }; - serde_json::from_str::>>(msg) - .inspect_err(|e| eprintln!("couldn't parse message from server: {e}")) - .map(|wrapper| wrapper.0) - .ok() +#[derive(serde::Serialize, Debug)] +struct SubscriptionTable { + deletes: Vec, + inserts: Vec, } -fn reformat_update<'a>( - msg: &'a ws_v1::DatabaseUpdate, - schema: &RawModuleDefV9, -) -> anyhow::Result> { - msg.tables - .iter() - .map(|upd| { - let table_ty = schema.typespace.resolve( - schema - .type_ref_for_table_like(&upd.table_name) - .context("table not found in schema")?, - ); +/// Concrete websocket stream type returned by `tokio_tungstenite::connect_async`. +type SubscribeWebSocket = WebSocketStream>; - let reformat_row = |row: &str| -> anyhow::Result { - // TODO: can the following two calls be merged into a single call to reduce allocations? - let row = serde_json::from_str::(row)?; - let row = serde::de::DeserializeSeed::deserialize(SeedWrapper(table_ty), row)?; - let row = table_ty.with_value(&row); - let row = serde_json::to_value(SerializeWrapper::from_ref(&row))?; - Ok(row) - }; +/// Active websocket connection for `spacetime subscribe`. +/// +/// The command prefers the v3 transport so smoketests and normal CLI usage +/// exercise the coalesced server path, but it keeps the old v1 text transport +/// as a fallback for older servers. +enum SubscribeConnection { + /// v3 uses BSATN-encoded v2 messages, possibly coalesced in one websocket payload. + V3 { + ws: SubscribeWebSocket, + /// Decoded messages left over from a coalesced v3 websocket payload. + pending: VecDeque, + }, + /// v1 is the historical JSON text protocol. + V1 { ws: SubscribeWebSocket }, +} - let mut deletes = Vec::new(); - let mut inserts = Vec::new(); - for upd in &upd.updates { - for s in &upd.deletes { - deletes.push(reformat_row(s)?); - } - for s in &upd.inserts { - inserts.push(reformat_row(s)?); - } - } +impl SubscribeConnection { + /// Send the subscribe request using whichever protocol was negotiated. + async fn subscribe(&mut self, query_strings: Box<[Box]>) -> Result<(), Error> { + match self { + Self::V3 { ws, .. } => subscribe_v3(ws, query_strings).await, + Self::V1 { ws } => subscribe_v1(ws, query_strings).await, + } + } - Ok((&*upd.table_name, SubscriptionTable { deletes, inserts })) - }) - .collect() -} + /// Wait for the initial subscription result and optionally print it. + async fn await_initial_update(&mut self, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> { + match self { + Self::V3 { ws, pending } => await_initial_update_v3(ws, pending, module_def).await, + Self::V1 { ws } => await_initial_update_v1(ws, module_def).await, + } + } -#[derive(serde::Serialize, Debug)] -struct SubscriptionTable { - deletes: Vec, - inserts: Vec, + /// Print transaction updates until the requested count is reached. + async fn consume_transaction_updates( + &mut self, + num: Option, + module_def: &RawModuleDefV9, + ) -> Result<(), Error> { + match self { + Self::V3 { ws, pending } => consume_transaction_updates_v3(ws, pending, num, module_def).await, + Self::V1 { ws } => consume_transaction_updates_v1(ws, num, module_def).await, + } + } + + /// Best-effort graceful websocket close. + async fn close(&mut self) { + match self { + Self::V3 { ws, .. } | Self::V1 { ws } => { + let _ = ws.close(None).await; + } + } + } } pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { @@ -160,36 +180,14 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error }; let api = ClientApi::new(conn); let module_def = api.module_def().await?; - - let mut url = Url::parse(&api.con.db_uri("subscribe"))?; - // Change the URI scheme from `http(s)` to `ws(s)`. - url.set_scheme(match url.scheme() { - "http" => "ws", - "https" => "wss", - unknown => unreachable!("Invalid URL scheme in `Connection::db_uri`: {unknown}"), - }) - .unwrap(); - if let Some(confirmed) = confirmed { - url.query_pairs_mut() - .append_pair("confirmed", if confirmed { "true" } else { "false" }); - } - - // Create the websocket request. - let mut req = url.into_client_request()?; - req.headers_mut().insert( - header::SEC_WEBSOCKET_PROTOCOL, - http::HeaderValue::from_static(ws_v1::TEXT_PROTOCOL), - ); - // Add the authorization header, if any. - if let Some(auth_header) = api.con.auth_header.to_header() { - req.headers_mut().insert(header::AUTHORIZATION, auth_header); - } - let mut ws = tokio_tungstenite::connect_async(req).await.map(|(ws, _)| ws)?; + let mut conn = connect_with_fallback(&api, confirmed).await?; let task = async { - subscribe(&mut ws, queries.iter().cloned().map(Into::into).collect()).await?; - await_initial_update(&mut ws, print_initial_update.then_some(&module_def)).await?; - consume_transaction_updates(&mut ws, num, &module_def).await + conn.subscribe(queries.iter().cloned().map(Into::into).collect()) + .await?; + conn.await_initial_update(print_initial_update.then_some(&module_def)) + .await?; + conn.consume_transaction_updates(num, &module_def).await }; let res = if let Some(timeout) = timeout { @@ -211,7 +209,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error // The error (if any) relevant to the user is already stored in `res`, // so we can ignore errors here -- graceful close is basically a // courtesy to the server. - let _ = ws.close(None).await; + conn.close().await; // The server closing the connection is not considered an error, // but any other error is. res.or_else(|e| { @@ -224,6 +222,90 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error .map_err(anyhow::Error::from) } +/// Connect using v3 when available, otherwise retry once with the v1 text protocol. +/// +/// Fallback is intentionally limited to connection setup and protocol +/// negotiation. After a v3 connection is accepted, malformed v3 data is a real +/// error and should not be hidden by silently reconnecting with v1. +async fn connect_with_fallback(api: &ClientApi, confirmed: Option) -> Result { + match connect_v3(api, confirmed).await { + Ok(conn) => Ok(conn), + Err(v3_error) => connect_v1(api, confirmed) + .await + .with_context(|| format!("v3 subscribe connection failed ({v3_error}); v1 fallback also failed")), + } +} + +/// Open a v3 subscribe websocket and validate that the server negotiated v3. +async fn connect_v3(api: &ClientApi, confirmed: Option) -> Result { + let req = subscribe_request(api, confirmed, ws_v3::BIN_PROTOCOL, true)?; + let (ws, response) = tokio_tungstenite::connect_async(req).await?; + if response + .headers() + .get(header::SEC_WEBSOCKET_PROTOCOL) + .and_then(|value| value.to_str().ok()) + != Some(ws_v3::BIN_PROTOCOL) + { + return Err(Error::Protocol { + details: "server did not negotiate the v3 websocket protocol", + } + .into()); + } + Ok(SubscribeConnection::V3 { + ws, + pending: VecDeque::new(), + }) +} + +/// Open a v1 text subscribe websocket for compatibility with older servers. +async fn connect_v1(api: &ClientApi, confirmed: Option) -> Result { + let req = subscribe_request(api, confirmed, ws_v1::TEXT_PROTOCOL, false)?; + let (ws, _) = tokio_tungstenite::connect_async(req).await?; + Ok(SubscribeConnection::V1 { ws }) +} + +/// Build a subscribe websocket request for a specific subprotocol. +/// +/// `request_uncompressed` is used only for the CLI v3 path. The CLI decodes +/// enough v3 to print JSON output, but does not implement brotli or gzip +/// decoding, so v3 asks the server for uncompressed payloads. The v1 fallback +/// leaves the query string in its historical shape. +fn subscribe_request( + api: &ClientApi, + confirmed: Option, + protocol: &'static str, + request_uncompressed: bool, +) -> Result { + let mut url = Url::parse(&api.con.db_uri("subscribe"))?; + // Change the URI scheme from `http(s)` to `ws(s)`. + url.set_scheme(match url.scheme() { + "http" => "ws", + "https" => "wss", + unknown => unreachable!("Invalid URL scheme in `Connection::db_uri`: {unknown}"), + }) + .unwrap(); + { + let mut query = url.query_pairs_mut(); + if request_uncompressed { + // The CLI v3 path only needs enough support to print updates as + // JSON, so request uncompressed payloads and avoid brotli/gzip + // decoding here. The v1 fallback preserves the old URL shape. + query.append_pair("compression", "None"); + } + if let Some(confirmed) = confirmed { + query.append_pair("confirmed", if confirmed { "true" } else { "false" }); + } + } + + let mut req = url.into_client_request()?; + req.headers_mut() + .insert(header::SEC_WEBSOCKET_PROTOCOL, http::HeaderValue::from_static(protocol)); + if let Some(auth_header) = api.con.auth_header.to_header() { + req.headers_mut().insert(header::AUTHORIZATION, auth_header); + } + Ok(req) +} + #[derive(Debug, Error)] enum Error { #[error("error sending subscription queries")] @@ -247,6 +329,16 @@ enum Error { #[source] source: anyhow::Error, }, + #[error("error encoding BSATN websocket message: {source}")] + BsatnEncode { + #[source] + source: spacetimedb_lib::bsatn::EncodeError, + }, + #[error("error decoding BSATN websocket message: {source}")] + BsatnDecode { + #[source] + source: spacetimedb_lib::bsatn::DecodeError, + }, #[error(transparent)] Serde(#[from] serde_json::Error), #[error(transparent)] @@ -264,8 +356,8 @@ impl Error { } } -/// Send the subscribe message. -async fn subscribe(ws: &mut S, query_strings: Box<[Box]>) -> Result<(), Error> +/// Send a v1 JSON subscribe message. +async fn subscribe_v1(ws: &mut S, query_strings: Box<[Box]>) -> Result<(), Error> where S: Sink + Unpin, { @@ -279,9 +371,34 @@ where ws.send(msg.into()).await.map_err(|source| Error::Subscribe { source }) } -/// Await the initial [`ServerMessage::SubscriptionUpdate`]. +/// Send a v3 BSATN subscribe message. +async fn subscribe_v3(ws: &mut S, query_strings: Box<[Box]>) -> Result<(), Error> +where + S: Sink + Unpin, +{ + let msg = ws_v2::ClientMessage::Subscribe(ws_v2::Subscribe { + request_id: 0, + query_set_id: ws_v2::QuerySetId::new(0), + query_strings, + }); + let msg = bsatn::to_vec(&msg).map_err(|source| Error::BsatnEncode { source })?; + ws.send(WsMessage::Binary(msg.into())) + .await + .map_err(|source| Error::Subscribe { source }) +} + +/// Parse a v1 text websocket message as JSON. +fn parse_msg_json(msg: &WsMessage) -> Option> { + let WsMessage::Text(msg) = msg else { return None }; + serde_json::from_str::>>(msg) + .inspect_err(|e| eprintln!("couldn't parse message from server: {e}")) + .map(|wrapper| wrapper.0) + .ok() +} + +/// Await the initial v1 [`ws_v1::ServerMessage::InitialSubscription`]. /// If `module_def` is `Some`, print a JSON representation to stdout. -async fn await_initial_update(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> +async fn await_initial_update_v1(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> where S: TryStream + Unpin, { @@ -292,7 +409,7 @@ where match msg { ws_v1::ServerMessage::InitialSubscription(sub) => { if let Some(module_def) = module_def { - let output = format_output_json(&sub.database_update, module_def)?; + let output = format_output_json_v1(&sub.database_update, module_def)?; tokio::io::stdout().write_all(output.as_bytes()).await? } break; @@ -320,9 +437,49 @@ where Ok(()) } -/// Print `num` [`ServerMessage::TransactionUpdate`] messages as JSON. +/// Await the initial [`ws_v2::ServerMessage::SubscribeApplied`]. +/// If `module_def` is `Some`, print a JSON representation to stdout. +async fn await_initial_update_v3( + ws: &mut S, + pending: &mut VecDeque, + module_def: Option<&RawModuleDefV9>, +) -> Result<(), Error> +where + S: TryStream + Unpin, +{ + const RECV_TX_UPDATE: &str = "received transaction update before initial subscription update"; + + while let Some(msg) = next_server_message(ws, pending).await? { + match msg { + ws_v2::ServerMessage::SubscribeApplied(sub) => { + if let Some(module_def) = module_def { + let output = format_output_json_query_rows(&sub.rows, module_def)?; + tokio::io::stdout().write_all(output.as_bytes()).await? + } + break; + } + ws_v2::ServerMessage::SubscriptionError(error) => { + return Err(Error::SubscribeFailure { reason: error.error }); + } + ws_v2::ServerMessage::TransactionUpdate(_) => { + return Err(Error::Protocol { + details: RECV_TX_UPDATE, + }) + } + _ => continue, + } + } + + Ok(()) +} + +/// Print `num` v1 [`ws_v1::ServerMessage::TransactionUpdate`] messages as JSON. /// If `num` is `None`, keep going indefinitely. -async fn consume_transaction_updates(ws: &mut S, num: Option, module_def: &RawModuleDefV9) -> Result<(), Error> +async fn consume_transaction_updates_v1( + ws: &mut S, + num: Option, + module_def: &RawModuleDefV9, +) -> Result<(), Error> where S: TryStream + Unpin, { @@ -354,7 +511,50 @@ where status: ws_v1::UpdateStatus::Committed(update), .. }) => { - let output = format_output_json(&update, module_def)?; + let output = format_output_json_v1(&update, module_def)?; + stdout.write_all(output.as_bytes()).await?; + num_received += 1; + } + _ => continue, + } + } +} + +/// Print `num` [`ws_v2::ServerMessage::TransactionUpdate`] messages as JSON. +/// If `num` is `None`, keep going indefinitely. +async fn consume_transaction_updates_v3( + ws: &mut S, + pending: &mut VecDeque, + num: Option, + module_def: &RawModuleDefV9, +) -> Result<(), Error> +where + S: TryStream + Unpin, +{ + let mut stdout = tokio::io::stdout(); + let mut num_received = 0; + loop { + if num.is_some_and(|n| num_received >= n) { + return Ok(()); + } + let Some(msg) = next_server_message(ws, pending).await? else { + eprintln!("disconnected by server"); + return Err(Error::Websocket { + source: WsError::ConnectionClosed, + }); + }; + + match msg { + ws_v2::ServerMessage::SubscribeApplied(_) => { + return Err(Error::Protocol { + details: "received a second initial subscription update", + }) + } + ws_v2::ServerMessage::SubscriptionError(error) => { + return Err(Error::SubscribeFailure { reason: error.error }); + } + ws_v2::ServerMessage::TransactionUpdate(update) => { + let output = format_output_json_transaction_update(&update, module_def)?; stdout.write_all(output.as_bytes()).await?; num_received += 1; } @@ -363,12 +563,207 @@ where } } -fn format_output_json( +/// Return the next decoded server message from a v3 websocket stream. +/// +/// A v3 websocket payload can contain multiple consecutive BSATN-encoded v2 +/// server messages, so decoded surplus messages are queued for the next call. +/// Non-binary messages are ignored because v3 server data is binary-only. +async fn next_server_message( + ws: &mut S, + pending: &mut VecDeque, +) -> Result, Error> +where + S: TryStream + Unpin, +{ + loop { + if let Some(msg) = pending.pop_front() { + return Ok(Some(msg)); + } + + let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + return Ok(None); + }; + let WsMessage::Binary(msg) = msg else { continue }; + decode_server_payload(msg, pending)?; + } +} + +/// Decode one uncompressed v3 websocket payload into queued v2 server messages. +/// +/// The server prefixes each binary payload with a compression tag. This CLI path +/// requests `compression=None`, so any compressed tag is treated as a protocol +/// error rather than decoded here. +fn decode_server_payload(msg: Bytes, pending: &mut VecDeque) -> Result<(), Error> { + let Some((&tag, mut remaining)) = msg.as_ref().split_first() else { + return Err(Error::Protocol { + details: "received empty v3 websocket payload", + }); + }; + if tag != ws_common::SERVER_MSG_COMPRESSION_TAG_NONE { + return Err(Error::Protocol { + details: "compressed v3 subscribe payload is not supported by this CLI path", + }); + } + if remaining.is_empty() { + return Err(Error::Protocol { + details: "received v3 websocket payload without a server message", + }); + } + + while !remaining.is_empty() { + let msg = bsatn::from_reader(&mut remaining).map_err(|source| Error::BsatnDecode { source })?; + pending.push_back(msg); + } + + Ok(()) +} + +/// Format a v1 database update using the legacy JSON row representation. +fn format_output_json_v1( msg: &ws_v1::DatabaseUpdate, schema: &RawModuleDefV9, ) -> Result { - let formatted = reformat_update(msg, schema).map_err(|source| Error::Reformat { source })?; - let output = serde_json::to_string(&formatted)? + "\n"; + let formatted = reformat_update_v1(msg, schema).map_err(|source| Error::Reformat { source })?; + format_output_json_from_tables(&formatted) +} + +/// Format initial v3 subscription rows using the CLI's existing JSON output shape. +fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &RawModuleDefV9) -> Result { + let formatted = reformat_query_rows(msg, schema).map_err(|source| Error::Reformat { source })?; + format_output_json_from_tables(&formatted) +} +/// Format a v3 transaction update using the CLI's existing JSON output shape. +fn format_output_json_transaction_update( + msg: &ws_v2::TransactionUpdate, + schema: &RawModuleDefV9, +) -> Result { + let formatted = reformat_transaction_update(msg, schema).map_err(|source| Error::Reformat { source })?; + format_output_json_from_tables(&formatted) +} + +/// Serialize the normalized table update map as one JSON object per output line. +fn format_output_json_from_tables(formatted: &HashMap<&str, SubscriptionTable>) -> Result { + let output = serde_json::to_string(formatted)? + "\n"; Ok(output) } + +/// Convert a v1 JSON-format database update to the normalized table output map. +fn reformat_update_v1<'a>( + msg: &'a ws_v1::DatabaseUpdate, + schema: &RawModuleDefV9, +) -> anyhow::Result> { + msg.tables + .iter() + .map(|upd| { + let table_ty = schema.typespace.resolve( + schema + .type_ref_for_table_like(&upd.table_name) + .context("table not found in schema")?, + ); + + let reformat_row = |row: &str| -> anyhow::Result { + // TODO: can the following two calls be merged into a single call to reduce allocations? + let row = serde_json::from_str::(row)?; + let row = serde::de::DeserializeSeed::deserialize(SeedWrapper(table_ty), row)?; + let row = table_ty.with_value(&row); + let row = serde_json::to_value(SerializeWrapper::from_ref(&row))?; + Ok(row) + }; + + let mut deletes = Vec::new(); + let mut inserts = Vec::new(); + for upd in &upd.updates { + for s in &upd.deletes { + deletes.push(reformat_row(s)?); + } + for s in &upd.inserts { + inserts.push(reformat_row(s)?); + } + } + + Ok((&*upd.table_name, SubscriptionTable { deletes, inserts })) + }) + .collect() +} + +/// Convert v3 initial subscription rows to the normalized table output map. +fn reformat_query_rows<'a>( + msg: &'a ws_v2::QueryRows, + schema: &RawModuleDefV9, +) -> anyhow::Result> { + let mut formatted = HashMap::default(); + + for table in &msg.tables { + let table_ty = schema.typespace.resolve( + schema + .type_ref_for_table_like(&table.table) + .context("table not found in schema")?, + ); + let table_output = formatted.entry(&*table.table).or_insert_with(|| SubscriptionTable { + deletes: Vec::new(), + inserts: Vec::new(), + }); + table_output.inserts.extend(reformat_bsatn_rows(&table.rows, table_ty)?); + } + + Ok(formatted) +} + +/// Convert a v3 transaction update to the normalized table output map. +fn reformat_transaction_update<'a>( + msg: &'a ws_v2::TransactionUpdate, + schema: &RawModuleDefV9, +) -> anyhow::Result> { + let mut formatted = HashMap::default(); + + for query_set in &msg.query_sets { + for table in &query_set.tables { + let table_ty = schema.typespace.resolve( + schema + .type_ref_for_table_like(&table.table_name) + .context("table not found in schema")?, + ); + let table_output = formatted + .entry(&*table.table_name) + .or_insert_with(|| SubscriptionTable { + deletes: Vec::new(), + inserts: Vec::new(), + }); + for rows in &table.rows { + match rows { + ws_v2::TableUpdateRows::PersistentTable(rows) => { + table_output + .deletes + .extend(reformat_bsatn_rows(&rows.deletes, table_ty)?); + table_output + .inserts + .extend(reformat_bsatn_rows(&rows.inserts, table_ty)?); + } + ws_v2::TableUpdateRows::EventTable(rows) => { + table_output + .inserts + .extend(reformat_bsatn_rows(&rows.events, table_ty)?); + } + } + } + } + } + + Ok(formatted) +} + +/// Decode BSATN row-list entries and re-encode them as schema-aware JSON values. +fn reformat_bsatn_rows( + rows: &ws_common::BsatnRowList, + table_ty: WithTypespace<'_, AlgebraicType>, +) -> anyhow::Result> { + rows.into_iter() + .map(|row| { + let mut row = row.as_ref(); + let row = BsatnDeserializeSeed::deserialize(table_ty, bsatn::Deserializer::new(&mut row))?; + let row = table_ty.with_value(&row); + Ok(serde_json::to_value(SerializeWrapper::from_ref(&row))?) + }) + .collect() +} diff --git a/crates/client-api/src/routes/subscribe.rs b/crates/client-api/src/routes/subscribe.rs index ce7c127be98..f29527bee42 100644 --- a/crates/client-api/src/routes/subscribe.rs +++ b/crates/client-api/src/routes/subscribe.rs @@ -23,7 +23,7 @@ use prometheus::{Histogram, IntGauge}; use scopeguard::{defer, ScopeGuard}; use serde::Deserialize; use spacetimedb::client::messages::{ - serialize, serialize_v2, IdentityTokenMessage, InUseSerializeBuffer, SerializeBuffer, SwitchedServerMessage, + serialize, serialize_v3, IdentityTokenMessage, InUseSerializeBuffer, SerializeBuffer, SwitchedServerMessage, ToProtocol, }; use spacetimedb::client::{ @@ -39,7 +39,7 @@ use spacetimedb::Identity; use spacetimedb_client_api_messages::websocket::v1 as ws_v1; use spacetimedb_client_api_messages::websocket::v2 as ws_v2; use spacetimedb_client_api_messages::websocket::v3 as ws_v3; -use spacetimedb_datastore::execution_context::WorkloadType; +use spacetimedb_lib::bsatn; use spacetimedb_lib::connection_id::{ConnectionId, ConnectionIdForUrl}; use tokio::sync::{mpsc, watch}; use tokio::task::JoinHandle; @@ -1290,13 +1290,143 @@ enum OutboundWsMessage { Message(OutboundMessage), } -/// Task that reads [`OutboundWsMessage`]s from `messages`, encodes them via -/// [`ws_encode_message`], and sends the resuling [`Frame`]s to `outgoing_frames`. +/// Controls how many binary protocol messages may be packed into a single +/// websocket payload. +/// +/// Protocol v2 requires one [`ws_v2::ServerMessage`] per websocket message. +/// Protocol v3 keeps the v2 message schema but permits multiple consecutive +/// v2 messages in a single websocket message. +#[derive(Clone, Copy, PartialEq, Eq)] +enum BinaryPayloadMode { + /// Flush after each binary server message. + Single, + /// Flush once after all available binary server messages are collected. + Coalesced, +} + +/// A binary websocket message plus the logical row count it contributes to +/// payload-level send metrics. +struct V2OutboundMessage { + message: ws_v2::ServerMessage, + num_rows: Option, +} + +/// Convert an outbound message into the binary websocket schema. +/// +/// v2 connections should only receive v2 server messages. +/// v1 messages are dropped. +/// +/// TODO: For better type safety, [`ClientConnectionReceiver`] should be made +/// generic over the protocol version. +fn v2_outbound_message(message: OutboundWsMessage) -> Option { + let message = match message { + OutboundWsMessage::Error(message) => { + log::error!("dropping v1 error message on v2 connection: {:?}", message); + return None; + } + OutboundWsMessage::Message(message) => message, + }; + + let num_rows = message.num_rows(); + match message { + OutboundMessage::V2(message) => Some(V2OutboundMessage { message, num_rows }), + OutboundMessage::V1(message) => { + log::error!("dropping v1 message on v2 connection: {:?}", message); + None + } + } +} + +/// Return the uncompressed payload size of `message`. +/// +/// v2 sends exactly one BSATN-encoded v2 server message per websocket payload. +/// v3 sends one or more of the same encoded messages in a coalesced payload. +fn message_size(message: &ws_v2::ServerMessage) -> usize { + bsatn::to_len(message).expect("should be able to measure bsatn-encoded v2 server message") +} + +/// Return whether appending the next message would cross the v3 coalescing cap. +/// +/// An empty payload is always allowed to accept one message, even when that +/// message alone is larger than the cap. +fn v3_payload_would_exceed_limit(total_bytes: usize, message_bytes: usize) -> bool { + total_bytes != 0 && total_bytes.saturating_add(message_bytes) > V3_MAX_UNCOMPRESSED_PAYLOAD_SIZE +} + +/// Return whether a binary websocket payload is large enough to encode on Rayon. +fn is_large_payload(num_bytes: usize) -> bool { + num_bytes >= V3_MAX_UNCOMPRESSED_PAYLOAD_SIZE +} + +/// Encoding receive batch size. +/// +/// This is deliberately tied to the client connection receive limit so the +/// websocket encoder can consume the batches produced by +/// [`ClientConnectionReceiver::recv_many`] without immediately re-batching +/// them to a different size. +const ENCODE_BATCH_SIZE: usize = ClientConnectionReceiver::DEFAULT_RECV_MANY_LIMIT; + +/// Target maximum uncompressed v3 payload body size. +/// +/// The v3 binary body is a sequence of BSATN-encoded v2 server messages. The +/// one-byte compression tag is not counted here. This is a target, not a hard +/// rejection limit. One logical server message may exceed it, in which case the +/// message is sent by itself. +const V3_MAX_UNCOMPRESSED_PAYLOAD_SIZE: usize = 512 * 1024; + +/// Tracks serialize buffers that may be reusable once their frames have been +/// copied to the wire. +struct SerializeBufferPool { + config: ClientConfig, + available: ArrayQueue, + in_use: Vec, +} + +impl SerializeBufferPool { + const CAPACITY: usize = 16; + + fn new(config: ClientConfig) -> Self { + Self { + config, + available: ArrayQueue::new(Self::CAPACITY), + in_use: Vec::with_capacity(Self::CAPACITY), + } + } + + fn get(&mut self) -> SerializeBuffer { + self.reclaim(); + self.available + .pop() + .unwrap_or_else(|| SerializeBuffer::new(self.config)) + } + + fn hold(&mut self, in_use: InUseSerializeBuffer) { + if self.in_use.len() < Self::CAPACITY { + self.in_use.push(in_use); + } + } + + fn reclaim(&mut self) { + let mut i = 0; + while i < self.in_use.len() { + if self.in_use[i].is_unique() { + let in_use = self.in_use.swap_remove(i); + let buf = in_use.try_reclaim().expect("buffer should be unique"); + let _ = self.available.push(buf); + } else { + i += 1; + } + } + } +} + +/// Task that reads [`OutboundWsMessage`]s from `messages`, encodes them, and +/// sends the resulting [`Frame`]s to `outgoing_frames`. /// /// Meant to be [`tokio::spawn`]ed. /// /// The function also takes care of reusing serialization buffers and reporting -/// metrics via [`SendMetrics`].. +/// metrics via [`SendMetrics`]. async fn ws_encode_task( metrics: SendMetrics, config: ClientConfig, @@ -1304,91 +1434,264 @@ async fn ws_encode_task( outgoing_frames: mpsc::UnboundedSender, bsatn_rlb_pool: BsatnRowListBuilderPool, ) { - // Serialize buffers can be reclaimed once all frames of a message are - // copied to the wire. Since we don't know when that will happen, we prepare - // for a few messages to be in-flight, i.e. encoded but not yet sent. - const BUF_POOL_CAPACITY: usize = 16; - let buf_pool = ArrayQueue::new(BUF_POOL_CAPACITY); - let mut in_use_bufs: Vec> = Vec::with_capacity(BUF_POOL_CAPACITY); - - 'send: while let Some(message) = messages.recv().await { - // Drop serialize buffers with no external referent, - // returning them to the pool. - in_use_bufs.retain(|in_use| !in_use.is_unique()); - // Get a serialize buffer from the pool, - // or create a fresh one. - let buf = buf_pool.pop().unwrap_or_else(|| SerializeBuffer::new(config)); - - let in_use_buf = match message { - OutboundWsMessage::Error(message) => { - if config.version != WsVersion::V1 { - log::error!( - "dropping v1 error message sent to a binary websocket client: {:?}", - message - ); - continue; + let mut encoder = WsEncoder { + config, + buffers: SerializeBufferPool::new(config), + metrics: &metrics, + outgoing_frames: &outgoing_frames, + bsatn_rlb_pool: &bsatn_rlb_pool, + binary_server_messages: Vec::new(), + }; + let mut message_batch = Vec::new(); + while messages.recv_many(&mut message_batch, ENCODE_BATCH_SIZE).await != 0 { + log::trace!("encoding batch of {} websocket messages", message_batch.len()); + // `encode_batch` drains `message_batch` on success. If forwarding to + // the websocket send loop fails, the receiver is gone, so the encode + // task can terminate. + if encoder.encode_batch(&mut message_batch).await.is_err() { + break; + } + } +} + +/// Stateful websocket encoder for one client connection. +/// +/// The encoder owns reusable scratch storage: +/// +/// - [`SerializeBufferPool`] reuses byte buffers once encoded frames have been +/// copied to the socket task. +/// - `binary_server_messages` reuses the vector allocation used to assemble +/// v2/v3 binary websocket payloads. +struct WsEncoder<'a> { + config: ClientConfig, + buffers: SerializeBufferPool, + metrics: &'a SendMetrics, + outgoing_frames: &'a mpsc::UnboundedSender, + bsatn_rlb_pool: &'a BsatnRowListBuilderPool, + binary_server_messages: Vec, +} + +impl WsEncoder<'_> { + /// Encode a drained batch according to the websocket version negotiated by + /// the client. + async fn encode_batch( + &mut self, + message_batch: &mut Vec, + ) -> Result<(), mpsc::error::SendError> { + match self.config.version { + WsVersion::V1 => self.encode_v1_batch(message_batch).await, + WsVersion::V2 => self.encode_v2_batch(message_batch).await, + WsVersion::V3 => self.encode_v3_batch(message_batch).await, + } + } + + /// Encode a batch for the original v1 websocket protocols. + /// + /// v1 text/binary messages are encoded one logical message at a time. This + /// path also handles reducer errors, which still use the v1 message schema. + async fn encode_v1_batch( + &mut self, + message_batch: &mut Vec, + ) -> Result<(), mpsc::error::SendError> { + for message in message_batch.drain(..) { + match message { + OutboundWsMessage::Error(message) => { + self.encode_and_forward_v1_message(None, message).await?; } - let Ok(in_use) = ws_forward_frames( - &metrics, - &outgoing_frames, - None, - None, - ws_encode_message(config, buf, message, false, &bsatn_rlb_pool).await, - ) else { - break 'send; - }; - in_use - } - OutboundWsMessage::Message(message) => { - let workload = message.workload(); - let num_rows = message.num_rows(); - match message { - OutboundMessage::V2(server_message) => { - if config.version == WsVersion::V1 { + OutboundWsMessage::Message(message) => { + let num_rows = message.num_rows(); + match message { + OutboundMessage::V2(_) => { log::error!("dropping v2 message on v1 connection"); continue; } - - let Ok(in_use) = ws_forward_frames( - &metrics, - &outgoing_frames, - workload, - num_rows, - ws_encode_binary_message(config, buf, server_message, false, &bsatn_rlb_pool).await, - ) else { - break 'send; - }; - in_use - } - OutboundMessage::V1(message) => { - if config.version != WsVersion::V1 { - log::error!("dropping v1 message for a binary websocket connection: {:?}", message); - continue; + OutboundMessage::V1(message) => { + self.encode_and_forward_v1_message(num_rows, message).await?; } + } + } + } + } + Ok(()) + } + + /// Encode a batch for protocol v2. + /// + /// v2 uses the binary server-message schema, but each logical server + /// message must still be sent as its own websocket message. + async fn encode_v2_batch( + &mut self, + message_batch: &mut Vec, + ) -> Result<(), mpsc::error::SendError> { + self.encode_binary_batch(message_batch, BinaryPayloadMode::Single).await + } + + /// Encode a batch for protocol v3. + /// + /// v3 uses the same binary server-message schema as v2, but coalesces all + /// messages currently available from the encoder input into one websocket + /// payload. + async fn encode_v3_batch( + &mut self, + message_batch: &mut Vec, + ) -> Result<(), mpsc::error::SendError> { + self.encode_binary_batch(message_batch, BinaryPayloadMode::Coalesced) + .await + } + + /// Encode binary websocket payloads from a batch of outbound messages. + /// + /// `mode` is the only protocol-specific choice here: + /// + /// - [`BinaryPayloadMode::Single`] preserves the v2 wire format by + /// flushing after each message. + /// - [`BinaryPayloadMode::Coalesced`] uses the v3 wire format by flushing + /// after the whole batch has been accumulated. + async fn encode_binary_batch( + &mut self, + message_batch: &mut Vec, + mode: BinaryPayloadMode, + ) -> Result<(), mpsc::error::SendError> { + self.binary_server_messages.clear(); + self.binary_server_messages.reserve(match mode { + BinaryPayloadMode::Single => 1, + BinaryPayloadMode::Coalesced => message_batch.len(), + }); + let mut total_rows = None; + let mut total_bytes = 0; + + for message in message_batch.drain(..) { + // Drop messages that are not valid for a binary websocket + // connection. The conversion logs the protocol mismatch. + let Some(v2_message) = v2_outbound_message(message) else { + continue; + }; - let is_large = num_rows.is_some_and(|n| n > 1024); - - let Ok(in_use) = ws_forward_frames( - &metrics, - &outgoing_frames, - workload, - num_rows, - ws_encode_message(config, buf, message, is_large, &bsatn_rlb_pool).await, - ) else { - break 'send; - }; - in_use + let message = v2_message.message; + let message_rows = v2_message.num_rows; + + let message_bytes = message_size(&message); + match mode { + BinaryPayloadMode::Coalesced => { + if v3_payload_would_exceed_limit(total_bytes, message_bytes) { + // v3 payload boundary: adding this message would cross the + // target byte limit, so flush the payload accumulated so far. + self.flush_binary_payload(&mut total_rows, &mut total_bytes).await?; } + self.append_binary_message(message, message_rows, message_bytes, &mut total_rows, &mut total_bytes); + } + BinaryPayloadMode::Single => { + // v2 payload boundary: exactly one binary server message per websocket message. + self.append_binary_message(message, message_rows, message_bytes, &mut total_rows, &mut total_bytes); + self.flush_binary_payload(&mut total_rows, &mut total_bytes).await?; } } - }; + } - if in_use_bufs.len() < BUF_POOL_CAPACITY { - in_use_bufs.push(scopeguard::guard(in_use_buf, |in_use| { - let buf = in_use.try_reclaim().expect("buffer should be unique"); - let _ = buf_pool.push(buf); - })); + // Final v3 payload boundary: flush the remaining coalesced messages. + // This is a no-op for v2 because `Single` mode flushes inside the loop. + self.flush_binary_payload(&mut total_rows, &mut total_bytes).await + } + + /// Append one v2 server message to the binary websocket payload currently being accumulated. + fn append_binary_message( + &mut self, + message: ws_v2::ServerMessage, + message_rows: Option, + message_bytes: usize, + total_rows: &mut Option, + total_bytes: &mut usize, + ) { + if let Some(message_rows) = message_rows { + // Payload metrics are emitted at websocket-payload granularity. + // In v3, one payload can contain several logical messages, so row + // counts are accumulated across the coalesced payload. + *total_rows.get_or_insert(0) += message_rows; } + self.binary_server_messages.push(message); + *total_bytes += message_bytes; + } + + /// Encode and forward the accumulated binary payload, then reset its counters. + async fn flush_binary_payload( + &mut self, + total_rows: &mut Option, + total_bytes: &mut usize, + ) -> Result<(), mpsc::error::SendError> { + if self.binary_server_messages.is_empty() { + return Ok(()); + } + let is_large = is_large_payload(*total_bytes); + self.encode_and_forward_binary_messages(total_rows.take(), is_large) + .await?; + *total_bytes = 0; + Ok(()) + } + + /// Encode and forward one v1 websocket message. + /// + /// v1 can produce either text or binary payloads depending on the client's + /// requested protocol, so it uses [`ws_encode_message`] rather than the + /// binary-only v2/v3 path. + async fn encode_and_forward_v1_message( + &mut self, + num_rows: Option, + message: impl ToProtocol + Send + 'static, + ) -> Result<(), mpsc::error::SendError> { + let config = self.config; + let bsatn_rlb_pool = self.bsatn_rlb_pool; + self.encode_and_forward_message(|buf| ws_encode_message(config, buf, message, true, bsatn_rlb_pool, num_rows)) + .await + } + + /// Encode and forward the currently accumulated binary server messages. + /// + /// This method is shared by v2 and v3. v2 calls it with exactly one + /// `binary_server_messages` entry; v3 calls it with the whole coalesced + /// batch. The actual bytes are produced by [`serialize_v3`], whose core + /// implementation also backs `serialize_v2`. + async fn encode_and_forward_binary_messages( + &mut self, + num_rows: Option, + is_large: bool, + ) -> Result<(), mpsc::error::SendError> { + let buf = self.buffers.get(); + // `spawn_rayon` requires a `'static` closure, so the message Vec cannot + // be borrowed from `self`. Move it into the closure and return the + // drained Vec afterward so its allocation is reused by the next batch. + let messages = std::mem::take(&mut self.binary_server_messages); + let compression = self.config.compression; + let bsatn_rlb_pool = self.bsatn_rlb_pool.clone(); + let (messages, timing, in_use, data) = maybe_spawn_encode(is_large, move || { + let mut messages = messages; + let (timing, in_use, data) = + time_encode(|| serialize_v3(&bsatn_rlb_pool, buf, messages.drain(..), compression)); + (messages, timing, in_use, data) + }) + .await; + self.binary_server_messages = messages; + let encoded = ws_encode_binary_frames(timing, in_use, data, num_rows); + let in_use = ws_forward_frames(self.metrics, self.outgoing_frames, encoded); + let in_use = in_use?; + self.buffers.hold(in_use); + Ok(()) + } + + /// Encode one websocket payload using a reusable serialization buffer, + /// forward its frames, then retain the buffer for later reuse. + async fn encode_and_forward_message( + &mut self, + encode: Encode, + ) -> Result<(), mpsc::error::SendError> + where + Encode: FnOnce(SerializeBuffer) -> Fut, + Fut: Future, + Frames: IntoIterator, + { + let buf = self.buffers.get(); + let in_use = ws_forward_frames(self.metrics, self.outgoing_frames, encode(buf).await)?; + self.buffers.hold(in_use); + Ok(()) } } @@ -1397,25 +1700,27 @@ async fn ws_encode_task( fn ws_forward_frames( metrics: &SendMetrics, outgoing_frames: &mpsc::UnboundedSender, - workload: Option, - num_rows: Option, - encoded: (EncodeMetrics, InUseSerializeBuffer, impl IntoIterator), + encoded: ( + EncodedPayloadMetrics, + InUseSerializeBuffer, + impl IntoIterator, + ), ) -> Result> { let (stats, in_use, frames) = encoded; - metrics.report(workload, num_rows, stats); + metrics.report(stats); frames.into_iter().try_for_each(|frame| outgoing_frames.send(frame))?; Ok(in_use) } -/// Some stats about serialization and compression. -/// -/// Returned by [`ws_encode_message`]. -struct EncodeMetrics { +/// Metrics for one encoded websocket payload. +struct EncodedPayloadMetrics { /// Time it took to serialize and (potentially) compress a message. /// Does not include scheduling overhead. timing: Duration, /// Length in bytes of the serialized and (potentially) compressed message. encoded_len: usize, + /// Number of logical rows included in the payload, if known. + num_rows: Option, } /// Encodes `message` into zero or more WebSocket [`Frame`]s. @@ -1432,7 +1737,7 @@ struct EncodeMetrics { /// of payload each, according to the rules laid out in [RFC6455], Section /// 5.4 Fragmentation. /// -/// Returns [`EncodeMetrics`], the [`InUseSerializeBuffer`] that was passed in +/// Returns [`EncodedPayloadMetrics`], the [`InUseSerializeBuffer`] that was passed in /// as `buf` for later reuse, and the [`Frame`]s. /// /// NOTE: When sending, the frames of a single message MUST NOT be interleaved @@ -1446,62 +1751,75 @@ async fn ws_encode_message( message: impl ToProtocol + Send + 'static, is_large_message: bool, bsatn_rlb_pool: &BsatnRowListBuilderPool, -) -> (EncodeMetrics, InUseSerializeBuffer, impl Iterator) { - const FRAGMENT_SIZE: usize = 4096; - - fn serialize_and_compress( - bsatn_rlb_pool: &BsatnRowListBuilderPool, - serialize_buf: SerializeBuffer, - message: impl ToProtocol + Send + 'static, - config: ClientConfig, - ) -> (Duration, InUseSerializeBuffer, DataMessage) { - let start = Instant::now(); - let (msg_alloc, msg_data) = serialize(bsatn_rlb_pool, serialize_buf, message, config); - (start.elapsed(), msg_alloc, msg_data) - } - let (timing, msg_alloc, msg_data) = if is_large_message { - let bsatn_rlb_pool = bsatn_rlb_pool.clone(); - spawn_rayon(move || serialize_and_compress(&bsatn_rlb_pool, buf, message, config)).await - } else { - serialize_and_compress(bsatn_rlb_pool, buf, message, config) - }; - - let metrics = EncodeMetrics { - timing, - encoded_len: msg_data.len(), - }; + num_rows: Option, +) -> (EncodedPayloadMetrics, InUseSerializeBuffer, impl Iterator) { + let bsatn_rlb_pool = bsatn_rlb_pool.clone(); + // Serialization/compression can dominate large subscription or query + // responses, so large payloads are offloaded to Rayon. + let (timing, in_use, msg_data) = maybe_spawn_encode(is_large_message, move || { + time_encode(|| serialize(&bsatn_rlb_pool, buf, message, config)) + }) + .await; + let encoded_len = msg_data.len(); let (data, ty) = match msg_data { DataMessage::Text(text) => (bytestring_to_utf8bytes(text).into(), Data::Text), DataMessage::Binary(bin) => (bin, Data::Binary), }; - let frames = fragment(data, ty, FRAGMENT_SIZE); + ws_encode_frames(timing, in_use, encoded_len, data, ty, num_rows) +} - (metrics, msg_alloc, frames) +/// Run `encode` on Rayon when the payload is expected to be large. +/// +/// Small payloads stay on the async task to avoid Rayon scheduling overhead. +async fn maybe_spawn_encode(is_large: bool, encode: impl FnOnce() -> T + Send + 'static) -> T { + if is_large { + spawn_rayon(encode).await + } else { + encode() + } } -async fn ws_encode_binary_message( - config: ClientConfig, - buf: SerializeBuffer, - message: ws_v2::ServerMessage, - is_large_message: bool, - bsatn_rlb_pool: &BsatnRowListBuilderPool, -) -> (EncodeMetrics, InUseSerializeBuffer, impl Iterator + use<>) { +/// Measure serialization/compression time for one websocket payload. +fn time_encode(encode: impl FnOnce() -> (InUseSerializeBuffer, T)) -> (Duration, InUseSerializeBuffer, T) { let start = Instant::now(); - let compression = config.compression; + let (in_use, data) = encode(); + (start.elapsed(), in_use, data) +} - let (in_use, data) = if is_large_message { - let bsatn_rlb_pool = bsatn_rlb_pool.clone(); - spawn_rayon(move || serialize_v2(&bsatn_rlb_pool, buf, message, compression)).await - } else { - serialize_v2(bsatn_rlb_pool, buf, message, compression) - }; +/// Build binary websocket frames and payload metrics for encoded bytes. +fn ws_encode_binary_frames( + timing: Duration, + in_use: InUseSerializeBuffer, + data: Bytes, + num_rows: Option, +) -> ( + EncodedPayloadMetrics, + InUseSerializeBuffer, + impl Iterator + use<>, +) { + ws_encode_frames(timing, in_use, data.len(), data, Data::Binary, num_rows) +} - let metrics = EncodeMetrics { - timing: start.elapsed(), - encoded_len: data.len(), +/// Build websocket frames and payload metrics for already-serialized bytes. +fn ws_encode_frames( + timing: Duration, + in_use: InUseSerializeBuffer, + encoded_len: usize, + data: Bytes, + ty: Data, + num_rows: Option, +) -> ( + EncodedPayloadMetrics, + InUseSerializeBuffer, + impl Iterator + use<>, +) { + let metrics = EncodedPayloadMetrics { + timing, + encoded_len, + num_rows, }; - let frames = fragment(data, Data::Binary, 4096); + let frames = fragment(data, ty, 4096); (metrics, in_use, frames) } @@ -1545,33 +1863,32 @@ impl ClientMessage { } } +/// Cached metric handles for the websocket send path. struct SendMetrics { - database: Identity, encode_timing: Histogram, + payload_size: Histogram, + payload_num_rows: Histogram, } impl SendMetrics { + /// Resolve metric handles for one database once per websocket send loop. fn new(database: Identity) -> Self { Self { encode_timing: WORKER_METRICS.websocket_serialize_secs.with_label_values(&database), - database, + payload_size: WORKER_METRICS.websocket_sent_msg_size.with_label_values(&database), + payload_num_rows: WORKER_METRICS.websocket_sent_num_rows.with_label_values(&database), } } - fn report(&self, workload: Option, num_rows: Option, encode: EncodeMetrics) { + /// Report one encoded websocket payload. + fn report(&self, encode: EncodedPayloadMetrics) { self.encode_timing.observe(encode.timing.as_secs_f64()); + self.payload_size.observe(encode.encoded_len as f64); - // These metrics should be updated together, - // or not at all. - if let (Some(workload), Some(num_rows)) = (workload, num_rows) { - WORKER_METRICS - .websocket_sent_num_rows - .with_label_values(&self.database, &workload) - .observe(num_rows as f64); - WORKER_METRICS - .websocket_sent_msg_size - .with_label_values(&self.database, &workload) - .observe(encode.encoded_len as f64); + if let Some(num_rows) = encode.num_rows { + // Some websocket payloads, such as control or error messages, do + // not correspond to a known logical row count. + self.payload_num_rows.observe(num_rows as f64); } } } diff --git a/crates/core/src/client/messages.rs b/crates/core/src/client/messages.rs index 798596b5bca..123c1c75d4a 100644 --- a/crates/core/src/client/messages.rs +++ b/crates/core/src/client/messages.rs @@ -196,20 +196,46 @@ pub fn serialize( /// conditional compression when configured. pub fn serialize_v2( bsatn_rlb_pool: &BsatnRowListBuilderPool, - mut buffer: SerializeBuffer, + buffer: SerializeBuffer, msg: ws_v2::ServerMessage, compression: ws_v1::Compression, +) -> (InUseSerializeBuffer, Bytes) { + serialize_v2_messages(bsatn_rlb_pool, buffer, std::iter::once(msg), compression) +} + +/// Serialize one or more [`ws_v2::ServerMessage`]s into a v3 websocket payload. +/// +/// Protocol v3 keeps the v2 message schema, but allows the uncompressed payload +/// body to contain consecutive BSATN-encoded server messages. +pub fn serialize_v3( + bsatn_rlb_pool: &BsatnRowListBuilderPool, + buffer: SerializeBuffer, + msgs: impl IntoIterator, + compression: ws_v1::Compression, +) -> (InUseSerializeBuffer, Bytes) { + serialize_v2_messages(bsatn_rlb_pool, buffer, msgs, compression) +} + +fn serialize_v2_messages( + bsatn_rlb_pool: &BsatnRowListBuilderPool, + mut buffer: SerializeBuffer, + msgs: impl IntoIterator, + compression: ws_v1::Compression, ) -> (InUseSerializeBuffer, Bytes) { let srv_msg = buffer.write_with_tag(ws_common::SERVER_MSG_COMPRESSION_TAG_NONE, |w| { - bsatn::to_writer(w.into_inner(), &msg).expect("should be able to bsatn encode v2 message"); + let out = w.into_inner(); + for msg in msgs { + write_v2_server_message(bsatn_rlb_pool, out, msg); + } }); let srv_msg_len = srv_msg.len(); + finalize_binary_serialize_buffer(buffer, srv_msg_len, compression) +} - // At this point, we no longer have a use for `msg`, - // so try to reclaim its buffers. +fn write_v2_server_message(bsatn_rlb_pool: &BsatnRowListBuilderPool, out: &mut BytesMut, msg: ws_v2::ServerMessage) { + bsatn::to_writer(out, &msg).expect("should be able to bsatn encode v2 message"); + // At this point, we no longer have a use for `msg`, so try to reclaim its buffers. msg.consume_each_list(&mut |buffer| bsatn_rlb_pool.try_put(buffer)); - - finalize_binary_serialize_buffer(buffer, srv_msg_len, compression) } #[derive(Debug, From)] diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index 14ce95c5ccf..c1847fa6d1c 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -204,8 +204,8 @@ metrics_group!( pub tokio_mean_polls_per_park: GaugeVec, #[name = spacetime_websocket_sent_msg_size_bytes] - #[help = "The size of messages sent to connected sessions"] - #[labels(db: Identity, workload: WorkloadType)] + #[help = "The size of websocket payloads sent to connected sessions"] + #[labels(db: Identity)] // Prometheus histograms have default buckets, // which broadly speaking, // are tailored to measure the response time of a network service. @@ -219,8 +219,8 @@ metrics_group!( pub websocket_sent_msg_size: HistogramVec, #[name = spacetime_websocket_sent_num_rows] - #[help = "The number of rows sent to connected sessions"] - #[labels(db: Identity, workload: WorkloadType)] + #[help = "The number of rows sent in websocket payloads"] + #[labels(db: Identity)] // Prometheus histograms have default buckets, // which broadly speaking, // are tailored to measure the response time of a network service. From bae0842d4620c2fc60dfbbb9db5bca60309f0c5c Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Wed, 20 May 2026 15:10:12 -0700 Subject: [PATCH 15/20] Clean up for Ryan's comments --- .../include/spacetimedb/http_client_impl.h | 13 ++++--------- .../include/spacetimedb/http_handler_macros.h | 2 +- .../spacetimedb/internal/runtime_registration.h | 6 ++++-- crates/bindings-cpp/include/spacetimedb/router.h | 3 +-- crates/bindings-cpp/src/internal/Module.cpp | 7 +++---- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/crates/bindings-cpp/include/spacetimedb/http_client_impl.h b/crates/bindings-cpp/include/spacetimedb/http_client_impl.h index 374b60a406c..e1ad619410b 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_client_impl.h +++ b/crates/bindings-cpp/include/spacetimedb/http_client_impl.h @@ -23,9 +23,9 @@ inline Outcome HttpClient::SendImpl(const HttpRequest& request) { // Prepare body bytes const std::vector& body_bytes = request.body.bytes; - // Call host function - // Note: For empty body, we need to pass a valid pointer, not null - const uint8_t* body_ptr = body_bytes.empty() ? reinterpret_cast("") : body_bytes.data(); + // The host ABI requires a non-null, in-bounds body pointer even when body_len == 0. + static const uint8_t empty_sentinel = 0; + const uint8_t* body_ptr = body_bytes.empty() ? &empty_sentinel : body_bytes.data(); BytesSource out[2] = {BytesSource{0}, BytesSource{0}}; Status status = procedure_http_request( @@ -40,15 +40,11 @@ inline Outcome HttpClient::SendImpl(const HttpRequest& request) { if (status.inner == 21) { // Read error message from out[0] std::vector error_bytes = Internal::ConsumeBytes(out[0]); - - LOG_INFO("HTTP: Error bytes: " + std::to_string(error_bytes.size())); - + // Decode BSATN string bsatn::Reader reader(error_bytes.data(), error_bytes.size()); std::string error_message = bsatn::deserialize(reader); - LOG_INFO("HTTP: Error message: " + error_message); - return Err(std::move(error_message)); } @@ -57,7 +53,6 @@ inline Outcome HttpClient::SendImpl(const HttpRequest& request) { return Err("HTTP requests are blocked inside transactions. Call HTTP before with_tx() or try_with_tx()."); } - LOG_INFO("HTTP: Unknown error code: " + std::to_string(status.inner)); return Err("HTTP request failed with status code: " + std::to_string(status.inner)); } diff --git a/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h b/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h index df9a1fc3a18..4cd88802f9f 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h +++ b/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h @@ -29,7 +29,7 @@ inline void RegisterHttpHandlerMacro(const char* handler_name, Func func) { [func](HandlerContext& ctx, HttpRequest request) -> HttpResponse { return func(ctx, std::move(request)); }; - RegisterHttpHandlerHandler(handler_name, reinterpret_cast(func), std::move(handler)); + RegisterHttpHandlerHandler(handler_name, func, std::move(handler)); getV10Builder().RegisterHttpHandlerDef(handler_name); } diff --git a/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h b/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h index ee00ff1d54a..005a6553172 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h @@ -20,6 +20,8 @@ struct HttpResponse; namespace Internal { +using HttpHandlerSymbol = HttpResponse (*)(HandlerContext, HttpRequest); + void RegisterReducerHandler(const std::string& name, std::function handler, std::optional lifecycle = std::nullopt); @@ -30,9 +32,9 @@ void RegisterAnonymousViewHandler(const std::string& name, void RegisterProcedureHandler(const std::string& name, std::function(ProcedureContext&, BytesSource)> handler); void RegisterHttpHandlerHandler(const std::string& name, - const void* handler_symbol, + HttpHandlerSymbol handler_symbol, std::function handler); -std::string LookupHttpHandlerName(const void* handler_symbol); +std::string LookupHttpHandlerName(HttpHandlerSymbol handler_symbol); size_t GetViewHandlerCount(); size_t GetAnonymousViewHandlerCount(); size_t GetProcedureHandlerCount(); diff --git a/crates/bindings-cpp/include/spacetimedb/router.h b/crates/bindings-cpp/include/spacetimedb/router.h index e3ee6ca8962..00808f8bdfd 100644 --- a/crates/bindings-cpp/include/spacetimedb/router.h +++ b/crates/bindings-cpp/include/spacetimedb/router.h @@ -228,8 +228,7 @@ class Router { template static std::string resolve_handler_name(Func handler) { - const void* symbol = reinterpret_cast(handler); - return Internal::LookupHttpHandlerName(symbol); + return Internal::LookupHttpHandlerName(handler); } }; diff --git a/crates/bindings-cpp/src/internal/Module.cpp b/crates/bindings-cpp/src/internal/Module.cpp index 421c4968c5d..901bb5e3e86 100644 --- a/crates/bindings-cpp/src/internal/Module.cpp +++ b/crates/bindings-cpp/src/internal/Module.cpp @@ -63,7 +63,7 @@ namespace Internal { struct HttpHandler { std::string name; - const void* symbol; + HttpHandlerSymbol symbol; std::function handler; }; static std::vector g_http_handlers; @@ -130,12 +130,12 @@ namespace Internal { } void RegisterHttpHandlerHandler(const std::string& name, - const void* handler_symbol, + HttpHandlerSymbol handler_symbol, std::function handler) { g_http_handlers.push_back({name, handler_symbol, handler}); } - std::string LookupHttpHandlerName(const void* handler_symbol) { + std::string LookupHttpHandlerName(HttpHandlerSymbol handler_symbol) { auto it = std::find_if(g_http_handlers.begin(), g_http_handlers.end(), [&](const auto& existing) { return existing.symbol == handler_symbol; }); @@ -726,4 +726,3 @@ void Module::RegisterExplicitIndexName(const std::string& source_name, const std - From c39141de6a333f7c0f62e62a4608ad902ada096b Mon Sep 17 00:00:00 2001 From: Zeke Foppa <196249+bfops@users.noreply.github.com> Date: Thu, 21 May 2026 09:21:22 -0700 Subject: [PATCH 16/20] Undo `.npmrc` in templates (#5084) # Description of Changes See https://github.com/clockworklabs/SpacetimeDB/issues/5083 for motivation. # API and ABI breaking changes None # Expected complexity level and risk 2 # Testing - [x] `cargo run -pspcaetimedb-cli -- init --template nodejs-ts` no longer creates a directory with `.npmrc` --------- Co-authored-by: Zeke Foppa --- crates/smoketests/src/lib.rs | 41 +++++++++++++++++-- .../smoketests/tests/smoketests/templates.rs | 18 +------- templates/angular-ts/.npmrc | 1 - templates/angular-ts/spacetimedb/.npmrc | 1 - templates/astro-ts/.npmrc | 1 - templates/astro-ts/spacetimedb/.npmrc | 1 - templates/basic-ts/.npmrc | 1 - templates/basic-ts/spacetimedb/.npmrc | 1 - templates/browser-ts/.npmrc | 1 - templates/browser-ts/spacetimedb/.npmrc | 1 - templates/bun-ts/.npmrc | 1 - templates/bun-ts/spacetimedb/.npmrc | 1 - templates/chat-react-ts/.npmrc | 1 - templates/chat-react-ts/spacetimedb/.npmrc | 1 - templates/deno-ts/.npmrc | 1 - templates/deno-ts/spacetimedb/.npmrc | 1 - templates/keynote-2/.npmrc | 1 - templates/keynote-2/convex-app/.npmrc | 1 - templates/keynote-2/spacetimedb/.npmrc | 1 - templates/nextjs-ts/.npmrc | 1 - templates/nextjs-ts/spacetimedb/.npmrc | 1 - templates/nodejs-ts/.npmrc | 1 - templates/nodejs-ts/spacetimedb/.npmrc | 1 - templates/nuxt-ts/.npmrc | 1 - templates/nuxt-ts/spacetimedb/.npmrc | 1 - templates/react-ts/.npmrc | 1 - templates/react-ts/spacetimedb/.npmrc | 1 - templates/remix-ts/.npmrc | 1 - templates/remix-ts/spacetimedb/.npmrc | 1 - templates/svelte-ts/.npmrc | 1 - templates/svelte-ts/spacetimedb/.npmrc | 1 - templates/tanstack-ts/.npmrc | 1 - templates/tanstack-ts/spacetimedb/.npmrc | 1 - templates/vue-ts/.npmrc | 1 - templates/vue-ts/spacetimedb/.npmrc | 1 - tools/ci/src/main.rs | 16 ++++++++ .../src/bench/publishers.rs | 28 ++++++++++++- 37 files changed, 82 insertions(+), 54 deletions(-) delete mode 100644 templates/angular-ts/.npmrc delete mode 100644 templates/angular-ts/spacetimedb/.npmrc delete mode 100644 templates/astro-ts/.npmrc delete mode 100644 templates/astro-ts/spacetimedb/.npmrc delete mode 100644 templates/basic-ts/.npmrc delete mode 100644 templates/basic-ts/spacetimedb/.npmrc delete mode 100644 templates/browser-ts/.npmrc delete mode 100644 templates/browser-ts/spacetimedb/.npmrc delete mode 100644 templates/bun-ts/.npmrc delete mode 100644 templates/bun-ts/spacetimedb/.npmrc delete mode 100644 templates/chat-react-ts/.npmrc delete mode 100644 templates/chat-react-ts/spacetimedb/.npmrc delete mode 100644 templates/deno-ts/.npmrc delete mode 100644 templates/deno-ts/spacetimedb/.npmrc delete mode 100644 templates/keynote-2/.npmrc delete mode 100644 templates/keynote-2/convex-app/.npmrc delete mode 100644 templates/keynote-2/spacetimedb/.npmrc delete mode 100644 templates/nextjs-ts/.npmrc delete mode 100644 templates/nextjs-ts/spacetimedb/.npmrc delete mode 100644 templates/nodejs-ts/.npmrc delete mode 100644 templates/nodejs-ts/spacetimedb/.npmrc delete mode 100644 templates/nuxt-ts/.npmrc delete mode 100644 templates/nuxt-ts/spacetimedb/.npmrc delete mode 100644 templates/react-ts/.npmrc delete mode 100644 templates/react-ts/spacetimedb/.npmrc delete mode 100644 templates/remix-ts/.npmrc delete mode 100644 templates/remix-ts/spacetimedb/.npmrc delete mode 100644 templates/svelte-ts/.npmrc delete mode 100644 templates/svelte-ts/spacetimedb/.npmrc delete mode 100644 templates/tanstack-ts/.npmrc delete mode 100644 templates/tanstack-ts/spacetimedb/.npmrc delete mode 100644 templates/vue-ts/.npmrc delete mode 100644 templates/vue-ts/spacetimedb/.npmrc diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 1ef17047d31..55f75b3ed73 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -280,6 +280,21 @@ pub fn pnpm_path() -> Option { PNPM_PATH.get_or_init(|| which("pnpm").ok()).clone() } +fn pnpm_minimum_release_age() -> Result { + let workspace = fs::read_to_string(workspace_root().join("pnpm-workspace.yaml"))?; + workspace + .lines() + .find_map(|line| { + line.trim() + .strip_prefix("minimumReleaseAge:")? + .trim() + .parse::() + .ok() + }) + .map(|age| age.to_string()) + .context("pnpm-workspace.yaml is missing minimumReleaseAge") +} + /// Runs a command and returns stdout as a string. pub fn run_cmd(args: &[&str], cwd: &Path) -> Result { run_cmd_inner(args, cwd, None) @@ -332,10 +347,28 @@ fn run_cmd_inner(args: &[&str], cwd: &Path, stdin_input: Option<&str>) -> Result /// Runs a `pnpm` command and returns stdout as a string. pub fn pnpm(args: &[&str], cwd: &Path) -> Result { let pnpm_path = pnpm_path().context("Could not locate pnpm")?; - let pnpm_path = pnpm_path.to_str().context("pnpm path is not valid UTF-8")?; - let mut full_args = vec![pnpm_path]; - full_args.extend(args); - run_cmd(&full_args, cwd) + let minimum_release_age = pnpm_minimum_release_age()?; + + // Smoketests often install inside temp projects created by `spacetime init`. + // Those projects intentionally do not carry the repo's .npmrc, so pass the + // repo policy through pnpm's environment variable instead. + let output = Command::new(&pnpm_path) + .args(args) + .current_dir(cwd) + .env("npm_config_minimum_release_age", minimum_release_age) + .output() + .with_context(|| format!("Failed to spawn pnpm {}", args.join(" ")))?; + + if !output.status.success() { + bail!( + "pnpm {} (in {:?}) failed:\nstdout: {}\nstderr: {}", + args.join(" "), + cwd, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) } /// Builds the local TypeScript bindings package. diff --git a/crates/smoketests/tests/smoketests/templates.rs b/crates/smoketests/tests/smoketests/templates.rs index 8de55871f58..e818000201d 100644 --- a/crates/smoketests/tests/smoketests/templates.rs +++ b/crates/smoketests/tests/smoketests/templates.rs @@ -12,7 +12,7 @@ use anyhow::{bail, Context, Result}; use regex::Regex; use serde_json::Value; -use spacetimedb_smoketests::{pnpm_path, random_string, workspace_root, Smoketest}; +use spacetimedb_smoketests::{pnpm, random_string, workspace_root, Smoketest}; use std::env; use std::fs; use std::path::{Path, PathBuf}; @@ -174,21 +174,7 @@ fn update_package_json_dependency(package_json_path: &Path, package_name: &str, /// Runs pnpm with the given arguments in the given working directory. fn run_pnpm(args: &[&str], cwd: &Path) -> Result<()> { - let pnpm = pnpm_path().context("pnpm not found")?; - let output = Command::new(&pnpm) - .args(args) - .current_dir(cwd) - .output() - .with_context(|| format!("Failed to spawn pnpm {}", args.join(" ")))?; - if !output.status.success() { - bail!( - "pnpm {} (in {:?}) failed:\nstdout: {}\nstderr: {}", - args.join(" "), - cwd, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } + pnpm(args, cwd)?; Ok(()) } diff --git a/templates/angular-ts/.npmrc b/templates/angular-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/angular-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/angular-ts/spacetimedb/.npmrc b/templates/angular-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/angular-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/astro-ts/.npmrc b/templates/astro-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/astro-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/astro-ts/spacetimedb/.npmrc b/templates/astro-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/astro-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/basic-ts/.npmrc b/templates/basic-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/basic-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/basic-ts/spacetimedb/.npmrc b/templates/basic-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/basic-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/browser-ts/.npmrc b/templates/browser-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/browser-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/browser-ts/spacetimedb/.npmrc b/templates/browser-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/browser-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/bun-ts/.npmrc b/templates/bun-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/bun-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/bun-ts/spacetimedb/.npmrc b/templates/bun-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/bun-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/chat-react-ts/.npmrc b/templates/chat-react-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/chat-react-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/chat-react-ts/spacetimedb/.npmrc b/templates/chat-react-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/chat-react-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/deno-ts/.npmrc b/templates/deno-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/deno-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/deno-ts/spacetimedb/.npmrc b/templates/deno-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/deno-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/keynote-2/.npmrc b/templates/keynote-2/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/keynote-2/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/keynote-2/convex-app/.npmrc b/templates/keynote-2/convex-app/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/keynote-2/convex-app/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/keynote-2/spacetimedb/.npmrc b/templates/keynote-2/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/keynote-2/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nextjs-ts/.npmrc b/templates/nextjs-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nextjs-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nextjs-ts/spacetimedb/.npmrc b/templates/nextjs-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nextjs-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nodejs-ts/.npmrc b/templates/nodejs-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nodejs-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nodejs-ts/spacetimedb/.npmrc b/templates/nodejs-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nodejs-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nuxt-ts/.npmrc b/templates/nuxt-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nuxt-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nuxt-ts/spacetimedb/.npmrc b/templates/nuxt-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nuxt-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/react-ts/.npmrc b/templates/react-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/react-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/react-ts/spacetimedb/.npmrc b/templates/react-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/react-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/remix-ts/.npmrc b/templates/remix-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/remix-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/remix-ts/spacetimedb/.npmrc b/templates/remix-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/remix-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/svelte-ts/.npmrc b/templates/svelte-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/svelte-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/svelte-ts/spacetimedb/.npmrc b/templates/svelte-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/svelte-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/tanstack-ts/.npmrc b/templates/tanstack-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/tanstack-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/tanstack-ts/spacetimedb/.npmrc b/templates/tanstack-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/tanstack-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/vue-ts/.npmrc b/templates/vue-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/vue-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/vue-ts/spacetimedb/.npmrc b/templates/vue-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/vue-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index e1bb914e3ff..8d20a0cae1b 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -131,6 +131,10 @@ fn is_npm_package_json(package_json: &Value) -> bool { .any(|key| package_json.get(key).is_some()) } +fn is_template_path(path: &Path) -> bool { + path.starts_with("templates") +} + fn minimum_release_age(path: &Path) -> Result { let workspace = fs::read_to_string(path)?; workspace @@ -221,6 +225,12 @@ fn check_pnpm_release_age_policy() -> Result<()> { } for npmrc_path in git_tracked_files(":(glob)**/.npmrc")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not embed this repo's package-age policy; smoketests enforce it + // at the pnpm process boundary instead. + if is_template_path(&npmrc_path) { + continue; + } let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; if found_minimum_release_age != root_minimum_release_age { bail!( @@ -233,6 +243,12 @@ fn check_pnpm_release_age_policy() -> Result<()> { } for package_json_path in git_tracked_files(":(glob)**/package.json")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not require adjacent .npmrc files for this repo's package-age + // policy; smoketests enforce it at the pnpm process boundary instead. + if is_template_path(&package_json_path) { + continue; + } let package_json = read_package_json(&package_json_path)?; if !is_npm_package_json(&package_json) { continue; diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index c2bcdcd90bc..68775ff631c 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -8,6 +8,29 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::LazyLock; +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .expect("xtask-llm-benchmark is under public/tools/xtask-llm-benchmark") + .to_path_buf() +} + +fn pnpm_minimum_release_age() -> Result { + let workspace = fs::read_to_string(workspace_root().join("pnpm-workspace.yaml"))?; + workspace + .lines() + .find_map(|line| { + line.trim() + .strip_prefix("minimumReleaseAge:")? + .trim() + .parse::() + .ok() + }) + .map(|age| age.to_string()) + .ok_or_else(|| anyhow::anyhow!("pnpm-workspace.yaml is missing minimumReleaseAge")) +} + /// Strip ANSI escape codes (color codes) from a string fn strip_ansi_codes(s: &str) -> Cow<'_, str> { static ANSI_RE: LazyLock = LazyLock::new(|| { @@ -298,7 +321,10 @@ impl Publisher for TypeScriptPublisher { .arg("install") .arg("--ignore-workspace") .current_dir(source) - .env("CI", "true"); + .env("CI", "true") + // This install runs in a materialized project with workspace config + // ignored, so pass the repo's pnpm package-age policy explicitly. + .env("npm_config_minimum_release_age", pnpm_minimum_release_age()?); // When using NODEJS_DIR, prepend it to PATH so pnpm.cmd can find node. if let Some(ref dir) = pnpm_exe && let Some(parent) = dir.parent() From 93a68ade002f3f334c69810aaabeb66ec62e49c5 Mon Sep 17 00:00:00 2001 From: Jeff Rooks Date: Thu, 21 May 2026 13:56:04 -0400 Subject: [PATCH 17/20] chore: enable HTTP/2 support for backend server (#5027) # Description of Changes Enables Axum's `http2` feature for SpacetimeDB, which enables Hyper's HTTP/2 support in the standalone/backend HTTP server. This allows clients that explicitly speak cleartext HTTP/2 (`h2c`) to connect to the local standalone server over HTTP/2. _Note: public HTTPS deployments may also require edge proxy or load balancer configuration, such as enabling HTTP/2 ALPN on nginx. For example, `curl --http2 -v https://maincloud.spacetimedb.com/v1/ping` currently negotiates `http/1.1` at the public edge._ Refs #4964 # API and ABI breaking changes None. # Expected complexity level and risk 1 This is a dependency feature flag change. It should not affect existing HTTP/1.1 clients, and the server continues to accept HTTP/1.1 requests. # Testing - Build the standalone server: `cargo build -p spacetimedb-standalone` - Start the local standalone server and verify HTTP/2 cleartext prior knowledge support: `cargo run -p spacetimedb-standalone -- start --data-dir /tmp/spacetimedb-data --jwt-key-dir /tmp/spacetimedb-data --listen-addr 127.0.0.1:3000 --in-memory --non-interactive` - Test using curl http prior knowledge flag with the local instance: `curl --http2-prior-knowledge -v http://127.0.0.1:3000/v1/ping` **Output** ```bash * Trying 127.0.0.1:3000... * Established connection to 127.0.0.1 (127.0.0.1 port 3000) from 127.0.0.1 port 42662 * using HTTP/2 * [HTTP/2] [1] OPENED stream for http://127.0.0.1:3000/v1/ping * [HTTP/2] [1] [:method: GET] * [HTTP/2] [1] [:scheme: http] * [HTTP/2] [1] [:authority: 127.0.0.1:3000] * [HTTP/2] [1] [:path: /v1/ping] * [HTTP/2] [1] [user-agent: curl/8.20.0] * [HTTP/2] [1] [accept: */*] > GET /v1/ping HTTP/2 > Host: 127.0.0.1:3000 > User-Agent: curl/8.20.0 > Accept: */* > * Request completely sent off < HTTP/2 200 < vary: origin, access-control-request-method, access-control-request-headers < access-control-allow-origin: * < content-length: 0 < date: Fri, 15 May 2026 10:25:12 GMT < * Connection #0 to host 127.0.0.1:3000 left intact ``` --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d1488e186df..3d1d20080d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,7 +159,7 @@ arrayvec = "0.7.2" async-channel = "2.5" async-stream = "0.3.6" async-trait = "0.1.68" -axum = { version = "0.7", features = ["tracing"] } +axum = { version = "0.7", features = ["tracing", "http2"] } axum-extra = { version = "0.9", features = ["typed-header"] } backtrace = "0.3.66" base64 = "0.21.2" From d3b4a96132e2b5ada0238e17f6ef81df47b97ba2 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Thu, 21 May 2026 15:13:29 -0400 Subject: [PATCH 18/20] Drop serde_json arbitrary_precision from workspace (#5001) Closes #4989 We were not using `arbitrary_precision` in the workspace: I only found the workspace dependency declaration itself plus a normal `serde_json::Number::from(3u8)` call in `crates/cli/src/spacetime_config.rs`, which does not require the feature. ## Summary - remove `serde_json/arbitrary_precision` from the workspace dependency declaration - keep `raw_value`, which is the only serde_json feature the workspace is clearly using here - avoid forcing a non-additive serde_json feature onto downstream consumers via feature unification ## Validation - `cargo tree -e features -p serde_json` - `cargo check -p spacetimedb-sdk -p spacetimedb-lib -p spacetimedb-sats -p spacetimedb-schema` --------- Co-authored-by: clockwork-labs-bot Co-authored-by: Zeke Foppa <196249+bfops@users.noreply.github.com> Co-authored-by: Zeke Foppa --- Cargo.toml | 2 +- crates/bindings/tests/deps.rs | 33 +++++++++++++++++++++++++ crates/cli/Cargo.toml | 2 +- crates/cli/src/subcommands/subscribe.rs | 29 ++++++++++++++++++++++ crates/lib/src/connection_id.rs | 11 ++++----- 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3d1d20080d6..3eac1a1bc1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,7 +278,7 @@ second-stack = "0.3" self-replace = "1.5" semver = "1" serde = { version = "1.0.136", features = ["derive"] } -serde_json = { version = "1.0.128", features = ["raw_value", "arbitrary_precision"] } +serde_json = { version = "1.0.128", features = ["raw_value"] } serde_path_to_error = "0.1.9" serde_with = { version = "3.3.0", features = ["base64", "hex"] } serial_test = "2.0.0" diff --git a/crates/bindings/tests/deps.rs b/crates/bindings/tests/deps.rs index 2383fdc7a1e..ccbb2b4b685 100644 --- a/crates/bindings/tests/deps.rs +++ b/crates/bindings/tests/deps.rs @@ -25,6 +25,39 @@ fn deptree_snapshot() -> std::io::Result<()> { Ok(()) } +#[test] +fn serde_json_arbitrary_precision_feature_boundaries() { + // https://github.com/clockworklabs/SpacetimeDB/issues/4989 + // `serde_json/arbitrary_precision` is fine for internal tooling like the CLI, + // but it should not be forced onto users compiling the Rust SDK or module + // bindings. Cargo features are additive, so guard those public dependency + // graphs explicitly. + + // The CLI opts into it because `spacetime subscribe` reformats JSON rows + // through `serde_json::Value`; without arbitrary precision, large SATS + // integers like `ConnectionId` can be rounded before typed deserialization. + assert_serde_json_arbitrary_precision("cargo tree -p spacetimedb-cli -e features,no-dev -i serde_json", true); + assert_serde_json_arbitrary_precision( + "cargo tree -p spacetimedb -e features,no-dev --target wasm32-unknown-unknown -i serde_json", + false, + ); + assert_serde_json_arbitrary_precision("cargo tree -p spacetimedb-sdk -e features,no-dev -i serde_json", false); + assert_serde_json_arbitrary_precision( + "cargo tree -p spacetimedb-sdk -e features,no-dev --features browser --target wasm32-unknown-unknown -i serde_json", + false, + ); +} + +#[track_caller] +fn assert_serde_json_arbitrary_precision(cmd: &str, expected: bool) { + let tree = run_cmd(cmd); + assert_eq!( + tree.contains("serde_json feature \"arbitrary_precision\""), + expected, + "`arbitrary_precision` expectation failed for `{cmd}`:\n{tree}" + ); +} + // runs a command string, split on spaces #[track_caller] fn run_cmd(cmd: &str) -> String { diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index b2b8d7e1d89..bb71fb8b6e5 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -58,7 +58,7 @@ regex.workspace = true reqwest.workspace = true rustyline.workspace = true serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true, features = ["raw_value", "preserve_order"] } +serde_json = { workspace = true, features = ["raw_value", "preserve_order", "arbitrary_precision"] } serde_with = { workspace = true, features = ["chrono_0_4"] } slab.workspace = true syntect.workspace = true diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index f8cbc61e0cf..6586d8edd5f 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -767,3 +767,32 @@ fn reformat_bsatn_rows( }) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_lib::sats::{ + algebraic_value::de::ValueDeserializer, de::Deserialize, GroundSpacetimeType, Typespace, WithTypespace, + }; + use spacetimedb_lib::ConnectionId; + + #[test] + fn serde_json_value_preserves_connection_id_u128() -> anyhow::Result<()> { + // `spacetime subscribe` reformats JSON rows through `serde_json::Value` + // before typed SATS deserialization. The CLI enables + // `serde_json/arbitrary_precision` so large `ConnectionId` values do not + // get rounded while inside `Value`. + let conn_id = ConnectionId::from_u128(u64::MAX as u128 + 1); + let json = serde_json::to_string(&SerializeWrapper::new(&conn_id))?; + let row = serde_json::from_str::(&json)?; + + let typespace = Typespace::default(); + let conn_id_ty = ConnectionId::get_type(); + let conn_id_ty = WithTypespace::new(&typespace, &conn_id_ty); + let de = serde::de::DeserializeSeed::deserialize(SeedWrapper(conn_id_ty), row)?; + let de = ConnectionId::deserialize(ValueDeserializer::new(de)).unwrap(); + + assert_eq!(conn_id, de); + Ok(()) + } +} diff --git a/crates/lib/src/connection_id.rs b/crates/lib/src/connection_id.rs index 2055e389127..13fda72362f 100644 --- a/crates/lib/src/connection_id.rs +++ b/crates/lib/src/connection_id.rs @@ -300,10 +300,9 @@ mod tests { use crate::WithTypespace; proptest! { - /// Tests the round-trip used when using the `spacetime subscribe` - /// CLI command. - /// Somewhat confusingly, this is distinct from the ser-de path - /// in `test_serde_roundtrip`. + /// Tests deserialization through the serde seed adapter without + /// relying on `serde_json::Value` to preserve arbitrary-precision + /// numbers. #[test] fn test_wrapper_roundtrip(val: u128) { let conn_id = ConnectionId::from_u128(val); @@ -313,12 +312,12 @@ mod tests { let empty = Typespace::default(); let conn_id_ty = ConnectionId::get_type(); let conn_id_ty = WithTypespace::new(&empty, &conn_id_ty); - let row = serde_json::from_str::(&ser[..])?; + let mut de = serde_json::Deserializer::from_str(&ser); let de = ::serde::de::DeserializeSeed::deserialize( crate::de::serde::SeedWrapper( conn_id_ty ), - row)?; + &mut de)?; let de = ConnectionId::deserialize(ValueDeserializer::new(de)).unwrap(); prop_assert_eq!(conn_id, de); } From 7d4a8c1efef10618c74f45e9c0c32fe342896f24 Mon Sep 17 00:00:00 2001 From: JasonAtClockwork Date: Thu, 21 May 2026 13:04:27 -0700 Subject: [PATCH 19/20] Fixes to get minimum requirement for namespace changes --- .../autogen/RawMiscModuleExportV9.g.h | 2 +- .../internal/autogen/RawModuleDef.g.h | 2 +- .../autogen/RawModuleDefV10Section.g.h | 19 ++--- .../internal/autogen/RawModuleDefV8.g.h | 4 +- .../internal/autogen/RawModuleDefV9.g.h | 6 +- .../internal/autogen/RawModuleMountV10.g.h | 32 ++++++++ .../internal/autogen/RawProcedureDefV10.g.h | 4 +- .../internal/autogen/RawProcedureDefV9.g.h | 2 +- .../internal/autogen/RawTableDefV10.g.h | 2 +- .../internal/autogen/RawTableDefV9.g.h | 4 +- .../bindings-cpp/src/internal/v10_builder.cpp | 4 +- crates/codegen/src/cpp.rs | 78 ++++++++++++++++--- 12 files changed, 124 insertions(+), 35 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleMountV10.g.h diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h index b8e391591df..654260b3532 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h @@ -12,9 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "RawProcedureDefV9.g.h" #include "RawViewDefV9.g.h" #include "RawColumnDefaultValueV9.g.h" -#include "RawProcedureDefV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h index c7f144eb07d..08b82e677c3 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h @@ -13,8 +13,8 @@ #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" #include "RawModuleDefV8.g.h" -#include "RawModuleDefV9.g.h" #include "RawModuleDefV10.g.h" +#include "RawModuleDefV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index 5ddbde5e11a..7480d2c2b43 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -12,21 +12,22 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawViewDefV10.g.h" -#include "CaseConversionPolicy.g.h" -#include "RawScheduleDefV10.g.h" -#include "RawTableDefV10.g.h" -#include "Typespace.g.h" #include "RawReducerDefV10.g.h" +#include "RawHttpRouteDefV10.g.h" +#include "RawScheduleDefV10.g.h" +#include "RawModuleMountV10.g.h" +#include "RawViewDefV10.g.h" #include "RawProcedureDefV10.g.h" -#include "RawTypeDefV10.g.h" +#include "Typespace.g.h" #include "RawLifeCycleReducerDefV10.g.h" -#include "RawRowLevelSecurityDefV9.g.h" +#include "RawTableDefV10.g.h" +#include "CaseConversionPolicy.g.h" #include "ExplicitNames.g.h" +#include "RawTypeDefV10.g.h" +#include "RawRowLevelSecurityDefV9.g.h" #include "RawHttpHandlerDefV10.g.h" -#include "RawHttpRouteDefV10.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h index e856af0fec5..2418ed03e8d 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h @@ -12,10 +12,10 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "ReducerDef.g.h" -#include "MiscModuleExport.g.h" #include "Typespace.g.h" #include "TableDesc.g.h" +#include "MiscModuleExport.g.h" +#include "ReducerDef.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h index 9ab21147e08..cfcee4fe7ec 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h @@ -12,12 +12,12 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "RawRowLevelSecurityDefV9.g.h" #include "RawTableDefV9.g.h" -#include "RawTypeDefV9.g.h" -#include "RawMiscModuleExportV9.g.h" #include "Typespace.g.h" #include "RawReducerDefV9.g.h" -#include "RawRowLevelSecurityDefV9.g.h" +#include "RawTypeDefV9.g.h" +#include "RawMiscModuleExportV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleMountV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleMountV10.g.h new file mode 100644 index 00000000000..0b1ea9dc053 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleMountV10.g.h @@ -0,0 +1,32 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { +struct RawModuleDefV10; +} // namespace SpacetimeDB::Internal + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawModuleMountV10) { + std::string namespace_; + std::shared_ptr module; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, namespace_); + ::SpacetimeDB::bsatn::serialize(writer, *module); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(namespace_, module) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h index f316264fc5c..463b0ec78b8 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h @@ -12,9 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "FunctionVisibility.g.h" -#include "ProductType.g.h" #include "AlgebraicType.g.h" +#include "ProductType.g.h" +#include "FunctionVisibility.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h index 667d9864a2a..a49d9d78970 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h @@ -12,8 +12,8 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "ProductType.g.h" #include "AlgebraicType.g.h" +#include "ProductType.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h index 715364b13cf..7b5b0f10ea3 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h @@ -13,9 +13,9 @@ #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" #include "RawIndexDefV10.g.h" -#include "TableType.g.h" #include "TableAccess.g.h" #include "RawColumnDefaultValueV10.g.h" +#include "TableType.g.h" #include "RawConstraintDefV10.g.h" #include "RawSequenceDefV10.g.h" diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h index a69a502fb0e..5fe2e061945 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h @@ -12,10 +12,10 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawScheduleDefV9.g.h" -#include "RawSequenceDefV9.g.h" #include "TableType.g.h" +#include "RawScheduleDefV9.g.h" #include "TableAccess.g.h" +#include "RawSequenceDefV9.g.h" #include "RawConstraintDefV9.g.h" #include "RawIndexDefV9.g.h" diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index eb22114e8b9..bde38715deb 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -286,12 +286,12 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { } if (!http_handlers_.empty()) { RawModuleDefV10Section section_http_handlers; - section_http_handlers.set<11>(http_handlers_); + section_http_handlers.set<12>(http_handlers_); v10_module.sections.push_back(std::move(section_http_handlers)); } if (!http_routes_.empty()) { RawModuleDefV10Section section_http_routes; - section_http_routes.set<12>(http_routes_); + section_http_routes.set<13>(http_routes_); v10_module.sections.push_back(std::move(section_http_routes)); } if (!row_level_security_.empty()) { diff --git a/crates/codegen/src/cpp.rs b/crates/codegen/src/cpp.rs index 3f20b4d271a..c007ea8c108 100644 --- a/crates/codegen/src/cpp.rs +++ b/crates/codegen/src/cpp.rs @@ -19,6 +19,40 @@ pub struct Cpp<'opts> { } impl<'opts> Cpp<'opts> { + fn cpp_field_name<'a>(&self, field_name: &'a str) -> &'a str { + match field_name { + "alignas" | "alignof" | "and" | "and_eq" | "asm" | "atomic_cancel" | "atomic_commit" + | "atomic_noexcept" | "auto" | "bitand" | "bitor" | "bool" | "break" | "case" + | "catch" | "char" | "char8_t" | "char16_t" | "char32_t" | "class" | "compl" + | "concept" | "const" | "consteval" | "constexpr" | "constinit" | "const_cast" + | "continue" | "co_await" | "co_return" | "co_yield" | "decltype" | "default" + | "delete" | "do" | "double" | "dynamic_cast" | "else" | "enum" | "explicit" + | "export" | "extern" | "false" | "float" | "for" | "friend" | "goto" | "if" + | "inline" | "int" | "long" | "mutable" | "namespace" | "new" | "noexcept" + | "not" | "not_eq" | "nullptr" | "operator" | "or" | "or_eq" | "private" + | "protected" | "public" | "register" | "reinterpret_cast" | "requires" + | "return" | "short" | "signed" | "sizeof" | "static" | "static_assert" + | "static_cast" | "struct" | "switch" | "template" | "this" | "thread_local" + | "throw" | "true" | "try" | "typedef" | "typeid" | "typename" | "union" + | "unsigned" | "using" | "virtual" | "void" | "volatile" | "wchar_t" | "while" + | "xor" | "xor_eq" => "", + _ => field_name, + } + } + + fn write_cpp_field_name(&self, output: &mut String, field_name: &str) -> fmt::Result { + let escaped = self.cpp_field_name(field_name); + if escaped.is_empty() { + write!(output, "{}_", field_name) + } else { + write!(output, "{}", escaped) + } + } + + fn is_recursive_mount_module_field(&self, type_name: &str, field_name: &str) -> bool { + type_name == "RawModuleMountV10" && field_name == "module" + } + fn write_header_comment(&self, output: &mut String) { writeln!( output, @@ -148,8 +182,16 @@ impl<'opts> Cpp<'opts> { // Write fields only for (field_name, field_type) in &product.elements { write!(output, " ").unwrap(); - self.write_algebraic_type(output, module, field_type).unwrap(); - writeln!(output, " {};", field_name).unwrap(); + if self.is_recursive_mount_module_field(type_name, field_name) { + // Temporary special-case to preserve the recursive RawModuleMountV10 -> + // RawModuleDefV10 shape while breaking the include cycle in generated C++. + write!(output, "std::shared_ptr<{}::RawModuleDefV10>", self.namespace).unwrap(); + } else { + self.write_algebraic_type(output, module, field_type).unwrap(); + } + write!(output, " ").unwrap(); + self.write_cpp_field_name(output, field_name).unwrap(); + writeln!(output, ";").unwrap(); } writeln!(output).unwrap(); @@ -161,23 +203,30 @@ impl<'opts> Cpp<'opts> { ) .unwrap(); for (field_name, _) in &product.elements { - writeln!( - output, - " ::SpacetimeDB::bsatn::serialize(writer, {});", - field_name - ) - .unwrap(); + if self.is_recursive_mount_module_field(type_name, field_name) { + write!(output, " ::SpacetimeDB::bsatn::serialize(writer, *").unwrap(); + self.write_cpp_field_name(output, field_name).unwrap(); + writeln!(output, ");").unwrap(); + } else { + write!(output, " ::SpacetimeDB::bsatn::serialize(writer, ").unwrap(); + self.write_cpp_field_name(output, field_name).unwrap(); + writeln!(output, ");").unwrap(); + } } writeln!(output, " }}").unwrap(); // Generate equality method - if !product.elements.is_empty() { + if type_name == "RawModuleMountV10" { + // Pointer equality is sufficient for this internal autogen type. Mounts are not + // emitted by the C++ module path yet; this exists to keep the schema shape aligned. + writeln!(output, " SPACETIMEDB_PRODUCT_TYPE_EQUALITY(namespace_, module)").unwrap(); + } else if !product.elements.is_empty() { write!(output, " SPACETIMEDB_PRODUCT_TYPE_EQUALITY(").unwrap(); for (i, (field_name, _)) in product.elements.iter().enumerate() { if i > 0 { write!(output, ", ").unwrap(); } - write!(output, "{}", field_name).unwrap(); + self.write_cpp_field_name(output, field_name).unwrap(); } writeln!(output, ")").unwrap(); } @@ -493,13 +542,20 @@ impl Lang for Cpp<'_> { None => HashSet::new(), }; + let type_name = name.to_string(); for dep in deps { - if dep != name.to_string() { + if dep != type_name && !(type_name == "RawModuleMountV10" && dep == "RawModuleDefV10") { writeln!(output, "#include \"{}.g.h\"", dep).unwrap(); } } writeln!(output).unwrap(); + if type_name == "RawModuleMountV10" { + writeln!(output, "namespace {} {{", self.namespace).unwrap(); + writeln!(output, "struct RawModuleDefV10;").unwrap(); + writeln!(output, "}} // namespace {}", self.namespace).unwrap(); + writeln!(output).unwrap(); + } writeln!(output, "namespace {} {{", self.namespace).unwrap(); writeln!(output).unwrap(); From 8c36f6e58e6d54453cfcf30894eb0d0cbf8cde59 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Thu, 21 May 2026 17:03:21 -0400 Subject: [PATCH 20/20] Format C++ codegen --- crates/codegen/src/cpp.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/crates/codegen/src/cpp.rs b/crates/codegen/src/cpp.rs index c007ea8c108..9ddbeae8bb0 100644 --- a/crates/codegen/src/cpp.rs +++ b/crates/codegen/src/cpp.rs @@ -22,19 +22,16 @@ impl<'opts> Cpp<'opts> { fn cpp_field_name<'a>(&self, field_name: &'a str) -> &'a str { match field_name { "alignas" | "alignof" | "and" | "and_eq" | "asm" | "atomic_cancel" | "atomic_commit" - | "atomic_noexcept" | "auto" | "bitand" | "bitor" | "bool" | "break" | "case" - | "catch" | "char" | "char8_t" | "char16_t" | "char32_t" | "class" | "compl" - | "concept" | "const" | "consteval" | "constexpr" | "constinit" | "const_cast" - | "continue" | "co_await" | "co_return" | "co_yield" | "decltype" | "default" - | "delete" | "do" | "double" | "dynamic_cast" | "else" | "enum" | "explicit" - | "export" | "extern" | "false" | "float" | "for" | "friend" | "goto" | "if" - | "inline" | "int" | "long" | "mutable" | "namespace" | "new" | "noexcept" - | "not" | "not_eq" | "nullptr" | "operator" | "or" | "or_eq" | "private" - | "protected" | "public" | "register" | "reinterpret_cast" | "requires" - | "return" | "short" | "signed" | "sizeof" | "static" | "static_assert" - | "static_cast" | "struct" | "switch" | "template" | "this" | "thread_local" - | "throw" | "true" | "try" | "typedef" | "typeid" | "typename" | "union" - | "unsigned" | "using" | "virtual" | "void" | "volatile" | "wchar_t" | "while" + | "atomic_noexcept" | "auto" | "bitand" | "bitor" | "bool" | "break" | "case" | "catch" | "char" + | "char8_t" | "char16_t" | "char32_t" | "class" | "compl" | "concept" | "const" | "consteval" + | "constexpr" | "constinit" | "const_cast" | "continue" | "co_await" | "co_return" | "co_yield" + | "decltype" | "default" | "delete" | "do" | "double" | "dynamic_cast" | "else" | "enum" | "explicit" + | "export" | "extern" | "false" | "float" | "for" | "friend" | "goto" | "if" | "inline" | "int" + | "long" | "mutable" | "namespace" | "new" | "noexcept" | "not" | "not_eq" | "nullptr" | "operator" + | "or" | "or_eq" | "private" | "protected" | "public" | "register" | "reinterpret_cast" | "requires" + | "return" | "short" | "signed" | "sizeof" | "static" | "static_assert" | "static_cast" | "struct" + | "switch" | "template" | "this" | "thread_local" | "throw" | "true" | "try" | "typedef" | "typeid" + | "typename" | "union" | "unsigned" | "using" | "virtual" | "void" | "volatile" | "wchar_t" | "while" | "xor" | "xor_eq" => "", _ => field_name, }