diff --git a/.github/workflows/attach-artifacts.yml b/.github/workflows/attach-artifacts.yml index 23ad34343de..90a331387fe 100644 --- a/.github/workflows/attach-artifacts.yml +++ b/.github/workflows/attach-artifacts.yml @@ -17,10 +17,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Download artifacts from private base URL + - name: Download artifacts from AWS base URL env: RELEASE_TAG: ${{ github.event.inputs.release_tag }} - BASE_URL: ${{ secrets.ARTIFACT_BASE_URL }} + BASE_URL: https://${{ vars.AWS_BUCKET }}.s3.amazonaws.com/refs/tags run: | set -euo pipefail diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 2785a84d95a..1a934842bfb 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -2,7 +2,6 @@ on: push: branches: - master - - jgilles/fix-callgrind-again workflow_dispatch: inputs: @@ -24,8 +23,10 @@ jobs: benchmark: name: run criterion benchmarks runs-on: benchmarks-runner + # disable until we fix the benchmarks + if: false # filter for a comment containing 'benchmarks please' - if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} + #if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} env: PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }} steps: @@ -185,8 +186,10 @@ jobs: container: image: rust:1.93.0 options: --privileged + # disable until we fix the benchmarks + if: false # filter for a comment containing 'benchmarks please' - if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} + #if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }} env: PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }} steps: diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 38632014506..1aa47cb0eae 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -142,12 +142,11 @@ jobs: run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT id: extract_branch - - name: Upload to DO Spaces + - name: Upload to AWS S3 uses: shallwefootball/s3-upload-action@master with: aws_key_id: ${{ secrets.AWS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} aws_bucket: ${{ vars.AWS_BUCKET }} source_dir: build - endpoint: https://nyc3.digitaloceanspaces.com destination_dir: ${{ steps.extract_branch.outputs.branch }} diff --git a/Cargo.toml b/Cargo.toml index d1488e186df..3eac1a1bc1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,7 +159,7 @@ arrayvec = "0.7.2" async-channel = "2.5" async-stream = "0.3.6" async-trait = "0.1.68" -axum = { version = "0.7", features = ["tracing"] } +axum = { version = "0.7", features = ["tracing", "http2"] } axum-extra = { version = "0.9", features = ["typed-header"] } backtrace = "0.3.66" base64 = "0.21.2" @@ -278,7 +278,7 @@ second-stack = "0.3" self-replace = "1.5" semver = "1" serde = { version = "1.0.136", features = ["derive"] } -serde_json = { version = "1.0.128", features = ["raw_value", "arbitrary_precision"] } +serde_json = { version = "1.0.128", features = ["raw_value"] } serde_path_to_error = "0.1.9" serde_with = { version = "3.3.0", features = ["base64", "hex"] } serial_test = "2.0.0" diff --git a/crates/bindings-cpp/CMakeLists.txt b/crates/bindings-cpp/CMakeLists.txt index 35a4e0a1701..a66b50ad9e2 100644 --- a/crates/bindings-cpp/CMakeLists.txt +++ b/crates/bindings-cpp/CMakeLists.txt @@ -30,6 +30,7 @@ target_sources(spacetimedb_cpp_library PRIVATE ${LIBRARY_SOURCES}) # Require C++20 for consumers of this library without forcing global flags target_compile_features(spacetimedb_cpp_library PUBLIC cxx_std_20) +target_compile_definitions(spacetimedb_cpp_library PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) # Set include directories target_include_directories(spacetimedb_cpp_library @@ -60,46 +61,5 @@ if(PROJECT_IS_TOP_LEVEL) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) endif() -# ---- Tests ---- -# Default: ON only when building this project directly; OFF when used via FetchContent/add_subdirectory -if(CMAKE_VERSION VERSION_LESS 3.21) - # Fallback heuristic for older CMake - set(_is_top_level FALSE) - if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - set(_is_top_level TRUE) - endif() -else() - set(_is_top_level ${PROJECT_IS_TOP_LEVEL}) -endif() - -option(BUILD_TESTS "Build the test suite" ${_is_top_level}) - -if(BUILD_TESTS AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - enable_testing() - - # Add test executable - add_executable(test_bsatn tests/main.cpp tests/module_library_unit_tests.cpp) - - # Link against the module library - target_link_libraries(test_bsatn PRIVATE spacetimedb_cpp_library) - - # Set C++20 standard for tests - target_compile_features(test_bsatn PRIVATE cxx_std_20) - - # Add test to CTest - add_test(NAME bsatn_tests COMMAND test_bsatn) - - # Add verbose test variant - add_test(NAME bsatn_tests_verbose COMMAND test_bsatn -v) - - # Set test properties - set_tests_properties(bsatn_tests PROPERTIES - TIMEOUT 30 - LABELS "unit" - ) - - set_tests_properties(bsatn_tests_verbose PROPERTIES - TIMEOUT 30 - LABELS "unit;verbose" - ) -endif() +# Unit/compile/smoke test harnesses live under `tests/` as standalone runners +# rather than being built through the top-level library CMake target. diff --git a/crates/bindings-cpp/include/spacetimedb.h b/crates/bindings-cpp/include/spacetimedb.h index a36d96074a8..4ed389d06fa 100644 --- a/crates/bindings-cpp/include/spacetimedb.h +++ b/crates/bindings-cpp/include/spacetimedb.h @@ -126,6 +126,11 @@ // Procedure context and macros #include "spacetimedb/procedure_macros.h" +#ifdef SPACETIMEDB_UNSTABLE_FEATURES +#include "spacetimedb/handler_context.h" +#include "spacetimedb/router.h" +#include "spacetimedb/http_handler_macros.h" +#endif // ============================================================================= // VIEW SYSTEM diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 54cefc8e9c3..e1aa12ac2de 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -216,6 +216,37 @@ int16_t __call_reducer__( BytesSource args, BytesSink error); +STDB_EXPORT(__call_view__) +int16_t __call_view__( + uint32_t id, + uint64_t sender_0, uint64_t sender_1, uint64_t sender_2, uint64_t sender_3, + BytesSource args, + BytesSink result); + +STDB_EXPORT(__call_view_anon__) +int16_t __call_view_anon__( + uint32_t id, + BytesSource args, + BytesSink result); + +STDB_EXPORT(__call_procedure__) +int16_t __call_procedure__( + uint32_t id, + uint64_t sender_0, uint64_t sender_1, uint64_t sender_2, uint64_t sender_3, + uint64_t conn_id_0, uint64_t conn_id_1, + uint64_t timestamp_microseconds, + BytesSource args_source, + BytesSink result_sink); + +STDB_EXPORT(__call_http_handler__) +int16_t __call_http_handler__( + uint32_t id, + uint64_t timestamp_microseconds, + BytesSource request_source, + BytesSource request_body_source, + BytesSink response_sink, + BytesSink response_body_sink); + // ======================================================================== // WASI SHIMS // ======================================================================== diff --git a/crates/bindings-cpp/include/spacetimedb/handler_context.h b/crates/bindings-cpp/include/spacetimedb/handler_context.h new file mode 100644 index 00000000000..cb9b4abe84c --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/handler_context.h @@ -0,0 +1,92 @@ +#ifndef SPACETIMEDB_HANDLER_CONTEXT_H +#define SPACETIMEDB_HANDLER_CONTEXT_H + +#ifndef SPACETIMEDB_UNSTABLE_FEATURES +#error "spacetimedb/handler_context.h requires SPACETIMEDB_UNSTABLE_FEATURES to be enabled" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { + +struct HandlerContext { + Timestamp timestamp; + HttpClient http; + +private: + mutable std::shared_ptr rng_instance; + mutable uint32_t counter_uuid_ = 0; + +public: + HandlerContext() = default; + explicit HandlerContext(Timestamp t) : timestamp(t) {} + + Identity identity() const { + std::array id_bytes; + ::identity(id_bytes.data()); + return Identity(id_bytes); + } + + StdbRng& rng() const { + if (!rng_instance) { + rng_instance = std::make_shared(timestamp); + } + return *rng_instance; + } + + Uuid new_uuid_v4() const { + std::array random_bytes; + rng().fill_bytes(random_bytes.data(), random_bytes.size()); + return Uuid::from_random_bytes_v4(random_bytes); + } + + Uuid new_uuid_v7() const { + std::array random_bytes; + rng().fill_bytes(random_bytes.data(), random_bytes.size()); + return Uuid::from_counter_v7(counter_uuid_, timestamp, random_bytes); + } + +#ifdef SPACETIMEDB_UNSTABLE_FEATURES + template + auto with_tx(Func&& body) -> decltype(body(std::declval())) { + auto make_reducer_ctx = [](Timestamp tx_timestamp) { + return ReducerContext( + Identity{}, + std::nullopt, + tx_timestamp, + AuthCtx::internal() + ); + }; + return Internal::with_tx(make_reducer_ctx, body); + } + + template + auto try_with_tx(Func&& body) -> decltype(body(std::declval())) { + auto make_reducer_ctx = [](Timestamp tx_timestamp) { + return ReducerContext( + Identity{}, + std::nullopt, + tx_timestamp, + AuthCtx::internal() + ); + }; + return Internal::try_with_tx(make_reducer_ctx, body); + } +#endif +}; + +} // namespace SpacetimeDB + +#endif // SPACETIMEDB_HANDLER_CONTEXT_H diff --git a/crates/bindings-cpp/include/spacetimedb/http.h b/crates/bindings-cpp/include/spacetimedb/http.h index cb51acafa0a..82a55beeb32 100644 --- a/crates/bindings-cpp/include/spacetimedb/http.h +++ b/crates/bindings-cpp/include/spacetimedb/http.h @@ -3,6 +3,10 @@ #pragma once +#ifndef SPACETIMEDB_UNSTABLE_FEATURES +#error "spacetimedb/http.h requires SPACETIMEDB_UNSTABLE_FEATURES to be enabled" +#endif + #include #include #include @@ -312,8 +316,10 @@ class HttpClient { } // namespace SpacetimeDB -// Include implementation after class definition to avoid circular dependencies -#ifdef SPACETIMEDB_UNSTABLE_FEATURES +// Include implementation dependencies after class definition to avoid circular dependencies +#if defined(SPACETIMEDB_UNSTABLE_FEATURES) && !defined(SPACETIMEDB_HTTP_CONVERT_H) +#include "spacetimedb/logger.h" +#include "spacetimedb/http_convert.h" #include "spacetimedb/http_client_impl.h" #endif diff --git a/crates/bindings-cpp/include/spacetimedb/http_client_impl.h b/crates/bindings-cpp/include/spacetimedb/http_client_impl.h index ff3e0faf3df..e1ad619410b 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_client_impl.h +++ b/crates/bindings-cpp/include/spacetimedb/http_client_impl.h @@ -7,7 +7,7 @@ #include "spacetimedb/http_convert.h" #include "spacetimedb/abi/abi.h" #include "spacetimedb/bsatn/bsatn.h" -#include "spacetimedb/internal/Module.h" +#include "spacetimedb/internal/runtime_registration.h" namespace SpacetimeDB { @@ -23,9 +23,9 @@ inline Outcome HttpClient::SendImpl(const HttpRequest& request) { // Prepare body bytes const std::vector& body_bytes = request.body.bytes; - // Call host function - // Note: For empty body, we need to pass a valid pointer, not null - const uint8_t* body_ptr = body_bytes.empty() ? reinterpret_cast("") : body_bytes.data(); + // The host ABI requires a non-null, in-bounds body pointer even when body_len == 0. + static const uint8_t empty_sentinel = 0; + const uint8_t* body_ptr = body_bytes.empty() ? &empty_sentinel : body_bytes.data(); BytesSource out[2] = {BytesSource{0}, BytesSource{0}}; Status status = procedure_http_request( @@ -40,15 +40,11 @@ inline Outcome HttpClient::SendImpl(const HttpRequest& request) { if (status.inner == 21) { // Read error message from out[0] std::vector error_bytes = Internal::ConsumeBytes(out[0]); - - LOG_INFO("HTTP: Error bytes: " + std::to_string(error_bytes.size())); - + // Decode BSATN string bsatn::Reader reader(error_bytes.data(), error_bytes.size()); std::string error_message = bsatn::deserialize(reader); - LOG_INFO("HTTP: Error message: " + error_message); - return Err(std::move(error_message)); } @@ -57,7 +53,6 @@ inline Outcome HttpClient::SendImpl(const HttpRequest& request) { return Err("HTTP requests are blocked inside transactions. Call HTTP before with_tx() or try_with_tx()."); } - LOG_INFO("HTTP: Unknown error code: " + std::to_string(status.inner)); return Err("HTTP request failed with status code: " + std::to_string(status.inner)); } diff --git a/crates/bindings-cpp/include/spacetimedb/http_convert.h b/crates/bindings-cpp/include/spacetimedb/http_convert.h index b479cf84d03..e514f4b864e 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_convert.h +++ b/crates/bindings-cpp/include/spacetimedb/http_convert.h @@ -237,6 +237,12 @@ inline HttpRequest from_wire(const wire::HttpRequest& request) { return result; } +inline HttpRequest from_wire(const wire::HttpRequest& request, std::vector body) { + HttpRequest result = from_wire(request); + result.body.bytes = std::move(body); + return result; +} + // ==================== HttpResponse Conversions ==================== /** @@ -268,7 +274,16 @@ inline HttpResponse from_wire(const wire::HttpResponse& response) { return result; } +inline std::pair> to_wire_split(const HttpResponse& response) { + return {to_wire(response), response.body.bytes}; +} + } // namespace convert } // namespace SpacetimeDB +#ifdef SPACETIMEDB_UNSTABLE_FEATURES +#include "spacetimedb/logger.h" +#include "spacetimedb/http_client_impl.h" +#endif + #endif // SPACETIMEDB_HTTP_CONVERT_H diff --git a/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h b/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h new file mode 100644 index 00000000000..4cd88802f9f --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/http_handler_macros.h @@ -0,0 +1,61 @@ +#pragma once + +#ifndef SPACETIMEDB_UNSTABLE_FEATURES +#error "spacetimedb/http_handler_macros.h requires SPACETIMEDB_UNSTABLE_FEATURES to be enabled" +#endif + +#include "spacetimedb/handler_context.h" +#include "spacetimedb/http.h" +#include "spacetimedb/internal/runtime_registration.h" +#include "spacetimedb/internal/template_utils.h" +#include "spacetimedb/internal/v10_builder.h" +#include "spacetimedb/macros.h" +#include "spacetimedb/router.h" + +namespace SpacetimeDB::Internal { + +template +inline void RegisterHttpHandlerMacro(const char* handler_name, Func func) { + using traits = function_traits; + static_assert(traits::arity == 2, "HTTP handlers must take exactly two arguments"); + using ContextType = typename traits::template arg_t<0>; + using RequestType = typename traits::template arg_t<1>; + using ReturnType = typename traits::result_type; + static_assert(std::is_same_v, "First parameter of HTTP handler must be HandlerContext"); + static_assert(std::is_same_v, "Second parameter of HTTP handler must be HttpRequest"); + static_assert(std::is_same_v, "HTTP handlers must return HttpResponse"); + + std::function handler = + [func](HandlerContext& ctx, HttpRequest request) -> HttpResponse { + return func(ctx, std::move(request)); + }; + RegisterHttpHandlerHandler(handler_name, func, std::move(handler)); + getV10Builder().RegisterHttpHandlerDef(handler_name); +} + +} // namespace SpacetimeDB::Internal + +#define SPACETIMEDB_HTTP_HANDLER(handler_name, ctx_param, request_param) \ + SpacetimeDB::HttpResponse handler_name(ctx_param, request_param); \ + __attribute__((export_name("__preinit__60_http_handler_" #handler_name))) \ + extern "C" void CONCAT(_spacetimedb_preinit_register_http_handler_, handler_name)() { \ + ::SpacetimeDB::Internal::RegisterHttpHandlerMacro(#handler_name, handler_name); \ + } \ + SpacetimeDB::HttpResponse handler_name(ctx_param, request_param) + +#define SPACETIMEDB_HTTP_HANDLER_NAMED(handler_name, canonical_name, ctx_param, request_param) \ + SpacetimeDB::HttpResponse handler_name(ctx_param, request_param); \ + __attribute__((export_name("__preinit__60_http_handler_" #handler_name))) \ + extern "C" void CONCAT(_spacetimedb_preinit_register_http_handler_, handler_name)() { \ + ::SpacetimeDB::Internal::RegisterHttpHandlerMacro(#handler_name, handler_name); \ + SpacetimeDB::Module::RegisterExplicitFunctionName(#handler_name, canonical_name); \ + } \ + SpacetimeDB::HttpResponse handler_name(ctx_param, request_param) + +#define SPACETIMEDB_HTTP_ROUTER(router_name) \ + SpacetimeDB::Router router_name(); \ + __attribute__((export_name("__preinit__61_http_router_" #router_name))) \ + extern "C" void CONCAT(_spacetimedb_preinit_register_http_router_, router_name)() { \ + ::SpacetimeDB::Internal::getV10Builder().RegisterHttpRouter(router_name()); \ + } \ + SpacetimeDB::Router router_name() diff --git a/crates/bindings-cpp/include/spacetimedb/http_wire.h b/crates/bindings-cpp/include/spacetimedb/http_wire.h index c4631a749aa..ae473512a37 100644 --- a/crates/bindings-cpp/include/spacetimedb/http_wire.h +++ b/crates/bindings-cpp/include/spacetimedb/http_wire.h @@ -59,6 +59,14 @@ struct HttpMethod { std::string extension; // Only valid when tag == Extension }; +inline bool operator==(const HttpMethod& lhs, const HttpMethod& rhs) { + return lhs.tag == rhs.tag && lhs.extension == rhs.extension; +} + +inline bool operator!=(const HttpMethod& lhs, const HttpMethod& rhs) { + return !(lhs == rhs); +} + /** * @brief Wire format for HTTP version * diff --git a/crates/bindings-cpp/include/spacetimedb/internal/Module.h b/crates/bindings-cpp/include/spacetimedb/internal/Module.h index dd27c18dc3a..7e02e858fb6 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/Module.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/Module.h @@ -75,6 +75,15 @@ class Module { BytesSource args_source, BytesSink result_sink ); + + static int16_t __call_http_handler__( + uint32_t id, + uint64_t timestamp_microseconds, + BytesSource request_source, + BytesSource request_body_source, + BytesSink response_sink, + BytesSink response_body_sink + ); // Internal registration methods (inline to avoid linking issues) template diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/HttpMethod.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/HttpMethod.g.h new file mode 100644 index 00000000000..bb53d566b39 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/HttpMethod.g.h @@ -0,0 +1,75 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Head_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Post_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Put_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Delete_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Connect_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Options_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Trace_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(HttpMethod_Patch_Wrapper) { + std::monostate value; + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, value); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(value) +}; +SPACETIMEDB_INTERNAL_TAGGED_ENUM(HttpMethod, std::monostate, HttpMethod_Head_Wrapper, HttpMethod_Post_Wrapper, HttpMethod_Put_Wrapper, HttpMethod_Delete_Wrapper, HttpMethod_Connect_Wrapper, HttpMethod_Options_Wrapper, HttpMethod_Trace_Wrapper, HttpMethod_Patch_Wrapper, std::string) +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h new file mode 100644 index 00000000000..347702f1a61 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/MethodOrAny.g.h @@ -0,0 +1,20 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" +#include "HttpMethod.g.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_TAGGED_ENUM(MethodOrAny, std::monostate, SpacetimeDB::Internal::HttpMethod) +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpHandlerDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpHandlerDefV10.g.h new file mode 100644 index 00000000000..b6495235e7b --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpHandlerDefV10.g.h @@ -0,0 +1,26 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawHttpHandlerDefV10) { + std::string source_name; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, source_name); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(source_name) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpRouteDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpRouteDefV10.g.h new file mode 100644 index 00000000000..791882e6dff --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawHttpRouteDefV10.g.h @@ -0,0 +1,31 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" +#include "MethodOrAny.g.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawHttpRouteDefV10) { + std::string handler_function; + SpacetimeDB::Internal::MethodOrAny method; + std::string path; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, handler_function); + ::SpacetimeDB::bsatn::serialize(writer, method); + ::SpacetimeDB::bsatn::serialize(writer, path); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(handler_function, method, path) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h index b8e391591df..654260b3532 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h @@ -12,9 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "RawProcedureDefV9.g.h" #include "RawViewDefV9.g.h" #include "RawColumnDefaultValueV9.g.h" -#include "RawProcedureDefV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h index c7f144eb07d..08b82e677c3 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h @@ -13,8 +13,8 @@ #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" #include "RawModuleDefV8.g.h" -#include "RawModuleDefV9.g.h" #include "RawModuleDefV10.g.h" +#include "RawModuleDefV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index 241466f467c..7480d2c2b43 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -12,19 +12,22 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawViewDefV10.g.h" -#include "CaseConversionPolicy.g.h" -#include "RawScheduleDefV10.g.h" -#include "RawTableDefV10.g.h" -#include "Typespace.g.h" #include "RawReducerDefV10.g.h" +#include "RawHttpRouteDefV10.g.h" +#include "RawScheduleDefV10.g.h" +#include "RawModuleMountV10.g.h" +#include "RawViewDefV10.g.h" #include "RawProcedureDefV10.g.h" -#include "RawTypeDefV10.g.h" +#include "Typespace.g.h" #include "RawLifeCycleReducerDefV10.g.h" -#include "RawRowLevelSecurityDefV9.g.h" +#include "RawTableDefV10.g.h" +#include "CaseConversionPolicy.g.h" #include "ExplicitNames.g.h" +#include "RawTypeDefV10.g.h" +#include "RawRowLevelSecurityDefV9.g.h" +#include "RawHttpHandlerDefV10.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h index e856af0fec5..2418ed03e8d 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h @@ -12,10 +12,10 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "ReducerDef.g.h" -#include "MiscModuleExport.g.h" #include "Typespace.g.h" #include "TableDesc.g.h" +#include "MiscModuleExport.g.h" +#include "ReducerDef.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h index 9ab21147e08..cfcee4fe7ec 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h @@ -12,12 +12,12 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "RawRowLevelSecurityDefV9.g.h" #include "RawTableDefV9.g.h" -#include "RawTypeDefV9.g.h" -#include "RawMiscModuleExportV9.g.h" #include "Typespace.g.h" #include "RawReducerDefV9.g.h" -#include "RawRowLevelSecurityDefV9.g.h" +#include "RawTypeDefV9.g.h" +#include "RawMiscModuleExportV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleMountV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleMountV10.g.h new file mode 100644 index 00000000000..0b1ea9dc053 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleMountV10.g.h @@ -0,0 +1,32 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { +struct RawModuleDefV10; +} // namespace SpacetimeDB::Internal + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawModuleMountV10) { + std::string namespace_; + std::shared_ptr module; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, namespace_); + ::SpacetimeDB::bsatn::serialize(writer, *module); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(namespace_, module) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h index f316264fc5c..463b0ec78b8 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h @@ -12,9 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "FunctionVisibility.g.h" -#include "ProductType.g.h" #include "AlgebraicType.g.h" +#include "ProductType.g.h" +#include "FunctionVisibility.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h index 667d9864a2a..a49d9d78970 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h @@ -12,8 +12,8 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "ProductType.g.h" #include "AlgebraicType.g.h" +#include "ProductType.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h index 715364b13cf..7b5b0f10ea3 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h @@ -13,9 +13,9 @@ #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" #include "RawIndexDefV10.g.h" -#include "TableType.g.h" #include "TableAccess.g.h" #include "RawColumnDefaultValueV10.g.h" +#include "TableType.g.h" #include "RawConstraintDefV10.g.h" #include "RawSequenceDefV10.g.h" diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h index a69a502fb0e..5fe2e061945 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h @@ -12,10 +12,10 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawScheduleDefV9.g.h" -#include "RawSequenceDefV9.g.h" #include "TableType.g.h" +#include "RawScheduleDefV9.g.h" #include "TableAccess.g.h" +#include "RawSequenceDefV9.g.h" #include "RawConstraintDefV9.g.h" #include "RawIndexDefV9.g.h" diff --git a/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h b/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h index 4d84cb975d4..005a6553172 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/runtime_registration.h @@ -14,9 +14,14 @@ struct ReducerContext; struct ViewContext; struct AnonymousViewContext; struct ProcedureContext; +struct HandlerContext; +struct HttpRequest; +struct HttpResponse; namespace Internal { +using HttpHandlerSymbol = HttpResponse (*)(HandlerContext, HttpRequest); + void RegisterReducerHandler(const std::string& name, std::function handler, std::optional lifecycle = std::nullopt); @@ -26,9 +31,14 @@ void RegisterAnonymousViewHandler(const std::string& name, std::function(AnonymousViewContext&, BytesSource)> handler); void RegisterProcedureHandler(const std::string& name, std::function(ProcedureContext&, BytesSource)> handler); +void RegisterHttpHandlerHandler(const std::string& name, + HttpHandlerSymbol handler_symbol, + std::function handler); +std::string LookupHttpHandlerName(HttpHandlerSymbol handler_symbol); size_t GetViewHandlerCount(); size_t GetAnonymousViewHandlerCount(); size_t GetProcedureHandlerCount(); +size_t GetHttpHandlerCount(); std::vector ConsumeBytes(BytesSource source); void SetMultiplePrimaryKeyError(const std::string& table_name); void SetConstraintRegistrationError(const std::string& code, const std::string& details); diff --git a/crates/bindings-cpp/include/spacetimedb/internal/tx_execution.h b/crates/bindings-cpp/include/spacetimedb/internal/tx_execution.h new file mode 100644 index 00000000000..4f0f8db2d0c --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/tx_execution.h @@ -0,0 +1,150 @@ +#ifndef SPACETIMEDB_INTERNAL_TX_EXECUTION_H +#define SPACETIMEDB_INTERNAL_TX_EXECUTION_H + +#include +#include +#include +#include +#include + +namespace SpacetimeDB::Internal { + +#ifdef SPACETIMEDB_UNSTABLE_FEATURES + +template +struct is_outcome : std::false_type {}; + +template +struct is_outcome> : std::true_type {}; + +template +inline constexpr bool is_outcome_v = is_outcome>>::value; + +template +bool tx_result_should_commit(const T& result) { + using ResultType = std::remove_cv_t>; + // TODO(http-handlers-cpp): Consider tightening try_with_tx in a future breaking release + // so rollback-aware callbacks use Outcome (and possibly bool for compatibility) + // instead of silently treating arbitrary return types as commit-on-success. + if constexpr (std::is_same_v) { + return result; + } else if constexpr (is_outcome_v) { + return result.is_ok(); + } else { + return true; + } +} + +class TxAbortGuard { +public: + TxAbortGuard() = default; + TxAbortGuard(const TxAbortGuard&) = delete; + TxAbortGuard& operator=(const TxAbortGuard&) = delete; + + ~TxAbortGuard() { + if (!armed_) { + return; + } + Status status = FFI::procedure_abort_mut_tx(); + if (is_error(status)) { + LOG_PANIC("Failed to abort transaction"); + } + } + + void disarm() { + armed_ = false; + } + +private: + bool armed_ = true; +}; + +inline void commit_tx_or_panic() { + Status status = FFI::procedure_commit_mut_tx(); + if (is_error(status)) { + LOG_PANIC("Failed to commit transaction"); + } +} + +inline bool try_commit_tx() { + return is_ok(FFI::procedure_commit_mut_tx()); +} + +inline void abort_tx_or_panic() { + Status status = FFI::procedure_abort_mut_tx(); + if (is_error(status)) { + LOG_PANIC("Failed to abort transaction"); + } +} + +template +auto run_tx_once(MakeReducerContext&& make_reducer_ctx, Func& body) -> decltype(body(std::declval())) { + using ResultType = decltype(body(std::declval())); + + int64_t tx_timestamp = 0; + Status status = FFI::procedure_start_mut_tx(&tx_timestamp); + if (is_error(status)) { + LOG_PANIC("Failed to start transaction"); + } + + TxAbortGuard abort_guard; + ReducerContext reducer_ctx = make_reducer_ctx(Timestamp::from_micros_since_epoch(tx_timestamp)); + TxContext tx{reducer_ctx}; + + if constexpr (std::is_void_v) { + body(tx); + abort_guard.disarm(); + } else { + ResultType result = body(tx); + abort_guard.disarm(); + return result; + } +} + +template +auto with_tx(MakeReducerContext&& make_reducer_ctx, Func& body) -> decltype(body(std::declval())) { + using ResultType = decltype(body(std::declval())); + + if constexpr (std::is_void_v) { + run_tx_once(std::forward(make_reducer_ctx), body); + if (!try_commit_tx()) { + run_tx_once(std::forward(make_reducer_ctx), body); + commit_tx_or_panic(); + } + } else { + ResultType result = run_tx_once(std::forward(make_reducer_ctx), body); + if (!try_commit_tx()) { + result = run_tx_once(std::forward(make_reducer_ctx), body); + commit_tx_or_panic(); + } + return result; + } +} + +template +auto try_with_tx(MakeReducerContext&& make_reducer_ctx, Func& body) -> decltype(body(std::declval())) { + using ResultType = decltype(body(std::declval())); + + ResultType result = run_tx_once(std::forward(make_reducer_ctx), body); + if (!tx_result_should_commit(result)) { + abort_tx_or_panic(); + return result; + } + + if (!try_commit_tx()) { + result = run_tx_once(std::forward(make_reducer_ctx), body); + if (tx_result_should_commit(result)) { + commit_tx_or_panic(); + } else { + abort_tx_or_panic(); + } + } + + return result; +} + +#endif + +} // namespace SpacetimeDB::Internal + +#endif // SPACETIMEDB_INTERNAL_TX_EXECUTION_H diff --git a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h index 9de0f0a2312..f746093c574 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h @@ -28,6 +28,8 @@ #include "autogen/RawViewDefV10.g.h" #include "autogen/RawScheduleDefV10.g.h" #include "autogen/RawLifeCycleReducerDefV10.g.h" +#include "autogen/RawHttpHandlerDefV10.g.h" +#include "autogen/RawHttpRouteDefV10.g.h" #include "autogen/RawColumnDefaultValueV10.g.h" #include "autogen/RawRowLevelSecurityDefV9.g.h" #include "autogen/RawTypeDefV10.g.h" @@ -39,6 +41,8 @@ namespace SpacetimeDB { +class Router; + void fail_reducer(std::string message); namespace Internal { @@ -584,6 +588,10 @@ class V10Builder { UpsertProcedure(procedure_def); } + void RegisterHttpHandlerDef(const std::string& handler_name); + void RegisterHttpRoute(const RawHttpRouteDefV10& route); + void RegisterHttpRouter(const ::SpacetimeDB::Router& router); + void RegisterSchedule(const std::string& table_name, uint16_t scheduled_at_column, const std::string& reducer_name) { if (g_circular_ref_error) { std::fprintf(stderr, "ERROR: Skipping schedule registration for table '%s' because circular reference error is set\n", @@ -628,6 +636,8 @@ class V10Builder { const std::vector& GetReducers() const { return reducers_; } const std::optional& GetCaseConversionPolicy() const { return case_conversion_policy_; } const std::vector& GetExplicitNames() const { return explicit_names_; } + const std::vector& GetHttpHandlers() const { return http_handlers_; } + const std::vector& GetHttpRoutes() const { return http_routes_; } private: std::vector::iterator FindTable(const std::string& table_name) { @@ -638,6 +648,7 @@ class V10Builder { void UpsertReducer(const RawReducerDefV10& reducer); void UpsertProcedure(const RawProcedureDefV10& procedure); void UpsertView(const RawViewDefV10& view); + void UpsertHttpHandler(const RawHttpHandlerDefV10& handler); RawIndexDefV10 CreateBTreeIndex(const std::string& table_name, const std::string& source_name, const std::vector& columns, @@ -656,6 +667,8 @@ class V10Builder { std::vector reducers_; std::vector procedures_; std::vector views_; + std::vector http_handlers_; + std::vector http_routes_; std::vector schedules_; std::vector lifecycle_reducers_; std::vector row_level_security_; diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index f9107d70251..ea189f8ef12 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -6,6 +6,7 @@ #include // For Uuid #include // For TxContext #include // For transaction syscalls +#include #include // For StdbRng #ifdef SPACETIMEDB_UNSTABLE_FEATURES #include // For HttpClient @@ -196,46 +197,14 @@ struct ProcedureContext { */ template auto with_tx(Func&& body) -> decltype(body(std::declval())) { - using ResultType = decltype(body(std::declval())); - - // Start transaction - int64_t tx_timestamp; - Status status = ::procedure_start_mut_tx(&tx_timestamp); - if (is_error(status)) { - LOG_PANIC("Failed to start transaction"); - } - - // Create a ReducerContext for this transaction - // Note: connection_id converted to std::optional - ReducerContext reducer_ctx( - sender(), - std::optional(connection_id), - Timestamp::from_micros_since_epoch(tx_timestamp) - ); - - // Create transaction context wrapping the reducer context - TxContext tx{reducer_ctx}; - - // Execute callback - if constexpr (std::is_void_v) { - body(tx); - - // Commit transaction - status = ::procedure_commit_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to commit transaction"); - } - } else { - ResultType result = body(tx); - - // Commit transaction - status = ::procedure_commit_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to commit transaction"); - } - - return result; - } + auto make_reducer_ctx = [this](Timestamp tx_timestamp) { + return ReducerContext( + sender(), + std::optional(connection_id), + tx_timestamp + ); + }; + return Internal::with_tx(make_reducer_ctx, body); } /** @@ -260,51 +229,14 @@ struct ProcedureContext { */ template auto try_with_tx(Func&& body) -> decltype(body(std::declval())) { - using ResultType = decltype(body(std::declval())); - - // Start transaction - int64_t tx_timestamp; - Status status = ::procedure_start_mut_tx(&tx_timestamp); - if (is_error(status)) { - LOG_PANIC("Failed to start transaction"); - } - - // Create a ReducerContext for this transaction - ReducerContext reducer_ctx( - sender(), - std::optional(connection_id), - Timestamp::from_micros_since_epoch(tx_timestamp) - ); - - // Create transaction context wrapping the reducer context - TxContext tx{reducer_ctx}; - - // Execute callback - ResultType result = body(tx); - - // For bool results, use the value to decide commit/rollback - // For other types, always commit (caller can use LOG_PANIC to abort) - if constexpr (std::is_same_v) { - if (result) { - status = ::procedure_commit_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to commit transaction"); - } - } else { - status = ::procedure_abort_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to rollback transaction"); - } - } - } else { - // For non-bool returns, always commit - status = ::procedure_commit_mut_tx(); - if (is_error(status)) { - LOG_PANIC("Failed to commit transaction"); - } - } - - return result; + auto make_reducer_ctx = [this](Timestamp tx_timestamp) { + return ReducerContext( + sender(), + std::optional(connection_id), + tx_timestamp + ); + }; + return Internal::try_with_tx(make_reducer_ctx, body); } #endif }; diff --git a/crates/bindings-cpp/include/spacetimedb/reducer_context.h b/crates/bindings-cpp/include/spacetimedb/reducer_context.h index 8c8fba26e72..41865b14f3a 100644 --- a/crates/bindings-cpp/include/spacetimedb/reducer_context.h +++ b/crates/bindings-cpp/include/spacetimedb/reducer_context.h @@ -124,6 +124,9 @@ struct ReducerContext { ReducerContext(Identity s, std::optional cid, Timestamp ts) : sender_(s), connection_id(cid), timestamp(ts), sender_auth_(AuthCtx::from_connection_id_opt(cid, s)) {} + + ReducerContext(Identity s, std::optional cid, Timestamp ts, AuthCtx auth) + : sender_(s), connection_id(cid), timestamp(ts), sender_auth_(std::move(auth)) {} }; } // namespace SpacetimeDB diff --git a/crates/bindings-cpp/include/spacetimedb/router.h b/crates/bindings-cpp/include/spacetimedb/router.h new file mode 100644 index 00000000000..00808f8bdfd --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/router.h @@ -0,0 +1,237 @@ +#ifndef SPACETIMEDB_ROUTER_H +#define SPACETIMEDB_ROUTER_H + +#ifndef SPACETIMEDB_UNSTABLE_FEATURES +#error "spacetimedb/router.h requires SPACETIMEDB_UNSTABLE_FEATURES to be enabled" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { + +class Router { +public: + struct RouteSpec { + Internal::MethodOrAny method; + std::string path; + std::string handler_name; + }; + + Router() = default; + + template + Router get(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::get()), std::move(path), handler); + } + + template + Router head(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::head()), std::move(path), handler); + } + + template + Router options(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::options()), std::move(path), handler); + } + + template + Router put(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::put()), std::move(path), handler); + } + + template + Router delete_(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::del()), std::move(path), handler); + } + + template + Router post(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::post()), std::move(path), handler); + } + + template + Router patch(std::string path, Func handler) const { + return add_route(make_method(HttpMethod::patch()), std::move(path), handler); + } + + template + Router any(std::string path, Func handler) const { + return add_route(make_any(), std::move(path), handler); + } + + Router nest(std::string path, const Router& sub_router) const { + assert_valid_path(path); + Router merged = *this; + for (const auto& route : routes_) { + if (route.path.starts_with(path)) { + fail_router_registration("Cannot nest router at `" + path + "`; existing routes overlap with nested path"); + } + } + for (const auto& route : sub_router.routes_) { + merged = merged.add_route(route.method, join_paths(path, route.path), route.handler_name); + } + return merged; + } + + Router merge(const Router& other) const { + Router merged = *this; + for (const auto& route : other.routes_) { + merged = merged.add_route(route.method, route.path, route.handler_name); + } + return merged; + } + + const std::vector& routes() const { + return routes_; + } + +private: + std::vector routes_; + + [[noreturn]] static void fail_router_registration(const std::string& message) { + std::fprintf(stderr, "Router registration failed: %s\n", message.c_str()); + std::abort(); + } + + static bool character_is_acceptable_for_route_path(char c) { + return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '~' || c == '/'; + } + + static void assert_valid_path(const std::string& path) { + if (!path.empty() && path[0] != '/') { + fail_router_registration("Route paths must start with `/`: " + path); + } + for (char c : path) { + if (!character_is_acceptable_for_route_path(c)) { + fail_router_registration("Route paths may contain only ASCII lowercase letters, digits and `-_~/`: " + path); + } + } + } + + static std::string join_paths(const std::string& prefix, const std::string& suffix) { + if (prefix == "/") { + return suffix; + } + if (suffix == "/") { + return prefix; + } + std::string trimmed_prefix = prefix; + while (!trimmed_prefix.empty() && trimmed_prefix.back() == '/') { + trimmed_prefix.pop_back(); + } + size_t start = 0; + while (start < suffix.size() && suffix[start] == '/') { + ++start; + } + return trimmed_prefix + "/" + suffix.substr(start); + } + + static bool routes_overlap(const RouteSpec& a, const RouteSpec& b) { + if (a.path != b.path) { + return false; + } + if (a.method.is<0>() || b.method.is<0>()) { + return true; + } + return method_key(a.method.template get<1>()) == method_key(b.method.template get<1>()); + } + + static std::string method_key(const Internal::HttpMethod& method) { + switch (method.get_tag()) { + case 0: + return "GET"; + case 1: + return "HEAD"; + case 2: + return "POST"; + case 3: + return "PUT"; + case 4: + return "DELETE"; + case 5: + return "CONNECT"; + case 6: + return "OPTIONS"; + case 7: + return "TRACE"; + case 8: + return "PATCH"; + case 9: + return method.template get<9>(); + default: + fail_router_registration("Unsupported internal HTTP method tag"); + } + } + + static Internal::MethodOrAny make_any() { + Internal::MethodOrAny method; + method.set<0>(std::monostate{}); + return method; + } + + static Internal::MethodOrAny make_method(const HttpMethod& method) { + Internal::MethodOrAny result; + result.set<1>(to_internal_http_method(method)); + return result; + } + + static Internal::HttpMethod to_internal_http_method(const HttpMethod& method) { + Internal::HttpMethod result; + if (method.value == "GET") { + result.set<0>(std::monostate{}); + } else if (method.value == "HEAD") { + result.set<1>(Internal::HttpMethod_Head_Wrapper{}); + } else if (method.value == "POST") { + result.set<2>(Internal::HttpMethod_Post_Wrapper{}); + } else if (method.value == "PUT") { + result.set<3>(Internal::HttpMethod_Put_Wrapper{}); + } else if (method.value == "DELETE") { + result.set<4>(Internal::HttpMethod_Delete_Wrapper{}); + } else if (method.value == "CONNECT") { + result.set<5>(Internal::HttpMethod_Connect_Wrapper{}); + } else if (method.value == "OPTIONS") { + result.set<6>(Internal::HttpMethod_Options_Wrapper{}); + } else if (method.value == "TRACE") { + result.set<7>(Internal::HttpMethod_Trace_Wrapper{}); + } else if (method.value == "PATCH") { + result.set<8>(Internal::HttpMethod_Patch_Wrapper{}); + } else { + result.set<9>(method.value); + } + return result; + } + + template + Router add_route(Internal::MethodOrAny method, std::string path, Func handler) const { + return add_route(std::move(method), std::move(path), resolve_handler_name(handler)); + } + + Router add_route(Internal::MethodOrAny method, std::string path, std::string handler_name) const { + assert_valid_path(path); + RouteSpec candidate{method, path, std::move(handler_name)}; + for (const auto& route : routes_) { + if (routes_overlap(route, candidate)) { + fail_router_registration("Route conflict for `" + candidate.path + "`"); + } + } + Router next = *this; + next.routes_.push_back(std::move(candidate)); + return next; + } + + template + static std::string resolve_handler_name(Func handler) { + return Internal::LookupHttpHandlerName(handler); + } +}; + +} // namespace SpacetimeDB + +#endif // SPACETIMEDB_ROUTER_H diff --git a/crates/bindings-cpp/src/abi/module_exports.cpp b/crates/bindings-cpp/src/abi/module_exports.cpp index 6156e32025a..a31d50be1fb 100644 --- a/crates/bindings-cpp/src/abi/module_exports.cpp +++ b/crates/bindings-cpp/src/abi/module_exports.cpp @@ -99,4 +99,23 @@ extern "C" { ); } + STDB_EXPORT(__call_http_handler__) + int16_t __call_http_handler__( + uint32_t id, + uint64_t timestamp_microseconds, + SpacetimeDB::BytesSource request_source, + SpacetimeDB::BytesSource request_body_source, + SpacetimeDB::BytesSink response_sink, + SpacetimeDB::BytesSink response_body_sink + ) { + return SpacetimeDB::Internal::Module::__call_http_handler__( + id, + timestamp_microseconds, + request_source, + request_body_source, + response_sink, + response_body_sink + ); + } + } // extern "C" diff --git a/crates/bindings-cpp/src/internal/Module.cpp b/crates/bindings-cpp/src/internal/Module.cpp index b0dcb1ceae3..901bb5e3e86 100644 --- a/crates/bindings-cpp/src/internal/Module.cpp +++ b/crates/bindings-cpp/src/internal/Module.cpp @@ -16,6 +16,11 @@ #include "spacetimedb/reducer_error.h" #include "spacetimedb/view_context.h" #include "spacetimedb/procedure_context.h" +#include "spacetimedb/handler_context.h" +#include "spacetimedb/http_convert.h" +#include "spacetimedb/http_wire.h" +#include +#include #include #include #include @@ -55,6 +60,13 @@ namespace Internal { std::function(ProcedureContext&, BytesSource)> handler; }; static std::vector g_procedure_handlers; + + struct HttpHandler { + std::string name; + HttpHandlerSymbol symbol; + std::function handler; + }; + static std::vector g_http_handlers; /** * @brief View result header for serializing view return values @@ -116,6 +128,23 @@ namespace Internal { std::function(ProcedureContext&, BytesSource)> handler) { g_procedure_handlers.push_back({name, handler}); } + + void RegisterHttpHandlerHandler(const std::string& name, + HttpHandlerSymbol handler_symbol, + std::function handler) { + g_http_handlers.push_back({name, handler_symbol, handler}); + } + + std::string LookupHttpHandlerName(HttpHandlerSymbol handler_symbol) { + auto it = std::find_if(g_http_handlers.begin(), g_http_handlers.end(), [&](const auto& existing) { + return existing.symbol == handler_symbol; + }); + if (it == g_http_handlers.end()) { + fprintf(stderr, "ERROR: HTTP handler must be registered before it is referenced by a router\n"); + std::abort(); + } + return it->name; + } // Get the number of registered view handlers size_t GetViewHandlerCount() { @@ -130,6 +159,10 @@ namespace Internal { size_t GetProcedureHandlerCount() { return g_procedure_handlers.size(); } + + size_t GetHttpHandlerCount() { + return g_http_handlers.size(); + } void SetTableIsEventFlag(const std::string& table_name, bool is_event) { getV10Builder().SetTableIsEventFlag(table_name, is_event); @@ -146,6 +179,7 @@ namespace Internal { g_view_handlers.clear(); // Clear view handlers g_view_anon_handlers.clear(); // Clear anonymous view handlers g_procedure_handlers.clear(); // Clear procedure handlers + g_http_handlers.clear(); // Clear http handlers g_multiple_primary_key_error = false; // Reset error flag g_multiple_primary_key_table_name = ""; // Reset error table name g_constraint_registration_error = false; @@ -630,6 +664,41 @@ int16_t Module::__call_procedure__( return 0; // Success (StatusCode::OK) } +int16_t Module::__call_http_handler__( + uint32_t id, + uint64_t timestamp_microseconds, + BytesSource request_source, + BytesSource request_body_source, + BytesSink response_sink, + BytesSink response_body_sink +) { + if (id >= g_http_handlers.size()) { + fprintf(stderr, "ERROR: Invalid http handler ID %u (have %zu handlers)\n", + id, g_http_handlers.size()); + return -1; + } + + Timestamp timestamp = Timestamp::from_micros_since_epoch(static_cast(timestamp_microseconds)); + HandlerContext ctx(timestamp); + + std::vector request_bytes = ConsumeBytes(request_source); + bsatn::Reader request_reader(request_bytes.data(), request_bytes.size()); + wire::HttpRequest wire_request = bsatn::deserialize(request_reader); + HttpRequest request = convert::from_wire(wire_request, ConsumeBytes(request_body_source)); + + HttpResponse response = g_http_handlers[id].handler(ctx, std::move(request)); + auto [wire_response, response_body] = convert::to_wire_split(response); + + std::vector response_metadata; + { + bsatn::Writer writer(response_metadata); + bsatn::serialize(writer, wire_response); + } + WriteBytes(response_sink, response_metadata); + WriteBytes(response_body_sink, response_body); + return 0; +} + void Module::SetCaseConversionPolicy(CaseConversionPolicy policy) { getV10Builder().SetCaseConversionPolicy(policy); } @@ -657,4 +726,3 @@ void Module::RegisterExplicitIndexName(const std::string& source_name, const std - diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index 65f1895b035..bde38715deb 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -7,6 +7,7 @@ #include "spacetimedb/internal/autogen/RawScopedTypeNameV10.g.h" #include "spacetimedb/internal/autogen/FunctionVisibility.g.h" #include "spacetimedb/internal/autogen/ExplicitNames.g.h" +#include "spacetimedb/router.h" #include #include @@ -37,6 +38,8 @@ void V10Builder::Clear() { reducers_.clear(); procedures_.clear(); views_.clear(); + http_handlers_.clear(); + http_routes_.clear(); schedules_.clear(); lifecycle_reducers_.clear(); row_level_security_.clear(); @@ -150,6 +153,31 @@ void V10Builder::UpsertView(const RawViewDefV10& view) { } } +void V10Builder::UpsertHttpHandler(const RawHttpHandlerDefV10& handler) { + auto it = std::find_if(http_handlers_.begin(), http_handlers_.end(), [&](const auto& existing) { + return existing.source_name == handler.source_name; + }); + if (it == http_handlers_.end()) { + http_handlers_.push_back(handler); + } else { + *it = handler; + } +} + +void V10Builder::RegisterHttpHandlerDef(const std::string& handler_name) { + UpsertHttpHandler(RawHttpHandlerDefV10{handler_name}); +} + +void V10Builder::RegisterHttpRoute(const RawHttpRouteDefV10& route) { + http_routes_.push_back(route); +} + +void V10Builder::RegisterHttpRouter(const ::SpacetimeDB::Router& router) { + for (const auto& route : router.routes()) { + RegisterHttpRoute(RawHttpRouteDefV10{route.handler_name, route.method, route.path}); + } +} + RawIndexDefV10 V10Builder::CreateBTreeIndex(const std::string& table_name, const std::string& source_name, const std::vector& columns, @@ -256,6 +284,16 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { section_explicit_names.set<10>(ExplicitNames{explicit_names_}); v10_module.sections.push_back(std::move(section_explicit_names)); } + if (!http_handlers_.empty()) { + RawModuleDefV10Section section_http_handlers; + section_http_handlers.set<12>(http_handlers_); + v10_module.sections.push_back(std::move(section_http_handlers)); + } + if (!http_routes_.empty()) { + RawModuleDefV10Section section_http_routes; + section_http_routes.set<13>(http_routes_); + v10_module.sections.push_back(std::move(section_http_routes)); + } if (!row_level_security_.empty()) { RawModuleDefV10Section section_rls; section_rls.set<8>(row_level_security_); diff --git a/crates/bindings-cpp/tests/compile/CMakeLists.module.txt b/crates/bindings-cpp/tests/compile/CMakeLists.module.txt new file mode 100644 index 00000000000..63d00912c90 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/CMakeLists.module.txt @@ -0,0 +1,49 @@ +cmake_minimum_required(VERSION 3.16) +project(module) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT DEFINED MODULE_SOURCE) + message(FATAL_ERROR "MODULE_SOURCE must be defined") +endif() + +if(NOT DEFINED OUTPUT_NAME) + set(OUTPUT_NAME "module") +endif() + +if(NOT DEFINED SPACETIMEDB_LIBRARY_DIR) + message(FATAL_ERROR "SPACETIMEDB_LIBRARY_DIR must be defined") +endif() + +if(NOT DEFINED SPACETIMEDB_INCLUDE_DIR) + message(FATAL_ERROR "SPACETIMEDB_INCLUDE_DIR must be defined") +endif() + +add_executable(${OUTPUT_NAME} ${MODULE_SOURCE}) +target_include_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_INCLUDE_DIR}) +target_link_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_LIBRARY_DIR}) +target_link_libraries(${OUTPUT_NAME} PRIVATE spacetimedb_cpp_library) +target_compile_definitions(${OUTPUT_NAME} PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) + +if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + set(EXPORTED_FUNCS "['_malloc','_free','___describe_module__','___call_reducer__','___call_http_handler__']") + + target_link_options(${OUTPUT_NAME} PRIVATE + "SHELL:-sSTANDALONE_WASM=1" + "SHELL:-sWASM=1" + "SHELL:--no-entry" + "SHELL:-sEXPORTED_FUNCTIONS=${EXPORTED_FUNCS}" + "SHELL:-sERROR_ON_UNDEFINED_SYMBOLS=1" + "SHELL:-sFILESYSTEM=0" + "SHELL:-sDISABLE_EXCEPTION_CATCHING=1" + "SHELL:-sALLOW_MEMORY_GROWTH=0" + "SHELL:-sINITIAL_MEMORY=16MB" + "SHELL:-sSUPPORT_LONGJMP=0" + "SHELL:-sSUPPORT_ERRNO=0" + "SHELL:-std=c++20" + "SHELL:-O2" + ) + + set_target_properties(${OUTPUT_NAME} PROPERTIES OUTPUT_NAME "lib" SUFFIX ".wasm") +endif() diff --git a/crates/bindings-cpp/tests/compile/README.md b/crates/bindings-cpp/tests/compile/README.md new file mode 100644 index 00000000000..34c011b0931 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/README.md @@ -0,0 +1,64 @@ +# SpacetimeDB C++ Compile Tests + +Focused compile-surface regression tests for the C++ bindings. + +This harness is intended for: +- authoring-time success cases +- compile-fail regression cases +- API surface checks that should fail before publish/runtime + +## HTTP Handler Coverage + +The `http-handlers` suite mirrors the Rust coverage in +`crates/bindings/tests/ui/http_handlers.rs` as closely as the C++ macro surface allows. + +Covered cases: +- valid handler/router authoring +- no handler args +- immutable handler context +- wrong handler context type +- missing request arg +- wrong request arg type +- missing return +- wrong return type +- forbidden `HandlerContext::sender()` +- forbidden `HandlerContext::connection_id` +- forbidden `HandlerContext::db` +- router authored with args +- router wrong return type +- router misuse in a non-function position + +## Run + +From Git Bash or Linux-style shells: + +```bash +./crates/bindings-cpp/tests/compile/run-compile-tests.sh --suite http-handlers +``` + +From PowerShell at the repo root: + +```powershell +.\crates\bindings-cpp\tests\compile\run-compile-tests.ps1 -Suite http-handlers +``` + +Or from the compile test directory: + +```powershell +.\run-compile-tests.ps1 -Suite http-handlers +``` + +## Output + +Build artifacts and logs are written under: + +```text +crates/bindings-cpp/tests/compile/build/ +``` + +Each case gets: +- `build//configure.log` +- `build//build.log` + +The shared bindings library build is under: +- `build/library/` diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp new file mode 100644 index 00000000000..acb7d2f731a --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_immutable_ctx.cpp @@ -0,0 +1,12 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_immutable_ctx, const HandlerContext& ctx, HttpRequest request) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_args.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_args.cpp new file mode 100644 index 00000000000..7391a55450f --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_args.cpp @@ -0,0 +1,12 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_no_args) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp new file mode 100644 index 00000000000..c775e6c3430 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_connection_id.cpp @@ -0,0 +1,13 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_no_connection_id, HandlerContext ctx, HttpRequest request) { + auto conn_id = ctx.connection_id(); + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string(conn_id.to_hex_string()), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp new file mode 100644 index 00000000000..4f650b55e71 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_db.cpp @@ -0,0 +1,19 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +struct TestRow { + uint32_t value; +}; +SPACETIMEDB_STRUCT(TestRow, value) +SPACETIMEDB_TABLE(TestRow, test_row, Public) + +SPACETIMEDB_HTTP_HANDLER(handler_no_db, HandlerContext ctx, HttpRequest request) { + auto count = ctx.db[test_row].count(); + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string(std::to_string(count)), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp new file mode 100644 index 00000000000..4543a97ef8c --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_request_arg.cpp @@ -0,0 +1,12 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_no_request_arg, HandlerContext ctx) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp new file mode 100644 index 00000000000..ea22473d38f --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_return_type.cpp @@ -0,0 +1,10 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +#if defined(__clang__) +#pragma clang diagnostic error "-Wreturn-type" +#endif + +SPACETIMEDB_HTTP_HANDLER(handler_no_return_type, HandlerContext ctx, HttpRequest request) { +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp new file mode 100644 index 00000000000..9f4ba51b5d0 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_no_sender.cpp @@ -0,0 +1,13 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_no_sender, HandlerContext ctx, HttpRequest request) { + auto sender = ctx.sender(); + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string(sender.to_hex_string()), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp new file mode 100644 index 00000000000..01893c6d278 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_ctx.cpp @@ -0,0 +1,12 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_wrong_ctx, ProcedureContext ctx, HttpRequest request) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp new file mode 100644 index 00000000000..6ee189776b6 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp @@ -0,0 +1,12 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_wrong_request_arg_type, HandlerContext ctx, uint32_t request) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp new file mode 100644 index 00000000000..b04c6bfaa3e --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_handler_wrong_return_type.cpp @@ -0,0 +1,7 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(handler_wrong_return_type, HandlerContext ctx, HttpRequest request) { + return 7u; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_not_a_function.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_not_a_function.cpp new file mode 100644 index 00000000000..a18cb84cd0f --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_not_a_function.cpp @@ -0,0 +1,5 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_ROUTER(register_http_routes) = Router(); diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp new file mode 100644 index 00000000000..c4f7fb6a5d3 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_with_args.cpp @@ -0,0 +1,16 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(hello_handler, HandlerContext ctx, HttpRequest request) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} + +SPACETIMEDB_HTTP_ROUTER(register_http_routes, HandlerContext ctx) { + return Router().get("/hello", hello_handler); +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_wrong_return_type.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_wrong_return_type.cpp new file mode 100644 index 00000000000..fd20453bc4c --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/error_http_router_wrong_return_type.cpp @@ -0,0 +1,7 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_ROUTER(register_http_routes) { + return 7u; +} diff --git a/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp b/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp new file mode 100644 index 00000000000..e482c0561df --- /dev/null +++ b/crates/bindings-cpp/tests/compile/cases/http-handlers/ok_http_handlers_basic.cpp @@ -0,0 +1,28 @@ +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(hello_handler, HandlerContext ctx, HttpRequest request) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string("ok"), + }; +} + +SPACETIMEDB_HTTP_ROUTER(register_http_routes) { + Router nested = Router() + .get("/nested", hello_handler); + + Router merged = Router() + .get("", hello_handler) + .head("/health", hello_handler); + + return Router() + .get("/hello", hello_handler) + .delete_("/delete", hello_handler) + .any("/", hello_handler) + .merge(merged) + .nest("/api", nested); +} diff --git a/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 b/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 new file mode 100644 index 00000000000..2dd8a40eaa4 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/run-compile-tests.ps1 @@ -0,0 +1,221 @@ +[CmdletBinding()] +param( + [ValidateSet("http-handlers")] + [string]$Suite = "http-handlers" +) + +$ErrorActionPreference = "Stop" + +function Find-Emcmake { + $candidates = @( + (Get-Command emcmake.bat -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Source -First 1), + (Get-Command emcmake -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Source -First 1) + ) | Where-Object { $_ } + + if ($candidates.Count -eq 0) { + throw "Unable to locate emcmake or emcmake.bat." + } + + return $candidates[0] +} + +function Invoke-LoggedCommand { + param( + [Parameter(Mandatory = $true)] + [string]$FilePath, + [Parameter(Mandatory = $true)] + [string[]]$Arguments, + [Parameter(Mandatory = $true)] + [string]$LogPath, + [string]$WorkingDirectory + ) + + if ($WorkingDirectory) { + Push-Location $WorkingDirectory + } + + try { + $quotedParts = @($FilePath) + $Arguments | ForEach-Object { + '"' + ($_ -replace '"', '\"') + '"' + } + $commandLine = ($quotedParts -join ' ') + " > `"$LogPath`" 2>&1" + + cmd /c $commandLine | Out-Null + return $LASTEXITCODE + } finally { + if ($WorkingDirectory) { + Pop-Location + } + } +} + +function New-CompileCase { + param( + [string]$Name, + [string]$RelativePath, + [ValidateSet("success", "failure")] + [string]$Expectation, + [string]$Marker = "" + ) + + return [pscustomobject]@{ + Name = $Name + RelativePath = $RelativePath + Expectation = $Expectation + Marker = $Marker + } +} + +function Convert-ToCMakePath { + param([Parameter(Mandatory = $true)][string]$Path) + return $Path.Replace('\', '/') +} + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$bindingsRoot = Split-Path -Parent (Split-Path -Parent $scriptDir) +$repoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $bindingsRoot)) +$includeDir = Join-Path $bindingsRoot "include" +$buildRoot = Join-Path $scriptDir "build" +$libraryBuildDir = Join-Path $buildRoot "library" +$libraryLogDir = Join-Path $buildRoot "logs" +$templatePath = Join-Path $scriptDir "CMakeLists.module.txt" +$emcmake = Find-Emcmake + +$cases = switch ($Suite) { + "http-handlers" { + @( + (New-CompileCase "ok_http_handlers_basic" "cases/http-handlers/ok_http_handlers_basic.cpp" "success") + (New-CompileCase "error_http_handler_no_args" "cases/http-handlers/error_http_handler_no_args.cpp" "failure" "too few arguments provided to function-like macro invocation") + (New-CompileCase "error_http_handler_immutable_ctx" "cases/http-handlers/error_http_handler_immutable_ctx.cpp" "failure" "First parameter of HTTP handler must be HandlerContext") + (New-CompileCase "error_http_handler_wrong_ctx" "cases/http-handlers/error_http_handler_wrong_ctx.cpp" "failure" "First parameter of HTTP handler must be HandlerContext") + (New-CompileCase "error_http_handler_no_request_arg" "cases/http-handlers/error_http_handler_no_request_arg.cpp" "failure" "too few arguments provided to function-like macro invocation") + (New-CompileCase "error_http_handler_wrong_request_arg_type" "cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp" "failure" "Second parameter of HTTP handler must be HttpRequest") + (New-CompileCase "error_http_handler_no_return_type" "cases/http-handlers/error_http_handler_no_return_type.cpp" "failure" "non-void function does not return a value") + (New-CompileCase "error_http_handler_wrong_return_type" "cases/http-handlers/error_http_handler_wrong_return_type.cpp" "failure" "no viable conversion from returned value of type 'unsigned int' to function return type 'SpacetimeDB::HttpResponse'") + (New-CompileCase "error_http_handler_no_sender" "cases/http-handlers/error_http_handler_no_sender.cpp" "failure" "no member named 'sender' in 'SpacetimeDB::HandlerContext'") + (New-CompileCase "error_http_handler_no_connection_id" "cases/http-handlers/error_http_handler_no_connection_id.cpp" "failure" "no member named 'connection_id' in 'SpacetimeDB::HandlerContext'") + (New-CompileCase "error_http_handler_no_db" "cases/http-handlers/error_http_handler_no_db.cpp" "failure" "no member named 'db' in 'SpacetimeDB::HandlerContext'") + (New-CompileCase "error_http_router_not_a_function" "cases/http-handlers/error_http_router_not_a_function.cpp" "failure" "illegal initializer") + (New-CompileCase "error_http_router_with_args" "cases/http-handlers/error_http_router_with_args.cpp" "failure" "too many arguments provided to function-like macro invocation") + (New-CompileCase "error_http_router_wrong_return_type" "cases/http-handlers/error_http_router_wrong_return_type.cpp" "failure" "no viable conversion from returned value of type 'unsigned int' to function return type 'SpacetimeDB::Router'") + ) + } +} + +New-Item -ItemType Directory -Force -Path $buildRoot | Out-Null +New-Item -ItemType Directory -Force -Path $libraryLogDir | Out-Null + +$libraryConfigureLog = Join-Path $libraryLogDir "library-configure.log" +$libraryBuildLog = Join-Path $libraryLogDir "library-build.log" + +Write-Host "Building bindings library..." +$configureExit = Invoke-LoggedCommand -FilePath $emcmake -Arguments @( + "cmake", + "-S", $bindingsRoot, + "-B", $libraryBuildDir +) -LogPath $libraryConfigureLog -WorkingDirectory $scriptDir + +if ($configureExit -ne 0) { + Write-Host "Library configure failed. See $libraryConfigureLog" + exit 1 +} + +$buildExit = Invoke-LoggedCommand -FilePath "cmake" -Arguments @( + "--build", $libraryBuildDir +) -LogPath $libraryBuildLog -WorkingDirectory $scriptDir + +if ($buildExit -ne 0) { + Write-Host "Library build failed. See $libraryBuildLog" + exit 1 +} + +$results = @() + +foreach ($case in $cases) { + $caseSource = Join-Path $scriptDir $case.RelativePath + $caseBuildDir = Join-Path $buildRoot $case.Name + $configureLog = Join-Path $caseBuildDir "configure.log" + $buildLog = Join-Path $caseBuildDir "build.log" + $caseSourceCMake = Convert-ToCMakePath $caseSource + $libraryBuildDirCMake = Convert-ToCMakePath $libraryBuildDir + $includeDirCMake = Convert-ToCMakePath $includeDir + + if (Test-Path $caseBuildDir) { + Remove-Item $caseBuildDir -Recurse -Force + } + + New-Item -ItemType Directory -Force -Path $caseBuildDir | Out-Null + Copy-Item $templatePath (Join-Path $caseBuildDir "CMakeLists.txt") + + Write-Host "Running $($case.Name)..." + $configureExit = Invoke-LoggedCommand -FilePath $emcmake -Arguments @( + "cmake", + "-S", $caseBuildDir, + "-B", $caseBuildDir, + "-DMODULE_SOURCE=$caseSourceCMake", + "-DOUTPUT_NAME=$($case.Name)", + "-DSPACETIMEDB_LIBRARY_DIR=$libraryBuildDirCMake", + "-DSPACETIMEDB_INCLUDE_DIR=$includeDirCMake" + ) -LogPath $configureLog -WorkingDirectory $scriptDir + + $buildExit = 0 + if ($configureExit -eq 0) { + $buildExit = Invoke-LoggedCommand -FilePath "cmake" -Arguments @( + "--build", $caseBuildDir + ) -LogPath $buildLog -WorkingDirectory $scriptDir + } + + $combinedLog = "" + if (Test-Path $configureLog) { + $combinedLog += Get-Content $configureLog -Raw + } + if (Test-Path $buildLog) { + $combinedLog += "`n" + $combinedLog += Get-Content $buildLog -Raw + } + + $passed = $false + $detail = "" + if ($case.Expectation -eq "success") { + $passed = ($configureExit -eq 0 -and $buildExit -eq 0) + if (-not $passed) { + $detail = "Expected build success." + } + } else { + $failedBuild = ($configureExit -ne 0 -or $buildExit -ne 0) + $matchedMarker = ($case.Marker -and $combinedLog.Contains($case.Marker)) + $passed = ($failedBuild -and $matchedMarker) + if (-not $passed) { + if (-not $failedBuild) { + $detail = "Expected build failure." + } else { + $detail = "Expected marker not found: $($case.Marker)" + } + } + } + + if (-not $passed -and -not $detail) { + $detail = (($combinedLog -split "`r?`n" | Where-Object { $_.Trim() }) | Select-Object -First 8) -join " " + } + + $results += [pscustomobject]@{ + Case = $case.Name + Expectation = $case.Expectation + Result = if ($passed) { "PASS" } else { "FAIL" } + Detail = $detail + } +} + +$results | Format-Table -AutoSize + +if ($results.Result -contains "FAIL") { + Write-Host "" + Write-Host "Failures:" + $results | Where-Object Result -eq "FAIL" | ForEach-Object { + Write-Host "- $($_.Case): $($_.Detail)" + } + exit 1 +} + +Write-Host "" +Write-Host "All compile tests passed." diff --git a/crates/bindings-cpp/tests/compile/run-compile-tests.sh b/crates/bindings-cpp/tests/compile/run-compile-tests.sh new file mode 100644 index 00000000000..d7c634bc9d7 --- /dev/null +++ b/crates/bindings-cpp/tests/compile/run-compile-tests.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BINDINGS_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +INCLUDE_DIR="$BINDINGS_ROOT/include" +BUILD_ROOT="$SCRIPT_DIR/build" +LIBRARY_BUILD_DIR="$BUILD_ROOT/library" +LIBRARY_LOG_DIR="$BUILD_ROOT/logs" +TEMPLATE_PATH="$SCRIPT_DIR/CMakeLists.module.txt" + +SUITE="http-handlers" + +while [[ $# -gt 0 ]]; do + case "$1" in + --suite) + SUITE="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +if [[ "$SUITE" != "http-handlers" ]]; then + echo "Unsupported suite: $SUITE" >&2 + exit 1 +fi + +if command -v emcmake >/dev/null 2>&1; then + EMCMAKE_CMD="emcmake" +elif command -v emcmake.bat >/dev/null 2>&1; then + EMCMAKE_CMD="emcmake.bat" +else + echo "Unable to locate emcmake or emcmake.bat" >&2 + exit 1 +fi + +mkdir -p "$BUILD_ROOT" "$LIBRARY_LOG_DIR" + +LIBRARY_CONFIGURE_LOG="$LIBRARY_LOG_DIR/library-configure.log" +LIBRARY_BUILD_LOG="$LIBRARY_LOG_DIR/library-build.log" + +echo "Building bindings library..." +if ! "$EMCMAKE_CMD" cmake -S "$BINDINGS_ROOT" -B "$LIBRARY_BUILD_DIR" >"$LIBRARY_CONFIGURE_LOG" 2>&1; then + echo "Library configure failed. See $LIBRARY_CONFIGURE_LOG" >&2 + exit 1 +fi + +if ! cmake --build "$LIBRARY_BUILD_DIR" >"$LIBRARY_BUILD_LOG" 2>&1; then + echo "Library build failed. See $LIBRARY_BUILD_LOG" >&2 + exit 1 +fi + +declare -a CASE_NAMES=( + "ok_http_handlers_basic" + "error_http_handler_no_args" + "error_http_handler_immutable_ctx" + "error_http_handler_wrong_ctx" + "error_http_handler_no_request_arg" + "error_http_handler_wrong_request_arg_type" + "error_http_handler_no_return_type" + "error_http_handler_wrong_return_type" + "error_http_handler_no_sender" + "error_http_handler_no_connection_id" + "error_http_handler_no_db" + "error_http_router_not_a_function" + "error_http_router_with_args" + "error_http_router_wrong_return_type" +) + +declare -A CASE_EXPECTATION +declare -A CASE_MARKER +declare -A CASE_SOURCE + +CASE_EXPECTATION["ok_http_handlers_basic"]="success" +CASE_SOURCE["ok_http_handlers_basic"]="$SCRIPT_DIR/cases/http-handlers/ok_http_handlers_basic.cpp" + +CASE_EXPECTATION["error_http_handler_no_args"]="failure" +CASE_MARKER["error_http_handler_no_args"]="too few arguments provided to function-like macro invocation" +CASE_SOURCE["error_http_handler_no_args"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_args.cpp" + +CASE_EXPECTATION["error_http_handler_immutable_ctx"]="failure" +CASE_MARKER["error_http_handler_immutable_ctx"]="First parameter of HTTP handler must be HandlerContext" +CASE_SOURCE["error_http_handler_immutable_ctx"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_immutable_ctx.cpp" + +CASE_EXPECTATION["error_http_handler_wrong_ctx"]="failure" +CASE_MARKER["error_http_handler_wrong_ctx"]="First parameter of HTTP handler must be HandlerContext" +CASE_SOURCE["error_http_handler_wrong_ctx"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_wrong_ctx.cpp" + +CASE_EXPECTATION["error_http_handler_no_request_arg"]="failure" +CASE_MARKER["error_http_handler_no_request_arg"]="too few arguments provided to function-like macro invocation" +CASE_SOURCE["error_http_handler_no_request_arg"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_request_arg.cpp" + +CASE_EXPECTATION["error_http_handler_wrong_request_arg_type"]="failure" +CASE_MARKER["error_http_handler_wrong_request_arg_type"]="Second parameter of HTTP handler must be HttpRequest" +CASE_SOURCE["error_http_handler_wrong_request_arg_type"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_wrong_request_arg_type.cpp" + +CASE_EXPECTATION["error_http_handler_no_return_type"]="failure" +CASE_MARKER["error_http_handler_no_return_type"]="non-void function does not return a value" +CASE_SOURCE["error_http_handler_no_return_type"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_return_type.cpp" + +CASE_EXPECTATION["error_http_handler_wrong_return_type"]="failure" +CASE_MARKER["error_http_handler_wrong_return_type"]="no viable conversion from returned value of type 'unsigned int' to function return type 'SpacetimeDB::HttpResponse'" +CASE_SOURCE["error_http_handler_wrong_return_type"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_wrong_return_type.cpp" + +CASE_EXPECTATION["error_http_handler_no_sender"]="failure" +CASE_MARKER["error_http_handler_no_sender"]="no member named 'sender' in 'SpacetimeDB::HandlerContext'" +CASE_SOURCE["error_http_handler_no_sender"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_sender.cpp" + +CASE_EXPECTATION["error_http_handler_no_connection_id"]="failure" +CASE_MARKER["error_http_handler_no_connection_id"]="no member named 'connection_id' in 'SpacetimeDB::HandlerContext'" +CASE_SOURCE["error_http_handler_no_connection_id"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_connection_id.cpp" + +CASE_EXPECTATION["error_http_handler_no_db"]="failure" +CASE_MARKER["error_http_handler_no_db"]="no member named 'db' in 'SpacetimeDB::HandlerContext'" +CASE_SOURCE["error_http_handler_no_db"]="$SCRIPT_DIR/cases/http-handlers/error_http_handler_no_db.cpp" + +CASE_EXPECTATION["error_http_router_not_a_function"]="failure" +CASE_MARKER["error_http_router_not_a_function"]="illegal initializer" +CASE_SOURCE["error_http_router_not_a_function"]="$SCRIPT_DIR/cases/http-handlers/error_http_router_not_a_function.cpp" + +CASE_EXPECTATION["error_http_router_with_args"]="failure" +CASE_MARKER["error_http_router_with_args"]="too many arguments provided to function-like macro invocation" +CASE_SOURCE["error_http_router_with_args"]="$SCRIPT_DIR/cases/http-handlers/error_http_router_with_args.cpp" + +CASE_EXPECTATION["error_http_router_wrong_return_type"]="failure" +CASE_MARKER["error_http_router_wrong_return_type"]="no viable conversion from returned value of type 'unsigned int' to function return type 'SpacetimeDB::Router'" +CASE_SOURCE["error_http_router_wrong_return_type"]="$SCRIPT_DIR/cases/http-handlers/error_http_router_wrong_return_type.cpp" + +FAILURES=0 + +for CASE_NAME in "${CASE_NAMES[@]}"; do + CASE_BUILD_DIR="$BUILD_ROOT/$CASE_NAME" + CONFIGURE_LOG="$CASE_BUILD_DIR/configure.log" + BUILD_LOG="$CASE_BUILD_DIR/build.log" + + rm -rf "$CASE_BUILD_DIR" + mkdir -p "$CASE_BUILD_DIR" + cp "$TEMPLATE_PATH" "$CASE_BUILD_DIR/CMakeLists.txt" + + echo "Running $CASE_NAME..." + + CONFIGURE_EXIT=0 + BUILD_EXIT=0 + + if "$EMCMAKE_CMD" cmake -S "$CASE_BUILD_DIR" -B "$CASE_BUILD_DIR" \ + -DMODULE_SOURCE="${CASE_SOURCE[$CASE_NAME]}" \ + -DOUTPUT_NAME="$CASE_NAME" \ + -DSPACETIMEDB_LIBRARY_DIR="$LIBRARY_BUILD_DIR" \ + -DSPACETIMEDB_INCLUDE_DIR="$INCLUDE_DIR" >"$CONFIGURE_LOG" 2>&1; then + CONFIGURE_EXIT=0 + else + CONFIGURE_EXIT=$? + fi + + if [[ $CONFIGURE_EXIT -eq 0 ]]; then + if cmake --build "$CASE_BUILD_DIR" >"$BUILD_LOG" 2>&1; then + BUILD_EXIT=0 + else + BUILD_EXIT=$? + fi + fi + + COMBINED_LOG="" + [[ -f "$CONFIGURE_LOG" ]] && COMBINED_LOG+="$(cat "$CONFIGURE_LOG")"$'\n' + [[ -f "$BUILD_LOG" ]] && COMBINED_LOG+="$(cat "$BUILD_LOG")" + + PASS=0 + DETAIL="" + + if [[ "${CASE_EXPECTATION[$CASE_NAME]}" == "success" ]]; then + if [[ $CONFIGURE_EXIT -eq 0 && $BUILD_EXIT -eq 0 ]]; then + PASS=1 + else + DETAIL="Expected build success." + fi + else + if [[ $CONFIGURE_EXIT -ne 0 || $BUILD_EXIT -ne 0 ]]; then + if [[ "$COMBINED_LOG" == *"${CASE_MARKER[$CASE_NAME]}"* ]]; then + PASS=1 + else + DETAIL="Expected marker not found: ${CASE_MARKER[$CASE_NAME]}" + fi + else + DETAIL="Expected build failure." + fi + fi + + if [[ $PASS -eq 1 ]]; then + printf '%-40s PASS\n' "$CASE_NAME" + else + printf '%-40s FAIL\n' "$CASE_NAME" + [[ -z "$DETAIL" ]] && DETAIL="$(printf '%s' "$COMBINED_LOG" | grep -v '^[[:space:]]*$' | head -n 8 | tr '\n' ' ')" + echo " $DETAIL" + FAILURES=1 + fi +done + +if [[ $FAILURES -ne 0 ]]; then + echo + echo "Compile test failures detected." + exit 1 +fi + +echo +echo "All compile tests passed." diff --git a/crates/bindings-cpp/tests/type-isolation-test/CMakeLists.module.txt b/crates/bindings-cpp/tests/type-isolation-test/CMakeLists.module.txt index 243fcc5b902..38db22d8b82 100644 --- a/crates/bindings-cpp/tests/type-isolation-test/CMakeLists.module.txt +++ b/crates/bindings-cpp/tests/type-isolation-test/CMakeLists.module.txt @@ -28,6 +28,7 @@ add_executable(${OUTPUT_NAME} ${MODULE_SOURCE}) # Include directories target_include_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_INCLUDE_DIR}) +target_compile_definitions(${OUTPUT_NAME} PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) # Link the pre-built library target_link_directories(${OUTPUT_NAME} PRIVATE ${SPACETIMEDB_LIBRARY_DIR}) @@ -61,4 +62,4 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") # Name the output lib.wasm set_target_properties(${OUTPUT_NAME} PROPERTIES OUTPUT_NAME "lib" SUFFIX ".wasm") -endif() \ No newline at end of file +endif() diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt new file mode 100644 index 00000000000..0ced4e0194c --- /dev/null +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 3.16) +project(bindings_cpp_unit_tests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + message(FATAL_ERROR "tests/unit is intended to be built with Emscripten via emcmake") +endif() + +add_executable(bindings_cpp_unit_tests + main.cpp + http_unit_tests.cpp +) + +target_include_directories(bindings_cpp_unit_tests PRIVATE + ../../include +) + +target_compile_definitions(bindings_cpp_unit_tests PRIVATE + SPACETIMEDB_UNSTABLE_FEATURES +) + +if(MSVC) + target_compile_options(bindings_cpp_unit_tests PRIVATE /W4) +else() + target_compile_options(bindings_cpp_unit_tests PRIVATE -Wall -Wextra) +endif() + +target_link_options(bindings_cpp_unit_tests PRIVATE + "SHELL:-sWASM=1" + "SHELL:-sENVIRONMENT=node" + "SHELL:-sEXIT_RUNTIME=1" + "SHELL:-sASSERTIONS=1" + "SHELL:-O2" +) + +set_target_properties(bindings_cpp_unit_tests PROPERTIES SUFFIX ".cjs") diff --git a/crates/bindings-cpp/tests/unit/README.md b/crates/bindings-cpp/tests/unit/README.md new file mode 100644 index 00000000000..558027cb113 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/README.md @@ -0,0 +1,52 @@ +# C++ Unit Tests + +Standalone unit-test harness for pure bindings/library behavior. + +This suite is the right home for: +- conversion helpers +- small pure-library regressions +- behavior that does not need wasm module compilation +- behavior that does not need a live SpacetimeDB server + +Current coverage includes the HTTP request/response split-body conversion checks that +mirror the Rust tests added next to `crates/bindings/src/http.rs`. + +This harness is intentionally separate from the top-level bindings CMake so that +small header-only/library tests do not need to build the full module ABI/export layer. + +It is built with Emscripten and run under Node, which matches the existing wasm-oriented +C++ test toolchain more closely than adding a separate native-MSVC path. + +The generated Node launcher uses a `.cjs` suffix so it is treated as CommonJS even though +the repo root sets `"type": "module"`. + +## Run + +Prerequisites: + +- `emcmake` on `PATH` +- `node` on `PATH` + +From PowerShell: + +```powershell +.\crates\bindings-cpp\tests\unit\run-unit-tests.ps1 +``` + +Verbose: + +```powershell +.\crates\bindings-cpp\tests\unit\run-unit-tests.ps1 -Detailed +``` + +From Git Bash: + +```bash +./crates/bindings-cpp/tests/unit/run-unit-tests.sh +``` + +Verbose: + +```bash +./crates/bindings-cpp/tests/unit/run-unit-tests.sh --verbose +``` diff --git a/crates/bindings-cpp/tests/unit/http_unit_tests.cpp b/crates/bindings-cpp/tests/unit/http_unit_tests.cpp new file mode 100644 index 00000000000..16d98db8f09 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/http_unit_tests.cpp @@ -0,0 +1,56 @@ +#include "test_harness.h" + +#include "spacetimedb/http_convert.h" + +#include +#include +#include + +using namespace SpacetimeDB; + +TEST_CASE(request_from_wire_preserves_metadata_and_body) { + wire::HttpRequest request; + request.method = wire::HttpMethod{wire::HttpMethod::Tag::Post, ""}; + request.headers.entries = { + wire::HttpHeaderPair{"content-type", std::vector{'a','p','p','l','i','c','a','t','i','o','n','/','o','c','t','e','t','-','s','t','r','e','a','m'}}, + wire::HttpHeaderPair{"x-echo", std::vector{'v','a','l','u','e'}}, + }; + request.timeout = std::nullopt; + request.uri = "https://example.invalid/upload?x=1"; + request.version = wire::HttpVersion{wire::HttpVersion::Tag::Http2}; + + HttpRequest converted = convert::from_wire(request, std::vector{'p','a','y','l','o','a','d'}); + + ASSERT_EQ(std::string("POST"), converted.method.value); + ASSERT_EQ(std::string("https://example.invalid/upload?x=1"), converted.uri); + ASSERT_EQ(HttpVersion::Http2, converted.version); + ASSERT_EQ(static_cast(2), converted.headers.size()); + ASSERT_EQ(std::string("content-type"), converted.headers[0].name); + ASSERT_EQ(std::vector({'a','p','p','l','i','c','a','t','i','o','n','/','o','c','t','e','t','-','s','t','r','e','a','m'}), converted.headers[0].value); + ASSERT_EQ(std::string("x-echo"), converted.headers[1].name); + ASSERT_EQ(std::vector({'v','a','l','u','e'}), converted.headers[1].value); + ASSERT_EQ(std::vector({'p','a','y','l','o','a','d'}), converted.body.bytes); +} + +TEST_CASE(response_into_wire_splits_metadata_and_body) { + HttpResponse response{ + 201, + HttpVersion::Http11, + { + HttpHeader{"content-type", "text/plain"}, + HttpHeader{"x-result", "ok"}, + }, + HttpBody::from_string("created"), + }; + + auto [response_meta, response_body] = convert::to_wire_split(response); + + ASSERT_EQ(static_cast(201), response_meta.code); + ASSERT_EQ(wire::HttpVersion::Tag::Http11, response_meta.version.tag); + ASSERT_EQ(static_cast(2), response_meta.headers.entries.size()); + ASSERT_EQ(std::string("content-type"), response_meta.headers.entries[0].name); + ASSERT_EQ(std::vector({'t','e','x','t','/','p','l','a','i','n'}), response_meta.headers.entries[0].value); + ASSERT_EQ(std::string("x-result"), response_meta.headers.entries[1].name); + ASSERT_EQ(std::vector({'o','k'}), response_meta.headers.entries[1].value); + ASSERT_EQ(std::vector({'c','r','e','a','t','e','d'}), response_body); +} diff --git a/crates/bindings-cpp/tests/unit/main.cpp b/crates/bindings-cpp/tests/unit/main.cpp new file mode 100644 index 00000000000..054370390b6 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/main.cpp @@ -0,0 +1,34 @@ +#include "test_harness.h" + +#include +#include + +int main(int argc, char** argv) { + bool verbose = argc > 1 && std::string(argv[1]) == "-v"; + int failures = 0; + + for (const auto& test : SpacetimeDB::UnitTests::all_tests()) { + try { + test.func(); + if (verbose) { + std::cout << "[PASS] " << test.name << '\n'; + } + } catch (const std::exception& ex) { + ++failures; + std::cerr << "[FAIL] " << test.name << ": " << ex.what() << '\n'; + } catch (...) { + ++failures; + std::cerr << "[FAIL] " << test.name << ": unknown exception\n"; + } + } + + if (!verbose) { + if (failures == 0) { + std::cout << "Passed " << SpacetimeDB::UnitTests::all_tests().size() << " unit tests\n"; + } else { + std::cerr << failures << " unit test(s) failed\n"; + } + } + + return failures == 0 ? 0 : 1; +} diff --git a/crates/bindings-cpp/tests/unit/run-unit-tests.ps1 b/crates/bindings-cpp/tests/unit/run-unit-tests.ps1 new file mode 100644 index 00000000000..39e023828d9 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/run-unit-tests.ps1 @@ -0,0 +1,52 @@ +[CmdletBinding()] +param( + [switch]$Detailed +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$buildDir = Join-Path $scriptDir 'build' +$launcherPath = Join-Path $buildDir 'bindings_cpp_unit_tests.cjs' + +$emcmake = Get-Command emcmake.bat -ErrorAction SilentlyContinue +if ($null -eq $emcmake) { + $emcmake = Get-Command emcmake -ErrorAction SilentlyContinue +} +if ($null -eq $emcmake) { + throw 'Unable to locate emcmake or emcmake.bat' +} + +$node = Get-Command node -ErrorAction SilentlyContinue +if ($null -eq $node) { + throw 'Unable to locate node' +} + +Write-Host '' +Write-Host '==> Configuring unit tests' -ForegroundColor Cyan +& $emcmake.Source cmake -S $scriptDir -B $buildDir +if ($LASTEXITCODE -ne 0) { + throw "cmake configure failed with exit code $LASTEXITCODE" +} + +Write-Host '' +Write-Host '==> Building unit tests' -ForegroundColor Cyan +cmake --build $buildDir --target bindings_cpp_unit_tests +if ($LASTEXITCODE -ne 0) { + throw "cmake build failed with exit code $LASTEXITCODE" +} + +Write-Host '' +Write-Host '==> Running unit tests' -ForegroundColor Cyan +if (-not (Test-Path $launcherPath)) { + throw "Could not find built bindings_cpp_unit_tests.cjs launcher at $launcherPath" +} +if ($Detailed) { + & $node.Source $launcherPath -v +} else { + & $node.Source $launcherPath +} +if ($LASTEXITCODE -ne 0) { + throw "unit tests failed with exit code $LASTEXITCODE" +} diff --git a/crates/bindings-cpp/tests/unit/run-unit-tests.sh b/crates/bindings-cpp/tests/unit/run-unit-tests.sh new file mode 100644 index 00000000000..7de9aba68b8 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/run-unit-tests.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build" +VERBOSE=0 + +if command -v emcmake >/dev/null 2>&1; then + EMCMAKE_CMD="emcmake" +elif command -v emcmake.bat >/dev/null 2>&1; then + EMCMAKE_CMD="emcmake.bat" +else + echo "Unable to locate emcmake or emcmake.bat" >&2 + exit 1 +fi + +if ! command -v node >/dev/null 2>&1; then + echo "Unable to locate node" >&2 + exit 1 +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + -v|--verbose) + VERBOSE=1 + shift + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +echo +echo "==> Configuring unit tests" +"$EMCMAKE_CMD" cmake -S "$SCRIPT_DIR" -B "$BUILD_DIR" + +echo +echo "==> Building unit tests" +cmake --build "$BUILD_DIR" --target bindings_cpp_unit_tests + +echo +echo "==> Running unit tests" +LAUNCHER="$BUILD_DIR/bindings_cpp_unit_tests.cjs" +if [[ ! -f "$LAUNCHER" ]]; then + echo "Could not find built bindings_cpp_unit_tests.cjs launcher" >&2 + exit 1 +fi + +if [[ $VERBOSE -eq 1 ]]; then + node "$LAUNCHER" -v +else + node "$LAUNCHER" +fi diff --git a/crates/bindings-cpp/tests/unit/test_harness.h b/crates/bindings-cpp/tests/unit/test_harness.h new file mode 100644 index 00000000000..c3e6dff4f64 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/test_harness.h @@ -0,0 +1,49 @@ +#ifndef SPACETIMEDB_TEST_HARNESS_H +#define SPACETIMEDB_TEST_HARNESS_H + +#include +#include +#include + +namespace SpacetimeDB::UnitTests { + +struct TestCase { + const char* name; + void (*func)(); +}; + +inline std::vector& all_tests() { + static std::vector tests; + return tests; +} + +struct TestRegistrar { + TestRegistrar(const char* name, void (*func)()) { + all_tests().push_back(TestCase{name, func}); + } +}; + +} // namespace SpacetimeDB::UnitTests + +#define TEST_CASE(name) \ + void name(); \ + static ::SpacetimeDB::UnitTests::TestRegistrar name##_registrar(#name, &name); \ + void name() + +#define ASSERT_TRUE(condition) \ + do { \ + if (!(condition)) { \ + throw std::runtime_error(std::string("Assertion failed: ") + #condition); \ + } \ + } while (0) + +#define ASSERT_EQ(expected, actual) \ + do { \ + auto expected_value = (expected); \ + auto actual_value = (actual); \ + if (!(expected_value == actual_value)) { \ + throw std::runtime_error(std::string("Assertion failed: ") + #expected " == " #actual); \ + } \ + } while (0) + +#endif // SPACETIMEDB_TEST_HARNESS_H diff --git a/crates/bindings-typescript/src/vue/index.ts b/crates/bindings-typescript/src/vue/index.ts index b32fbdf4b80..3b3e64aba48 100644 --- a/crates/bindings-typescript/src/vue/index.ts +++ b/crates/bindings-typescript/src/vue/index.ts @@ -2,3 +2,4 @@ export * from './SpacetimeDBProvider.ts'; export { useSpacetimeDB } from './useSpacetimeDB.ts'; export { useTable } from './useTable.ts'; export { useReducer } from './useReducer.ts'; +export { useProcedure } from './useProcedure.ts'; diff --git a/crates/bindings-typescript/src/vue/useProcedure.ts b/crates/bindings-typescript/src/vue/useProcedure.ts new file mode 100644 index 00000000000..385b0c96ae8 --- /dev/null +++ b/crates/bindings-typescript/src/vue/useProcedure.ts @@ -0,0 +1,62 @@ +import { shallowRef, watch, onUnmounted } from 'vue'; +import { useSpacetimeDB } from './useSpacetimeDB'; +import type { UntypedProcedureDef } from '../sdk/procedures'; +import type { + ProcedureParamsType, + ProcedureReturnType, +} from '../sdk/type_utils'; + +export function useProcedure( + procedureDef: ProcedureDef +): ( + ...params: ProcedureParamsType +) => Promise> { + const conn = useSpacetimeDB(); + const procedureName = procedureDef.accessorName; + + const queueRef = shallowRef< + { + params: ProcedureParamsType; + resolve: (val: any) => void; + reject: (err: unknown) => void; + }[] + >([]); + + const stopWatch = watch( + () => conn.isActive, + () => { + const connection = conn.getConnection(); + if (!connection) return; + + const fn = (connection.procedures as any)[procedureName] as ( + ...p: ProcedureParamsType + ) => Promise>; + if (queueRef.value.length) { + const pending = queueRef.value.splice(0); + for (const item of pending) { + fn(...item.params).then(item.resolve, item.reject); + } + } + }, + { immediate: true } + ); + + onUnmounted(() => { + stopWatch(); + }); + + return (...params: ProcedureParamsType) => { + const connection = conn.getConnection(); + if (!connection) { + return new Promise>( + (resolve, reject) => { + queueRef.value.push({ params, resolve, reject }); + } + ); + } + const fn = (connection.procedures as any)[procedureName] as ( + ...p: ProcedureParamsType + ) => Promise>; + return fn(...params); + }; +} diff --git a/crates/bindings/tests/deps.rs b/crates/bindings/tests/deps.rs index 2383fdc7a1e..ccbb2b4b685 100644 --- a/crates/bindings/tests/deps.rs +++ b/crates/bindings/tests/deps.rs @@ -25,6 +25,39 @@ fn deptree_snapshot() -> std::io::Result<()> { Ok(()) } +#[test] +fn serde_json_arbitrary_precision_feature_boundaries() { + // https://github.com/clockworklabs/SpacetimeDB/issues/4989 + // `serde_json/arbitrary_precision` is fine for internal tooling like the CLI, + // but it should not be forced onto users compiling the Rust SDK or module + // bindings. Cargo features are additive, so guard those public dependency + // graphs explicitly. + + // The CLI opts into it because `spacetime subscribe` reformats JSON rows + // through `serde_json::Value`; without arbitrary precision, large SATS + // integers like `ConnectionId` can be rounded before typed deserialization. + assert_serde_json_arbitrary_precision("cargo tree -p spacetimedb-cli -e features,no-dev -i serde_json", true); + assert_serde_json_arbitrary_precision( + "cargo tree -p spacetimedb -e features,no-dev --target wasm32-unknown-unknown -i serde_json", + false, + ); + assert_serde_json_arbitrary_precision("cargo tree -p spacetimedb-sdk -e features,no-dev -i serde_json", false); + assert_serde_json_arbitrary_precision( + "cargo tree -p spacetimedb-sdk -e features,no-dev --features browser --target wasm32-unknown-unknown -i serde_json", + false, + ); +} + +#[track_caller] +fn assert_serde_json_arbitrary_precision(cmd: &str, expected: bool) { + let tree = run_cmd(cmd); + assert_eq!( + tree.contains("serde_json feature \"arbitrary_precision\""), + expected, + "`arbitrary_precision` expectation failed for `{cmd}`:\n{tree}" + ); +} + // runs a command string, split on spaces #[track_caller] fn run_cmd(cmd: &str) -> String { diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index b2b8d7e1d89..bb71fb8b6e5 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -58,7 +58,7 @@ regex.workspace = true reqwest.workspace = true rustyline.workspace = true serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true, features = ["raw_value", "preserve_order"] } +serde_json = { workspace = true, features = ["raw_value", "preserve_order", "arbitrary_precision"] } serde_with = { workspace = true, features = ["chrono_0_4"] } slab.workspace = true syntect.workspace = true diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index 5b22b4cd8a5..6586d8edd5f 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -1,20 +1,28 @@ use anyhow::Context; +use bytes::Bytes; use clap::{value_parser, Arg, ArgAction, ArgMatches}; use futures::{Sink, SinkExt, TryStream, TryStreamExt}; use http::header; use reqwest::Url; use serde_json::Value; -use spacetimedb_client_api_messages::websocket::v1 as ws_v1; +use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2, v3 as ws_v3}; use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::{DeserializeWrapper, SeedWrapper}; +use spacetimedb_lib::de::DeserializeSeed as BsatnDeserializeSeed; +use spacetimedb_lib::sats::WithTypespace; use spacetimedb_lib::ser::serde::SerializeWrapper; +use spacetimedb_lib::{bsatn, AlgebraicType}; +use std::collections::VecDeque; use std::io; use std::time::Duration; use thiserror::Error; use tokio::io::AsyncWriteExt; +use tokio::net::TcpStream; use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::handshake::client::Request as WsRequest; use tokio_tungstenite::tungstenite::{Error as WsError, Message as WsMessage}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; use crate::api::ClientApi; use crate::common_args; @@ -72,56 +80,68 @@ pub fn cli() -> clap::Command { .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) } -fn parse_msg_json(msg: &WsMessage) -> Option> { - let WsMessage::Text(msg) = msg else { return None }; - serde_json::from_str::>>(msg) - .inspect_err(|e| eprintln!("couldn't parse message from server: {e}")) - .map(|wrapper| wrapper.0) - .ok() +#[derive(serde::Serialize, Debug)] +struct SubscriptionTable { + deletes: Vec, + inserts: Vec, } -fn reformat_update<'a>( - msg: &'a ws_v1::DatabaseUpdate, - schema: &RawModuleDefV9, -) -> anyhow::Result> { - msg.tables - .iter() - .map(|upd| { - let table_ty = schema.typespace.resolve( - schema - .type_ref_for_table_like(&upd.table_name) - .context("table not found in schema")?, - ); +/// Concrete websocket stream type returned by `tokio_tungstenite::connect_async`. +type SubscribeWebSocket = WebSocketStream>; - let reformat_row = |row: &str| -> anyhow::Result { - // TODO: can the following two calls be merged into a single call to reduce allocations? - let row = serde_json::from_str::(row)?; - let row = serde::de::DeserializeSeed::deserialize(SeedWrapper(table_ty), row)?; - let row = table_ty.with_value(&row); - let row = serde_json::to_value(SerializeWrapper::from_ref(&row))?; - Ok(row) - }; +/// Active websocket connection for `spacetime subscribe`. +/// +/// The command prefers the v3 transport so smoketests and normal CLI usage +/// exercise the coalesced server path, but it keeps the old v1 text transport +/// as a fallback for older servers. +enum SubscribeConnection { + /// v3 uses BSATN-encoded v2 messages, possibly coalesced in one websocket payload. + V3 { + ws: SubscribeWebSocket, + /// Decoded messages left over from a coalesced v3 websocket payload. + pending: VecDeque, + }, + /// v1 is the historical JSON text protocol. + V1 { ws: SubscribeWebSocket }, +} - let mut deletes = Vec::new(); - let mut inserts = Vec::new(); - for upd in &upd.updates { - for s in &upd.deletes { - deletes.push(reformat_row(s)?); - } - for s in &upd.inserts { - inserts.push(reformat_row(s)?); - } - } +impl SubscribeConnection { + /// Send the subscribe request using whichever protocol was negotiated. + async fn subscribe(&mut self, query_strings: Box<[Box]>) -> Result<(), Error> { + match self { + Self::V3 { ws, .. } => subscribe_v3(ws, query_strings).await, + Self::V1 { ws } => subscribe_v1(ws, query_strings).await, + } + } - Ok((&*upd.table_name, SubscriptionTable { deletes, inserts })) - }) - .collect() -} + /// Wait for the initial subscription result and optionally print it. + async fn await_initial_update(&mut self, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> { + match self { + Self::V3 { ws, pending } => await_initial_update_v3(ws, pending, module_def).await, + Self::V1 { ws } => await_initial_update_v1(ws, module_def).await, + } + } -#[derive(serde::Serialize, Debug)] -struct SubscriptionTable { - deletes: Vec, - inserts: Vec, + /// Print transaction updates until the requested count is reached. + async fn consume_transaction_updates( + &mut self, + num: Option, + module_def: &RawModuleDefV9, + ) -> Result<(), Error> { + match self { + Self::V3 { ws, pending } => consume_transaction_updates_v3(ws, pending, num, module_def).await, + Self::V1 { ws } => consume_transaction_updates_v1(ws, num, module_def).await, + } + } + + /// Best-effort graceful websocket close. + async fn close(&mut self) { + match self { + Self::V3 { ws, .. } | Self::V1 { ws } => { + let _ = ws.close(None).await; + } + } + } } pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { @@ -160,36 +180,14 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error }; let api = ClientApi::new(conn); let module_def = api.module_def().await?; - - let mut url = Url::parse(&api.con.db_uri("subscribe"))?; - // Change the URI scheme from `http(s)` to `ws(s)`. - url.set_scheme(match url.scheme() { - "http" => "ws", - "https" => "wss", - unknown => unreachable!("Invalid URL scheme in `Connection::db_uri`: {unknown}"), - }) - .unwrap(); - if let Some(confirmed) = confirmed { - url.query_pairs_mut() - .append_pair("confirmed", if confirmed { "true" } else { "false" }); - } - - // Create the websocket request. - let mut req = url.into_client_request()?; - req.headers_mut().insert( - header::SEC_WEBSOCKET_PROTOCOL, - http::HeaderValue::from_static(ws_v1::TEXT_PROTOCOL), - ); - // Add the authorization header, if any. - if let Some(auth_header) = api.con.auth_header.to_header() { - req.headers_mut().insert(header::AUTHORIZATION, auth_header); - } - let mut ws = tokio_tungstenite::connect_async(req).await.map(|(ws, _)| ws)?; + let mut conn = connect_with_fallback(&api, confirmed).await?; let task = async { - subscribe(&mut ws, queries.iter().cloned().map(Into::into).collect()).await?; - await_initial_update(&mut ws, print_initial_update.then_some(&module_def)).await?; - consume_transaction_updates(&mut ws, num, &module_def).await + conn.subscribe(queries.iter().cloned().map(Into::into).collect()) + .await?; + conn.await_initial_update(print_initial_update.then_some(&module_def)) + .await?; + conn.consume_transaction_updates(num, &module_def).await }; let res = if let Some(timeout) = timeout { @@ -211,7 +209,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error // The error (if any) relevant to the user is already stored in `res`, // so we can ignore errors here -- graceful close is basically a // courtesy to the server. - let _ = ws.close(None).await; + conn.close().await; // The server closing the connection is not considered an error, // but any other error is. res.or_else(|e| { @@ -224,6 +222,90 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error .map_err(anyhow::Error::from) } +/// Connect using v3 when available, otherwise retry once with the v1 text protocol. +/// +/// Fallback is intentionally limited to connection setup and protocol +/// negotiation. After a v3 connection is accepted, malformed v3 data is a real +/// error and should not be hidden by silently reconnecting with v1. +async fn connect_with_fallback(api: &ClientApi, confirmed: Option) -> Result { + match connect_v3(api, confirmed).await { + Ok(conn) => Ok(conn), + Err(v3_error) => connect_v1(api, confirmed) + .await + .with_context(|| format!("v3 subscribe connection failed ({v3_error}); v1 fallback also failed")), + } +} + +/// Open a v3 subscribe websocket and validate that the server negotiated v3. +async fn connect_v3(api: &ClientApi, confirmed: Option) -> Result { + let req = subscribe_request(api, confirmed, ws_v3::BIN_PROTOCOL, true)?; + let (ws, response) = tokio_tungstenite::connect_async(req).await?; + if response + .headers() + .get(header::SEC_WEBSOCKET_PROTOCOL) + .and_then(|value| value.to_str().ok()) + != Some(ws_v3::BIN_PROTOCOL) + { + return Err(Error::Protocol { + details: "server did not negotiate the v3 websocket protocol", + } + .into()); + } + Ok(SubscribeConnection::V3 { + ws, + pending: VecDeque::new(), + }) +} + +/// Open a v1 text subscribe websocket for compatibility with older servers. +async fn connect_v1(api: &ClientApi, confirmed: Option) -> Result { + let req = subscribe_request(api, confirmed, ws_v1::TEXT_PROTOCOL, false)?; + let (ws, _) = tokio_tungstenite::connect_async(req).await?; + Ok(SubscribeConnection::V1 { ws }) +} + +/// Build a subscribe websocket request for a specific subprotocol. +/// +/// `request_uncompressed` is used only for the CLI v3 path. The CLI decodes +/// enough v3 to print JSON output, but does not implement brotli or gzip +/// decoding, so v3 asks the server for uncompressed payloads. The v1 fallback +/// leaves the query string in its historical shape. +fn subscribe_request( + api: &ClientApi, + confirmed: Option, + protocol: &'static str, + request_uncompressed: bool, +) -> Result { + let mut url = Url::parse(&api.con.db_uri("subscribe"))?; + // Change the URI scheme from `http(s)` to `ws(s)`. + url.set_scheme(match url.scheme() { + "http" => "ws", + "https" => "wss", + unknown => unreachable!("Invalid URL scheme in `Connection::db_uri`: {unknown}"), + }) + .unwrap(); + { + let mut query = url.query_pairs_mut(); + if request_uncompressed { + // The CLI v3 path only needs enough support to print updates as + // JSON, so request uncompressed payloads and avoid brotli/gzip + // decoding here. The v1 fallback preserves the old URL shape. + query.append_pair("compression", "None"); + } + if let Some(confirmed) = confirmed { + query.append_pair("confirmed", if confirmed { "true" } else { "false" }); + } + } + + let mut req = url.into_client_request()?; + req.headers_mut() + .insert(header::SEC_WEBSOCKET_PROTOCOL, http::HeaderValue::from_static(protocol)); + if let Some(auth_header) = api.con.auth_header.to_header() { + req.headers_mut().insert(header::AUTHORIZATION, auth_header); + } + Ok(req) +} + #[derive(Debug, Error)] enum Error { #[error("error sending subscription queries")] @@ -247,6 +329,16 @@ enum Error { #[source] source: anyhow::Error, }, + #[error("error encoding BSATN websocket message: {source}")] + BsatnEncode { + #[source] + source: spacetimedb_lib::bsatn::EncodeError, + }, + #[error("error decoding BSATN websocket message: {source}")] + BsatnDecode { + #[source] + source: spacetimedb_lib::bsatn::DecodeError, + }, #[error(transparent)] Serde(#[from] serde_json::Error), #[error(transparent)] @@ -264,8 +356,8 @@ impl Error { } } -/// Send the subscribe message. -async fn subscribe(ws: &mut S, query_strings: Box<[Box]>) -> Result<(), Error> +/// Send a v1 JSON subscribe message. +async fn subscribe_v1(ws: &mut S, query_strings: Box<[Box]>) -> Result<(), Error> where S: Sink + Unpin, { @@ -279,9 +371,34 @@ where ws.send(msg.into()).await.map_err(|source| Error::Subscribe { source }) } -/// Await the initial [`ServerMessage::SubscriptionUpdate`]. +/// Send a v3 BSATN subscribe message. +async fn subscribe_v3(ws: &mut S, query_strings: Box<[Box]>) -> Result<(), Error> +where + S: Sink + Unpin, +{ + let msg = ws_v2::ClientMessage::Subscribe(ws_v2::Subscribe { + request_id: 0, + query_set_id: ws_v2::QuerySetId::new(0), + query_strings, + }); + let msg = bsatn::to_vec(&msg).map_err(|source| Error::BsatnEncode { source })?; + ws.send(WsMessage::Binary(msg.into())) + .await + .map_err(|source| Error::Subscribe { source }) +} + +/// Parse a v1 text websocket message as JSON. +fn parse_msg_json(msg: &WsMessage) -> Option> { + let WsMessage::Text(msg) = msg else { return None }; + serde_json::from_str::>>(msg) + .inspect_err(|e| eprintln!("couldn't parse message from server: {e}")) + .map(|wrapper| wrapper.0) + .ok() +} + +/// Await the initial v1 [`ws_v1::ServerMessage::InitialSubscription`]. /// If `module_def` is `Some`, print a JSON representation to stdout. -async fn await_initial_update(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> +async fn await_initial_update_v1(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> where S: TryStream + Unpin, { @@ -292,7 +409,7 @@ where match msg { ws_v1::ServerMessage::InitialSubscription(sub) => { if let Some(module_def) = module_def { - let output = format_output_json(&sub.database_update, module_def)?; + let output = format_output_json_v1(&sub.database_update, module_def)?; tokio::io::stdout().write_all(output.as_bytes()).await? } break; @@ -320,9 +437,49 @@ where Ok(()) } -/// Print `num` [`ServerMessage::TransactionUpdate`] messages as JSON. +/// Await the initial [`ws_v2::ServerMessage::SubscribeApplied`]. +/// If `module_def` is `Some`, print a JSON representation to stdout. +async fn await_initial_update_v3( + ws: &mut S, + pending: &mut VecDeque, + module_def: Option<&RawModuleDefV9>, +) -> Result<(), Error> +where + S: TryStream + Unpin, +{ + const RECV_TX_UPDATE: &str = "received transaction update before initial subscription update"; + + while let Some(msg) = next_server_message(ws, pending).await? { + match msg { + ws_v2::ServerMessage::SubscribeApplied(sub) => { + if let Some(module_def) = module_def { + let output = format_output_json_query_rows(&sub.rows, module_def)?; + tokio::io::stdout().write_all(output.as_bytes()).await? + } + break; + } + ws_v2::ServerMessage::SubscriptionError(error) => { + return Err(Error::SubscribeFailure { reason: error.error }); + } + ws_v2::ServerMessage::TransactionUpdate(_) => { + return Err(Error::Protocol { + details: RECV_TX_UPDATE, + }) + } + _ => continue, + } + } + + Ok(()) +} + +/// Print `num` v1 [`ws_v1::ServerMessage::TransactionUpdate`] messages as JSON. /// If `num` is `None`, keep going indefinitely. -async fn consume_transaction_updates(ws: &mut S, num: Option, module_def: &RawModuleDefV9) -> Result<(), Error> +async fn consume_transaction_updates_v1( + ws: &mut S, + num: Option, + module_def: &RawModuleDefV9, +) -> Result<(), Error> where S: TryStream + Unpin, { @@ -354,7 +511,50 @@ where status: ws_v1::UpdateStatus::Committed(update), .. }) => { - let output = format_output_json(&update, module_def)?; + let output = format_output_json_v1(&update, module_def)?; + stdout.write_all(output.as_bytes()).await?; + num_received += 1; + } + _ => continue, + } + } +} + +/// Print `num` [`ws_v2::ServerMessage::TransactionUpdate`] messages as JSON. +/// If `num` is `None`, keep going indefinitely. +async fn consume_transaction_updates_v3( + ws: &mut S, + pending: &mut VecDeque, + num: Option, + module_def: &RawModuleDefV9, +) -> Result<(), Error> +where + S: TryStream + Unpin, +{ + let mut stdout = tokio::io::stdout(); + let mut num_received = 0; + loop { + if num.is_some_and(|n| num_received >= n) { + return Ok(()); + } + let Some(msg) = next_server_message(ws, pending).await? else { + eprintln!("disconnected by server"); + return Err(Error::Websocket { + source: WsError::ConnectionClosed, + }); + }; + + match msg { + ws_v2::ServerMessage::SubscribeApplied(_) => { + return Err(Error::Protocol { + details: "received a second initial subscription update", + }) + } + ws_v2::ServerMessage::SubscriptionError(error) => { + return Err(Error::SubscribeFailure { reason: error.error }); + } + ws_v2::ServerMessage::TransactionUpdate(update) => { + let output = format_output_json_transaction_update(&update, module_def)?; stdout.write_all(output.as_bytes()).await?; num_received += 1; } @@ -363,12 +563,236 @@ where } } -fn format_output_json( +/// Return the next decoded server message from a v3 websocket stream. +/// +/// A v3 websocket payload can contain multiple consecutive BSATN-encoded v2 +/// server messages, so decoded surplus messages are queued for the next call. +/// Non-binary messages are ignored because v3 server data is binary-only. +async fn next_server_message( + ws: &mut S, + pending: &mut VecDeque, +) -> Result, Error> +where + S: TryStream + Unpin, +{ + loop { + if let Some(msg) = pending.pop_front() { + return Ok(Some(msg)); + } + + let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + return Ok(None); + }; + let WsMessage::Binary(msg) = msg else { continue }; + decode_server_payload(msg, pending)?; + } +} + +/// Decode one uncompressed v3 websocket payload into queued v2 server messages. +/// +/// The server prefixes each binary payload with a compression tag. This CLI path +/// requests `compression=None`, so any compressed tag is treated as a protocol +/// error rather than decoded here. +fn decode_server_payload(msg: Bytes, pending: &mut VecDeque) -> Result<(), Error> { + let Some((&tag, mut remaining)) = msg.as_ref().split_first() else { + return Err(Error::Protocol { + details: "received empty v3 websocket payload", + }); + }; + if tag != ws_common::SERVER_MSG_COMPRESSION_TAG_NONE { + return Err(Error::Protocol { + details: "compressed v3 subscribe payload is not supported by this CLI path", + }); + } + if remaining.is_empty() { + return Err(Error::Protocol { + details: "received v3 websocket payload without a server message", + }); + } + + while !remaining.is_empty() { + let msg = bsatn::from_reader(&mut remaining).map_err(|source| Error::BsatnDecode { source })?; + pending.push_back(msg); + } + + Ok(()) +} + +/// Format a v1 database update using the legacy JSON row representation. +fn format_output_json_v1( msg: &ws_v1::DatabaseUpdate, schema: &RawModuleDefV9, ) -> Result { - let formatted = reformat_update(msg, schema).map_err(|source| Error::Reformat { source })?; - let output = serde_json::to_string(&formatted)? + "\n"; + let formatted = reformat_update_v1(msg, schema).map_err(|source| Error::Reformat { source })?; + format_output_json_from_tables(&formatted) +} +/// Format initial v3 subscription rows using the CLI's existing JSON output shape. +fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &RawModuleDefV9) -> Result { + let formatted = reformat_query_rows(msg, schema).map_err(|source| Error::Reformat { source })?; + format_output_json_from_tables(&formatted) +} + +/// Format a v3 transaction update using the CLI's existing JSON output shape. +fn format_output_json_transaction_update( + msg: &ws_v2::TransactionUpdate, + schema: &RawModuleDefV9, +) -> Result { + let formatted = reformat_transaction_update(msg, schema).map_err(|source| Error::Reformat { source })?; + format_output_json_from_tables(&formatted) +} + +/// Serialize the normalized table update map as one JSON object per output line. +fn format_output_json_from_tables(formatted: &HashMap<&str, SubscriptionTable>) -> Result { + let output = serde_json::to_string(formatted)? + "\n"; Ok(output) } + +/// Convert a v1 JSON-format database update to the normalized table output map. +fn reformat_update_v1<'a>( + msg: &'a ws_v1::DatabaseUpdate, + schema: &RawModuleDefV9, +) -> anyhow::Result> { + msg.tables + .iter() + .map(|upd| { + let table_ty = schema.typespace.resolve( + schema + .type_ref_for_table_like(&upd.table_name) + .context("table not found in schema")?, + ); + + let reformat_row = |row: &str| -> anyhow::Result { + // TODO: can the following two calls be merged into a single call to reduce allocations? + let row = serde_json::from_str::(row)?; + let row = serde::de::DeserializeSeed::deserialize(SeedWrapper(table_ty), row)?; + let row = table_ty.with_value(&row); + let row = serde_json::to_value(SerializeWrapper::from_ref(&row))?; + Ok(row) + }; + + let mut deletes = Vec::new(); + let mut inserts = Vec::new(); + for upd in &upd.updates { + for s in &upd.deletes { + deletes.push(reformat_row(s)?); + } + for s in &upd.inserts { + inserts.push(reformat_row(s)?); + } + } + + Ok((&*upd.table_name, SubscriptionTable { deletes, inserts })) + }) + .collect() +} + +/// Convert v3 initial subscription rows to the normalized table output map. +fn reformat_query_rows<'a>( + msg: &'a ws_v2::QueryRows, + schema: &RawModuleDefV9, +) -> anyhow::Result> { + let mut formatted = HashMap::default(); + + for table in &msg.tables { + let table_ty = schema.typespace.resolve( + schema + .type_ref_for_table_like(&table.table) + .context("table not found in schema")?, + ); + let table_output = formatted.entry(&*table.table).or_insert_with(|| SubscriptionTable { + deletes: Vec::new(), + inserts: Vec::new(), + }); + table_output.inserts.extend(reformat_bsatn_rows(&table.rows, table_ty)?); + } + + Ok(formatted) +} + +/// Convert a v3 transaction update to the normalized table output map. +fn reformat_transaction_update<'a>( + msg: &'a ws_v2::TransactionUpdate, + schema: &RawModuleDefV9, +) -> anyhow::Result> { + let mut formatted = HashMap::default(); + + for query_set in &msg.query_sets { + for table in &query_set.tables { + let table_ty = schema.typespace.resolve( + schema + .type_ref_for_table_like(&table.table_name) + .context("table not found in schema")?, + ); + let table_output = formatted + .entry(&*table.table_name) + .or_insert_with(|| SubscriptionTable { + deletes: Vec::new(), + inserts: Vec::new(), + }); + for rows in &table.rows { + match rows { + ws_v2::TableUpdateRows::PersistentTable(rows) => { + table_output + .deletes + .extend(reformat_bsatn_rows(&rows.deletes, table_ty)?); + table_output + .inserts + .extend(reformat_bsatn_rows(&rows.inserts, table_ty)?); + } + ws_v2::TableUpdateRows::EventTable(rows) => { + table_output + .inserts + .extend(reformat_bsatn_rows(&rows.events, table_ty)?); + } + } + } + } + } + + Ok(formatted) +} + +/// Decode BSATN row-list entries and re-encode them as schema-aware JSON values. +fn reformat_bsatn_rows( + rows: &ws_common::BsatnRowList, + table_ty: WithTypespace<'_, AlgebraicType>, +) -> anyhow::Result> { + rows.into_iter() + .map(|row| { + let mut row = row.as_ref(); + let row = BsatnDeserializeSeed::deserialize(table_ty, bsatn::Deserializer::new(&mut row))?; + let row = table_ty.with_value(&row); + Ok(serde_json::to_value(SerializeWrapper::from_ref(&row))?) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_lib::sats::{ + algebraic_value::de::ValueDeserializer, de::Deserialize, GroundSpacetimeType, Typespace, WithTypespace, + }; + use spacetimedb_lib::ConnectionId; + + #[test] + fn serde_json_value_preserves_connection_id_u128() -> anyhow::Result<()> { + // `spacetime subscribe` reformats JSON rows through `serde_json::Value` + // before typed SATS deserialization. The CLI enables + // `serde_json/arbitrary_precision` so large `ConnectionId` values do not + // get rounded while inside `Value`. + let conn_id = ConnectionId::from_u128(u64::MAX as u128 + 1); + let json = serde_json::to_string(&SerializeWrapper::new(&conn_id))?; + let row = serde_json::from_str::(&json)?; + + let typespace = Typespace::default(); + let conn_id_ty = ConnectionId::get_type(); + let conn_id_ty = WithTypespace::new(&typespace, &conn_id_ty); + let de = serde::de::DeserializeSeed::deserialize(SeedWrapper(conn_id_ty), row)?; + let de = ConnectionId::deserialize(ValueDeserializer::new(de)).unwrap(); + + assert_eq!(conn_id, de); + Ok(()) + } +} diff --git a/crates/client-api/src/routes/subscribe.rs b/crates/client-api/src/routes/subscribe.rs index e2131abfa31..f29527bee42 100644 --- a/crates/client-api/src/routes/subscribe.rs +++ b/crates/client-api/src/routes/subscribe.rs @@ -23,7 +23,7 @@ use prometheus::{Histogram, IntGauge}; use scopeguard::{defer, ScopeGuard}; use serde::Deserialize; use spacetimedb::client::messages::{ - serialize, serialize_v2, IdentityTokenMessage, InUseSerializeBuffer, SerializeBuffer, SwitchedServerMessage, + serialize, serialize_v3, IdentityTokenMessage, InUseSerializeBuffer, SerializeBuffer, SwitchedServerMessage, ToProtocol, }; use spacetimedb::client::{ @@ -39,7 +39,7 @@ use spacetimedb::Identity; use spacetimedb_client_api_messages::websocket::v1 as ws_v1; use spacetimedb_client_api_messages::websocket::v2 as ws_v2; use spacetimedb_client_api_messages::websocket::v3 as ws_v3; -use spacetimedb_datastore::execution_context::WorkloadType; +use spacetimedb_lib::bsatn; use spacetimedb_lib::connection_id::{ConnectionId, ConnectionIdForUrl}; use tokio::sync::{mpsc, watch}; use tokio::task::JoinHandle; @@ -1064,13 +1064,13 @@ enum UnorderedWsMessage { /// Abstraction over [`ClientConnectionReceiver`], so tests can use a plain /// [`mpsc::Receiver`]. trait Receiver { - fn recv(&mut self) -> impl Future> + Send; + fn recv_many(&mut self, buf: &mut Vec, max: usize) -> impl Future + Send; fn close(&mut self); } impl Receiver for ClientConnectionReceiver { - async fn recv(&mut self) -> Option { - ClientConnectionReceiver::recv(self).await + async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + ClientConnectionReceiver::recv_many(self, buf, max).await } fn close(&mut self) { @@ -1079,8 +1079,8 @@ impl Receiver for ClientConnectionReceiver { } impl Receiver for mpsc::Receiver { - async fn recv(&mut self) -> Option { - mpsc::Receiver::recv(self).await + async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + mpsc::Receiver::recv_many(self, buf, max).await } fn close(&mut self) { @@ -1148,6 +1148,8 @@ async fn ws_send_loop_inner( // The default frame size is 4KiB, hence we write in batches of 32KiB. const FRAME_BATCH_SIZE: usize = 8; let mut frames_batch = Vec::with_capacity(FRAME_BATCH_SIZE); + const MESSAGE_BATCH_SIZE: usize = ClientConnectionReceiver::DEFAULT_RECV_MANY_LIMIT; + let mut message_batch = Vec::new(); let (frames_tx, mut frames_rx) = mpsc::unbounded_channel(); let (encode_tx, encode_rx) = mpsc::unbounded_channel(); @@ -1262,12 +1264,15 @@ async fn ws_send_loop_inner( // Take on more work. // // Branch is disabled if we already sent a close frame. - Some(message) = messages.recv(), if !closed => { - encode_tx - .send(message.into()) - // `ws_encode_task` shouldn't terminate until - // `encode_tx` is dropped, except by panicking. - .expect("encode task panicked"); + n = messages.recv_many(&mut message_batch, MESSAGE_BATCH_SIZE), if !closed => { + log::trace!("encoding batch of {n} messages"); + for message in message_batch.drain(..n) { + encode_tx + .send(message.into()) + // `ws_encode_task` shouldn't terminate until + // `encode_tx` is dropped, except by panicking. + .expect("encode task panicked"); + } }, } @@ -1285,13 +1290,143 @@ enum OutboundWsMessage { Message(OutboundMessage), } -/// Task that reads [`OutboundWsMessage`]s from `messages`, encodes them via -/// [`ws_encode_message`], and sends the resuling [`Frame`]s to `outgoing_frames`. +/// Controls how many binary protocol messages may be packed into a single +/// websocket payload. +/// +/// Protocol v2 requires one [`ws_v2::ServerMessage`] per websocket message. +/// Protocol v3 keeps the v2 message schema but permits multiple consecutive +/// v2 messages in a single websocket message. +#[derive(Clone, Copy, PartialEq, Eq)] +enum BinaryPayloadMode { + /// Flush after each binary server message. + Single, + /// Flush once after all available binary server messages are collected. + Coalesced, +} + +/// A binary websocket message plus the logical row count it contributes to +/// payload-level send metrics. +struct V2OutboundMessage { + message: ws_v2::ServerMessage, + num_rows: Option, +} + +/// Convert an outbound message into the binary websocket schema. +/// +/// v2 connections should only receive v2 server messages. +/// v1 messages are dropped. +/// +/// TODO: For better type safety, [`ClientConnectionReceiver`] should be made +/// generic over the protocol version. +fn v2_outbound_message(message: OutboundWsMessage) -> Option { + let message = match message { + OutboundWsMessage::Error(message) => { + log::error!("dropping v1 error message on v2 connection: {:?}", message); + return None; + } + OutboundWsMessage::Message(message) => message, + }; + + let num_rows = message.num_rows(); + match message { + OutboundMessage::V2(message) => Some(V2OutboundMessage { message, num_rows }), + OutboundMessage::V1(message) => { + log::error!("dropping v1 message on v2 connection: {:?}", message); + None + } + } +} + +/// Return the uncompressed payload size of `message`. +/// +/// v2 sends exactly one BSATN-encoded v2 server message per websocket payload. +/// v3 sends one or more of the same encoded messages in a coalesced payload. +fn message_size(message: &ws_v2::ServerMessage) -> usize { + bsatn::to_len(message).expect("should be able to measure bsatn-encoded v2 server message") +} + +/// Return whether appending the next message would cross the v3 coalescing cap. +/// +/// An empty payload is always allowed to accept one message, even when that +/// message alone is larger than the cap. +fn v3_payload_would_exceed_limit(total_bytes: usize, message_bytes: usize) -> bool { + total_bytes != 0 && total_bytes.saturating_add(message_bytes) > V3_MAX_UNCOMPRESSED_PAYLOAD_SIZE +} + +/// Return whether a binary websocket payload is large enough to encode on Rayon. +fn is_large_payload(num_bytes: usize) -> bool { + num_bytes >= V3_MAX_UNCOMPRESSED_PAYLOAD_SIZE +} + +/// Encoding receive batch size. +/// +/// This is deliberately tied to the client connection receive limit so the +/// websocket encoder can consume the batches produced by +/// [`ClientConnectionReceiver::recv_many`] without immediately re-batching +/// them to a different size. +const ENCODE_BATCH_SIZE: usize = ClientConnectionReceiver::DEFAULT_RECV_MANY_LIMIT; + +/// Target maximum uncompressed v3 payload body size. +/// +/// The v3 binary body is a sequence of BSATN-encoded v2 server messages. The +/// one-byte compression tag is not counted here. This is a target, not a hard +/// rejection limit. One logical server message may exceed it, in which case the +/// message is sent by itself. +const V3_MAX_UNCOMPRESSED_PAYLOAD_SIZE: usize = 512 * 1024; + +/// Tracks serialize buffers that may be reusable once their frames have been +/// copied to the wire. +struct SerializeBufferPool { + config: ClientConfig, + available: ArrayQueue, + in_use: Vec, +} + +impl SerializeBufferPool { + const CAPACITY: usize = 16; + + fn new(config: ClientConfig) -> Self { + Self { + config, + available: ArrayQueue::new(Self::CAPACITY), + in_use: Vec::with_capacity(Self::CAPACITY), + } + } + + fn get(&mut self) -> SerializeBuffer { + self.reclaim(); + self.available + .pop() + .unwrap_or_else(|| SerializeBuffer::new(self.config)) + } + + fn hold(&mut self, in_use: InUseSerializeBuffer) { + if self.in_use.len() < Self::CAPACITY { + self.in_use.push(in_use); + } + } + + fn reclaim(&mut self) { + let mut i = 0; + while i < self.in_use.len() { + if self.in_use[i].is_unique() { + let in_use = self.in_use.swap_remove(i); + let buf = in_use.try_reclaim().expect("buffer should be unique"); + let _ = self.available.push(buf); + } else { + i += 1; + } + } + } +} + +/// Task that reads [`OutboundWsMessage`]s from `messages`, encodes them, and +/// sends the resulting [`Frame`]s to `outgoing_frames`. /// /// Meant to be [`tokio::spawn`]ed. /// /// The function also takes care of reusing serialization buffers and reporting -/// metrics via [`SendMetrics`].. +/// metrics via [`SendMetrics`]. async fn ws_encode_task( metrics: SendMetrics, config: ClientConfig, @@ -1299,91 +1434,264 @@ async fn ws_encode_task( outgoing_frames: mpsc::UnboundedSender, bsatn_rlb_pool: BsatnRowListBuilderPool, ) { - // Serialize buffers can be reclaimed once all frames of a message are - // copied to the wire. Since we don't know when that will happen, we prepare - // for a few messages to be in-flight, i.e. encoded but not yet sent. - const BUF_POOL_CAPACITY: usize = 16; - let buf_pool = ArrayQueue::new(BUF_POOL_CAPACITY); - let mut in_use_bufs: Vec> = Vec::with_capacity(BUF_POOL_CAPACITY); - - 'send: while let Some(message) = messages.recv().await { - // Drop serialize buffers with no external referent, - // returning them to the pool. - in_use_bufs.retain(|in_use| !in_use.is_unique()); - // Get a serialize buffer from the pool, - // or create a fresh one. - let buf = buf_pool.pop().unwrap_or_else(|| SerializeBuffer::new(config)); - - let in_use_buf = match message { - OutboundWsMessage::Error(message) => { - if config.version != WsVersion::V1 { - log::error!( - "dropping v1 error message sent to a binary websocket client: {:?}", - message - ); - continue; + let mut encoder = WsEncoder { + config, + buffers: SerializeBufferPool::new(config), + metrics: &metrics, + outgoing_frames: &outgoing_frames, + bsatn_rlb_pool: &bsatn_rlb_pool, + binary_server_messages: Vec::new(), + }; + let mut message_batch = Vec::new(); + while messages.recv_many(&mut message_batch, ENCODE_BATCH_SIZE).await != 0 { + log::trace!("encoding batch of {} websocket messages", message_batch.len()); + // `encode_batch` drains `message_batch` on success. If forwarding to + // the websocket send loop fails, the receiver is gone, so the encode + // task can terminate. + if encoder.encode_batch(&mut message_batch).await.is_err() { + break; + } + } +} + +/// Stateful websocket encoder for one client connection. +/// +/// The encoder owns reusable scratch storage: +/// +/// - [`SerializeBufferPool`] reuses byte buffers once encoded frames have been +/// copied to the socket task. +/// - `binary_server_messages` reuses the vector allocation used to assemble +/// v2/v3 binary websocket payloads. +struct WsEncoder<'a> { + config: ClientConfig, + buffers: SerializeBufferPool, + metrics: &'a SendMetrics, + outgoing_frames: &'a mpsc::UnboundedSender, + bsatn_rlb_pool: &'a BsatnRowListBuilderPool, + binary_server_messages: Vec, +} + +impl WsEncoder<'_> { + /// Encode a drained batch according to the websocket version negotiated by + /// the client. + async fn encode_batch( + &mut self, + message_batch: &mut Vec, + ) -> Result<(), mpsc::error::SendError> { + match self.config.version { + WsVersion::V1 => self.encode_v1_batch(message_batch).await, + WsVersion::V2 => self.encode_v2_batch(message_batch).await, + WsVersion::V3 => self.encode_v3_batch(message_batch).await, + } + } + + /// Encode a batch for the original v1 websocket protocols. + /// + /// v1 text/binary messages are encoded one logical message at a time. This + /// path also handles reducer errors, which still use the v1 message schema. + async fn encode_v1_batch( + &mut self, + message_batch: &mut Vec, + ) -> Result<(), mpsc::error::SendError> { + for message in message_batch.drain(..) { + match message { + OutboundWsMessage::Error(message) => { + self.encode_and_forward_v1_message(None, message).await?; } - let Ok(in_use) = ws_forward_frames( - &metrics, - &outgoing_frames, - None, - None, - ws_encode_message(config, buf, message, false, &bsatn_rlb_pool).await, - ) else { - break 'send; - }; - in_use - } - OutboundWsMessage::Message(message) => { - let workload = message.workload(); - let num_rows = message.num_rows(); - match message { - OutboundMessage::V2(server_message) => { - if config.version == WsVersion::V1 { + OutboundWsMessage::Message(message) => { + let num_rows = message.num_rows(); + match message { + OutboundMessage::V2(_) => { log::error!("dropping v2 message on v1 connection"); continue; } - - let Ok(in_use) = ws_forward_frames( - &metrics, - &outgoing_frames, - workload, - num_rows, - ws_encode_binary_message(config, buf, server_message, false, &bsatn_rlb_pool).await, - ) else { - break 'send; - }; - in_use - } - OutboundMessage::V1(message) => { - if config.version != WsVersion::V1 { - log::error!("dropping v1 message for a binary websocket connection: {:?}", message); - continue; + OutboundMessage::V1(message) => { + self.encode_and_forward_v1_message(num_rows, message).await?; } + } + } + } + } + Ok(()) + } + + /// Encode a batch for protocol v2. + /// + /// v2 uses the binary server-message schema, but each logical server + /// message must still be sent as its own websocket message. + async fn encode_v2_batch( + &mut self, + message_batch: &mut Vec, + ) -> Result<(), mpsc::error::SendError> { + self.encode_binary_batch(message_batch, BinaryPayloadMode::Single).await + } + + /// Encode a batch for protocol v3. + /// + /// v3 uses the same binary server-message schema as v2, but coalesces all + /// messages currently available from the encoder input into one websocket + /// payload. + async fn encode_v3_batch( + &mut self, + message_batch: &mut Vec, + ) -> Result<(), mpsc::error::SendError> { + self.encode_binary_batch(message_batch, BinaryPayloadMode::Coalesced) + .await + } + + /// Encode binary websocket payloads from a batch of outbound messages. + /// + /// `mode` is the only protocol-specific choice here: + /// + /// - [`BinaryPayloadMode::Single`] preserves the v2 wire format by + /// flushing after each message. + /// - [`BinaryPayloadMode::Coalesced`] uses the v3 wire format by flushing + /// after the whole batch has been accumulated. + async fn encode_binary_batch( + &mut self, + message_batch: &mut Vec, + mode: BinaryPayloadMode, + ) -> Result<(), mpsc::error::SendError> { + self.binary_server_messages.clear(); + self.binary_server_messages.reserve(match mode { + BinaryPayloadMode::Single => 1, + BinaryPayloadMode::Coalesced => message_batch.len(), + }); + let mut total_rows = None; + let mut total_bytes = 0; + + for message in message_batch.drain(..) { + // Drop messages that are not valid for a binary websocket + // connection. The conversion logs the protocol mismatch. + let Some(v2_message) = v2_outbound_message(message) else { + continue; + }; - let is_large = num_rows.is_some_and(|n| n > 1024); - - let Ok(in_use) = ws_forward_frames( - &metrics, - &outgoing_frames, - workload, - num_rows, - ws_encode_message(config, buf, message, is_large, &bsatn_rlb_pool).await, - ) else { - break 'send; - }; - in_use + let message = v2_message.message; + let message_rows = v2_message.num_rows; + + let message_bytes = message_size(&message); + match mode { + BinaryPayloadMode::Coalesced => { + if v3_payload_would_exceed_limit(total_bytes, message_bytes) { + // v3 payload boundary: adding this message would cross the + // target byte limit, so flush the payload accumulated so far. + self.flush_binary_payload(&mut total_rows, &mut total_bytes).await?; } + self.append_binary_message(message, message_rows, message_bytes, &mut total_rows, &mut total_bytes); + } + BinaryPayloadMode::Single => { + // v2 payload boundary: exactly one binary server message per websocket message. + self.append_binary_message(message, message_rows, message_bytes, &mut total_rows, &mut total_bytes); + self.flush_binary_payload(&mut total_rows, &mut total_bytes).await?; } } - }; + } - if in_use_bufs.len() < BUF_POOL_CAPACITY { - in_use_bufs.push(scopeguard::guard(in_use_buf, |in_use| { - let buf = in_use.try_reclaim().expect("buffer should be unique"); - let _ = buf_pool.push(buf); - })); + // Final v3 payload boundary: flush the remaining coalesced messages. + // This is a no-op for v2 because `Single` mode flushes inside the loop. + self.flush_binary_payload(&mut total_rows, &mut total_bytes).await + } + + /// Append one v2 server message to the binary websocket payload currently being accumulated. + fn append_binary_message( + &mut self, + message: ws_v2::ServerMessage, + message_rows: Option, + message_bytes: usize, + total_rows: &mut Option, + total_bytes: &mut usize, + ) { + if let Some(message_rows) = message_rows { + // Payload metrics are emitted at websocket-payload granularity. + // In v3, one payload can contain several logical messages, so row + // counts are accumulated across the coalesced payload. + *total_rows.get_or_insert(0) += message_rows; } + self.binary_server_messages.push(message); + *total_bytes += message_bytes; + } + + /// Encode and forward the accumulated binary payload, then reset its counters. + async fn flush_binary_payload( + &mut self, + total_rows: &mut Option, + total_bytes: &mut usize, + ) -> Result<(), mpsc::error::SendError> { + if self.binary_server_messages.is_empty() { + return Ok(()); + } + let is_large = is_large_payload(*total_bytes); + self.encode_and_forward_binary_messages(total_rows.take(), is_large) + .await?; + *total_bytes = 0; + Ok(()) + } + + /// Encode and forward one v1 websocket message. + /// + /// v1 can produce either text or binary payloads depending on the client's + /// requested protocol, so it uses [`ws_encode_message`] rather than the + /// binary-only v2/v3 path. + async fn encode_and_forward_v1_message( + &mut self, + num_rows: Option, + message: impl ToProtocol + Send + 'static, + ) -> Result<(), mpsc::error::SendError> { + let config = self.config; + let bsatn_rlb_pool = self.bsatn_rlb_pool; + self.encode_and_forward_message(|buf| ws_encode_message(config, buf, message, true, bsatn_rlb_pool, num_rows)) + .await + } + + /// Encode and forward the currently accumulated binary server messages. + /// + /// This method is shared by v2 and v3. v2 calls it with exactly one + /// `binary_server_messages` entry; v3 calls it with the whole coalesced + /// batch. The actual bytes are produced by [`serialize_v3`], whose core + /// implementation also backs `serialize_v2`. + async fn encode_and_forward_binary_messages( + &mut self, + num_rows: Option, + is_large: bool, + ) -> Result<(), mpsc::error::SendError> { + let buf = self.buffers.get(); + // `spawn_rayon` requires a `'static` closure, so the message Vec cannot + // be borrowed from `self`. Move it into the closure and return the + // drained Vec afterward so its allocation is reused by the next batch. + let messages = std::mem::take(&mut self.binary_server_messages); + let compression = self.config.compression; + let bsatn_rlb_pool = self.bsatn_rlb_pool.clone(); + let (messages, timing, in_use, data) = maybe_spawn_encode(is_large, move || { + let mut messages = messages; + let (timing, in_use, data) = + time_encode(|| serialize_v3(&bsatn_rlb_pool, buf, messages.drain(..), compression)); + (messages, timing, in_use, data) + }) + .await; + self.binary_server_messages = messages; + let encoded = ws_encode_binary_frames(timing, in_use, data, num_rows); + let in_use = ws_forward_frames(self.metrics, self.outgoing_frames, encoded); + let in_use = in_use?; + self.buffers.hold(in_use); + Ok(()) + } + + /// Encode one websocket payload using a reusable serialization buffer, + /// forward its frames, then retain the buffer for later reuse. + async fn encode_and_forward_message( + &mut self, + encode: Encode, + ) -> Result<(), mpsc::error::SendError> + where + Encode: FnOnce(SerializeBuffer) -> Fut, + Fut: Future, + Frames: IntoIterator, + { + let buf = self.buffers.get(); + let in_use = ws_forward_frames(self.metrics, self.outgoing_frames, encode(buf).await)?; + self.buffers.hold(in_use); + Ok(()) } } @@ -1392,25 +1700,27 @@ async fn ws_encode_task( fn ws_forward_frames( metrics: &SendMetrics, outgoing_frames: &mpsc::UnboundedSender, - workload: Option, - num_rows: Option, - encoded: (EncodeMetrics, InUseSerializeBuffer, impl IntoIterator), + encoded: ( + EncodedPayloadMetrics, + InUseSerializeBuffer, + impl IntoIterator, + ), ) -> Result> { let (stats, in_use, frames) = encoded; - metrics.report(workload, num_rows, stats); + metrics.report(stats); frames.into_iter().try_for_each(|frame| outgoing_frames.send(frame))?; Ok(in_use) } -/// Some stats about serialization and compression. -/// -/// Returned by [`ws_encode_message`]. -struct EncodeMetrics { +/// Metrics for one encoded websocket payload. +struct EncodedPayloadMetrics { /// Time it took to serialize and (potentially) compress a message. /// Does not include scheduling overhead. timing: Duration, /// Length in bytes of the serialized and (potentially) compressed message. encoded_len: usize, + /// Number of logical rows included in the payload, if known. + num_rows: Option, } /// Encodes `message` into zero or more WebSocket [`Frame`]s. @@ -1427,7 +1737,7 @@ struct EncodeMetrics { /// of payload each, according to the rules laid out in [RFC6455], Section /// 5.4 Fragmentation. /// -/// Returns [`EncodeMetrics`], the [`InUseSerializeBuffer`] that was passed in +/// Returns [`EncodedPayloadMetrics`], the [`InUseSerializeBuffer`] that was passed in /// as `buf` for later reuse, and the [`Frame`]s. /// /// NOTE: When sending, the frames of a single message MUST NOT be interleaved @@ -1441,62 +1751,75 @@ async fn ws_encode_message( message: impl ToProtocol + Send + 'static, is_large_message: bool, bsatn_rlb_pool: &BsatnRowListBuilderPool, -) -> (EncodeMetrics, InUseSerializeBuffer, impl Iterator) { - const FRAGMENT_SIZE: usize = 4096; - - fn serialize_and_compress( - bsatn_rlb_pool: &BsatnRowListBuilderPool, - serialize_buf: SerializeBuffer, - message: impl ToProtocol + Send + 'static, - config: ClientConfig, - ) -> (Duration, InUseSerializeBuffer, DataMessage) { - let start = Instant::now(); - let (msg_alloc, msg_data) = serialize(bsatn_rlb_pool, serialize_buf, message, config); - (start.elapsed(), msg_alloc, msg_data) - } - let (timing, msg_alloc, msg_data) = if is_large_message { - let bsatn_rlb_pool = bsatn_rlb_pool.clone(); - spawn_rayon(move || serialize_and_compress(&bsatn_rlb_pool, buf, message, config)).await - } else { - serialize_and_compress(bsatn_rlb_pool, buf, message, config) - }; - - let metrics = EncodeMetrics { - timing, - encoded_len: msg_data.len(), - }; + num_rows: Option, +) -> (EncodedPayloadMetrics, InUseSerializeBuffer, impl Iterator) { + let bsatn_rlb_pool = bsatn_rlb_pool.clone(); + // Serialization/compression can dominate large subscription or query + // responses, so large payloads are offloaded to Rayon. + let (timing, in_use, msg_data) = maybe_spawn_encode(is_large_message, move || { + time_encode(|| serialize(&bsatn_rlb_pool, buf, message, config)) + }) + .await; + let encoded_len = msg_data.len(); let (data, ty) = match msg_data { DataMessage::Text(text) => (bytestring_to_utf8bytes(text).into(), Data::Text), DataMessage::Binary(bin) => (bin, Data::Binary), }; - let frames = fragment(data, ty, FRAGMENT_SIZE); + ws_encode_frames(timing, in_use, encoded_len, data, ty, num_rows) +} - (metrics, msg_alloc, frames) +/// Run `encode` on Rayon when the payload is expected to be large. +/// +/// Small payloads stay on the async task to avoid Rayon scheduling overhead. +async fn maybe_spawn_encode(is_large: bool, encode: impl FnOnce() -> T + Send + 'static) -> T { + if is_large { + spawn_rayon(encode).await + } else { + encode() + } } -async fn ws_encode_binary_message( - config: ClientConfig, - buf: SerializeBuffer, - message: ws_v2::ServerMessage, - is_large_message: bool, - bsatn_rlb_pool: &BsatnRowListBuilderPool, -) -> (EncodeMetrics, InUseSerializeBuffer, impl Iterator + use<>) { +/// Measure serialization/compression time for one websocket payload. +fn time_encode(encode: impl FnOnce() -> (InUseSerializeBuffer, T)) -> (Duration, InUseSerializeBuffer, T) { let start = Instant::now(); - let compression = config.compression; + let (in_use, data) = encode(); + (start.elapsed(), in_use, data) +} - let (in_use, data) = if is_large_message { - let bsatn_rlb_pool = bsatn_rlb_pool.clone(); - spawn_rayon(move || serialize_v2(&bsatn_rlb_pool, buf, message, compression)).await - } else { - serialize_v2(bsatn_rlb_pool, buf, message, compression) - }; +/// Build binary websocket frames and payload metrics for encoded bytes. +fn ws_encode_binary_frames( + timing: Duration, + in_use: InUseSerializeBuffer, + data: Bytes, + num_rows: Option, +) -> ( + EncodedPayloadMetrics, + InUseSerializeBuffer, + impl Iterator + use<>, +) { + ws_encode_frames(timing, in_use, data.len(), data, Data::Binary, num_rows) +} - let metrics = EncodeMetrics { - timing: start.elapsed(), - encoded_len: data.len(), +/// Build websocket frames and payload metrics for already-serialized bytes. +fn ws_encode_frames( + timing: Duration, + in_use: InUseSerializeBuffer, + encoded_len: usize, + data: Bytes, + ty: Data, + num_rows: Option, +) -> ( + EncodedPayloadMetrics, + InUseSerializeBuffer, + impl Iterator + use<>, +) { + let metrics = EncodedPayloadMetrics { + timing, + encoded_len, + num_rows, }; - let frames = fragment(data, Data::Binary, 4096); + let frames = fragment(data, ty, 4096); (metrics, in_use, frames) } @@ -1540,33 +1863,32 @@ impl ClientMessage { } } +/// Cached metric handles for the websocket send path. struct SendMetrics { - database: Identity, encode_timing: Histogram, + payload_size: Histogram, + payload_num_rows: Histogram, } impl SendMetrics { + /// Resolve metric handles for one database once per websocket send loop. fn new(database: Identity) -> Self { Self { encode_timing: WORKER_METRICS.websocket_serialize_secs.with_label_values(&database), - database, + payload_size: WORKER_METRICS.websocket_sent_msg_size.with_label_values(&database), + payload_num_rows: WORKER_METRICS.websocket_sent_num_rows.with_label_values(&database), } } - fn report(&self, workload: Option, num_rows: Option, encode: EncodeMetrics) { + /// Report one encoded websocket payload. + fn report(&self, encode: EncodedPayloadMetrics) { self.encode_timing.observe(encode.timing.as_secs_f64()); + self.payload_size.observe(encode.encoded_len as f64); - // These metrics should be updated together, - // or not at all. - if let (Some(workload), Some(num_rows)) = (workload, num_rows) { - WORKER_METRICS - .websocket_sent_num_rows - .with_label_values(&self.database, &workload) - .observe(num_rows as f64); - WORKER_METRICS - .websocket_sent_msg_size - .with_label_values(&self.database, &workload) - .observe(encode.encoded_len as f64); + if let Some(num_rows) = encode.num_rows { + // Some websocket payloads, such as control or error messages, do + // not correspond to a known logical row count. + self.payload_num_rows.observe(num_rows as f64); } } } diff --git a/crates/codegen/src/cpp.rs b/crates/codegen/src/cpp.rs index 3f20b4d271a..9ddbeae8bb0 100644 --- a/crates/codegen/src/cpp.rs +++ b/crates/codegen/src/cpp.rs @@ -19,6 +19,37 @@ pub struct Cpp<'opts> { } impl<'opts> Cpp<'opts> { + fn cpp_field_name<'a>(&self, field_name: &'a str) -> &'a str { + match field_name { + "alignas" | "alignof" | "and" | "and_eq" | "asm" | "atomic_cancel" | "atomic_commit" + | "atomic_noexcept" | "auto" | "bitand" | "bitor" | "bool" | "break" | "case" | "catch" | "char" + | "char8_t" | "char16_t" | "char32_t" | "class" | "compl" | "concept" | "const" | "consteval" + | "constexpr" | "constinit" | "const_cast" | "continue" | "co_await" | "co_return" | "co_yield" + | "decltype" | "default" | "delete" | "do" | "double" | "dynamic_cast" | "else" | "enum" | "explicit" + | "export" | "extern" | "false" | "float" | "for" | "friend" | "goto" | "if" | "inline" | "int" + | "long" | "mutable" | "namespace" | "new" | "noexcept" | "not" | "not_eq" | "nullptr" | "operator" + | "or" | "or_eq" | "private" | "protected" | "public" | "register" | "reinterpret_cast" | "requires" + | "return" | "short" | "signed" | "sizeof" | "static" | "static_assert" | "static_cast" | "struct" + | "switch" | "template" | "this" | "thread_local" | "throw" | "true" | "try" | "typedef" | "typeid" + | "typename" | "union" | "unsigned" | "using" | "virtual" | "void" | "volatile" | "wchar_t" | "while" + | "xor" | "xor_eq" => "", + _ => field_name, + } + } + + fn write_cpp_field_name(&self, output: &mut String, field_name: &str) -> fmt::Result { + let escaped = self.cpp_field_name(field_name); + if escaped.is_empty() { + write!(output, "{}_", field_name) + } else { + write!(output, "{}", escaped) + } + } + + fn is_recursive_mount_module_field(&self, type_name: &str, field_name: &str) -> bool { + type_name == "RawModuleMountV10" && field_name == "module" + } + fn write_header_comment(&self, output: &mut String) { writeln!( output, @@ -148,8 +179,16 @@ impl<'opts> Cpp<'opts> { // Write fields only for (field_name, field_type) in &product.elements { write!(output, " ").unwrap(); - self.write_algebraic_type(output, module, field_type).unwrap(); - writeln!(output, " {};", field_name).unwrap(); + if self.is_recursive_mount_module_field(type_name, field_name) { + // Temporary special-case to preserve the recursive RawModuleMountV10 -> + // RawModuleDefV10 shape while breaking the include cycle in generated C++. + write!(output, "std::shared_ptr<{}::RawModuleDefV10>", self.namespace).unwrap(); + } else { + self.write_algebraic_type(output, module, field_type).unwrap(); + } + write!(output, " ").unwrap(); + self.write_cpp_field_name(output, field_name).unwrap(); + writeln!(output, ";").unwrap(); } writeln!(output).unwrap(); @@ -161,23 +200,30 @@ impl<'opts> Cpp<'opts> { ) .unwrap(); for (field_name, _) in &product.elements { - writeln!( - output, - " ::SpacetimeDB::bsatn::serialize(writer, {});", - field_name - ) - .unwrap(); + if self.is_recursive_mount_module_field(type_name, field_name) { + write!(output, " ::SpacetimeDB::bsatn::serialize(writer, *").unwrap(); + self.write_cpp_field_name(output, field_name).unwrap(); + writeln!(output, ");").unwrap(); + } else { + write!(output, " ::SpacetimeDB::bsatn::serialize(writer, ").unwrap(); + self.write_cpp_field_name(output, field_name).unwrap(); + writeln!(output, ");").unwrap(); + } } writeln!(output, " }}").unwrap(); // Generate equality method - if !product.elements.is_empty() { + if type_name == "RawModuleMountV10" { + // Pointer equality is sufficient for this internal autogen type. Mounts are not + // emitted by the C++ module path yet; this exists to keep the schema shape aligned. + writeln!(output, " SPACETIMEDB_PRODUCT_TYPE_EQUALITY(namespace_, module)").unwrap(); + } else if !product.elements.is_empty() { write!(output, " SPACETIMEDB_PRODUCT_TYPE_EQUALITY(").unwrap(); for (i, (field_name, _)) in product.elements.iter().enumerate() { if i > 0 { write!(output, ", ").unwrap(); } - write!(output, "{}", field_name).unwrap(); + self.write_cpp_field_name(output, field_name).unwrap(); } writeln!(output, ")").unwrap(); } @@ -493,13 +539,20 @@ impl Lang for Cpp<'_> { None => HashSet::new(), }; + let type_name = name.to_string(); for dep in deps { - if dep != name.to_string() { + if dep != type_name && !(type_name == "RawModuleMountV10" && dep == "RawModuleDefV10") { writeln!(output, "#include \"{}.g.h\"", dep).unwrap(); } } writeln!(output).unwrap(); + if type_name == "RawModuleMountV10" { + writeln!(output, "namespace {} {{", self.namespace).unwrap(); + writeln!(output, "struct RawModuleDefV10;").unwrap(); + writeln!(output, "}} // namespace {}", self.namespace).unwrap(); + writeln!(output).unwrap(); + } writeln!(output, "namespace {} {{", self.namespace).unwrap(); writeln!(output).unwrap(); diff --git a/crates/commitlog/src/index/indexfile.rs b/crates/commitlog/src/index/indexfile.rs index 080fac97472..338c3e4552a 100644 --- a/crates/commitlog/src/index/indexfile.rs +++ b/crates/commitlog/src/index/indexfile.rs @@ -161,7 +161,7 @@ impl + From> IndexFileMut { /// Errors /// - `IndexError::InvalidInput`: Either Key or Value is 0 /// - `IndexError::OutOfMemory`: Append after index file is already full. - pub fn append(&mut self, key: Key, value: u64) -> Result<(), IndexError> { + pub fn append(&mut self, key: Key, value: u64) -> Result { let key = key.into(); let last_key = self.last_key()?; if last_key >= key { @@ -179,7 +179,7 @@ impl + From> IndexFileMut { self.inner[start..start + KEY_SIZE].copy_from_slice(&key_bytes); self.inner[start + KEY_SIZE..start + ENTRY_SIZE].copy_from_slice(&value_bytes); self.num_entries += 1; - Ok(()) + Ok(start) } /// Asynchronously flushes any pending changes to the index file @@ -190,6 +190,21 @@ impl + From> IndexFileMut { self.inner.flush_async() } + /// Asynchronously flushes the index entry starting at `offset` to the index file. + /// + /// On linux, the underlying `msync` is a documented no-op since the kernel already + /// tracks dirty pages and flushes them as needed. + /// + /// See https://man7.org/linux/man-pages/man2/msync.2.html for details. + /// + /// On macOS, it is not a documented no-op, and it explicitly states that `msync` + /// will only examine pages covered by the provided address range. Hence this should + /// be preferred over [`Self::async_flush`] when only flushing a single entry at a + /// time, since it may avoid examining pages across the whole mapping. + pub fn async_flush_entry(&self, offset: usize) -> io::Result<()> { + self.inner.flush_async_range(offset, ENTRY_SIZE) + } + /// Truncates the index file starting from the entry with a key greater than /// or equal to the given key. /// diff --git a/crates/commitlog/src/lib.rs b/crates/commitlog/src/lib.rs index abc8729c978..599c71c0136 100644 --- a/crates/commitlog/src/lib.rs +++ b/crates/commitlog/src/lib.rs @@ -70,7 +70,7 @@ pub struct Options { /// If `true`, require that the segment must be synced to disk before an /// index entry is added. /// - /// Setting this to `false` (the default) will update the index every + /// Setting this to `false` will update the index every /// `offset_index_interval_bytes`, even if the commitlog wasn't synced. /// This means that the index could contain non-existent entries in the /// event of a crash. @@ -80,7 +80,7 @@ pub struct Options { /// This means that the index could contain fewer index entries than /// strictly every `offset_index_interval_bytes`. /// - /// Default: false + /// Default: true #[cfg_attr( feature = "serde", serde(default = "Options::default_offset_index_require_segment_fsync") @@ -95,7 +95,7 @@ pub struct Options { /// Size in bytes of the memory buffer holding commit data before flushing /// to storage. /// - /// Default: 8KiB + /// Default: 128KiB #[cfg_attr(feature = "serde", serde(default = "Options::default_write_buffer_size"))] pub write_buffer_size: usize, } @@ -109,9 +109,9 @@ impl Default for Options { impl Options { pub const DEFAULT_MAX_SEGMENT_SIZE: u64 = 1024 * 1024 * 1024; pub const DEFAULT_OFFSET_INDEX_INTERVAL_BYTES: NonZeroU64 = NonZeroU64::new(4096).expect("4096 > 0, qed"); - pub const DEFAULT_OFFSET_INDEX_REQUIRE_SEGMENT_FSYNC: bool = false; + pub const DEFAULT_OFFSET_INDEX_REQUIRE_SEGMENT_FSYNC: bool = true; pub const DEFAULT_PREALLOCATE_SEGMENTS: bool = false; - pub const DEFAULT_WRITE_BUFFER_SIZE: usize = 8 * 1024; + pub const DEFAULT_WRITE_BUFFER_SIZE: usize = 128 * 1024; pub const DEFAULT: Self = Self { log_format_version: DEFAULT_LOG_FORMAT_VERSION, diff --git a/crates/commitlog/src/segment.rs b/crates/commitlog/src/segment.rs index 7913a3799e5..d4f85bf8fa0 100644 --- a/crates/commitlog/src/segment.rs +++ b/crates/commitlog/src/segment.rs @@ -356,9 +356,10 @@ impl OffsetIndexWriter { return Ok(()); } - self.head + let entry_offset = self + .head .append(self.candidate_min_tx_offset, self.candidate_byte_offset)?; - self.head.async_flush()?; + self.head.async_flush_entry(entry_offset)?; self.reset(); Ok(()) @@ -371,10 +372,6 @@ impl FileLike for OffsetIndexWriter { let _ = self.append_internal().map_err(|e| { warn!("failed to append to offset index: {e:?}"); }); - let _ = self - .head - .async_flush() - .map_err(|e| warn!("failed to flush offset index: {e:?}")); Ok(()) } diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 0a3fa198b01..8f19156d6bb 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -19,6 +19,7 @@ use bytes::Bytes; use bytestring::ByteString; use derive_more::From; use futures::prelude::*; +use log::warn; use prometheus::{Histogram, IntCounter, IntGauge}; use spacetimedb_auth::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2}; @@ -29,7 +30,7 @@ use spacetimedb_lib::Identity; use tokio::sync::mpsc::error::{SendError, TrySendError}; use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::AbortHandle; -use tracing::{trace, warn}; +use tracing::trace; #[derive(PartialEq, Eq, Clone, Copy, Hash, Debug)] pub enum Protocol { @@ -154,11 +155,13 @@ impl DurableOffsetSupply for Arc { pub struct ClientConnectionReceiver { confirmed_reads: bool, channel: MeteredReceiver, - current: Option, + pending: Vec, offset_supply: Box, } impl ClientConnectionReceiver { + pub const DEFAULT_RECV_MANY_LIMIT: usize = 4096; + fn new( confirmed_reads: bool, channel: MeteredReceiver, @@ -167,82 +170,115 @@ impl ClientConnectionReceiver { Self { confirmed_reads, channel, - current: None, + pending: Vec::new(), offset_supply: Box::new(offset_supply), } } - /// Receive the next message from this channel. - /// - /// If this method returns `None`, the channel is closed and no more messages - /// are in the internal buffers. No more messages can ever be received from - /// the channel. + #[cfg(test)] + pub(crate) async fn recv(&mut self) -> Option { + let mut buf = Vec::with_capacity(1); + (self.recv_many(&mut buf, 1).await != 0).then(|| buf.remove(0)) + } + + /// Receive multiple messages from this channel. /// /// Messages are returned immediately if: /// - /// - The (internal) [`ClientUpdate`] does not have a `tx_offset` + /// - The [`ClientUpdate`] does not have a `tx_offset` /// (such as for error messages). - /// - The client hasn't requested confirmed reads (i.e. - /// [`ClientConfig::confirmed_reads`] is `false`). + /// - The client hasn't requested confirmed reads + /// (i.e. [`ClientConfig::confirmed_reads`] is `false`). /// - The database is configured to not persist transactions. /// - /// Otherwise, the update's `tx_offset` is compared against the module's + /// Otherwise, the last `tx_offset` in the batch is compared against the module's /// durable offset. If the durable offset is behind the `tx_offset`, the /// method waits until it catches up before returning the message. /// /// If the database is shut down while waiting for the durable offset, - /// `None` is returned. In this case, no more messages can ever be received + /// 0 is returned. In this case, no more messages can ever be received /// from the channel. /// + /// For non-zero values of `max`, this method will never return `0` unless the + /// input channel has been closed and there are no pending messages, or if the + /// database goes away. This indicates that no further values can ever be received + /// from this `Receiver`. + /// /// # Cancel safety /// /// This method is cancel safe, as long as `self` is not dropped. /// - /// If `recv` is used in a [`tokio::select!`] statement, it may get + /// If `recv_many` is used in a [`tokio::select!`] statement, it may get /// cancelled while waiting for the durable offset to catch up. At this - /// point, it has already received a value from the underlying channel. - /// This value is stored internally, so calling `recv` again will not lose - /// data. - // - // TODO: Can we make a cancel-safe `recv_many` with confirmed reads semantics? - pub async fn recv(&mut self) -> Option { - let ClientUpdate { tx_offset, message } = match self.current.take() { - None => self.channel.recv().await?, - Some(update) => update, - }; + /// point, it has already received values from the underlying channel. + /// These values are stored internally, so calling `recv_many` again will + /// not lose data. + pub async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + // If there are no pending updates and the input channel has been closed, + // no more messages can be received from this receiver. + if max == 0 || (self.pending.is_empty() && self.channel.recv_many(&mut self.pending, max).await == 0) { + return 0; + } + + // If we don't have to wait for txns to be made durable, + // drain the pending updates. if !self.confirmed_reads { - return Some(message); + return self.drain_pending(buf, max); + } + + // If we do have to wait for txns to be made durable, + // but the next client update doesn't have a tx offset, + // there's no reason to wait - just send it. + if !self.pending_update_has_offset() { + return self.drain_pending(buf, 1); } - if let Some(tx_offset) = tx_offset { - match self.offset_supply.durable_offset() { - Ok(Some(mut durable)) => { - // Store the current update in case we get cancelled while - // waiting for the durable offset. - self.current = Some(ClientUpdate { - tx_offset: Some(tx_offset), - message, - }); - trace!("waiting for offset {tx_offset} to become durable"); - durable - .wait_for(tx_offset) - .await - .inspect_err(|_| { - warn!("database went away while waiting for durable offset"); - }) - .ok()?; - self.current.take().map(|update| update.message) + // Otherwise, grab the next offset that we should wait for. + let (n, wait_for_offset) = self.next_confirmed_reads_batch(max); + + match self.offset_supply.durable_offset() { + Ok(Some(mut durable)) => { + trace!("waiting for offset {wait_for_offset} to become durable"); + if durable.wait_for(wait_for_offset).await.is_err() { + warn!("database went away while waiting for durable offset"); + return 0; } - // Database shut down or crashed. - Err(NoSuchModule) => None, - // In-memory database. - Ok(None) => Some(message), + self.drain_pending(buf, n) } - } else { - Some(message) + // Database shut down or crashed. + Err(NoSuchModule) => 0, + // In-memory database. + Ok(None) => self.drain_pending(buf, max), } } + /// Compute the next batch of pending client updates that have a tx offset. + /// What is the size of the batch and what is the max offset? + fn next_confirmed_reads_batch(&self, max: usize) -> (usize, TxOffset) { + self.pending + .iter() + .take(max) + .map_while(|update| update.tx_offset) + .fold((0, 0), |(count, max_offset), tx_offset| { + (count + 1, max_offset.max(tx_offset)) + }) + } + + /// Drain the pending [`ClientUpdate`]s, up to `max, into `buf`. + fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + let n = self.pending.len().min(max); + buf.reserve(n); + buf.extend(self.pending.drain(..n).map(|u| u.message)); + n + } + + /// Does the next pending update have a tx offset? + /// + /// Assumes that [`Self::pending`] is not empty. + fn pending_update_has_offset(&self) -> bool { + self.pending.first().is_some_and(|update| update.tx_offset.is_some()) + } + /// Close the receiver without dropping it. /// /// This is used to notify the [`ClientConnectionSender`] that the receiver diff --git a/crates/core/src/client/messages.rs b/crates/core/src/client/messages.rs index 798596b5bca..123c1c75d4a 100644 --- a/crates/core/src/client/messages.rs +++ b/crates/core/src/client/messages.rs @@ -196,20 +196,46 @@ pub fn serialize( /// conditional compression when configured. pub fn serialize_v2( bsatn_rlb_pool: &BsatnRowListBuilderPool, - mut buffer: SerializeBuffer, + buffer: SerializeBuffer, msg: ws_v2::ServerMessage, compression: ws_v1::Compression, +) -> (InUseSerializeBuffer, Bytes) { + serialize_v2_messages(bsatn_rlb_pool, buffer, std::iter::once(msg), compression) +} + +/// Serialize one or more [`ws_v2::ServerMessage`]s into a v3 websocket payload. +/// +/// Protocol v3 keeps the v2 message schema, but allows the uncompressed payload +/// body to contain consecutive BSATN-encoded server messages. +pub fn serialize_v3( + bsatn_rlb_pool: &BsatnRowListBuilderPool, + buffer: SerializeBuffer, + msgs: impl IntoIterator, + compression: ws_v1::Compression, +) -> (InUseSerializeBuffer, Bytes) { + serialize_v2_messages(bsatn_rlb_pool, buffer, msgs, compression) +} + +fn serialize_v2_messages( + bsatn_rlb_pool: &BsatnRowListBuilderPool, + mut buffer: SerializeBuffer, + msgs: impl IntoIterator, + compression: ws_v1::Compression, ) -> (InUseSerializeBuffer, Bytes) { let srv_msg = buffer.write_with_tag(ws_common::SERVER_MSG_COMPRESSION_TAG_NONE, |w| { - bsatn::to_writer(w.into_inner(), &msg).expect("should be able to bsatn encode v2 message"); + let out = w.into_inner(); + for msg in msgs { + write_v2_server_message(bsatn_rlb_pool, out, msg); + } }); let srv_msg_len = srv_msg.len(); + finalize_binary_serialize_buffer(buffer, srv_msg_len, compression) +} - // At this point, we no longer have a use for `msg`, - // so try to reclaim its buffers. +fn write_v2_server_message(bsatn_rlb_pool: &BsatnRowListBuilderPool, out: &mut BytesMut, msg: ws_v2::ServerMessage) { + bsatn::to_writer(out, &msg).expect("should be able to bsatn encode v2 message"); + // At this point, we no longer have a use for `msg`, so try to reclaim its buffers. msg.consume_each_list(&mut |buffer| bsatn_rlb_pool.try_put(buffer)); - - finalize_binary_serialize_buffer(buffer, srv_msg_len, compression) } #[derive(Debug, From)] diff --git a/crates/core/src/db/persistence.rs b/crates/core/src/db/persistence.rs index 5b0daa5145c..d7c51b34e98 100644 --- a/crates/core/src/db/persistence.rs +++ b/crates/core/src/db/persistence.rs @@ -1,4 +1,8 @@ -use std::{io, sync::Arc}; +use std::{ + io, + num::{NonZeroU64, NonZeroUsize}, + sync::Arc, +}; use async_trait::async_trait; use spacetimedb_commitlog::SizeOnDisk; @@ -13,6 +17,59 @@ use super::{ snapshot::{self, SnapshotDatabaseState, SnapshotWorker}, }; +/// Local durability configuration exposed through server config. +#[derive(Clone, Copy, Debug, Default, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct DurabilityConfig { + #[serde(default)] + pub commitlog: CommitlogConfig, +} + +/// Commitlog configuration exposed through server config. +#[derive(Clone, Copy, Debug, Default, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct CommitlogConfig { + pub log_format_version: Option, + pub max_segment_size: Option, + #[serde(alias = "offset-interval-bytes")] + pub offset_index_interval_bytes: Option, + #[serde(alias = "offset-index-require-fsync")] + pub offset_index_require_segment_fsync: Option, + pub preallocate_segments: Option, + pub write_buffer_size: Option, +} + +impl DurabilityConfig { + fn into_options(self) -> spacetimedb_durability::local::Options { + let mut opts = spacetimedb_durability::local::Options::default(); + self.commitlog.apply_to(&mut opts.commitlog); + opts + } +} + +impl CommitlogConfig { + fn apply_to(self, opts: &mut spacetimedb_commitlog::Options) { + if let Some(log_format_version) = self.log_format_version { + opts.log_format_version = log_format_version; + } + if let Some(max_segment_size) = self.max_segment_size { + opts.max_segment_size = max_segment_size.get(); + } + if let Some(offset_index_interval_bytes) = self.offset_index_interval_bytes { + opts.offset_index_interval_bytes = offset_index_interval_bytes; + } + if let Some(offset_index_require_segment_fsync) = self.offset_index_require_segment_fsync { + opts.offset_index_require_segment_fsync = offset_index_require_segment_fsync; + } + if let Some(preallocate_segments) = self.preallocate_segments { + opts.preallocate_segments = preallocate_segments; + } + if let Some(write_buffer_size) = self.write_buffer_size { + opts.write_buffer_size = write_buffer_size.get(); + } + } +} + /// [spacetimedb_durability::Durability] impls with a [`Txdata`] transaction /// payload, suitable for use in the [`relational_db::RelationalDB`]. pub type Durability = dyn spacetimedb_durability::Durability; @@ -128,14 +185,21 @@ pub trait PersistenceProvider: Send + Sync { /// [compresses]: relational_db::snapshot_watching_commitlog_compressor pub struct LocalPersistenceProvider { data_dir: Arc, + durability: DurabilityConfig, } impl LocalPersistenceProvider { pub fn new(data_dir: impl Into>) -> Self { Self { data_dir: data_dir.into(), + durability: DurabilityConfig::default(), } } + + pub fn with_durability_config(mut self, durability: DurabilityConfig) -> Self { + self.durability = durability; + self + } } #[async_trait] @@ -149,7 +213,12 @@ impl PersistenceProvider for LocalPersistenceProvider { asyncify(move || relational_db::open_snapshot_repo(snapshot_dir, database_identity, replica_id)) .await .map(|repo| SnapshotWorker::new(repo, snapshot::Compression::Enabled))?; - let (durability, disk_size) = relational_db::local_durability(replica_dir, Some(&snapshot_worker)).await?; + let (durability, disk_size) = relational_db::local_durability_with_options( + replica_dir, + Some(&snapshot_worker), + self.durability.into_options(), + ) + .await?; tokio::spawn(relational_db::snapshot_watching_commitlog_compressor( snapshot_worker.subscribe(), diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 9f041c92ccb..a2b3a28db4c 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -1670,6 +1670,20 @@ pub type LocalDurability = Arc>; pub async fn local_durability( replica_dir: ReplicaDir, snapshot_worker: Option<&SnapshotWorker>, +) -> Result<(LocalDurability, DiskSizeFn), DBError> { + local_durability_with_options(replica_dir, snapshot_worker, <_>::default()).await +} + +/// Initialize local durability with explicit parameters. +/// +/// Also returned is a [`DiskSizeFn`] as required by [`RelationalDB::open`]. +/// +/// Note that this operation can be expensive, as it needs to traverse a suffix +/// of the commitlog. +pub async fn local_durability_with_options( + replica_dir: ReplicaDir, + snapshot_worker: Option<&SnapshotWorker>, + opts: durability::local::Options, ) -> Result<(LocalDurability, DiskSizeFn), DBError> { let rt = tokio::runtime::Handle::current(); let on_new_segment = snapshot_worker.map(|snapshot_worker| { @@ -1684,7 +1698,7 @@ pub async fn local_durability( durability::Local::open( replica_dir.clone(), rt, - <_>::default(), + opts, // Give the durability a handle to request a new snapshot run, // which it will send down whenever we rotate commitlog segments. on_new_segment, diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index 14ce95c5ccf..c1847fa6d1c 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -204,8 +204,8 @@ metrics_group!( pub tokio_mean_polls_per_park: GaugeVec, #[name = spacetime_websocket_sent_msg_size_bytes] - #[help = "The size of messages sent to connected sessions"] - #[labels(db: Identity, workload: WorkloadType)] + #[help = "The size of websocket payloads sent to connected sessions"] + #[labels(db: Identity)] // Prometheus histograms have default buckets, // which broadly speaking, // are tailored to measure the response time of a network service. @@ -219,8 +219,8 @@ metrics_group!( pub websocket_sent_msg_size: HistogramVec, #[name = spacetime_websocket_sent_num_rows] - #[help = "The number of rows sent to connected sessions"] - #[labels(db: Identity, workload: WorkloadType)] + #[help = "The number of rows sent in websocket payloads"] + #[labels(db: Identity)] // Prometheus histograms have default buckets, // which broadly speaking, // are tailored to measure the response time of a network service. diff --git a/crates/lib/src/connection_id.rs b/crates/lib/src/connection_id.rs index 2055e389127..13fda72362f 100644 --- a/crates/lib/src/connection_id.rs +++ b/crates/lib/src/connection_id.rs @@ -300,10 +300,9 @@ mod tests { use crate::WithTypespace; proptest! { - /// Tests the round-trip used when using the `spacetime subscribe` - /// CLI command. - /// Somewhat confusingly, this is distinct from the ser-de path - /// in `test_serde_roundtrip`. + /// Tests deserialization through the serde seed adapter without + /// relying on `serde_json::Value` to preserve arbitrary-precision + /// numbers. #[test] fn test_wrapper_roundtrip(val: u128) { let conn_id = ConnectionId::from_u128(val); @@ -313,12 +312,12 @@ mod tests { let empty = Typespace::default(); let conn_id_ty = ConnectionId::get_type(); let conn_id_ty = WithTypespace::new(&empty, &conn_id_ty); - let row = serde_json::from_str::(&ser[..])?; + let mut de = serde_json::Deserializer::from_str(&ser); let de = ::serde::de::DeserializeSeed::deserialize( crate::de::serde::SeedWrapper( conn_id_ty ), - row)?; + &mut de)?; let de = ConnectionId::deserialize(ValueDeserializer::new(de)).unwrap(); prop_assert_eq!(conn_id, de); } diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index 6d169a902cc..f2fba813fa8 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -1101,6 +1101,7 @@ mod tests { AlgebraicType, AlgebraicValue, ProductType, ScheduleAt, }; use spacetimedb_primitives::ColId; + use v10::{ExplicitNames, RawModuleDefV10Builder}; use v9::{RawModuleDefV9Builder, TableAccess}; use validate::tests::expect_identifier; @@ -1113,6 +1114,15 @@ mod tests { .expect("new_def should be a valid database definition") } + fn create_module_def_v10(build_module: impl Fn(&mut RawModuleDefV10Builder)) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + build_module(&mut builder); + builder + .finish() + .try_into() + .expect("new_def should be a valid database definition") + } + fn initial_module_def() -> ModuleDef { let mut builder = RawModuleDefV9Builder::new(); let schedule_at = builder.add_type::(); @@ -1982,6 +1992,53 @@ mod tests { } } + #[test] + fn migrate_view_with_explicit_name() { + fn module_def() -> ModuleDef { + create_module_def_v10(|builder| { + let return_type_ref = builder.add_algebraic_type( + [], + "Person", + AlgebraicType::product([("PersonId", AlgebraicType::U64)]), + true, + ); + builder.add_view( + "PersonAtLevel2", + 0, + true, + true, + ProductType::from([("Level", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(return_type_ref)), + ); + + let mut explicit = ExplicitNames::default(); + explicit.insert_function("PersonAtLevel2", "Level2Person"); + builder.add_explicit_names(explicit); + }) + } + + let old_def = module_def(); + let new_def = module_def(); + let level_2_person = expect_identifier("Level2Person"); + + let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::UpdateView(&level_2_person)), + "steps: {steps:?}" + ); + assert!( + !steps.contains(&AutoMigrateStep::AddView(&level_2_person)), + "steps: {steps:?}" + ); + assert!( + !steps.contains(&AutoMigrateStep::RemoveView(&level_2_person)), + "steps: {steps:?}" + ); + } + #[test] fn migrate_view_disconnect_clients() { struct TestCase { diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index e180c14dda9..1214ea0cab7 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -465,8 +465,13 @@ impl<'a> ModuleValidatorV10<'a> { }) })?; - let mut table_validator = - TableValidator::new(raw_table_name.clone(), product_type_ref, product_type, &mut self.core)?; + let mut table_validator = TableValidator::new( + raw_table_name.clone(), + product_type_ref, + product_type, + &mut self.core, + CoreValidator::resolve_table_ident, + )?; let table_ident = table_validator.table_ident.clone(); @@ -847,10 +852,12 @@ impl<'a> ModuleValidatorV10<'a> { }) })?; + let name = self.core.resolve_function_ident(accessor_name.clone())?; + let params_for_generate = self.core .params_for_generate(¶ms, |position, arg_name| TypeLocation::ViewArg { - view_name: accessor_name.clone(), + view_name: name.as_raw().clone(), position, arg_name, })?; @@ -864,13 +871,11 @@ impl<'a> ModuleValidatorV10<'a> { let return_type_for_generate = self.core.validate_for_type_use( || TypeLocation::ViewReturn { - view_name: accessor_name.clone(), + view_name: name.as_raw().clone(), }, &return_type_for_generate_input, ); - let name = self.core.resolve_function_ident(accessor_name.clone())?; - let mut view_validator = ViewValidator::new( accessor_name.clone(), product_type_ref, @@ -2449,4 +2454,41 @@ mod tests { assert_eq!(schedule.at_column, 1.into()); assert_eq!(schedule.function_kind, FunctionKind::Reducer); } + + #[test] + fn test_child_defs_use_explicit_view_name() { + use spacetimedb_lib::db::raw_def::v10::ExplicitNames; + + let id = |s: &str| Identifier::for_test(s); + + let mut builder = RawModuleDefV10Builder::new(); + let return_type_ref = builder.add_algebraic_type( + [], + "Person", + AlgebraicType::product([("PersonId", AlgebraicType::U64)]), + true, + ); + builder.add_view( + "PersonAtLevel2", + 0, + true, + true, + ProductType::from([("Level", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(return_type_ref)), + ); + + let mut explicit = ExplicitNames::default(); + explicit.insert_function("PersonAtLevel2", "Level2Person"); + builder.add_explicit_names(explicit); + + let def: ModuleDef = builder.finish().try_into().unwrap(); + let view = def + .view("Level2Person") + .expect("view should use explicit canonical name"); + + assert_eq!(view.name, id("Level2Person")); + assert_eq!(view.accessor_name, id("PersonAtLevel2")); + assert_eq!(view.return_columns[0].view_name, id("Level2Person")); + assert_eq!(view.param_columns[0].view_name, id("Level2Person")); + } } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 3786ad129bc..13961bd08be 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -204,8 +204,13 @@ impl ModuleValidatorV9<'_> { }) })?; - let mut table_in_progress = - TableValidator::new(raw_table_name.clone(), product_type_ref, product_type, &mut self.core)?; + let mut table_in_progress = TableValidator::new( + raw_table_name.clone(), + product_type_ref, + product_type, + &mut self.core, + CoreValidator::resolve_table_ident, + )?; let table_ident = table_in_progress.table_ident.clone(); @@ -931,6 +936,7 @@ impl CoreValidator<'_> { /// 2. Insert view names into the global namespace. pub(crate) struct ViewValidator<'a, 'b> { inner: TableValidator<'a, 'b>, + view_name: Identifier, params: &'a ProductType, params_for_generate: &'a [(Identifier, AlgebraicTypeUse)], } @@ -944,8 +950,12 @@ impl<'a, 'b> ViewValidator<'a, 'b> { params_for_generate: &'a [(Identifier, AlgebraicTypeUse)], module_validator: &'a mut CoreValidator<'b>, ) -> Result { + let view_name = module_validator.resolve_function_ident(raw_name.clone())?; Ok(Self { - inner: TableValidator::new(raw_name, product_type_ref, product_type, module_validator)?, + inner: TableValidator::new(raw_name, product_type_ref, product_type, module_validator, |_, _| { + Ok(view_name.clone()) + })?, + view_name, params, params_for_generate, }) @@ -970,25 +980,14 @@ impl<'a, 'b> ViewValidator<'a, 'b> { .unwrap_or_else(|| RawIdentifier::new(format!("param_{}", col_id))), ); - // This error will be created multiple times if the view name is invalid, - // but we sort and deduplicate the error stream afterwards, - // so it isn't a huge deal. - // - // This is necessary because we require `ErrorStream` to be nonempty. - // We need to put something in there if the view name is invalid. - let view_name = self - .inner - .module_validator - .resolve_identifier_with_case(self.inner.raw_name.clone()); - - let (name, view_name) = (name, view_name).combine_errors()?; + let name = name?; Ok(ViewParamDef { name, ty: column.algebraic_type.clone(), ty_for_generate: ty_for_generate.clone(), col_id, - view_name, + view_name: self.view_name.clone(), }) } @@ -1007,7 +1006,11 @@ impl<'a, 'b> ViewValidator<'a, 'b> { } } -/// A partially validated table. +/// A partially validated table-shaped definition. +/// +/// This is also used by [`ViewValidator`]. Tables and views do not resolve +/// their source names in the same namespace, so callers provide the +/// appropriate name resolver. pub(crate) struct TableValidator<'a, 'b> { pub(crate) module_validator: &'a mut CoreValidator<'b>, raw_name: RawIdentifier, @@ -1023,8 +1026,9 @@ impl<'a, 'b> TableValidator<'a, 'b> { product_type_ref: AlgebraicTypeRef, product_type: &'a ProductType, module_validator: &'a mut CoreValidator<'b>, + resolve_name: impl FnOnce(&CoreValidator<'b>, RawIdentifier) -> Result, ) -> Result { - let table_ident = module_validator.resolve_table_ident(raw_name.clone())?; + let table_ident = resolve_name(module_validator, raw_name.clone())?; Ok(Self { raw_name, product_type_ref, diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 1ef17047d31..ad3f0f0ca8f 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -280,6 +280,21 @@ pub fn pnpm_path() -> Option { PNPM_PATH.get_or_init(|| which("pnpm").ok()).clone() } +fn pnpm_minimum_release_age() -> Result { + let workspace = fs::read_to_string(workspace_root().join("pnpm-workspace.yaml"))?; + workspace + .lines() + .find_map(|line| { + line.trim() + .strip_prefix("minimumReleaseAge:")? + .trim() + .parse::() + .ok() + }) + .map(|age| age.to_string()) + .context("pnpm-workspace.yaml is missing minimumReleaseAge") +} + /// Runs a command and returns stdout as a string. pub fn run_cmd(args: &[&str], cwd: &Path) -> Result { run_cmd_inner(args, cwd, None) @@ -332,10 +347,28 @@ fn run_cmd_inner(args: &[&str], cwd: &Path, stdin_input: Option<&str>) -> Result /// Runs a `pnpm` command and returns stdout as a string. pub fn pnpm(args: &[&str], cwd: &Path) -> Result { let pnpm_path = pnpm_path().context("Could not locate pnpm")?; - let pnpm_path = pnpm_path.to_str().context("pnpm path is not valid UTF-8")?; - let mut full_args = vec![pnpm_path]; - full_args.extend(args); - run_cmd(&full_args, cwd) + let minimum_release_age = pnpm_minimum_release_age()?; + + // Smoketests often install inside temp projects created by `spacetime init`. + // Those projects intentionally do not carry the repo's .npmrc, so pass the + // repo policy through pnpm's environment variable instead. + let output = Command::new(&pnpm_path) + .args(args) + .current_dir(cwd) + .env("npm_config_minimum_release_age", minimum_release_age) + .output() + .with_context(|| format!("Failed to spawn pnpm {}", args.join(" ")))?; + + if !output.status.success() { + bail!( + "pnpm {} (in {:?}) failed:\nstdout: {}\nstderr: {}", + args.join(" "), + cwd, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) } /// Builds the local TypeScript bindings package. @@ -353,6 +386,63 @@ pub fn have_emscripten() -> bool { *HAVE_EMSCRIPTEN.get_or_init(|| which("emcc").is_ok() || which("emcc.bat").is_ok()) } +const CPP_SMOKETEST_CMAKELISTS: &str = r#"cmake_minimum_required(VERSION 3.16) +project(smoketest_cpp_module) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(SPACETIMEDB_CPP_LIBRARY_PATH "@SPACETIMEDB_CPP_LIBRARY_PATH@") + +add_executable(lib src/lib.cpp) + +target_include_directories(lib PRIVATE + ${SPACETIMEDB_CPP_LIBRARY_PATH}/include +) + +if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + target_compile_options(lib PRIVATE -fno-exceptions -O2 -g0) + target_compile_definitions(lib PRIVATE SPACETIMEDB_UNSTABLE_FEATURES) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSPACETIMEDB_UNSTABLE_FEATURES") +endif() + +add_subdirectory(${SPACETIMEDB_CPP_LIBRARY_PATH} ${CMAKE_CURRENT_BINARY_DIR}/spacetimedb_cpp_library) +target_link_libraries(lib PRIVATE spacetimedb_cpp_library) + +if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + set(EXPORTED_FUNCS + "['_malloc','_free','___describe_module__','___call_reducer__','___call_procedure__','___call_http_handler__']" + ) + + target_link_options(lib PRIVATE + "SHELL:-sSTANDALONE_WASM=1" + "SHELL:-sWASM=1" + "SHELL:--no-entry" + "SHELL:-sEXPORTED_FUNCTIONS=${EXPORTED_FUNCS}" + "SHELL:-sERROR_ON_UNDEFINED_SYMBOLS=1" + "SHELL:-sFILESYSTEM=0" + "SHELL:-sDISABLE_EXCEPTION_CATCHING=1" + "SHELL:-sALLOW_MEMORY_GROWTH=0" + "SHELL:-sINITIAL_MEMORY=16MB" + "SHELL:-sSUPPORT_LONGJMP=0" + "SHELL:-sSUPPORT_ERRNO=0" + "SHELL:-std=c++20" + "SHELL:-O2" + "SHELL:-g0" + ) + + set_target_properties(lib PROPERTIES OUTPUT_NAME "lib" SUFFIX ".wasm") +endif() +"#; + +fn parse_identity_from_publish_output(publish_output: &str) -> Result { + let re = Regex::new(r"identity: ([0-9a-fA-F]+)").unwrap(); + re.captures(publish_output) + .and_then(|caps| caps.get(1)) + .map(|m| m.as_str().to_string()) + .context("Failed to parse database identity from publish output") +} + /// A smoketest instance that manages a SpacetimeDB server and module project. pub struct Smoketest { /// The SpacetimeDB server guard (stops server on drop). @@ -929,12 +1019,49 @@ impl Smoketest { ])?; csharp::verify_csharp_module_restore(&module_path)?; - let re = Regex::new(r"identity: ([0-9a-fA-F]+)").unwrap(); - let identity = re - .captures(&publish_output) - .and_then(|caps| caps.get(1)) - .map(|m| m.as_str().to_string()) - .context("Failed to parse database identity from publish output")?; + let identity = parse_identity_from_publish_output(&publish_output)?; + self.database_identity = Some(identity.clone()); + + Ok(identity) + } + + /// Writes and publishes a C++ module from source. + /// + /// The module is created at `/`. + /// On success this updates `self.database_identity`. + pub fn publish_cpp_module_source( + &mut self, + project_dir_name: &str, + module_name: &str, + module_source: &str, + ) -> Result { + let module_path = self.project_dir.path().join(project_dir_name); + let src_dir = module_path.join("src"); + fs::create_dir_all(&src_dir).context("Failed to create C++ source directory")?; + + let bindings_cpp_path = workspace_root() + .join("crates/bindings-cpp") + .display() + .to_string() + .replace('\\', "/"); + let cmakelists = CPP_SMOKETEST_CMAKELISTS.replace("@SPACETIMEDB_CPP_LIBRARY_PATH@", &bindings_cpp_path); + + fs::write(module_path.join("CMakeLists.txt"), cmakelists).context("Failed to write C++ CMakeLists.txt")?; + fs::write(src_dir.join("lib.cpp"), module_source).context("Failed to write C++ module code")?; + + let module_path_str = module_path.to_str().context("Invalid C++ module path")?; + let publish_output = self.spacetime(&[ + "publish", + "--server", + &self.server_url, + "--module-path", + module_path_str, + "--yes", + "--clear-database", + module_name, + ])?; + + let identity = parse_identity_from_publish_output(&publish_output)?; self.database_identity = Some(identity.clone()); Ok(identity) diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/smoketests/http_routes.rs index aab3e0bcbe3..9c5622088bc 100644 --- a/crates/smoketests/tests/smoketests/http_routes.rs +++ b/crates/smoketests/tests/smoketests/http_routes.rs @@ -1,5 +1,5 @@ use regex::Regex; -use spacetimedb_smoketests::{require_dotnet, workspace_root, Smoketest}; +use spacetimedb_smoketests::{require_dotnet, require_emscripten, workspace_root, Smoketest}; use std::{fs, path::Path}; const MODULE_CODE: &str = r#" @@ -230,6 +230,335 @@ fn router() -> Router { } "#; +const CPP_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +struct Entry { + uint64_t id; + std::string value; +}; +SPACETIMEDB_STRUCT(Entry, id, value) +SPACETIMEDB_TABLE(Entry, entry, Public) + +namespace { + +std::string header_value_utf8(const HttpRequest& request, const std::string& header_name) { + for (const auto& header : request.headers) { + if (header.name == header_name) { + return std::string(header.value.begin(), header.value.end()); + } + } + return ""; +} + +HttpResponse text_response(uint16_t status_code, std::string body) { + return HttpResponse{ + status_code, + HttpVersion::Http11, + { HttpHeader{"content-type", "text/plain; charset=utf-8"} }, + HttpBody::from_string(body), + }; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(get_simple, HandlerContext ctx, HttpRequest request) { + return text_response(200, "ok"); +} + +SPACETIMEDB_HTTP_HANDLER(post_insert, HandlerContext ctx, HttpRequest request) { + ctx.with_tx([](TxContext& tx) { + uint64_t id = tx.db[entry].count(); + tx.db[entry].insert(Entry{ id, "posted" }); + }); + return text_response(200, "inserted"); +} + +SPACETIMEDB_HTTP_HANDLER(get_count, HandlerContext ctx, HttpRequest request) { + uint64_t count = ctx.with_tx([](TxContext& tx) -> uint64_t { + return tx.db[entry].count(); + }); + return text_response(200, std::to_string(count)); +} + +SPACETIMEDB_HTTP_HANDLER(any_handler, HandlerContext ctx, HttpRequest request) { + return text_response(200, "any"); +} + +SPACETIMEDB_HTTP_HANDLER(header_echo, HandlerContext ctx, HttpRequest request) { + return text_response(200, header_value_utf8(request, "x-echo")); +} + +SPACETIMEDB_HTTP_HANDLER(set_response_header, HandlerContext ctx, HttpRequest request) { + return HttpResponse{ + 200, + HttpVersion::Http11, + { HttpHeader{"x-response", "set"} }, + HttpBody::from_string("header-set"), + }; +} + +SPACETIMEDB_HTTP_HANDLER(body_handler, HandlerContext ctx, HttpRequest request) { + return text_response(200, "non-empty"); +} + +SPACETIMEDB_HTTP_HANDLER(teapot, HandlerContext ctx, HttpRequest request) { + return text_response(418, "teapot"); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .get("/get", get_simple) + .post("/post", post_insert) + .get("/count", get_count) + .any("/any", any_handler) + .get("/header", header_echo) + .get("/set-header", set_response_header) + .get("/body", body_handler) + .get("/teapot", teapot); +} +"#; + +const CPP_EXAMPLE_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +struct Data { + uint64_t id; + std::vector body; +}; +SPACETIMEDB_STRUCT(Data, id, body) +SPACETIMEDB_TABLE(Data, data, Public) +FIELD_PrimaryKeyAutoInc(data, id) + +namespace { + +HttpResponse bytes_response(uint16_t status_code, std::vector body) { + return HttpResponse{ + status_code, + HttpVersion::Http11, + {}, + HttpBody{std::move(body)}, + }; +} + +HttpResponse text_response(uint16_t status_code, std::string body) { + return HttpResponse{ + status_code, + HttpVersion::Http11, + {}, + HttpBody::from_string(body), + }; +} + +std::string query_value(const std::string& uri, const std::string& key) { + std::string needle = "?" + key + "="; + size_t pos = uri.find(needle); + if (pos == std::string::npos) { + needle = "&" + key + "="; + pos = uri.find(needle); + } + if (pos == std::string::npos) { + return ""; + } + pos += needle.size(); + size_t end = uri.find('&', pos); + return uri.substr(pos, end == std::string::npos ? std::string::npos : end - pos); +} + +bool try_parse_u64(const std::string& text, uint64_t& value) { + if (text.empty()) { + return false; + } + uint64_t result = 0; + for (char c : text) { + if (c < '0' || c > '9') { + return false; + } + result = (result * 10) + static_cast(c - '0'); + } + value = result; + return true; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(insert, HandlerContext ctx, HttpRequest request) { + std::vector body = request.body.to_bytes(); + uint64_t id = ctx.with_tx([&](TxContext& tx) -> uint64_t { + return tx.db[data].insert(Data{0, body}).id; + }); + return text_response(200, std::to_string(id)); +} + +SPACETIMEDB_HTTP_HANDLER(retrieve, HandlerContext ctx, HttpRequest request) { + uint64_t id = 0; + if (!try_parse_u64(query_value(request.uri, "id"), id)) { + return text_response(500, "invalid id"); + } + + auto body = ctx.with_tx([&](TxContext& tx) -> std::optional> { + auto row = tx.db[data_id].find(id); + if (row.has_value()) { + return row->body; + } + return std::nullopt; + }); + + if (body.has_value()) { + return bytes_response(200, std::move(body.value())); + } + return bytes_response(404, {}); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router().post("/insert", insert).get("/retrieve", retrieve); +} +"#; + +const CPP_STRICT_ROOT_ROUTING_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +namespace { + +HttpResponse text_response(const std::string& body) { + return HttpResponse{200, HttpVersion::Http11, {}, HttpBody::from_string(body)}; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(empty_root, HandlerContext ctx, HttpRequest request) { + return text_response("empty"); +} + +SPACETIMEDB_HTTP_HANDLER(slash_root, HandlerContext ctx, HttpRequest request) { + return text_response("slash"); +} + +SPACETIMEDB_HTTP_HANDLER(foo, HandlerContext ctx, HttpRequest request) { + return text_response("foo"); +} + +SPACETIMEDB_HTTP_HANDLER(foo_slash, HandlerContext ctx, HttpRequest request) { + return text_response("foo-slash"); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .get("", empty_root) + .get("/", slash_root) + .get("/foo", foo) + .get("/foo/", foo_slash); +} +"#; + +const CPP_STRICT_NON_ROOT_ROUTING_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +namespace { + +HttpResponse text_response(const std::string& body) { + return HttpResponse{200, HttpVersion::Http11, {}, HttpBody::from_string(body)}; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(foo, HandlerContext ctx, HttpRequest request) { + return text_response("foo"); +} + +SPACETIMEDB_HTTP_HANDLER(foo_slash, HandlerContext ctx, HttpRequest request) { + return text_response("foo-slash"); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .get("/foo", foo) + .get("/foo/", foo_slash); +} +"#; + +const CPP_FULL_URI_MODULE_CODE: &str = r#"#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(echo_uri, HandlerContext ctx, HttpRequest request) { + return HttpResponse{ + 200, + HttpVersion::Http11, + {}, + HttpBody::from_string(request.uri), + }; +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router().get("/echo-uri", echo_uri); +} +"#; + +const CPP_HANDLE_REQUEST_BODY_MODULE_CODE: &str = r#"#include "spacetimedb.h" +#include + +using namespace SpacetimeDB; + +namespace { + +HttpResponse bytes_response(uint16_t status_code, std::vector body) { + return HttpResponse{status_code, HttpVersion::Http11, {}, HttpBody{std::move(body)}}; +} + +HttpResponse text_response(uint16_t status_code, const std::string& body) { + return HttpResponse{status_code, HttpVersion::Http11, {}, HttpBody::from_string(body)}; +} + +} // namespace + +SPACETIMEDB_HTTP_HANDLER(reverse_bytes, HandlerContext ctx, HttpRequest request) { + std::vector reversed = request.body.to_bytes(); + std::reverse(reversed.begin(), reversed.end()); + return bytes_response(200, std::move(reversed)); +} + +SPACETIMEDB_HTTP_HANDLER(reverse_words, HandlerContext ctx, HttpRequest request) { + const std::vector bytes = request.body.to_bytes(); + std::string body(bytes.begin(), bytes.end()); + if (body.find(static_cast(0x80)) != std::string::npos) { + return text_response(400, "request body must be valid UTF-8"); + } + + std::vector words; + size_t start = 0; + while (true) { + size_t pos = body.find(' ', start); + words.push_back(body.substr(start, pos == std::string::npos ? std::string::npos : pos - start)); + if (pos == std::string::npos) { + break; + } + start = pos + 1; + } + std::reverse(words.begin(), words.end()); + + std::string reversed; + for (size_t i = 0; i < words.size(); ++i) { + if (i != 0) { + reversed += " "; + } + reversed += words[i]; + } + + return text_response(200, reversed); +} + +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .post("/reverse-bytes", reverse_bytes) + .post("/reverse-words", reverse_words); +} +"#; + const CS_MODULE_CODE: &str = r#" using System; using System.Collections.Generic; @@ -571,6 +900,13 @@ fn rust_http_test(module_code: &str) -> (Smoketest, String) { (test, identity) } +fn cpp_http_test(name: &str, module_code: &str) -> (Smoketest, String) { + require_emscripten!(); + let mut test = Smoketest::builder().autopublish(false).build(); + let identity = test.publish_cpp_module_source(name, name, module_code).unwrap(); + (test, identity) +} + fn csharp_http_test(name: &str, module_code: &str) -> (Smoketest, String) { let mut test = Smoketest::builder().autopublish(false).build(); let identity = test.publish_csharp_module_source(name, name, module_code).unwrap(); @@ -835,6 +1171,12 @@ fn handle_request_body() { assert_handle_request_body(&test.server_url, &identity); } +#[test] +fn cpp_http_routes_end_to_end() { + let (test, identity) = cpp_http_test("http-routes-cpp-basic", CPP_MODULE_CODE); + assert_http_routes_end_to_end(&test.server_url, &identity); +} + #[test] fn csharp_http_routes_end_to_end() { require_dotnet!(); @@ -842,6 +1184,12 @@ fn csharp_http_routes_end_to_end() { assert_http_routes_end_to_end(&test.server_url, &identity); } +#[test] +fn cpp_http_routes_pr_example_round_trip() { + let (test, identity) = cpp_http_test("http-routes-cpp-example", CPP_EXAMPLE_MODULE_CODE); + assert_http_routes_pr_example_round_trip(&test.server_url, &identity); +} + #[test] fn csharp_http_routes_pr_example_round_trip() { require_dotnet!(); @@ -849,6 +1197,15 @@ fn csharp_http_routes_pr_example_round_trip() { assert_http_routes_pr_example_round_trip(&test.server_url, &identity); } +#[test] +fn cpp_http_routes_are_strict_for_non_root_paths() { + let (test, identity) = cpp_http_test( + "http-routes-cpp-strict-non-root", + CPP_STRICT_NON_ROOT_ROUTING_MODULE_CODE, + ); + assert_http_routes_are_strict_for_non_root_paths(&test.server_url, &identity); +} + #[test] fn csharp_http_routes_are_strict_for_non_root_paths() { require_dotnet!(); @@ -859,6 +1216,12 @@ fn csharp_http_routes_are_strict_for_non_root_paths() { assert_http_routes_are_strict_for_non_root_paths(&test.server_url, &identity); } +#[test] +fn cpp_http_routes_are_strict_for_root_paths() { + let (test, identity) = cpp_http_test("http-routes-cpp-strict-root", CPP_STRICT_ROOT_ROUTING_MODULE_CODE); + assert_http_routes_are_strict_for_root_paths(&test.server_url, &identity); +} + #[test] fn csharp_http_routes_are_strict_for_root_paths() { require_dotnet!(); @@ -866,6 +1229,12 @@ fn csharp_http_routes_are_strict_for_root_paths() { assert_http_routes_are_strict_for_root_paths(&test.server_url, &identity); } +#[test] +fn cpp_http_handler_observes_full_external_uri() { + let (test, identity) = cpp_http_test("http-routes-cpp-full-uri", CPP_FULL_URI_MODULE_CODE); + assert_http_handler_observes_full_external_uri(&test.server_url, &identity); +} + #[test] fn csharp_http_handler_observes_full_external_uri() { require_dotnet!(); @@ -873,6 +1242,12 @@ fn csharp_http_handler_observes_full_external_uri() { assert_http_handler_observes_full_external_uri(&test.server_url, &identity); } +#[test] +fn cpp_handle_request_body() { + let (test, identity) = cpp_http_test("http-routes-cpp-request-body", CPP_HANDLE_REQUEST_BODY_MODULE_CODE); + assert_handle_request_body(&test.server_url, &identity); +} + #[test] fn csharp_handle_request_body() { require_dotnet!(); @@ -899,6 +1274,29 @@ fn http_handlers_tutorial_say_hello_route_works() { assert_eq!(resp.text().expect("say-hello body"), "Hello!"); } +/// Validates the C++ example from `docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md`. +#[test] +fn cpp_http_handlers_tutorial_say_hello_route_works() { + require_emscripten!(); + + let module_code = extract_code_blocks( + &workspace_root().join("docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md"), + r"```(?:cpp|c\+\+)\n([\s\S]*?)\n```", + "cpp", + ); + let mut test = Smoketest::builder().autopublish(false).build(); + let identity = test + .publish_cpp_module_source("http-handlers-docs-cpp", "http-handlers-docs-cpp", &module_code) + .unwrap(); + + let url = format!("{}/v1/database/{identity}/route/say-hello", test.server_url); + let client = reqwest::blocking::Client::new(); + + let resp = client.get(&url).send().expect("say-hello failed"); + assert!(resp.status().is_success()); + assert_eq!(resp.text().expect("say-hello body"), "Hello!"); +} + /// Validates the C# example from `docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md`. #[test] fn csharp_http_handlers_tutorial_say_hello_route_works() { diff --git a/crates/smoketests/tests/smoketests/templates.rs b/crates/smoketests/tests/smoketests/templates.rs index 8de55871f58..e818000201d 100644 --- a/crates/smoketests/tests/smoketests/templates.rs +++ b/crates/smoketests/tests/smoketests/templates.rs @@ -12,7 +12,7 @@ use anyhow::{bail, Context, Result}; use regex::Regex; use serde_json::Value; -use spacetimedb_smoketests::{pnpm_path, random_string, workspace_root, Smoketest}; +use spacetimedb_smoketests::{pnpm, random_string, workspace_root, Smoketest}; use std::env; use std::fs; use std::path::{Path, PathBuf}; @@ -174,21 +174,7 @@ fn update_package_json_dependency(package_json_path: &Path, package_name: &str, /// Runs pnpm with the given arguments in the given working directory. fn run_pnpm(args: &[&str], cwd: &Path) -> Result<()> { - let pnpm = pnpm_path().context("pnpm not found")?; - let output = Command::new(&pnpm) - .args(args) - .current_dir(cwd) - .output() - .with_context(|| format!("Failed to spawn pnpm {}", args.join(" ")))?; - if !output.status.success() { - bail!( - "pnpm {} (in {:?}) failed:\nstdout: {}\nstderr: {}", - args.join(" "), - cwd, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } + pnpm(args, cwd)?; Ok(()) } diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index 29a4bf3556e..9eeef5d3535 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -44,4 +44,24 @@ directives = [ # Apply a V8 heap limit in MiB. Set to 0 to use V8's default limit. # heap-limit-mb = 0 +[commitlog] +# The maximum supported commitlog format version, also used for writing. +# log-format-version = 1 + +# Maximum size in bytes for each commitlog segment. +# max-segment-size = 1073741824 + +# Number of bytes written to the commitlog after which an entry is added to the offset index. +# offset-index-interval-bytes = 4096 + +# Require that the commitlog segment is synced before adding an offset index entry. +# offset-index-require-segment-fsync = true + +# Preallocate disk space for commitlog segments up to max-segment-size. +# Has no effect unless commitlog fallocate support is enabled. +# preallocate-segments = false + +# Size in bytes of the memory buffer holding commit data before flushing to storage. +# write-buffer-size = 131072 + # vim: set nowritebackup: << otherwise triggers cargo-watch diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index de4b80ce78c..189bafa11bd 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -12,7 +12,7 @@ use http::StatusCode; use spacetimedb::client::ClientActorIndex; use spacetimedb::config::{CertificateAuthority, MetadataFile, V8Config, WasmConfig}; use spacetimedb::db; -use spacetimedb::db::persistence::LocalPersistenceProvider; +use spacetimedb::db::persistence::{DurabilityConfig, LocalPersistenceProvider}; use spacetimedb::energy::{EnergyBalance, EnergyQuanta, NullEnergyMonitor}; use spacetimedb::host::{DiskStorage, HostController, HostRuntimeConfig, MigratePlanResult, UpdateDatabaseResult}; use spacetimedb::identity::{AuthCtx, Identity}; @@ -41,6 +41,7 @@ pub use spacetimedb_client_api::routes::subscribe::{BIN_PROTOCOL, TEXT_PROTOCOL} #[derive(Clone, Copy)] pub struct StandaloneOptions { pub db_config: db::Config, + pub durability: DurabilityConfig, pub websocket: WebSocketOptions, pub wasm: WasmConfig, pub v8: V8Config, @@ -76,7 +77,8 @@ impl StandaloneEnv { let energy_monitor = Arc::new(NullEnergyMonitor); let program_store = Arc::new(DiskStorage::new(data_dir.program_bytes().0).await?); - let persistence_provider = Arc::new(LocalPersistenceProvider::new(data_dir.clone())); + let persistence_provider = + Arc::new(LocalPersistenceProvider::new(data_dir.clone()).with_durability_config(config.durability)); let host_controller = HostController::new( data_dir, config.db_config, @@ -650,6 +652,7 @@ mod tests { storage: Storage::Memory, page_pool_max_size: None, }, + durability: DurabilityConfig::default(), websocket: WebSocketOptions::default(), wasm: WasmConfig::default(), v8: V8Config::default(), diff --git a/crates/standalone/src/subcommands/start.rs b/crates/standalone/src/subcommands/start.rs index 50f6db19257..2373d11d26d 100644 --- a/crates/standalone/src/subcommands/start.rs +++ b/crates/standalone/src/subcommands/start.rs @@ -11,6 +11,7 @@ use axum::extract::DefaultBodyLimit; use clap::ArgAction::SetTrue; use clap::{Arg, ArgMatches}; use spacetimedb::config::{parse_config, CertificateAuthority}; +use spacetimedb::db::persistence::{CommitlogConfig, DurabilityConfig}; use spacetimedb::db::{self, Storage}; use spacetimedb::startup::{self, TracingOptions}; use spacetimedb::util::jobs::JobCores; @@ -99,6 +100,8 @@ struct ConfigFile { #[serde(flatten)] common: spacetimedb::config::ConfigFile, #[serde(default)] + commitlog: CommitlogConfig, + #[serde(default)] websocket: WebSocketOptions, } @@ -181,6 +184,9 @@ pub async fn exec(args: &ArgMatches, db_cores: JobCores) -> anyhow::Result<()> { let ctx = StandaloneEnv::init( StandaloneOptions { db_config, + durability: DurabilityConfig { + commitlog: config.commitlog, + }, websocket: config.websocket, wasm: config.common.wasm, v8: config.common.v8, @@ -525,6 +531,14 @@ mod tests { heap-gc-trigger-fraction = 0.6 heap-retire-fraction = 0.8 heap-limit-mb = 128 + + [commitlog] + log-format-version = 1 + max-segment-size = 1048576 + offset-index-interval-bytes = 8192 + offset-index-require-segment-fsync = false + preallocate-segments = true + write-buffer-size = 131072 "#; let config: ConfigFile = toml::from_str(toml).unwrap(); @@ -543,6 +557,21 @@ mod tests { assert_eq!(config.common.v8.heap_policy.heap_gc_trigger_fraction, 0.6); assert_eq!(config.common.v8.heap_policy.heap_retire_fraction, 0.8); assert_eq!(config.common.v8.heap_policy.heap_limit_bytes, 128 * 1024 * 1024); + assert_eq!(config.commitlog.log_format_version, Some(1)); + assert_eq!( + config.commitlog.max_segment_size.map(|val| val.get()), + Some(1024 * 1024) + ); + assert_eq!( + config.commitlog.offset_index_interval_bytes.map(|val| val.get()), + Some(8192) + ); + assert_eq!(config.commitlog.offset_index_require_segment_fsync, Some(false)); + assert_eq!(config.commitlog.preallocate_segments, Some(true)); + assert_eq!( + config.commitlog.write_buffer_size.map(|val| val.get()), + Some(128 * 1024) + ); assert_eq!( config.websocket, @@ -553,4 +582,20 @@ mod tests { } ); } + + #[test] + fn commitlog_options_accept_aliases() { + let toml = r#" + [commitlog] + offset-interval-bytes = 16384 + offset-index-require-fsync = true +"#; + + let config: ConfigFile = toml::from_str(toml).unwrap(); + assert_eq!( + config.commitlog.offset_index_interval_bytes.map(|val| val.get()), + Some(16 * 1024) + ); + assert_eq!(config.commitlog.offset_index_require_segment_fsync, Some(true)); + } } diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 750d9ca7606..21fea57fe96 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -101,7 +101,8 @@ impl ModuleHandle { } pub async fn recv_message(&mut self) -> Option { - self.receiver.recv().await + let mut buf = Vec::with_capacity(1); + (self.receiver.recv_many(&mut buf, 1).await != 0).then(|| buf.remove(0)) } pub async fn recv_reducer_update(&mut self, request_id: RequestId) -> anyhow::Result<()> { @@ -244,6 +245,7 @@ impl CompiledModule { let env = spacetimedb_standalone::StandaloneEnv::init( spacetimedb_standalone::StandaloneOptions { db_config: config, + durability: Default::default(), websocket: WebSocketOptions::default(), wasm: Default::default(), v8: Default::default(), diff --git a/crates/update/spacetime-install.ps1 b/crates/update/spacetime-install.ps1 index c5f9ecf6c14..2ade395ca25 100644 --- a/crates/update/spacetime-install.ps1 +++ b/crates/update/spacetime-install.ps1 @@ -32,7 +32,7 @@ function Install { $AssetName = "spacetimedb-update-x86_64-pc-windows-msvc.exe" $DownloadUrl = "https://github.com/clockworklabs/SpacetimeDB/releases/latest/download/$AssetName" - $MirrorBase = "https://spacetimedb-client-binaries.nyc3.digitaloceanspaces.com" + $MirrorBase = "https://spacetimedb-client-binaries.s3.amazonaws.com" Write-Output "Downloading installer..." function UpdatePathIfNotExists { diff --git a/crates/update/spacetime-install.sh b/crates/update/spacetime-install.sh index 47d3bc60739..d171a0f5467 100644 --- a/crates/update/spacetime-install.sh +++ b/crates/update/spacetime-install.sh @@ -65,7 +65,7 @@ main() { # Define the latest SpacetimeDB download url local _asset_name="spacetimedb-update-$_host$_ext" local _url="$SPACETIME_DOWNLOAD_ROOT/$_asset_name" - local _mirror_base="https://spacetimedb-client-binaries.nyc3.digitaloceanspaces.com" + local _mirror_base="https://spacetimedb-client-binaries.s3.amazonaws.com" echo "Downloading installer..." local _ok=false if [ "$_downloader" = curl ]; then diff --git a/crates/update/src/cli/install.rs b/crates/update/src/cli/install.rs index 3a75ce84c53..c2cadb5bb3d 100644 --- a/crates/update/src/cli/install.rs +++ b/crates/update/src/cli/install.rs @@ -59,7 +59,7 @@ fn releases_url() -> String { .unwrap_or_else(|_| "https://api.github.com/repos/clockworklabs/SpacetimeDB/releases".to_owned()) } -const MIRROR_BASE_URL: &str = "https://spacetimedb-client-binaries.nyc3.digitaloceanspaces.com"; +const MIRROR_BASE_URL: &str = "https://spacetimedb-client-binaries.s3.amazonaws.com"; /// Fetch the latest version tag from the mirror. /// diff --git a/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md b/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md index aab6d67690f..c8bc481d338 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md +++ b/docs/docs/00200-core-concepts/00200-functions/00600-HTTP-handlers.md @@ -66,6 +66,35 @@ fn say_hello(_ctx: &mut HandlerContext, _req: Request) -> Response { } ``` + + + +Because HTTP handlers are unstable, C++ modules that define them must enable `SPACETIMEDB_UNSTABLE_FEATURES` when compiling. + +Define an HTTP handler with `SPACETIMEDB_HTTP_HANDLER`. + +The function must accept exactly two arguments: + +1. A `SpacetimeDB::HandlerContext`. +2. A `SpacetimeDB::HttpRequest`. + +The function must return a `SpacetimeDB::HttpResponse`. + +```cpp +#include "spacetimedb.h" + +using namespace SpacetimeDB; + +SPACETIMEDB_HTTP_HANDLER(say_hello, HandlerContext ctx, HttpRequest request) { + return HttpResponse{ + 200, + HttpVersion::Http11, + { HttpHeader{"content-type", "text/plain; charset=utf-8"} }, + HttpBody::from_string("Hello!"), + }; +} +``` + @@ -148,6 +177,24 @@ Nest routers with `router.nest(prefix, sub_router)`, which causes `sub_router` t Combine routers with `router.merge(other_router)`, which combines both routers. + + + +All routes exposed by your module are declared in a `SpacetimeDB::Router`. Register the `Router` for your database by returning it from a function defined with `SPACETIMEDB_HTTP_ROUTER`. + +```cpp +SPACETIMEDB_HTTP_ROUTER(router) { + return Router() + .get("/say-hello", say_hello); +} +``` + +Add routes within a router with the `get`, `head`, `options`, `put`, `delete_`, `post`, `patch` and `any` methods, which register an HTTP handler for that HTTP method at a given path. + +Nest routers with `router.nest(prefix, sub_router)`, which causes `sub_router` to handle routing for all paths that start with `prefix`. + +Combine routers with `router.merge(other_router)`, which combines both routers. + diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md index dce6afe8c5f..6863d642a98 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00200-standalone-config.md @@ -18,6 +18,10 @@ On Linux and macOS, this directory is by default `~/.local/share/spacetime/data` - [`logs`](#logs) +- [`commitlog`](#commitlog) + +- [`websocket`](#websocket) + ### `certificate-authority` ```toml @@ -47,6 +51,56 @@ Can be one of `"error"`, `"warn"`, `"info"`, `"debug"`, `"trace"`, or `"off"`, c A list of filtering directives controlling what messages get logged, which overwrite the global [`logs.level`](#logslevel). See [`tracing documentation`](https://docs.rs/tracing-subscriber/0.3/tracing_subscriber/filter/struct.EnvFilter.html#directives) for syntax. Note that this is primarily intended as a debugging tool, and log message fields and targets are not considered stable. +### `commitlog` + +```toml +[commitlog] +log-format-version = 1 +max-segment-size = 1073741824 # 1GiB +offset-index-interval-bytes = 4096 +offset-index-require-segment-fsync = true +preallocate-segments = false +write-buffer-size = 131072 # 128KiB +``` + +The `commitlog` table configures local durability. These settings are advanced and may affect recovery behavior, disk usage, memory usage, and write throughput. Omitted fields use the server's built-in defaults. + +#### `commitlog.log-format-version` + +The maximum supported commitlog format version, also used for writing. + +::::caution +This setting should not normally be changed from the commitlog crate's default. A reason to change it could be to make the server accept an older, incompatible commitlog. +:::: + +#### `commitlog.max-segment-size` + +The maximum size in bytes to which commitlog segments should be allowed to grow. + +#### `commitlog.offset-index-interval-bytes` + +Number of bytes written to the commitlog after which an entry is added to the offset index. + +#### `commitlog.offset-index-require-segment-fsync` + +If `true`, require that the segment must be synced to disk before an index entry is added. + +Setting this to `false` will update the index every `offset-index-interval-bytes`, even if the commitlog was not synced. This means that the index could contain non-existent entries in the event of a crash. + +Setting this to `true` will update the index when the commitlog is synced, and `offset-index-interval-bytes` have been written. This means that the index could contain fewer index entries than strictly every `offset-index-interval-bytes`. + +::::note +The commitlog operates correctly under both settings, but the choice can have performance implications. +:::: + +#### `commitlog.preallocate-segments` + +If `true`, preallocate disk space for commitlog segments up to `commitlog.max-segment-size`. This has no effect unless commitlog fallocate support is enabled. + +#### `commitlog.write-buffer-size` + +Size in bytes of the memory buffer holding commit data before flushing to storage. + ### `websocket` ```toml diff --git a/docs/docs/00300-resources/00200-reference/00300-internals/00400-commitlog.md b/docs/docs/00300-resources/00200-reference/00300-internals/00400-commitlog.md index 01fc3063807..59ceb1add10 100644 --- a/docs/docs/00300-resources/00200-reference/00300-internals/00400-commitlog.md +++ b/docs/docs/00300-resources/00200-reference/00300-internals/00400-commitlog.md @@ -238,7 +238,7 @@ The offset index is controlled by two parameters: | Parameter | Default | Description | |-----------|---------|-------------| | `offset_index_interval_bytes` | 4,096 | An index entry is written whenever this many bytes have been flushed to the active segment. | -| `offset_index_require_segment_fsync` | false | If true, the segment must be synced to disk before an index entry is written. | +| `offset_index_require_segment_fsync` | true | If true, the segment must be synced to disk before an index entry is written. | ## Wire Format diff --git a/templates/angular-ts/.npmrc b/templates/angular-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/angular-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/angular-ts/spacetimedb/.npmrc b/templates/angular-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/angular-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/astro-ts/.npmrc b/templates/astro-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/astro-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/astro-ts/spacetimedb/.npmrc b/templates/astro-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/astro-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/basic-ts/.npmrc b/templates/basic-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/basic-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/basic-ts/spacetimedb/.npmrc b/templates/basic-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/basic-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/browser-ts/.npmrc b/templates/browser-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/browser-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/browser-ts/spacetimedb/.npmrc b/templates/browser-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/browser-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/bun-ts/.npmrc b/templates/bun-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/bun-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/bun-ts/spacetimedb/.npmrc b/templates/bun-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/bun-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/chat-react-ts/.npmrc b/templates/chat-react-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/chat-react-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/chat-react-ts/spacetimedb/.npmrc b/templates/chat-react-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/chat-react-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/deno-ts/.npmrc b/templates/deno-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/deno-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/deno-ts/spacetimedb/.npmrc b/templates/deno-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/deno-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/keynote-2/.npmrc b/templates/keynote-2/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/keynote-2/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/keynote-2/convex-app/.npmrc b/templates/keynote-2/convex-app/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/keynote-2/convex-app/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/keynote-2/spacetimedb/.npmrc b/templates/keynote-2/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/keynote-2/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nextjs-ts/.npmrc b/templates/nextjs-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nextjs-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nextjs-ts/spacetimedb/.npmrc b/templates/nextjs-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nextjs-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nodejs-ts/.npmrc b/templates/nodejs-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nodejs-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nodejs-ts/spacetimedb/.npmrc b/templates/nodejs-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nodejs-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nuxt-ts/.npmrc b/templates/nuxt-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nuxt-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/nuxt-ts/spacetimedb/.npmrc b/templates/nuxt-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/nuxt-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/react-ts/.npmrc b/templates/react-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/react-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/react-ts/spacetimedb/.npmrc b/templates/react-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/react-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/remix-ts/.npmrc b/templates/remix-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/remix-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/remix-ts/spacetimedb/.npmrc b/templates/remix-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/remix-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/svelte-ts/.npmrc b/templates/svelte-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/svelte-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/svelte-ts/spacetimedb/.npmrc b/templates/svelte-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/svelte-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/tanstack-ts/.npmrc b/templates/tanstack-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/tanstack-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/tanstack-ts/spacetimedb/.npmrc b/templates/tanstack-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/tanstack-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/vue-ts/.npmrc b/templates/vue-ts/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/vue-ts/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/templates/vue-ts/spacetimedb/.npmrc b/templates/vue-ts/spacetimedb/.npmrc deleted file mode 100644 index 44bdf80d1df..00000000000 --- a/templates/vue-ts/spacetimedb/.npmrc +++ /dev/null @@ -1 +0,0 @@ -minimum-release-age=1440 diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 2f5c381f823..9239058f039 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -131,6 +131,10 @@ fn is_npm_package_json(package_json: &Value) -> bool { .any(|key| package_json.get(key).is_some()) } +fn is_template_path(path: &Path) -> bool { + path.starts_with("templates") +} + fn minimum_release_age(path: &Path) -> Result { let workspace = fs::read_to_string(path)?; workspace @@ -221,6 +225,12 @@ fn check_pnpm_release_age_policy() -> Result<()> { } for npmrc_path in git_tracked_files(":(glob)**/.npmrc")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not embed this repo's package-age policy; smoketests enforce it + // at the pnpm process boundary instead. + if is_template_path(&npmrc_path) { + continue; + } let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; if found_minimum_release_age != root_minimum_release_age { bail!( @@ -233,6 +243,12 @@ fn check_pnpm_release_age_policy() -> Result<()> { } for package_json_path in git_tracked_files(":(glob)**/package.json")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not require adjacent .npmrc files for this repo's package-age + // policy; smoketests enforce it at the pnpm process boundary instead. + if is_template_path(&package_json_path) { + continue; + } let package_json = read_package_json(&package_json_path)?; if !is_npm_package_json(&package_json) { continue; diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index c2bcdcd90bc..68775ff631c 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -8,6 +8,29 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::LazyLock; +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .expect("xtask-llm-benchmark is under public/tools/xtask-llm-benchmark") + .to_path_buf() +} + +fn pnpm_minimum_release_age() -> Result { + let workspace = fs::read_to_string(workspace_root().join("pnpm-workspace.yaml"))?; + workspace + .lines() + .find_map(|line| { + line.trim() + .strip_prefix("minimumReleaseAge:")? + .trim() + .parse::() + .ok() + }) + .map(|age| age.to_string()) + .ok_or_else(|| anyhow::anyhow!("pnpm-workspace.yaml is missing minimumReleaseAge")) +} + /// Strip ANSI escape codes (color codes) from a string fn strip_ansi_codes(s: &str) -> Cow<'_, str> { static ANSI_RE: LazyLock = LazyLock::new(|| { @@ -298,7 +321,10 @@ impl Publisher for TypeScriptPublisher { .arg("install") .arg("--ignore-workspace") .current_dir(source) - .env("CI", "true"); + .env("CI", "true") + // This install runs in a materialized project with workspace config + // ignored, so pass the repo's pnpm package-age policy explicitly. + .env("npm_config_minimum_release_age", pnpm_minimum_release_age()?); // When using NODEJS_DIR, prepend it to PATH so pnpm.cmd can find node. if let Some(ref dir) = pnpm_exe && let Some(parent) = dir.parent()