From 917afbbc31b366207bc4a60a6f12213ce7283a3c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 12 Mar 2025 13:09:54 +0100 Subject: [PATCH 1/2] Support compressing HTTP contents --- CHANGELOG.md | 4 ++ Cargo.lock | 82 +++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- trino-lb/src/http_server/mod.rs | 3 +- 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb8a38e..3084dea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,13 @@ All notable changes to this project will be documented in this file. - Support configuring compression for OTEL ([#70]). - Improve tracing details by adding a `tower_http::trace::TraceLayer` that creates spans for every HTTP request ([#71]). - Improve tracing for running queries on Trino, adding spans for the request to Trino and parsing ([#71]). +- Support compressing HTTP contents, previously content was always uncompressed. + This consumes more CPU, but also reduces the data amount send to Trino clients. + E.g. `trino-cli` by default asks for `gzip` compressed content ([#74]). [#70]: https://github.com/stackabletech/trino-lb/pull/70 [#71]: https://github.com/stackabletech/trino-lb/pull/71 +[#74]: https://github.com/stackabletech/trino-lb/pull/74 ## [0.4.1] - 2025-03-03 diff --git a/Cargo.lock b/Cargo.lock index a0bc3bc..80b7f0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -184,11 +199,14 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ + "brotli", "flate2", "futures-core", "memchr", "pin-project-lite", "tokio", + "zstd", + "zstd-safe", ] [[package]] @@ -476,6 +494,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -500,6 +539,8 @@ version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "755717a7de9ec452bf7f3f1a3099085deabd7f2962b861dae91ecd7a365903d2" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -1786,6 +1827,15 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -4147,13 +4197,17 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ + "async-compression", "base64 0.22.1", "bitflags 2.8.0", "bytes", + "futures-core", "http 1.2.0", "http-body 1.0.1", "mime", "pin-project-lite", + "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -5023,3 +5077,31 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 9961fa7..9f9d5b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,7 @@ sqlx = { version = "0.8.2", features = [ strum = { version = "0.27", features = ["derive"] } tokio = "1.39" tower = "0.5" -tower-http = { version = "0.6", features = ["tracing"] } +tower-http = { version = "0.6", features = ["compression-full", "tracing"] } tracing = "0.1" tracing-opentelemetry = "0.25" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/trino-lb/src/http_server/mod.rs b/trino-lb/src/http_server/mod.rs index 37ff20e..2037722 100644 --- a/trino-lb/src/http_server/mod.rs +++ b/trino-lb/src/http_server/mod.rs @@ -15,7 +15,7 @@ use axum_server::{tls_rustls::RustlsConfig, Handle}; use futures::FutureExt; use snafu::{OptionExt, ResultExt, Snafu}; use tokio::time::sleep; -use tower_http::trace::TraceLayer; +use tower_http::{compression::CompressionLayer, trace::TraceLayer}; use tracing::info; use trino_lb_persistence::PersistenceImplementation; @@ -124,6 +124,7 @@ pub async fn start_http_server( .route("/ui/index.html", get(ui::index::get_ui_index)) .route("/ui/query.html", get(ui::query::get_ui_query)) .layer(TraceLayer::new_for_http()) + .layer(CompressionLayer::new()) .with_state(app_state); if tls_config.enabled { From 1e79c1b10646ff11239f73c255b1ccf7bac28ea4 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 12 Mar 2025 13:17:19 +0100 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Malte Sander --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3084dea..8fb280a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ All notable changes to this project will be documented in this file. - Support configuring compression for OTEL ([#70]). - Improve tracing details by adding a `tower_http::trace::TraceLayer` that creates spans for every HTTP request ([#71]). - Improve tracing for running queries on Trino, adding spans for the request to Trino and parsing ([#71]). -- Support compressing HTTP contents, previously content was always uncompressed. - This consumes more CPU, but also reduces the data amount send to Trino clients. +- Support compressing HTTP contents, previously the content was always uncompressed. + This consumes more CPU, but also reduces the data amount sent to Trino clients. E.g. `trino-cli` by default asks for `gzip` compressed content ([#74]). [#70]: https://github.com/stackabletech/trino-lb/pull/70