From 57b87970f5e8202309b69f687cf7ce3d6ae20bdd Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 11 Mar 2025 10:19:34 +0100 Subject: [PATCH 1/4] chore: Bump stackable-operator to get updated s3 region structs --- Cargo.lock | 16 +++++----- Cargo.nix | 38 ++++++++++------------- Cargo.toml | 8 ++--- crate-hashes.json | 6 ++-- deploy/helm/druid-operator/crds/crds.yaml | 28 +++++++++++++++++ 5 files changed, 58 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b8934c4..7c5ec1e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2619,8 +2619,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.85.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +version = "0.87.2" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "chrono", "clap", @@ -2658,7 +2658,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "darling", "proc-macro2", @@ -2669,7 +2669,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "kube 0.98.0", "semver", @@ -2710,18 +2710,18 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.nix b/Cargo.nix index 450d23d8..ead1cdcc 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1601,7 +1601,7 @@ rec { "default" = [ "Debug" "Clone" "Copy" "PartialEq" "Eq" "PartialOrd" "Ord" "Hash" "Default" "Deref" "DerefMut" "Into" ]; "full" = [ "syn/full" ]; }; - resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Hash" "PartialEq" ]; + resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Eq" "Hash" "PartialEq" ]; }; "either" = rec { crateName = "either"; @@ -8331,13 +8331,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.85.0"; + version = "0.87.2"; edition = "2021"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; libName = "stackable_operator"; authors = [ @@ -8370,7 +8370,7 @@ rec { name = "educe"; packageId = "educe"; usesDefaultFeatures = false; - features = [ "Clone" "Debug" "Default" "PartialEq" ]; + features = [ "Clone" "Debug" "Default" "PartialEq" "Eq" ]; } { name = "either"; @@ -8496,8 +8496,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8531,8 +8531,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; libName = "stackable_shared"; authors = [ @@ -8678,9 +8678,9 @@ rec { }; "strum" = rec { crateName = "strum"; - version = "0.26.3"; - edition = "2018"; - sha256 = "01lgl6jvrf4j28v5kmx9bp480ygf1nhvac8b4p7rcj9hxw50zv4g"; + version = "0.27.1"; + edition = "2021"; + sha256 = "0cic9r2sc2h17nnpjm2yfp7rsd35gkbcbqvrhl553jaiih4fykgn"; authors = [ "Peter Glotfelty " ]; @@ -8691,12 +8691,6 @@ rec { optional = true; } ]; - devDependencies = [ - { - name = "strum_macros"; - packageId = "strum_macros"; - } - ]; features = { "default" = [ "std" ]; "derive" = [ "strum_macros" ]; @@ -8707,9 +8701,9 @@ rec { }; "strum_macros" = rec { crateName = "strum_macros"; - version = "0.26.4"; - edition = "2018"; - sha256 = "1gl1wmq24b8md527cpyd5bw9rkbqldd7k1h38kf5ajd2ln2ywssc"; + version = "0.27.1"; + edition = "2021"; + sha256 = "1s7x07nkrgjfvxrvcdjw6qanad4c55yjnd32bph9q3xgpid8qyn7"; procMacro = true; authors = [ "Peter Glotfelty " @@ -8734,7 +8728,7 @@ rec { { name = "syn"; packageId = "syn 2.0.98"; - features = [ "parsing" "extra-traits" ]; + features = [ "parsing" ]; } ]; diff --git a/Cargo.toml b/Cargo.toml index 3fe410c1..dadb23ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,10 +10,8 @@ edition = "2021" repository = "https://github.com/stackabletech/druid-operator" [workspace.dependencies] -stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = [ - "k8s", -], tag = "stackable-versioned-0.5.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } +stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.5.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.87.2" } product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } anyhow = "1.0" @@ -31,7 +29,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -strum = { version = "0.26", features = ["derive"] } +strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" diff --git a/crate-hashes.json b/crate-hashes.json index c7d32c3a..e02a942d 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,7 +1,7 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator-derive@0.3.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator@0.85.0": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-shared@0.0.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-operator-derive@0.3.1": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-operator@0.87.2": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-shared@0.0.1": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#k8s-version@0.1.2": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned-macros@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", diff --git a/deploy/helm/druid-operator/crds/crds.yaml b/deploy/helm/druid-operator/crds/crds.yaml index e08b349b..d3df0746 100644 --- a/deploy/helm/druid-operator/crds/crds.yaml +++ b/deploy/helm/druid-operator/crds/crds.yaml @@ -665,6 +665,20 @@ spec: minimum: 0.0 nullable: true type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object tls: description: Use a TLS connection. If not specified no TLS will be used. nullable: true @@ -795,6 +809,20 @@ spec: minimum: 0.0 nullable: true type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object tls: description: Use a TLS connection. If not specified no TLS will be used. nullable: true From 77b8f1a6ac87ffe5f8de9c454ba999c68eab3f9f Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 11 Mar 2025 10:46:26 +0100 Subject: [PATCH 2/4] chore: Add S3 bucket region to config --- rust/operator-binary/src/crd/mod.rs | 2 ++ rust/operator-binary/src/druid_controller.rs | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index ddb38f24..ea1128e2 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -98,6 +98,7 @@ pub const DS_DIRECTORY: &str = "druid.storage.storageDirectory"; pub const DS_BUCKET: &str = "druid.storage.bucket"; pub const DS_BASE_KEY: &str = "druid.storage.baseKey"; pub const S3_ENDPOINT_URL: &str = "druid.s3.endpoint.url"; +pub const S3_REGION_NAME: &str = "aws.region"; pub const S3_ACCESS_KEY: &str = "druid.s3.accessKey"; pub const S3_SECRET_KEY: &str = "druid.s3.secretKey"; pub const S3_PATH_STYLE_ACCESS: &str = "druid.s3.enablePathStyleAccess"; @@ -389,6 +390,7 @@ impl v1alpha1::DruidCluster { Ok(result) } + #[allow(clippy::type_complexity)] pub fn build_role_properties( &self, ) -> HashMap< diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs index 54f49f0b..9397f49c 100644 --- a/rust/operator-binary/src/druid_controller.rs +++ b/rust/operator-binary/src/druid_controller.rs @@ -76,7 +76,8 @@ use crate::{ DRUID_CONFIG_DIRECTORY, DS_BUCKET, EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL, - S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, ZOOKEEPER_CONNECTION_STRING, + S3_PATH_STYLE_ACCESS, S3_REGION_NAME, S3_SECRET_KEY, STACKABLE_LOG_DIR, + ZOOKEEPER_CONNECTION_STRING, }, discovery::{self, build_discovery_configmaps}, extensions::get_extension_list, @@ -734,6 +735,8 @@ fn build_rolegroup_config_map( }; if let Some(s3) = s3_conn { + conf.insert(S3_REGION_NAME.to_string(), Some(s3.region.name.to_string())); + conf.insert( S3_ENDPOINT_URL.to_string(), Some(s3.endpoint().context(ConfigureS3Snafu)?.to_string()), From 6d0358f3e2276a89f8e374ba04b9bc4b0a735b39 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 11 Mar 2025 10:48:41 +0100 Subject: [PATCH 3/4] chore: Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de0def6a..5dbd9fd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file. - Support Apache Druid `31.0.1` and `30.0.1`, remove `26.0.0` ([#685]). - BREAKING: Adjust default memory limits of coordinator from `512Mi` to `768Mi` and middlemanager from `1Gi` to `1500Mi` ([#685]). - Support configuring JVM arguments ([#693]). +- Add s3 bucket region to config ([#696]). ### Changed @@ -24,6 +25,7 @@ All notable changes to this project will be documented in this file. [#677]: https://github.com/stackabletech/druid-operator/pull/677 [#693]: https://github.com/stackabletech/druid-operator/pull/693 [#685]: https://github.com/stackabletech/druid-operator/pull/685 +[#696]: https://github.com/stackabletech/druid-operator/pull/696 ## [24.11.1] - 2025-01-09 From 4dde62e7cbe37c106130f18ef2f0133bc06a5185 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 11 Mar 2025 11:27:54 +0100 Subject: [PATCH 4/4] chore: Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dbd9fd4..2fe92c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ All notable changes to this project will be documented in this file. - Support Apache Druid `31.0.1` and `30.0.1`, remove `26.0.0` ([#685]). - BREAKING: Adjust default memory limits of coordinator from `512Mi` to `768Mi` and middlemanager from `1Gi` to `1500Mi` ([#685]). - Support configuring JVM arguments ([#693]). -- Add s3 bucket region to config ([#696]). +- BREAKING: Add s3 bucket region to config ([#696]). ### Changed