From 8fc972414aa866ca7a25d43889dff7c1f3766503 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 29 Jan 2025 11:44:30 +0100 Subject: [PATCH 1/7] chore: Update Druid product version to 31.0.1 and adjust resource limits --- CHANGELOG.md | 2 ++ docs/modules/druid/examples/getting_started/druid.yaml | 2 +- docs/modules/druid/examples/getting_started/druid.yaml.j2 | 2 +- docs/modules/druid/partials/supported-versions.adoc | 1 + examples/psql-s3/psql-s3-druid-cluster.yaml | 2 +- examples/psql/psql-hdfs-druid-cluster.yaml | 2 +- examples/tls/tls-druid-cluster.yaml | 2 +- rust/operator-binary/src/crd/resource.rs | 4 ++-- tests/test-definition.yaml | 3 ++- 9 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 816be139..55d13aa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file.  config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#660]). - Run a `containerdebug` process in the background of each "druid" container to collect debugging information ([#667]). - Aggregate emitted Kubernetes events on the CustomResources ([#677]). +- Support Apache Druid `31.0.1` as experimental version and adjust memory limits of coordinator and middlemanager ([#685]). ### Changed @@ -19,6 +20,7 @@ All notable changes to this project will be documented in this file. [#667]: https://github.com/stackabletech/druid-operator/pull/667 [#676]: https://github.com/stackabletech/druid-operator/pull/676 [#677]: https://github.com/stackabletech/druid-operator/pull/677 +[#685]: https://github.com/stackabletech/druid-operator/pull/685 ## [24.11.1] - 2025-01-09 diff --git a/docs/modules/druid/examples/getting_started/druid.yaml b/docs/modules/druid/examples/getting_started/druid.yaml index f01d56f3..c16c7268 100644 --- a/docs/modules/druid/examples/getting_started/druid.yaml +++ b/docs/modules/druid/examples/getting_started/druid.yaml @@ -5,7 +5,7 @@ metadata: name: simple-druid spec: image: - productVersion: 30.0.0 + productVersion: 31.0.1 clusterConfig: listenerClass: external-stable # This exposes your Stacklet outside of Kubernetes. Remove this configuration if this is not desired zookeeperConfigMapName: simple-druid-znode diff --git a/docs/modules/druid/examples/getting_started/druid.yaml.j2 b/docs/modules/druid/examples/getting_started/druid.yaml.j2 index f01d56f3..c16c7268 100644 --- a/docs/modules/druid/examples/getting_started/druid.yaml.j2 +++ b/docs/modules/druid/examples/getting_started/druid.yaml.j2 @@ -5,7 +5,7 @@ metadata: name: simple-druid spec: image: - productVersion: 30.0.0 + productVersion: 31.0.1 clusterConfig: listenerClass: external-stable # This exposes your Stacklet outside of Kubernetes. Remove this configuration if this is not desired zookeeperConfigMapName: simple-druid-znode diff --git a/docs/modules/druid/partials/supported-versions.adoc b/docs/modules/druid/partials/supported-versions.adoc index 46add761..4f18bc92 100644 --- a/docs/modules/druid/partials/supported-versions.adoc +++ b/docs/modules/druid/partials/supported-versions.adoc @@ -2,5 +2,6 @@ // This is a separate file, since it is used by both the direct Druid documentation, and the overarching // Stackable Platform documentation. +- 31.0.1 (experimental) - 30.0.0 (LTS) - 26.0.0 (deprecated) diff --git a/examples/psql-s3/psql-s3-druid-cluster.yaml b/examples/psql-s3/psql-s3-druid-cluster.yaml index 49a6cf7c..a8de8534 100644 --- a/examples/psql-s3/psql-s3-druid-cluster.yaml +++ b/examples/psql-s3/psql-s3-druid-cluster.yaml @@ -45,7 +45,7 @@ metadata: name: psql-s3-druid spec: image: - productVersion: 30.0.0 + productVersion: 31.0.1 clusterConfig: deepStorage: s3: diff --git a/examples/psql/psql-hdfs-druid-cluster.yaml b/examples/psql/psql-hdfs-druid-cluster.yaml index 505dc94f..a5637f4c 100644 --- a/examples/psql/psql-hdfs-druid-cluster.yaml +++ b/examples/psql/psql-hdfs-druid-cluster.yaml @@ -56,7 +56,7 @@ metadata: name: psql-druid spec: image: - productVersion: 30.0.0 + productVersion: 31.0.1 clusterConfig: deepStorage: hdfs: diff --git a/examples/tls/tls-druid-cluster.yaml b/examples/tls/tls-druid-cluster.yaml index bec8b443..d2c97d70 100644 --- a/examples/tls/tls-druid-cluster.yaml +++ b/examples/tls/tls-druid-cluster.yaml @@ -78,7 +78,7 @@ metadata: name: derby-druid spec: image: - productVersion: 30.0.0 + productVersion: 31.0.1 clusterConfig: authentication: - authenticationClass: druid-mtls-authentication-class diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index a8454a3a..60a6c46e 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -180,7 +180,7 @@ pub static MIDDLE_MANAGER_RESOURCES: LazyLock< max: Some(Quantity("1200m".to_owned())), }, memory: MemoryLimitsFragment { - limit: Some(Quantity("1Gi".to_owned())), + limit: Some(Quantity("1500Mi".to_owned())), runtime_limits: NoRuntimeLimitsFragment {}, }, storage: storage::DruidStorageFragment {}, @@ -223,7 +223,7 @@ pub static COORDINATOR_RESOURCES: LazyLock< max: Some(Quantity("400m".to_owned())), }, memory: MemoryLimitsFragment { - limit: Some(Quantity("512Mi".to_owned())), + limit: Some(Quantity("768Mi".to_owned())), runtime_limits: NoRuntimeLimitsFragment {}, }, storage: storage::DruidStorageFragment {}, diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 45ae3368..937895e2 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -15,11 +15,12 @@ dimensions: values: - 26.0.0 - 30.0.0 + - 31.0.1 # To use a custom image, add a comma and the full name after the product version # - 30.0.0,oci.stackable.tech/sdp/druid:30.0.0-stackable0.0.0-dev - name: druid-latest values: - - 30.0.0 + - 31.0.1 # To use a custom image, add a comma and the full name after the product version # - 30.0.0,oci.stackable.tech/sdp/druid:30.0.0-stackable0.0.0-dev - name: zookeeper From b047973042362dc9875e448292a5c63b2cc7a939 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 10 Feb 2025 09:20:16 +0100 Subject: [PATCH 2/7] docs: don't mark 31.0.1 as experimental --- CHANGELOG.md | 2 +- docs/modules/druid/partials/supported-versions.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55d13aa4..8d5fa910 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project will be documented in this file.  config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#660]). - Run a `containerdebug` process in the background of each "druid" container to collect debugging information ([#667]). - Aggregate emitted Kubernetes events on the CustomResources ([#677]). -- Support Apache Druid `31.0.1` as experimental version and adjust memory limits of coordinator and middlemanager ([#685]). +- Support Apache Druid `31.0.1` and adjust memory limits of coordinator and middlemanager ([#685]). ### Changed diff --git a/docs/modules/druid/partials/supported-versions.adoc b/docs/modules/druid/partials/supported-versions.adoc index 4f18bc92..72254f57 100644 --- a/docs/modules/druid/partials/supported-versions.adoc +++ b/docs/modules/druid/partials/supported-versions.adoc @@ -2,6 +2,6 @@ // This is a separate file, since it is used by both the direct Druid documentation, and the overarching // Stackable Platform documentation. -- 31.0.1 (experimental) +- 31.0.1 - 30.0.0 (LTS) - 26.0.0 (deprecated) From d5cfd09154499773d2bde86b2143b480f5368b00 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 26 Feb 2025 17:37:28 +0100 Subject: [PATCH 3/7] remove 26.0.0 / fix opa integration test --- CHANGELOG.md | 2 +- docs/modules/druid/partials/supported-versions.adoc | 6 +++--- .../kuttl/authorizer/02-install-opa.yaml.j2 | 12 ++++++------ tests/test-definition.yaml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d5fa910..3c062b6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project will be documented in this file.  config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#660]). - Run a `containerdebug` process in the background of each "druid" container to collect debugging information ([#667]). - Aggregate emitted Kubernetes events on the CustomResources ([#677]). -- Support Apache Druid `31.0.1` and adjust memory limits of coordinator and middlemanager ([#685]). +- Support Apache Druid `31.0.1` and adjust memory limits of coordinator and middlemanager, remove `26.0.0` ([#685]). ### Changed diff --git a/docs/modules/druid/partials/supported-versions.adoc b/docs/modules/druid/partials/supported-versions.adoc index 72254f57..fae0270b 100644 --- a/docs/modules/druid/partials/supported-versions.adoc +++ b/docs/modules/druid/partials/supported-versions.adoc @@ -2,6 +2,6 @@ // This is a separate file, since it is used by both the direct Druid documentation, and the overarching // Stackable Platform documentation. -- 31.0.1 -- 30.0.0 (LTS) -- 26.0.0 (deprecated) +- 31.0.1 (LTS) +- 30.0.1 +- 30.0.0 (deprecated) diff --git a/tests/templates/kuttl/authorizer/02-install-opa.yaml.j2 b/tests/templates/kuttl/authorizer/02-install-opa.yaml.j2 index 2425de4e..97f1aa67 100644 --- a/tests/templates/kuttl/authorizer/02-install-opa.yaml.j2 +++ b/tests/templates/kuttl/authorizer/02-install-opa.yaml.j2 @@ -11,16 +11,16 @@ data: default allow = false - allow if { - input.user == "alice" + allow { + input.authenticationResult.identity == "alice" } - allow if { - input.user == "admin" + allow { + input.authenticationResult.identity == "admin" } - allow if { - input.user == "druid_system" + allow { + input.authenticationResult.identity == "druid_system" } --- apiVersion: opa.stackable.tech/v1alpha1 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 937895e2..6a91ac11 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -13,8 +13,8 @@ dimensions: - name: druid values: - - 26.0.0 - 30.0.0 + - 30.0.1 - 31.0.1 # To use a custom image, add a comma and the full name after the product version # - 30.0.0,oci.stackable.tech/sdp/druid:30.0.0-stackable0.0.0-dev From 496bcd12c3e5147e3ab6e145cf7eaf617b08bd19 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 27 Feb 2025 15:07:54 +0100 Subject: [PATCH 4/7] docs: split changelog entry into separate entries --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c062b6e..b064007f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ All notable changes to this project will be documented in this file.  config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#660]). - Run a `containerdebug` process in the background of each "druid" container to collect debugging information ([#667]). - Aggregate emitted Kubernetes events on the CustomResources ([#677]). -- Support Apache Druid `31.0.1` and adjust memory limits of coordinator and middlemanager, remove `26.0.0` ([#685]). +- Support Apache Druid `31.0.1` and `30.0.1`, remove `26.0.0` ([#685]). +- BREAKING: Adjust memory limits of coordinator and middlemanager ([#685]). ### Changed From d66d60fc1dcdd751153fba334a22453c4bae4021 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Thu, 27 Feb 2025 15:29:42 +0100 Subject: [PATCH 5/7] docs: improved changelog entry Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b064007f..8863158d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ All notable changes to this project will be documented in this file. - Run a `containerdebug` process in the background of each "druid" container to collect debugging information ([#667]). - Aggregate emitted Kubernetes events on the CustomResources ([#677]). - Support Apache Druid `31.0.1` and `30.0.1`, remove `26.0.0` ([#685]). -- BREAKING: Adjust memory limits of coordinator and middlemanager ([#685]). +- BREAKING: Adjust default memory limits of coordinator from `512Mi` to `768Mi` and middlemanager from `1Gi` to `1500Mi` ([#685]). ### Changed From cb4d110cdaa8d5e38970cb62fcf4eda677306c3d Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 27 Feb 2025 15:32:25 +0100 Subject: [PATCH 6/7] fix: adjusted docs to reflect new memory limits --- .../druid/pages/usage-guide/resources-and-storage.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modules/druid/pages/usage-guide/resources-and-storage.adoc b/docs/modules/druid/pages/usage-guide/resources-and-storage.adoc index b54d49ea..ef4d1f25 100644 --- a/docs/modules/druid/pages/usage-guide/resources-and-storage.adoc +++ b/docs/modules/druid/pages/usage-guide/resources-and-storage.adoc @@ -13,7 +13,7 @@ A minimal HA setup consisting of 2 Pods of each role has the following https://k * `4700m` CPU request * `13800m` CPU limit -* `11376m` memory request and limit +* `12144Mi` memory request and limit Of course, additional services require additional resources. For Stackable components, see the corresponding documentation on further resource requirements. @@ -54,7 +54,7 @@ spec: min: 300m max: 1200m memory: - limit: 1Gi + limit: 1500Mi coordinators: config: resources: @@ -62,7 +62,7 @@ spec: min: 100m max: 400m memory: - limit: 512Mi + limit: 768Mi ---- The operator may configure an additional container for log aggregation. From 2c027a237170ffb8ffa7af57b7737abc5046b304 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Thu, 27 Feb 2025 15:52:35 +0100 Subject: [PATCH 7/7] fix: LTS version Co-authored-by: Sebastian Bernauer --- docs/modules/druid/partials/supported-versions.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/druid/partials/supported-versions.adoc b/docs/modules/druid/partials/supported-versions.adoc index fae0270b..2aabffdd 100644 --- a/docs/modules/druid/partials/supported-versions.adoc +++ b/docs/modules/druid/partials/supported-versions.adoc @@ -2,6 +2,6 @@ // This is a separate file, since it is used by both the direct Druid documentation, and the overarching // Stackable Platform documentation. -- 31.0.1 (LTS) -- 30.0.1 +- 31.0.1 +- 30.0.1 (LTS) - 30.0.0 (deprecated)