From 7412b28bb5944a6a512691f083b7293cdc2c7f62 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 10:16:10 +0200 Subject: [PATCH 01/54] fix(docs): Correct the release notes template --- .../release-notes/release-template.adoc | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-template.adoc b/modules/ROOT/partials/release-notes/release-template.adoc index 62c691e4a..a4a5949aa 100644 --- a/modules/ROOT/partials/release-notes/release-template.adoc +++ b/modules/ROOT/partials/release-notes/release-template.adoc @@ -77,7 +77,7 @@ Of the changes mentioned above, the following are breaking (or could lead to bre * `spec.b`: This field has been changed to a number. ==== -==== Upgrade from YY.M +==== Upgrade from OO.M ===== Using stackablectl @@ -110,21 +110,21 @@ This can be done using `kubectl replace`. [source] ---- -kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/OO.M.X/deploy/helm/airflow-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/OO.M.X/deploy/helm/commons-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/OO.M.X/deploy/helm/druid-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/OO.M.X/deploy/helm/hbase-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/OO.M.X/deploy/helm/hdfs-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/OO.M.X/deploy/helm/hive-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/OO.M.X/deploy/helm/kafka-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/OO.M.X/deploy/helm/listener-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/OO.M.X/deploy/helm/nifi-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/OO.M.X/deploy/helm/opa-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/OO.M.X/deploy/helm/secret-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/OO.M.X/deploy/helm/spark-k8s-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/OO.M.X/deploy/helm/superset-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/OO.M.X/deploy/helm/trino-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/OO.M.X/deploy/helm/zookeeper-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/YY.M.X/deploy/helm/airflow-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/YY.M.X/deploy/helm/commons-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/YY.M.X/deploy/helm/druid-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/YY.M.X/deploy/helm/hbase-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/YY.M.X/deploy/helm/hdfs-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/YY.M.X/deploy/helm/hive-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/YY.M.X/deploy/helm/kafka-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/YY.M.X/deploy/helm/listener-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/YY.M.X/deploy/helm/nifi-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/YY.M.X/deploy/helm/opa-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/YY.M.X/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/YY.M.X/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/YY.M.X/deploy/helm/superset-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/YY.M.X/deploy/helm/trino-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/YY.M.X/deploy/helm/zookeeper-operator/crds/crds.yaml ---- [source,console] @@ -165,21 +165,21 @@ The reason for this is that helm will uninstall the operators but not the CRDs. [source] ---- -kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/OO.M.X/deploy/helm/airflow-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/OO.M.X/deploy/helm/commons-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/OO.M.X/deploy/helm/druid-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/OO.M.X/deploy/helm/hbase-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/OO.M.X/deploy/helm/hdfs-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/OO.M.X/deploy/helm/hive-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/OO.M.X/deploy/helm/kafka-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/OO.M.X/deploy/helm/listener-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/OO.M.X/deploy/helm/nifi-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/OO.M.X/deploy/helm/opa-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/OO.M.X/deploy/helm/secret-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/OO.M.X/deploy/helm/spark-k8s-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/OO.M.X/deploy/helm/superset-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/OO.M.X/deploy/helm/trino-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/OO.M.X/deploy/helm/zookeeper-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/YY.M.X/deploy/helm/airflow-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/YY.M.X/deploy/helm/commons-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/YY.M.X/deploy/helm/druid-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/YY.M.X/deploy/helm/hbase-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/YY.M.X/deploy/helm/hdfs-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/YY.M.X/deploy/helm/hive-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/YY.M.X/deploy/helm/kafka-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/YY.M.X/deploy/helm/listener-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/YY.M.X/deploy/helm/nifi-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/YY.M.X/deploy/helm/opa-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/YY.M.X/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/YY.M.X/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/YY.M.X/deploy/helm/superset-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/YY.M.X/deploy/helm/trino-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/YY.M.X/deploy/helm/zookeeper-operator/crds/crds.yaml ---- [source,console] @@ -196,21 +196,21 @@ NOTE: `helm repo` subcommands are not supported for OCI registries. The operator [source,console] ---- -helm install --wait airflow-operator oci://oci.stackable.tech/sdp-charts/airflow-operator --version OO.M.X -helm install --wait commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator --version OO.M.X -helm install --wait druid-operator oci://oci.stackable.tech/sdp-charts/druid-operator --version OO.M.X -helm install --wait hbase-operator oci://oci.stackable.tech/sdp-charts/hbase-operator --version OO.M.X -helm install --wait hdfs-operator oci://oci.stackable.tech/sdp-charts/hdfs-operator --version OO.M.X -helm install --wait hive-operator oci://oci.stackable.tech/sdp-charts/hive-operator --version OO.M.X -helm install --wait kafka-operator oci://oci.stackable.tech/sdp-charts/kafka-operator --version OO.M.X -helm install --wait listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator --version OO.M.X -helm install --wait nifi-operator oci://oci.stackable.tech/sdp-charts/nifi-operator --version OO.M.X -helm install --wait opa-operator oci://oci.stackable.tech/sdp-charts/opa-operator --version OO.M.X -helm install --wait secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator --version OO.M.X -helm install --wait spark-k8s-operator oci://oci.stackable.tech/sdp-charts/spark-k8s-operator --version OO.M.X -helm install --wait superset-operator oci://oci.stackable.tech/sdp-charts/superset-operator --version OO.M.X -helm install --wait trino-operator oci://oci.stackable.tech/sdp-charts/trino-operator --version OO.M.X -helm install --wait zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version OO.M.X +helm install --wait airflow-operator oci://oci.stackable.tech/sdp-charts/airflow-operator --version YY.M.X +helm install --wait commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator --version YY.M.X +helm install --wait druid-operator oci://oci.stackable.tech/sdp-charts/druid-operator --version YY.M.X +helm install --wait hbase-operator oci://oci.stackable.tech/sdp-charts/hbase-operator --version YY.M.X +helm install --wait hdfs-operator oci://oci.stackable.tech/sdp-charts/hdfs-operator --version YY.M.X +helm install --wait hive-operator oci://oci.stackable.tech/sdp-charts/hive-operator --version YY.M.X +helm install --wait kafka-operator oci://oci.stackable.tech/sdp-charts/kafka-operator --version YY.M.X +helm install --wait listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator --version YY.M.X +helm install --wait nifi-operator oci://oci.stackable.tech/sdp-charts/nifi-operator --version YY.M.X +helm install --wait opa-operator oci://oci.stackable.tech/sdp-charts/opa-operator --version YY.M.X +helm install --wait secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator --version YY.M.X +helm install --wait spark-k8s-operator oci://oci.stackable.tech/sdp-charts/spark-k8s-operator --version YY.M.X +helm install --wait superset-operator oci://oci.stackable.tech/sdp-charts/superset-operator --version YY.M.X +helm install --wait trino-operator oci://oci.stackable.tech/sdp-charts/trino-operator --version YY.M.X +helm install --wait zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version YY.M.X ---- ==== Known issues From 68a8c1751c4b9c83d2d5a668d397d36a9aa7cc60 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 10:16:59 +0200 Subject: [PATCH 02/54] docs: Add skeleton 25.7.0 release notes --- modules/ROOT/pages/release-notes.adoc | 2 + .../partials/release-notes/release-25.7.adoc | 216 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 modules/ROOT/partials/release-notes/release-25.7.adoc diff --git a/modules/ROOT/pages/release-notes.adoc b/modules/ROOT/pages/release-notes.adoc index 0d02dbba6..2212ed303 100644 --- a/modules/ROOT/pages/release-notes.adoc +++ b/modules/ROOT/pages/release-notes.adoc @@ -7,6 +7,8 @@ The Stackable Data Platform consists of multiple operators that work together. Periodically a platform release is made, including all components of the platform at a specific version. // WARNING: Please keep the empty newlines, otherwise headings are broken. +include::partial$release-notes/release-25.7.adoc[] + include::partial$release-notes/release-25.3.adoc[] include::partial$release-notes/release-24.11.adoc[] diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc new file mode 100644 index 000000000..8af5dc0ad --- /dev/null +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -0,0 +1,216 @@ +// Here are the headings you can use for the next release. Saves time checking indentation levels. +// Take a look at release 24.11 to see how to structure patch releases. + +== Release 25.7 + +=== 25.7.0 + +Released on YYYY-MM-DD. +(Optional description / introduction) + +==== New platform features + +==== Platform improvements + +==== Platform deprecations + +==== Product versions + +As with previous SDP releases, many product images have been updated to their latest versions. +The LTS version has in many cases also been adjusted in line with our xref:ROOT:policies.adoc[support policy]. + +Refer to the xref:operators:supported_versions.adoc[supported versions] documentation for a complete overview including LTS versions or deprecations. + +===== New versions + +The following new product versions are now supported: + +* ... + +===== Deprecated versions + +The following product versions are deprecated and will be removed in a later release: + +* ... + +===== Removed versions + +The following product versions are no longer supported (although images for released product versions remain available https://oci.stackable.tech/[here,window=_blank]. Information on how to browse the registry can be found xref:contributor:project-overview.adoc#docker-images[here,window=_blank].): + +* ... + +==== stackablectl + +==== Supported Kubernetes versions + +This release supports the following Kubernetes versions: + +* `1.XX` + +These Kubernetes versions are no longer supported: + +* `1.XX` + +==== Supported OpenShift versions + +This release is available in the RedHat Certified Operator Catalog for the following OpenShift versions: + +* `4.XX` + +These OpenShift versions are no longer supported: + +* `4.XX` + +==== Breaking changes + +Of the changes mentioned above, the following are breaking (or could lead to breaking behaviour), and you will need to adapt your existing CRDs accordingly: + +===== Stackable Operator for Example Product + +* Description of the change 1 +* Description of the change 2 + +.Breaking changes details +[%collapsible] +==== +* `spec.a`: This field has been removed. +* `spec.b`: This field has been changed to a number. +==== + +==== Upgrade from 25.3 + +===== Using stackablectl + +====== Upgrade with a single command + +Starting with stackablectl Release 1.0.0 the multiple consecutive commands described below can be shortened to just one command, which executes exactly those steps on its own. + +[source,console] +---- +$ stackablectl release upgrade 25.7 +---- + +====== Upgrade with multiple consecutive commands + +Uninstall the `25.3` release + +[source,console] +---- +$ stackablectl release uninstall 25.3 + +Uninstalled release '25.3' + +Use "stackablectl release list" to list available releases. +# ... +---- + +Afterwards you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. +The reason for this is that helm will uninstall the operators but not the CRDs. +This can be done using `kubectl replace`. + +[source] +---- +kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/25.7.0/deploy/helm/airflow-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/25.7.0/deploy/helm/commons-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/25.7.0/deploy/helm/druid-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/25.7.0/deploy/helm/hbase-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/25.7.0/deploy/helm/hdfs-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/25.7.0/deploy/helm/hive-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/25.7.0/deploy/helm/kafka-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/25.7.0/deploy/helm/listener-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/25.7.0/deploy/helm/nifi-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/25.7.0/deploy/helm/opa-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/25.7.0/deploy/helm/superset-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/25.7.0/deploy/helm/trino-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/25.7.0/deploy/helm/zookeeper-operator/crds/crds.yaml +---- + +[source,console] +---- +customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced +... +---- + +Install the `25.7` release + +[source,console] +---- +$ stackablectl release install 25.7 + +Installed release '25.7' + +Use "stackablectl operator installed" to list installed operators. +---- + +===== Using Helm + +Use `helm list` to list the currently installed operators. + +You can use the following command to uninstall all operators that are part of the `25.3` release: + +[source,console] +---- +$ helm uninstall airflow-operator commons-operator druid-operator hbase-operator hdfs-operator hive-operator kafka-operator listener-operator nifi-operator opa-operator secret-operator spark-k8s-operator superset-operator trino-operator zookeeper-operator +release "airflow-operator" uninstalled +release "commons-operator" uninstalled +... +---- + +Afterward you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. +The reason for this is that helm will uninstall the operators but not the CRDs. This can be done using `kubectl replace`: + +[source] +---- +kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/25.7.0/deploy/helm/airflow-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/25.7.0/deploy/helm/commons-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/25.7.0/deploy/helm/druid-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/25.7.0/deploy/helm/hbase-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/25.7.0/deploy/helm/hdfs-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/25.7.0/deploy/helm/hive-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/25.7.0/deploy/helm/kafka-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/25.7.0/deploy/helm/listener-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/25.7.0/deploy/helm/nifi-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/25.7.0/deploy/helm/opa-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/25.7.0/deploy/helm/superset-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/25.7.0/deploy/helm/trino-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/25.7.0/deploy/helm/zookeeper-operator/crds/crds.yaml +---- + +[source,console] +---- +customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced +... +---- + +Install the `25.7` release + +NOTE: `helm repo` subcommands are not supported for OCI registries. The operators are installed directly, without adding the Helm Chart repository first. + +[source,console] +---- +helm install --wait airflow-operator oci://oci.stackable.tech/sdp-charts/airflow-operator --version 25.7.0 +helm install --wait commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator --version 25.7.0 +helm install --wait druid-operator oci://oci.stackable.tech/sdp-charts/druid-operator --version 25.7.0 +helm install --wait hbase-operator oci://oci.stackable.tech/sdp-charts/hbase-operator --version 25.7.0 +helm install --wait hdfs-operator oci://oci.stackable.tech/sdp-charts/hdfs-operator --version 25.7.0 +helm install --wait hive-operator oci://oci.stackable.tech/sdp-charts/hive-operator --version 25.7.0 +helm install --wait kafka-operator oci://oci.stackable.tech/sdp-charts/kafka-operator --version 25.7.0 +helm install --wait listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator --version 25.7.0 +helm install --wait nifi-operator oci://oci.stackable.tech/sdp-charts/nifi-operator --version 25.7.0 +helm install --wait opa-operator oci://oci.stackable.tech/sdp-charts/opa-operator --version 25.7.0 +helm install --wait secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator --version 25.7.0 +helm install --wait spark-k8s-operator oci://oci.stackable.tech/sdp-charts/spark-k8s-operator --version 25.7.0 +helm install --wait superset-operator oci://oci.stackable.tech/sdp-charts/superset-operator --version 25.7.0 +helm install --wait trino-operator oci://oci.stackable.tech/sdp-charts/trino-operator --version 25.7.0 +helm install --wait zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version 25.7.0 +---- + +==== Known issues From c39a2843a292b7975ce0c522da47653e53eb4e24 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 10:24:34 +0200 Subject: [PATCH 03/54] chore(template): Remove taskslists from release notes PR template --- .github/PULL_REQUEST_TEMPLATE/release-notes.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/release-notes.md b/.github/PULL_REQUEST_TEMPLATE/release-notes.md index d71ce7b08..861548e97 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-notes.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-notes.md @@ -16,8 +16,8 @@ [template]: https://github.com/stackabletech/documentation/blob/8dc93f28ac6d20a587f54d0a697c71fe47e8643a/modules/ROOT/pages/release-notes.adoc?plain=1#L11-L56 -```[tasklist] -#### Release note compilation tasks +## Release note compilation tasks + - [ ] Check [Issues](https://github.com/search?q=org%3Astackabletech+label%3Arelease-note%2Crelease-note%2Faction-required+label%3Arelease%YY.M.X%2Cscheduled-for%YY.M.X&type=issues) for Product and Platform release notes - [ ] Check [PRs](https://github.com/search?q=org%3Astackabletech+label%3Arelease-note%2Crelease-note%2Faction-required+label%3Arelease%YY.M.X%2Cscheduled-for%YY.M.X&type=pullrequests) for Product and Platform release notes - [ ] Optionally check the [Changelogs](https://github.com/search?q=org%3Astackabletech+path%3A*CHANGELOG.md+%22YY.M.X%22&type=code) in case release notes were missed @@ -28,15 +28,13 @@ - [ ] Upgrade guide: List removed product versions (if there are any) - [ ] Upgrade guide: List removed operators (if there are any) - [ ] Upgrade guide: List supported Kubernetes versions -``` Each of the following tasks focuses on a specific goal and should be done once the items above have been completed. -```[tasklist] -#### Release note review tasks +## Release note review tasks + - [ ] Check overall document structure - [ ] Check spelling, grammar, and correct wording - [ ] Check that internal links are xrefs - [ ] Check that rendered links are valid - [ ] Check that each sentence begins on a new line -``` From ee874b0d80baa1d037200d57dfcd7b3860b9cb2e Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 10:29:46 +0200 Subject: [PATCH 04/54] chore: Update 25.3(.0) to 25.7(.0) on getting-started page --- modules/ROOT/pages/getting-started.adoc | 29 +++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/modules/ROOT/pages/getting-started.adoc b/modules/ROOT/pages/getting-started.adoc index e09f7afd9..b1e669391 100644 --- a/modules/ROOT/pages/getting-started.adoc +++ b/modules/ROOT/pages/getting-started.adoc @@ -32,11 +32,12 @@ Install the Stackable command line utility xref:management:stackablectl:index.ad The Stackable operators are components that translate the service definitions deployed via Kubernetes into deploy services on the worker nodes. These can be installed on any node that has access to the Kubernetes control plane. In this example we will install them on the controller node. -Stackable operators can be installed using `stackablectl`. Run the following commands to install ZooKeeper, Kafka and NiFi from the Stackable 25.3 release. +Stackable operators can be installed using `stackablectl`. +Run the following commands to install ZooKeeper, Kafka and NiFi from the Stackable 25.7 release. [source,bash] ---- -stackablectl release install -i commons -i secret -i listener -i zookeeper -i kafka -i nifi 25.3 +stackablectl release install -i commons -i secret -i listener -i zookeeper -i kafka -i nifi 25.7 ---- .Using Helm instead @@ -50,12 +51,12 @@ Install the operators: [source,bash] ---- -helm install zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version=25.3.0 -helm install kafka-operator oci://oci.stackable.tech/sdp-charts/kafka-operator --version=25.3.0 -helm install secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator --version=25.3.0 -helm install listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator --version=25.3.0 -helm install commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator --version=25.3.0 -helm install nifi-operator oci://oci.stackable.tech/sdp-charts/nifi-operator --version=25.3.0 +helm install zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version=25.7.0 +helm install kafka-operator oci://oci.stackable.tech/sdp-charts/kafka-operator --version=25.7.0 +helm install secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator --version=25.7.0 +helm install listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator --version=25.7.0 +helm install commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator --version=25.7.0 +helm install nifi-operator oci://oci.stackable.tech/sdp-charts/nifi-operator --version=25.7.0 ---- ==== @@ -64,12 +65,12 @@ You can check which operators are installed using `stackablectl operator install [source,console] ---- OPERATOR VERSION NAMESPACE STATUS LAST UPDATED -commons 25.3.0 default deployed 2024-11-30 17:58:32.916032854 +0100 CET -kafka 25.3.0 default deployed 2024-11-30 17:58:55.036115353 +0100 CET -listener 25.3.0 default deployed 2024-11-30 17:59:18.136775259 +0100 CET -nifi 25.3.0 default deployed 2024-11-30 17:59:51.927081648 +0100 CET -secret 25.3.0 default deployed 2024-11-30 18:00:05.060241771 +0100 CET -zookeeper 25.3.0 default deployed 2024-11-30 18:00:08.425686918 +0100 CET +commons 25.7.0 default deployed 2024-11-30 17:58:32.916032854 +0100 CET +kafka 25.7.0 default deployed 2024-11-30 17:58:55.036115353 +0100 CET +listener 25.7.0 default deployed 2024-11-30 17:59:18.136775259 +0100 CET +nifi 25.7.0 default deployed 2024-11-30 17:59:51.927081648 +0100 CET +secret 25.7.0 default deployed 2024-11-30 18:00:05.060241771 +0100 CET +zookeeper 25.7.0 default deployed 2024-11-30 18:00:08.425686918 +0100 CET ---- == Deploying Stackable Services From db24d23f54634684263628502288989b736565f1 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 10:30:41 +0200 Subject: [PATCH 05/54] chore: Put sentences on single lines on getting-started page --- modules/ROOT/pages/getting-started.adoc | 37 +++++++++++++++++-------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/modules/ROOT/pages/getting-started.adoc b/modules/ROOT/pages/getting-started.adoc index b1e669391..28f51eca7 100644 --- a/modules/ROOT/pages/getting-started.adoc +++ b/modules/ROOT/pages/getting-started.adoc @@ -2,7 +2,8 @@ :page-aliases: getting_started.adoc :description: Learn how to set up and test a Stackable Data Platform with Apache ZooKeeper, Kafka, and NiFi using Kubernetes. Ideal for lab environments. -One of the best ways of getting started with a new platform is to try it out. Any big data platform has a lot of moving parts and getting some hands on keyboard time with it helps reinforce learning. +One of the best ways of getting started with a new platform is to try it out. +Any big data platform has a lot of moving parts and getting some hands on keyboard time with it helps reinforce learning. == About this guide @@ -13,7 +14,8 @@ Firstly, let's cover whether this *Getting Started* guide is right for you. This == Overview -Stackable is based on Kubernetes and uses this as the control plane to manage clusters. In this guide we will build a simple cluster with 3 services; Apache ZooKeeper, Apache Kafka and Apache NiFi. +Stackable is based on Kubernetes and uses this as the control plane to manage clusters. +In this guide we will build a simple cluster with 3 services; Apache ZooKeeper, Apache Kafka and Apache NiFi. == Installing Kubernetes and kubectl @@ -30,7 +32,9 @@ Install the Stackable command line utility xref:management:stackablectl:index.ad === Installing Stackable Operators -The Stackable operators are components that translate the service definitions deployed via Kubernetes into deploy services on the worker nodes. These can be installed on any node that has access to the Kubernetes control plane. In this example we will install them on the controller node. +The Stackable operators are components that translate the service definitions deployed via Kubernetes into deploy services on the worker nodes. +These can be installed on any node that has access to the Kubernetes control plane. +In this example we will install them on the controller node. Stackable operators can be installed using `stackablectl`. Run the following commands to install ZooKeeper, Kafka and NiFi from the Stackable 25.7 release. @@ -45,7 +49,8 @@ stackablectl release install -i commons -i secret -i listener -i zookeeper -i ka ==== NOTE: These examples assume Helm version 3. They will not work with Helm version 2. -`helm repo` subcommands are not supported for OCI registries. The operators are installed directly, without adding the Helm Chart repository first. +`helm repo` subcommands are not supported for OCI registries. +The operators are installed directly, without adding the Helm Chart repository first. Install the operators: @@ -75,7 +80,8 @@ zookeeper 25.7.0 default deployed == Deploying Stackable Services -At this point you've successfully deployed Kubernetes and the Stackable operators we need and are ready to deploy services to the cluster. To do this we provide service descriptions to Kubernetes for each of the services we wish to deploy. +At this point you've successfully deployed Kubernetes and the Stackable operators we need and are ready to deploy services to the cluster. +To do this we provide service descriptions to Kubernetes for each of the services we wish to deploy. === Apache ZooKeeper @@ -114,7 +120,8 @@ EOF === Apache Kafka -We will deploy an Apache Kafka broker that depends on the ZooKeeper service we just deployed. The zookeeperReference property below points to the namespace and name we gave to the ZooKeeper service deployed previously. +We will deploy an Apache Kafka broker that depends on the ZooKeeper service we just deployed. +The zookeeperReference property below points to the namespace and name we gave to the ZooKeeper service deployed previously. [source,bash] ---- @@ -208,7 +215,8 @@ stringData: EOF ---- -You can check the status of the services using `kubectl get pods`. This will retrieve the status of all pods running in the default namespace. +You can check the status of the services using `kubectl get pods`. +This will retrieve the status of all pods running in the default namespace. ---- NAME READY STATUS RESTARTS AGE @@ -228,7 +236,8 @@ simple-zk-server-primary-0 1/1 Running 0 zookeeper-operator-deployment-7bcdcbb558-xc77h 1/1 Running 0 13m ---- -Since this is the first time that each of these services has been deployed to these nodes, it will take some time to download the software from the Stackable repository and deploy the services. Once all the pods are in the running state your cluster is ready to use. +Since this is the first time that each of these services has been deployed to these nodes, it will take some time to download the software from the Stackable repository and deploy the services. +Once all the pods are in the running state your cluster is ready to use. == Testing your cluster @@ -236,14 +245,16 @@ If all has gone well then you will have successfully deployed a Stackable cluste === Apache ZooKeeper -We can test ZooKeeper by running the ZooKeeper CLI shell. The easiest way to do this is to run the CLI shell on the pod that is running ZooKeeper. +We can test ZooKeeper by running the ZooKeeper CLI shell. +The easiest way to do this is to run the CLI shell on the pod that is running ZooKeeper. [source,bash] ---- kubectl exec -i -t simple-zk-server-primary-0 -- bin/zkCli.sh ---- -The shell should connect automatically to the ZooKeeper server running on the pod. You can run the `ls /` command to see the list of znodes in the root path, which should include those created by Apache Kafka and Apache NiFi. +The shell should connect automatically to the ZooKeeper server running on the pod. +You can run the `ls /` command to see the list of znodes in the root path, which should include those created by Apache Kafka and Apache NiFi. ---- [zk: localhost:2181(CONNECTED) 0] ls / @@ -261,7 +272,8 @@ kubectl exec -i -t simple-kafka-broker-brokers-0 -c kafka -- \ bin/kafka-topics.sh --bootstrap-server localhost:9092 --create --topic demo ---- -You should see the message, "Created topic demo." on the console. Now let's check to see if it was actually created: +You should see the message, "Created topic demo." on the console. +Now let's check to see if it was actually created: [source,bash] ---- @@ -299,7 +311,8 @@ Browse to the address of your Kubernetes node on port `31931` e.g. https://172.1 image:nifi_login_screen.png[The Apache NiFi web interface login screen] -If a password has not been specified for the admin user the Apache NiFi operator will automatically generate the admin user credentials with a random password and store it as a Kubernetes secret in order to provide some security out of the box. In the example above we have provided our own secret, but you can retrieve and confirm this password for the `admin` user with the following kubectl command. +If a password has not been specified for the admin user the Apache NiFi operator will automatically generate the admin user credentials with a random password and store it as a Kubernetes secret in order to provide some security out of the box. +In the example above we have provided our own secret, but you can retrieve and confirm this password for the `admin` user with the following kubectl command. [source,bash] ---- From 2e7975dd658b19acedf02890d6e1f9b806f2a573 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 10:41:55 +0200 Subject: [PATCH 06/54] fix(template): Correct the issue and PR search links --- .github/PULL_REQUEST_TEMPLATE/release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/release-notes.md b/.github/PULL_REQUEST_TEMPLATE/release-notes.md index 861548e97..dcf811874 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-notes.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-notes.md @@ -18,8 +18,8 @@ ## Release note compilation tasks -- [ ] Check [Issues](https://github.com/search?q=org%3Astackabletech+label%3Arelease-note%2Crelease-note%2Faction-required+label%3Arelease%YY.M.X%2Cscheduled-for%YY.M.X&type=issues) for Product and Platform release notes -- [ ] Check [PRs](https://github.com/search?q=org%3Astackabletech+label%3Arelease-note%2Crelease-note%2Faction-required+label%3Arelease%YY.M.X%2Cscheduled-for%YY.M.X&type=pullrequests) for Product and Platform release notes +- [ ] Check [Issues](https://github.com/search?q=org%3Astackabletech+label%3Arelease-note%2Crelease-note%2Faction-required+label%3Arelease%2FYY.M.X%2Cscheduled-for%2FYY.M.X&type=issues) for Product and Platform release notes +- [ ] Check [PRs](https://github.com/search?q=org%3Astackabletech+label%3Arelease-note%2Crelease-note%2Faction-required+label%3Arelease%2FYY.M.X%2Cscheduled-for%2FYY.M.X&type=pullrequests) for Product and Platform release notes - [ ] Optionally check the [Changelogs](https://github.com/search?q=org%3Astackabletech+path%3A*CHANGELOG.md+%22YY.M.X%22&type=code) in case release notes were missed - [ ] Compile list of new product versions that are supported and compile a list of new product features to include in the Release Highlights - [ ] Upgrade guide: Document how to use stackablectl to uninstall all and install new release From 0c23bce67d98326f6d56a03f490404ff775b790d Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 14:40:17 +0200 Subject: [PATCH 07/54] docs: Add initial set of new platform features --- .../partials/release-notes/release-25.7.adoc | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 8af5dc0ad..2b969ee85 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -10,6 +10,66 @@ Released on YYYY-MM-DD. ==== New platform features +===== General + +====== Security + +Users (and services) can now use the new TrustStore resource to request the trust root certificates associated with a SecretClass. +See the xref:secret-operator:truststore.adoc[TrustStore documentation] and https://github.com/stackabletech/secret-operator/issues/410[secret-operator#410]. + +====== Automatic cluster domain detection + +The default Kubernetes cluster domain name is now fetched from the kubelet API unless explicitly configured. +Operators are now required to have the RBAC permission to get nodes/proxy in the apiGroup `""`. +The helm-charts takes care of this. + +The CLI argument `--kubernetes-node-name` or env variable `KUBERNETES_NODE_NAME` needs to be set. +The helm-charts takes care of this. + +See https://github.com/stackabletech/issues/issues/662[issues#662]. + +====== Observability + +* All operators support sending OpenTelemetry logs and traces. +* All operators use unified CLI arguments and environment variables to configure the exporters. +* All operators allow exporter configuration via Helm values. +* All exporters are disabled by default, except for console logs, which remain enabled by default. + +Additionally see the official xref:concepts:observability/telemetry.adoc[telemetry documentation page] and the tracking issue https://github.com/stackabletech/issues/issues/639[issues#639] for further details. + +===== Apache Airflow + +The recent release of Airflow 3 brings with it a raft of changes - with both new features and significant changes to architecture. +SDP offers early experimental support for Airflow 3.0.1 as of SDP 25.7. +This gives users the chance to use the new UI, to https://airflow.apache.org/docs/apache-airflow/3.0.1/installation/upgrading_to_airflow3.html#step-3-dag-authors-check-your-airflow-dags-for-compatibility[update] their DAGs and to test the new OPA authenticator. +See https://github.com/stackabletech/docker-images/issues/1074[docker-images#1074]. + +===== Apache NiFi + +* Starting with NiFi 2.0, custom Python extensions are supported. + See https://github.com/stackabletech/nifi-operator/issues/739[nifi-operator#739] and documentation on xref:nifi:usage_guide/custom-components.adoc[custom components]. +* Custom NiFi Archives (NARs) and Python components can be loaded from Git repositories with git-sync. + See https://github.com/stackabletech/nifi-operator/issues/739[nifi-operator#739] and documentation on xref:nifi:usage_guide/custom-components.adoc[custom components]. +* The Stackable operator for Apache NiFi now supports using Kubernetes for cluster coordination and storage, rather than ZooKeeper. + (Requires NiFi 2.x, migrating existing installs is currently not supported) + See xref:nifi:usage_guide/clustering.adoc#backend-kubernetes[NiFi backend documentation] and https://github.com/stackabletech/nifi-operator/issues/737[nifi-operator#737]. +* Rolling upgrades are now supported between NiFi 2 versions. + This is the default update strategy for NiFi 2 clusters. + See the xref:nifi:usage_guide/updating.adoc[NiFi updating documentation] and https://github.com/stackabletech/nifi-operator/issues/736[nifi-operator#736]. +* The Stackable operator for Apache NiFi now supports authorization using the Open Policy Agent (OPA). + See the xref:nifi:usage_guide/security.adoc#authorization-opa[NiFi authorization documentation] and https://github.com/stackabletech/issues/issues/47[issues#47]. + +===== Apache Spark + +This release adds experimental support for Spark Connect. +The Spark operator watches for SparkConnectServer custom resources. +Preliminary documentation is xref:spark-k8s:usage-guide/spark-connect.adoc[available] and the existing xref:demos:jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc[Taxi Data Anomaly Detection demo] has been retrofitted to use a JupterLab client running against a Spark Connect server. +Also see https://github.com/stackabletech/spark-k8s-operator/issues/284[spark-k8s-operator#284]. + +===== Stackable listener operator + +The Stackable Listener Operator can now use custom xref:listener-operator:listenerclass.adoc#servicetype-loadbalancer-class[LoadBalancer classes] and https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation[disable NodePort allocation]. + ==== Platform improvements ==== Platform deprecations From dc2969e2025fe162b54a7b3b0799c5d6cf005bab Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 14:40:54 +0200 Subject: [PATCH 08/54] docs: Add initial set of platform improvements --- .../partials/release-notes/release-25.7.adoc | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 2b969ee85..17fb63c16 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -72,6 +72,37 @@ The Stackable Listener Operator can now use custom xref:listener-operator:listen ==== Platform improvements +===== General + +* All products that are built from source in SDP, which is at the time of this release everything except Airflow and Superset, now have a version suffix to indicate they include custom modifications made by Stackable. + The suffix the same as for SDP container images, so Zookeeper 3.9.3 in SDP 25.7.0 will for example report `3.9.3-stackable25.7.0` as its version. + See https://github.com/stackabletech/docker-images/issues/1068[docker-images#1068]. +* The Stackable operators for Apache Airflow, Apache Druid, Apache HBase, Apache Hadoop, Apache Hive, Apache NiFi, Apache Spark, Apache ZooKeeper, and Trino are now integrated with the Stackable listener operator. + This is a breaking change across the board. + Consult the breaking changes section for more details. + +===== Vulnerabilities + +230 CVEs were fixed in the Stackable product images. +This includes 3 critical and 97 high-severity CVEs. + +===== Apache Hadoop + +Various setting defaults have been updated for better performance and reliability. +See https://github.com/stackabletech/hdfs-operator/issues/685[hdfs-operator#685]. + +===== Apache NiFi + +* The operator now emits a warning for deprecated sensitive property algorithms in NiFi version 1.x.x and errors out for version 2.x.x if an unsupported sensitive properties algorithm is used. + See the official https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#property-encryption-algorithms[NiFi documentation] and https://github.com/stackabletech/nifi-operator/issues/791[nifi-operator#791]. +* Re-add Iceberg support to NiFi 2.4.0, which was removed upstream starting with version 2.0.0. + See xref:nifi:usage_guide/writing-to-iceberg-tables.adoc[NiFi Iceberg documentation] and https://github.com/stackabletech/nifi-operator/issues/738[nifi-operator#738]. + +===== Apache ZooKeeper + +The image for Apache ZooKeeper backports various 3.9.4 changes into the supported 3.9.3 version. +See https://github.com/stackabletech/docker-images/issues/1143[docker-images#1143]. + ==== Platform deprecations ==== Product versions From 9bfcce5f67a5fd1e190da429940f9a3f29430efe Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 14:41:34 +0200 Subject: [PATCH 09/54] docs: Add new, deprecated and removed product versions --- .../partials/release-notes/release-25.7.adoc | 43 +++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 17fb63c16..4be2bdab6 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -108,7 +108,6 @@ See https://github.com/stackabletech/docker-images/issues/1143[docker-images#114 ==== Product versions As with previous SDP releases, many product images have been updated to their latest versions. -The LTS version has in many cases also been adjusted in line with our xref:ROOT:policies.adoc[support policy]. Refer to the xref:operators:supported_versions.adoc[supported versions] documentation for a complete overview including LTS versions or deprecations. @@ -116,21 +115,49 @@ Refer to the xref:operators:supported_versions.adoc[supported versions] document The following new product versions are now supported: -* ... +* Apache Airflow: https://github.com/stackabletech/docker-images/issues/1074[2.10.5], https://github.com/stackabletech/docker-images/issues/1074[3.0.1 (experimental)] +* Apache Druid: https://github.com/stackabletech/docker-images/issues/1075[33.0.0] +* Apache HBase: https://github.com/stackabletech/docker-images/issues/1076[2.6.2] +* Apache Kafka: https://github.com/stackabletech/docker-images/issues/1078[4.0.0 (experimental)^1^] +* Apache NiFi: https://github.com/stackabletech/docker-images/issues/1079[2.4.0] +* Apache Superset: https://github.com/stackabletech/docker-images/issues/1081[4.1.2] +* Open Policy Agent: https://github.com/stackabletech/docker-images/issues/1084[1.4.2] +* Trino: https://github.com/stackabletech/docker-images/issues/1082[476] ===== Deprecated versions The following product versions are deprecated and will be removed in a later release: -* ... +* Apache Airflow: https://github.com/stackabletech/docker-images/issues/1074[2.10.4] +* Apache Druid: https://github.com/stackabletech/docker-images/issues/1075[31.0.1] +* Apache Hbase: https://github.com/stackabletech/docker-images/issues/1076[2.6.1] +* Apache Superset: https://github.com/stackabletech/docker-images/issues/1081[4.2.1] +* Open Policy Agent: https://github.com/stackabletech/docker-images/issues/1084[1.0.1] +* Trino: https://github.com/stackabletech/docker-images/issues/1082[470] ===== Removed versions -The following product versions are no longer supported (although images for released product versions remain available https://oci.stackable.tech/[here,window=_blank]. Information on how to browse the registry can be found xref:contributor:project-overview.adoc#docker-images[here,window=_blank].): - -* ... - -==== stackablectl +The following product versions are no longer supported. +These images for released product versions remain available https://oci.stackable.tech/[here,window=_blank] +Information on how to browse the registry can be found xref:contributor:project-overview.adoc#docker-images[here,window=_blank] + +* Apache Druid: https://github.com/stackabletech/docker-images/issues/1075[30.0.0] +* Apache Hbase: https://github.com/stackabletech/docker-images/issues/1076[2.4.18] +* Apache Hadoop: https://github.com/stackabletech/docker-images/issues/1077[3.3.4], https://github.com/stackabletech/docker-images/issues/1077[3.4.0], https://github.com/stackabletech/docker-images/issues/1077[3.3.6^2^] +* Apache Kafka: https://github.com/stackabletech/docker-images/issues/1078[3.7.1], https://github.com/stackabletech/docker-images/issues/1078[3.8.0] +* Apache NiFi: https://github.com/stackabletech/docker-images/issues/1079[2.2.0 (experimental)] +* Apache Spark: https://github.com/stackabletech/docker-images/issues/1080[3.5.2] +* Apache ZooKeeper: https://github.com/stackabletech/docker-images/issues/1083[3.9.2] +* Open Policy Agent: https://github.com/stackabletech/docker-images/issues/1084[0.67.1] +* Trino: https://github.com/stackabletech/docker-images/issues/1082[455] + +''' + +[.small] +-- +. Unused in SDP, see https://github.com/stackabletech/kafka-operator/issues/690[kafka-operator#690] and https://github.com/stackabletech/docker-images/pull/1117[docker-images#1117] +. Remove support for 3.3.6, it is still required by HBase, Hive, and Spark in this release. +-- ==== Supported Kubernetes versions From 6f0eaae4864415779be56885517bf2c625a92f80 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 17 Jul 2025 14:42:00 +0200 Subject: [PATCH 10/54] docs: Add (un)supported Kubernetes and OpenShift versions --- .../ROOT/partials/release-notes/release-25.7.adoc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 4be2bdab6..37dc2871b 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -163,21 +163,27 @@ Information on how to browse the registry can be found xref:contributor:project- This release supports the following Kubernetes versions: -* `1.XX` +* `1.33` +* `1.32` +* `1.31` +* `1.30` These Kubernetes versions are no longer supported: -* `1.XX` +* `1.29` ==== Supported OpenShift versions This release is available in the RedHat Certified Operator Catalog for the following OpenShift versions: -* `4.XX` +* `4.18` +* `4.17` +* `4.16` These OpenShift versions are no longer supported: -* `4.XX` +* `4.15` +* `4.14` ==== Breaking changes From 1467aeed3d1e6e046ff4088985e10dffcbb32b84 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 09:57:50 +0200 Subject: [PATCH 11/54] docs: Add OCI registry note --- modules/ROOT/partials/release-notes/release-25.7.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 37dc2871b..1cce93d01 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -8,6 +8,11 @@ Released on YYYY-MM-DD. (Optional description / introduction) +[NOTE] +==== +The SDP 25.7.0 release is the first release which will **only** be published on `oci.stackable.tech`. +==== + ==== New platform features ===== General From d3c9b6ac5095e8679ba1595a9a68550ad33646f9 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 09:58:30 +0200 Subject: [PATCH 12/54] docs: Add adjusted commands for the secret-operator CRD --- .../partials/release-notes/release-25.7.adoc | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 1cce93d01..f40189317 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -237,6 +237,13 @@ Afterwards you will need to upgrade the CustomResourceDefinitions (CRDs) install The reason for this is that helm will uninstall the operators but not the CRDs. This can be done using `kubectl replace`. +[IMPORTANT] +==== +The Stackable secret operator CRD needs to be applied in addition to being replaced, because it introduces the new TrustStore custom resource. +The commands below are adjusted accordingly. +This issue is also mentioned in the known issues section below. +==== + [source] ---- kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/25.7.0/deploy/helm/airflow-operator/crds/crds.yaml @@ -249,7 +256,8 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operato kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/25.7.0/deploy/helm/listener-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/25.7.0/deploy/helm/nifi-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/25.7.0/deploy/helm/opa-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml || true +kubectl apply -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/25.7.0/deploy/helm/superset-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/25.7.0/deploy/helm/trino-operator/crds/crds.yaml @@ -290,7 +298,14 @@ release "commons-operator" uninstalled ---- Afterward you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. -The reason for this is that helm will uninstall the operators but not the CRDs. This can be done using `kubectl replace`: +The reason for this is that helm will uninstall the operators but not the CRDs. This can be done using `kubectl replace`. + +[IMPORTANT] +==== +The Stackable secret operator CRD needs to be applied in addition to being replaced, because it introduces the new TrustStore custom resource. +The commands below are adjusted accordingly. +This issue is also mentioned in the known issues section below. +==== [source] ---- @@ -304,7 +319,8 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operato kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/25.7.0/deploy/helm/listener-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/25.7.0/deploy/helm/nifi-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/25.7.0/deploy/helm/opa-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml || true +kubectl apply -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/25.7.0/deploy/helm/superset-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/25.7.0/deploy/helm/trino-operator/crds/crds.yaml From 727655192993c9af42da2e2f25a06a8718e0f654 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 09:59:14 +0200 Subject: [PATCH 13/54] docs: Add initial set of known issues --- modules/ROOT/partials/release-notes/release-25.7.adoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index f40189317..4b3fabf5c 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -359,3 +359,12 @@ helm install --wait zookeeper-operator oci://oci.stackable.tech/sdp-charts/zooke ---- ==== Known issues + +* During the upgrade of the secret-operator CRD, special care must be taken because the CRD introduces the new TrustStore custom resource. + It is recommended to apply it instead of replacing it: `kubectl apply -f ...`. +* There are strong indicators that TLS CA handling is broken in products using `keytool`. + As of now, we confirmed that Trino has an issue in properly handling CA rotations due to a limitation with `keytool`. + The report in https://github.com/stackabletech/issues/issues/744[issues#744] outlines a workaround which requires the following steps: + . Identify the secretClass used by the secret-operator to distribute CAs. + . Delete this secret causing secret-operator to re-create a new CA secret. + . Restart all stackable pods manually. From d97716975878244f272e8b27d1e1e1dd6d0d779c Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 10:49:01 +0200 Subject: [PATCH 14/54] docs: Add breaking changes section --- .../partials/release-notes/release-25.7.adoc | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 4b3fabf5c..d14555418 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -194,17 +194,52 @@ These OpenShift versions are no longer supported: Of the changes mentioned above, the following are breaking (or could lead to breaking behaviour), and you will need to adapt your existing CRDs accordingly: -===== Stackable Operator for Example Product - -* Description of the change 1 -* Description of the change 2 +===== General -.Breaking changes details -[%collapsible] -==== -* `spec.a`: This field has been removed. -* `spec.b`: This field has been changed to a number. -==== +* As mentioned in the platform improvements section, many of our product operators now integrate with the listener-operator. + See the overall tracking issue https://github.com/stackabletech/issues/issues/692[issues#692] and https://github.com/stackabletech/issues/issues/692#issuecomment-3068662411[comment] summarizing the breaking changes. + - *Apache Airflow and Apache Superset:* + Listener classes are now declared at role level: this replaces the previous implementation where this was defined at cluster level and used a specific role service. + Metrics are not exposed via the listener service and are - as previously - only available within the cluster. + The service naming has been changed to reflect this. + See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. + - *Apache Druid:* TBD. + - *Apache HBase:*: TBD. + - *Apache Hive:* + Introduces a listener service on the `metastore` role level. + Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. + See https://github.com/stackabletech/hive-operator/issues/566[hive-operator#566]. + - *Apache NiFi:* TBD. + - *Apache Spark:* + The history server UI is now exposed using listeners instead of services created by the operator directly. + This implementation also changed the CRD by moving the `spec.clusterConfig.listenerClass` field to `spec.node.config.listenerClass`. + The new and experimental Spark connect servers in this release are also exposed using listeners in the same fashion as the history server. + See https://github.com/stackabletech/issues/issues/692#issuecomment-2847025199[issues#692 (comment)]. + - *Apache ZooKeeper:* + ListenerClasses are now defined in `spec.server.roleConfig.listenerClass`. + Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per RoleGroup (eg: `my-zk-server-default-metrics`). + A separate Headless Service is now created for each RoleGroup for internal communications (eg: access to the leader and leader-election). + See https://github.com/stackabletech/issues/issues/692#issuecomment-3034672506[issues#692 (comment)]. + - *Trino:* + Coordinator Listener classes are now declared at role level `spec.coordinators.roleConfig.listenerClass` instead of `spec.clusterConfig.listenerClass`. + Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called `---metrics`. + See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. +* The `lastUpdatedTime` was removed from the `ClusterStatus` of each operator. + Users previously relying on this value should migrate to `lastTransitionTime`. + See https://github.com/stackabletech/operator-rs/issues/1032[operator-rs#1032] and https://github.com/stackabletech/operator-rs/pull/1054[operator-rs#1054]. + +===== Container images + +The `oci.stackable.tech/sdp/kcat` image has been discontinued starting with SDP 25.7.0. +Use `oci.stackable.tech/sdp/kafka-testing-tools` instead, which includes kcat. +Existing `oci.stackable.tech/sdp/kcat` images for previous SDP releases remain available. +See https://github.com/stackabletech/docker-images/pull/1124[docker-images#1124]. + +===== Stackable Operator for Apache Kafka + +The `-nodeport` discovery ConfigMap has been deprecated in 25.3.0 and is removed as of this release. +Use the primary discovery ConfigMap instead. +See https://github.com/stackabletech/kafka-operator/pull/868[kafka-operator#868]. ==== Upgrade from 25.3 From 536b0c92f380e9c0dd4a5b48333956dbb6fc8581 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 10:49:20 +0200 Subject: [PATCH 15/54] chore(template): Adjust punctuation --- modules/ROOT/partials/release-notes/release-template.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-template.adoc b/modules/ROOT/partials/release-notes/release-template.adoc index a4a5949aa..200ca0711 100644 --- a/modules/ROOT/partials/release-notes/release-template.adoc +++ b/modules/ROOT/partials/release-notes/release-template.adoc @@ -161,7 +161,8 @@ release "commons-operator" uninstalled ---- Afterward you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. -The reason for this is that helm will uninstall the operators but not the CRDs. This can be done using `kubectl replace`: +The reason for this is that helm will uninstall the operators but not the CRDs. +This can be done using `kubectl replace`. [source] ---- From fe83f2b091d998d4d2eee1b4111009b50b61c456 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 12:06:20 +0200 Subject: [PATCH 16/54] docs: Add Airflow improvement --- modules/ROOT/partials/release-notes/release-25.7.adoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index d14555418..7e9f8d927 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -44,10 +44,11 @@ Additionally see the official xref:concepts:observability/telemetry.adoc[telemet ===== Apache Airflow -The recent release of Airflow 3 brings with it a raft of changes - with both new features and significant changes to architecture. -SDP offers early experimental support for Airflow 3.0.1 as of SDP 25.7. -This gives users the chance to use the new UI, to https://airflow.apache.org/docs/apache-airflow/3.0.1/installation/upgrading_to_airflow3.html#step-3-dag-authors-check-your-airflow-dags-for-compatibility[update] their DAGs and to test the new OPA authenticator. -See https://github.com/stackabletech/docker-images/issues/1074[docker-images#1074]. +* The recent release of Airflow 3 brings with it a raft of changes - with both new features and significant changes to architecture. + SDP offers early experimental support for Airflow 3.0.1 as of SDP 25.7. + This gives users the chance to use the new UI, to https://airflow.apache.org/docs/apache-airflow/3.0.1/installation/upgrading_to_airflow3.html#step-3-dag-authors-check-your-airflow-dags-for-compatibility[update] their DAGs and to test the new OPA authenticator. + See https://github.com/stackabletech/docker-images/issues/1074[docker-images#1074]. +* Set the default `airflow.task` log level to `INFO` to not spam the Airflow UI with task logs. ===== Apache NiFi From e27d3b2f79824143863a5d76496582bd733f1330 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 12:07:15 +0200 Subject: [PATCH 17/54] docs: Combine breaking changes, deprecations and removals --- .../partials/release-notes/release-25.7.adoc | 149 +++++++++++------- 1 file changed, 90 insertions(+), 59 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 7e9f8d927..6ec141249 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -83,9 +83,38 @@ The Stackable Listener Operator can now use custom xref:listener-operator:listen * All products that are built from source in SDP, which is at the time of this release everything except Airflow and Superset, now have a version suffix to indicate they include custom modifications made by Stackable. The suffix the same as for SDP container images, so Zookeeper 3.9.3 in SDP 25.7.0 will for example report `3.9.3-stackable25.7.0` as its version. See https://github.com/stackabletech/docker-images/issues/1068[docker-images#1068]. -* The Stackable operators for Apache Airflow, Apache Druid, Apache HBase, Apache Hadoop, Apache Hive, Apache NiFi, Apache Spark, Apache ZooKeeper, and Trino are now integrated with the Stackable listener operator. - This is a breaking change across the board. - Consult the breaking changes section for more details. +* *Breaking:* The Stackable operators for Apache Airflow, Apache Druid, Apache HBase, Apache Hadoop, Apache Hive, Apache NiFi, Apache Spark, Apache ZooKeeper, and Trino are now integrated with the Stackable listener operator. + See the overall tracking issue https://github.com/stackabletech/issues/issues/692[issues#692] and https://github.com/stackabletech/issues/issues/692#issuecomment-3068662411[comment] summarizing the breaking changes. +** *Apache Airflow and Apache Superset:* +Listener classes are now declared at role level: this replaces the previous implementation where this was defined at cluster level and used a specific role service. +Metrics are not exposed via the listener service and are - as previously - only available within the cluster. +The service naming has been changed to reflect this. +See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. +** *Apache Druid:* TBD. +** *Apache HBase:*: TBD. +** *Apache Hive:* + Introduces a listener service on the `metastore` role level. + Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. + See https://github.com/stackabletech/hive-operator/issues/566[hive-operator#566]. +** *Apache NiFi:* TBD. +** *Apache Spark:* + The history server UI is now exposed using listeners instead of services created by the operator directly. + This implementation also changed the CRD by moving the `spec.clusterConfig.listenerClass` field to `spec.node.config.listenerClass`. + The new and experimental Spark connect servers in this release are also exposed using listeners in the same fashion as the history server. + See https://github.com/stackabletech/issues/issues/692#issuecomment-2847025199[issues#692 (comment)]. +** *Apache ZooKeeper:* + ListenerClasses are now defined in `spec.server.roleConfig.listenerClass`. + Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per RoleGroup (eg: `my-zk-server-default-metrics`). + A separate Headless Service is now created for each RoleGroup for internal communications (eg: access to the leader and leader-election). + See https://github.com/stackabletech/issues/issues/692#issuecomment-3034672506[issues#692 (comment)]. +** *Trino:* + Coordinator Listener classes are now declared at role level `spec.coordinators.roleConfig.listenerClass` instead of `spec.clusterConfig.listenerClass`. + Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called`---metrics`. + See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. +* Our containers include a tool which regularly logs relevant system information to aid in debugging. + This was introduced in SDP 25.3.0 and was running every minute. + The signal to noise ratio was off which is why we are switching to run this every 30min in this release. + See https://github.com/stackabletech/docker-images/pull/1128[docker-images#1128]. ===== Vulnerabilities @@ -94,8 +123,10 @@ This includes 3 critical and 97 high-severity CVEs. ===== Apache Hadoop -Various setting defaults have been updated for better performance and reliability. -See https://github.com/stackabletech/hdfs-operator/issues/685[hdfs-operator#685]. +* Various setting defaults have been updated for better performance and reliability. + See https://github.com/stackabletech/hdfs-operator/issues/685[hdfs-operator#685]. +* The operator now defaults `dfs.encrypt.data.transfer.cipher.suite` to `AES/CTR/NoPadding` to improve security and performance. + See https://github.com/stackabletech/hdfs-operator/pull/693[hdfs-operator#693]. ===== Apache NiFi @@ -106,12 +137,63 @@ See https://github.com/stackabletech/hdfs-operator/issues/685[hdfs-operator#685] ===== Apache ZooKeeper -The image for Apache ZooKeeper backports various 3.9.4 changes into the supported 3.9.3 version. -See https://github.com/stackabletech/docker-images/issues/1143[docker-images#1143]. +* The image for Apache ZooKeeper backports various 3.9.4 changes into the supported 3.9.3 version. + See https://github.com/stackabletech/docker-images/issues/1143[docker-images#1143]. +* Prometheus support was added using the built-in Prometheus provider in Apache ZooKeeper. + This appends another port (native-metrics). + To access the metrics this port in combination with the `/metrics` endpoint can be used. + See https://github.com/stackabletech/zookeeper-operator/pull/955[zookeeper-operator#955]. ==== Platform deprecations -==== Product versions +===== Apache Hadoop + +The built-in Prometheus servlet is now enabled by default and metrics are available under the `/prom` path of all UI services. +The metrics exposed by the JMX exporter are now considered deprecated and will be removed in a future release. +See https://github.com/stackabletech/hdfs-operator/pull/695[hdfs-operator#695]. + +===== Apache Spark + +This release enables the built in Prometheus servlet for metric scraping. +The existing JMX exporter is still enabled but it is deprecated and it will be removed in a future release. +See https://github.com/stackabletech/spark-k8s-operator/pull/584[spark-k8s-operator#584]. + +==== Platform removals + +===== General + +* Remove CSI registration path migration job (init-container) from secret-operator DaemonSet. + See https://github.com/stackabletech/secret-operator/pull/610[secret-operator#610]. +* This release removes support for previously deprecated product service accounts. + Deprecated service accounts named `-serviceaccount` are ignored. + Instead, product workloads use service accounts named `-serviceaccount` where `resource_name` is the stacklet (or cluster) name as defined in the cluster resource. + See https://github.com/stackabletech/operator-rs/pull/1060[operator-rs#1060]. +* *Breaking:* + The `lastUpdatedTime` was removed from the `ClusterStatus` of each operator. + Users previously relying on this value should migrate to `lastTransitionTime`. + See https://github.com/stackabletech/operator-rs/issues/1032[operator-rs#1032] and https://github.com/stackabletech/operator-rs/pull/1054[operator-rs#1054]. + +===== Container images + +*Breaking:* +The `oci.stackable.tech/sdp/kcat` image has been discontinued starting with SDP 25.7.0. +Use `oci.stackable.tech/sdp/kafka-testing-tools` instead, which includes kcat. +Existing `oci.stackable.tech/sdp/kcat` images for previous SDP releases remain available. +See https://github.com/stackabletech/docker-images/pull/1124[docker-images#1124]. + +===== Apache HBase + +Starting with this release, support for HBase 2.4.x has been removed. +This includes metrics exposed via the JMX exporter. +All supported HBase versions expose metrics via the built in Prometheus servlet. +See https://github.com/stackabletech/hbase-operator/pull/672[hbase-operator#672]. + +===== Apache ZooKeeper + +*Breaking:* +The `-nodeport` discovery ConfigMap has been deprecated in 25.3.0 and is removed as of this release. +Use the primary discovery ConfigMap instead. +See https://github.com/stackabletech/kafka-operator/pull/868[kafka-operator#868]. As with previous SDP releases, many product images have been updated to their latest versions. @@ -191,57 +273,6 @@ These OpenShift versions are no longer supported: * `4.15` * `4.14` -==== Breaking changes - -Of the changes mentioned above, the following are breaking (or could lead to breaking behaviour), and you will need to adapt your existing CRDs accordingly: - -===== General - -* As mentioned in the platform improvements section, many of our product operators now integrate with the listener-operator. - See the overall tracking issue https://github.com/stackabletech/issues/issues/692[issues#692] and https://github.com/stackabletech/issues/issues/692#issuecomment-3068662411[comment] summarizing the breaking changes. - - *Apache Airflow and Apache Superset:* - Listener classes are now declared at role level: this replaces the previous implementation where this was defined at cluster level and used a specific role service. - Metrics are not exposed via the listener service and are - as previously - only available within the cluster. - The service naming has been changed to reflect this. - See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. - - *Apache Druid:* TBD. - - *Apache HBase:*: TBD. - - *Apache Hive:* - Introduces a listener service on the `metastore` role level. - Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. - See https://github.com/stackabletech/hive-operator/issues/566[hive-operator#566]. - - *Apache NiFi:* TBD. - - *Apache Spark:* - The history server UI is now exposed using listeners instead of services created by the operator directly. - This implementation also changed the CRD by moving the `spec.clusterConfig.listenerClass` field to `spec.node.config.listenerClass`. - The new and experimental Spark connect servers in this release are also exposed using listeners in the same fashion as the history server. - See https://github.com/stackabletech/issues/issues/692#issuecomment-2847025199[issues#692 (comment)]. - - *Apache ZooKeeper:* - ListenerClasses are now defined in `spec.server.roleConfig.listenerClass`. - Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per RoleGroup (eg: `my-zk-server-default-metrics`). - A separate Headless Service is now created for each RoleGroup for internal communications (eg: access to the leader and leader-election). - See https://github.com/stackabletech/issues/issues/692#issuecomment-3034672506[issues#692 (comment)]. - - *Trino:* - Coordinator Listener classes are now declared at role level `spec.coordinators.roleConfig.listenerClass` instead of `spec.clusterConfig.listenerClass`. - Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called `---metrics`. - See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. -* The `lastUpdatedTime` was removed from the `ClusterStatus` of each operator. - Users previously relying on this value should migrate to `lastTransitionTime`. - See https://github.com/stackabletech/operator-rs/issues/1032[operator-rs#1032] and https://github.com/stackabletech/operator-rs/pull/1054[operator-rs#1054]. - -===== Container images - -The `oci.stackable.tech/sdp/kcat` image has been discontinued starting with SDP 25.7.0. -Use `oci.stackable.tech/sdp/kafka-testing-tools` instead, which includes kcat. -Existing `oci.stackable.tech/sdp/kcat` images for previous SDP releases remain available. -See https://github.com/stackabletech/docker-images/pull/1124[docker-images#1124]. - -===== Stackable Operator for Apache Kafka - -The `-nodeport` discovery ConfigMap has been deprecated in 25.3.0 and is removed as of this release. -Use the primary discovery ConfigMap instead. -See https://github.com/stackabletech/kafka-operator/pull/868[kafka-operator#868]. - ==== Upgrade from 25.3 ===== Using stackablectl From a868972d28598097575ca472b1fc2b8a3ec8779e Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 12:08:00 +0200 Subject: [PATCH 18/54] docs: Group supported product, Kubernetes and OpenShift versions --- .../partials/release-notes/release-25.7.adoc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 6ec141249..2857662c9 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -195,24 +195,26 @@ The `-nodeport` discovery ConfigMap has been deprecated in 25.3.0 and is removed Use the primary discovery ConfigMap instead. See https://github.com/stackabletech/kafka-operator/pull/868[kafka-operator#868]. -As with previous SDP releases, many product images have been updated to their latest versions. +==== Supported versions + +===== Product versions +As with previous SDP releases, many product images have been updated to their latest versions. Refer to the xref:operators:supported_versions.adoc[supported versions] documentation for a complete overview including LTS versions or deprecations. -===== New versions +====== New versions The following new product versions are now supported: * Apache Airflow: https://github.com/stackabletech/docker-images/issues/1074[2.10.5], https://github.com/stackabletech/docker-images/issues/1074[3.0.1 (experimental)] * Apache Druid: https://github.com/stackabletech/docker-images/issues/1075[33.0.0] * Apache HBase: https://github.com/stackabletech/docker-images/issues/1076[2.6.2] -* Apache Kafka: https://github.com/stackabletech/docker-images/issues/1078[4.0.0 (experimental)^1^] * Apache NiFi: https://github.com/stackabletech/docker-images/issues/1079[2.4.0] * Apache Superset: https://github.com/stackabletech/docker-images/issues/1081[4.1.2] * Open Policy Agent: https://github.com/stackabletech/docker-images/issues/1084[1.4.2] * Trino: https://github.com/stackabletech/docker-images/issues/1082[476] -===== Deprecated versions +====== Deprecated versions The following product versions are deprecated and will be removed in a later release: @@ -223,7 +225,7 @@ The following product versions are deprecated and will be removed in a later rel * Open Policy Agent: https://github.com/stackabletech/docker-images/issues/1084[1.0.1] * Trino: https://github.com/stackabletech/docker-images/issues/1082[470] -===== Removed versions +====== Removed versions The following product versions are no longer supported. These images for released product versions remain available https://oci.stackable.tech/[here,window=_blank] @@ -231,7 +233,7 @@ Information on how to browse the registry can be found xref:contributor:project- * Apache Druid: https://github.com/stackabletech/docker-images/issues/1075[30.0.0] * Apache Hbase: https://github.com/stackabletech/docker-images/issues/1076[2.4.18] -* Apache Hadoop: https://github.com/stackabletech/docker-images/issues/1077[3.3.4], https://github.com/stackabletech/docker-images/issues/1077[3.4.0], https://github.com/stackabletech/docker-images/issues/1077[3.3.6^2^] +* Apache Hadoop: https://github.com/stackabletech/docker-images/issues/1077[3.3.4], https://github.com/stackabletech/docker-images/issues/1077[3.4.0], https://github.com/stackabletech/docker-images/issues/1077[3.3.6^1^] * Apache Kafka: https://github.com/stackabletech/docker-images/issues/1078[3.7.1], https://github.com/stackabletech/docker-images/issues/1078[3.8.0] * Apache NiFi: https://github.com/stackabletech/docker-images/issues/1079[2.2.0 (experimental)] * Apache Spark: https://github.com/stackabletech/docker-images/issues/1080[3.5.2] @@ -243,11 +245,10 @@ Information on how to browse the registry can be found xref:contributor:project- [.small] -- -. Unused in SDP, see https://github.com/stackabletech/kafka-operator/issues/690[kafka-operator#690] and https://github.com/stackabletech/docker-images/pull/1117[docker-images#1117] . Remove support for 3.3.6, it is still required by HBase, Hive, and Spark in this release. -- -==== Supported Kubernetes versions +===== Kubernetes versions This release supports the following Kubernetes versions: @@ -260,7 +261,7 @@ These Kubernetes versions are no longer supported: * `1.29` -==== Supported OpenShift versions +===== OpenShift versions This release is available in the RedHat Certified Operator Catalog for the following OpenShift versions: From fcff2a348289001187879c8d8ad36d95f7375cfb Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 12:08:29 +0200 Subject: [PATCH 19/54] docs: Add note about stackablectl --- modules/ROOT/partials/release-notes/release-25.7.adoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 2857662c9..fe50d7346 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -13,6 +13,15 @@ Released on YYYY-MM-DD. The SDP 25.7.0 release is the first release which will **only** be published on `oci.stackable.tech`. ==== +[NOTE] +==== +In previous releases, releases of the `stackablectl` CLI tool were synced with platform releases. +The tool itself was never tied to a particular version of SDP. +As such, it was decided to version `stackablectl` independently. +// TODO (@Techassi): Actually create and link that page +Starting with this release, the release notes for `stackablectl` are located on a separate page. +==== + ==== New platform features ===== General From 560a3258c9bf3d8a0a20093aa2aa122e4ba7fcf7 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 12:08:50 +0200 Subject: [PATCH 20/54] chore: Add comment about spelling of core operators --- modules/ROOT/partials/release-notes/release-25.7.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index fe50d7346..de1e21f59 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -81,6 +81,9 @@ The Spark operator watches for SparkConnectServer custom resources. Preliminary documentation is xref:spark-k8s:usage-guide/spark-connect.adoc[available] and the existing xref:demos:jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc[Taxi Data Anomaly Detection demo] has been retrofitted to use a JupterLab client running against a Spark Connect server. Also see https://github.com/stackabletech/spark-k8s-operator/issues/284[spark-k8s-operator#284]. +// TODO (@Techassi): Define a single way of spelling internal operators, like listener, secret, and commons. +// Currently, throughout this page variations like Stackable Listener Operator, Listener Operator, listener operator, +// listener-operator are used. Only ONE must be used. ===== Stackable listener operator The Stackable Listener Operator can now use custom xref:listener-operator:listenerclass.adoc#servicetype-loadbalancer-class[LoadBalancer classes] and https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation[disable NodePort allocation]. From 0489fce742d22a98780e6191d56453e519238955 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 12:27:57 +0200 Subject: [PATCH 21/54] docs: Add a few missing changes --- .../partials/release-notes/release-25.7.adoc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index de1e21f59..23427feaa 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -28,8 +28,11 @@ Starting with this release, the release notes for `stackablectl` are located on ====== Security -Users (and services) can now use the new TrustStore resource to request the trust root certificates associated with a SecretClass. -See the xref:secret-operator:truststore.adoc[TrustStore documentation] and https://github.com/stackabletech/secret-operator/issues/410[secret-operator#410]. +* Users (and services) can now use the new TrustStore resource to request the trust root certificates associated with a SecretClass. + See the xref:secret-operator:truststore.adoc[TrustStore documentation] and https://github.com/stackabletech/secret-operator/issues/410[secret-operator#410]. +* Add support for format-specific annotations to override secret file names. + Names can be customized using secret volume annotations which are listed xref:secret-operator:volume.adoc[in our documentation]. + See https://github.com/stackabletech/secret-operator/pull/572[secret-operator#572]. ====== Automatic cluster domain detection @@ -48,9 +51,15 @@ See https://github.com/stackabletech/issues/issues/662[issues#662]. * All operators use unified CLI arguments and environment variables to configure the exporters. * All operators allow exporter configuration via Helm values. * All exporters are disabled by default, except for console logs, which remain enabled by default. +* Customization via Helm values is supported for all exporters. Additionally see the official xref:concepts:observability/telemetry.adoc[telemetry documentation page] and the tracking issue https://github.com/stackabletech/issues/issues/639[issues#639] for further details. +====== AuthZ + +The user-info-fetcher backend now supports fetching user groups from Microsoft Entra (experimental). +See https://github.com/stackabletech/opa-operator/pull/712[opa-operator#712]. + ===== Apache Airflow * The recent release of Airflow 3 brings with it a raft of changes - with both new features and significant changes to architecture. @@ -58,6 +67,7 @@ Additionally see the official xref:concepts:observability/telemetry.adoc[telemet This gives users the chance to use the new UI, to https://airflow.apache.org/docs/apache-airflow/3.0.1/installation/upgrading_to_airflow3.html#step-3-dag-authors-check-your-airflow-dags-for-compatibility[update] their DAGs and to test the new OPA authenticator. See https://github.com/stackabletech/docker-images/issues/1074[docker-images#1074]. * Set the default `airflow.task` log level to `INFO` to not spam the Airflow UI with task logs. + See https://github.com/stackabletech/airflow-operator/pull/649[airflow-operator#649]. ===== Apache NiFi @@ -146,6 +156,8 @@ This includes 3 critical and 97 high-severity CVEs. See the official https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#property-encryption-algorithms[NiFi documentation] and https://github.com/stackabletech/nifi-operator/issues/791[nifi-operator#791]. * Re-add Iceberg support to NiFi 2.4.0, which was removed upstream starting with version 2.0.0. See xref:nifi:usage_guide/writing-to-iceberg-tables.adoc[NiFi Iceberg documentation] and https://github.com/stackabletech/nifi-operator/issues/738[nifi-operator#738]. +* Update patch that allows bypassing the host header validation starting with NiFi 2.4.0. + See https://github.com/stackabletech/docker-images/pull/1125[docker-images#1125]. ===== Apache ZooKeeper From baf5e1066bfd89dc93edbae88b319b596040c439 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 16:34:29 +0200 Subject: [PATCH 22/54] docs: Add HBase listener changes --- modules/ROOT/partials/release-notes/release-25.7.adoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 23427feaa..b1f1e11fc 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -113,7 +113,10 @@ Metrics are not exposed via the listener service and are - as previously - only The service naming has been changed to reflect this. See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. ** *Apache Druid:* TBD. -** *Apache HBase:*: TBD. +** *Apache HBase:*: + Coordinator Listener classes are now declared at role level `spec..config.listenerClass` instead of `spec.clusterConfig.listenerClass`. + HBase has been patched so that the URLs in the UI screens now use listener endpoints (which are externally reachable as long as the appropriate listenerClass has been selected). + See xref:hbase:usage-guide/listenerclass.adoc[HBase ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3089441868[docker-images#692 (comment)]. ** *Apache Hive:* Introduces a listener service on the `metastore` role level. Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. From ad89318a67d488534b88d0a3872320b747d90d1e Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 18 Jul 2025 16:34:50 +0200 Subject: [PATCH 23/54] chore: Fix indentation in list --- .../partials/release-notes/release-25.7.adoc | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index b1f1e11fc..7bcb3fd7e 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -108,34 +108,34 @@ The Stackable Listener Operator can now use custom xref:listener-operator:listen * *Breaking:* The Stackable operators for Apache Airflow, Apache Druid, Apache HBase, Apache Hadoop, Apache Hive, Apache NiFi, Apache Spark, Apache ZooKeeper, and Trino are now integrated with the Stackable listener operator. See the overall tracking issue https://github.com/stackabletech/issues/issues/692[issues#692] and https://github.com/stackabletech/issues/issues/692#issuecomment-3068662411[comment] summarizing the breaking changes. ** *Apache Airflow and Apache Superset:* -Listener classes are now declared at role level: this replaces the previous implementation where this was defined at cluster level and used a specific role service. -Metrics are not exposed via the listener service and are - as previously - only available within the cluster. -The service naming has been changed to reflect this. -See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. + Listener classes are now declared at role level: this replaces the previous implementation where this was defined at cluster level and used a specific role service. + Metrics are not exposed via the listener service and are - as previously - only available within the cluster. + The service naming has been changed to reflect this. + See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. ** *Apache Druid:* TBD. ** *Apache HBase:*: Coordinator Listener classes are now declared at role level `spec..config.listenerClass` instead of `spec.clusterConfig.listenerClass`. HBase has been patched so that the URLs in the UI screens now use listener endpoints (which are externally reachable as long as the appropriate listenerClass has been selected). See xref:hbase:usage-guide/listenerclass.adoc[HBase ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3089441868[docker-images#692 (comment)]. ** *Apache Hive:* - Introduces a listener service on the `metastore` role level. - Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. - See https://github.com/stackabletech/hive-operator/issues/566[hive-operator#566]. + Introduces a listener service on the `metastore` role level. + Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. + See https://github.com/stackabletech/hive-operator/issues/566[hive-operator#566]. ** *Apache NiFi:* TBD. ** *Apache Spark:* - The history server UI is now exposed using listeners instead of services created by the operator directly. - This implementation also changed the CRD by moving the `spec.clusterConfig.listenerClass` field to `spec.node.config.listenerClass`. - The new and experimental Spark connect servers in this release are also exposed using listeners in the same fashion as the history server. - See https://github.com/stackabletech/issues/issues/692#issuecomment-2847025199[issues#692 (comment)]. + The history server UI is now exposed using listeners instead of services created by the operator directly. + This implementation also changed the CRD by moving the `spec.clusterConfig.listenerClass` field to `spec.node.config.listenerClass`. + The new and experimental Spark connect servers in this release are also exposed using listeners in the same fashion as the history server. + See https://github.com/stackabletech/issues/issues/692#issuecomment-2847025199[issues#692 (comment)]. ** *Apache ZooKeeper:* - ListenerClasses are now defined in `spec.server.roleConfig.listenerClass`. - Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per RoleGroup (eg: `my-zk-server-default-metrics`). - A separate Headless Service is now created for each RoleGroup for internal communications (eg: access to the leader and leader-election). - See https://github.com/stackabletech/issues/issues/692#issuecomment-3034672506[issues#692 (comment)]. + ListenerClasses are now defined in `spec.server.roleConfig.listenerClass`. + Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per RoleGroup (eg: `my-zk-server-default-metrics`). + A separate Headless Service is now created for each RoleGroup for internal communications (eg: access to the leader and leader-election). + See https://github.com/stackabletech/issues/issues/692#issuecomment-3034672506[issues#692 (comment)]. ** *Trino:* - Coordinator Listener classes are now declared at role level `spec.coordinators.roleConfig.listenerClass` instead of `spec.clusterConfig.listenerClass`. - Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called`---metrics`. - See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. + Coordinator Listener classes are now declared at role level `spec.coordinators.roleConfig.listenerClass` instead of `spec.clusterConfig.listenerClass`. + Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called `---metrics`. + See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. * Our containers include a tool which regularly logs relevant system information to aid in debugging. This was introduced in SDP 25.3.0 and was running every minute. The signal to noise ratio was off which is why we are switching to run this every 30min in this release. From 601321ceb1719bd39611464d2fa69f1c6f3d0888 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 21 Jul 2025 11:26:56 +0200 Subject: [PATCH 24/54] docs: Add RBAC permission change --- modules/ROOT/partials/release-notes/release-25.7.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 7bcb3fd7e..708d0ddf2 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -60,6 +60,11 @@ Additionally see the official xref:concepts:observability/telemetry.adoc[telemet The user-info-fetcher backend now supports fetching user groups from Microsoft Entra (experimental). See https://github.com/stackabletech/opa-operator/pull/712[opa-operator#712]. +===== Miscellaneous + +The operator Helm charts now grant RBAC `patch` permissions on `events.k8s.io/events`, so events can be aggregated (e.g. "error happened 10 times over the last 5 minutes"). +See https://github.com/stackabletech/issues/issues/748[issues#748]. + ===== Apache Airflow * The recent release of Airflow 3 brings with it a raft of changes - with both new features and significant changes to architecture. From 52968c15380cf8e655055dc84352aa7891497d6e Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 21 Jul 2025 11:54:00 +0200 Subject: [PATCH 25/54] noop From 22a2960a33c6ebf3e23c4a160af086f98aae6488 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 21 Jul 2025 12:34:21 +0200 Subject: [PATCH 26/54] fix(docs): Use correct indentation --- modules/ROOT/partials/release-notes/release-25.7.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 708d0ddf2..9dd33b2bb 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -60,7 +60,7 @@ Additionally see the official xref:concepts:observability/telemetry.adoc[telemet The user-info-fetcher backend now supports fetching user groups from Microsoft Entra (experimental). See https://github.com/stackabletech/opa-operator/pull/712[opa-operator#712]. -===== Miscellaneous +====== Miscellaneous The operator Helm charts now grant RBAC `patch` permissions on `events.k8s.io/events`, so events can be aggregated (e.g. "error happened 10 times over the last 5 minutes"). See https://github.com/stackabletech/issues/issues/748[issues#748]. From 2b236b48b6d10035c41a4a95be4c74d52e9ad3cc Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 21 Jul 2025 13:03:31 +0200 Subject: [PATCH 27/54] chore: Remove superfluous list item --- modules/ROOT/partials/release-notes/release-25.7.adoc | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 9dd33b2bb..5ea81ca6f 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -51,7 +51,6 @@ See https://github.com/stackabletech/issues/issues/662[issues#662]. * All operators use unified CLI arguments and environment variables to configure the exporters. * All operators allow exporter configuration via Helm values. * All exporters are disabled by default, except for console logs, which remain enabled by default. -* Customization via Helm values is supported for all exporters. Additionally see the official xref:concepts:observability/telemetry.adoc[telemetry documentation page] and the tracking issue https://github.com/stackabletech/issues/issues/639[issues#639] for further details. From bb2ca65908c2a3697a886757a9a3e0c14a0915dc Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 22 Jul 2025 10:43:54 +0200 Subject: [PATCH 28/54] chore: Restructure platform improvement sections --- .../partials/release-notes/release-25.7.adoc | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 5ea81ca6f..508171f05 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -106,40 +106,46 @@ The Stackable Listener Operator can now use custom xref:listener-operator:listen ===== General +====== Listener integration + +*Breaking:* The Stackable operators for Apache Airflow, Apache Druid, Apache HBase, Apache Hadoop, Apache Hive, Apache NiFi, Apache Spark, Apache ZooKeeper, and Trino are now integrated with the Stackable listener operator. +See the overall tracking issue https://github.com/stackabletech/issues/issues/692[issues#692] and https://github.com/stackabletech/issues/issues/692#issuecomment-3068662411[comment] summarizing the breaking changes. + +* *Apache Airflow and Apache Superset:* + Listener classes are now declared at role level: this replaces the previous implementation where this was defined at cluster level and used a specific role service. + Metrics are not exposed via the listener service and are - as previously - only available within the cluster. + The service naming has been changed to reflect this. + See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. +* *Apache Druid:* TBD. +* *Apache HBase:*: + Coordinator Listener classes are now declared at role level `spec..config.listenerClass` instead of `spec.clusterConfig.listenerClass`. + HBase has been patched so that the URLs in the UI screens now use listener endpoints (which are externally reachable as long as the appropriate listenerClass has been selected). + See xref:hbase:usage-guide/listenerclass.adoc[HBase ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3089441868[docker-images#692 (comment)]. +* *Apache Hive:* + Introduces a listener service on the `metastore` role level. + Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. + See https://github.com/stackabletech/hive-operator/issues/566[hive-operator#566]. +* *Apache NiFi:* TBD. +* *Apache Spark:* + The history server UI is now exposed using listeners instead of services created by the operator directly. + This implementation also changed the CRD by moving the `spec.clusterConfig.listenerClass` field to `spec.node.config.listenerClass`. + The new and experimental Spark connect servers in this release are also exposed using listeners in the same fashion as the history server. + See https://github.com/stackabletech/issues/issues/692#issuecomment-2847025199[issues#692 (comment)]. +* *Apache ZooKeeper:* + ListenerClasses are now defined in `spec.server.roleConfig.listenerClass`. + Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per RoleGroup (eg: `my-zk-server-default-metrics`). + A separate Headless Service is now created for each RoleGroup for internal communications (eg: access to the leader and leader-election). + See https://github.com/stackabletech/issues/issues/692#issuecomment-3034672506[issues#692 (comment)]. +* *Trino:* + Coordinator Listener classes are now declared at role level `spec.coordinators.roleConfig.listenerClass` instead of `spec.clusterConfig.listenerClass`. + Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called `---metrics`. + See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. + +====== Miscellaneous + * All products that are built from source in SDP, which is at the time of this release everything except Airflow and Superset, now have a version suffix to indicate they include custom modifications made by Stackable. The suffix the same as for SDP container images, so Zookeeper 3.9.3 in SDP 25.7.0 will for example report `3.9.3-stackable25.7.0` as its version. See https://github.com/stackabletech/docker-images/issues/1068[docker-images#1068]. -* *Breaking:* The Stackable operators for Apache Airflow, Apache Druid, Apache HBase, Apache Hadoop, Apache Hive, Apache NiFi, Apache Spark, Apache ZooKeeper, and Trino are now integrated with the Stackable listener operator. - See the overall tracking issue https://github.com/stackabletech/issues/issues/692[issues#692] and https://github.com/stackabletech/issues/issues/692#issuecomment-3068662411[comment] summarizing the breaking changes. -** *Apache Airflow and Apache Superset:* - Listener classes are now declared at role level: this replaces the previous implementation where this was defined at cluster level and used a specific role service. - Metrics are not exposed via the listener service and are - as previously - only available within the cluster. - The service naming has been changed to reflect this. - See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. -** *Apache Druid:* TBD. -** *Apache HBase:*: - Coordinator Listener classes are now declared at role level `spec..config.listenerClass` instead of `spec.clusterConfig.listenerClass`. - HBase has been patched so that the URLs in the UI screens now use listener endpoints (which are externally reachable as long as the appropriate listenerClass has been selected). - See xref:hbase:usage-guide/listenerclass.adoc[HBase ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3089441868[docker-images#692 (comment)]. -** *Apache Hive:* - Introduces a listener service on the `metastore` role level. - Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. - See https://github.com/stackabletech/hive-operator/issues/566[hive-operator#566]. -** *Apache NiFi:* TBD. -** *Apache Spark:* - The history server UI is now exposed using listeners instead of services created by the operator directly. - This implementation also changed the CRD by moving the `spec.clusterConfig.listenerClass` field to `spec.node.config.listenerClass`. - The new and experimental Spark connect servers in this release are also exposed using listeners in the same fashion as the history server. - See https://github.com/stackabletech/issues/issues/692#issuecomment-2847025199[issues#692 (comment)]. -** *Apache ZooKeeper:* - ListenerClasses are now defined in `spec.server.roleConfig.listenerClass`. - Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per RoleGroup (eg: `my-zk-server-default-metrics`). - A separate Headless Service is now created for each RoleGroup for internal communications (eg: access to the leader and leader-election). - See https://github.com/stackabletech/issues/issues/692#issuecomment-3034672506[issues#692 (comment)]. -** *Trino:* - Coordinator Listener classes are now declared at role level `spec.coordinators.roleConfig.listenerClass` instead of `spec.clusterConfig.listenerClass`. - Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called `---metrics`. - See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. * Our containers include a tool which regularly logs relevant system information to aid in debugging. This was introduced in SDP 25.3.0 and was running every minute. The signal to noise ratio was off which is why we are switching to run this every 30min in this release. From 3927d1d681a85213bb6306231158bab856c032b1 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 22 Jul 2025 10:44:47 +0200 Subject: [PATCH 29/54] docs: Mention included source code in images --- modules/ROOT/partials/release-notes/release-25.7.adoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 508171f05..5073ca216 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -146,6 +146,10 @@ See the overall tracking issue https://github.com/stackabletech/issues/issues/69 * All products that are built from source in SDP, which is at the time of this release everything except Airflow and Superset, now have a version suffix to indicate they include custom modifications made by Stackable. The suffix the same as for SDP container images, so Zookeeper 3.9.3 in SDP 25.7.0 will for example report `3.9.3-stackable25.7.0` as its version. See https://github.com/stackabletech/docker-images/issues/1068[docker-images#1068]. +* We've added source code snapshots directly into our container images to further improve transparency and the debugging experience. + Each container image now includes `.tar.gz` archives containing the exact source code used to build that specific version of the product. + They can be found within the `/stackable` directory of each container image. + See the xref:guides:viewing-and-verifying-sboms.adoc[SBOM documentation] and https://github.com/stackabletech/docker-images/issues/1069[docker-images#1069]. * Our containers include a tool which regularly logs relevant system information to aid in debugging. This was introduced in SDP 25.3.0 and was running every minute. The signal to noise ratio was off which is why we are switching to run this every 30min in this release. From c7bac8eace93702050c0f931bf214fb662982331 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 22 Jul 2025 10:45:18 +0200 Subject: [PATCH 30/54] docs: Mention user-info-fetcher AD backend filtering --- modules/ROOT/partials/release-notes/release-25.7.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 5073ca216..63c064b89 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -141,6 +141,11 @@ See the overall tracking issue https://github.com/stackabletech/issues/issues/69 Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called `---metrics`. See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. +====== AuthZ + +The user-info-fetcher AD backend now filters by both UPN and SAN fields when fetching user groups. +See the xref:opa:usage-guide/user-info-fetcher.adoc#backend-activedirectory[documentation] and https://github.com/stackabletech/opa-operator/issues/702[opa-operator#702]. + ====== Miscellaneous * All products that are built from source in SDP, which is at the time of this release everything except Airflow and Superset, now have a version suffix to indicate they include custom modifications made by Stackable. From fe83b0230ad08cd8a92035d0eaa18a87a153b0b6 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 22 Jul 2025 10:45:53 +0200 Subject: [PATCH 31/54] docs: Mention Apache Hadoop OpenSSL fixes --- modules/ROOT/partials/release-notes/release-25.7.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 63c064b89..cc88f1779 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -171,6 +171,9 @@ This includes 3 critical and 97 high-severity CVEs. See https://github.com/stackabletech/hdfs-operator/issues/685[hdfs-operator#685]. * The operator now defaults `dfs.encrypt.data.transfer.cipher.suite` to `AES/CTR/NoPadding` to improve security and performance. See https://github.com/stackabletech/hdfs-operator/pull/693[hdfs-operator#693]. +* Previously Apache Hadoop wouldn't find and use the native OpenSSL library to speed up crypto operations. + This was fixed by adding a https://issues.apache.org/jira/browse/HADOOP-12845[missing symlink] and backporting https://issues.apache.org/jira/browse/HADOOP-18583[HADOOP-18583]. + See https://github.com/stackabletech/docker-images/pull/1209[docker-images#1209]. ===== Apache NiFi From d555cccc1928f60106466e2e3c8afcb5ae7e138b Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 22 Jul 2025 12:51:22 +0200 Subject: [PATCH 32/54] docs: Add Hive 3 deprecation --- modules/ROOT/partials/release-notes/release-25.7.adoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index cc88f1779..8a9f7ac10 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -201,6 +201,10 @@ The built-in Prometheus servlet is now enabled by default and metrics are availa The metrics exposed by the JMX exporter are now considered deprecated and will be removed in a future release. See https://github.com/stackabletech/hdfs-operator/pull/695[hdfs-operator#695]. +===== Apache Hive + +Hive 3 is deprecated in this release and we plan to remove it in the upcoming release. + ===== Apache Spark This release enables the built in Prometheus servlet for metric scraping. From 93966cab3ec37c0023cc4ff01d6f5d54f53baf7f Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 22 Jul 2025 13:07:01 +0200 Subject: [PATCH 33/54] docs: Adjust Hive 3 deprecation section --- modules/ROOT/partials/release-notes/release-25.7.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 8a9f7ac10..9ae087cc1 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -203,7 +203,8 @@ See https://github.com/stackabletech/hdfs-operator/pull/695[hdfs-operator#695]. ===== Apache Hive -Hive 3 is deprecated in this release and we plan to remove it in the upcoming release. +Hive 3 has been deprecated since SDP 25.3.0 and was marked as End of Life on 2024-10-08 by the upstream project. +We might remove it in the upcoming release, the final decision will be based on metastore client compatibility and customer feedback. ===== Apache Spark From f64ce8abe65d38d48f0a08aed02b0e5ed894d166 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 23 Jul 2025 08:59:10 +0200 Subject: [PATCH 34/54] docs: Mention the addition of missing properties in Druid middleManager --- modules/ROOT/partials/release-notes/release-25.7.adoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 9ae087cc1..1b1d4b5cc 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -165,6 +165,12 @@ See the xref:opa:usage-guide/user-info-fetcher.adoc#backend-activedirectory[docu 230 CVEs were fixed in the Stackable product images. This includes 3 critical and 97 high-severity CVEs. +===== Apache Druid + +We noticed that a bunch of configuration properties were missing on the middleManager roles. +The properties are now available. +See https://github.com/stackabletech/druid-operator/pull/740[druid-operator#740]. + ===== Apache Hadoop * Various setting defaults have been updated for better performance and reliability. From bf461c03b079cd3656abd75612ac683fc116e434 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 23 Jul 2025 09:01:03 +0200 Subject: [PATCH 35/54] chore: Prepare ref to stackablectl release notes --- modules/ROOT/partials/release-notes/release-25.7.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 1b1d4b5cc..609275fe0 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -18,7 +18,7 @@ The SDP 25.7.0 release is the first release which will **only** be published on In previous releases, releases of the `stackablectl` CLI tool were synced with platform releases. The tool itself was never tied to a particular version of SDP. As such, it was decided to version `stackablectl` independently. -// TODO (@Techassi): Actually create and link that page +// Starting with this release, the release notes for `stackablectl` are located on a separate xref:management:stackablectl/release-notes.adoc[page]. Starting with this release, the release notes for `stackablectl` are located on a separate page. ==== From 5f035ad2c91926a1f3884f01c40adce2f3115bd9 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 10:27:22 +0200 Subject: [PATCH 36/54] docs: Add Druid and NiFi listener changes --- .../partials/release-notes/release-25.7.adoc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 609275fe0..37b522189 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -116,7 +116,11 @@ See the overall tracking issue https://github.com/stackabletech/issues/issues/69 Metrics are not exposed via the listener service and are - as previously - only available within the cluster. The service naming has been changed to reflect this. See https://github.com/stackabletech/issues/issues/692#issuecomment-2854266967[issues#692 (comment)]. -* *Apache Druid:* TBD. +* *Apache Druid:* + Listener classes for Brokers, Coordinators and Routers are now defined in `spec.brokers|coordinators|routers.roleConfig.listenerClass`. + Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per role group (eg: my-druid-broker-default-metrics). + A separate headless Service is now created for each role group for internal communications (e.g. between different Druid services). + See https://github.com/stackabletech/issues/issues/692#issuecomment-3106382988[issues#692 (comment)]. * *Apache HBase:*: Coordinator Listener classes are now declared at role level `spec..config.listenerClass` instead of `spec.clusterConfig.listenerClass`. HBase has been patched so that the URLs in the UI screens now use listener endpoints (which are externally reachable as long as the appropriate listenerClass has been selected). @@ -125,7 +129,11 @@ See the overall tracking issue https://github.com/stackabletech/issues/issues/69 Introduces a listener service on the `metastore` role level. Additionally, a `-headless` service for internal communications as well as a `-metrics` service for monitoring is introduced at the role group level. See https://github.com/stackabletech/hive-operator/issues/566[hive-operator#566]. -* *Apache NiFi:* TBD. +* *Apache NiFi:* + Listener classes are now defined in spec.nodes.roleConfig.listenerClass. + Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per role group (eg: my-nifi-node-default-metrics). + A separate headless Service is now created for each role group for internal communications (between NiFi nodes). + See https://github.com/stackabletech/issues/issues/692#issuecomment-3106375374[issues#692 (comment)]. * *Apache Spark:* The history server UI is now exposed using listeners instead of services created by the operator directly. This implementation also changed the CRD by moving the `spec.clusterConfig.listenerClass` field to `spec.node.config.listenerClass`. @@ -133,8 +141,8 @@ See the overall tracking issue https://github.com/stackabletech/issues/issues/69 See https://github.com/stackabletech/issues/issues/692#issuecomment-2847025199[issues#692 (comment)]. * *Apache ZooKeeper:* ListenerClasses are now defined in `spec.server.roleConfig.listenerClass`. - Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per RoleGroup (eg: `my-zk-server-default-metrics`). - A separate Headless Service is now created for each RoleGroup for internal communications (eg: access to the leader and leader-election). + Metrics are no longer exposed by the listener Service and are instead now defined on a dedicated ClusterIP service per role group (eg: `my-zk-server-default-metrics`). + A separate headless Service is now created for each role group for internal communications (eg: access to the leader and leader-election). See https://github.com/stackabletech/issues/issues/692#issuecomment-3034672506[issues#692 (comment)]. * *Trino:* Coordinator Listener classes are now declared at role level `spec.coordinators.roleConfig.listenerClass` instead of `spec.clusterConfig.listenerClass`. From 65f3317aed9922b19ad4a4952952b89000522ecc Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 10:33:17 +0200 Subject: [PATCH 37/54] chore: Apply suggestions Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- .../partials/release-notes/release-25.7.adoc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 37b522189..f3adaa2f9 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -5,7 +5,7 @@ === 25.7.0 -Released on YYYY-MM-DD. +Released on 2025-07-23. (Optional description / introduction) [NOTE] @@ -15,7 +15,7 @@ The SDP 25.7.0 release is the first release which will **only** be published on [NOTE] ==== -In previous releases, releases of the `stackablectl` CLI tool were synced with platform releases. +Previously, releases of the `stackablectl` CLI tool were synced with platform releases. The tool itself was never tied to a particular version of SDP. As such, it was decided to version `stackablectl` independently. // Starting with this release, the release notes for `stackablectl` are located on a separate xref:management:stackablectl/release-notes.adoc[page]. @@ -37,7 +37,7 @@ Starting with this release, the release notes for `stackablectl` are located on ====== Automatic cluster domain detection The default Kubernetes cluster domain name is now fetched from the kubelet API unless explicitly configured. -Operators are now required to have the RBAC permission to get nodes/proxy in the apiGroup `""`. +Operators are now required to have the RBAC permission to get `nodes/proxy`. The helm-charts takes care of this. The CLI argument `--kubernetes-node-name` or env variable `KUBERNETES_NODE_NAME` needs to be set. @@ -47,14 +47,14 @@ See https://github.com/stackabletech/issues/issues/662[issues#662]. ====== Observability -* All operators support sending OpenTelemetry logs and traces. +* All operators support sending OpenTelemetry logs and traces via OTLP. * All operators use unified CLI arguments and environment variables to configure the exporters. * All operators allow exporter configuration via Helm values. * All exporters are disabled by default, except for console logs, which remain enabled by default. Additionally see the official xref:concepts:observability/telemetry.adoc[telemetry documentation page] and the tracking issue https://github.com/stackabletech/issues/issues/639[issues#639] for further details. -====== AuthZ +====== Authorization The user-info-fetcher backend now supports fetching user groups from Microsoft Entra (experimental). See https://github.com/stackabletech/opa-operator/pull/712[opa-operator#712]. @@ -80,7 +80,7 @@ See https://github.com/stackabletech/issues/issues/748[issues#748]. * Custom NiFi Archives (NARs) and Python components can be loaded from Git repositories with git-sync. See https://github.com/stackabletech/nifi-operator/issues/739[nifi-operator#739] and documentation on xref:nifi:usage_guide/custom-components.adoc[custom components]. * The Stackable operator for Apache NiFi now supports using Kubernetes for cluster coordination and storage, rather than ZooKeeper. - (Requires NiFi 2.x, migrating existing installs is currently not supported) + (Requires NiFi 2, migrating existing installs is currently not supported) See xref:nifi:usage_guide/clustering.adoc#backend-kubernetes[NiFi backend documentation] and https://github.com/stackabletech/nifi-operator/issues/737[nifi-operator#737]. * Rolling upgrades are now supported between NiFi 2 versions. This is the default update strategy for NiFi 2 clusters. @@ -149,7 +149,7 @@ See the overall tracking issue https://github.com/stackabletech/issues/issues/69 Both Coordinators and Workers have two services at role group level, a headless service called `---headless` and a service for metrics called `---metrics`. See xref:trino:usage-guide/listenerclass.adoc[Trino ListenerClass documentation] and https://github.com/stackabletech/issues/issues/692#issuecomment-3001089387[issues#692 (comment)]. -====== AuthZ +====== Authorization The user-info-fetcher AD backend now filters by both UPN and SAN fields when fetching user groups. See the xref:opa:usage-guide/user-info-fetcher.adoc#backend-activedirectory[documentation] and https://github.com/stackabletech/opa-operator/issues/702[opa-operator#702]. @@ -191,7 +191,7 @@ See https://github.com/stackabletech/druid-operator/pull/740[druid-operator#740] ===== Apache NiFi -* The operator now emits a warning for deprecated sensitive property algorithms in NiFi version 1.x.x and errors out for version 2.x.x if an unsupported sensitive properties algorithm is used. +* The operator now emits a warning for deprecated sensitive property algorithms in NiFi version 1 and errors out for version 2 if an unsupported sensitive properties algorithm is used. See the official https://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#property-encryption-algorithms[NiFi documentation] and https://github.com/stackabletech/nifi-operator/issues/791[nifi-operator#791]. * Re-add Iceberg support to NiFi 2.4.0, which was removed upstream starting with version 2.0.0. See xref:nifi:usage_guide/writing-to-iceberg-tables.adoc[NiFi Iceberg documentation] and https://github.com/stackabletech/nifi-operator/issues/738[nifi-operator#738]. @@ -251,7 +251,7 @@ See https://github.com/stackabletech/docker-images/pull/1124[docker-images#1124] ===== Apache HBase -Starting with this release, support for HBase 2.4.x has been removed. +Starting with this release, support for HBase 2.4 has been removed. This includes metrics exposed via the JMX exporter. All supported HBase versions expose metrics via the built in Prometheus servlet. See https://github.com/stackabletech/hbase-operator/pull/672[hbase-operator#672]. From ff2f13a80324ebec9d7c8231c4be7c6a1bfe75d1 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 24 Jul 2025 10:35:35 +0200 Subject: [PATCH 38/54] mention bouncy castle fix for spark images --- modules/ROOT/partials/release-notes/release-25.7.adoc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index f3adaa2f9..5352a5d7e 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -90,10 +90,13 @@ See https://github.com/stackabletech/issues/issues/748[issues#748]. ===== Apache Spark -This release adds experimental support for Spark Connect. -The Spark operator watches for SparkConnectServer custom resources. -Preliminary documentation is xref:spark-k8s:usage-guide/spark-connect.adoc[available] and the existing xref:demos:jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc[Taxi Data Anomaly Detection demo] has been retrofitted to use a JupterLab client running against a Spark Connect server. -Also see https://github.com/stackabletech/spark-k8s-operator/issues/284[spark-k8s-operator#284]. +* This release adds experimental support for Spark Connect. + The Spark operator watches for SparkConnectServer custom resources. + Preliminary documentation is xref:spark-k8s:usage-guide/spark-connect.adoc[available] and the existing xref:demos:jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc[Taxi Data Anomaly Detection demo] has been retrofitted to use a JupterLab client running against a Spark Connect server. + Also see https://github.com/stackabletech/spark-k8s-operator/issues/284[the issue] tracking support for Spark Connect. +* This release also ensures that Bouncy Castle libraries are included in the Spark images. + This was causing errors in kerberized environments previously. + See https://github.com/stackabletech/docker-images/pull/1212[this pull request] for details. // TODO (@Techassi): Define a single way of spelling internal operators, like listener, secret, and commons. // Currently, throughout this page variations like Stackable Listener Operator, Listener Operator, listener operator, From f64eb396f4487f48b2ead355c24abbc4006099ae Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 11:00:09 +0200 Subject: [PATCH 39/54] docs: Add call-to-action to delete orphaned SAs --- modules/ROOT/partials/release-notes/release-25.7.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 5352a5d7e..3ca4f45e5 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -238,6 +238,7 @@ See https://github.com/stackabletech/spark-k8s-operator/pull/584[spark-k8s-opera * This release removes support for previously deprecated product service accounts. Deprecated service accounts named `-serviceaccount` are ignored. Instead, product workloads use service accounts named `-serviceaccount` where `resource_name` is the stacklet (or cluster) name as defined in the cluster resource. + We advise deleting these orphaned ServiceAccounts manually. See https://github.com/stackabletech/operator-rs/pull/1060[operator-rs#1060]. * *Breaking:* The `lastUpdatedTime` was removed from the `ClusterStatus` of each operator. From 760e9f1c35aae96e5c1bc9a0f4e2d4352d541061 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 11:00:54 +0200 Subject: [PATCH 40/54] docs: Add link to "known issues" section --- modules/ROOT/partials/release-notes/release-25.7.adoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 3ca4f45e5..3c26295c8 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -381,7 +381,7 @@ This can be done using `kubectl replace`. ==== The Stackable secret operator CRD needs to be applied in addition to being replaced, because it introduces the new TrustStore custom resource. The commands below are adjusted accordingly. -This issue is also mentioned in the known issues section below. +This issue is also mentioned in the link:#known-issues-25_7_0[known issues section] below. ==== [source] @@ -444,7 +444,7 @@ The reason for this is that helm will uninstall the operators but not the CRDs. ==== The Stackable secret operator CRD needs to be applied in addition to being replaced, because it introduces the new TrustStore custom resource. The commands below are adjusted accordingly. -This issue is also mentioned in the known issues section below. +This issue is also mentioned in the link:#known-issues-25_7_0[known issues section] below. ==== [source] @@ -498,6 +498,7 @@ helm install --wait trino-operator oci://oci.stackable.tech/sdp-charts/trino-ope helm install --wait zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version 25.7.0 ---- +[#known-issues-25_7_0] ==== Known issues * During the upgrade of the secret-operator CRD, special care must be taken because the CRD introduces the new TrustStore custom resource. From 34ca82d9e4c6eb5f06213c3be8d5cef7e4d22063 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 11:01:28 +0200 Subject: [PATCH 41/54] chore: Update link text --- modules/ROOT/partials/release-notes/release-25.7.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 3c26295c8..c9dc1ba9d 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -96,7 +96,7 @@ See https://github.com/stackabletech/issues/issues/748[issues#748]. Also see https://github.com/stackabletech/spark-k8s-operator/issues/284[the issue] tracking support for Spark Connect. * This release also ensures that Bouncy Castle libraries are included in the Spark images. This was causing errors in kerberized environments previously. - See https://github.com/stackabletech/docker-images/pull/1212[this pull request] for details. + See https://github.com/stackabletech/docker-images/pull/1212[docker-images#1212] for details. // TODO (@Techassi): Define a single way of spelling internal operators, like listener, secret, and commons. // Currently, throughout this page variations like Stackable Listener Operator, Listener Operator, listener operator, From 95e029be255eedf9ad38df5a5624dceb5d72b87f Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 11:07:19 +0200 Subject: [PATCH 42/54] chore: Update CRD replacement commands --- .../partials/release-notes/release-25.7.adoc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index c9dc1ba9d..6185ac021 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -396,9 +396,12 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operato kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/25.7.0/deploy/helm/listener-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/25.7.0/deploy/helm/nifi-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/25.7.0/deploy/helm/opa-operator/crds/crds.yaml +# We have a new custom resource: truststores.secrets.stackable.tech kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml || true -kubectl apply -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl create -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml || true +# We have a new custom resource: sparkconnectservers.spark.stackable.tech +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml || true +kubectl create -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml || true kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/25.7.0/deploy/helm/superset-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/25.7.0/deploy/helm/trino-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/25.7.0/deploy/helm/zookeeper-operator/crds/crds.yaml @@ -459,9 +462,12 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operato kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/25.7.0/deploy/helm/listener-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/25.7.0/deploy/helm/nifi-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/25.7.0/deploy/helm/opa-operator/crds/crds.yaml +# We have a new custom resource: truststores.secrets.stackable.tech kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml || true -kubectl apply -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml -kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl create -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.7.0/deploy/helm/secret-operator/crds/crds.yaml || true +# We have a new custom resource: sparkconnectservers.spark.stackable.tech +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml || true +kubectl create -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.7.0/deploy/helm/spark-k8s-operator/crds/crds.yaml || true kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/25.7.0/deploy/helm/superset-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/25.7.0/deploy/helm/trino-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/25.7.0/deploy/helm/zookeeper-operator/crds/crds.yaml @@ -502,7 +508,7 @@ helm install --wait zookeeper-operator oci://oci.stackable.tech/sdp-charts/zooke ==== Known issues * During the upgrade of the secret-operator CRD, special care must be taken because the CRD introduces the new TrustStore custom resource. - It is recommended to apply it instead of replacing it: `kubectl apply -f ...`. + It is recommended to apply/create it in addition to replacing it. * There are strong indicators that TLS CA handling is broken in products using `keytool`. As of now, we confirmed that Trino has an issue in properly handling CA rotations due to a limitation with `keytool`. The report in https://github.com/stackabletech/issues/issues/744[issues#744] outlines a workaround which requires the following steps: From 1bdc488b94c275757991be51169bc7b9bdaae8b4 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 14:27:54 +0200 Subject: [PATCH 43/54] chore: Adjust Hadoop 3.3.6 support note --- .../partials/release-notes/release-25.7.adoc | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 6185ac021..2afb78bc9 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -300,12 +300,17 @@ The following product versions are deprecated and will be removed in a later rel ====== Removed versions The following product versions are no longer supported. -These images for released product versions remain available https://oci.stackable.tech/[here,window=_blank] -Information on how to browse the registry can be found xref:contributor:project-overview.adoc#docker-images[here,window=_blank] +These images for released product versions remain available https://oci.stackable.tech/[here,window=_blank]. +Information on how to browse the registry can be found xref:contributor:project-overview.adoc#docker-images[here,window=_blank]. + +[NOTE] +==== +We removed support for Apache Hadoop 3.3.6 in our operator, but it is still required by HBase, Hive, and Spark in this release. +==== * Apache Druid: https://github.com/stackabletech/docker-images/issues/1075[30.0.0] * Apache Hbase: https://github.com/stackabletech/docker-images/issues/1076[2.4.18] -* Apache Hadoop: https://github.com/stackabletech/docker-images/issues/1077[3.3.4], https://github.com/stackabletech/docker-images/issues/1077[3.4.0], https://github.com/stackabletech/docker-images/issues/1077[3.3.6^1^] +* Apache Hadoop: https://github.com/stackabletech/docker-images/issues/1077[3.3.4], https://github.com/stackabletech/docker-images/issues/1077[3.4.0], https://github.com/stackabletech/docker-images/issues/1077[3.3.6] * Apache Kafka: https://github.com/stackabletech/docker-images/issues/1078[3.7.1], https://github.com/stackabletech/docker-images/issues/1078[3.8.0] * Apache NiFi: https://github.com/stackabletech/docker-images/issues/1079[2.2.0 (experimental)] * Apache Spark: https://github.com/stackabletech/docker-images/issues/1080[3.5.2] @@ -313,13 +318,6 @@ Information on how to browse the registry can be found xref:contributor:project- * Open Policy Agent: https://github.com/stackabletech/docker-images/issues/1084[0.67.1] * Trino: https://github.com/stackabletech/docker-images/issues/1082[455] -''' - -[.small] --- -. Remove support for 3.3.6, it is still required by HBase, Hive, and Spark in this release. --- - ===== Kubernetes versions This release supports the following Kubernetes versions: From d6328a2903d8e4d615dde0466c1d8776d403b2b0 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 14:29:28 +0200 Subject: [PATCH 44/54] chore: Apply suggestion Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- modules/ROOT/pages/getting-started.adoc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/ROOT/pages/getting-started.adoc b/modules/ROOT/pages/getting-started.adoc index 28f51eca7..b05fe0bf5 100644 --- a/modules/ROOT/pages/getting-started.adoc +++ b/modules/ROOT/pages/getting-started.adoc @@ -70,12 +70,12 @@ You can check which operators are installed using `stackablectl operator install [source,console] ---- OPERATOR VERSION NAMESPACE STATUS LAST UPDATED -commons 25.7.0 default deployed 2024-11-30 17:58:32.916032854 +0100 CET -kafka 25.7.0 default deployed 2024-11-30 17:58:55.036115353 +0100 CET -listener 25.7.0 default deployed 2024-11-30 17:59:18.136775259 +0100 CET -nifi 25.7.0 default deployed 2024-11-30 17:59:51.927081648 +0100 CET -secret 25.7.0 default deployed 2024-11-30 18:00:05.060241771 +0100 CET -zookeeper 25.7.0 default deployed 2024-11-30 18:00:08.425686918 +0100 CET +commons 25.7.0 default deployed 2025-07-24 17:58:32.916032854 +0100 CET +kafka 25.7.0 default deployed 2025-07-24 17:58:55.036115353 +0100 CET +listener 25.7.0 default deployed 2025-07-24 17:59:18.136775259 +0100 CET +nifi 25.7.0 default deployed 2025-07-24 17:59:51.927081648 +0100 CET +secret 25.7.0 default deployed 2025-07-24 18:00:05.060241771 +0100 CET +zookeeper 25.7.0 default deployed 2025-07-24 18:00:08.425686918 +0100 CET ---- == Deploying Stackable Services From 9609fc45c746b478557691cc79cab16ee7440773 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 15:35:56 +0200 Subject: [PATCH 45/54] chore: Indicate that Apache HBase 2.6.2 is the LTS version --- modules/ROOT/partials/release-notes/release-25.7.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 2afb78bc9..83cac6ce6 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -280,7 +280,7 @@ The following new product versions are now supported: * Apache Airflow: https://github.com/stackabletech/docker-images/issues/1074[2.10.5], https://github.com/stackabletech/docker-images/issues/1074[3.0.1 (experimental)] * Apache Druid: https://github.com/stackabletech/docker-images/issues/1075[33.0.0] -* Apache HBase: https://github.com/stackabletech/docker-images/issues/1076[2.6.2] +* Apache HBase: https://github.com/stackabletech/docker-images/issues/1076[2.6.2 (LTS)] * Apache NiFi: https://github.com/stackabletech/docker-images/issues/1079[2.4.0] * Apache Superset: https://github.com/stackabletech/docker-images/issues/1081[4.1.2] * Open Policy Agent: https://github.com/stackabletech/docker-images/issues/1084[1.4.2] From 0b147473b8fbf9fd69b2bb6afc471f8ed478dcf4 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 15:50:28 +0200 Subject: [PATCH 46/54] chore: Link to stackablectl release notes page --- modules/ROOT/partials/release-notes/release-25.7.adoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 83cac6ce6..19a12ee3a 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -18,8 +18,7 @@ The SDP 25.7.0 release is the first release which will **only** be published on Previously, releases of the `stackablectl` CLI tool were synced with platform releases. The tool itself was never tied to a particular version of SDP. As such, it was decided to version `stackablectl` independently. -// Starting with this release, the release notes for `stackablectl` are located on a separate xref:management:stackablectl/release-notes.adoc[page]. -Starting with this release, the release notes for `stackablectl` are located on a separate page. +Starting with this release, the release notes for `stackablectl` are located on a separate xref:management:stackablectl:release-notes.adoc[page]. ==== ==== New platform features From c8f524c199742af3b57371cea5ef49b47617abc9 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 16:25:04 +0200 Subject: [PATCH 47/54] chore: Move sentences to individual lines --- modules/ROOT/partials/release-notes/release-25.7.adoc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 19a12ee3a..c6f6cbaea 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -1,6 +1,3 @@ -// Here are the headings you can use for the next release. Saves time checking indentation levels. -// Take a look at release 24.11 to see how to structure patch releases. - == Release 25.7 === 25.7.0 @@ -438,7 +435,8 @@ release "commons-operator" uninstalled ---- Afterward you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. -The reason for this is that helm will uninstall the operators but not the CRDs. This can be done using `kubectl replace`. +The reason for this is that helm will uninstall the operators but not the CRDs. +This can be done using `kubectl replace`. [IMPORTANT] ==== @@ -480,7 +478,8 @@ customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" Install the `25.7` release -NOTE: `helm repo` subcommands are not supported for OCI registries. The operators are installed directly, without adding the Helm Chart repository first. +NOTE: `helm repo` subcommands are not supported for OCI registries. +The operators are installed directly, without adding the Helm Chart repository first. [source,console] ---- From 5ad066036d455348c179ed55ab41b2fa32608400 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Jul 2025 16:32:26 +0200 Subject: [PATCH 48/54] chore: Spelling of Jupyter --- modules/ROOT/partials/release-notes/release-25.7.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index c6f6cbaea..4ef8b0a46 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -88,7 +88,7 @@ See https://github.com/stackabletech/issues/issues/748[issues#748]. * This release adds experimental support for Spark Connect. The Spark operator watches for SparkConnectServer custom resources. - Preliminary documentation is xref:spark-k8s:usage-guide/spark-connect.adoc[available] and the existing xref:demos:jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc[Taxi Data Anomaly Detection demo] has been retrofitted to use a JupterLab client running against a Spark Connect server. + Preliminary documentation is xref:spark-k8s:usage-guide/spark-connect.adoc[available] and the existing xref:demos:jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data.adoc[Taxi Data Anomaly Detection demo] has been retrofitted to use a JupyterLab client running against a Spark Connect server. Also see https://github.com/stackabletech/spark-k8s-operator/issues/284[the issue] tracking support for Spark Connect. * This release also ensures that Bouncy Castle libraries are included in the Spark images. This was causing errors in kerberized environments previously. From 48f74a97129571646f1433694b271aacfa5f08a1 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 16:33:00 +0200 Subject: [PATCH 49/54] chore: Unify spelling for listener, secret and commons operator --- modules/ROOT/partials/release-notes/release-25.7.adoc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 4ef8b0a46..9fd44c254 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -94,12 +94,9 @@ See https://github.com/stackabletech/issues/issues/748[issues#748]. This was causing errors in kerberized environments previously. See https://github.com/stackabletech/docker-images/pull/1212[docker-images#1212] for details. -// TODO (@Techassi): Define a single way of spelling internal operators, like listener, secret, and commons. -// Currently, throughout this page variations like Stackable Listener Operator, Listener Operator, listener operator, -// listener-operator are used. Only ONE must be used. -===== Stackable listener operator +===== Stackable listener-operator -The Stackable Listener Operator can now use custom xref:listener-operator:listenerclass.adoc#servicetype-loadbalancer-class[LoadBalancer classes] and https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation[disable NodePort allocation]. +The Stackable listener-operator can now use custom xref:listener-operator:listenerclass.adoc#servicetype-loadbalancer-class[LoadBalancer classes] and https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation[disable NodePort allocation]. ==== Platform improvements @@ -107,7 +104,7 @@ The Stackable Listener Operator can now use custom xref:listener-operator:listen ====== Listener integration -*Breaking:* The Stackable operators for Apache Airflow, Apache Druid, Apache HBase, Apache Hadoop, Apache Hive, Apache NiFi, Apache Spark, Apache ZooKeeper, and Trino are now integrated with the Stackable listener operator. +*Breaking:* The Stackable operators for Apache Airflow, Apache Druid, Apache HBase, Apache Hadoop, Apache Hive, Apache NiFi, Apache Spark, Apache ZooKeeper, and Trino are now integrated with the Stackable listener-operator. See the overall tracking issue https://github.com/stackabletech/issues/issues/692[issues#692] and https://github.com/stackabletech/issues/issues/692#issuecomment-3068662411[comment] summarizing the breaking changes. * *Apache Airflow and Apache Superset:* From 84b4c788362811edf8285184b70db9c44c19ad94 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 16:33:25 +0200 Subject: [PATCH 50/54] docs: Adjust note about new CRs --- .../ROOT/partials/release-notes/release-25.7.adoc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 9fd44c254..e29d2a697 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -370,7 +370,11 @@ This can be done using `kubectl replace`. [IMPORTANT] ==== -The Stackable secret operator CRD needs to be applied in addition to being replaced, because it introduces the new TrustStore custom resource. +In this release, two CRDs must be applied/created in addition to being replaced: + +* The Stackable secret-operator CRD introduces the new TrustStore custom resource. +* The Stackable Operator for Apache Spark CRD introduces a new SparkConnectServer custom resource. + The commands below are adjusted accordingly. This issue is also mentioned in the link:#known-issues-25_7_0[known issues section] below. ==== @@ -437,7 +441,11 @@ This can be done using `kubectl replace`. [IMPORTANT] ==== -The Stackable secret operator CRD needs to be applied in addition to being replaced, because it introduces the new TrustStore custom resource. +In this release, two CRDs must be applied/created in addition to being replaced: + +* The Stackable secret-operator CRD introduces the new TrustStore custom resource. +* The Stackable Operator for Apache Spark CRD introduces a new SparkConnectServer custom resource. + The commands below are adjusted accordingly. This issue is also mentioned in the link:#known-issues-25_7_0[known issues section] below. ==== From 87e7665cb949187c06be8ec92f444eda5a885524 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Jul 2025 16:36:23 +0200 Subject: [PATCH 51/54] chore: Spelling of ZooKeeper --- modules/ROOT/partials/release-notes/release-25.7.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index e29d2a697..78979fd1a 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -153,7 +153,7 @@ See the xref:opa:usage-guide/user-info-fetcher.adoc#backend-activedirectory[docu ====== Miscellaneous * All products that are built from source in SDP, which is at the time of this release everything except Airflow and Superset, now have a version suffix to indicate they include custom modifications made by Stackable. - The suffix the same as for SDP container images, so Zookeeper 3.9.3 in SDP 25.7.0 will for example report `3.9.3-stackable25.7.0` as its version. + The suffix the same as for SDP container images, so ZooKeeper 3.9.3 in SDP 25.7.0 will for example report `3.9.3-stackable25.7.0` as its version. See https://github.com/stackabletech/docker-images/issues/1068[docker-images#1068]. * We've added source code snapshots directly into our container images to further improve transparency and the debugging experience. Each container image now includes `.tar.gz` archives containing the exact source code used to build that specific version of the product. From aa5e1680ed22da722098bda09ce2920049b7bee7 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Jul 2025 16:38:16 +0200 Subject: [PATCH 52/54] chore: Syntax highlight commands --- modules/ROOT/partials/release-notes/release-25.7.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index 78979fd1a..f75201095 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -379,7 +379,7 @@ The commands below are adjusted accordingly. This issue is also mentioned in the link:#known-issues-25_7_0[known issues section] below. ==== -[source] +[source,shell] ---- kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/25.7.0/deploy/helm/airflow-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/25.7.0/deploy/helm/commons-operator/crds/crds.yaml From cf9e544525cc60d5e43746ab29e022bd68d47b84 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Jul 2025 16:39:37 +0200 Subject: [PATCH 53/54] chore: Syntax highlight commands --- modules/ROOT/partials/release-notes/release-25.7.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index f75201095..e439f171b 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -402,7 +402,7 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operato kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/25.7.0/deploy/helm/zookeeper-operator/crds/crds.yaml ---- -[source,console] +[source] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced @@ -450,7 +450,7 @@ The commands below are adjusted accordingly. This issue is also mentioned in the link:#known-issues-25_7_0[known issues section] below. ==== -[source] +[source,shell] ---- kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/25.7.0/deploy/helm/airflow-operator/crds/crds.yaml kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/25.7.0/deploy/helm/commons-operator/crds/crds.yaml @@ -473,7 +473,7 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operato kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/25.7.0/deploy/helm/zookeeper-operator/crds/crds.yaml ---- -[source,console] +[source] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced From 2eb7c49cc47d1f7e2f14f112d6ee42f808ff1ba8 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 24 Jul 2025 16:49:27 +0200 Subject: [PATCH 54/54] chore: Remove optional description TODO --- modules/ROOT/partials/release-notes/release-25.7.adoc | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/ROOT/partials/release-notes/release-25.7.adoc b/modules/ROOT/partials/release-notes/release-25.7.adoc index e439f171b..5a91fc7a1 100644 --- a/modules/ROOT/partials/release-notes/release-25.7.adoc +++ b/modules/ROOT/partials/release-notes/release-25.7.adoc @@ -3,7 +3,6 @@ === 25.7.0 Released on 2025-07-23. -(Optional description / introduction) [NOTE] ====