From d008047a6d37f88afe33f68fe3f660342236db8d Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 27 May 2025 14:55:51 +0200 Subject: [PATCH 01/50] WIP: Listener integration --- Cargo.lock | 20 +-- deploy/helm/hive-operator/crds/crds.yaml | 25 +-- rust/operator-binary/src/controller.rs | 186 ++++++++++++++--------- rust/operator-binary/src/crd/mod.rs | 35 ++--- rust/operator-binary/src/discovery.rs | 21 +-- 5 files changed, 151 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 78425d60..51cf5652 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,9 +319,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.23" +version = "1.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766" +checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" dependencies = [ "jobserver", "libc", @@ -1100,11 +1100,10 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.5" +version = "0.27.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +checksum = "03a01595e11bdcec50946522c32dde3fc6914743000a68b93000965f2f02406d" dependencies = [ - "futures-util", "http", "hyper", "hyper-util", @@ -1686,12 +1685,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2611b99ab098a31bdc8be48b4f1a285ca0ced28bd5b4f23e45efa8c63b09efa5" -dependencies = [ - "once_cell", -] +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" [[package]] name = "openssl-probe" @@ -2315,9 +2311,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "ryu" diff --git a/deploy/helm/hive-operator/crds/crds.yaml b/deploy/helm/hive-operator/crds/crds.yaml index f0ca0792..cf23b837 100644 --- a/deploy/helm/hive-operator/crds/crds.yaml +++ b/deploy/helm/hive-operator/crds/crds.yaml @@ -77,23 +77,6 @@ spec: required: - configMap type: object - listenerClass: - default: cluster-internal - description: |- - This field controls which type of Service the Operator creates for this HiveCluster: - - * cluster-internal: Use a ClusterIP service - - * external-unstable: Use a NodePort service - - * external-stable: Use a LoadBalancer service - - This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - enum: - - cluster-internal - - external-unstable - - external-stable - type: string s3: description: S3 connection specification. This can be either `inline` or a `reference` to an S3Connection object. Read the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3) to learn more. nullable: true @@ -333,6 +316,10 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + nullable: true + type: string logging: default: containers: {} @@ -627,6 +614,10 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + nullable: true + type: string logging: default: containers: {} diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 0e2bf485..988073a7 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -3,6 +3,7 @@ use std::{ borrow::Cow, collections::{BTreeMap, HashMap}, + default, hash::Hasher, sync::Arc, }; @@ -22,8 +23,14 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::VolumeBuilder, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, VolumeBuilder, + }, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -31,7 +38,13 @@ use stackable_operator::{ product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, tls_verification::TlsClientDetailsError, }, - crd::s3, + crd::{ + listener::{ + self, + v1alpha1::{Listener, ListenerPort, ListenerSpec}, + }, + s3, + }, k8s_openapi::{ DeepMerge, api::{ @@ -82,8 +95,9 @@ use crate::{ crd::{ APP_NAME, CORE_SITE_XML, Container, DB_PASSWORD_ENV, DB_USERNAME_ENV, HIVE_PORT, HIVE_PORT_NAME, HIVE_SITE_XML, HiveClusterStatus, HiveRole, JVM_SECURITY_PROPERTIES_FILE, - METRICS_PORT, METRICS_PORT_NAME, MetaStoreConfig, STACKABLE_CONFIG_DIR, - STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, + MetaStoreConfig, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, + STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR, STACKABLE_LOG_DIR_NAME, v1alpha1, }, @@ -327,6 +341,16 @@ pub enum Error { #[snafu(display("failed to construct JVM arguments"))] ConstructJvmArguments { source: crate::config::jvm::Error }, + + #[snafu(display("failed to apply group listener for {rolegroup}"))] + ApplyGroupListener { + source: stackable_operator::cluster_resources::Error, + rolegroup: RoleGroupRef, + }, + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, } type Result = std::result::Result; @@ -428,16 +452,10 @@ pub async fn reconcile_hive( .await .context(ApplyRoleBindingSnafu)?; - let metastore_role_service = build_metastore_role_service(hive, &resolved_product_image)?; - - // we have to get the assigned ports - let metastore_role_service = cluster_resources - .add(client, metastore_role_service) - .await - .context(ApplyRoleServiceSnafu)?; - let mut ss_cond_builder = StatefulSetConditionBuilder::default(); + // TODO: You want to add service creation in this for loop + // for each rg a service needs to be build for hive for (rolegroup_name, rolegroup_config) in metastore_config.iter() { let rolegroup = hive.metastore_rolegroup_ref(rolegroup_name); @@ -467,6 +485,20 @@ pub async fn reconcile_hive( &rbac_sa.name_any(), )?; + let rg_group_listener = build_group_listener( + hive, + &resolved_product_image, + &rolegroup, + config.listener_class, + )?; + + cluster_resources + .add(client, rg_group_listener) + .await + .with_context(|_| ApplyGroupListenerSnafu { + rolegroup: rolegroup.clone(), + })?; + cluster_resources .add(client, rg_service) .await @@ -504,16 +536,10 @@ pub async fn reconcile_hive( // std's SipHasher is deprecated, and DefaultHasher is unstable across Rust releases. // We don't /need/ stability, but it's still nice to avoid spurious changes where possible. let mut discovery_hash = FnvHasher::with_key(0); - for discovery_cm in discovery::build_discovery_configmaps( - client, - hive, - hive, - &resolved_product_image, - &metastore_role_service, - None, - ) - .await - .context(BuildDiscoveryConfigSnafu)? + for discovery_cm in + discovery::build_discovery_configmaps(client, hive, hive, &resolved_product_image, None) + .await + .context(BuildDiscoveryConfigSnafu)? { let discovery_cm = cluster_resources .add(client, discovery_cm) @@ -547,43 +573,47 @@ pub async fn reconcile_hive( Ok(Action::await_change()) } -/// The server-role service is the primary endpoint that should be used by clients that do not -/// perform internal load balancing including targets outside of the cluster. -pub fn build_metastore_role_service( +pub fn build_group_listener( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, -) -> Result { - let role_name = HiveRole::MetaStore.to_string(); + rolegroup: &RoleGroupRef, + listener_class: String, +) -> Result { + let metadata = ObjectMetaBuilder::new() + .name_and_namespace(hive) + .name(hive.group_listener_name(rolegroup)) + .ownerreference_from_resource(hive, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .build(); - let role_svc_name = hive - .metastore_role_service_name() - .context(GlobalServiceNameNotFoundSnafu)?; - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(hive) - .name(role_svc_name) - .ownerreference_from_resource(hive, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &role_name, - "global", - )) - .context(MetadataBuildSnafu)? - .build(), - spec: Some(ServiceSpec { - type_: Some(hive.spec.cluster_config.listener_class.k8s_service_type()), - ports: Some(service_ports()), - selector: Some( - Labels::role_selector(hive, APP_NAME, &role_name) - .context(LabelBuildSnafu)? - .into(), - ), - ..ServiceSpec::default() - }), + let spec = ListenerSpec { + class_name: Some(listener_class), + ports: Some(listener_ports()), + ..Default::default() + }; + + let listener = Listener { + metadata, + spec, status: None, - }) + }; + + Ok(listener) +} + +fn listener_ports() -> Vec { + vec![ListenerPort { + name: HIVE_PORT_NAME.to_owned(), + port: HIVE_PORT.into(), + protocol: Some("TCP".to_owned()), + }] } /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator @@ -729,6 +759,7 @@ fn build_metastore_rolegroup_config_map( /// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. +// TODO: However it looks like I need to add it here as services will be created here by rolegroup fn build_rolegroup_service( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, @@ -949,16 +980,39 @@ fn build_metastore_rolegroup_statefulset( } } + let recommended_object_labels = build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + // Used for PVC templates that cannot be modified once they are deployed + let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( + hive, + // A version value is required, and we do want to use the "recommended" format for the other desired labels + "none", + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .context(LabelBuildSnafu)?; + let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels.clone()) .context(MetadataBuildSnafu)? .build(); + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(hive.group_listener_name(rolegroup_ref)), + &unversioned_recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_owned()) + .context(BuildListenerVolumeSnafu)?; + + container_builder + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + pod_builder .metadata(metadata) .image_pull_secrets_from_product_image(resolved_product_image) @@ -1069,12 +1123,7 @@ fn build_metastore_rolegroup_statefulset( .name(rolegroup_ref.object_name()) .ownerreference_from_resource(hive, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels) .context(MetadataBuildSnafu)? .build(), spec: Some(StatefulSetSpec { @@ -1095,6 +1144,7 @@ fn build_metastore_rolegroup_statefulset( }, service_name: Some(rolegroup_ref.object_name()), template: pod_template, + volume_claim_templates: Some(vec![pvc]), ..StatefulSetSpec::default() }), status: None, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index fd285a68..b43461f7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -60,6 +60,10 @@ pub const HIVE_PORT: u16 = 9083; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9084; +// Listener volumes +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; + // Certificates and trust stores pub const SYSTEM_TRUST_STORE: &str = "/etc/pki/java/cacerts"; pub const SYSTEM_TRUST_STORE_PASSWORD: &str = "changeit"; @@ -155,20 +159,6 @@ pub mod versioned { #[serde(skip_serializing_if = "Option::is_none")] pub vector_aggregator_config_map_name: Option, - /// This field controls which type of Service the Operator creates for this HiveCluster: - /// - /// * cluster-internal: Use a ClusterIP service - /// - /// * external-unstable: Use a NodePort service - /// - /// * external-stable: Use a LoadBalancer service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - #[serde(default)] - pub listener_class: CurrentlySupportedListenerClasses, - /// Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/usage-guide/security). pub authentication: Option, } @@ -184,11 +174,6 @@ impl HasStatusCondition for v1alpha1::HiveCluster { } impl v1alpha1::HiveCluster { - /// The name of the role-level load-balanced Kubernetes `Service` - pub fn metastore_role_service_name(&self) -> Option<&str> { - self.metadata.name.as_deref() - } - /// Metadata about a metastore rolegroup pub fn metastore_rolegroup_ref(&self, group_name: impl Into) -> RoleGroupRef { RoleGroupRef { @@ -235,6 +220,13 @@ impl v1alpha1::HiveCluster { }) } + /// The name of the group-listener provided for a specific role-group. + /// The UI will use this group listener so that only one load balancer + /// is needed (per role group). + pub fn group_listener_name(&self, rolegroup: &RoleGroupRef) -> String { + rolegroup.object_name() + } + pub fn rolegroup( &self, rolegroup_ref: &RoleGroupRef, @@ -448,6 +440,10 @@ pub struct MetaStoreConfig { /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. + #[serde(default)] + pub listener_class: String, } impl MetaStoreConfig { @@ -489,6 +485,7 @@ impl MetaStoreConfig { logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, role), graceful_shutdown_timeout: Some(DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT), + listener_class: Some("cluster-internal".to_owned()), } } } diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 147024f6..d21987d9 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -64,7 +64,6 @@ pub async fn build_discovery_configmaps( owner: &impl Resource, hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, - svc: &Service, chroot: Option<&str>, ) -> Result, Error> { let name = owner @@ -78,7 +77,7 @@ pub async fn build_discovery_configmaps( .as_deref() .context(NoNamespaceSnafu)?; let cluster_domain = &client.kubernetes_cluster_info.cluster_domain; - let mut discovery_configmaps = vec![build_discovery_configmap( + let discovery_configmaps = vec![build_discovery_configmap( name, owner, hive, @@ -90,24 +89,6 @@ pub async fn build_discovery_configmaps( )], )?]; - // TODO: Temporary solution until listener-operator is finished - if let Some(ServiceSpec { - type_: Some(service_type), - .. - }) = svc.spec.as_ref() - { - if service_type == &ServiceType::NodePort.to_string() { - discovery_configmaps.push(build_discovery_configmap( - &format!("{}-nodeport", name), - owner, - hive, - resolved_product_image, - chroot, - nodeport_hosts(client, svc, HIVE_PORT_NAME).await?, - )?); - } - } - Ok(discovery_configmaps) } From 04558d992c192ac39eb0675a03cce5bb4786bc31 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 27 May 2025 15:30:33 +0200 Subject: [PATCH 02/50] Adding apiGroups for listeners --- deploy/helm/hive-operator/templates/roles.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deploy/helm/hive-operator/templates/roles.yaml b/deploy/helm/hive-operator/templates/roles.yaml index 421f7360..053ccd43 100644 --- a/deploy/helm/hive-operator/templates/roles.yaml +++ b/deploy/helm/hive-operator/templates/roles.yaml @@ -122,6 +122,17 @@ rules: - bind resourceNames: - {{ include "operator.name" . }}-clusterrole + - apiGroups: + - listeners.stackable.tech + resources: + - listeners + verbs: + - get + - list + - watch + - patch + - create + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole From 4cf40c4a7749143a22d0d43e7679001f2f8a1024 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 27 May 2025 15:38:09 +0200 Subject: [PATCH 03/50] Making clippy happy --- rust/operator-binary/src/controller.rs | 6 +----- rust/operator-binary/src/discovery.rs | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 988073a7..7160095b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -3,7 +3,6 @@ use std::{ borrow::Cow, collections::{BTreeMap, HashMap}, - default, hash::Hasher, sync::Arc, }; @@ -39,10 +38,7 @@ use stackable_operator::{ tls_verification::TlsClientDetailsError, }, crd::{ - listener::{ - self, - v1alpha1::{Listener, ListenerPort, ListenerSpec}, - }, + listener::v1alpha1::{Listener, ListenerPort, ListenerSpec}, s3, }, k8s_openapi::{ diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index d21987d9..c807e865 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -4,13 +4,13 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, - k8s_openapi::api::core::v1::{ConfigMap, Endpoints, Service, ServiceSpec}, + k8s_openapi::api::core::v1::{ConfigMap, Endpoints, Service}, kube::{Resource, runtime::reflector::ObjectRef}, }; use crate::{ controller::build_recommended_labels, - crd::{HIVE_PORT, HIVE_PORT_NAME, HiveRole, ServiceType, v1alpha1}, + crd::{HIVE_PORT, HIVE_PORT_NAME, HiveRole, v1alpha1}, }; #[derive(Snafu, Debug)] From 2af8e4a62c4bedbedd6eeee0c0bcf4a0fa24e3c6 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 28 May 2025 08:45:49 +0200 Subject: [PATCH 04/50] Working code --- rust/operator-binary/src/controller.rs | 40 ++++++++++++-------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 7160095b..d26f5f88 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -764,7 +764,7 @@ fn build_rolegroup_service( Ok(Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(hive) - .name(rolegroup.object_name()) + .name(format!("{name}-metrics", name = rolegroup.object_name())) .ownerreference_from_resource(hive, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( @@ -976,12 +976,13 @@ fn build_metastore_rolegroup_statefulset( } } - let recommended_object_labels = build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ); + let recommended_object_labels: ObjectLabels<'_, v1alpha1::HiveCluster> = + build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); // Used for PVC templates that cannot be modified once they are deployed let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( hive, @@ -1138,7 +1139,10 @@ fn build_metastore_rolegroup_statefulset( ), ..LabelSelector::default() }, - service_name: Some(rolegroup_ref.object_name()), + service_name: Some(format!( + "{name}-metrics", + name = rolegroup_ref.object_name() + )), template: pod_template, volume_claim_templates: Some(vec![pvc]), ..StatefulSetSpec::default() @@ -1160,20 +1164,12 @@ pub fn error_policy( } pub fn service_ports() -> Vec { - vec![ - ServicePort { - name: Some(HIVE_PORT_NAME.to_string()), - port: HIVE_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }, - ServicePort { - name: Some(METRICS_PORT_NAME.to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }, - ] + vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] } /// Creates recommended `ObjectLabels` to be used in deployed resources From 42017f88019f3fb5907f823f31e12fe23148073a Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 28 May 2025 08:46:16 +0200 Subject: [PATCH 05/50] Working test --- .../kuttl/external-access/00-limit-range.yaml | 11 ++++ .../kuttl/external-access/00-patch-ns.yaml.j2 | 9 +++ .../external-access/10-listener-classes.yaml | 6 ++ .../kuttl/external-access/20-assert.yaml | 63 +++++++++++++++++++ .../external-access/20-install-hive.yaml | 8 +++ .../helm-bitnami-postgresql-values.yaml.j2 | 31 +++++++++ .../external-access/install-hive.yaml.j2 | 45 +++++++++++++ .../external-access/listener-classes.yaml | 21 +++++++ 8 files changed, 194 insertions(+) create mode 100644 tests/templates/kuttl/external-access/00-limit-range.yaml create mode 100644 tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/10-listener-classes.yaml create mode 100644 tests/templates/kuttl/external-access/20-assert.yaml create mode 100644 tests/templates/kuttl/external-access/20-install-hive.yaml create mode 100644 tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/install-hive.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/listener-classes.yaml diff --git a/tests/templates/kuttl/external-access/00-limit-range.yaml b/tests/templates/kuttl/external-access/00-limit-range.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-limit-range.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/external-access/10-listener-classes.yaml b/tests/templates/kuttl/external-access/10-listener-classes.yaml new file mode 100644 index 00000000..4af48da3 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-listener-classes.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - \ No newline at end of file diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml new file mode 100644 index 00000000..e780987a --- /dev/null +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +metadata: + name: install-hive +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available hiveclusters.hive.stackable.tech/test-hive --timeout 601s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hive-metastore-default +status: + readyReplicas: 2 + replicas: 2 + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hive-metastore-external-unstable +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hive-metastore-cluster-internal +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: test-hive-metastore +status: + expectedPods: 4 + currentHealthy: 4 + disruptionsAllowed: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hive-metastore-cluster-internal +spec: + type: ClusterIP # cluster-internal +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hive-metastore-default +spec: + type: NodePort # external-stable +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hive-metastore-external-unstable +spec: + type: NodePort # external-unstable \ No newline at end of file diff --git a/tests/templates/kuttl/external-access/20-install-hive.yaml b/tests/templates/kuttl/external-access/20-install-hive.yaml new file mode 100644 index 00000000..eb5b3df7 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-install-hive.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: > + envsubst < install-hive.yaml | + kubectl apply -n $NAMESPACE -f - \ No newline at end of file diff --git a/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..215a9199 --- /dev/null +++ b/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,31 @@ +--- +volumePermissions: + enabled: false + securityContext: + runAsUser: auto + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + +shmVolume: + chmod: + enabled: false + +auth: + username: hive + password: hive + database: hive \ No newline at end of file diff --git a/tests/templates/kuttl/external-access/install-hive.yaml.j2 b/tests/templates/kuttl/external-access/install-hive.yaml.j2 new file mode 100644 index 00000000..17ab4950 --- /dev/null +++ b/tests/templates/kuttl/external-access/install-hive.yaml.j2 @@ -0,0 +1,45 @@ +--- +apiVersion: hive.stackable.tech/v1alpha1 +kind: HiveCluster +metadata: + name: test-hive +spec: + image: +{% if test_scenario['values']['hive-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hive-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hive-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hive-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + database: + connString: jdbc:derby:;databaseName=/tmp/hive;create=true + credentialsSecret: hive-credentials + dbType: derby +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + metastore: + config: + listenerClass: test-external-stable-$NAMESPACE + roleGroups: + default: + replicas: 2 + external-unstable: + replicas: 1 + config: + listenerClass: test-external-unstable-$NAMESPACE + cluster-internal: + replicas: 1 + config: + listenerClass: test-cluster-internal-$NAMESPACE +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: APP + password: mine diff --git a/tests/templates/kuttl/external-access/listener-classes.yaml b/tests/templates/kuttl/external-access/listener-classes.yaml new file mode 100644 index 00000000..4131526a --- /dev/null +++ b/tests/templates/kuttl/external-access/listener-classes.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-cluster-internal-$NAMESPACE +spec: + serviceType: ClusterIP +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-stable-$NAMESPACE +spec: + serviceType: NodePort +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable-$NAMESPACE +spec: + serviceType: NodePort From 5ac30df794abae633ba8e6f00b87f02a5f6306e2 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 28 May 2025 08:46:32 +0200 Subject: [PATCH 06/50] test definition --- tests/test-definition.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index ae1b406a..7a9106ea 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -105,6 +105,10 @@ tests: dimensions: - hive-latest - openshift + - name: external-access + dimensions: + - hive-latest + - openshift suites: - name: nightly patch: From de413d38aaf8c76b58ea85c656588609cf3c6be1 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 28 May 2025 08:55:38 +0200 Subject: [PATCH 07/50] Cleaning up leftovers --- rust/operator-binary/src/controller.rs | 2 -- rust/operator-binary/src/crd/mod.rs | 37 -------------------------- 2 files changed, 39 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index d26f5f88..8ca6bb7d 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -450,7 +450,6 @@ pub async fn reconcile_hive( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - // TODO: You want to add service creation in this for loop // for each rg a service needs to be build for hive for (rolegroup_name, rolegroup_config) in metastore_config.iter() { let rolegroup = hive.metastore_rolegroup_ref(rolegroup_name); @@ -755,7 +754,6 @@ fn build_metastore_rolegroup_config_map( /// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. -// TODO: However it looks like I need to add it here as services will be created here by rolegroup fn build_rolegroup_service( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index b43461f7..c67e5132 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -299,29 +299,6 @@ impl v1alpha1::HiveCluster { } } -// TODO: Temporary solution until listener-operator is finished -#[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "PascalCase")] -pub enum CurrentlySupportedListenerClasses { - #[default] - #[serde(rename = "cluster-internal")] - ClusterInternal, - #[serde(rename = "external-unstable")] - ExternalUnstable, - #[serde(rename = "external-stable")] - ExternalStable, -} - -impl CurrentlySupportedListenerClasses { - pub fn k8s_service_type(&self) -> String { - match self { - CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), - CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), - CurrentlySupportedListenerClasses::ExternalStable => "LoadBalancer".to_string(), - } - } -} - #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HdfsConnection { @@ -490,20 +467,6 @@ impl MetaStoreConfig { } } -// TODO: Temporary solution until listener-operator is finished -#[derive(Clone, Debug, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "PascalCase")] -pub enum ServiceType { - NodePort, - ClusterIP, -} - -impl Default for ServiceType { - fn default() -> Self { - Self::NodePort - } -} - #[derive( Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize, Display, EnumString, )] From e88fd8f1dba23f51a0631109229be3eb5577a68e Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 28 May 2025 09:50:45 +0200 Subject: [PATCH 08/50] Fixing tests --- .../kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 | 6 +++--- .../kuttl/kerberos-s3/70-install-access-hive.yaml.j2 | 6 +++--- tests/templates/kuttl/smoke/80-assert.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 index 0c6c0647..e5536516 100644 --- a/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 @@ -69,7 +69,7 @@ data: kinit -kt /stackable/kerberos/keytab access-hive/access-hive."$NAMESPACE".svc.cluster.local klist - python /tmp/scripts/metastore.py -m hive-metastore-default-0.hive-metastore-default.$NAMESPACE.svc.cluster.local + python /tmp/scripts/metastore.py -m hive-metastore-default.$NAMESPACE.svc.cluster.local metastore.py: | from hive_metastore_client import HiveMetastoreClient from hive_metastore_client.builders import ( @@ -92,9 +92,9 @@ data: @staticmethod def _init_protocol(host: str, port: int) -> TBinaryProtocol: transport = TSocket.TSocket(host, int(port)) - # host is something like "hive-metastore-default-0.hive-metastore-default.kuttl-test-brave-caribou.svc.cluster.local" + # host is something like "hive-metastore-default..kuttl-test-brave-caribou.svc.cluster.local" # We need to change it to the host part of the HMS principal e.g. "hive.kuttl-test-brave-caribou.svc.cluster.local" - host = host.replace("hive-metastore-default-0.hive-metastore-default", "hive") + host = host.replace("hive-metastore-default", "hive") transport = TTransport.TSaslClientTransport(transport, # host part of the HMS principal (not the HMS client) host=host, diff --git a/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 b/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 index 371f4b4e..ed316cb9 100644 --- a/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 @@ -69,7 +69,7 @@ data: kinit -kt /stackable/kerberos/keytab access-hive/access-hive."$NAMESPACE".svc.cluster.local klist - python /tmp/scripts/metastore.py -m hive-metastore-default-0.hive-metastore-default.$NAMESPACE.svc.cluster.local + python /tmp/scripts/metastore.py -m hive-metastore-default.$NAMESPACE.svc.cluster.local metastore.py: | from hive_metastore_client import HiveMetastoreClient from hive_metastore_client.builders import ( @@ -92,9 +92,9 @@ data: @staticmethod def _init_protocol(host: str, port: int) -> TBinaryProtocol: transport = TSocket.TSocket(host, int(port)) - # host is something like "hive-metastore-default-0.hive-metastore-default.kuttl-test-brave-caribou.svc.cluster.local" + # host is something like "hive-metastore-default.kuttl-test-brave-caribou.svc.cluster.local" # We need to change it to the host part of the HMS principal e.g. "hive.kuttl-test-brave-caribou.svc.cluster.local" - host = host.replace("hive-metastore-default-0.hive-metastore-default", "hive") + host = host.replace("hive-metastore-default", "hive") transport = TTransport.TSaslClientTransport(transport, # host part of the HMS principal (not the HMS client) host=host, diff --git a/tests/templates/kuttl/smoke/80-assert.yaml b/tests/templates/kuttl/smoke/80-assert.yaml index a5404d02..2bf0f4dc 100644 --- a/tests/templates/kuttl/smoke/80-assert.yaml +++ b/tests/templates/kuttl/smoke/80-assert.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - - script: kubectl exec -n $NAMESPACE test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore-default-0.hive-metastore-default.$NAMESPACE.svc.cluster.local + - script: kubectl exec -n $NAMESPACE test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore-default.$NAMESPACE.svc.cluster.local From ff8ebe024c6445ca0e69ae17510c88ec6ad56948 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 28 May 2025 09:59:43 +0200 Subject: [PATCH 09/50] Adding Cargo.nix --- Cargo.nix | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 075c3173..5a63f756 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1035,9 +1035,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.23"; + version = "1.2.24"; edition = "2018"; - sha256 = "0ripmr35ix8lhszbkmr8pg8y61d4lv9xgfcs8jry5havkrmchjjz"; + sha256 = "1irvbn8y9sg6f1070yg5469fxk5c3ximh24ds04kph21w0xmsn8n"; authors = [ "Alex Crichton " ]; @@ -3416,16 +3416,11 @@ rec { }; "hyper-rustls" = rec { crateName = "hyper-rustls"; - version = "0.27.5"; + version = "0.27.6"; edition = "2021"; - sha256 = "1cjr3yf3x5mr3194llsfibacl6j7n2dknii2dwjha4ysyf1ia69d"; + sha256 = "0va008pmz5h062wnh2h08d3r3iizvqnw68k5ji8frp0vw6aib803"; libName = "hyper_rustls"; dependencies = [ - { - name = "futures-util"; - packageId = "futures-util"; - usesDefaultFeatures = false; - } { name = "http"; packageId = "http"; @@ -5497,15 +5492,9 @@ rec { }; "once_cell_polyfill" = rec { crateName = "once_cell_polyfill"; - version = "1.70.0"; + version = "1.70.1"; edition = "2021"; - sha256 = "19gg14xwda7g8lzg5d6mig9cx82w50d4z2z4igf1p8wqn2dbj496"; - dependencies = [ - { - name = "once_cell"; - packageId = "once_cell"; - } - ]; + sha256 = "1bg0w99srq8h4mkl68l1mza2n2f2hvrg0n8vfa3izjr5nism32d4"; features = { }; resolvedDefaultFeatures = [ "default" ]; @@ -7622,9 +7611,9 @@ rec { }; "rustversion" = rec { crateName = "rustversion"; - version = "1.0.20"; + version = "1.0.21"; edition = "2018"; - sha256 = "1lhwjb16dsm8brd18bn2bh0ryzc7qi29bi2jjsc6ny2zbwn3ivgd"; + sha256 = "07bb1xx05hhwpnl43sqrhsmxyk5sd5m5baadp19nxp69s9xij3ca"; procMacro = true; build = "build/build.rs"; authors = [ @@ -13926,3 +13915,4 @@ rec { # }; } + From 8116051326d05655a96723027a365f83f60969ab Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 4 Jun 2025 15:49:11 +0200 Subject: [PATCH 10/50] Cleaning up and renaming vars. First working example --- rust/operator-binary/src/controller.rs | 13 ++-- rust/operator-binary/src/discovery.rs | 88 +++++++++----------------- 2 files changed, 37 insertions(+), 64 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 8ca6bb7d..f48484e4 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -449,7 +449,7 @@ pub async fn reconcile_hive( .context(ApplyRoleBindingSnafu)?; let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - + let mut listeners = Vec::::new(); // for each rg a service needs to be build for hive for (rolegroup_name, rolegroup_config) in metastore_config.iter() { let rolegroup = hive.metastore_rolegroup_ref(rolegroup_name); @@ -480,14 +480,14 @@ pub async fn reconcile_hive( &rbac_sa.name_any(), )?; - let rg_group_listener = build_group_listener( + let rg_group_listener: Listener = build_group_listener( hive, &resolved_product_image, &rolegroup, config.listener_class, )?; - - cluster_resources + // TODO: Listener name might be wrong + let listener = cluster_resources .add(client, rg_group_listener) .await .with_context(|_| ApplyGroupListenerSnafu { @@ -501,6 +501,8 @@ pub async fn reconcile_hive( rolegroup: rolegroup.clone(), })?; + listeners.push(listener); + cluster_resources .add(client, rg_configmap) .await @@ -531,8 +533,9 @@ pub async fn reconcile_hive( // std's SipHasher is deprecated, and DefaultHasher is unstable across Rust releases. // We don't /need/ stability, but it's still nice to avoid spurious changes where possible. let mut discovery_hash = FnvHasher::with_key(0); + for discovery_cm in - discovery::build_discovery_configmaps(client, hive, hive, &resolved_product_image, None) + discovery::build_discovery_configmaps(hive, hive, &resolved_product_image, None, &listeners) .await .context(BuildDiscoveryConfigSnafu)? { diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index c807e865..1f09fd2e 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -4,13 +4,14 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, + crd::listener, k8s_openapi::api::core::v1::{ConfigMap, Endpoints, Service}, kube::{Resource, runtime::reflector::ObjectRef}, }; use crate::{ controller::build_recommended_labels, - crd::{HIVE_PORT, HIVE_PORT_NAME, HiveRole, v1alpha1}, + crd::{HIVE_PORT_NAME, HiveRole, v1alpha1}, }; #[derive(Snafu, Debug)] @@ -31,10 +32,10 @@ pub enum Error { source: stackable_operator::builder::configmap::Error, obj_ref: ObjectRef, }, - #[snafu(display("could not find service [{obj_ref}] port [{port_name}]"))] + #[snafu(display("could not find port [{port_name}]"))] NoServicePort { port_name: String, - obj_ref: ObjectRef, + //obj_ref: ObjectRef, }, #[snafu(display("service [{obj_ref}] port [{port_name}] does not have a nodePort "))] NoNodePort { @@ -60,33 +61,25 @@ pub enum Error { /// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::HiveCluster`] for all expected /// scenarios. pub async fn build_discovery_configmaps( - client: &stackable_operator::client::Client, owner: &impl Resource, hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, chroot: Option<&str>, + listeners: &[listener::v1alpha1::Listener], ) -> Result, Error> { let name = owner .meta() .name .as_ref() .context(InvalidOwnerNameForDiscoveryConfigMapSnafu)?; - let namespace = owner - .meta() - .namespace - .as_deref() - .context(NoNamespaceSnafu)?; - let cluster_domain = &client.kubernetes_cluster_info.cluster_domain; + let discovery_configmaps = vec![build_discovery_configmap( name, owner, hive, resolved_product_image, chroot, - vec![( - format!("{name}.{namespace}.svc.{cluster_domain}"), - HIVE_PORT, - )], + listener_hosts(listeners, &HIVE_PORT_NAME)?, )?]; Ok(discovery_configmaps) @@ -95,7 +88,7 @@ pub async fn build_discovery_configmaps( /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain /// [`v1alpha1::HiveCluster`]. /// -/// `hosts` will usually come from the cluster role service or [`nodeport_hosts`]. +/// `hosts` will usually come from the cluster role service or [`listener_hosts`]. fn build_discovery_configmap( name: &str, owner: &impl Resource, @@ -140,52 +133,29 @@ fn build_discovery_configmap( }) } -/// Lists all nodes currently hosting Pods participating in the [`Service`] -async fn nodeport_hosts( - client: &stackable_operator::client::Client, - svc: &Service, +fn listener_hosts( + listeners: &[listener::v1alpha1::Listener], port_name: &str, ) -> Result, Error> { - let svc_port = svc - .spec - .as_ref() - .and_then(|svc_spec| { - svc_spec - .ports - .as_ref()? - .iter() - .find(|port| port.name.as_deref() == Some(HIVE_PORT_NAME)) + listeners + .iter() + .flat_map(|listener| { + listener + .status + .as_ref() + .and_then(|s| s.ingress_addresses.as_deref()) }) - .context(NoServicePortSnafu { - port_name, - obj_ref: ObjectRef::from_obj(svc), - })?; - - let node_port = svc_port.node_port.context(NoNodePortSnafu { - port_name, - obj_ref: ObjectRef::from_obj(svc), - })?; - let endpoints = client - .get::( - svc.metadata.name.as_deref().context(NoNameSnafu)?, - svc.metadata - .namespace - .as_deref() - .context(NoNamespaceSnafu)?, - ) - .await - .with_context(|_| FindEndpointsSnafu { - svc: ObjectRef::from_obj(svc), - })?; - let nodes = endpoints - .subsets - .into_iter() .flatten() - .flat_map(|subset| subset.addresses) - .flatten() - .flat_map(|addr| addr.node_name); - let addrs = nodes - .map(|node| Ok((node, node_port.try_into().context(InvalidNodePortSnafu)?))) - .collect::, _>>()?; - Ok(addrs) + .map(|addr| { + Ok(( + addr.address.clone(), + addr.ports + .get(port_name) + .copied() + .context(NoServicePortSnafu { port_name })? + .try_into() + .context(InvalidNodePortSnafu)?, + )) + }) + .collect::, _>>() } From 505471c651059668c43df58b5d419db0be6d55f7 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 09:18:12 +0200 Subject: [PATCH 11/50] Making clippy happy --- rust/operator-binary/src/discovery.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 1f09fd2e..26101f48 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -5,7 +5,7 @@ use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, crd::listener, - k8s_openapi::api::core::v1::{ConfigMap, Endpoints, Service}, + k8s_openapi::api::core::v1::{ConfigMap, Service}, kube::{Resource, runtime::reflector::ObjectRef}, }; @@ -79,7 +79,7 @@ pub async fn build_discovery_configmaps( hive, resolved_product_image, chroot, - listener_hosts(listeners, &HIVE_PORT_NAME)?, + listener_hosts(listeners, HIVE_PORT_NAME)?, )?]; Ok(discovery_configmaps) From 0d1b223299714e65fe84455f5512d7fcbe5c795d Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 09:19:48 +0200 Subject: [PATCH 12/50] remove unused input --- rust/operator-binary/src/discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 26101f48..1bdeb2d5 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeSet, num::TryFromIntError}; +use std::num::TryFromIntError; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ From 4b201a10946371a94e5d01e2603aba0a5fa5f059 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 13:27:31 +0200 Subject: [PATCH 13/50] Discovery configMap is filled properly --- rust/operator-binary/src/controller.rs | 21 ++-- rust/operator-binary/src/discovery.rs | 111 ++++++++---------- tests/ansible/roles/expand-tests/CHANGELOG.md | 12 ++ tests/ansible/roles/expand-tests/README.md | 38 ++++++ .../roles/expand-tests/defaults/main.yml | 4 + .../expand-tests/files/generate_tests.py | 91 ++++++++++++++ .../roles/expand-tests/handlers/main.yml | 2 + .../expand-tests/meta/.galaxy_install_info | 2 + .../ansible/roles/expand-tests/meta/main.yml | 52 ++++++++ .../expand-tests/tasks/create_scenario.yaml | 50 ++++++++ .../ansible/roles/expand-tests/tasks/main.yml | 36 ++++++ .../roles/expand-tests/tests/inventory | 2 + .../ansible/roles/expand-tests/tests/test.yml | 5 + .../ansible/roles/expand-tests/vars/main.yml | 1 + .../external-access/10-listener-classes.yaml | 2 +- .../kuttl/external-access/20-assert.yaml | 2 +- .../external-access/20-install-hive.yaml | 2 +- 17 files changed, 362 insertions(+), 71 deletions(-) create mode 100644 tests/ansible/roles/expand-tests/CHANGELOG.md create mode 100644 tests/ansible/roles/expand-tests/README.md create mode 100644 tests/ansible/roles/expand-tests/defaults/main.yml create mode 100755 tests/ansible/roles/expand-tests/files/generate_tests.py create mode 100644 tests/ansible/roles/expand-tests/handlers/main.yml create mode 100644 tests/ansible/roles/expand-tests/meta/.galaxy_install_info create mode 100644 tests/ansible/roles/expand-tests/meta/main.yml create mode 100644 tests/ansible/roles/expand-tests/tasks/create_scenario.yaml create mode 100644 tests/ansible/roles/expand-tests/tasks/main.yml create mode 100644 tests/ansible/roles/expand-tests/tests/inventory create mode 100644 tests/ansible/roles/expand-tests/tests/test.yml create mode 100644 tests/ansible/roles/expand-tests/vars/main.yml diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index f48484e4..0c55b985 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -449,8 +449,8 @@ pub async fn reconcile_hive( .context(ApplyRoleBindingSnafu)?; let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - let mut listeners = Vec::::new(); - // for each rg a service needs to be build for hive + // Collecting listener objects with corresponding rolegroup to fill the discovery configMap later on + let mut listener_refs = BTreeMap::<&String, Listener>::new(); for (rolegroup_name, rolegroup_config) in metastore_config.iter() { let rolegroup = hive.metastore_rolegroup_ref(rolegroup_name); @@ -486,7 +486,7 @@ pub async fn reconcile_hive( &rolegroup, config.listener_class, )?; - // TODO: Listener name might be wrong + let listener = cluster_resources .add(client, rg_group_listener) .await @@ -501,7 +501,7 @@ pub async fn reconcile_hive( rolegroup: rolegroup.clone(), })?; - listeners.push(listener); + listener_refs.insert(rolegroup_name, listener); cluster_resources .add(client, rg_configmap) @@ -534,10 +534,15 @@ pub async fn reconcile_hive( // We don't /need/ stability, but it's still nice to avoid spurious changes where possible. let mut discovery_hash = FnvHasher::with_key(0); - for discovery_cm in - discovery::build_discovery_configmaps(hive, hive, &resolved_product_image, None, &listeners) - .await - .context(BuildDiscoveryConfigSnafu)? + for discovery_cm in discovery::build_discovery_configmaps( + hive, + hive, + &resolved_product_image, + None, + listener_refs, + ) + .await + .context(BuildDiscoveryConfigSnafu)? { let discovery_cm = cluster_resources .add(client, discovery_cm) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 1bdeb2d5..ebd6d274 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -1,10 +1,10 @@ -use std::num::TryFromIntError; +use std::{collections::BTreeMap, num::TryFromIntError}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, - crd::listener, + crd::listener::v1alpha1::Listener, k8s_openapi::api::core::v1::{ConfigMap, Service}, kube::{Resource, runtime::reflector::ObjectRef}, }; @@ -56,6 +56,8 @@ pub enum Error { MetadataBuild { source: stackable_operator::builder::meta::Error, }, + #[snafu(display("failed to apply group listener for {rolegroup}"))] + RoleGroupListenerHasNoAddress { rolegroup: String }, } /// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::HiveCluster`] for all expected @@ -65,7 +67,7 @@ pub async fn build_discovery_configmaps( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, chroot: Option<&str>, - listeners: &[listener::v1alpha1::Listener], + listener_refs: BTreeMap<&String, Listener>, ) -> Result, Error> { let name = owner .meta() @@ -79,7 +81,7 @@ pub async fn build_discovery_configmaps( hive, resolved_product_image, chroot, - listener_hosts(listeners, HIVE_PORT_NAME)?, + listener_refs, )?]; Ok(discovery_configmaps) @@ -95,67 +97,56 @@ fn build_discovery_configmap( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, chroot: Option<&str>, - hosts: impl IntoIterator, u16)>, + listener_refs: BTreeMap<&String, Listener>, ) -> Result { - let mut conn_str = hosts - .into_iter() - .map(|(host, port)| format!("thrift://{}:{}", host.into(), port)) - .collect::>() - .join("\n"); - if let Some(chroot) = chroot { - if !chroot.starts_with('/') { - return RelativeChrootSnafu { chroot }.fail(); + let mut discovery_configmap = ConfigMapBuilder::new(); + + discovery_configmap.metadata( + ObjectMetaBuilder::new() + .name_and_namespace(hive) + .name(name) + .ownerreference_from_resource(owner, None, Some(true)) + .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { + hive: ObjectRef::from_obj(hive), + })? + .with_recommended_labels(build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &HiveRole::MetaStore.to_string(), + "discovery", + )) + .context(MetadataBuildSnafu)? + .build(), + ); + + for (rolegroup, listener_ref) in listener_refs { + let listener_address = listener_ref + .status + .and_then(|s| s.ingress_addresses?.into_iter().next()) + .context(RoleGroupListenerHasNoAddressSnafu { rolegroup })?; + let mut conn_str = format!( + "thrift://{}:{}", + listener_address.address, + listener_address + .ports + .get(HIVE_PORT_NAME) + .copied() + .context(NoServicePortSnafu { + port_name: HIVE_PORT_NAME + })? + ); + if let Some(chroot) = chroot { + if !chroot.starts_with('/') { + return RelativeChrootSnafu { chroot }.fail(); + } + conn_str.push_str(chroot); } - conn_str.push_str(chroot); + discovery_configmap.add_data(rolegroup, conn_str); } - ConfigMapBuilder::new() - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(hive) - .name(name) - .ownerreference_from_resource(owner, None, Some(true)) - .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - hive: ObjectRef::from_obj(hive), - })? - .with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &HiveRole::MetaStore.to_string(), - "discovery", - )) - .context(MetadataBuildSnafu)? - .build(), - ) - .add_data("HIVE", conn_str) + + discovery_configmap .build() .with_context(|_| DiscoveryConfigMapSnafu { obj_ref: ObjectRef::from_obj(hive), }) } - -fn listener_hosts( - listeners: &[listener::v1alpha1::Listener], - port_name: &str, -) -> Result, Error> { - listeners - .iter() - .flat_map(|listener| { - listener - .status - .as_ref() - .and_then(|s| s.ingress_addresses.as_deref()) - }) - .flatten() - .map(|addr| { - Ok(( - addr.address.clone(), - addr.ports - .get(port_name) - .copied() - .context(NoServicePortSnafu { port_name })? - .try_into() - .context(InvalidNodePortSnafu)?, - )) - }) - .collect::, _>>() -} diff --git a/tests/ansible/roles/expand-tests/CHANGELOG.md b/tests/ansible/roles/expand-tests/CHANGELOG.md new file mode 100644 index 00000000..6e90657b --- /dev/null +++ b/tests/ansible/roles/expand-tests/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [Unreleased] + +### Changed + +- Generated kuttl tests prepend the original test name to the generated kuttl test folders ([#1]) + +[#1]: https://github.com/stackabletech/expand-tests/pull/1 + diff --git a/tests/ansible/roles/expand-tests/README.md b/tests/ansible/roles/expand-tests/README.md new file mode 100644 index 00000000..225dd44b --- /dev/null +++ b/tests/ansible/roles/expand-tests/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/tests/ansible/roles/expand-tests/defaults/main.yml b/tests/ansible/roles/expand-tests/defaults/main.yml new file mode 100644 index 00000000..64f6ec50 --- /dev/null +++ b/tests/ansible/roles/expand-tests/defaults/main.yml @@ -0,0 +1,4 @@ +--- +work_dir: "../test-work" +test_dir: "../tests" +template_dir: "{{ test_dir }}/templates" \ No newline at end of file diff --git a/tests/ansible/roles/expand-tests/files/generate_tests.py b/tests/ansible/roles/expand-tests/files/generate_tests.py new file mode 100755 index 00000000..22bd162e --- /dev/null +++ b/tests/ansible/roles/expand-tests/files/generate_tests.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import argparse +from argparse import Namespace +from dataclasses import dataclass, field, asdict +import itertools +import logging +import sys +from typing import Dict +import yaml +from itertools import chain + + +@dataclass +class TestCase: + testcase: str + values: Dict[str, str] + name: str = field(init=False) + + def __post_init__(self): + self.name = "_".join( + chain( + [self.testcase], + ["-".join([x, self.values.get(x)]) for x in self.values.keys()], + ) + ) + + +def check_args() -> Namespace: + parser = argparse.ArgumentParser( + description="This tool is used by the Stackable integration tests to create the final matrix of tests to run " + "based on test dimensions defined for the test suite it is running in." + ) + parser.add_argument( + "--input", + "-i", + required=True, + help="The input file (yaml format) which specifies the dimensions and their values", + ) + parser.add_argument( + "--output", + "-o", + required=True, + help="A yaml file to be read by Ansible which contains the test cases", + ) + parser.add_argument( + "--debug", + "-d", + action="store_true", + required=False, + help="Will print additional debug statements (e.g. output from all run commands)", + ) + args = parser.parse_args() + + log_level = "DEBUG" if args.debug else "INFO" + logging.basicConfig( + level=log_level, + format="%(asctime)s %(levelname)s: %(message)s", + stream=sys.stdout, + ) + return args + + +def main() -> int: + args = check_args() + result = [] + with open(args.input, encoding="utf8") as stream: + input_dimensions = yaml.safe_load(stream) + + for test_case in input_dimensions["tests"]: + dimensions = test_case["dimensions"] + used_dimensions = [ + v for v in input_dimensions["dimensions"] if v["name"] in dimensions + ] + tmp = [] + for value in used_dimensions: + tmp.append([(value["name"], x) for x in value["values"]]) + + for materialized_case in itertools.product(*tmp): + result.append( + TestCase(testcase=test_case["name"], values=dict(materialized_case)) + ) + + with open(args.output, "w", encoding="utf8") as outstream: + outputstruct = {"tests": [asdict(r) for r in result]} + logging.debug(f"Got the following output: {outputstruct}") + yaml.dump(outputstruct, outstream) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/ansible/roles/expand-tests/handlers/main.yml b/tests/ansible/roles/expand-tests/handlers/main.yml new file mode 100644 index 00000000..48dcb995 --- /dev/null +++ b/tests/ansible/roles/expand-tests/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for expand-tests diff --git a/tests/ansible/roles/expand-tests/meta/.galaxy_install_info b/tests/ansible/roles/expand-tests/meta/.galaxy_install_info new file mode 100644 index 00000000..6a2b53b7 --- /dev/null +++ b/tests/ansible/roles/expand-tests/meta/.galaxy_install_info @@ -0,0 +1,2 @@ +install_date: Wed Jul 13 08:18:32 2022 +version: '' diff --git a/tests/ansible/roles/expand-tests/meta/main.yml b/tests/ansible/roles/expand-tests/meta/main.yml new file mode 100644 index 00000000..c572acc9 --- /dev/null +++ b/tests/ansible/roles/expand-tests/meta/main.yml @@ -0,0 +1,52 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/tests/ansible/roles/expand-tests/tasks/create_scenario.yaml b/tests/ansible/roles/expand-tests/tasks/create_scenario.yaml new file mode 100644 index 00000000..e78bfc60 --- /dev/null +++ b/tests/ansible/roles/expand-tests/tasks/create_scenario.yaml @@ -0,0 +1,50 @@ +- name: Create test scenario folders + file: + path: "{{ work_dir }}/tests/{{ test_scenario.testcase }}/{{ test_scenario.name }}" + state: directory + with_items: "{{ testdefinition.tests }}" + +- name: Find template files + find: + paths: "{{ template_dir }}/kuttl/{{ test_scenario.testcase }}" + patterns: "*.j2" + file_type: file + hidden: true + use_regex: false + recurse: true + register: files_j2 + +- name: Find regular files that need no templating + find: + paths: "{{ template_dir }}/kuttl/{{ test_scenario.testcase }}" + excludes: "*.j2" + file_type: file + hidden: true + use_regex: false + recurse: true + register: files_normal + +- name: Ensure directories exist + file: + path: "{{ item.path | replace(template_dir +'/kuttl/' + test_scenario.testcase, work_dir + '/tests/' + test_scenario.testcase + '/' + test_scenario.name) | dirname }}" + state: directory + with_items: + - "{{ files_j2.files }}" + - "{{ files_normal.files }}" + register: directory_result + +- name: Copy templates files + template: + src: "{{ item.path }}" + mode: "preserve" + dest: "{{ item.path | replace(template_dir +'/kuttl/' + test_scenario.testcase, work_dir + '/tests/' + test_scenario.testcase + '/' + test_scenario.name) | regex_replace('.j2$', '') }}" + with_items: "{{ files_j2.files }}" + register: template_result + +- name: Copy normal files to temp directory + copy: + src: "{{ item.path }}" + mode: "preserve" + dest: "{{ item.path | replace(template_dir +'/kuttl/' + test_scenario.testcase, work_dir + '/tests/' + test_scenario.testcase + '/' + test_scenario.name) }}" + with_items: "{{ files_normal.files }}" + register: file_result \ No newline at end of file diff --git a/tests/ansible/roles/expand-tests/tasks/main.yml b/tests/ansible/roles/expand-tests/tasks/main.yml new file mode 100644 index 00000000..4b72c56b --- /dev/null +++ b/tests/ansible/roles/expand-tests/tasks/main.yml @@ -0,0 +1,36 @@ +--- +- name: Delete target dir for tests if it exists + file: + path: "{{ work_dir }}/tests" + state: absent + +- name: Create target dir for tests + file: + path: "{{ work_dir }}/tests" + state: directory + +- name: Create test scenarios from dimensions file + ansible.builtin.script: generate_tests.py -i {{ test_dir }}/test-definition.yaml -o {{ work_dir }}/dimensions.yaml + +- name: Read test definitions + include_vars: + file: "{{ test_dir }}/test-definition.yaml" + name: testinput + +- name: Read test scenario definitions + include_vars: + file: "{{ work_dir }}/dimensions.yaml" + name: testdefinition + +- name: Create kuttl-test.yaml + template: + src: "{{ test_dir}}/kuttl-test.yaml.jinja2" + mode: "preserve" + dest: "{{ work_dir }}/kuttl-test.yaml" + +- name: Create test scenarios + include_tasks: create_scenario.yaml + loop: "{{ testdefinition.tests }}" + loop_control: + loop_var: test_scenario + diff --git a/tests/ansible/roles/expand-tests/tests/inventory b/tests/ansible/roles/expand-tests/tests/inventory new file mode 100644 index 00000000..878877b0 --- /dev/null +++ b/tests/ansible/roles/expand-tests/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/tests/ansible/roles/expand-tests/tests/test.yml b/tests/ansible/roles/expand-tests/tests/test.yml new file mode 100644 index 00000000..bdbdb354 --- /dev/null +++ b/tests/ansible/roles/expand-tests/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - expand-tests diff --git a/tests/ansible/roles/expand-tests/vars/main.yml b/tests/ansible/roles/expand-tests/vars/main.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/tests/ansible/roles/expand-tests/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/tests/templates/kuttl/external-access/10-listener-classes.yaml b/tests/templates/kuttl/external-access/10-listener-classes.yaml index 4af48da3..893032c5 100644 --- a/tests/templates/kuttl/external-access/10-listener-classes.yaml +++ b/tests/templates/kuttl/external-access/10-listener-classes.yaml @@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - \ No newline at end of file + envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index e780987a..11ed8224 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -60,4 +60,4 @@ kind: Service metadata: name: test-hive-metastore-external-unstable spec: - type: NodePort # external-unstable \ No newline at end of file + type: NodePort # external-unstable diff --git a/tests/templates/kuttl/external-access/20-install-hive.yaml b/tests/templates/kuttl/external-access/20-install-hive.yaml index eb5b3df7..5566841d 100644 --- a/tests/templates/kuttl/external-access/20-install-hive.yaml +++ b/tests/templates/kuttl/external-access/20-install-hive.yaml @@ -5,4 +5,4 @@ timeout: 600 commands: - script: > envsubst < install-hive.yaml | - kubectl apply -n $NAMESPACE -f - \ No newline at end of file + kubectl apply -n $NAMESPACE -f - From 19b09549825f1437cb67e276407dcbc0e8665dbd Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 13:31:49 +0200 Subject: [PATCH 14/50] remove uneccessary files --- tests/ansible/roles/expand-tests/CHANGELOG.md | 12 --- tests/ansible/roles/expand-tests/README.md | 38 -------- .../roles/expand-tests/defaults/main.yml | 4 - .../expand-tests/files/generate_tests.py | 91 ------------------- .../roles/expand-tests/handlers/main.yml | 2 - .../expand-tests/meta/.galaxy_install_info | 2 - .../ansible/roles/expand-tests/meta/main.yml | 52 ----------- .../expand-tests/tasks/create_scenario.yaml | 50 ---------- .../ansible/roles/expand-tests/tasks/main.yml | 36 -------- .../roles/expand-tests/tests/inventory | 2 - .../ansible/roles/expand-tests/tests/test.yml | 5 - .../ansible/roles/expand-tests/vars/main.yml | 1 - 12 files changed, 295 deletions(-) delete mode 100644 tests/ansible/roles/expand-tests/CHANGELOG.md delete mode 100644 tests/ansible/roles/expand-tests/README.md delete mode 100644 tests/ansible/roles/expand-tests/defaults/main.yml delete mode 100755 tests/ansible/roles/expand-tests/files/generate_tests.py delete mode 100644 tests/ansible/roles/expand-tests/handlers/main.yml delete mode 100644 tests/ansible/roles/expand-tests/meta/.galaxy_install_info delete mode 100644 tests/ansible/roles/expand-tests/meta/main.yml delete mode 100644 tests/ansible/roles/expand-tests/tasks/create_scenario.yaml delete mode 100644 tests/ansible/roles/expand-tests/tasks/main.yml delete mode 100644 tests/ansible/roles/expand-tests/tests/inventory delete mode 100644 tests/ansible/roles/expand-tests/tests/test.yml delete mode 100644 tests/ansible/roles/expand-tests/vars/main.yml diff --git a/tests/ansible/roles/expand-tests/CHANGELOG.md b/tests/ansible/roles/expand-tests/CHANGELOG.md deleted file mode 100644 index 6e90657b..00000000 --- a/tests/ansible/roles/expand-tests/CHANGELOG.md +++ /dev/null @@ -1,12 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -## [Unreleased] - -### Changed - -- Generated kuttl tests prepend the original test name to the generated kuttl test folders ([#1]) - -[#1]: https://github.com/stackabletech/expand-tests/pull/1 - diff --git a/tests/ansible/roles/expand-tests/README.md b/tests/ansible/roles/expand-tests/README.md deleted file mode 100644 index 225dd44b..00000000 --- a/tests/ansible/roles/expand-tests/README.md +++ /dev/null @@ -1,38 +0,0 @@ -Role Name -========= - -A brief description of the role goes here. - -Requirements ------------- - -Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. - -Role Variables --------------- - -A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. - -Dependencies ------------- - -A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. - -Example Playbook ----------------- - -Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: - - - hosts: servers - roles: - - { role: username.rolename, x: 42 } - -License -------- - -BSD - -Author Information ------------------- - -An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/tests/ansible/roles/expand-tests/defaults/main.yml b/tests/ansible/roles/expand-tests/defaults/main.yml deleted file mode 100644 index 64f6ec50..00000000 --- a/tests/ansible/roles/expand-tests/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -work_dir: "../test-work" -test_dir: "../tests" -template_dir: "{{ test_dir }}/templates" \ No newline at end of file diff --git a/tests/ansible/roles/expand-tests/files/generate_tests.py b/tests/ansible/roles/expand-tests/files/generate_tests.py deleted file mode 100755 index 22bd162e..00000000 --- a/tests/ansible/roles/expand-tests/files/generate_tests.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -from argparse import Namespace -from dataclasses import dataclass, field, asdict -import itertools -import logging -import sys -from typing import Dict -import yaml -from itertools import chain - - -@dataclass -class TestCase: - testcase: str - values: Dict[str, str] - name: str = field(init=False) - - def __post_init__(self): - self.name = "_".join( - chain( - [self.testcase], - ["-".join([x, self.values.get(x)]) for x in self.values.keys()], - ) - ) - - -def check_args() -> Namespace: - parser = argparse.ArgumentParser( - description="This tool is used by the Stackable integration tests to create the final matrix of tests to run " - "based on test dimensions defined for the test suite it is running in." - ) - parser.add_argument( - "--input", - "-i", - required=True, - help="The input file (yaml format) which specifies the dimensions and their values", - ) - parser.add_argument( - "--output", - "-o", - required=True, - help="A yaml file to be read by Ansible which contains the test cases", - ) - parser.add_argument( - "--debug", - "-d", - action="store_true", - required=False, - help="Will print additional debug statements (e.g. output from all run commands)", - ) - args = parser.parse_args() - - log_level = "DEBUG" if args.debug else "INFO" - logging.basicConfig( - level=log_level, - format="%(asctime)s %(levelname)s: %(message)s", - stream=sys.stdout, - ) - return args - - -def main() -> int: - args = check_args() - result = [] - with open(args.input, encoding="utf8") as stream: - input_dimensions = yaml.safe_load(stream) - - for test_case in input_dimensions["tests"]: - dimensions = test_case["dimensions"] - used_dimensions = [ - v for v in input_dimensions["dimensions"] if v["name"] in dimensions - ] - tmp = [] - for value in used_dimensions: - tmp.append([(value["name"], x) for x in value["values"]]) - - for materialized_case in itertools.product(*tmp): - result.append( - TestCase(testcase=test_case["name"], values=dict(materialized_case)) - ) - - with open(args.output, "w", encoding="utf8") as outstream: - outputstruct = {"tests": [asdict(r) for r in result]} - logging.debug(f"Got the following output: {outputstruct}") - yaml.dump(outputstruct, outstream) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/ansible/roles/expand-tests/handlers/main.yml b/tests/ansible/roles/expand-tests/handlers/main.yml deleted file mode 100644 index 48dcb995..00000000 --- a/tests/ansible/roles/expand-tests/handlers/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# handlers file for expand-tests diff --git a/tests/ansible/roles/expand-tests/meta/.galaxy_install_info b/tests/ansible/roles/expand-tests/meta/.galaxy_install_info deleted file mode 100644 index 6a2b53b7..00000000 --- a/tests/ansible/roles/expand-tests/meta/.galaxy_install_info +++ /dev/null @@ -1,2 +0,0 @@ -install_date: Wed Jul 13 08:18:32 2022 -version: '' diff --git a/tests/ansible/roles/expand-tests/meta/main.yml b/tests/ansible/roles/expand-tests/meta/main.yml deleted file mode 100644 index c572acc9..00000000 --- a/tests/ansible/roles/expand-tests/meta/main.yml +++ /dev/null @@ -1,52 +0,0 @@ -galaxy_info: - author: your name - description: your role description - company: your company (optional) - - # If the issue tracker for your role is not on github, uncomment the - # next line and provide a value - # issue_tracker_url: http://example.com/issue/tracker - - # Choose a valid license ID from https://spdx.org - some suggested licenses: - # - BSD-3-Clause (default) - # - MIT - # - GPL-2.0-or-later - # - GPL-3.0-only - # - Apache-2.0 - # - CC-BY-4.0 - license: license (GPL-2.0-or-later, MIT, etc) - - min_ansible_version: 2.1 - - # If this a Container Enabled role, provide the minimum Ansible Container version. - # min_ansible_container_version: - - # - # Provide a list of supported platforms, and for each platform a list of versions. - # If you don't wish to enumerate all versions for a particular platform, use 'all'. - # To view available platforms and versions (or releases), visit: - # https://galaxy.ansible.com/api/v1/platforms/ - # - # platforms: - # - name: Fedora - # versions: - # - all - # - 25 - # - name: SomePlatform - # versions: - # - all - # - 1.0 - # - 7 - # - 99.99 - - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes - # and categorizes the role. Users find roles by searching for tags. Be sure to - # remove the '[]' above, if you add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. - -dependencies: [] - # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. diff --git a/tests/ansible/roles/expand-tests/tasks/create_scenario.yaml b/tests/ansible/roles/expand-tests/tasks/create_scenario.yaml deleted file mode 100644 index e78bfc60..00000000 --- a/tests/ansible/roles/expand-tests/tasks/create_scenario.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- name: Create test scenario folders - file: - path: "{{ work_dir }}/tests/{{ test_scenario.testcase }}/{{ test_scenario.name }}" - state: directory - with_items: "{{ testdefinition.tests }}" - -- name: Find template files - find: - paths: "{{ template_dir }}/kuttl/{{ test_scenario.testcase }}" - patterns: "*.j2" - file_type: file - hidden: true - use_regex: false - recurse: true - register: files_j2 - -- name: Find regular files that need no templating - find: - paths: "{{ template_dir }}/kuttl/{{ test_scenario.testcase }}" - excludes: "*.j2" - file_type: file - hidden: true - use_regex: false - recurse: true - register: files_normal - -- name: Ensure directories exist - file: - path: "{{ item.path | replace(template_dir +'/kuttl/' + test_scenario.testcase, work_dir + '/tests/' + test_scenario.testcase + '/' + test_scenario.name) | dirname }}" - state: directory - with_items: - - "{{ files_j2.files }}" - - "{{ files_normal.files }}" - register: directory_result - -- name: Copy templates files - template: - src: "{{ item.path }}" - mode: "preserve" - dest: "{{ item.path | replace(template_dir +'/kuttl/' + test_scenario.testcase, work_dir + '/tests/' + test_scenario.testcase + '/' + test_scenario.name) | regex_replace('.j2$', '') }}" - with_items: "{{ files_j2.files }}" - register: template_result - -- name: Copy normal files to temp directory - copy: - src: "{{ item.path }}" - mode: "preserve" - dest: "{{ item.path | replace(template_dir +'/kuttl/' + test_scenario.testcase, work_dir + '/tests/' + test_scenario.testcase + '/' + test_scenario.name) }}" - with_items: "{{ files_normal.files }}" - register: file_result \ No newline at end of file diff --git a/tests/ansible/roles/expand-tests/tasks/main.yml b/tests/ansible/roles/expand-tests/tasks/main.yml deleted file mode 100644 index 4b72c56b..00000000 --- a/tests/ansible/roles/expand-tests/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- name: Delete target dir for tests if it exists - file: - path: "{{ work_dir }}/tests" - state: absent - -- name: Create target dir for tests - file: - path: "{{ work_dir }}/tests" - state: directory - -- name: Create test scenarios from dimensions file - ansible.builtin.script: generate_tests.py -i {{ test_dir }}/test-definition.yaml -o {{ work_dir }}/dimensions.yaml - -- name: Read test definitions - include_vars: - file: "{{ test_dir }}/test-definition.yaml" - name: testinput - -- name: Read test scenario definitions - include_vars: - file: "{{ work_dir }}/dimensions.yaml" - name: testdefinition - -- name: Create kuttl-test.yaml - template: - src: "{{ test_dir}}/kuttl-test.yaml.jinja2" - mode: "preserve" - dest: "{{ work_dir }}/kuttl-test.yaml" - -- name: Create test scenarios - include_tasks: create_scenario.yaml - loop: "{{ testdefinition.tests }}" - loop_control: - loop_var: test_scenario - diff --git a/tests/ansible/roles/expand-tests/tests/inventory b/tests/ansible/roles/expand-tests/tests/inventory deleted file mode 100644 index 878877b0..00000000 --- a/tests/ansible/roles/expand-tests/tests/inventory +++ /dev/null @@ -1,2 +0,0 @@ -localhost - diff --git a/tests/ansible/roles/expand-tests/tests/test.yml b/tests/ansible/roles/expand-tests/tests/test.yml deleted file mode 100644 index bdbdb354..00000000 --- a/tests/ansible/roles/expand-tests/tests/test.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- hosts: localhost - remote_user: root - roles: - - expand-tests diff --git a/tests/ansible/roles/expand-tests/vars/main.yml b/tests/ansible/roles/expand-tests/vars/main.yml deleted file mode 100644 index ed97d539..00000000 --- a/tests/ansible/roles/expand-tests/vars/main.yml +++ /dev/null @@ -1 +0,0 @@ ---- From 84e187e96e68513f84acdf7e6d4d915aa8df04bd Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 15:14:46 +0200 Subject: [PATCH 15/50] Refactor code, adding Cargo.lock and Cargo.nix --- Cargo.lock | 279 +++++-------- Cargo.nix | 548 ++++++++++---------------- rust/operator-binary/src/discovery.rs | 60 +-- 3 files changed, 358 insertions(+), 529 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51cf5652..6cbcba2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -77,33 +77,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.8" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", "once_cell_polyfill", @@ -229,9 +229,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd0b50b1b78dbadd44ab18b3c794e496f3a139abb9fbc27d9c94c4eebbb96496" +checksum = "302eaff5357a264a2c42f127ecb8bac761cf99749fc3dc95677e2743991f99e7" dependencies = [ "fastrand", "gloo-timers", @@ -250,15 +250,9 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets 0.52.6", + "windows-targets", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -319,9 +313,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.24" +version = "1.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" +checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" dependencies = [ "jobserver", "libc", @@ -349,9 +343,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.38" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" +checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" dependencies = [ "clap_builder", "clap_derive", @@ -359,9 +353,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.38" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" +checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" dependencies = [ "anstream", "anstyle", @@ -389,9 +383,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "concurrent-queue" @@ -443,9 +437,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -963,11 +957,11 @@ dependencies = [ [[package]] name = "headers" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.21.7", + "base64", "bytes", "headers-core", "http", @@ -1131,17 +1125,21 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9f1e950e0d9d1d3c47184416723cf29c0d1f93bd8cccf37e4beb6b44f31710" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ + "base64", "bytes", "futures-channel", + "futures-core", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -1318,6 +1316,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -1411,7 +1419,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa60a41b57ae1a0a071af77dbcf89fc9819cfe66edaf2beeb204c34459dcf0b2" dependencies = [ - "base64 0.22.1", + "base64", "chrono", "schemars", "serde", @@ -1425,14 +1433,14 @@ source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-ope dependencies = [ "darling", "regex", - "snafu 0.8.5", + "snafu 0.8.6", ] [[package]] name = "kube" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b49c39074089233c2bb7b1791d1b6c06c84dbab26757491fad9d233db0d432f" +checksum = "778f98664beaf4c3c11372721e14310d1ae00f5e2d9aabcf8906c881aa4e9f51" dependencies = [ "k8s-openapi", "kube-client", @@ -1443,11 +1451,11 @@ dependencies = [ [[package]] name = "kube-client" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e199797b1b08865041c9c698f0d11a91de0a8143e808b71e250cd4a1d7ce2b9f" +checksum = "7cb276b85b6e94ded00ac8ea2c68fcf4697ea0553cb25fddc35d4a0ab718db8d" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "chrono", "either", @@ -1480,9 +1488,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bdefbba89dea2d99ea822a1d7cd6945535efbfb10b790056ee9284bf9e698e7" +checksum = "e3c56ff45deb0031f2a476017eed60c06872251f271b8387ad8020b8fef60960" dependencies = [ "chrono", "derive_more", @@ -1499,9 +1507,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e609a3633689a50869352a3c16e01d863b6137863c80eeb038383d5ab9f83bf" +checksum = "079fc8c1c397538628309cfdee20696ebdcc26745f9fb17f89b78782205bd995" dependencies = [ "darling", "proc-macro2", @@ -1513,9 +1521,9 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d4bd8a4554786f8f9a87bfa977fb7dbaa1d7f102a30477338b044b65de29d8e" +checksum = "2f1326e946fadf6248febdf8a1c001809c3899ccf48cb9768cbc536b741040dc" dependencies = [ "ahash", "async-broadcast", @@ -1582,9 +1590,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -1634,13 +1642,13 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1810,9 +1818,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -1820,15 +1828,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -1837,7 +1845,7 @@ version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" dependencies = [ - "base64 0.22.1", + "base64", "serde", ] @@ -1984,7 +1992,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.8.5", + "snafu 0.8.6", "xml-rs", ] @@ -2146,11 +2154,11 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-core", @@ -2173,12 +2181,12 @@ dependencies = [ "sync_wrapper", "tokio", "tower 0.5.2", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "windows-registry", ] [[package]] @@ -2390,7 +2398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags", - "core-foundation 0.10.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -2573,11 +2581,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +checksum = "320b01e011bf8d5d7a4a4a4be966d9160968935849c83b918827f6a435e7f627" dependencies = [ - "snafu-derive 0.8.5", + "snafu-derive 0.8.6", ] [[package]] @@ -2593,9 +2601,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7" dependencies = [ "heck", "proc-macro2", @@ -2605,9 +2613,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2637,7 +2645,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.8.5", + "snafu 0.8.6", "stackable-operator", "strum", "tokio", @@ -2668,7 +2676,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.8.5", + "snafu 0.8.6", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -2701,7 +2709,7 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.8.5", + "snafu 0.8.6", ] [[package]] @@ -2717,7 +2725,7 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry_sdk", "pin-project", - "snafu 0.8.5", + "snafu 0.8.6", "strum", "tokio", "tower 0.5.2", @@ -2920,9 +2928,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.45.0" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", @@ -3006,7 +3014,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-trait", - "base64 0.22.1", + "base64", "bytes", "flate2", "http", @@ -3065,17 +3073,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ - "base64 0.22.1", + "base64", "bitflags", "bytes", + "futures-util", "http", "http-body", + "iri-string", "mime", "pin-project-lite", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -3435,7 +3446,7 @@ dependencies = [ "windows-interface", "windows-link", "windows-result", - "windows-strings 0.4.2", + "windows-strings", ] [[package]] @@ -3466,17 +3477,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" -[[package]] -name = "windows-registry" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" -dependencies = [ - "windows-result", - "windows-strings 0.3.1", - "windows-targets 0.53.0", -] - [[package]] name = "windows-result" version = "0.3.4" @@ -3486,15 +3486,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-strings" version = "0.4.2" @@ -3510,7 +3501,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -3519,7 +3510,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -3528,30 +3519,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" -dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -3560,96 +3535,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - [[package]] name = "winnow" version = "0.7.10" diff --git a/Cargo.nix b/Cargo.nix index 5a63f756..fefccf6d 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -253,9 +253,9 @@ rec { }; "anstream" = rec { crateName = "anstream"; - version = "0.6.18"; + version = "0.6.19"; edition = "2021"; - sha256 = "16sjk4x3ns2c3ya1x28a44kh6p47c7vhk27251i015hik1lm7k4a"; + sha256 = "0crr9a207dyn8k66xgvhvmlxm9raiwpss3syfa35c6265s9z26ih"; dependencies = [ { name = "anstyle"; @@ -298,9 +298,9 @@ rec { }; "anstyle" = rec { crateName = "anstyle"; - version = "1.0.10"; + version = "1.0.11"; edition = "2021"; - sha256 = "1yai2vppmd7zlvlrp9grwll60knrmscalf8l2qpfz8b7y5lkpk2m"; + sha256 = "1gbbzi0zbgff405q14v8hhpi1kz2drzl9a75r3qhks47lindjbl6"; features = { "default" = [ "std" ]; }; @@ -308,9 +308,9 @@ rec { }; "anstyle-parse" = rec { crateName = "anstyle-parse"; - version = "0.2.6"; + version = "0.2.7"; edition = "2021"; - sha256 = "1acqayy22fwzsrvr6n0lz6a4zvjjcvgr5sm941m7m0b2fr81cb9v"; + sha256 = "1hhmkkfr95d462b3zf6yl2vfzdqfy5726ya572wwg8ha9y148xjf"; libName = "anstyle_parse"; dependencies = [ { @@ -328,9 +328,9 @@ rec { }; "anstyle-query" = rec { crateName = "anstyle-query"; - version = "1.1.2"; + version = "1.1.3"; edition = "2021"; - sha256 = "036nm3lkyk43xbps1yql3583fp4hg3b1600is7mcyxs1gzrpm53r"; + sha256 = "1sgs2hq54wayrmpvy784ww2ccv9f8yhhpasv12z872bx0jvdx2vc"; libName = "anstyle_query"; dependencies = [ { @@ -344,9 +344,9 @@ rec { }; "anstyle-wincon" = rec { crateName = "anstyle-wincon"; - version = "3.0.8"; + version = "3.0.9"; edition = "2021"; - sha256 = "1ykkvih20kykgfix7j8y74av90m2v8ji72hv373f8vmx659dx036"; + sha256 = "10n8mcgr89risdf35i73zc67aaa392bhggwzqlri1fv79297ags0"; libName = "anstyle_wincon"; dependencies = [ { @@ -761,9 +761,9 @@ rec { }; "backon" = rec { crateName = "backon"; - version = "1.5.0"; + version = "1.5.1"; edition = "2021"; - sha256 = "15k4p6xyxi4lkiyw5yxrmcws3wwnwjacgcqqmd2dvfldnyqm02zx"; + sha256 = "1rwr3ycl69vycyaxrhwzfjcwyqf7pawfq9zi88n4l9ks6pssybih"; dependencies = [ { name = "fastrand"; @@ -856,7 +856,7 @@ rec { } { name = "windows-targets"; - packageId = "windows-targets 0.52.6"; + packageId = "windows-targets"; target = { target, features }: ((target."windows" or false) || ("cygwin" == target."os" or null)); } ]; @@ -869,22 +869,7 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "base64 0.21.7" = rec { - crateName = "base64"; - version = "0.21.7"; - edition = "2018"; - sha256 = "0rw52yvsk75kar9wgqfwgb414kvil1gn7mqkrhn9zf1537mpsacx"; - authors = [ - "Alice Maz " - "Marshall Pierce " - ]; - features = { - "default" = [ "std" ]; - "std" = [ "alloc" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; - "base64 0.22.1" = rec { + "base64" = rec { crateName = "base64"; version = "0.22.1"; edition = "2018"; @@ -1035,9 +1020,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.24"; + version = "1.2.25"; edition = "2018"; - sha256 = "1irvbn8y9sg6f1070yg5469fxk5c3ximh24ds04kph21w0xmsn8n"; + sha256 = "0la999i153miffa4maj2wk5z679zkmnq6np0q1kwqrg8q5yqkz6h"; authors = [ "Alex Crichton " ]; @@ -1143,10 +1128,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.5.38"; + version = "4.5.39"; edition = "2021"; crateBin = []; - sha256 = "0060d8gx692via31il45pdnc6pix8l2hynf28bgk1acbby0bk4zd"; + sha256 = "17raqwxkhhhm80iyblp1v83fvpddkg7rgqr2cjsmz3p6kczfcq7x"; dependencies = [ { name = "clap_builder"; @@ -1185,9 +1170,9 @@ rec { }; "clap_builder" = rec { crateName = "clap_builder"; - version = "4.5.38"; + version = "4.5.39"; edition = "2021"; - sha256 = "0821n0ri2nf1xqj11q1fn78i2hhw6qs96qpan08zdb1z53zjd41p"; + sha256 = "0lggb5vscs21jliisvjjphcazzb1iw8347yp42wbwazpl6967k49"; dependencies = [ { name = "anstream"; @@ -1263,9 +1248,9 @@ rec { }; "colorchoice" = rec { crateName = "colorchoice"; - version = "1.0.3"; + version = "1.0.4"; edition = "2021"; - sha256 = "1439m3r3jy3xqck8aa13q658visn71ki76qa93cy55wkmalwlqsv"; + sha256 = "0x8ymkz1xr77rcj1cfanhf416pc4v681gmkc9dzb3jqja7f62nxh"; }; "concurrent-queue" = rec { @@ -1377,11 +1362,11 @@ rec { "random" = [ "rand" ]; }; }; - "core-foundation 0.10.0" = rec { + "core-foundation 0.10.1" = rec { crateName = "core-foundation"; - version = "0.10.0"; - edition = "2018"; - sha256 = "0qscay14s2rwkg8nd8ljhiaf149hj8sfy95d70zssy64r3jp2lmm"; + version = "0.10.1"; + edition = "2021"; + sha256 = "1xjns6dqf36rni2x9f47b65grxwdm20kwdg9lhmzdrrkwadcv9mj"; libName = "core_foundation"; authors = [ "The Servo Project Developers" @@ -1402,8 +1387,7 @@ rec { "link" = [ "core-foundation-sys/link" ]; "mac_os_10_7_support" = [ "core-foundation-sys/mac_os_10_7_support" ]; "mac_os_10_8_features" = [ "core-foundation-sys/mac_os_10_8_features" ]; - "uuid" = [ "dep:uuid" ]; - "with-uuid" = [ "uuid" ]; + "with-uuid" = [ "dep:uuid" ]; }; resolvedDefaultFeatures = [ "default" "link" ]; }; @@ -2988,16 +2972,16 @@ rec { }; "headers" = rec { crateName = "headers"; - version = "0.4.0"; - edition = "2015"; - sha256 = "1abari69kjl2yv2dg06g2x17qgd1a20xp7aqmmg2vfhcppk0c89j"; + version = "0.4.1"; + edition = "2018"; + sha256 = "1sr4zygaq1b2f0k7b5l8vx5vp05wvd82w7vpavgvr52xvdd4scdk"; authors = [ "Sean McArthur " ]; dependencies = [ { name = "base64"; - packageId = "base64 0.21.7"; + packageId = "base64"; } { name = "bytes"; @@ -3560,14 +3544,19 @@ rec { }; "hyper-util" = rec { crateName = "hyper-util"; - version = "0.1.12"; + version = "0.1.14"; edition = "2021"; - sha256 = "040pyd26pssbgvrwr35xjcghv77j7ir1ci0q8wy1v78d1saix7yg"; + sha256 = "1nqvf5azmv8p7hs5ghjlbgfya7xaafq377vppdazxbq8zzdxybyw"; libName = "hyper_util"; authors = [ "Sean McArthur " ]; dependencies = [ + { + name = "base64"; + packageId = "base64"; + optional = true; + } { name = "bytes"; packageId = "bytes"; @@ -3577,9 +3566,14 @@ rec { packageId = "futures-channel"; optional = true; } + { + name = "futures-core"; + packageId = "futures-core"; + } { name = "futures-util"; packageId = "futures-util"; + optional = true; usesDefaultFeatures = false; } { @@ -3594,11 +3588,21 @@ rec { name = "hyper"; packageId = "hyper"; } + { + name = "ipnet"; + packageId = "ipnet"; + optional = true; + } { name = "libc"; packageId = "libc"; optional = true; } + { + name = "percent-encoding"; + packageId = "percent-encoding"; + optional = true; + } { name = "pin-project-lite"; packageId = "pin-project-lite"; @@ -3633,6 +3637,12 @@ rec { name = "bytes"; packageId = "bytes"; } + { + name = "futures-util"; + packageId = "futures-util"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } { name = "hyper"; packageId = "hyper"; @@ -3646,7 +3656,7 @@ rec { ]; features = { "client" = [ "hyper/client" "dep:tracing" "dep:futures-channel" "dep:tower-service" ]; - "client-legacy" = [ "client" "dep:socket2" "tokio/sync" "dep:libc" ]; + "client-legacy" = [ "client" "dep:socket2" "tokio/sync" "dep:libc" "dep:futures-util" ]; "client-proxy" = [ "client" "dep:base64" "dep:ipnet" "dep:percent-encoding" ]; "client-proxy-system" = [ "dep:system-configuration" "dep:windows-registry" ]; "full" = [ "client" "client-legacy" "server" "server-auto" "server-graceful" "service" "http1" "http2" "tokio" "tracing" ]; @@ -3654,12 +3664,12 @@ rec { "http2" = [ "hyper/http2" ]; "server" = [ "hyper/server" ]; "server-auto" = [ "server" "http1" "http2" ]; - "server-graceful" = [ "server" "tokio/sync" "futures-util/alloc" ]; + "server-graceful" = [ "server" "tokio/sync" ]; "service" = [ "dep:tower-service" ]; "tokio" = [ "dep:tokio" "tokio/net" "tokio/rt" "tokio/time" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "client" "client-legacy" "default" "http1" "server" "service" "tokio" "tracing" ]; + resolvedDefaultFeatures = [ "client" "client-legacy" "client-proxy" "default" "http1" "server" "service" "tokio" "tracing" ]; }; "iana-time-zone" = rec { crateName = "iana-time-zone"; @@ -4186,6 +4196,39 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "iri-string" = rec { + crateName = "iri-string"; + version = "0.7.8"; + edition = "2021"; + sha256 = "1cl0wfq97wq4s1p4dl0ix5cfgsc5fn7l22ljgw9ab9x1qglypifv"; + libName = "iri_string"; + authors = [ + "YOSHIOKA Takuma " + ]; + dependencies = [ + { + name = "memchr"; + packageId = "memchr"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + optional = true; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "alloc" = [ "serde?/alloc" ]; + "default" = [ "std" ]; + "memchr" = [ "dep:memchr" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "alloc" "memchr?/std" "serde?/std" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; "is_terminal_polyfill" = rec { crateName = "is_terminal_polyfill"; version = "1.70.1"; @@ -4426,7 +4469,7 @@ rec { dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64"; usesDefaultFeatures = false; features = [ "alloc" ]; } @@ -4488,7 +4531,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.8.5"; + packageId = "snafu 0.8.6"; } ]; features = { @@ -4499,9 +4542,9 @@ rec { }; "kube" = rec { crateName = "kube"; - version = "1.0.0"; + version = "1.1.0"; edition = "2021"; - sha256 = "0bs31pdk7lnrza8p8x96mgdq8v60nv8r25vvpg1374h8fj8c6j8v"; + sha256 = "0lcz9sm83j06i77sp6idbq7y06hd64a1wwkj2g0w7x7a9dk9i3vp"; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -4572,9 +4615,9 @@ rec { }; "kube-client" = rec { crateName = "kube-client"; - version = "1.0.0"; + version = "1.1.0"; edition = "2021"; - sha256 = "17rbrvbs3m0c4lgbf2788f0hmpli3b8z1666r50m11h83dxpk6g1"; + sha256 = "13fv32vhljjxqgfmzciwanh7wsglzil2rsn81b8dx53fbfw7dckw"; libName = "kube_client"; authors = [ "clux " @@ -4584,7 +4627,7 @@ rec { dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64"; optional = true; } { @@ -4813,9 +4856,9 @@ rec { }; "kube-core" = rec { crateName = "kube-core"; - version = "1.0.0"; + version = "1.1.0"; edition = "2021"; - sha256 = "1rwqwvwlna79dq2r1dqhzgxmwls5d76xg892m2gdk8nyi6xgpphv"; + sha256 = "0q09yvzbh840mn3q66r73wjp4s60c3npw0bnlkr3207bbps6zig3"; libName = "kube_core"; authors = [ "clux " @@ -4895,9 +4938,9 @@ rec { }; "kube-derive" = rec { crateName = "kube-derive"; - version = "1.0.0"; + version = "1.1.0"; edition = "2021"; - sha256 = "1gw3kymxb0w30gmhxj33g09vcqyq05pc38sjjf3516k86cv9lq4f"; + sha256 = "15frbch851xpi5zv37szfhkcrgbfd4hfxzcw60l8clwpqg0wi7q7"; procMacro = true; libName = "kube_derive"; authors = [ @@ -4944,9 +4987,9 @@ rec { }; "kube-runtime" = rec { crateName = "kube-runtime"; - version = "1.0.0"; + version = "1.1.0"; edition = "2021"; - sha256 = "13lxw9fvci5h71rlfc1a21zivanvnxzrgykvm3wzi1j7anjdhjqx"; + sha256 = "1p2021s6nlxwiivbk37lrjcki740070a3y5xzr465pzs8vljc4rg"; libName = "kube_runtime"; authors = [ "clux " @@ -5195,9 +5238,9 @@ rec { }; "lock_api" = rec { crateName = "lock_api"; - version = "0.4.12"; + version = "0.4.13"; edition = "2021"; - sha256 = "05qvxa6g27yyva25a5ghsg85apdxkvr77yhkyhapj6r8vnf8pbq7"; + sha256 = "0rd73p4299mjwl4hhlfj9qr88v3r0kc8s1nszkfmnq2ky43nb4wn"; authors = [ "Amanieu d'Antras " ]; @@ -5333,9 +5376,9 @@ rec { }; "mio" = rec { crateName = "mio"; - version = "1.0.3"; + version = "1.0.4"; edition = "2021"; - sha256 = "1gah0h4ia3avxbwym0b6bi6lr6rpysmj9zvw6zis5yq0z0xq91i8"; + sha256 = "073n3kam3nz8j8had35fd2nn7j6a33pi3y5w3kq608cari2d9gkq"; authors = [ "Carl Lerche " "Thomas de Zeeuw " @@ -5364,7 +5407,7 @@ rec { } { name = "windows-sys"; - packageId = "windows-sys 0.52.0"; + packageId = "windows-sys 0.59.0"; target = { target, features }: (target."windows" or false); features = [ "Wdk_Foundation" "Wdk_Storage_FileSystem" "Wdk_System_IO" "Win32_Foundation" "Win32_Networking_WinSock" "Win32_Storage_FileSystem" "Win32_System_IO" "Win32_System_WindowsProgramming" ]; } @@ -5998,9 +6041,9 @@ rec { }; "parking_lot" = rec { crateName = "parking_lot"; - version = "0.12.3"; + version = "0.12.4"; edition = "2021"; - sha256 = "09ws9g6245iiq8z975h8ycf818a66q3c6zv4b5h8skpm7hc1igzi"; + sha256 = "04sab1c7304jg8k0d5b2pxbj1fvgzcf69l3n2mfpkdb96vs8pmbh"; authors = [ "Amanieu d'Antras " ]; @@ -6025,9 +6068,9 @@ rec { }; "parking_lot_core" = rec { crateName = "parking_lot_core"; - version = "0.9.10"; + version = "0.9.11"; edition = "2021"; - sha256 = "1y3cf9ld9ijf7i4igwzffcn0xl16dxyn4c5bwgjck1dkgabiyh0y"; + sha256 = "19g4d6m5k4ggacinqprnn8xvdaszc3y5smsmbz1adcdmaqm8v0xw"; authors = [ "Amanieu d'Antras " ]; @@ -6052,7 +6095,7 @@ rec { } { name = "windows-targets"; - packageId = "windows-targets 0.52.6"; + packageId = "windows-targets"; target = { target, features }: (target."windows" or false); } ]; @@ -6074,7 +6117,7 @@ rec { dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64"; usesDefaultFeatures = false; features = [ "alloc" ]; } @@ -6454,7 +6497,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.8.5"; + packageId = "snafu 0.8.6"; } { name = "xml-rs"; @@ -6970,16 +7013,16 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.15"; + version = "0.12.19"; edition = "2021"; - sha256 = "1fvvrl3jmsnlm99ldl0ariklrlsmrky06qabp7dc92ylznk4d76i"; + sha256 = "06b1mn98nsqki42233i5jm7giwzaczf0clgbhsrybwk37m8yby52"; authors = [ "Sean McArthur " ]; dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64"; } { name = "bytes"; @@ -6999,6 +7042,7 @@ rec { { name = "futures-util"; packageId = "futures-util"; + optional = true; usesDefaultFeatures = false; } { @@ -7025,7 +7069,7 @@ rec { name = "hyper-util"; packageId = "hyper-util"; target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "http1" "client" "client-legacy" "tokio" ]; + features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "ipnet"; @@ -7099,6 +7143,13 @@ rec { target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "timeout" "util" ]; } + { + name = "tower-http"; + packageId = "tower-http"; + usesDefaultFeatures = false; + target = { target, features }: (!("wasm32" == target."arch" or null)); + features = [ "follow-redirect" ]; + } { name = "tower-service"; packageId = "tower-service"; @@ -7123,11 +7174,6 @@ rec { target = { target, features }: ("wasm32" == target."arch" or null); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" ]; } - { - name = "windows-registry"; - packageId = "windows-registry"; - target = { target, features }: (target."windows" or false); - } ]; devDependencies = [ { @@ -7148,7 +7194,7 @@ rec { name = "hyper-util"; packageId = "hyper-util"; target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "http1" "http2" "client" "client-legacy" "server-auto" "tokio" ]; + features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } { name = "serde"; @@ -7177,24 +7223,24 @@ rec { } ]; features = { - "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" "dep:rustls-pemfile" "dep:rustls-pki-types" ]; + "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; - "__tls" = [ "dep:rustls-pemfile" "tokio/io-util" ]; - "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "futures-util/io" "futures-util/sink" "tokio/sync" ]; - "brotli" = [ "dep:async-compression" "async-compression?/brotli" "dep:tokio-util" ]; + "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; + "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; + "brotli" = [ "dep:async-compression" "async-compression?/brotli" "dep:futures-util" "dep:tokio-util" ]; "charset" = [ "dep:encoding_rs" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; - "default" = [ "default-tls" "charset" "http2" "macos-system-configuration" ]; + "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; - "deflate" = [ "dep:async-compression" "async-compression?/zlib" "dep:tokio-util" ]; - "gzip" = [ "dep:async-compression" "async-compression?/gzip" "dep:tokio-util" ]; + "deflate" = [ "dep:async-compression" "async-compression?/zlib" "dep:futures-util" "dep:tokio-util" ]; + "gzip" = [ "dep:async-compression" "async-compression?/gzip" "dep:futures-util" "dep:tokio-util" ]; "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" ]; "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "dep:slab" "dep:futures-channel" ]; + "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "dep:slab" "dep:futures-channel" "tokio/macros" ]; "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "dep:system-configuration" ]; - "multipart" = [ "dep:mime_guess" ]; + "macos-system-configuration" = [ "system-proxy" ]; + "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; "native-tls" = [ "default-tls" ]; "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; @@ -7207,8 +7253,9 @@ rec { "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; "socks" = [ "dep:tokio-socks" ]; - "stream" = [ "tokio/fs" "dep:tokio-util" "dep:wasm-streams" ]; - "zstd" = [ "dep:async-compression" "async-compression?/zstd" "dep:tokio-util" ]; + "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; + "system-proxy" = [ "hyper-util/client-proxy-system" ]; + "zstd" = [ "dep:async-compression" "async-compression?/zstd" "dep:futures-util" "dep:tokio-util" ]; }; resolvedDefaultFeatures = [ "blocking" ]; }; @@ -7850,7 +7897,7 @@ rec { } { name = "core-foundation"; - packageId = "core-foundation 0.10.0"; + packageId = "core-foundation 0.10.1"; } { name = "core-foundation-sys"; @@ -8374,18 +8421,18 @@ rec { }; resolvedDefaultFeatures = [ "default" "guide" "std" ]; }; - "snafu 0.8.5" = rec { + "snafu 0.8.6" = rec { crateName = "snafu"; - version = "0.8.5"; + version = "0.8.6"; edition = "2018"; - sha256 = "06ahz9g4f6475rspfcay8512x776wpxc205rizzc6a9abv492f12"; + sha256 = "09znwwss9xi7i28kpj29b29nh28nv5kfjjsa99x5v3dz27h022rj"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.8.5"; + packageId = "snafu-derive 0.8.6"; } ]; features = { @@ -8400,9 +8447,10 @@ rec { "rust_1_61" = [ "snafu-derive/rust_1_61" ]; "rust_1_65" = [ "rust_1_61" ]; "rust_1_81" = [ "rust_1_65" ]; + "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; - resolvedDefaultFeatures = [ "default" "rust_1_61" "rust_1_65" "std" ]; + resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; "snafu-derive 0.6.10" = rec { crateName = "snafu-derive"; @@ -8432,11 +8480,11 @@ rec { features = { }; }; - "snafu-derive 0.8.5" = rec { + "snafu-derive 0.8.6" = rec { crateName = "snafu-derive"; - version = "0.8.5"; + version = "0.8.6"; edition = "2018"; - sha256 = "05zr38bcngn8ha4mfi7lr2pqqfysjhwhxvk9lz57xzkzjavwdhq3"; + sha256 = "1xs7w5hg9sw45lw34mzza1nnpx9lz5snjp9s9lh2852c8bpy4q8r"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -8468,9 +8516,9 @@ rec { }; "socket2" = rec { crateName = "socket2"; - version = "0.5.9"; + version = "0.5.10"; edition = "2021"; - sha256 = "1vzds1wwwi0a51fn10r98j7cx3ir4shvhykpbk7md2h5h1ydapsg"; + sha256 = "0y067ki5q946w91xlz2sb175pnfazizva6fi3kfp639mxnmpc8z2"; authors = [ "Alex Crichton " "Thomas de Zeeuw " @@ -8570,7 +8618,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.8.5"; + packageId = "snafu 0.8.6"; } { name = "stackable-operator"; @@ -8714,7 +8762,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.8.5"; + packageId = "snafu 0.8.6"; } { name = "stackable-operator-derive"; @@ -8841,7 +8889,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.8.5"; + packageId = "snafu 0.8.6"; } ]; @@ -8900,7 +8948,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.8.5"; + packageId = "snafu 0.8.6"; } { name = "strum"; @@ -9510,9 +9558,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.45.0"; + version = "1.45.1"; edition = "2021"; - sha256 = "0rg1i83awynp1xnhz4y1klmi1jq787pa8wgy4gxy1vgr9rlwl4r5"; + sha256 = "0yb7h0mr0m0gfwdl1jir2k37gcrwhcib2kiyx9f95npi7sim3vvm"; authors = [ "Tokio Contributors " ]; @@ -9841,7 +9889,7 @@ rec { } { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64"; } { name = "bytes"; @@ -10187,9 +10235,9 @@ rec { }; "tower-http" = rec { crateName = "tower-http"; - version = "0.6.4"; + version = "0.6.6"; edition = "2018"; - sha256 = "0bladfcd75dkh3ikmf2m4f971nc0zn8b5pb9mdbryym27hhhrnqg"; + sha256 = "1wh51y4rf03f91c6rvli6nwzsarx7097yx6sqlm75ag27pbjzj5d"; libName = "tower_http"; authors = [ "Tower Maintainers " @@ -10197,7 +10245,7 @@ rec { dependencies = [ { name = "base64"; - packageId = "base64 0.22.1"; + packageId = "base64"; optional = true; } { @@ -10208,6 +10256,12 @@ rec { name = "bytes"; packageId = "bytes"; } + { + name = "futures-util"; + packageId = "futures-util"; + optional = true; + usesDefaultFeatures = false; + } { name = "http"; packageId = "http"; @@ -10217,6 +10271,11 @@ rec { packageId = "http-body"; optional = true; } + { + name = "iri-string"; + packageId = "iri-string"; + optional = true; + } { name = "mime"; packageId = "mime"; @@ -10227,6 +10286,11 @@ rec { name = "pin-project-lite"; packageId = "pin-project-lite"; } + { + name = "tower"; + packageId = "tower 0.5.2"; + optional = true; + } { name = "tower-layer"; packageId = "tower-layer"; @@ -10247,10 +10311,19 @@ rec { name = "bytes"; packageId = "bytes"; } + { + name = "futures-util"; + packageId = "futures-util"; + } { name = "http-body"; packageId = "http-body"; } + { + name = "tower"; + packageId = "tower 0.5.2"; + features = [ "buffer" "util" "retry" "make" "timeout" ]; + } ]; features = { "async-compression" = [ "dep:async-compression" ]; @@ -10268,7 +10341,7 @@ rec { "decompression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; "decompression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; "follow-redirect" = [ "futures-util" "dep:http-body" "iri-string" "tower/util" ]; - "fs" = [ "futures-util" "dep:http-body" "dep:http-body-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; + "fs" = [ "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; "futures-core" = [ "dep:futures-core" ]; "futures-util" = [ "dep:futures-util" ]; @@ -10290,7 +10363,7 @@ rec { "uuid" = [ "dep:uuid" ]; "validate-request" = [ "mime" ]; }; - resolvedDefaultFeatures = [ "auth" "base64" "default" "map-response-body" "mime" "trace" "tracing" "validate-request" ]; + resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "iri-string" "map-response-body" "mime" "tower" "trace" "tracing" "validate-request" ]; }; "tower-layer" = rec { crateName = "tower-layer"; @@ -11799,7 +11872,7 @@ rec { } { name = "windows-strings"; - packageId = "windows-strings 0.4.2"; + packageId = "windows-strings"; usesDefaultFeatures = false; } ]; @@ -11880,37 +11953,6 @@ rec { ]; }; - "windows-registry" = rec { - crateName = "windows-registry"; - version = "0.4.0"; - edition = "2021"; - sha256 = "18wbgr6z6765qdnasi8mmvxhvp82xd1zlvd6s7pp2l5lvn8av1j2"; - libName = "windows_registry"; - authors = [ - "Microsoft" - ]; - dependencies = [ - { - name = "windows-result"; - packageId = "windows-result"; - usesDefaultFeatures = false; - } - { - name = "windows-strings"; - packageId = "windows-strings 0.3.1"; - usesDefaultFeatures = false; - } - { - name = "windows-targets"; - packageId = "windows-targets 0.53.0"; - } - ]; - features = { - "default" = [ "std" ]; - "std" = [ "windows-result/std" "windows-strings/std" ]; - }; - resolvedDefaultFeatures = [ "default" "std" ]; - }; "windows-result" = rec { crateName = "windows-result"; version = "0.3.4"; @@ -11932,27 +11974,7 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; - "windows-strings 0.3.1" = rec { - crateName = "windows-strings"; - version = "0.3.1"; - edition = "2021"; - sha256 = "06bkhkyclbfchcsv5bnhz77r290k20m15glj2xq60ra0bp64iyl7"; - libName = "windows_strings"; - authors = [ - "Microsoft" - ]; - dependencies = [ - { - name = "windows-link"; - packageId = "windows-link"; - } - ]; - features = { - "default" = [ "std" ]; - }; - resolvedDefaultFeatures = [ "std" ]; - }; - "windows-strings 0.4.2" = rec { + "windows-strings" = rec { crateName = "windows-strings"; version = "0.4.2"; edition = "2021"; @@ -11985,7 +12007,7 @@ rec { dependencies = [ { name = "windows-targets"; - packageId = "windows-targets 0.52.6"; + packageId = "windows-targets"; } ]; features = { @@ -12219,7 +12241,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; + resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-sys 0.59.0" = rec { crateName = "windows-sys"; @@ -12233,7 +12255,7 @@ rec { dependencies = [ { name = "windows-targets"; - packageId = "windows-targets 0.52.6"; + packageId = "windows-targets"; } ]; features = { @@ -12478,9 +12500,9 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_SystemInformation" "Win32_UI" "Win32_UI_Shell" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Shell" "default" ]; }; - "windows-targets 0.52.6" = rec { + "windows-targets" = rec { crateName = "windows-targets"; version = "0.52.6"; edition = "2021"; @@ -12492,101 +12514,48 @@ rec { dependencies = [ { name = "windows_aarch64_gnullvm"; - packageId = "windows_aarch64_gnullvm 0.52.6"; + packageId = "windows_aarch64_gnullvm"; target = { target, features }: (target.name == "aarch64-pc-windows-gnullvm"); } { name = "windows_aarch64_msvc"; - packageId = "windows_aarch64_msvc 0.52.6"; + packageId = "windows_aarch64_msvc"; target = { target, features }: (("aarch64" == target."arch" or null) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); } { name = "windows_i686_gnu"; - packageId = "windows_i686_gnu 0.52.6"; + packageId = "windows_i686_gnu"; target = { target, features }: (("x86" == target."arch" or null) && ("gnu" == target."env" or null) && (!("llvm" == target."abi" or null)) && (!(target."windows_raw_dylib" or false))); } { name = "windows_i686_gnullvm"; - packageId = "windows_i686_gnullvm 0.52.6"; + packageId = "windows_i686_gnullvm"; target = { target, features }: (target.name == "i686-pc-windows-gnullvm"); } { name = "windows_i686_msvc"; - packageId = "windows_i686_msvc 0.52.6"; + packageId = "windows_i686_msvc"; target = { target, features }: (("x86" == target."arch" or null) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); } { name = "windows_x86_64_gnu"; - packageId = "windows_x86_64_gnu 0.52.6"; + packageId = "windows_x86_64_gnu"; target = { target, features }: (("x86_64" == target."arch" or null) && ("gnu" == target."env" or null) && (!("llvm" == target."abi" or null)) && (!(target."windows_raw_dylib" or false))); } { name = "windows_x86_64_gnullvm"; - packageId = "windows_x86_64_gnullvm 0.52.6"; + packageId = "windows_x86_64_gnullvm"; target = { target, features }: (target.name == "x86_64-pc-windows-gnullvm"); } { name = "windows_x86_64_msvc"; - packageId = "windows_x86_64_msvc 0.52.6"; + packageId = "windows_x86_64_msvc"; target = { target, features }: ((("x86_64" == target."arch" or null) || ("arm64ec" == target."arch" or null)) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); } ]; }; - "windows-targets 0.53.0" = rec { - crateName = "windows-targets"; - version = "0.53.0"; - edition = "2021"; - sha256 = "12yakpjizhfpppz1i3zgcwxlbar8axrp9j87fmywpydarvlcgr5i"; - libName = "windows_targets"; - authors = [ - "Microsoft" - ]; - dependencies = [ - { - name = "windows_aarch64_gnullvm"; - packageId = "windows_aarch64_gnullvm 0.53.0"; - target = { target, features }: (target.name == "aarch64-pc-windows-gnullvm"); - } - { - name = "windows_aarch64_msvc"; - packageId = "windows_aarch64_msvc 0.53.0"; - target = { target, features }: (("aarch64" == target."arch" or null) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); - } - { - name = "windows_i686_gnu"; - packageId = "windows_i686_gnu 0.53.0"; - target = { target, features }: (("x86" == target."arch" or null) && ("gnu" == target."env" or null) && (!("llvm" == target."abi" or null)) && (!(target."windows_raw_dylib" or false))); - } - { - name = "windows_i686_gnullvm"; - packageId = "windows_i686_gnullvm 0.53.0"; - target = { target, features }: (target.name == "i686-pc-windows-gnullvm"); - } - { - name = "windows_i686_msvc"; - packageId = "windows_i686_msvc 0.53.0"; - target = { target, features }: (("x86" == target."arch" or null) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); - } - { - name = "windows_x86_64_gnu"; - packageId = "windows_x86_64_gnu 0.53.0"; - target = { target, features }: (("x86_64" == target."arch" or null) && ("gnu" == target."env" or null) && (!("llvm" == target."abi" or null)) && (!(target."windows_raw_dylib" or false))); - } - { - name = "windows_x86_64_gnullvm"; - packageId = "windows_x86_64_gnullvm 0.53.0"; - target = { target, features }: (target.name == "x86_64-pc-windows-gnullvm"); - } - { - name = "windows_x86_64_msvc"; - packageId = "windows_x86_64_msvc 0.53.0"; - target = { target, features }: ((("x86_64" == target."arch" or null) || ("arm64ec" == target."arch" or null)) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); - } - ]; - - }; - "windows_aarch64_gnullvm 0.52.6" = rec { + "windows_aarch64_gnullvm" = rec { crateName = "windows_aarch64_gnullvm"; version = "0.52.6"; edition = "2021"; @@ -12596,17 +12565,7 @@ rec { ]; }; - "windows_aarch64_gnullvm 0.53.0" = rec { - crateName = "windows_aarch64_gnullvm"; - version = "0.53.0"; - edition = "2021"; - sha256 = "0r77pbpbcf8bq4yfwpz2hpq3vns8m0yacpvs2i5cn6fx1pwxbf46"; - authors = [ - "Microsoft" - ]; - - }; - "windows_aarch64_msvc 0.52.6" = rec { + "windows_aarch64_msvc" = rec { crateName = "windows_aarch64_msvc"; version = "0.52.6"; edition = "2021"; @@ -12616,17 +12575,7 @@ rec { ]; }; - "windows_aarch64_msvc 0.53.0" = rec { - crateName = "windows_aarch64_msvc"; - version = "0.53.0"; - edition = "2021"; - sha256 = "0v766yqw51pzxxwp203yqy39ijgjamp54hhdbsyqq6x1c8gilrf7"; - authors = [ - "Microsoft" - ]; - - }; - "windows_i686_gnu 0.52.6" = rec { + "windows_i686_gnu" = rec { crateName = "windows_i686_gnu"; version = "0.52.6"; edition = "2021"; @@ -12636,17 +12585,7 @@ rec { ]; }; - "windows_i686_gnu 0.53.0" = rec { - crateName = "windows_i686_gnu"; - version = "0.53.0"; - edition = "2021"; - sha256 = "1hvjc8nv95sx5vdd79fivn8bpm7i517dqyf4yvsqgwrmkmjngp61"; - authors = [ - "Microsoft" - ]; - - }; - "windows_i686_gnullvm 0.52.6" = rec { + "windows_i686_gnullvm" = rec { crateName = "windows_i686_gnullvm"; version = "0.52.6"; edition = "2021"; @@ -12656,17 +12595,7 @@ rec { ]; }; - "windows_i686_gnullvm 0.53.0" = rec { - crateName = "windows_i686_gnullvm"; - version = "0.53.0"; - edition = "2021"; - sha256 = "04df1in2k91qyf1wzizvh560bvyzq20yf68k8xa66vdzxnywrrlw"; - authors = [ - "Microsoft" - ]; - - }; - "windows_i686_msvc 0.52.6" = rec { + "windows_i686_msvc" = rec { crateName = "windows_i686_msvc"; version = "0.52.6"; edition = "2021"; @@ -12676,17 +12605,7 @@ rec { ]; }; - "windows_i686_msvc 0.53.0" = rec { - crateName = "windows_i686_msvc"; - version = "0.53.0"; - edition = "2021"; - sha256 = "0pcvb25fkvqnp91z25qr5x61wyya12lx8p7nsa137cbb82ayw7sq"; - authors = [ - "Microsoft" - ]; - - }; - "windows_x86_64_gnu 0.52.6" = rec { + "windows_x86_64_gnu" = rec { crateName = "windows_x86_64_gnu"; version = "0.52.6"; edition = "2021"; @@ -12696,17 +12615,7 @@ rec { ]; }; - "windows_x86_64_gnu 0.53.0" = rec { - crateName = "windows_x86_64_gnu"; - version = "0.53.0"; - edition = "2021"; - sha256 = "1flh84xkssn1n6m1riddipydcksp2pdl45vdf70jygx3ksnbam9f"; - authors = [ - "Microsoft" - ]; - - }; - "windows_x86_64_gnullvm 0.52.6" = rec { + "windows_x86_64_gnullvm" = rec { crateName = "windows_x86_64_gnullvm"; version = "0.52.6"; edition = "2021"; @@ -12716,17 +12625,7 @@ rec { ]; }; - "windows_x86_64_gnullvm 0.53.0" = rec { - crateName = "windows_x86_64_gnullvm"; - version = "0.53.0"; - edition = "2021"; - sha256 = "0mvc8119xpbi3q2m6mrjcdzl6afx4wffacp13v76g4jrs1fh6vha"; - authors = [ - "Microsoft" - ]; - - }; - "windows_x86_64_msvc 0.52.6" = rec { + "windows_x86_64_msvc" = rec { crateName = "windows_x86_64_msvc"; version = "0.52.6"; edition = "2021"; @@ -12735,16 +12634,6 @@ rec { "Microsoft" ]; - }; - "windows_x86_64_msvc 0.53.0" = rec { - crateName = "windows_x86_64_msvc"; - version = "0.53.0"; - edition = "2021"; - sha256 = "11h4i28hq0zlnjcaqi2xdxr7ibnpa8djfggch9rki1zzb8qi8517"; - authors = [ - "Microsoft" - ]; - }; "winnow" = rec { crateName = "winnow"; @@ -13915,4 +13804,3 @@ rec { # }; } - diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index ebd6d274..07f0872b 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -56,7 +56,7 @@ pub enum Error { MetadataBuild { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("failed to apply group listener for {rolegroup}"))] + #[snafu(display("{rolegroup} listener has no adress"))] RoleGroupListenerHasNoAddress { rolegroup: String }, } @@ -90,7 +90,7 @@ pub async fn build_discovery_configmaps( /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain /// [`v1alpha1::HiveCluster`]. /// -/// `hosts` will usually come from the cluster role service or [`listener_hosts`]. +/// Data is coming from the [`Listener`] objects. Connection string is only build by [`build_listener_connection_string`] fn build_discovery_configmap( name: &str, owner: &impl Resource, @@ -120,28 +120,11 @@ fn build_discovery_configmap( ); for (rolegroup, listener_ref) in listener_refs { - let listener_address = listener_ref - .status - .and_then(|s| s.ingress_addresses?.into_iter().next()) - .context(RoleGroupListenerHasNoAddressSnafu { rolegroup })?; - let mut conn_str = format!( - "thrift://{}:{}", - listener_address.address, - listener_address - .ports - .get(HIVE_PORT_NAME) - .copied() - .context(NoServicePortSnafu { - port_name: HIVE_PORT_NAME - })? + // Names are equal to role group listener name test + discovery_configmap.add_data( + rolegroup, + build_listener_connection_string(listener_ref, rolegroup, chroot)?, ); - if let Some(chroot) = chroot { - if !chroot.starts_with('/') { - return RelativeChrootSnafu { chroot }.fail(); - } - conn_str.push_str(chroot); - } - discovery_configmap.add_data(rolegroup, conn_str); } discovery_configmap @@ -150,3 +133,34 @@ fn build_discovery_configmap( obj_ref: ObjectRef::from_obj(hive), }) } + +// Builds the connection string with respect to the listener provided objects +fn build_listener_connection_string( + listener_ref: Listener, + rolegroup: &String, + chroot: Option<&str>, +) -> Result { + // We'd need only the first address corresponding to the rolegroup + let listener_address = listener_ref + .status + .and_then(|s| s.ingress_addresses?.into_iter().next()) + .context(RoleGroupListenerHasNoAddressSnafu { rolegroup })?; + let mut conn_str = format!( + "thrift://{}:{}", + listener_address.address, + listener_address + .ports + .get(HIVE_PORT_NAME) + .copied() + .context(NoServicePortSnafu { + port_name: HIVE_PORT_NAME + })? + ); + if let Some(chroot) = chroot { + if !chroot.starts_with('/') { + return RelativeChrootSnafu { chroot }.fail(); + } + conn_str.push_str(chroot); + } + Ok(conn_str) +} From 01a73290d7623d4eca7776d1f02808344e4a211c Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 15:26:24 +0200 Subject: [PATCH 16/50] Better comments --- rust/operator-binary/src/discovery.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 07f0872b..2edb7b27 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -90,7 +90,7 @@ pub async fn build_discovery_configmaps( /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain /// [`v1alpha1::HiveCluster`]. /// -/// Data is coming from the [`Listener`] objects. Connection string is only build by [`build_listener_connection_string`] +/// Data is coming from the [`Listener`] objects. Connection string is only build by [`build_listener_connection_string`]. fn build_discovery_configmap( name: &str, owner: &impl Resource, @@ -140,7 +140,7 @@ fn build_listener_connection_string( rolegroup: &String, chroot: Option<&str>, ) -> Result { - // We'd need only the first address corresponding to the rolegroup + // We only need the first address corresponding to the rolegroup let listener_address = listener_ref .status .and_then(|s| s.ingress_addresses?.into_iter().next()) From 33e73b54cc022dcefe5f2ab89b7764b2c4c85565 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 15:27:53 +0200 Subject: [PATCH 17/50] Adding changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb23aca8..036a946c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. +- BREAKING: Added listener support for Hive ([#605]) ### Changed @@ -41,6 +42,7 @@ All notable changes to this project will be documented in this file. [#599]: https://github.com/stackabletech/hive-operator/pull/599 [#603]: https://github.com/stackabletech/hive-operator/pull/603 [#604]: https://github.com/stackabletech/hive-operator/pull/604 +[#605]: https://github.com/stackabletech/hive-operator/pull/605 ## [25.3.0] - 2025-03-21 From deb0ae71215942c9ec6452ee27713318670ee4ff Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 15:35:17 +0200 Subject: [PATCH 18/50] Adding docs --- .../hive/pages/usage-guide/listenerclass.adoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/modules/hive/pages/usage-guide/listenerclass.adoc b/docs/modules/hive/pages/usage-guide/listenerclass.adoc index 02b75e53..4add5709 100644 --- a/docs/modules/hive/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/hive/pages/usage-guide/listenerclass.adoc @@ -1,16 +1,16 @@ = Service exposition with ListenerClasses +:description: Configure the Hive service exposure with listener classes: cluster-internal, external-unstable or external-stable Apache Hive offers an API. -The Operator deploys a service called `` (where `` is the name of the HiveCluster) through which Hive can be reached. +The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Nodes pod. -This service can have three different types: `cluster-internal`, `external-unstable` and `external-stable`. Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level. - -This is how the ListenerClass is configured: +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.config.listenerClass`: [source,yaml] ---- spec: - clusterConfig: - listenerClass: cluster-internal # <1> + metastore: + config: + listenerClass: external-stable # <1> ---- -<1> The default `cluster-internal` setting. +<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). From 1e4ad32991b4defe8ff2d3d57b51aa9ec2e4f0f0 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 5 Jun 2025 15:46:17 +0200 Subject: [PATCH 19/50] Making pre-commit happy --- .../external-access/helm-bitnami-postgresql-values.yaml.j2 | 2 +- tests/templates/kuttl/external-access/install-hive.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 index 215a9199..9e821599 100644 --- a/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 +++ b/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 @@ -28,4 +28,4 @@ shmVolume: auth: username: hive password: hive - database: hive \ No newline at end of file + database: hive diff --git a/tests/templates/kuttl/external-access/install-hive.yaml.j2 b/tests/templates/kuttl/external-access/install-hive.yaml.j2 index 17ab4950..dd01273b 100644 --- a/tests/templates/kuttl/external-access/install-hive.yaml.j2 +++ b/tests/templates/kuttl/external-access/install-hive.yaml.j2 @@ -41,5 +41,5 @@ metadata: name: hive-credentials type: Opaque stringData: - username: APP + username: APP password: mine From 67523a8a1d03dfc2d9b5da2030d57411ac9833a3 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Wed, 11 Jun 2025 15:27:43 +0200 Subject: [PATCH 20/50] Update CHANGELOG.md Co-authored-by: Malte Sander --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 036a946c..08bff4a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project will be documented in this file. - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. -- BREAKING: Added listener support for Hive ([#605]) +- BREAKING: Added Listener support for Hive ([#605]) ### Changed From 807278718f6e77a75e747ae3a5a641a93d8eef9b Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 15:47:46 +0200 Subject: [PATCH 21/50] Moving headless name building in method --- rust/operator-binary/src/controller.rs | 11 ++++++----- rust/operator-binary/src/discovery.rs | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 0c55b985..ab9a9962 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -97,7 +97,7 @@ use crate::{ STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR, STACKABLE_LOG_DIR_NAME, v1alpha1, }, - discovery, + discovery::{self, build_headless_listener_service_name}, kerberos::{ self, add_kerberos_pod_config, kerberos_config_properties, kerberos_container_start_commands, @@ -770,7 +770,9 @@ fn build_rolegroup_service( Ok(Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(hive) - .name(format!("{name}-metrics", name = rolegroup.object_name())) + .name(build_headless_listener_service_name( + rolegroup.object_name(), + )) .ownerreference_from_resource(hive, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( @@ -1145,9 +1147,8 @@ fn build_metastore_rolegroup_statefulset( ), ..LabelSelector::default() }, - service_name: Some(format!( - "{name}-metrics", - name = rolegroup_ref.object_name() + service_name: Some(build_headless_listener_service_name( + rolegroup_ref.object_name(), )), template: pod_template, volume_claim_templates: Some(vec![pvc]), diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 2edb7b27..a6a94eba 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -164,3 +164,7 @@ fn build_listener_connection_string( } Ok(conn_str) } + +pub fn build_headless_listener_service_name(name: String) -> String { + format!("{name}-metrics", name = name) +} From 5fff93b6796eef54378e69e0efe7d6b15e0851b8 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 15:56:08 +0200 Subject: [PATCH 22/50] Adressing some feedback --- rust/operator-binary/src/controller.rs | 3 --- rust/operator-binary/src/discovery.rs | 11 ++++------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index ab9a9962..66f373df 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -131,9 +131,6 @@ pub enum Error { #[snafu(display("object defines no metastore role"))] NoMetaStoreRole, - #[snafu(display("failed to calculate global service name"))] - GlobalServiceNameNotFound, - #[snafu(display("failed to calculate service name for role {rolegroup}"))] RoleGroupServiceNameNotFound { rolegroup: RoleGroupRef, diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index a6a94eba..32643dac 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -16,10 +16,6 @@ use crate::{ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object has no name associated"))] - NoName, - #[snafu(display("object has no namespace associated"))] - NoNamespace, #[snafu(display("object is missing metadata to build owner reference {hive}"))] ObjectMissingMetadataForOwnerRef { source: stackable_operator::builder::meta::Error, @@ -32,10 +28,10 @@ pub enum Error { source: stackable_operator::builder::configmap::Error, obj_ref: ObjectRef, }, - #[snafu(display("could not find port [{port_name}]"))] + #[snafu(display("could not find port [{port_name}] for rolegroup listener {rolegroup}"))] NoServicePort { port_name: String, - //obj_ref: ObjectRef, + rolegroup: String, }, #[snafu(display("service [{obj_ref}] port [{port_name}] does not have a nodePort "))] NoNodePort { @@ -153,7 +149,8 @@ fn build_listener_connection_string( .get(HIVE_PORT_NAME) .copied() .context(NoServicePortSnafu { - port_name: HIVE_PORT_NAME + port_name: HIVE_PORT_NAME, + rolegroup })? ); if let Some(chroot) = chroot { From 71a3463802c02f11656a75b326129f55313a1011 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Wed, 11 Jun 2025 16:05:20 +0200 Subject: [PATCH 23/50] Update tests/templates/kuttl/external-access/20-assert.yaml Co-authored-by: Malte Sander --- tests/templates/kuttl/external-access/20-assert.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index 11ed8224..6450b8cd 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -14,7 +14,6 @@ metadata: status: readyReplicas: 2 replicas: 2 - --- apiVersion: apps/v1 kind: StatefulSet From 49ead86215db35597b790ea5c02c576f001f672b Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 16:10:06 +0200 Subject: [PATCH 24/50] Fixing tests --- .../templates/kuttl/external-access/install-hive.yaml.j2 | 8 ++++---- tests/test-definition.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/templates/kuttl/external-access/install-hive.yaml.j2 b/tests/templates/kuttl/external-access/install-hive.yaml.j2 index dd01273b..b5c188a8 100644 --- a/tests/templates/kuttl/external-access/install-hive.yaml.j2 +++ b/tests/templates/kuttl/external-access/install-hive.yaml.j2 @@ -5,11 +5,11 @@ metadata: name: test-hive spec: image: -{% if test_scenario['values']['hive-latest'].find(",") > 0 %} - custom: "{{ test_scenario['values']['hive-latest'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['hive-latest'].split(',')[0] }}" +{% if test_scenario['values']['hive'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hive'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hive'].split(',')[0] }}" {% else %} - productVersion: "{{ test_scenario['values']['hive-latest'] }}" + productVersion: "{{ test_scenario['values']['hive'] }}" {% endif %} pullPolicy: IfNotPresent clusterConfig: diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 7a9106ea..b93b7b55 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -107,7 +107,7 @@ tests: - openshift - name: external-access dimensions: - - hive-latest + - hive - openshift suites: - name: nightly From 50029efebc06dc8bcfcca873be202169ae5c6f13 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 16:10:55 +0200 Subject: [PATCH 25/50] Adding vector configmap --- ...install-vector-aggregator-discovery-configmap.yaml.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 diff --git a/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} From 5996f78b93be5ef4a37784c9e4a82ddd0f9972f8 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 14:10:35 +0200 Subject: [PATCH 26/50] Adopting to listenerClass per role rather then roleGroup --- rust/operator-binary/src/config/jvm.rs | 10 ++-- rust/operator-binary/src/controller.rs | 73 +++++++++++++++----------- rust/operator-binary/src/crd/mod.rs | 55 ++++++++++++++----- rust/operator-binary/src/discovery.rs | 43 +++++++-------- 4 files changed, 108 insertions(+), 73 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index b95a4b40..ecc4d15b 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -1,13 +1,13 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, - role_utils::{self, GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role}, + role_utils::{self, JavaCommonConfig, JvmArgumentOverrides, Role}, }; use crate::crd::{ JVM_SECURITY_PROPERTIES_FILE, METRICS_PORT, MetaStoreConfig, MetaStoreConfigFragment, STACKABLE_CONFIG_DIR, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, - v1alpha1::HiveCluster, + v1alpha1::{HiveCluster, HiveMetastoreRoleConfig}, }; const JAVA_HEAP_FACTOR: f32 = 0.8; @@ -29,7 +29,7 @@ pub enum Error { /// All JVM arguments. fn construct_jvm_args( hive: &HiveCluster, - role: &Role, + role: &Role, role_group: &str, ) -> Result, Error> { let mut jvm_args = vec![ @@ -60,7 +60,7 @@ fn construct_jvm_args( /// [`construct_hadoop_heapsize_env`]). pub fn construct_non_heap_jvm_args( hive: &HiveCluster, - role: &Role, + role: &Role, role_group: &str, ) -> Result { let mut jvm_args = construct_jvm_args(hive, role, role_group)?; @@ -193,7 +193,7 @@ mod tests { ) -> ( HiveCluster, MetaStoreConfig, - Role, + Role, String, ) { let hive: HiveCluster = serde_yaml::from_str(hive_cluster).expect("illegal test input"); diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 66f373df..efd93ec5 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -92,10 +92,11 @@ use crate::{ APP_NAME, CORE_SITE_XML, Container, DB_PASSWORD_ENV, DB_USERNAME_ENV, HIVE_PORT, HIVE_PORT_NAME, HIVE_SITE_XML, HiveClusterStatus, HiveRole, JVM_SECURITY_PROPERTIES_FILE, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, - MetaStoreConfig, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, + MetaStoreConfig, MetaStoreConfigFragment, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR, - STACKABLE_LOG_DIR_NAME, v1alpha1, + STACKABLE_LOG_DIR_NAME, + v1alpha1::{self, HiveMetastoreRoleConfig}, }, discovery::{self, build_headless_listener_service_name}, kerberos::{ @@ -335,10 +336,10 @@ pub enum Error { #[snafu(display("failed to construct JVM arguments"))] ConstructJvmArguments { source: crate::config::jvm::Error }, - #[snafu(display("failed to apply group listener for {rolegroup}"))] + #[snafu(display("failed to apply group listener for {role}"))] ApplyGroupListener { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + role: String, }, #[snafu(display("failed to build listener volume"))] BuildListenerVolume { @@ -370,7 +371,11 @@ pub async fn reconcile_hive( .spec .image .resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::PKG_VERSION); - let role = hive.spec.metastore.as_ref().context(NoMetaStoreRoleSnafu)?; + let role: &stackable_operator::role_utils::Role< + MetaStoreConfigFragment, + v1alpha1::HiveMetastoreRoleConfig, + stackable_operator::role_utils::JavaCommonConfig, + > = hive.spec.metastore.as_ref().context(NoMetaStoreRoleSnafu)?; let hive_role = HiveRole::MetaStore; let s3_connection_spec: Option = @@ -440,6 +445,7 @@ pub async fn reconcile_hive( .add(client, rbac_sa) .await .context(ApplyServiceAccountSnafu)?; + cluster_resources .add(client, rbac_rolebinding) .await @@ -447,7 +453,7 @@ pub async fn reconcile_hive( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); // Collecting listener objects with corresponding rolegroup to fill the discovery configMap later on - let mut listener_refs = BTreeMap::<&String, Listener>::new(); + // let mut listener_refs = BTreeMap::<&String, Listener>::new(); for (rolegroup_name, rolegroup_config) in metastore_config.iter() { let rolegroup = hive.metastore_rolegroup_ref(rolegroup_name); @@ -477,20 +483,6 @@ pub async fn reconcile_hive( &rbac_sa.name_any(), )?; - let rg_group_listener: Listener = build_group_listener( - hive, - &resolved_product_image, - &rolegroup, - config.listener_class, - )?; - - let listener = cluster_resources - .add(client, rg_group_listener) - .await - .with_context(|_| ApplyGroupListenerSnafu { - rolegroup: rolegroup.clone(), - })?; - cluster_resources .add(client, rg_service) .await @@ -498,7 +490,7 @@ pub async fn reconcile_hive( rolegroup: rolegroup.clone(), })?; - listener_refs.insert(rolegroup_name, listener); + // listener_refs.insert(rolegroup_name, listener); cluster_resources .add(client, rg_configmap) @@ -516,15 +508,29 @@ pub async fn reconcile_hive( })?, ); } - + // Init listener struct. Collect listener after applied to cluster_resources + // to use listener object in later created discovery configMap + let mut listener = Listener::new("name", ListenerSpec::default()); let role_config = hive.role_config(&hive_role); - if let Some(GenericRoleConfig { - pod_disruption_budget: pdb, + if let Some(HiveMetastoreRoleConfig { + common: GenericRoleConfig { + pod_disruption_budget: pdb, + }, + listener_class, }) = role_config { add_pdbs(pdb, hive, &hive_role, client, &mut cluster_resources) .await .context(FailedToCreatePdbSnafu)?; + + let group_listener: Listener = + build_group_listener(hive, &resolved_product_image, &hive_role, listener_class)?; + listener = cluster_resources + .add(client, group_listener) + .await + .with_context(|_| ApplyGroupListenerSnafu { + role: hive_role.to_string(), + })?; } // std's SipHasher is deprecated, and DefaultHasher is unstable across Rust releases. @@ -534,9 +540,10 @@ pub async fn reconcile_hive( for discovery_cm in discovery::build_discovery_configmaps( hive, hive, + hive_role, &resolved_product_image, None, - listener_refs, + listener, ) .await .context(BuildDiscoveryConfigSnafu)? @@ -573,28 +580,30 @@ pub async fn reconcile_hive( Ok(Action::await_change()) } +// Designed to build a listener per role +// In case of Hive we expect only one role: Metastore pub fn build_group_listener( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - listener_class: String, + hive_role: &HiveRole, + listener_class: &String, ) -> Result { let metadata = ObjectMetaBuilder::new() .name_and_namespace(hive) - .name(hive.group_listener_name(rolegroup)) + .name(hive.group_listener_name(hive_role)) .ownerreference_from_resource(hive, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( hive, &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, + &hive_role.to_string(), + "none", )) .context(MetadataBuildSnafu)? .build(); let spec = ListenerSpec { - class_name: Some(listener_class), + class_name: Some(listener_class.to_owned()), ports: Some(listener_ports()), ..Default::default() }; @@ -1004,7 +1013,7 @@ fn build_metastore_rolegroup_statefulset( .build(); let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(hive.group_listener_name(rolegroup_ref)), + &ListenerReference::ListenerName(hive.group_listener_name(&hive_role)), &unversioned_recommended_labels, ) .context(BuildListenerVolumeSnafu)? diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index c67e5132..737fbde9 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -30,6 +30,7 @@ use stackable_operator::{ versioned::versioned, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; +use v1alpha1::HiveMetastoreRoleConfig; use crate::crd::affinity::get_affinity; @@ -134,7 +135,20 @@ pub mod versioned { // no doc - docs in Role struct. #[serde(default, skip_serializing_if = "Option::is_none")] - pub metastore: Option>, + pub metastore: + Option>, + } + + // TODO: move generic version to op-rs? + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct HiveMetastoreRoleConfig { + #[serde(flatten)] + pub common: GenericRoleConfig, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the coordinator. + #[serde(default = "metastore_default_listener_class")] + pub listener_class: String, } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] @@ -164,6 +178,19 @@ pub mod versioned { } } +impl Default for v1alpha1::HiveMetastoreRoleConfig { + fn default() -> Self { + v1alpha1::HiveMetastoreRoleConfig { + listener_class: metastore_default_listener_class(), + common: Default::default(), + } + } +} + +fn metastore_default_listener_class() -> String { + "cluster-internal".to_string() +} + impl HasStatusCondition for v1alpha1::HiveCluster { fn conditions(&self) -> Vec { match &self.status { @@ -211,7 +238,8 @@ impl v1alpha1::HiveCluster { pub fn role( &self, role_variant: &HiveRole, - ) -> Result<&Role, Error> { + ) -> Result<&Role, Error> + { match role_variant { HiveRole::MetaStore => self.spec.metastore.as_ref(), } @@ -220,11 +248,14 @@ impl v1alpha1::HiveCluster { }) } - /// The name of the group-listener provided for a specific role-group. - /// The UI will use this group listener so that only one load balancer - /// is needed (per role group). - pub fn group_listener_name(&self, rolegroup: &RoleGroupRef) -> String { - rolegroup.object_name() + /// The name of the group-listener provided for a specific role. + /// returns a name - + pub fn group_listener_name(&self, hive_role: &HiveRole) -> String { + format!( + "{name}-{role}", + name = self.name_any(), + role = hive_role.to_string() + ) } pub fn rolegroup( @@ -246,7 +277,7 @@ impl v1alpha1::HiveCluster { .cloned() } - pub fn role_config(&self, role: &HiveRole) -> Option<&GenericRoleConfig> { + pub fn role_config(&self, role: &HiveRole) -> Option<&HiveMetastoreRoleConfig> { match role { HiveRole::MetaStore => self.spec.metastore.as_ref().map(|m| &m.role_config), } @@ -417,10 +448,9 @@ pub struct MetaStoreConfig { /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, - - /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. - #[serde(default)] - pub listener_class: String, + // This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. + // #[serde(default)] + // pub listener_class: String, } impl MetaStoreConfig { @@ -462,7 +492,6 @@ impl MetaStoreConfig { logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, role), graceful_shutdown_timeout: Some(DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT), - listener_class: Some("cluster-internal".to_owned()), } } } diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 32643dac..56f06e50 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, num::TryFromIntError}; +use std::num::TryFromIntError; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -28,11 +28,8 @@ pub enum Error { source: stackable_operator::builder::configmap::Error, obj_ref: ObjectRef, }, - #[snafu(display("could not find port [{port_name}] for rolegroup listener {rolegroup}"))] - NoServicePort { - port_name: String, - rolegroup: String, - }, + #[snafu(display("could not find port [{port_name}] for rolegroup listener {role}"))] + NoServicePort { port_name: String, role: String }, #[snafu(display("service [{obj_ref}] port [{port_name}] does not have a nodePort "))] NoNodePort { port_name: String, @@ -52,8 +49,8 @@ pub enum Error { MetadataBuild { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("{rolegroup} listener has no adress"))] - RoleGroupListenerHasNoAddress { rolegroup: String }, + #[snafu(display("{role} listener has no adress"))] + RoleGroupListenerHasNoAddress { role: String }, } /// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::HiveCluster`] for all expected @@ -61,9 +58,10 @@ pub enum Error { pub async fn build_discovery_configmaps( owner: &impl Resource, hive: &v1alpha1::HiveCluster, + hive_role: HiveRole, resolved_product_image: &ResolvedProductImage, chroot: Option<&str>, - listener_refs: BTreeMap<&String, Listener>, + listener: Listener, ) -> Result, Error> { let name = owner .meta() @@ -75,9 +73,10 @@ pub async fn build_discovery_configmaps( name, owner, hive, + hive_role, resolved_product_image, chroot, - listener_refs, + listener, )?]; Ok(discovery_configmaps) @@ -91,9 +90,10 @@ fn build_discovery_configmap( name: &str, owner: &impl Resource, hive: &v1alpha1::HiveCluster, + hive_role: HiveRole, resolved_product_image: &ResolvedProductImage, chroot: Option<&str>, - listener_refs: BTreeMap<&String, Listener>, + listener: Listener, ) -> Result { let mut discovery_configmap = ConfigMapBuilder::new(); @@ -108,20 +108,17 @@ fn build_discovery_configmap( .with_recommended_labels(build_recommended_labels( hive, &resolved_product_image.app_version_label, - &HiveRole::MetaStore.to_string(), + &hive_role.to_string(), "discovery", )) .context(MetadataBuildSnafu)? .build(), ); - for (rolegroup, listener_ref) in listener_refs { - // Names are equal to role group listener name test - discovery_configmap.add_data( - rolegroup, - build_listener_connection_string(listener_ref, rolegroup, chroot)?, - ); - } + discovery_configmap.add_data( + "HIVE".to_string(), + build_listener_connection_string(listener, &hive_role.to_string(), chroot)?, + ); discovery_configmap .build() @@ -133,14 +130,14 @@ fn build_discovery_configmap( // Builds the connection string with respect to the listener provided objects fn build_listener_connection_string( listener_ref: Listener, - rolegroup: &String, + role: &String, chroot: Option<&str>, ) -> Result { - // We only need the first address corresponding to the rolegroup + // We only need the first address corresponding to the role let listener_address = listener_ref .status .and_then(|s| s.ingress_addresses?.into_iter().next()) - .context(RoleGroupListenerHasNoAddressSnafu { rolegroup })?; + .context(RoleGroupListenerHasNoAddressSnafu { role })?; let mut conn_str = format!( "thrift://{}:{}", listener_address.address, @@ -150,7 +147,7 @@ fn build_listener_connection_string( .copied() .context(NoServicePortSnafu { port_name: HIVE_PORT_NAME, - rolegroup + role })? ); if let Some(chroot) = chroot { From 7cd2af82f7822c3a8fc85a30cb04459fad0324e3 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 14:10:55 +0200 Subject: [PATCH 27/50] Updating helmcharts, regenerate charts --- deploy/helm/hive-operator/crds/crds.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/deploy/helm/hive-operator/crds/crds.yaml b/deploy/helm/hive-operator/crds/crds.yaml index cf23b837..408d443a 100644 --- a/deploy/helm/hive-operator/crds/crds.yaml +++ b/deploy/helm/hive-operator/crds/crds.yaml @@ -316,10 +316,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - nullable: true - type: string logging: default: containers: {} @@ -541,11 +537,16 @@ spec: x-kubernetes-preserve-unknown-fields: true roleConfig: default: + listenerClass: cluster-internal podDisruptionBudget: enabled: true maxUnavailable: null description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. properties: + listenerClass: + default: cluster-internal + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the coordinator. + type: string podDisruptionBudget: default: enabled: true @@ -614,10 +615,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - nullable: true - type: string logging: default: containers: {} From 42d4cbdd4736d7712752e44005336fb54ef25ac6 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 14:11:09 +0200 Subject: [PATCH 28/50] Adapting docs --- docs/modules/hive/pages/usage-guide/listenerclass.adoc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/modules/hive/pages/usage-guide/listenerclass.adoc b/docs/modules/hive/pages/usage-guide/listenerclass.adoc index 4add5709..55c197cc 100644 --- a/docs/modules/hive/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/hive/pages/usage-guide/listenerclass.adoc @@ -4,13 +4,16 @@ Apache Hive offers an API. The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Nodes pod. -The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.config.listenerClass`: +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleConfig.listenerClass`: [source,yaml] ---- spec: metastore: - config: - listenerClass: external-stable # <1> + roleConfig: + listenerClass: "cluster-internal" <1> + roleGroups: + default: + replicas: 1 ---- <1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). From 9b550d64da74ec47fcb22ed50e3975cf77b7c1d2 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 14:14:46 +0200 Subject: [PATCH 29/50] Remove some leftovers --- rust/operator-binary/src/crd/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 737fbde9..44ec366f 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -448,9 +448,6 @@ pub struct MetaStoreConfig { /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, - // This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. - // #[serde(default)] - // pub listener_class: String, } impl MetaStoreConfig { From 381aa56ce685b48985214027fab02e147a5b7f79 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 14:18:10 +0200 Subject: [PATCH 30/50] Making linting happy --- rust/operator-binary/src/controller.rs | 2 +- rust/operator-binary/src/crd/mod.rs | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index efd93ec5..1ad8f12e 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1013,7 +1013,7 @@ fn build_metastore_rolegroup_statefulset( .build(); let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(hive.group_listener_name(&hive_role)), + &ListenerReference::ListenerName(hive.group_listener_name(hive_role)), &unversioned_recommended_labels, ) .context(BuildListenerVolumeSnafu)? diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 44ec366f..7e4586ee 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -251,11 +251,7 @@ impl v1alpha1::HiveCluster { /// The name of the group-listener provided for a specific role. /// returns a name - pub fn group_listener_name(&self, hive_role: &HiveRole) -> String { - format!( - "{name}-{role}", - name = self.name_any(), - role = hive_role.to_string() - ) + format!("{name}-{role}", name = self.name_any(), role = hive_role) } pub fn rolegroup( From b08fe44e2f31ac1f26f92988f3f8d8fdf5725421 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 15:24:18 +0200 Subject: [PATCH 31/50] cleaning up old rg listener leftovers --- rust/operator-binary/src/controller.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 1ad8f12e..b893916d 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -452,8 +452,7 @@ pub async fn reconcile_hive( .context(ApplyRoleBindingSnafu)?; let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - // Collecting listener objects with corresponding rolegroup to fill the discovery configMap later on - // let mut listener_refs = BTreeMap::<&String, Listener>::new(); + for (rolegroup_name, rolegroup_config) in metastore_config.iter() { let rolegroup = hive.metastore_rolegroup_ref(rolegroup_name); @@ -490,8 +489,6 @@ pub async fn reconcile_hive( rolegroup: rolegroup.clone(), })?; - // listener_refs.insert(rolegroup_name, listener); - cluster_resources .add(client, rg_configmap) .await From d8fde0b561d3a50ec10454557ee8fb4a16404860 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 15:24:49 +0200 Subject: [PATCH 32/50] Adopting tests to role listener --- .../kuttl/external-access/20-assert.yaml | 37 ++----------------- .../external-access/install-hive.yaml.j2 | 12 +----- .../external-access/listener-classes.yaml | 15 +------- 3 files changed, 7 insertions(+), 57 deletions(-) diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index 6450b8cd..074cd864 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -11,22 +11,6 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: test-hive-metastore-default -status: - readyReplicas: 2 - replicas: 2 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test-hive-metastore-external-unstable -status: - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test-hive-metastore-cluster-internal status: readyReplicas: 1 replicas: 1 @@ -36,27 +20,14 @@ kind: PodDisruptionBudget metadata: name: test-hive-metastore status: - expectedPods: 4 - currentHealthy: 4 + expectedPods: 1 + currentHealthy: 1 disruptionsAllowed: 1 --- apiVersion: v1 kind: Service metadata: - name: test-hive-metastore-cluster-internal + name: test-hive-metastore spec: type: ClusterIP # cluster-internal ---- -apiVersion: v1 -kind: Service -metadata: - name: test-hive-metastore-default -spec: - type: NodePort # external-stable ---- -apiVersion: v1 -kind: Service -metadata: - name: test-hive-metastore-external-unstable -spec: - type: NodePort # external-unstable + diff --git a/tests/templates/kuttl/external-access/install-hive.yaml.j2 b/tests/templates/kuttl/external-access/install-hive.yaml.j2 index b5c188a8..b6f977d8 100644 --- a/tests/templates/kuttl/external-access/install-hive.yaml.j2 +++ b/tests/templates/kuttl/external-access/install-hive.yaml.j2 @@ -21,19 +21,11 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} metastore: - config: - listenerClass: test-external-stable-$NAMESPACE + roleConfig: + listenerClass: test-cluster-internal-$NAMESPACE roleGroups: default: - replicas: 2 - external-unstable: replicas: 1 - config: - listenerClass: test-external-unstable-$NAMESPACE - cluster-internal: - replicas: 1 - config: - listenerClass: test-cluster-internal-$NAMESPACE --- apiVersion: v1 kind: Secret diff --git a/tests/templates/kuttl/external-access/listener-classes.yaml b/tests/templates/kuttl/external-access/listener-classes.yaml index 4131526a..616b3fcf 100644 --- a/tests/templates/kuttl/external-access/listener-classes.yaml +++ b/tests/templates/kuttl/external-access/listener-classes.yaml @@ -5,17 +5,4 @@ metadata: name: test-cluster-internal-$NAMESPACE spec: serviceType: ClusterIP ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: ListenerClass -metadata: - name: test-external-stable-$NAMESPACE -spec: - serviceType: NodePort ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: ListenerClass -metadata: - name: test-external-unstable-$NAMESPACE -spec: - serviceType: NodePort + From 705811fb58f07b7c5301a92d28a50a0eb54fef6b Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Fri, 20 Jun 2025 16:29:18 +0200 Subject: [PATCH 33/50] Apply minor review changes Co-authored-by: Malte Sander --- CHANGELOG.md | 2 +- docs/modules/hive/pages/usage-guide/listenerclass.adoc | 4 ++-- rust/operator-binary/src/controller.rs | 6 +----- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08bff4a3..484fe0af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project will be documented in this file. - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. -- BREAKING: Added Listener support for Hive ([#605]) +- BREAKING: Add Listener support for Hive ([#605]) ### Changed diff --git a/docs/modules/hive/pages/usage-guide/listenerclass.adoc b/docs/modules/hive/pages/usage-guide/listenerclass.adoc index 55c197cc..50aa7579 100644 --- a/docs/modules/hive/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/hive/pages/usage-guide/listenerclass.adoc @@ -2,9 +2,9 @@ :description: Configure the Hive service exposure with listener classes: cluster-internal, external-unstable or external-stable Apache Hive offers an API. -The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Nodes pod. +The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Metastore pods. -The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleConfig.listenerClass`: +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.metastore.roleConfig.listenerClass`: [source,yaml] ---- diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b893916d..2c51a91c 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -371,11 +371,7 @@ pub async fn reconcile_hive( .spec .image .resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::PKG_VERSION); - let role: &stackable_operator::role_utils::Role< - MetaStoreConfigFragment, - v1alpha1::HiveMetastoreRoleConfig, - stackable_operator::role_utils::JavaCommonConfig, - > = hive.spec.metastore.as_ref().context(NoMetaStoreRoleSnafu)?; + let role = hive.spec.metastore.as_ref().context(NoMetaStoreRoleSnafu)?; let hive_role = HiveRole::MetaStore; let s3_connection_spec: Option = From aa56092ac6d67eb05174aa79794f11fdb93a5cb1 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 16:48:10 +0200 Subject: [PATCH 34/50] Adopting to review, fixing lints --- rust/operator-binary/src/controller.rs | 58 +++++++++++++------------- rust/operator-binary/src/discovery.rs | 17 +------- 2 files changed, 31 insertions(+), 44 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 2c51a91c..9ab73c8b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -92,7 +92,7 @@ use crate::{ APP_NAME, CORE_SITE_XML, Container, DB_PASSWORD_ENV, DB_USERNAME_ENV, HIVE_PORT, HIVE_PORT_NAME, HIVE_SITE_XML, HiveClusterStatus, HiveRole, JVM_SECURITY_PROPERTIES_FILE, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, - MetaStoreConfig, MetaStoreConfigFragment, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, + MetaStoreConfig, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR, STACKABLE_LOG_DIR_NAME, @@ -501,52 +501,52 @@ pub async fn reconcile_hive( })?, ); } - // Init listener struct. Collect listener after applied to cluster_resources - // to use listener object in later created discovery configMap - let mut listener = Listener::new("name", ListenerSpec::default()); + let role_config = hive.role_config(&hive_role); if let Some(HiveMetastoreRoleConfig { common: GenericRoleConfig { pod_disruption_budget: pdb, }, - listener_class, + .. }) = role_config { add_pdbs(pdb, hive, &hive_role, client, &mut cluster_resources) .await .context(FailedToCreatePdbSnafu)?; + } + + // std's SipHasher is deprecated, and DefaultHasher is unstable across Rust releases. + // We don't /need/ stability, but it's still nice to avoid spurious changes where possible. + let mut discovery_hash = FnvHasher::with_key(0); + if let Some(HiveMetastoreRoleConfig { listener_class, .. }) = role_config { let group_listener: Listener = build_group_listener(hive, &resolved_product_image, &hive_role, listener_class)?; - listener = cluster_resources + let listener = cluster_resources .add(client, group_listener) .await - .with_context(|_| ApplyGroupListenerSnafu { + .context(ApplyGroupListenerSnafu { role: hive_role.to_string(), })?; - } - - // std's SipHasher is deprecated, and DefaultHasher is unstable across Rust releases. - // We don't /need/ stability, but it's still nice to avoid spurious changes where possible. - let mut discovery_hash = FnvHasher::with_key(0); - for discovery_cm in discovery::build_discovery_configmaps( - hive, - hive, - hive_role, - &resolved_product_image, - None, - listener, - ) - .await - .context(BuildDiscoveryConfigSnafu)? - { - let discovery_cm = cluster_resources - .add(client, discovery_cm) - .await - .context(ApplyDiscoveryConfigSnafu)?; - if let Some(generation) = discovery_cm.metadata.resource_version { - discovery_hash.write(generation.as_bytes()) + for discovery_cm in discovery::build_discovery_configmaps( + hive, + hive, + hive_role, + &resolved_product_image, + None, + listener, + ) + .await + .context(BuildDiscoveryConfigSnafu)? + { + let discovery_cm = cluster_resources + .add(client, discovery_cm) + .await + .context(ApplyDiscoveryConfigSnafu)?; + if let Some(generation) = discovery_cm.metadata.resource_version { + discovery_hash.write(generation.as_bytes()) + } } } diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 56f06e50..6c2de5d1 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -1,11 +1,9 @@ -use std::num::TryFromIntError; - use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, crd::listener::v1alpha1::Listener, - k8s_openapi::api::core::v1::{ConfigMap, Service}, + k8s_openapi::api::core::v1::ConfigMap, kube::{Resource, runtime::reflector::ObjectRef}, }; @@ -30,18 +28,7 @@ pub enum Error { }, #[snafu(display("could not find port [{port_name}] for rolegroup listener {role}"))] NoServicePort { port_name: String, role: String }, - #[snafu(display("service [{obj_ref}] port [{port_name}] does not have a nodePort "))] - NoNodePort { - port_name: String, - obj_ref: ObjectRef, - }, - #[snafu(display("could not find Endpoints for {svc}"))] - FindEndpoints { - source: stackable_operator::client::Error, - svc: ObjectRef, - }, - #[snafu(display("nodePort was out of range"))] - InvalidNodePort { source: TryFromIntError }, + #[snafu(display("invalid owner name for discovery ConfigMap"))] InvalidOwnerNameForDiscoveryConfigMap, From 760573ffb54ca4663423038d1ab8ff1394ef807f Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 16:48:37 +0200 Subject: [PATCH 35/50] Adopting tests to role listener --- tests/templates/kuttl/external-access/20-assert.yaml | 1 - tests/templates/kuttl/external-access/listener-classes.yaml | 1 - .../kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 | 6 +++--- .../kuttl/kerberos-s3/70-install-access-hive.yaml.j2 | 6 +++--- tests/templates/kuttl/smoke/80-assert.yaml | 2 +- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index 074cd864..5f588b7b 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -30,4 +30,3 @@ metadata: name: test-hive-metastore spec: type: ClusterIP # cluster-internal - diff --git a/tests/templates/kuttl/external-access/listener-classes.yaml b/tests/templates/kuttl/external-access/listener-classes.yaml index 616b3fcf..588af5f5 100644 --- a/tests/templates/kuttl/external-access/listener-classes.yaml +++ b/tests/templates/kuttl/external-access/listener-classes.yaml @@ -5,4 +5,3 @@ metadata: name: test-cluster-internal-$NAMESPACE spec: serviceType: ClusterIP - diff --git a/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 index e5536516..c258f6fe 100644 --- a/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 @@ -69,7 +69,7 @@ data: kinit -kt /stackable/kerberos/keytab access-hive/access-hive."$NAMESPACE".svc.cluster.local klist - python /tmp/scripts/metastore.py -m hive-metastore-default.$NAMESPACE.svc.cluster.local + python /tmp/scripts/metastore.py -m hive-metastore.$NAMESPACE.svc.cluster.local metastore.py: | from hive_metastore_client import HiveMetastoreClient from hive_metastore_client.builders import ( @@ -92,9 +92,9 @@ data: @staticmethod def _init_protocol(host: str, port: int) -> TBinaryProtocol: transport = TSocket.TSocket(host, int(port)) - # host is something like "hive-metastore-default..kuttl-test-brave-caribou.svc.cluster.local" + # host is something like "hive-metastore..kuttl-test-brave-caribou.svc.cluster.local" # We need to change it to the host part of the HMS principal e.g. "hive.kuttl-test-brave-caribou.svc.cluster.local" - host = host.replace("hive-metastore-default", "hive") + host = host.replace("hive-metastore", "hive") transport = TTransport.TSaslClientTransport(transport, # host part of the HMS principal (not the HMS client) host=host, diff --git a/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 b/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 index ed316cb9..8bb592e1 100644 --- a/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-s3/70-install-access-hive.yaml.j2 @@ -69,7 +69,7 @@ data: kinit -kt /stackable/kerberos/keytab access-hive/access-hive."$NAMESPACE".svc.cluster.local klist - python /tmp/scripts/metastore.py -m hive-metastore-default.$NAMESPACE.svc.cluster.local + python /tmp/scripts/metastore.py -m hive-metastore.$NAMESPACE.svc.cluster.local metastore.py: | from hive_metastore_client import HiveMetastoreClient from hive_metastore_client.builders import ( @@ -92,9 +92,9 @@ data: @staticmethod def _init_protocol(host: str, port: int) -> TBinaryProtocol: transport = TSocket.TSocket(host, int(port)) - # host is something like "hive-metastore-default.kuttl-test-brave-caribou.svc.cluster.local" + # host is something like "hive-metastore.kuttl-test-brave-caribou.svc.cluster.local" # We need to change it to the host part of the HMS principal e.g. "hive.kuttl-test-brave-caribou.svc.cluster.local" - host = host.replace("hive-metastore-default", "hive") + host = host.replace("hive-metastore", "hive") transport = TTransport.TSaslClientTransport(transport, # host part of the HMS principal (not the HMS client) host=host, diff --git a/tests/templates/kuttl/smoke/80-assert.yaml b/tests/templates/kuttl/smoke/80-assert.yaml index 2bf0f4dc..ee864121 100644 --- a/tests/templates/kuttl/smoke/80-assert.yaml +++ b/tests/templates/kuttl/smoke/80-assert.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - - script: kubectl exec -n $NAMESPACE test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore-default.$NAMESPACE.svc.cluster.local + - script: kubectl exec -n $NAMESPACE test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore.$NAMESPACE.svc.cluster.local From cae1450842a862526daa47b330039fd74dd34287 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 16:54:41 +0200 Subject: [PATCH 36/50] Adressing feedback --- rust/operator-binary/src/controller.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 9ab73c8b..22a6b16d 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -983,13 +983,12 @@ fn build_metastore_rolegroup_statefulset( } } - let recommended_object_labels: ObjectLabels<'_, v1alpha1::HiveCluster> = - build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ); + let recommended_object_labels = build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); // Used for PVC templates that cannot be modified once they are deployed let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( hive, From 625c225736561ea1aac50d12bd52dcd3e219f6ee Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 17:01:42 +0200 Subject: [PATCH 37/50] fix ophaned ressources test --- tests/templates/kuttl/orphaned-resources/04-assert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/orphaned-resources/04-assert.yaml b/tests/templates/kuttl/orphaned-resources/04-assert.yaml index fccf7ec2..763a8795 100644 --- a/tests/templates/kuttl/orphaned-resources/04-assert.yaml +++ b/tests/templates/kuttl/orphaned-resources/04-assert.yaml @@ -23,4 +23,4 @@ metadata: apiVersion: v1 kind: Service metadata: - name: test-hive-metastore-newrolegroup + name: test-hive-metastore From 40e28de8310ca04bcc0aac748304a5037dae934d Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 20 Jun 2025 17:08:33 +0200 Subject: [PATCH 38/50] making rustdocs happy --- rust/operator-binary/src/crd/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 7e4586ee..17ec98a9 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -249,7 +249,7 @@ impl v1alpha1::HiveCluster { } /// The name of the group-listener provided for a specific role. - /// returns a name - + /// returns a name `-` pub fn group_listener_name(&self, hive_role: &HiveRole) -> String { format!("{name}-{role}", name = self.name_any(), role = hive_role) } From 675cb1be594eaeda60732201cf18cfda627442e8 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Mon, 23 Jun 2025 11:04:51 +0200 Subject: [PATCH 39/50] Update tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 Co-authored-by: Malte Sander --- .../kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 index c258f6fe..2af52b52 100644 --- a/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/70-install-access-hive.yaml.j2 @@ -92,7 +92,7 @@ data: @staticmethod def _init_protocol(host: str, port: int) -> TBinaryProtocol: transport = TSocket.TSocket(host, int(port)) - # host is something like "hive-metastore..kuttl-test-brave-caribou.svc.cluster.local" + # host is something like "hive-metastore.kuttl-test-brave-caribou.svc.cluster.local" # We need to change it to the host part of the HMS principal e.g. "hive.kuttl-test-brave-caribou.svc.cluster.local" host = host.replace("hive-metastore", "hive") transport = TTransport.TSaslClientTransport(transport, From 99eeb9229cc9baa4044b183f0ad77362cf59521c Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Mon, 23 Jun 2025 11:05:03 +0200 Subject: [PATCH 40/50] Update CHANGELOG.md Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 484fe0af..d7866a8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project will be documented in this file. - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. -- BREAKING: Add Listener support for Hive ([#605]) +- BREAKING: Add Listener support for Hive ([#605]). ### Changed From 35c859237f6377f14a2aff9d7698b6210f56cecb Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Jun 2025 11:10:49 +0200 Subject: [PATCH 41/50] Test for metrics port, rename metrics svc name function --- rust/operator-binary/src/controller.rs | 6 +++--- rust/operator-binary/src/discovery.rs | 2 +- tests/templates/kuttl/external-access/20-assert.yaml | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 22a6b16d..a2589123 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -98,7 +98,7 @@ use crate::{ STACKABLE_LOG_DIR_NAME, v1alpha1::{self, HiveMetastoreRoleConfig}, }, - discovery::{self, build_headless_listener_service_name}, + discovery::{self, build_headless_role_group_metrics_service_name}, kerberos::{ self, add_kerberos_pod_config, kerberos_config_properties, kerberos_container_start_commands, @@ -769,7 +769,7 @@ fn build_rolegroup_service( Ok(Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(hive) - .name(build_headless_listener_service_name( + .name(build_headless_role_group_metrics_service_name( rolegroup.object_name(), )) .ownerreference_from_resource(hive, None, Some(true)) @@ -1145,7 +1145,7 @@ fn build_metastore_rolegroup_statefulset( ), ..LabelSelector::default() }, - service_name: Some(build_headless_listener_service_name( + service_name: Some(build_headless_role_group_metrics_service_name( rolegroup_ref.object_name(), )), template: pod_template, diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 6c2de5d1..f687ef7e 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -146,6 +146,6 @@ fn build_listener_connection_string( Ok(conn_str) } -pub fn build_headless_listener_service_name(name: String) -> String { +pub fn build_headless_role_group_metrics_service_name(name: String) -> String { format!("{name}-metrics", name = name) } diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index 5f588b7b..45ac3df2 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -30,3 +30,10 @@ metadata: name: test-hive-metastore spec: type: ClusterIP # cluster-internal +--- +apiVersion: v1 +kind: Service +metadata: + name: test-hive-metastore-default-metrics +spec: + type: ClusterIP # exposed metrics From 43d5e383336e47b862dad059e9ae66fbe72787e2 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Jun 2025 11:14:28 +0200 Subject: [PATCH 42/50] Trailing whitespace --- tests/templates/kuttl/external-access/20-assert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index 45ac3df2..e6e05c21 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -34,6 +34,6 @@ spec: apiVersion: v1 kind: Service metadata: - name: test-hive-metastore-default-metrics + name: test-hive-metastore-default-metrics spec: type: ClusterIP # exposed metrics From ff4e092d64440617df253dd2b8944136889b2d04 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Mon, 23 Jun 2025 12:41:40 +0200 Subject: [PATCH 43/50] Update error to `RoleListenerHasNoAddress` Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- rust/operator-binary/src/discovery.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index f687ef7e..947324e7 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -37,7 +37,7 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, #[snafu(display("{role} listener has no adress"))] - RoleGroupListenerHasNoAddress { role: String }, + RoleListenerHasNoAddress { role: String }, } /// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::HiveCluster`] for all expected @@ -124,7 +124,7 @@ fn build_listener_connection_string( let listener_address = listener_ref .status .and_then(|s| s.ingress_addresses?.into_iter().next()) - .context(RoleGroupListenerHasNoAddressSnafu { role })?; + .context(RoleListenerHasNoAddressSnafu { role })?; let mut conn_str = format!( "thrift://{}:{}", listener_address.address, From 8567a5288ef9fde1ca7a6d8350cc224b7603a893 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Mon, 23 Jun 2025 12:42:51 +0200 Subject: [PATCH 44/50] Use named placeholders in discovery.rs Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- rust/operator-binary/src/discovery.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 947324e7..8db4809f 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -126,9 +126,9 @@ fn build_listener_connection_string( .and_then(|s| s.ingress_addresses?.into_iter().next()) .context(RoleListenerHasNoAddressSnafu { role })?; let mut conn_str = format!( - "thrift://{}:{}", - listener_address.address, - listener_address + "thrift://{address}:{port}", + address = listener_address.address, + port = listener_address .ports .get(HIVE_PORT_NAME) .copied() From 8637ecdf1f80cb942ef2cf78c75344d9266ae4af Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Jun 2025 12:49:30 +0200 Subject: [PATCH 45/50] Using const for default listener class --- rust/operator-binary/src/crd/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 17ec98a9..d564fe14 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -65,6 +65,9 @@ pub const METRICS_PORT: u16 = 9084; pub const LISTENER_VOLUME_NAME: &str = "listener"; pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; +// Listener defaults +pub const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; + // Certificates and trust stores pub const SYSTEM_TRUST_STORE: &str = "/etc/pki/java/cacerts"; pub const SYSTEM_TRUST_STORE_PASSWORD: &str = "changeit"; @@ -188,7 +191,7 @@ impl Default for v1alpha1::HiveMetastoreRoleConfig { } fn metastore_default_listener_class() -> String { - "cluster-internal".to_string() + DEFAULT_LISTENER_CLASS.to_owned() } impl HasStatusCondition for v1alpha1::HiveCluster { From 247b45002dedea00642e628b7ba6e3c291f74576 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Jun 2025 13:30:58 +0200 Subject: [PATCH 46/50] using "" through all integration tests with shell interaction --- .../kuttl/cluster-operation/00-patch-ns.yaml.j2 | 2 +- .../kuttl/cluster-operation/10-assert.yaml | 2 +- .../kuttl/cluster-operation/20-assert.yaml | 2 +- .../kuttl/cluster-operation/30-assert.yaml | 2 +- .../kuttl/cluster-operation/40-assert.yaml | 2 +- .../kuttl/external-access/00-patch-ns.yaml.j2 | 2 +- .../kuttl/external-access/10-listener-classes.yaml | 2 +- .../templates/kuttl/external-access/20-assert.yaml | 2 +- .../kuttl/external-access/20-install-hive.yaml | 2 +- .../kuttl/kerberos-hdfs/00-patch-ns.yaml.j2 | 2 +- .../02-create-kerberos-secretclass.yaml.j2 | 2 +- .../kuttl/kerberos-hdfs/30-install-hdfs.yaml.j2 | 2 +- .../kuttl/kerberos-hdfs/35-access-hdfs.yaml.j2 | 2 +- .../kerberos-hdfs/40-install-postgres.yaml.j2 | 2 +- .../kuttl/kerberos-hdfs/60-install-hive.yaml.j2 | 2 +- .../kerberos-hdfs/70-install-access-hive.yaml.j2 | 2 +- .../kuttl/kerberos-s3/00-patch-ns.yaml.j2 | 2 +- .../02-create-kerberos-secretclass.yaml.j2 | 2 +- .../kuttl/kerberos-s3/30-setup-minio.yaml.j2 | 2 +- .../kuttl/kerberos-s3/40-install-postgres.yaml.j2 | 2 +- .../kuttl/kerberos-s3/60-install-hive.yaml.j2 | 2 +- .../kerberos-s3/70-install-access-hive.yaml.j2 | 2 +- tests/templates/kuttl/logging/00-patch-ns.yaml.j2 | 2 +- .../01-install-hbase-vector-aggregator.yaml | 2 +- .../kuttl/logging/02-install-postgres.yaml.j2 | 2 +- .../03-create-configmap-with-prepared-logs.yaml | 2 +- tests/templates/kuttl/logging/06-assert.yaml | 4 ++-- .../kuttl/logging/06-test-log-aggregation.yaml | 2 +- .../kuttl/orphaned-resources/00-patch-ns.yaml.j2 | 2 +- .../templates/kuttl/resources/00-patch-ns.yaml.j2 | 2 +- tests/templates/kuttl/resources/20-assert.yaml | 4 ++-- tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 | 2 +- tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 | 2 +- .../kuttl/smoke/40-install-postgres.yaml.j2 | 2 +- tests/templates/kuttl/smoke/61-assert.yaml | 14 +++++++------- tests/templates/kuttl/smoke/62-assert.yaml | 2 +- tests/templates/kuttl/smoke/80-assert.yaml | 2 +- .../kuttl/smoke/80-prepare-test-metastore.yaml | 2 +- tests/templates/kuttl/upgrade/00-patch-ns.yaml.j2 | 2 +- .../kuttl/upgrade/20-install-postgres.yaml.j2 | 2 +- 40 files changed, 48 insertions(+), 48 deletions(-) diff --git a/tests/templates/kuttl/cluster-operation/00-patch-ns.yaml.j2 b/tests/templates/kuttl/cluster-operation/00-patch-ns.yaml.j2 index 67185acf..7400f1ee 100644 --- a/tests/templates/kuttl/cluster-operation/00-patch-ns.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/00-patch-ns.yaml.j2 @@ -4,6 +4,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + - script: kubectl patch namespace "$NAMESPACE" -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' timeout: 120 {% endif %} diff --git a/tests/templates/kuttl/cluster-operation/10-assert.yaml b/tests/templates/kuttl/cluster-operation/10-assert.yaml index 0cfb577a..9aa93dcc 100644 --- a/tests/templates/kuttl/cluster-operation/10-assert.yaml +++ b/tests/templates/kuttl/cluster-operation/10-assert.yaml @@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available hiveclusters.hive.stackable.tech/test-hive --timeout 601s + - script: kubectl -n "$NAMESPACE" wait --for=condition=available hiveclusters.hive.stackable.tech/test-hive --timeout 601s --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/templates/kuttl/cluster-operation/20-assert.yaml b/tests/templates/kuttl/cluster-operation/20-assert.yaml index bb7b1fd0..81aed253 100644 --- a/tests/templates/kuttl/cluster-operation/20-assert.yaml +++ b/tests/templates/kuttl/cluster-operation/20-assert.yaml @@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 commands: - - script: kubectl -n $NAMESPACE wait --for=condition=stopped hiveclusters.hive.stackable.tech/test-hive --timeout 301s + - script: kubectl -n "$NAMESPACE" wait --for=condition=stopped hiveclusters.hive.stackable.tech/test-hive --timeout 301s --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/templates/kuttl/cluster-operation/30-assert.yaml b/tests/templates/kuttl/cluster-operation/30-assert.yaml index 9be3bef9..b14c9f26 100644 --- a/tests/templates/kuttl/cluster-operation/30-assert.yaml +++ b/tests/templates/kuttl/cluster-operation/30-assert.yaml @@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 commands: - - script: kubectl -n $NAMESPACE wait --for=condition=reconciliationPaused hiveclusters.hive.stackable.tech/test-hive --timeout 301s + - script: kubectl -n "$NAMESPACE" wait --for=condition=reconciliationPaused hiveclusters.hive.stackable.tech/test-hive --timeout 301s --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/templates/kuttl/cluster-operation/40-assert.yaml b/tests/templates/kuttl/cluster-operation/40-assert.yaml index 0cfb577a..9aa93dcc 100644 --- a/tests/templates/kuttl/cluster-operation/40-assert.yaml +++ b/tests/templates/kuttl/cluster-operation/40-assert.yaml @@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available hiveclusters.hive.stackable.tech/test-hive --timeout 601s + - script: kubectl -n "$NAMESPACE" wait --for=condition=available hiveclusters.hive.stackable.tech/test-hive --timeout 601s --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 index 67185acf..7400f1ee 100644 --- a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 @@ -4,6 +4,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + - script: kubectl patch namespace "$NAMESPACE" -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' timeout: 120 {% endif %} diff --git a/tests/templates/kuttl/external-access/10-listener-classes.yaml b/tests/templates/kuttl/external-access/10-listener-classes.yaml index 893032c5..842e17e5 100644 --- a/tests/templates/kuttl/external-access/10-listener-classes.yaml +++ b/tests/templates/kuttl/external-access/10-listener-classes.yaml @@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - + envsubst < listener-classes.yaml | kubectl apply -n "$NAMESPACE" -f - diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index e6e05c21..58842f02 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -5,7 +5,7 @@ timeout: 600 metadata: name: install-hive commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available hiveclusters.hive.stackable.tech/test-hive --timeout 601s + - script: kubectl -n "$NAMESPACE" wait --for=condition=available hiveclusters.hive.stackable.tech/test-hive --timeout 601s --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/templates/kuttl/external-access/20-install-hive.yaml b/tests/templates/kuttl/external-access/20-install-hive.yaml index 5566841d..73d580d6 100644 --- a/tests/templates/kuttl/external-access/20-install-hive.yaml +++ b/tests/templates/kuttl/external-access/20-install-hive.yaml @@ -5,4 +5,4 @@ timeout: 600 commands: - script: > envsubst < install-hive.yaml | - kubectl apply -n $NAMESPACE -f - + kubectl apply -n "$NAMESPACE" -f - diff --git a/tests/templates/kuttl/kerberos-hdfs/00-patch-ns.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/00-patch-ns.yaml.j2 index 67185acf..7400f1ee 100644 --- a/tests/templates/kuttl/kerberos-hdfs/00-patch-ns.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/00-patch-ns.yaml.j2 @@ -4,6 +4,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + - script: kubectl patch namespace "$NAMESPACE" -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' timeout: 120 {% endif %} diff --git a/tests/templates/kuttl/kerberos-hdfs/02-create-kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/02-create-kerberos-secretclass.yaml.j2 index 560704b3..c016e25a 100644 --- a/tests/templates/kuttl/kerberos-hdfs/02-create-kerberos-secretclass.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/02-create-kerberos-secretclass.yaml.j2 @@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - kubectl apply -n $NAMESPACE -f - <- helm install postgresql --version={{ test_scenario['values']['postgres'] }} - --namespace $NAMESPACE + --namespace "$NAMESPACE" -f helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 index 1a47cb0c..840be987 100644 --- a/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 @@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - kubectl apply -n $NAMESPACE -f - <- helm install minio - --namespace $NAMESPACE + --namespace "$NAMESPACE" --version 12.6.4 -f helm-bitnami-minio-values.yaml --repo https://charts.bitnami.com/bitnami minio diff --git a/tests/templates/kuttl/kerberos-s3/40-install-postgres.yaml.j2 b/tests/templates/kuttl/kerberos-s3/40-install-postgres.yaml.j2 index 0076b435..aa7e4108 100644 --- a/tests/templates/kuttl/kerberos-s3/40-install-postgres.yaml.j2 +++ b/tests/templates/kuttl/kerberos-s3/40-install-postgres.yaml.j2 @@ -5,6 +5,6 @@ commands: - script: >- helm install postgresql --version={{ test_scenario['values']['postgres'] }} - --namespace $NAMESPACE + --namespace "$NAMESPACE" -f helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/kerberos-s3/60-install-hive.yaml.j2 b/tests/templates/kuttl/kerberos-s3/60-install-hive.yaml.j2 index 5dc77ede..d60c0ef9 100644 --- a/tests/templates/kuttl/kerberos-s3/60-install-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-s3/60-install-hive.yaml.j2 @@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - kubectl apply -n $NAMESPACE -f - <- helm install hive-vector-aggregator vector - --namespace $NAMESPACE + --namespace "$NAMESPACE" --version 0.43.0 --repo https://helm.vector.dev --values hive-vector-aggregator-values.yaml diff --git a/tests/templates/kuttl/logging/02-install-postgres.yaml.j2 b/tests/templates/kuttl/logging/02-install-postgres.yaml.j2 index 3bcd0134..0e88ca58 100644 --- a/tests/templates/kuttl/logging/02-install-postgres.yaml.j2 +++ b/tests/templates/kuttl/logging/02-install-postgres.yaml.j2 @@ -5,6 +5,6 @@ commands: - script: >- helm install hive --version={{ test_scenario['values']['postgres'] }} - --namespace $NAMESPACE + --namespace "$NAMESPACE" -f helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/logging/03-create-configmap-with-prepared-logs.yaml b/tests/templates/kuttl/logging/03-create-configmap-with-prepared-logs.yaml index df6308c2..e6af2aee 100644 --- a/tests/templates/kuttl/logging/03-create-configmap-with-prepared-logs.yaml +++ b/tests/templates/kuttl/logging/03-create-configmap-with-prepared-logs.yaml @@ -5,4 +5,4 @@ commands: - script: > kubectl create configmap prepared-logs --from-file=prepared-logs.log4j2.xml - --namespace=$NAMESPACE + --namespace="$NAMESPACE" diff --git a/tests/templates/kuttl/logging/06-assert.yaml b/tests/templates/kuttl/logging/06-assert.yaml index 2262ea03..d7adc692 100644 --- a/tests/templates/kuttl/logging/06-assert.yaml +++ b/tests/templates/kuttl/logging/06-assert.yaml @@ -3,5 +3,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - script: >- - kubectl exec --namespace=$NAMESPACE hive-test-runner-0 -- - python /tmp/test_log_aggregation.py -n $NAMESPACE + kubectl exec --namespace="$NAMESPACE" hive-test-runner-0 -- + python /tmp/test_log_aggregation.py -n "$NAMESPACE" diff --git a/tests/templates/kuttl/logging/06-test-log-aggregation.yaml b/tests/templates/kuttl/logging/06-test-log-aggregation.yaml index 80ca473f..fb124ab1 100644 --- a/tests/templates/kuttl/logging/06-test-log-aggregation.yaml +++ b/tests/templates/kuttl/logging/06-test-log-aggregation.yaml @@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - kubectl cp ./test_log_aggregation.py $NAMESPACE/hive-test-runner-0:/tmp + kubectl cp ./test_log_aggregation.py "$NAMESPACE/hive-test-runner-0:/tmp" diff --git a/tests/templates/kuttl/orphaned-resources/00-patch-ns.yaml.j2 b/tests/templates/kuttl/orphaned-resources/00-patch-ns.yaml.j2 index 67185acf..7400f1ee 100644 --- a/tests/templates/kuttl/orphaned-resources/00-patch-ns.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/00-patch-ns.yaml.j2 @@ -4,6 +4,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + - script: kubectl patch namespace "$NAMESPACE" -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' timeout: 120 {% endif %} diff --git a/tests/templates/kuttl/resources/00-patch-ns.yaml.j2 b/tests/templates/kuttl/resources/00-patch-ns.yaml.j2 index 67185acf..7400f1ee 100644 --- a/tests/templates/kuttl/resources/00-patch-ns.yaml.j2 +++ b/tests/templates/kuttl/resources/00-patch-ns.yaml.j2 @@ -4,6 +4,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + - script: kubectl patch namespace "$NAMESPACE" -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' timeout: 120 {% endif %} diff --git a/tests/templates/kuttl/resources/20-assert.yaml b/tests/templates/kuttl/resources/20-assert.yaml index 34cfcdad..8aa5c645 100644 --- a/tests/templates/kuttl/resources/20-assert.yaml +++ b/tests/templates/kuttl/resources/20-assert.yaml @@ -3,5 +3,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - - script: kubectl -n $NAMESPACE get sts hive-metastore-resources-from-role -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "HADOOP_HEAPSIZE" and .value == "3276")' - - script: kubectl -n $NAMESPACE get sts hive-metastore-resources-from-role-group -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "HADOOP_HEAPSIZE" and .value == "2457")' + - script: kubectl -n "$NAMESPACE" get sts hive-metastore-resources-from-role -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "HADOOP_HEAPSIZE" and .value == "3276")' + - script: kubectl -n "$NAMESPACE" get sts hive-metastore-resources-from-role-group -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "HADOOP_HEAPSIZE" and .value == "2457")' diff --git a/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 index 67185acf..7400f1ee 100644 --- a/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 +++ b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 @@ -4,6 +4,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + - script: kubectl patch namespace "$NAMESPACE" -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' timeout: 120 {% endif %} diff --git a/tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 b/tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 index 1cf27c53..d5070277 100644 --- a/tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-setup-minio.yaml.j2 @@ -7,7 +7,7 @@ kind: TestStep commands: - script: >- helm install minio - --namespace $NAMESPACE + --namespace "$NAMESPACE" --version 12.6.4 -f helm-bitnami-minio-values.yaml --repo https://charts.bitnami.com/bitnami minio diff --git a/tests/templates/kuttl/smoke/40-install-postgres.yaml.j2 b/tests/templates/kuttl/smoke/40-install-postgres.yaml.j2 index 0076b435..aa7e4108 100644 --- a/tests/templates/kuttl/smoke/40-install-postgres.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-install-postgres.yaml.j2 @@ -5,6 +5,6 @@ commands: - script: >- helm install postgresql --version={{ test_scenario['values']['postgres'] }} - --namespace $NAMESPACE + --namespace "$NAMESPACE" -f helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/smoke/61-assert.yaml b/tests/templates/kuttl/smoke/61-assert.yaml index 22c0cb95..c35e9a5f 100644 --- a/tests/templates/kuttl/smoke/61-assert.yaml +++ b/tests/templates/kuttl/smoke/61-assert.yaml @@ -7,14 +7,14 @@ commands: # Test envOverrides # - script: | - kubectl -n $NAMESPACE get sts hive-metastore-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' - kubectl -n $NAMESPACE get sts hive-metastore-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' - kubectl -n $NAMESPACE get sts hive-metastore-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' + kubectl -n "$NAMESPACE" get sts hive-metastore-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' + kubectl -n "$NAMESPACE" get sts hive-metastore-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' + kubectl -n "$NAMESPACE" get sts hive-metastore-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "hive") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' # # Test configOverrides # - script: | - kubectl -n $NAMESPACE get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "hive.metastore.warehouse.dir") | .value' | grep -qx "/stackable/warehouse/override" - kubectl -n $NAMESPACE get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "role-var") | .value' | grep -qx "role-value" - kubectl -n $NAMESPACE get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "group-var") | .value' | grep -qx "group-value" - kubectl -n $NAMESPACE get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "common-var") | .value' | grep -qx "group-value" + kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "hive.metastore.warehouse.dir") | .value' | grep -qx "/stackable/warehouse/override" + kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "role-var") | .value' | grep -qx "role-value" + kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "group-var") | .value' | grep -qx "group-value" + kubectl -n "$NAMESPACE" get cm hive-metastore-default -o yaml | yq -e '.data."hive-site.xml"' | yq -p=xml '.configuration.property[] | select(.name == "common-var") | .value' | grep -qx "group-value" diff --git a/tests/templates/kuttl/smoke/62-assert.yaml b/tests/templates/kuttl/smoke/62-assert.yaml index f729d72f..dc483ddb 100644 --- a/tests/templates/kuttl/smoke/62-assert.yaml +++ b/tests/templates/kuttl/smoke/62-assert.yaml @@ -4,4 +4,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 60 commands: - - script: kubectl exec -n $NAMESPACE --container hive hive-metastore-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' + - script: kubectl exec -n "$NAMESPACE" --container hive hive-metastore-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' diff --git a/tests/templates/kuttl/smoke/80-assert.yaml b/tests/templates/kuttl/smoke/80-assert.yaml index ee864121..a6494eb5 100644 --- a/tests/templates/kuttl/smoke/80-assert.yaml +++ b/tests/templates/kuttl/smoke/80-assert.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - - script: kubectl exec -n $NAMESPACE test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore.$NAMESPACE.svc.cluster.local + - script: kubectl exec -n "$NAMESPACE" test-metastore-0 -- python /tmp/test_metastore.py -m hive-metastore.$NAMESPACE.svc.cluster.local diff --git a/tests/templates/kuttl/smoke/80-prepare-test-metastore.yaml b/tests/templates/kuttl/smoke/80-prepare-test-metastore.yaml index cd88fb1c..45da6773 100644 --- a/tests/templates/kuttl/smoke/80-prepare-test-metastore.yaml +++ b/tests/templates/kuttl/smoke/80-prepare-test-metastore.yaml @@ -2,4 +2,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl cp -n $NAMESPACE ./test_metastore.py test-metastore-0:/tmp + - script: kubectl cp -n "$NAMESPACE" ./test_metastore.py test-metastore-0:/tmp diff --git a/tests/templates/kuttl/upgrade/00-patch-ns.yaml.j2 b/tests/templates/kuttl/upgrade/00-patch-ns.yaml.j2 index 67185acf..7400f1ee 100644 --- a/tests/templates/kuttl/upgrade/00-patch-ns.yaml.j2 +++ b/tests/templates/kuttl/upgrade/00-patch-ns.yaml.j2 @@ -4,6 +4,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + - script: kubectl patch namespace "$NAMESPACE" -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' timeout: 120 {% endif %} diff --git a/tests/templates/kuttl/upgrade/20-install-postgres.yaml.j2 b/tests/templates/kuttl/upgrade/20-install-postgres.yaml.j2 index 0076b435..aa7e4108 100644 --- a/tests/templates/kuttl/upgrade/20-install-postgres.yaml.j2 +++ b/tests/templates/kuttl/upgrade/20-install-postgres.yaml.j2 @@ -5,6 +5,6 @@ commands: - script: >- helm install postgresql --version={{ test_scenario['values']['postgres'] }} - --namespace $NAMESPACE + --namespace "$NAMESPACE" -f helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql From 546c733e032bb0e77a5afe236253cd9971a598a7 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Jun 2025 14:30:59 +0200 Subject: [PATCH 47/50] Consolidate listener to module --- rust/operator-binary/src/controller.rs | 62 ++----------- rust/operator-binary/src/crd/mod.rs | 13 +-- rust/operator-binary/src/discovery.rs | 48 ++-------- rust/operator-binary/src/listener.rs | 118 +++++++++++++++++++++++++ rust/operator-binary/src/main.rs | 1 + 5 files changed, 136 insertions(+), 106 deletions(-) create mode 100644 rust/operator-binary/src/listener.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a2589123..d3eb5e4b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -37,10 +37,7 @@ use stackable_operator::{ product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, tls_verification::TlsClientDetailsError, }, - crd::{ - listener::v1alpha1::{Listener, ListenerPort, ListenerSpec}, - s3, - }, + crd::{listener::v1alpha1::Listener, s3}, k8s_openapi::{ DeepMerge, api::{ @@ -91,9 +88,8 @@ use crate::{ crd::{ APP_NAME, CORE_SITE_XML, Container, DB_PASSWORD_ENV, DB_USERNAME_ENV, HIVE_PORT, HIVE_PORT_NAME, HIVE_SITE_XML, HiveClusterStatus, HiveRole, JVM_SECURITY_PROPERTIES_FILE, - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, - MetaStoreConfig, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, - STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, + METRICS_PORT, METRICS_PORT_NAME, MetaStoreConfig, STACKABLE_CONFIG_DIR, + STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR, STACKABLE_LOG_DIR_NAME, v1alpha1::{self, HiveMetastoreRoleConfig}, @@ -103,6 +99,7 @@ use crate::{ self, add_kerberos_pod_config, kerberos_config_properties, kerberos_container_start_commands, }, + listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::extend_role_group_config_map, }; @@ -341,6 +338,9 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, role: String, }, + #[snafu(display("failed to configure listener"))] + ListenerConfiguration { source: crate::listener::Error }, + #[snafu(display("failed to build listener volume"))] BuildListenerVolume { source: ListenerOperatorVolumeSourceBuilderError, @@ -521,7 +521,8 @@ pub async fn reconcile_hive( if let Some(HiveMetastoreRoleConfig { listener_class, .. }) = role_config { let group_listener: Listener = - build_group_listener(hive, &resolved_product_image, &hive_role, listener_class)?; + build_group_listener(hive, &resolved_product_image, &hive_role, listener_class) + .context(ListenerConfigurationSnafu)?; let listener = cluster_resources .add(client, group_listener) .await @@ -573,51 +574,6 @@ pub async fn reconcile_hive( Ok(Action::await_change()) } -// Designed to build a listener per role -// In case of Hive we expect only one role: Metastore -pub fn build_group_listener( - hive: &v1alpha1::HiveCluster, - resolved_product_image: &ResolvedProductImage, - hive_role: &HiveRole, - listener_class: &String, -) -> Result { - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(hive) - .name(hive.group_listener_name(hive_role)) - .ownerreference_from_resource(hive, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &hive_role.to_string(), - "none", - )) - .context(MetadataBuildSnafu)? - .build(); - - let spec = ListenerSpec { - class_name: Some(listener_class.to_owned()), - ports: Some(listener_ports()), - ..Default::default() - }; - - let listener = Listener { - metadata, - spec, - status: None, - }; - - Ok(listener) -} - -fn listener_ports() -> Vec { - vec![ListenerPort { - name: HIVE_PORT_NAME.to_owned(), - port: HIVE_PORT.into(), - protocol: Some("TCP".to_owned()), - }] -} - /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] fn build_metastore_rolegroup_config_map( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index d564fe14..f7ff9f52 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -32,7 +32,7 @@ use stackable_operator::{ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use v1alpha1::HiveMetastoreRoleConfig; -use crate::crd::affinity::get_affinity; +use crate::{crd::affinity::get_affinity, listener::metastore_default_listener_class}; pub mod affinity; pub mod security; @@ -61,13 +61,6 @@ pub const HIVE_PORT: u16 = 9083; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9084; -// Listener volumes -pub const LISTENER_VOLUME_NAME: &str = "listener"; -pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; - -// Listener defaults -pub const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; - // Certificates and trust stores pub const SYSTEM_TRUST_STORE: &str = "/etc/pki/java/cacerts"; pub const SYSTEM_TRUST_STORE_PASSWORD: &str = "changeit"; @@ -190,10 +183,6 @@ impl Default for v1alpha1::HiveMetastoreRoleConfig { } } -fn metastore_default_listener_class() -> String { - DEFAULT_LISTENER_CLASS.to_owned() -} - impl HasStatusCondition for v1alpha1::HiveCluster { fn conditions(&self) -> Vec { match &self.status { diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 8db4809f..f6f14522 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -9,7 +9,8 @@ use stackable_operator::{ use crate::{ controller::build_recommended_labels, - crd::{HIVE_PORT_NAME, HiveRole, v1alpha1}, + crd::{HiveRole, v1alpha1}, + listener::build_listener_connection_string, }; #[derive(Snafu, Debug)] @@ -19,16 +20,12 @@ pub enum Error { source: stackable_operator::builder::meta::Error, hive: ObjectRef, }, - #[snafu(display("chroot path {chroot} was relative (must be absolute)"))] - RelativeChroot { chroot: String }, + #[snafu(display("could not build discovery config map for {obj_ref}"))] DiscoveryConfigMap { source: stackable_operator::builder::configmap::Error, obj_ref: ObjectRef, }, - #[snafu(display("could not find port [{port_name}] for rolegroup listener {role}"))] - NoServicePort { port_name: String, role: String }, - #[snafu(display("invalid owner name for discovery ConfigMap"))] InvalidOwnerNameForDiscoveryConfigMap, @@ -36,8 +33,8 @@ pub enum Error { MetadataBuild { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("{role} listener has no adress"))] - RoleListenerHasNoAddress { role: String }, + #[snafu(display("failed to configure listener discovery configmap"))] + ListenerConfiguration { source: crate::listener::Error }, } /// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::HiveCluster`] for all expected @@ -104,7 +101,8 @@ fn build_discovery_configmap( discovery_configmap.add_data( "HIVE".to_string(), - build_listener_connection_string(listener, &hive_role.to_string(), chroot)?, + build_listener_connection_string(listener, &hive_role.to_string(), chroot) + .context(ListenerConfigurationSnafu)?, ); discovery_configmap @@ -114,38 +112,6 @@ fn build_discovery_configmap( }) } -// Builds the connection string with respect to the listener provided objects -fn build_listener_connection_string( - listener_ref: Listener, - role: &String, - chroot: Option<&str>, -) -> Result { - // We only need the first address corresponding to the role - let listener_address = listener_ref - .status - .and_then(|s| s.ingress_addresses?.into_iter().next()) - .context(RoleListenerHasNoAddressSnafu { role })?; - let mut conn_str = format!( - "thrift://{address}:{port}", - address = listener_address.address, - port = listener_address - .ports - .get(HIVE_PORT_NAME) - .copied() - .context(NoServicePortSnafu { - port_name: HIVE_PORT_NAME, - role - })? - ); - if let Some(chroot) = chroot { - if !chroot.starts_with('/') { - return RelativeChrootSnafu { chroot }.fail(); - } - conn_str.push_str(chroot); - } - Ok(conn_str) -} - pub fn build_headless_role_group_metrics_service_name(name: String) -> String { format!("{name}-metrics", name = name) } diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs new file mode 100644 index 00000000..0f780af8 --- /dev/null +++ b/rust/operator-binary/src/listener.rs @@ -0,0 +1,118 @@ +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + commons::product_image_selection::ResolvedProductImage, + crd::listener::v1alpha1::{Listener, ListenerPort, ListenerSpec}, +}; + +use crate::{ + controller::build_recommended_labels, + crd::{HIVE_PORT, HIVE_PORT_NAME, HiveRole, v1alpha1}, +}; + +// Listener volumes +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; + +// Listener defaults +pub const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + #[snafu(display("{role} listener has no adress"))] + RoleListenerHasNoAddress { role: String }, + #[snafu(display("could not find port [{port_name}] for rolegroup listener {role}"))] + NoServicePort { port_name: String, role: String }, + #[snafu(display("chroot path {chroot} was relative (must be absolute)"))] + RelativeChroot { chroot: String }, +} + +// Builds the connection string with respect to the listener provided objects +pub fn build_listener_connection_string( + listener_ref: Listener, + role: &String, + chroot: Option<&str>, +) -> Result { + // We only need the first address corresponding to the role + let listener_address = listener_ref + .status + .and_then(|s| s.ingress_addresses?.into_iter().next()) + .context(RoleListenerHasNoAddressSnafu { role })?; + let mut conn_str = format!( + "thrift://{address}:{port}", + address = listener_address.address, + port = listener_address + .ports + .get(HIVE_PORT_NAME) + .copied() + .context(NoServicePortSnafu { + port_name: HIVE_PORT_NAME, + role + })? + ); + if let Some(chroot) = chroot { + if !chroot.starts_with('/') { + return RelativeChrootSnafu { chroot }.fail(); + } + conn_str.push_str(chroot); + } + Ok(conn_str) +} + +// Designed to build a listener per role +// In case of Hive we expect only one role: Metastore +pub fn build_group_listener( + hive: &v1alpha1::HiveCluster, + resolved_product_image: &ResolvedProductImage, + hive_role: &HiveRole, + listener_class: &String, +) -> Result { + let metadata = ObjectMetaBuilder::new() + .name_and_namespace(hive) + .name(hive.group_listener_name(hive_role)) + .ownerreference_from_resource(hive, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &hive_role.to_string(), + "none", + )) + .context(MetadataBuildSnafu)? + .build(); + + let spec = ListenerSpec { + class_name: Some(listener_class.to_owned()), + ports: Some(listener_ports()), + ..Default::default() + }; + + let listener = Listener { + metadata, + spec, + status: None, + }; + + Ok(listener) +} + +fn listener_ports() -> Vec { + vec![ListenerPort { + name: HIVE_PORT_NAME.to_owned(), + port: HIVE_PORT.into(), + protocol: Some("TCP".to_owned()), + }] +} + +// used by crds to define a default listener_class name +pub fn metastore_default_listener_class() -> String { + DEFAULT_LISTENER_CLASS.to_owned() +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 4f17a34b..22d6376a 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -4,6 +4,7 @@ mod controller; mod crd; mod discovery; mod kerberos; +mod listener; mod operations; mod product_logging; From 1eddee063ed52e29ae3b08a9407231454b8d2ce1 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Jun 2025 14:36:39 +0200 Subject: [PATCH 48/50] update function to build_role_listener --- rust/operator-binary/src/controller.rs | 4 ++-- rust/operator-binary/src/listener.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index d3eb5e4b..fbc8e271 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -99,7 +99,7 @@ use crate::{ self, add_kerberos_pod_config, kerberos_config_properties, kerberos_container_start_commands, }, - listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener}, + listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_role_listener}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::extend_role_group_config_map, }; @@ -521,7 +521,7 @@ pub async fn reconcile_hive( if let Some(HiveMetastoreRoleConfig { listener_class, .. }) = role_config { let group_listener: Listener = - build_group_listener(hive, &resolved_product_image, &hive_role, listener_class) + build_role_listener(hive, &resolved_product_image, &hive_role, listener_class) .context(ListenerConfigurationSnafu)?; let listener = cluster_resources .add(client, group_listener) diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs index 0f780af8..0a7b816f 100644 --- a/rust/operator-binary/src/listener.rs +++ b/rust/operator-binary/src/listener.rs @@ -69,7 +69,7 @@ pub fn build_listener_connection_string( // Designed to build a listener per role // In case of Hive we expect only one role: Metastore -pub fn build_group_listener( +pub fn build_role_listener( hive: &v1alpha1::HiveCluster, resolved_product_image: &ResolvedProductImage, hive_role: &HiveRole, From 00375d1157d4cdaebb7402dfc64dc1c0c9ab7399 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Jun 2025 14:42:12 +0200 Subject: [PATCH 49/50] more consistent wording --- rust/operator-binary/src/controller.rs | 13 ++++++------- rust/operator-binary/src/crd/mod.rs | 4 ++-- rust/operator-binary/src/listener.rs | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index fbc8e271..4c41390e 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -520,15 +520,14 @@ pub async fn reconcile_hive( let mut discovery_hash = FnvHasher::with_key(0); if let Some(HiveMetastoreRoleConfig { listener_class, .. }) = role_config { - let group_listener: Listener = + let role_listener: Listener = build_role_listener(hive, &resolved_product_image, &hive_role, listener_class) .context(ListenerConfigurationSnafu)?; - let listener = cluster_resources - .add(client, group_listener) - .await - .context(ApplyGroupListenerSnafu { + let listener = cluster_resources.add(client, role_listener).await.context( + ApplyGroupListenerSnafu { role: hive_role.to_string(), - })?; + }, + )?; for discovery_cm in discovery::build_discovery_configmaps( hive, @@ -961,7 +960,7 @@ fn build_metastore_rolegroup_statefulset( .build(); let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(hive.group_listener_name(hive_role)), + &ListenerReference::ListenerName(hive.role_listener_name(hive_role)), &unversioned_recommended_labels, ) .context(BuildListenerVolumeSnafu)? diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f7ff9f52..de82443f 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -240,9 +240,9 @@ impl v1alpha1::HiveCluster { }) } - /// The name of the group-listener provided for a specific role. + /// The name of the role-listener provided for a specific role. /// returns a name `-` - pub fn group_listener_name(&self, hive_role: &HiveRole) -> String { + pub fn role_listener_name(&self, hive_role: &HiveRole) -> String { format!("{name}-{role}", name = self.name_any(), role = hive_role) } diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs index 0a7b816f..0b24920e 100644 --- a/rust/operator-binary/src/listener.rs +++ b/rust/operator-binary/src/listener.rs @@ -77,7 +77,7 @@ pub fn build_role_listener( ) -> Result { let metadata = ObjectMetaBuilder::new() .name_and_namespace(hive) - .name(hive.group_listener_name(hive_role)) + .name(hive.role_listener_name(hive_role)) .ownerreference_from_resource(hive, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( From 65b795fc225bfb5d594387e80c7d2bedb6ef3dbe Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Mon, 23 Jun 2025 14:43:32 +0200 Subject: [PATCH 50/50] fix: headless service function Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- rust/operator-binary/src/discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index f6f14522..2e9e16fb 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -113,5 +113,5 @@ fn build_discovery_configmap( } pub fn build_headless_role_group_metrics_service_name(name: String) -> String { - format!("{name}-metrics", name = name) + format!("{name}-metrics") }