From 2c253e87fcd0bc67ff2ee703aedfd27160dac8d5 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 25 Jul 2025 16:08:10 +0200 Subject: [PATCH 01/16] added listener dirs and service --- .../opensearch-operator/templates/roles.yaml | 11 + rust/operator-binary/src/controller.rs | 3 + rust/operator-binary/src/controller/apply.rs | 3 + rust/operator-binary/src/controller/build.rs | 3 + .../src/controller/build/node_config.rs | 1 + .../controller/build/role_group_builder.rs | 73 ++- .../src/controller/validate.rs | 1 + rust/operator-binary/src/crd/mod.rs | 11 + .../src/framework/role_group_utils.rs | 10 + tests/templates/kuttl/smoke/10-assert.yaml | 511 ------------------ .../kuttl/smoke/10-install-opensearch.yaml | 30 +- tests/test-definition.yaml | 2 +- 12 files changed, 129 insertions(+), 530 deletions(-) diff --git a/deploy/helm/opensearch-operator/templates/roles.yaml b/deploy/helm/opensearch-operator/templates/roles.yaml index c74dd92..97ea65f 100644 --- a/deploy/helm/opensearch-operator/templates/roles.yaml +++ b/deploy/helm/opensearch-operator/templates/roles.yaml @@ -70,6 +70,17 @@ rules: - customresourcedefinitions verbs: - get + - apiGroups: + - listeners.stackable.tech + resources: + - listeners + verbs: + - get + - list + - watch + - patch + - create + - delete - apiGroups: - events.k8s.io resources: diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 6b092b1..4e23072 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -6,6 +6,7 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ cluster_resources::ClusterResourceApplyStrategy, commons::{affinity::StackableAffinity, product_image_selection::ProductImage}, + crd::listener::v1alpha1::Listener, k8s_openapi::api::{ apps::v1::StatefulSet, core::v1::{ConfigMap, Service, ServiceAccount}, @@ -111,6 +112,7 @@ pub struct ValidatedOpenSearchConfig { pub node_roles: NodeRoles, pub resources: stackable_operator::commons::resources::Resources, pub termination_grace_period_seconds: i64, + pub listener_class: String, } // validated and converted to validated and safe types @@ -275,6 +277,7 @@ struct Applied; struct KubernetesResources { stateful_sets: Vec, services: Vec, + listeners: Vec, config_maps: Vec, service_accounts: Vec, role_bindings: Vec, diff --git a/rust/operator-binary/src/controller/apply.rs b/rust/operator-binary/src/controller/apply.rs index 23b6f2e..f0bd893 100644 --- a/rust/operator-binary/src/controller/apply.rs +++ b/rust/operator-binary/src/controller/apply.rs @@ -62,6 +62,8 @@ impl<'a> Applier<'a> { let services = self.add_resources(resources.services).await?; + let listeners = self.add_resources(resources.listeners).await?; + let config_maps = self.add_resources(resources.config_maps).await?; let service_accounts = self.add_resources(resources.service_accounts).await?; @@ -78,6 +80,7 @@ impl<'a> Applier<'a> { Ok(KubernetesResources { stateful_sets, services, + listeners, config_maps, service_accounts, role_bindings, diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 28aa9ad..db8e35e 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -12,6 +12,7 @@ pub fn build(names: &ContextNames, cluster: ValidatedCluster) -> KubernetesResou let mut config_maps = vec![]; let mut stateful_sets = vec![]; let mut services = vec![]; + let mut listeners = vec![]; let role_builder = RoleBuilder::new(cluster.clone(), names); @@ -19,6 +20,7 @@ pub fn build(names: &ContextNames, cluster: ValidatedCluster) -> KubernetesResou config_maps.push(role_group_builder.build_config_map()); stateful_sets.push(role_group_builder.build_stateful_set()); services.push(role_group_builder.build_headless_service()); + listeners.push(role_group_builder.build_listener()); } let cluster_manager_service = role_builder.build_cluster_manager_service(); @@ -33,6 +35,7 @@ pub fn build(names: &ContextNames, cluster: ValidatedCluster) -> KubernetesResou KubernetesResources { stateful_sets, services, + listeners, config_maps, service_accounts, role_bindings, diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 49da561..6eb9ba6 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -286,6 +286,7 @@ mod tests { node_roles: NodeRoles::default(), resources: Resources::default(), termination_grace_period_seconds: 30, + listener_class: "cluster-internal".to_string(), }, config_overrides: HashMap::default(), env_overrides: [("TEST".to_owned(), "value".to_owned())].into(), diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 6ca788a..51884f3 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -1,13 +1,20 @@ use stackable_operator::{ - builder::{meta::ObjectMetaBuilder, pod::container::ContainerBuilder}, + builder::{ + meta::ObjectMetaBuilder, + pod::{ + container::ContainerBuilder, + volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, + }, + }, + crd::listener::{self}, k8s_openapi::{ DeepMerge, api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ Affinity, ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, - PodSecurityContext, PodSpec, PodTemplateSpec, Probe, Service, ServicePort, - ServiceSpec, TCPSocketAction, Volume, VolumeMount, + PersistentVolumeClaim, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, + Service, ServicePort, ServiceSpec, TCPSocketAction, Volume, VolumeMount, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -36,8 +43,11 @@ pub const TRANSPORT_PORT: u16 = 9300; const CONFIG_VOLUME_NAME: &str = "config"; const DATA_VOLUME_NAME: &str = "data"; +const LISTENER_VOLUME_NAME: &str = "listener"; +const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; + // Path in opensearchproject/opensearch:3.0.0 -const OPENSEARCH_BASE_PATH: &str = "/usr/share/opensearch"; +const OPENSEARCH_BASE_PATH: &str = "/stackable/opensearch/"; pub struct RoleGroupBuilder<'a> { service_account_name: String, @@ -107,6 +117,27 @@ impl<'a> RoleGroupBuilder<'a> { .data .build_pvc(DATA_VOLUME_NAME, Some(vec!["ReadWriteOnce"])); + let listener_group_name = self.resource_names.listener_service_name(); + + // Listener endpoints for the all rolegroups will use persistent + // volumes so that load balancers can hard-code the target + // addresses. This will be the case even when no class is set (and + // the value defaults to cluster-internal) as the address should + // still be consistent. + let listener_volume_claim_template = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(listener_group_name), + // TODO should be unversioned + &self.recommended_labels(), + ) + .expect("should be a listener group name") + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .expect("should be a valid annotation"); + + let pvcs: Option> = Some(vec![ + data_volume_claim_template, + listener_volume_claim_template, + ]); + let spec = StatefulSetSpec { // Order does not matter for OpenSearch pod_management_policy: Some("Parallel".to_string()), @@ -117,7 +148,7 @@ impl<'a> RoleGroupBuilder<'a> { }, service_name: Some(self.resource_names.headless_service_name()), template, - volume_claim_templates: Some(vec![data_volume_claim_template]), + volume_claim_templates: pvcs, ..StatefulSetSpec::default() }; @@ -271,6 +302,11 @@ impl<'a> RoleGroupBuilder<'a> { name: DATA_VOLUME_NAME.to_owned(), ..VolumeMount::default() }, + VolumeMount { + mount_path: LISTENER_VOLUME_DIR.to_owned(), + name: LISTENER_VOLUME_NAME.to_owned(), + ..VolumeMount::default() + }, ]) .expect("The mount paths are statically defined and there should be no duplicates.") .add_container_ports(vec![ @@ -337,6 +373,33 @@ impl<'a> RoleGroupBuilder<'a> { } } + pub fn build_listener(&self) -> listener::v1alpha1::Listener { + let metadata = + self.common_metadata(self.resource_names.listener_service_name(), Labels::new()); + + let listener_class = self.role_group_config.config.listener_class.to_owned(); + + listener::v1alpha1::Listener { + metadata, + spec: listener::v1alpha1::ListenerSpec { + class_name: Some(listener_class), + ports: Some(self.listener_ports()), + ..listener::v1alpha1::ListenerSpec::default() + }, + status: None, + } + } + + /// We only use the http port here and intentionally omit + /// the metrics one. + fn listener_ports(&self) -> Vec { + vec![listener::v1alpha1::ListenerPort { + name: HTTP_PORT_NAME.to_string(), + port: HTTP_PORT.into(), + protocol: Some("TCP".to_string()), + }] + } + fn common_metadata( &self, resource_name: impl Into, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index d171898..959ef80 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -122,6 +122,7 @@ fn validate_role_group_config( node_roles: merged_role_group.config.config.node_roles, resources: merged_role_group.config.config.resources, termination_grace_period_seconds, + listener_class: merged_role_group.config.config.listener_class, }; Ok(RoleGroupConfig { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index ce1eabb..942a4a6 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -30,6 +30,8 @@ use crate::framework::{ role_utils::GenericProductSpecificCommonConfig, }; +const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; + #[versioned(version(name = "v1alpha1"))] pub mod versioned { @@ -130,6 +132,10 @@ pub mod versioned { #[fragment_attrs(serde(default))] pub resources: Resources, + + /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + #[serde(default = "default_listener_class")] + pub listener_class: String, } #[derive(Clone, Debug, Default, JsonSchema, PartialEq, Fragment)] @@ -162,6 +168,10 @@ pub mod versioned { } } +fn default_listener_class() -> String { + DEFAULT_LISTENER_CLASS.to_string() +} + impl HasStatusCondition for v1alpha1::OpenSearchCluster { fn conditions(&self) -> Vec { match &self.status { @@ -232,6 +242,7 @@ impl v1alpha1::OpenSearchConfig { }, }, }, + listener_class: Some("cluster-internal".to_string()), } } } diff --git a/rust/operator-binary/src/framework/role_group_utils.rs b/rust/operator-binary/src/framework/role_group_utils.rs index e4b522a..7cd26df 100644 --- a/rust/operator-binary/src/framework/role_group_utils.rs +++ b/rust/operator-binary/src/framework/role_group_utils.rs @@ -65,6 +65,16 @@ impl ResourceNames { format!("{}{SUFFIX}", self.qualified_role_group_name()) } + + pub fn listener_service_name(&self) -> String { + // Compile-time check + const _: () = assert!( + ResourceNames::MAX_QUALIFIED_ROLE_GROUP_NAME_LENGTH <= MAX_OBJECT_NAME_LENGTH, + "The listener name `--` must not exceed 253 characters." + ); + + self.qualified_role_group_name() + } } #[cfg(test)] diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml index 99038c7..66d4042 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -10,159 +10,9 @@ timeout: 600 apiVersion: apps/v1 kind: StatefulSet metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable name: opensearch-nodes-cluster-manager - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch spec: - podManagementPolicy: Parallel replicas: 3 - selector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - serviceName: opensearch-nodes-cluster-manager-headless - template: - metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: 3.0.0 - stackable.tech/opensearch-role.cluster_manager: "true" - stackable.tech/vendor: Stackable - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - topologyKey: kubernetes.io/hostname - weight: 1 - containers: - - command: - - /usr/share/opensearch/opensearch-docker-entrypoint.sh - env: - - name: DISABLE_INSTALL_DEMO_CONFIG - value: "true" - - name: cluster.initial_cluster_manager_nodes - value: opensearch-nodes-cluster-manager-0,opensearch-nodes-cluster-manager-1,opensearch-nodes-cluster-manager-2 - - name: discovery.seed_hosts - value: opensearch - - name: node.name - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: node.roles - value: cluster_manager - image: opensearchproject/opensearch:3.0.0 - imagePullPolicy: Always - name: opensearch - ports: - - containerPort: 9200 - name: http - protocol: TCP - - containerPort: 9300 - name: transport - protocol: TCP - readinessProbe: - failureThreshold: 3 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: http - timeoutSeconds: 3 - resources: - limits: - cpu: "4" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - startupProbe: - failureThreshold: 30 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - tcpSocket: - port: http - timeoutSeconds: 3 - volumeMounts: - - mountPath: /usr/share/opensearch/config/opensearch.yml - name: config - readOnly: true - subPath: opensearch.yml - - mountPath: /usr/share/opensearch/data - name: data - - mountPath: /usr/share/opensearch/config/opensearch-security - name: security-config - readOnly: true - - mountPath: /usr/share/opensearch/config/tls - name: tls - readOnly: true - securityContext: - fsGroup: 1000 - serviceAccount: opensearch-serviceaccount - serviceAccountName: opensearch-serviceaccount - terminationGracePeriodSeconds: 120 - volumes: - - configMap: - defaultMode: 420 - name: opensearch-nodes-cluster-manager - name: config - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: node,pod,service=opensearch-cluster-manager,service=opensearch-nodes-cluster-manager - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" - storageClassName: secrets.stackable.tech - volumeMode: Filesystem - name: tls - - name: security-config - secret: - defaultMode: 420 - secretName: opensearch-security-config - volumeClaimTemplates: - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi - volumeMode: Filesystem - status: - phase: Pending status: readyReplicas: 3 replicas: 3 @@ -170,160 +20,9 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable name: opensearch-nodes-data - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch spec: - podManagementPolicy: Parallel replicas: 2 - selector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - serviceName: opensearch-nodes-data-headless - template: - metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: 3.0.0 - stackable.tech/opensearch-role.data: "true" - stackable.tech/opensearch-role.ingest: "true" - stackable.tech/opensearch-role.remote_cluster_client: "true" - stackable.tech/vendor: Stackable - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - topologyKey: kubernetes.io/hostname - weight: 1 - containers: - - command: - - /usr/share/opensearch/opensearch-docker-entrypoint.sh - env: - - name: DISABLE_INSTALL_DEMO_CONFIG - value: "true" - - name: cluster.initial_cluster_manager_nodes - - name: discovery.seed_hosts - value: opensearch - - name: node.name - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: node.roles - value: ingest,data,remote_cluster_client - image: opensearchproject/opensearch:3.0.0 - imagePullPolicy: Always - name: opensearch - ports: - - containerPort: 9200 - name: http - protocol: TCP - - containerPort: 9300 - name: transport - protocol: TCP - readinessProbe: - failureThreshold: 3 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: http - timeoutSeconds: 3 - resources: - limits: - cpu: "4" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - startupProbe: - failureThreshold: 30 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - tcpSocket: - port: http - timeoutSeconds: 3 - volumeMounts: - - mountPath: /usr/share/opensearch/config/opensearch.yml - name: config - readOnly: true - subPath: opensearch.yml - - mountPath: /usr/share/opensearch/data - name: data - - mountPath: /usr/share/opensearch/config/opensearch-security - name: security-config - readOnly: true - - mountPath: /usr/share/opensearch/config/tls - name: tls - readOnly: true - securityContext: - fsGroup: 1000 - serviceAccount: opensearch-serviceaccount - serviceAccountName: opensearch-serviceaccount - terminationGracePeriodSeconds: 120 - volumes: - - configMap: - defaultMode: 420 - name: opensearch-nodes-data - name: config - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" - storageClassName: secrets.stackable.tech - volumeMode: Filesystem - name: tls - - name: security-config - secret: - defaultMode: 420 - secretName: opensearch-security-config - volumeClaimTemplates: - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi - volumeMode: Filesystem - status: - phase: Pending status: readyReplicas: 2 replicas: 2 @@ -331,219 +30,9 @@ status: apiVersion: v1 kind: ConfigMap metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable name: opensearch-nodes-cluster-manager - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -data: - opensearch.yml: |- - cluster.name: "opensearch" - discovery.type: "zen" - network.host: "0.0.0.0" - node.store.allow_mmap: "false" - plugins.security.allow_default_init_securityindex: "true" - plugins.security.nodes_dn: ["CN=generated certificate for pod"] - plugins.security.ssl.http.enabled: "true" - plugins.security.ssl.http.pemcert_filepath: "/usr/share/opensearch/config/tls/tls.crt" - plugins.security.ssl.http.pemkey_filepath: "/usr/share/opensearch/config/tls/tls.key" - plugins.security.ssl.http.pemtrustedcas_filepath: "/usr/share/opensearch/config/tls/ca.crt" - plugins.security.ssl.transport.enabled: "true" - plugins.security.ssl.transport.pemcert_filepath: "/usr/share/opensearch/config/tls/tls.crt" - plugins.security.ssl.transport.pemkey_filepath: "/usr/share/opensearch/config/tls/tls.key" - plugins.security.ssl.transport.pemtrustedcas_filepath: "/usr/share/opensearch/config/tls/ca.crt" --- apiVersion: v1 kind: ConfigMap metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable name: opensearch-nodes-data - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -data: - opensearch.yml: |- - cluster.name: "opensearch" - discovery.type: "zen" - network.host: "0.0.0.0" - node.store.allow_mmap: "false" - plugins.security.allow_default_init_securityindex: "true" - plugins.security.nodes_dn: ["CN=generated certificate for pod"] - plugins.security.ssl.http.enabled: "true" - plugins.security.ssl.http.pemcert_filepath: "/usr/share/opensearch/config/tls/tls.crt" - plugins.security.ssl.http.pemkey_filepath: "/usr/share/opensearch/config/tls/tls.key" - plugins.security.ssl.http.pemtrustedcas_filepath: "/usr/share/opensearch/config/tls/ca.crt" - plugins.security.ssl.transport.enabled: "true" - plugins.security.ssl.transport.pemcert_filepath: "/usr/share/opensearch/config/tls/tls.crt" - plugins.security.ssl.transport.pemkey_filepath: "/usr/share/opensearch/config/tls/tls.key" - plugins.security.ssl.transport.pemtrustedcas_filepath: "/usr/share/opensearch/config/tls/ca.crt" ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable - name: opensearch-nodes-cluster-manager-headless -spec: - ports: - - name: http - port: 9200 - protocol: TCP - targetPort: 9200 - - name: transport - port: 9300 - protocol: TCP - targetPort: 9300 - publishNotReadyAddresses: true - selector: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable - name: opensearch-nodes-data-headless -spec: - ports: - - name: http - port: 9200 - protocol: TCP - targetPort: 9200 - - name: transport - port: 9300 - protocol: TCP - targetPort: 9300 - publishNotReadyAddresses: true - selector: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable - name: opensearch - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -spec: - ports: - - name: http - port: 9200 - protocol: TCP - targetPort: 9200 - - name: transport - port: 9300 - protocol: TCP - targetPort: 9300 - publishNotReadyAddresses: true - selector: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - stackable.tech/opensearch-role.cluster_manager: "true" - type: ClusterIP ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable - name: opensearch-serviceaccount - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/version: 3.0.0 - stackable.tech/vendor: Stackable - name: opensearch-rolebinding - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: opensearch-clusterrole -subjects: -- kind: ServiceAccount - name: opensearch-serviceaccount ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - name: opensearch-nodes -spec: - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch diff --git a/tests/templates/kuttl/smoke/10-install-opensearch.yaml b/tests/templates/kuttl/smoke/10-install-opensearch.yaml index d390b29..9c32793 100644 --- a/tests/templates/kuttl/smoke/10-install-opensearch.yaml +++ b/tests/templates/kuttl/smoke/10-install-opensearch.yaml @@ -5,8 +5,8 @@ metadata: name: opensearch spec: image: - custom: opensearchproject/opensearch:3.0.0 - productVersion: 3.0.0 + productVersion: 3.1.0 + pullPolicy: IfNotPresent nodes: roleGroups: cluster-manager: @@ -17,6 +17,7 @@ spec: storage: data: capacity: 100Mi + listenerClass: external-stable replicas: 3 podOverrides: spec: @@ -26,7 +27,7 @@ spec: volumeClaimTemplate: metadata: annotations: - secrets.stackable.tech/scope: node,pod,service=opensearch-cluster-manager,service=opensearch-nodes-cluster-manager + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-cluster-manager-headless data: config: nodeRoles: @@ -37,6 +38,7 @@ spec: storage: data: capacity: 2Gi + listenerClass: external-stable replicas: 2 podOverrides: spec: @@ -46,7 +48,7 @@ spec: volumeClaimTemplate: metadata: annotations: - secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data + secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data-headless envOverrides: # TODO Make these the defaults in the image DISABLE_INSTALL_DEMO_CONFIG: "true" @@ -59,24 +61,26 @@ spec: # TODO Check that this is safe despite the warning in the documentation plugins.security.allow_default_init_securityindex: "true" plugins.security.ssl.transport.enabled: "true" - plugins.security.ssl.transport.pemcert_filepath: /usr/share/opensearch/config/tls/tls.crt - plugins.security.ssl.transport.pemkey_filepath: /usr/share/opensearch/config/tls/tls.key - plugins.security.ssl.transport.pemtrustedcas_filepath: /usr/share/opensearch/config/tls/ca.crt + plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt plugins.security.ssl.http.enabled: "true" - plugins.security.ssl.http.pemcert_filepath: /usr/share/opensearch/config/tls/tls.crt - plugins.security.ssl.http.pemkey_filepath: /usr/share/opensearch/config/tls/tls.key - plugins.security.ssl.http.pemtrustedcas_filepath: /usr/share/opensearch/config/tls/ca.crt + plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt + plugins.security.authcz.admin_dn: "CN=generated certificate for pod" podOverrides: spec: containers: - name: opensearch + command: + - /stackable/opensearch/opensearch-docker-entrypoint.sh volumeMounts: - name: security-config - mountPath: /usr/share/opensearch/config/opensearch-security + mountPath: /stackable/opensearch/config/opensearch-security readOnly: true - name: tls - # The Java policy allows reading from /usr/share/opensearch/config. - mountPath: /usr/share/opensearch/config/tls + mountPath: /stackable/opensearch/config/tls readOnly: true securityContext: fsGroup: 1000 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 505167a..b7bcbe1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,7 +2,7 @@ dimensions: - name: opensearch values: - - 3.0.0 + - 3.1.0 - name: openshift values: - "false" From 02a1b2205209f57d13480a0b92459737dc51347c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 25 Jul 2025 17:59:18 +0200 Subject: [PATCH 02/16] added integration test; use unversioned labels for STS listener volume --- .../helm/opensearch-operator/crds/crds.yaml | 8 + .../controller/build/role_group_builder.rs | 16 +- .../kuttl/external-access/00-patch-ns.yaml | 15 + .../kuttl/external-access/01-rbac.yaml | 31 ++ .../external-access/10-listener-classes.yaml | 6 + .../kuttl/external-access/20-assert.yaml | 49 ++ .../20-install-opensearch.yaml | 8 + .../external-access/listener-classes.yaml | 21 + .../kuttl/external-access/opensearch.yaml.j2 | 187 ++++++++ tests/templates/kuttl/smoke/10-assert.yaml | 417 ++++++++++++++++++ .../kuttl/smoke/10-install-opensearch.yaml | 4 +- tests/test-definition.yaml | 4 + 12 files changed, 756 insertions(+), 10 deletions(-) create mode 100644 tests/templates/kuttl/external-access/00-patch-ns.yaml create mode 100644 tests/templates/kuttl/external-access/01-rbac.yaml create mode 100644 tests/templates/kuttl/external-access/10-listener-classes.yaml create mode 100644 tests/templates/kuttl/external-access/20-assert.yaml create mode 100644 tests/templates/kuttl/external-access/20-install-opensearch.yaml create mode 100644 tests/templates/kuttl/external-access/listener-classes.yaml create mode 100644 tests/templates/kuttl/external-access/opensearch.yaml.j2 diff --git a/deploy/helm/opensearch-operator/crds/crds.yaml b/deploy/helm/opensearch-operator/crds/crds.yaml index e67d288..102d8f0 100644 --- a/deploy/helm/opensearch-operator/crds/crds.yaml +++ b/deploy/helm/opensearch-operator/crds/crds.yaml @@ -133,6 +133,10 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + nullable: true + type: string nodeRoles: items: enum: @@ -325,6 +329,10 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + nullable: true + type: string nodeRoles: items: enum: diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 51884f3..c19df81 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + use stackable_operator::{ builder::{ meta::ObjectMetaBuilder, @@ -28,7 +30,7 @@ use crate::{ controller::{ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster}, crd::v1alpha1, framework::{ - RoleGroupName, + ProductVersion, RoleGroupName, builder::meta::ownerreference_from_resource, kvp::label::{recommended_labels, role_group_selector, role_selector}, role_group_utils::ResourceNames, @@ -126,8 +128,8 @@ impl<'a> RoleGroupBuilder<'a> { // still be consistent. let listener_volume_claim_template = ListenerOperatorVolumeSourceBuilder::new( &ListenerReference::ListenerName(listener_group_name), - // TODO should be unversioned - &self.recommended_labels(), + &self + .recommended_labels(ProductVersion::from_str("none").expect("version is supplied")), ) .expect("should be a listener group name") .build_pvc(LISTENER_VOLUME_NAME.to_string()) @@ -166,7 +168,7 @@ impl<'a> RoleGroupBuilder<'a> { } let metadata = ObjectMetaBuilder::new() - .with_labels(self.recommended_labels()) + .with_labels(self.recommended_labels(self.cluster.product_version.clone())) .with_labels(node_role_labels) .build(); @@ -413,16 +415,16 @@ impl<'a> RoleGroupBuilder<'a> { None, Some(true), )) - .with_labels(self.recommended_labels()) + .with_labels(self.recommended_labels(self.cluster.product_version.clone())) .with_labels(extra_labels) .build() } - fn recommended_labels(&self) -> Labels { + fn recommended_labels(&self, product_version: ProductVersion) -> Labels { recommended_labels( &self.cluster, &self.context_names.product_name, - &self.cluster.product_version, + &product_version, &self.context_names.operator_name, &self.context_names.controller_name, &ValidatedCluster::role_name(), diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml b/tests/templates/kuttl/external-access/00-patch-ns.yaml new file mode 100644 index 0000000..d4f91fa --- /dev/null +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml @@ -0,0 +1,15 @@ +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl patch namespace $NAMESPACE --patch=' + { + "metadata": { + "labels": { + "pod-security.kubernetes.io/enforce": "privileged" + } + } + }' + timeout: 120 diff --git a/tests/templates/kuttl/external-access/01-rbac.yaml b/tests/templates/kuttl/external-access/01-rbac.yaml new file mode 100644 index 0000000..64eced8 --- /dev/null +++ b/tests/templates/kuttl/external-access/01-rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-service-account +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - privileged + verbs: + - use +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role-binding +subjects: + - kind: ServiceAccount + name: test-service-account +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-role diff --git a/tests/templates/kuttl/external-access/10-listener-classes.yaml b/tests/templates/kuttl/external-access/10-listener-classes.yaml new file mode 100644 index 0000000..893032c --- /dev/null +++ b/tests/templates/kuttl/external-access/10-listener-classes.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml new file mode 100644 index 0000000..ce0c0d7 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -0,0 +1,49 @@ +# All fields are checked that are set by the operator. +# This helps to detect unintentional changes. +# The maintenance effort should be okay as long as it is only done in the smoke test. +# TODO Check individual field in unit tests +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-nodes-cluster-manager +spec: + replicas: 1 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-nodes-data +spec: + replicas: 1 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: opensearch-nodes-cluster-manager +spec: + type: NodePort # external-stable +--- +apiVersion: v1 +kind: Service +metadata: + name: opensearch-nodes-data +spec: + type: NodePort # external-unstable +--- +apiVersion: v1 +kind: Service +metadata: + name: opensearch +spec: + type: ClusterIP diff --git a/tests/templates/kuttl/external-access/20-install-opensearch.yaml b/tests/templates/kuttl/external-access/20-install-opensearch.yaml new file mode 100644 index 0000000..78a7a56 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-install-opensearch.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: > + envsubst < opensearch.yaml | + kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/listener-classes.yaml b/tests/templates/kuttl/external-access/listener-classes.yaml new file mode 100644 index 0000000..4131526 --- /dev/null +++ b/tests/templates/kuttl/external-access/listener-classes.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-cluster-internal-$NAMESPACE +spec: + serviceType: ClusterIP +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-stable-$NAMESPACE +spec: + serviceType: NodePort +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable-$NAMESPACE +spec: + serviceType: NodePort diff --git a/tests/templates/kuttl/external-access/opensearch.yaml.j2 b/tests/templates/kuttl/external-access/opensearch.yaml.j2 new file mode 100644 index 0000000..73370cd --- /dev/null +++ b/tests/templates/kuttl/external-access/opensearch.yaml.j2 @@ -0,0 +1,187 @@ +--- +apiVersion: opensearch.stackable.tech/v1alpha1 +kind: OpenSearchCluster +metadata: + name: opensearch +spec: + image: + productVersion: 3.1.0 + pullPolicy: IfNotPresent + nodes: + roleGroups: + cluster-manager: + config: + nodeRoles: + - cluster_manager + listenerClass: test-external-stable-$NAMESPACE + replicas: 1 + podOverrides: + spec: + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-cluster-manager-headless + data: + config: + nodeRoles: + - data + listenerClass: test-external-unstable-$NAMESPACE + replicas: 1 + podOverrides: + spec: + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data-headless + envOverrides: + # TODO Make these the defaults in the image + DISABLE_INSTALL_DEMO_CONFIG: "true" + configOverrides: + # TODO Add the required options to the operator + opensearch.yml: + # Disable memory mapping in this test; If memory mapping were activated, the kernel setting + # vm.max_map_count would have to be increased to 262144 on the node. + node.store.allow_mmap: "false" + # TODO Check that this is safe despite the warning in the documentation + plugins.security.allow_default_init_securityindex: "true" + plugins.security.ssl.transport.enabled: "true" + plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt + plugins.security.ssl.http.enabled: "true" + plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt + plugins.security.authcz.admin_dn: "CN=generated certificate for pod" + podOverrides: + spec: + containers: + - name: opensearch + volumeMounts: + - name: security-config + mountPath: /stackable/opensearch/config/opensearch-security + readOnly: true + - name: tls + mountPath: /stackable/opensearch/config/tls + readOnly: true + securityContext: + fsGroup: 1000 + volumes: + - name: security-config + secret: + secretName: opensearch-security-config + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" +--- +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-security-config +stringData: + action_groups.yml: | + --- + _meta: + type: actiongroups + config_version: 2 + allowlist.yml: | + --- + _meta: + type: allowlist + config_version: 2 + + config: + enabled: false + audit.yml: | + --- + _meta: + type: audit + config_version: 2 + + config: + enabled: false + config.yml: | + --- + _meta: + type: config + config_version: 2 + + config: + dynamic: + authc: + basic_internal_auth_domain: + description: Authenticate via HTTP Basic against internal users database + http_enabled: true + transport_enabled: true + order: 1 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + authz: {} + internal_users.yml: | + --- + # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh + + _meta: + type: internalusers + config_version: 2 + + admin: + hash: $2y$10$xRtHZFJ9QhG9GcYhRpAGpufCZYsk//nxsuel5URh0GWEBgmiI4Q/e + reserved: true + backend_roles: + - admin + description: OpenSearch admin user + + kibanaserver: + hash: $2y$10$vPgQ/6ilKDM5utawBqxoR.7euhVQ0qeGl8mPTeKhmFT475WUDrfQS + reserved: true + description: OpenSearch Dashboards user + nodes_dn.yml: | + --- + _meta: + type: nodesdn + config_version: 2 + roles.yml: | + --- + _meta: + type: roles + config_version: 2 + roles_mapping.yml: | + --- + _meta: + type: rolesmapping + config_version: 2 + + all_access: + reserved: false + backend_roles: + - admin + + kibana_server: + reserved: true + users: + - kibanaserver + tenants.yml: | + --- + _meta: + type: tenants + config_version: 2 diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml index 66d4042..41c5fa8 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -10,9 +10,185 @@ timeout: 600 apiVersion: apps/v1 kind: StatefulSet metadata: + generation: 1 + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable name: opensearch-nodes-cluster-manager + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch spec: + podManagementPolicy: Parallel replicas: 3 + selector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + serviceName: opensearch-nodes-cluster-manager-headless + template: + metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: 3.1.0 + stackable.tech/opensearch-role.cluster_manager: "true" + stackable.tech/vendor: Stackable + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + topologyKey: kubernetes.io/hostname + weight: 1 + containers: + - command: + - /stackable/opensearch//opensearch-docker-entrypoint.sh + env: + - name: DISABLE_INSTALL_DEMO_CONFIG + value: "true" + - name: cluster.initial_cluster_manager_nodes + value: opensearch-nodes-cluster-manager-0,opensearch-nodes-cluster-manager-1,opensearch-nodes-cluster-manager-2 + - name: discovery.seed_hosts + value: opensearch + - name: node.name + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: node.roles + value: cluster_manager + image: oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev + imagePullPolicy: IfNotPresent + name: opensearch + ports: + - containerPort: 9200 + name: http + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + readinessProbe: + failureThreshold: 3 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 3 + resources: + limits: + cpu: "4" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + startupProbe: + failureThreshold: 30 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 3 + volumeMounts: + - mountPath: /stackable/opensearch//config/opensearch.yml + name: config + readOnly: true + subPath: opensearch.yml + - mountPath: /stackable/opensearch//data + name: data + - mountPath: /stackable/listener + name: listener + - mountPath: /stackable/opensearch/config/opensearch-security + name: security-config + readOnly: true + - mountPath: /stackable/opensearch/config/tls + name: tls + readOnly: true + securityContext: + fsGroup: 1000 + serviceAccount: opensearch-serviceaccount + serviceAccountName: opensearch-serviceaccount + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + defaultMode: 420 + name: opensearch-nodes-cluster-manager + name: config + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-cluster-manager-headless + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem + name: tls + - name: security-config + secret: + defaultMode: 420 + secretName: opensearch-security-config + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi + volumeMode: Filesystem + status: + phase: Pending + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + annotations: + listeners.stackable.tech/listener-name: opensearch-nodes-cluster-manager + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: none + stackable.tech/vendor: Stackable + name: listener + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: "1" + storageClassName: listeners.stackable.tech + volumeMode: Filesystem + status: + phase: Pending status: readyReplicas: 3 replicas: 3 @@ -20,9 +196,190 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: + generation: 1 + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable name: opensearch-nodes-data + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch spec: + podManagementPolicy: Parallel replicas: 2 + selector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + serviceName: opensearch-nodes-data-headless + template: + metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: 3.1.0 + stackable.tech/opensearch-role.data: "true" + stackable.tech/opensearch-role.ingest: "true" + stackable.tech/opensearch-role.remote_cluster_client: "true" + stackable.tech/vendor: Stackable + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + topologyKey: kubernetes.io/hostname + weight: 1 + containers: + - command: + - /stackable/opensearch//opensearch-docker-entrypoint.sh + env: + - name: DISABLE_INSTALL_DEMO_CONFIG + value: "true" + - name: cluster.initial_cluster_manager_nodes + - name: discovery.seed_hosts + value: opensearch + - name: node.name + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: node.roles + value: ingest,data,remote_cluster_client + image: oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev + imagePullPolicy: IfNotPresent + name: opensearch + ports: + - containerPort: 9200 + name: http + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + readinessProbe: + failureThreshold: 3 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 3 + resources: + limits: + cpu: "4" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + startupProbe: + failureThreshold: 30 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 3 + volumeMounts: + - mountPath: /stackable/opensearch//config/opensearch.yml + name: config + readOnly: true + subPath: opensearch.yml + - mountPath: /stackable/opensearch//data + name: data + - mountPath: /stackable/listener + name: listener + - mountPath: /stackable/opensearch/config/opensearch-security + name: security-config + readOnly: true + - mountPath: /stackable/opensearch/config/tls + name: tls + readOnly: true + securityContext: + fsGroup: 1000 + serviceAccount: opensearch-serviceaccount + serviceAccountName: opensearch-serviceaccount + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + defaultMode: 420 + name: opensearch-nodes-data + name: config + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data-headless + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem + name: tls + - name: security-config + secret: + defaultMode: 420 + secretName: opensearch-security-config + updateStrategy: + rollingUpdate: + partition: 0 + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + volumeMode: Filesystem + status: + phase: Pending + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + annotations: + listeners.stackable.tech/listener-name: opensearch-nodes-data + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: none + stackable.tech/vendor: Stackable + name: listener + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: "1" + storageClassName: listeners.stackable.tech + volumeMode: Filesystem + status: + phase: Pending status: readyReplicas: 2 replicas: 2 @@ -30,9 +387,69 @@ status: apiVersion: v1 kind: ConfigMap metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable name: opensearch-nodes-cluster-manager + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +data: + opensearch.yml: |- + cluster.name: "opensearch" + discovery.type: "zen" + network.host: "0.0.0.0" + node.store.allow_mmap: "false" + plugins.security.allow_default_init_securityindex: "true" + plugins.security.authcz.admin_dn: "CN=generated certificate for pod" + plugins.security.nodes_dn: ["CN=generated certificate for pod"] + plugins.security.ssl.http.enabled: "true" + plugins.security.ssl.http.pemcert_filepath: "/stackable/opensearch/config/tls/tls.crt" + plugins.security.ssl.http.pemkey_filepath: "/stackable/opensearch/config/tls/tls.key" + plugins.security.ssl.http.pemtrustedcas_filepath: "/stackable/opensearch/config/tls/ca.crt" + plugins.security.ssl.transport.enabled: "true" + plugins.security.ssl.transport.pemcert_filepath: "/stackable/opensearch/config/tls/tls.crt" + plugins.security.ssl.transport.pemkey_filepath: "/stackable/opensearch/config/tls/tls.key" + plugins.security.ssl.transport.pemtrustedcas_filepath: "/stackable/opensearch/config/tls/ca.crt" --- apiVersion: v1 kind: ConfigMap metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable name: opensearch-nodes-data + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +data: + opensearch.yml: |- + cluster.name: "opensearch" + discovery.type: "zen" + network.host: "0.0.0.0" + node.store.allow_mmap: "false" + plugins.security.allow_default_init_securityindex: "true" + plugins.security.authcz.admin_dn: "CN=generated certificate for pod" + plugins.security.nodes_dn: ["CN=generated certificate for pod"] + plugins.security.ssl.http.enabled: "true" + plugins.security.ssl.http.pemcert_filepath: "/stackable/opensearch/config/tls/tls.crt" + plugins.security.ssl.http.pemkey_filepath: "/stackable/opensearch/config/tls/tls.key" + plugins.security.ssl.http.pemtrustedcas_filepath: "/stackable/opensearch/config/tls/ca.crt" + plugins.security.ssl.transport.enabled: "true" + plugins.security.ssl.transport.pemcert_filepath: "/stackable/opensearch/config/tls/tls.crt" + plugins.security.ssl.transport.pemkey_filepath: "/stackable/opensearch/config/tls/tls.key" + plugins.security.ssl.transport.pemtrustedcas_filepath: "/stackable/opensearch/config/tls/ca.crt" diff --git a/tests/templates/kuttl/smoke/10-install-opensearch.yaml b/tests/templates/kuttl/smoke/10-install-opensearch.yaml index 9c32793..dbc0c88 100644 --- a/tests/templates/kuttl/smoke/10-install-opensearch.yaml +++ b/tests/templates/kuttl/smoke/10-install-opensearch.yaml @@ -38,7 +38,7 @@ spec: storage: data: capacity: 2Gi - listenerClass: external-stable + listenerClass: cluster-internal replicas: 2 podOverrides: spec: @@ -73,8 +73,6 @@ spec: spec: containers: - name: opensearch - command: - - /stackable/opensearch/opensearch-docker-entrypoint.sh volumeMounts: - name: security-config mountPath: /stackable/opensearch/config/opensearch-security diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index b7bcbe1..4e66bf6 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -11,6 +11,10 @@ tests: dimensions: - opensearch - openshift + - name: external-access + dimensions: + - opensearch + - openshift suites: - name: nightly patch: From 2316cc252eba91bf386d34e260883baf889c67b1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 28 Jul 2025 12:52:49 +0200 Subject: [PATCH 03/16] changelog, docs --- CHANGELOG.md | 2 ++ .../pages/usage-guide/listenerclass.adoc | 17 +++++++++++++++++ docs/modules/opensearch/partials/nav.adoc | 1 + rust/operator-binary/src/crd/mod.rs | 2 +- 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 docs/modules/opensearch/pages/usage-guide/listenerclass.adoc diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f0b5b1..0cafe3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,5 +16,7 @@ All notable changes to this project will be documented in this file. - Resources (CPU, memory, storage) - PodDisruptionBudgets - Replicas +- Add Listener support ([#17]). [#10]: https://github.com/stackabletech/opensearch-operator/pull/10 +[#17]: https://github.com/stackabletech/opensearch-operator/pull/17 diff --git a/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc b/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc new file mode 100644 index 0000000..d4ea7ac --- /dev/null +++ b/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc @@ -0,0 +1,17 @@ += Service exposition with ListenerClasses +:description: Configure OpenSearch service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable. + +The operator deploys a xref:listener-operator:listener.adoc[Listener] for OpenSearch role-groups. +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleGroups.{role-group-name}.config.listenerClass`: + +[source,yaml] +---- +spec: + nodes: + roleGroups: + cluster-manager: + config: + listenerClass: external-stable +---- +<1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`) at role-group level. +This can be set for all role-groups individually. diff --git a/docs/modules/opensearch/partials/nav.adoc b/docs/modules/opensearch/partials/nav.adoc index c21e7d5..c096205 100644 --- a/docs/modules/opensearch/partials/nav.adoc +++ b/docs/modules/opensearch/partials/nav.adoc @@ -2,6 +2,7 @@ ** xref:opensearch:getting_started/installation.adoc[] ** xref:opensearch:getting_started/first_steps.adoc[] * xref:opensearch:usage-guide/index.adoc[] +** xref:opensearch:usage-guide/listenerclass.adoc[] ** xref:opensearch:usage-guide/node-roles.adoc[] ** xref:opensearch:usage-guide/storage-resource-configuration.adoc[] ** xref:opensearch:usage-guide/configuration-environment-overrides.adoc[] diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 942a4a6..24dad98 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -242,7 +242,7 @@ impl v1alpha1::OpenSearchConfig { }, }, }, - listener_class: Some("cluster-internal".to_string()), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), } } } From 31eceb1ce1eb9a82d5f894932e53e69a5652103b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:18:54 +0200 Subject: [PATCH 04/16] Update docs/modules/opensearch/pages/usage-guide/listenerclass.adoc Co-authored-by: Siegfried Weber --- docs/modules/opensearch/pages/usage-guide/listenerclass.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc b/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc index d4ea7ac..577b115 100644 --- a/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc @@ -2,7 +2,7 @@ :description: Configure OpenSearch service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable. The operator deploys a xref:listener-operator:listener.adoc[Listener] for OpenSearch role-groups. -The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleGroups.{role-group-name}.config.listenerClass`: +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleGroups.\{role-group-name}.config.listenerClass`: [source,yaml] ---- From 03e7070509d78f538ec5cc6911b6620323d435f4 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:19:18 +0200 Subject: [PATCH 05/16] Update docs/modules/opensearch/pages/usage-guide/listenerclass.adoc Co-authored-by: Siegfried Weber --- docs/modules/opensearch/pages/usage-guide/listenerclass.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc b/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc index 577b115..0d4312f 100644 --- a/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/opensearch/pages/usage-guide/listenerclass.adoc @@ -11,7 +11,7 @@ spec: roleGroups: cluster-manager: config: - listenerClass: external-stable + listenerClass: external-stable # <1> ---- <1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`) at role-group level. This can be set for all role-groups individually. From ae186fcaa24b239b7373b2e1337e4006036aa5b3 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:19:57 +0200 Subject: [PATCH 06/16] Update docs/modules/opensearch/partials/nav.adoc Co-authored-by: Siegfried Weber --- docs/modules/opensearch/partials/nav.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/opensearch/partials/nav.adoc b/docs/modules/opensearch/partials/nav.adoc index c096205..3ddf9b3 100644 --- a/docs/modules/opensearch/partials/nav.adoc +++ b/docs/modules/opensearch/partials/nav.adoc @@ -2,8 +2,8 @@ ** xref:opensearch:getting_started/installation.adoc[] ** xref:opensearch:getting_started/first_steps.adoc[] * xref:opensearch:usage-guide/index.adoc[] -** xref:opensearch:usage-guide/listenerclass.adoc[] ** xref:opensearch:usage-guide/node-roles.adoc[] +** xref:opensearch:usage-guide/listenerclass.adoc[] ** xref:opensearch:usage-guide/storage-resource-configuration.adoc[] ** xref:opensearch:usage-guide/configuration-environment-overrides.adoc[] ** xref:opensearch:usage-guide/operations/index.adoc[] From 469ae2713bbb6142fc8223929377d38e983a351c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:20:16 +0200 Subject: [PATCH 07/16] Update rust/operator-binary/src/controller/build/role_group_builder.rs Co-authored-by: Siegfried Weber --- rust/operator-binary/src/controller/build/role_group_builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index c19df81..b224627 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -49,7 +49,7 @@ const LISTENER_VOLUME_NAME: &str = "listener"; const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; // Path in opensearchproject/opensearch:3.0.0 -const OPENSEARCH_BASE_PATH: &str = "/stackable/opensearch/"; +const OPENSEARCH_BASE_PATH: &str = "/stackable/opensearch"; pub struct RoleGroupBuilder<'a> { service_account_name: String, From 7d0981a4c0fb7beb52a94c8709c51261dd33d32e Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:21:06 +0200 Subject: [PATCH 08/16] Update rust/operator-binary/src/controller/build/role_group_builder.rs Co-authored-by: Siegfried Weber --- rust/operator-binary/src/controller/build/role_group_builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index b224627..69d2d49 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -131,7 +131,7 @@ impl<'a> RoleGroupBuilder<'a> { &self .recommended_labels(ProductVersion::from_str("none").expect("version is supplied")), ) - .expect("should be a listener group name") + .expect("should return Ok independent of the given parameters") .build_pvc(LISTENER_VOLUME_NAME.to_string()) .expect("should be a valid annotation"); From 59596c45344c6f62a68d34722102a29eb72ab93b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:21:47 +0200 Subject: [PATCH 09/16] Update rust/operator-binary/src/crd/mod.rs Co-authored-by: Siegfried Weber --- rust/operator-binary/src/crd/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 24dad98..e2f365b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -133,7 +133,7 @@ pub mod versioned { #[fragment_attrs(serde(default))] pub resources: Resources, - /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. #[serde(default = "default_listener_class")] pub listener_class: String, } From ddcb8587115f8bbd02f28bd094bc4fee3927b731 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:25:19 +0200 Subject: [PATCH 10/16] Update tests/templates/kuttl/external-access/20-assert.yaml Co-authored-by: Siegfried Weber --- tests/templates/kuttl/external-access/20-assert.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index ce0c0d7..dc82170 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -1,7 +1,3 @@ -# All fields are checked that are set by the operator. -# This helps to detect unintentional changes. -# The maintenance effort should be okay as long as it is only done in the smoke test. -# TODO Check individual field in unit tests --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert From 576aa9e08948ac266e330723b498a1c473ed4efc Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:26:19 +0200 Subject: [PATCH 11/16] Update tests/templates/kuttl/smoke/10-assert.yaml Co-authored-by: Siegfried Weber --- tests/templates/kuttl/smoke/10-assert.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml index 41c5fa8..5b93e7d 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -10,7 +10,6 @@ timeout: 600 apiVersion: apps/v1 kind: StatefulSet metadata: - generation: 1 labels: app.kubernetes.io/component: nodes app.kubernetes.io/instance: opensearch From e17ac67da45f44923147180ab8771c41e3faf617 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 30 Jul 2025 11:51:43 +0200 Subject: [PATCH 12/16] Update rust/operator-binary/src/crd/mod.rs Co-authored-by: Siegfried Weber --- rust/operator-binary/src/crd/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index e2f365b..d6a851f 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -134,7 +134,7 @@ pub mod versioned { pub resources: Resources, /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. - #[serde(default = "default_listener_class")] + #[fragment_attrs(serde(default))] pub listener_class: String, } From bbfb9f8d36a19954e016b4fb49be88367cc4c6c7 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 30 Jul 2025 12:39:30 +0200 Subject: [PATCH 13/16] reverted assert, removed default function --- .../helm/opensearch-operator/crds/crds.yaml | 4 +- rust/operator-binary/src/crd/mod.rs | 4 - tests/templates/kuttl/smoke/10-assert.yaml | 164 ++++++++++++++++-- 3 files changed, 155 insertions(+), 17 deletions(-) diff --git a/deploy/helm/opensearch-operator/crds/crds.yaml b/deploy/helm/opensearch-operator/crds/crds.yaml index 102d8f0..7b9d514 100644 --- a/deploy/helm/opensearch-operator/crds/crds.yaml +++ b/deploy/helm/opensearch-operator/crds/crds.yaml @@ -134,7 +134,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. nullable: true type: string nodeRoles: @@ -330,7 +330,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. nullable: true type: string nodeRoles: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index d6a851f..84a6efa 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -168,10 +168,6 @@ pub mod versioned { } } -fn default_listener_class() -> String { - DEFAULT_LISTENER_CLASS.to_string() -} - impl HasStatusCondition for v1alpha1::OpenSearchCluster { fn conditions(&self) -> Vec { match &self.status { diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml index 5b93e7d..00c6beb 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -59,7 +59,7 @@ spec: weight: 1 containers: - command: - - /stackable/opensearch//opensearch-docker-entrypoint.sh + - /stackable/opensearch/opensearch-docker-entrypoint.sh env: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" @@ -107,11 +107,11 @@ spec: port: http timeoutSeconds: 3 volumeMounts: - - mountPath: /stackable/opensearch//config/opensearch.yml + - mountPath: /stackable/opensearch/config/opensearch.yml name: config readOnly: true subPath: opensearch.yml - - mountPath: /stackable/opensearch//data + - mountPath: /stackable/opensearch/data name: data - mountPath: /stackable/listener name: listener @@ -195,7 +195,6 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - generation: 1 labels: app.kubernetes.io/component: nodes app.kubernetes.io/instance: opensearch @@ -247,7 +246,7 @@ spec: weight: 1 containers: - command: - - /stackable/opensearch//opensearch-docker-entrypoint.sh + - /stackable/opensearch/opensearch-docker-entrypoint.sh env: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" @@ -294,11 +293,11 @@ spec: port: http timeoutSeconds: 3 volumeMounts: - - mountPath: /stackable/opensearch//config/opensearch.yml + - mountPath: /stackable/opensearch/config/opensearch.yml name: config readOnly: true subPath: opensearch.yml - - mountPath: /stackable/opensearch//data + - mountPath: /stackable/opensearch/data name: data - mountPath: /stackable/listener name: listener @@ -337,10 +336,6 @@ spec: secret: defaultMode: 420 secretName: opensearch-security-config - updateStrategy: - rollingUpdate: - partition: 0 - type: RollingUpdate volumeClaimTemplates: - apiVersion: v1 kind: PersistentVolumeClaim @@ -452,3 +447,150 @@ data: plugins.security.ssl.transport.pemcert_filepath: "/stackable/opensearch/config/tls/tls.crt" plugins.security.ssl.transport.pemkey_filepath: "/stackable/opensearch/config/tls/tls.key" plugins.security.ssl.transport.pemtrustedcas_filepath: "/stackable/opensearch/config/tls/ca.crt" +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable + name: opensearch-nodes-cluster-manager-headless +spec: + ports: + - name: http + port: 9200 + protocol: TCP + targetPort: 9200 + - name: transport + port: 9300 + protocol: TCP + targetPort: 9300 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable + name: opensearch-nodes-data-headless +spec: + ports: + - name: http + port: 9200 + protocol: TCP + targetPort: 9200 + - name: transport + port: 9300 + protocol: TCP + targetPort: 9300 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable + name: opensearch +spec: + ports: + - name: http + port: 9200 + protocol: TCP + targetPort: 9200 + - name: transport + port: 9300 + protocol: TCP + targetPort: 9300 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + stackable.tech/opensearch-role.cluster_manager: "true" + type: ClusterIP +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable + name: opensearch-serviceaccount + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable + name: opensearch-rolebinding + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: opensearch-clusterrole +subjects: +- kind: ServiceAccount + name: opensearch-serviceaccount +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + name: opensearch-nodes +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch From d3ac257b1d2bbfd33bf94d81d78a953fa608a5a6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 30 Jul 2025 12:41:46 +0200 Subject: [PATCH 14/16] missed owner ref --- tests/templates/kuttl/smoke/10-assert.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml index 00c6beb..e8c34e8 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -519,6 +519,11 @@ metadata: app.kubernetes.io/version: 3.1.0 stackable.tech/vendor: Stackable name: opensearch + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch spec: ports: - name: http From b5e20402fa6c6c42b37f0f67fb7ba3269032e8a6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 30 Jul 2025 12:53:26 +0200 Subject: [PATCH 15/16] added listener test case --- .../kuttl/external-access/20-assert.yaml | 21 +++++++++++++++++-- .../kuttl/external-access/opensearch.yaml.j2 | 19 +++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index dc82170..6e7c514 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -16,7 +16,17 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: opensearch-nodes-data + name: opensearch-nodes-data1 +spec: + replicas: 1 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-nodes-data2 spec: replicas: 1 status: @@ -33,12 +43,19 @@ spec: apiVersion: v1 kind: Service metadata: - name: opensearch-nodes-data + name: opensearch-nodes-data1 spec: type: NodePort # external-unstable --- apiVersion: v1 kind: Service +metadata: + name: opensearch-nodes-data2 +spec: + type: ClusterIP # cluster-internal +--- +apiVersion: v1 +kind: Service metadata: name: opensearch spec: diff --git a/tests/templates/kuttl/external-access/opensearch.yaml.j2 b/tests/templates/kuttl/external-access/opensearch.yaml.j2 index 73370cd..463cf67 100644 --- a/tests/templates/kuttl/external-access/opensearch.yaml.j2 +++ b/tests/templates/kuttl/external-access/opensearch.yaml.j2 @@ -24,7 +24,7 @@ spec: metadata: annotations: secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-cluster-manager-headless - data: + data1: config: nodeRoles: - data @@ -38,7 +38,22 @@ spec: volumeClaimTemplate: metadata: annotations: - secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data-headless + secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data1-headless + data2: + config: + nodeRoles: + - data + listenerClass: test-cluster-internal-$NAMESPACE + replicas: 1 + podOverrides: + spec: + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data2-headless envOverrides: # TODO Make these the defaults in the image DISABLE_INSTALL_DEMO_CONFIG: "true" From ca3e9b1cae2f1e0a23bb59c030653aa17cc3959a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 30 Jul 2025 14:19:56 +0200 Subject: [PATCH 16/16] use versioned labels, move listener_pvc to framework module --- .../controller/build/role_group_builder.rs | 34 +++++++------------ rust/operator-binary/src/framework.rs | 1 + .../operator-binary/src/framework/listener.rs | 19 +++++++++++ tests/templates/kuttl/smoke/10-assert.yaml | 4 +-- 4 files changed, 34 insertions(+), 24 deletions(-) create mode 100644 rust/operator-binary/src/framework/listener.rs diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 69d2d49..503ae97 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -1,13 +1,5 @@ -use std::str::FromStr; - use stackable_operator::{ - builder::{ - meta::ObjectMetaBuilder, - pod::{ - container::ContainerBuilder, - volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, - }, - }, + builder::{meta::ObjectMetaBuilder, pod::container::ContainerBuilder}, crd::listener::{self}, k8s_openapi::{ DeepMerge, @@ -30,9 +22,10 @@ use crate::{ controller::{ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster}, crd::v1alpha1, framework::{ - ProductVersion, RoleGroupName, + RoleGroupName, builder::meta::ownerreference_from_resource, kvp::label::{recommended_labels, role_group_selector, role_selector}, + listener::listener_pvc, role_group_utils::ResourceNames, }, }; @@ -126,14 +119,11 @@ impl<'a> RoleGroupBuilder<'a> { // addresses. This will be the case even when no class is set (and // the value defaults to cluster-internal) as the address should // still be consistent. - let listener_volume_claim_template = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(listener_group_name), - &self - .recommended_labels(ProductVersion::from_str("none").expect("version is supplied")), - ) - .expect("should return Ok independent of the given parameters") - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .expect("should be a valid annotation"); + let listener_volume_claim_template = listener_pvc( + listener_group_name, + &self.recommended_labels(), + LISTENER_VOLUME_NAME.to_string(), + ); let pvcs: Option> = Some(vec![ data_volume_claim_template, @@ -168,7 +158,7 @@ impl<'a> RoleGroupBuilder<'a> { } let metadata = ObjectMetaBuilder::new() - .with_labels(self.recommended_labels(self.cluster.product_version.clone())) + .with_labels(self.recommended_labels()) .with_labels(node_role_labels) .build(); @@ -415,16 +405,16 @@ impl<'a> RoleGroupBuilder<'a> { None, Some(true), )) - .with_labels(self.recommended_labels(self.cluster.product_version.clone())) + .with_labels(self.recommended_labels()) .with_labels(extra_labels) .build() } - fn recommended_labels(&self, product_version: ProductVersion) -> Labels { + fn recommended_labels(&self) -> Labels { recommended_labels( &self.cluster, &self.context_names.product_name, - &product_version, + &self.cluster.product_version, &self.context_names.operator_name, &self.context_names.controller_name, &ValidatedCluster::role_name(), diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index aad601d..07c5699 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -12,6 +12,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; pub mod builder; pub mod cluster_resources; pub mod kvp; +pub mod listener; pub mod role_group_utils; pub mod role_utils; diff --git a/rust/operator-binary/src/framework/listener.rs b/rust/operator-binary/src/framework/listener.rs new file mode 100644 index 0000000..5151aaa --- /dev/null +++ b/rust/operator-binary/src/framework/listener.rs @@ -0,0 +1,19 @@ +use stackable_operator::{ + builder::pod::volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, + k8s_openapi::api::core::v1::PersistentVolumeClaim, + kvp::Labels, +}; + +pub fn listener_pvc( + listener_group_name: String, + labels: &Labels, + pvc_name: String, +) -> PersistentVolumeClaim { + ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(listener_group_name), + labels, + ) + .expect("should return Ok independent of the given parameters") + .build_pvc(pvc_name.to_string()) + .expect("should be a valid annotation") +} diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml index e8c34e8..7f33e9a 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -175,7 +175,7 @@ spec: app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster app.kubernetes.io/name: opensearch app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: none + app.kubernetes.io/version: 3.1.0 stackable.tech/vendor: Stackable name: listener spec: @@ -361,7 +361,7 @@ spec: app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster app.kubernetes.io/name: opensearch app.kubernetes.io/role-group: data - app.kubernetes.io/version: none + app.kubernetes.io/version: 3.1.0 stackable.tech/vendor: Stackable name: listener spec: