From a457de33a3d15183f6e9e2e2ee0fb038dc75ec0b Mon Sep 17 00:00:00 2001 From: dervoeti Date: Tue, 28 Oct 2025 19:40:14 +0100 Subject: [PATCH 01/13] feat: tls support --- CHANGELOG.md | 2 + deploy/helm/opa-operator/crds/crds.yaml | 14 +++ docs/modules/opa/pages/usage-guide/tls.adoc | 71 +++++++++++++++ docs/modules/opa/partials/nav.adoc | 1 + rust/operator-binary/src/controller.rs | 88 ++++++++++++++++--- rust/operator-binary/src/crd/mod.rs | 12 +++ rust/operator-binary/src/discovery.rs | 29 ++++-- rust/operator-binary/src/service.rs | 47 +++++++--- .../smoke/09-install-secretclass.yaml.j2 | 19 ++++ .../kuttl/smoke/10-install-opa.yaml.j2 | 4 +- .../kuttl/smoke/20-install-test-opa.yaml | 19 ++++ tests/templates/kuttl/smoke/30-assert.yaml | 2 +- .../templates/kuttl/smoke/30_test-metrics.py | 24 +++-- .../templates/kuttl/smoke/30_test-regorule.py | 10 +-- tests/templates/kuttl/smoke/31-assert.yaml | 2 +- 15 files changed, 294 insertions(+), 50 deletions(-) create mode 100644 docs/modules/opa/pages/usage-guide/tls.adoc create mode 100644 tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 01de4f75..42dd2e22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file. - Helm: Allow Pod `priorityClassName` to be configured ([#762]). - Add support for OPA `1.8.0` ([#765]). - Add `prometheus.io/path|port|scheme` annotations to metrics service ([#767]). +- Add support for TLS ([#774]) ### Changed @@ -41,6 +42,7 @@ All notable changes to this project will be documented in this file. [#767]: https://github.com/stackabletech/opa-operator/pull/767 [#771]: https://github.com/stackabletech/opa-operator/pull/771 [#772]: https://github.com/stackabletech/opa-operator/pull/772 +[#774]: https://github.com/stackabletech/opa-operator/pull/774 ## [25.7.0] - 2025-07-23 diff --git a/deploy/helm/opa-operator/crds/crds.yaml b/deploy/helm/opa-operator/crds/crds.yaml index 13deb8b5..7e2b9c38 100644 --- a/deploy/helm/opa-operator/crds/crds.yaml +++ b/deploy/helm/opa-operator/crds/crds.yaml @@ -27,6 +27,7 @@ spec: clusterConfig: default: listenerClass: cluster-internal + tls: null userInfo: null description: Global OPA cluster configuration that applies to all roles and role groups. properties: @@ -49,6 +50,19 @@ spec: - external-unstable - external-stable type: string + tls: + description: |- + TLS encryption settings for the OPA server. + When configured, OPA will use HTTPS (port 8443) instead of HTTP (port 8081). + Clients must connect using HTTPS and trust the certificates provided by the configured SecretClass. + nullable: true + properties: + serverSecretClass: + description: Name of the SecretClass which will provide TLS certificates for the OPA server. + type: string + required: + - serverSecretClass + type: object userInfo: description: |- Configures how to fetch additional metadata about users (such as group memberships) diff --git a/docs/modules/opa/pages/usage-guide/tls.adoc b/docs/modules/opa/pages/usage-guide/tls.adoc new file mode 100644 index 00000000..63ce31ca --- /dev/null +++ b/docs/modules/opa/pages/usage-guide/tls.adoc @@ -0,0 +1,71 @@ += Enabling TLS Encryption +:description: Learn how to enable TLS encryption for your OPA cluster to secure client connections. + +TLS encryption for securing connections between clients and the OPA server can be configured in the `OpaCluster` resource. When TLS is enabled, OPA will serve requests over HTTPS instead of HTTP. + +== Overview + +TLS encryption in OPA is disabled by default. To enable it, you need to: + +1. Create a `SecretClass` that provides TLS certificates +2. Reference the `SecretClass` in your `OpaCluster` specification + +The operator integrates with the xref:secret-operator:index.adoc[Secret Operator] to automatically provision and mount TLS certificates to the OPA pods. + +== Configuration + +=== Creating a SecretClass + +First, create a `SecretClass` that will provide TLS certificates. Here's an example using xref:secret-operator:secretclass.adoc#backend-autotls[autoTls]: + +[source,yaml] +---- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: opa-tls +spec: + backend: + autoTls: + ca: + autoGenerate: true + secret: + name: opa-tls-ca + namespace: default +---- + +This SecretClass uses the autoTls backend, which automatically generates a Certificate Authority (CA) and signs certificates for your OPA cluster. + +Similarly, you can also use xref:secret-operator:secretclass.adoc#backend[other backends] supported by Secret Operator. + +=== Enabling TLS in OpaCluster + +Once you have a SecretClass, enable TLS in your OpaCluster by setting the `.spec.clusterConfig.tls.serverSecretClass` field: + +[source,yaml] +---- +kind: OpaCluster +name: opa-with-tls +spec: + clusterConfig: + tls: + serverSecretClass: opa-tls # <1> +---- +<1> Reference the SecretClass created above + +== Discovery ConfigMap + +The operator automatically creates a discovery ConfigMap, with the same name as the OPA cluster, that contains the connection URL for your cluster. When TLS is enabled, this ConfigMap will contain an HTTPS URL and the SecretClass name: + +[source,yaml] +---- +apiVersion: v1 +kind: ConfigMap +metadata: + name: opa-with-tls +data: + OPA: "https://opa-with-tls.default.svc.cluster.local:8443/" + OPA_SECRET_CLASS: "opa-tls" +---- + +Applications can use this ConfigMap to discover and connect to the OPA cluster automatically. diff --git a/docs/modules/opa/partials/nav.adoc b/docs/modules/opa/partials/nav.adoc index f354d583..2f7dd58c 100644 --- a/docs/modules/opa/partials/nav.adoc +++ b/docs/modules/opa/partials/nav.adoc @@ -10,6 +10,7 @@ ** xref:opa:usage-guide/monitoring.adoc[] ** xref:opa:usage-guide/OpenTelemetry.adoc[] ** xref:opa:usage-guide/configuration-environment-overrides.adoc[] +** xref:opa:usage-guide/tls.adoc[] ** xref:opa:usage-guide/operations/index.adoc[] *** xref:opa:usage-guide/operations/cluster-operations.adoc[] // *** xref:hdfs:usage-guide/operations/pod-placement.adoc[] Missing diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index c092e924..c455897e 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -23,7 +23,7 @@ use stackable_operator::{ container::{ContainerBuilder, FieldPathEnvVar}, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, - volume::VolumeBuilder, + volume::{SecretOperatorVolumeSourceBuilder, VolumeBuilder}, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -104,6 +104,8 @@ const USER_INFO_FETCHER_CREDENTIALS_VOLUME_NAME: &str = "credentials"; const USER_INFO_FETCHER_CREDENTIALS_DIR: &str = "/stackable/credentials"; const USER_INFO_FETCHER_KERBEROS_VOLUME_NAME: &str = "kerberos"; const USER_INFO_FETCHER_KERBEROS_DIR: &str = "/stackable/kerberos"; +const TLS_VOLUME_NAME: &str = "tls"; +const TLS_STORE_DIR: &str = "/stackable/tls"; const DOCKER_IMAGE_BASE_NAME: &str = "opa"; @@ -329,6 +331,11 @@ pub enum Error { #[snafu(display("failed to build service"))] BuildService { source: service::Error }, + + #[snafu(display("failed to build TLS volume"))] + TlsVolumeBuild { + source: builder::pod::volume::SecretOperatorVolumeSourceBuilderError, + }, } type Result = std::result::Result; @@ -823,6 +830,8 @@ fn build_server_rolegroup_daemonset( &v1alpha1::Container::BundleBuilder, ); + let opa_tls_config = opa.spec.cluster_config.tls.as_ref(); + cb_opa .image_from_product_image(resolved_product_image) .command(vec![ @@ -835,29 +844,50 @@ fn build_server_rolegroup_daemonset( .args(vec![build_opa_start_command( merged_config, &opa_container_name, + opa_tls_config, )]) .add_env_vars(env) .add_env_var( "CONTAINERDEBUG_LOG_DIRECTORY", format!("{STACKABLE_LOG_DIR}/containerdebug"), - ) - .add_container_port(APP_PORT_NAME, APP_PORT.into()) - // If we also add a container port "metrics" pointing to the same port number, we get a - // - // .spec.template.spec.containers[name="opa"].ports: duplicate entries for key [containerPort=8081,protocol="TCP"] - // - // So we don't do that + ); + + // Add appropriate container port based on TLS configuration + // If we also add a container port "metrics" pointing to the same port number, we get a + // + // .spec.template.spec.containers[name="opa"].ports: duplicate entries for key [containerPort=8081,protocol="TCP"] + // + // So we don't do that + if opa_tls_config.is_some() { + cb_opa.add_container_port(service::APP_TLS_PORT_NAME, service::APP_TLS_PORT.into()); + cb_opa + .add_volume_mount(TLS_VOLUME_NAME, TLS_STORE_DIR) + .context(AddVolumeMountSnafu)?; + } else { + cb_opa.add_container_port(APP_PORT_NAME, APP_PORT.into()); + } + + cb_opa .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_DIR) .context(AddVolumeMountSnafu)? .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? - .resources(merged_config.resources.to_owned().into()) + .resources(merged_config.resources.to_owned().into()); + + let (probe_port_name, probe_scheme) = if opa_tls_config.is_some() { + (service::APP_TLS_PORT_NAME, Some("HTTPS".to_string())) + } else { + (APP_PORT_NAME, Some("HTTP".to_string())) + }; + + cb_opa .readiness_probe(Probe { initial_delay_seconds: Some(5), period_seconds: Some(10), failure_threshold: Some(5), http_get: Some(HTTPGetAction { - port: IntOrString::String(APP_PORT_NAME.to_string()), + port: IntOrString::String(probe_port_name.to_string()), + scheme: probe_scheme.clone(), ..HTTPGetAction::default() }), ..Probe::default() @@ -866,7 +896,8 @@ fn build_server_rolegroup_daemonset( initial_delay_seconds: Some(30), period_seconds: Some(10), http_get: Some(HTTPGetAction { - port: IntOrString::String(APP_PORT_NAME.to_string()), + port: IntOrString::String(probe_port_name.to_string()), + scheme: probe_scheme, ..HTTPGetAction::default() }), ..Probe::default() @@ -918,6 +949,22 @@ fn build_server_rolegroup_daemonset( .service_account_name(service_account.name_any()) .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + if let Some(tls) = opa_tls_config { + pb.add_volume( + VolumeBuilder::new(TLS_VOLUME_NAME) + .ephemeral( + SecretOperatorVolumeSourceBuilder::new(&tls.server_secret_class) + .with_service_scope(opa.server_role_service_name()) + .with_service_scope(rolegroup_ref.rolegroup_headless_service_name()) + .with_service_scope(rolegroup_ref.rolegroup_metrics_service_name()) + .build() + .context(TlsVolumeBuildSnafu)?, + ) + .build(), + ) + .context(AddVolumeSnafu)?; + } + if let Some(user_info) = &opa.spec.cluster_config.user_info { let mut cb_user_info_fetcher = ContainerBuilder::new("user-info-fetcher").context(IllegalContainerNameSnafu)?; @@ -1146,7 +1193,11 @@ fn build_config_file(merged_config: &v1alpha1::OpaConfig) -> String { serde_json::to_string_pretty(&json!(config)).unwrap() } -fn build_opa_start_command(merged_config: &v1alpha1::OpaConfig, container_name: &str) -> String { +fn build_opa_start_command( + merged_config: &v1alpha1::OpaConfig, + container_name: &str, + tls_config: Option<&v1alpha1::OpaTls>, +) -> String { let mut file_log_level = DEFAULT_FILE_LOG_LEVEL; let mut console_log_level = DEFAULT_CONSOLE_LOG_LEVEL; let mut server_log_level = DEFAULT_SERVER_LOG_LEVEL; @@ -1187,6 +1238,17 @@ fn build_opa_start_command(merged_config: &v1alpha1::OpaConfig, container_name: } } + let (bind_port, tls_flags) = if tls_config.is_some() { + ( + service::APP_TLS_PORT, + format!( + "--tls-cert-file {TLS_STORE_DIR}/tls.crt --tls-private-key-file {TLS_STORE_DIR}/tls.key" + ), + ) + } else { + (APP_PORT, String::new()) + }; + // Redirects matter! // We need to watch out, that the following "$!" call returns the PID of the main (opa-bundle-builder) process, // and not some utility (e.g. multilog or tee) process. @@ -1202,7 +1264,7 @@ fn build_opa_start_command(merged_config: &v1alpha1::OpaConfig, container_name: {remove_vector_shutdown_file_command} prepare_signal_handlers containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - opa run -s -a 0.0.0.0:{APP_PORT} -c {CONFIG_DIR}/{CONFIG_FILE} -l {opa_log_level} --shutdown-grace-period {shutdown_grace_period_s} --disable-telemetry {logging_redirects} & + opa run -s -a 0.0.0.0:{bind_port} -c {CONFIG_DIR}/{CONFIG_FILE} -l {opa_log_level} --shutdown-grace-period {shutdown_grace_period_s} --disable-telemetry {tls_flags} {logging_redirects} & wait_for_termination $! {create_vector_shutdown_file_command} ", diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 0cd9951f..a6b08d34 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -115,6 +115,18 @@ pub mod versioned { /// from an external directory service. #[serde(default)] pub user_info: Option, + /// TLS encryption settings for the OPA server. + /// When configured, OPA will use HTTPS (port 8443) instead of HTTP (port 8081). + /// Clients must connect using HTTPS and trust the certificates provided by the configured SecretClass. + #[serde(default)] + pub tls: Option, + } + + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct OpaTls { + /// Name of the SecretClass which will provide TLS certificates for the OPA server. + pub server_secret_class: String, } // TODO: Temporary solution until listener-operator is finished diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index f9c1023b..05fb3805 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -8,7 +8,10 @@ use stackable_operator::{ utils::cluster_info::KubernetesClusterInfo, }; -use crate::{controller::build_recommended_labels, service::APP_PORT}; +use crate::{ + controller::build_recommended_labels, + service::{APP_PORT, APP_TLS_PORT}, +}; #[derive(Snafu, Debug)] pub enum Error { @@ -63,8 +66,15 @@ fn build_discovery_configmap( svc: &Service, cluster_info: &KubernetesClusterInfo, ) -> Result { + let tls_config = opa.spec.cluster_config.tls.as_ref(); + let (scheme, port) = if tls_config.is_some() { + ("https", APP_TLS_PORT) + } else { + ("http", APP_PORT) + }; + let url = format!( - "http://{name}.{namespace}.svc.{cluster_domain}:{port}/", + "{scheme}://{name}.{namespace}.svc.{cluster_domain}:{port}/", name = svc.metadata.name.as_deref().context(NoNameSnafu)?, namespace = svc .metadata @@ -72,7 +82,6 @@ fn build_discovery_configmap( .as_deref() .context(NoNamespaceSnafu)?, cluster_domain = cluster_info.cluster_domain, - port = APP_PORT ); let metadata = ObjectMetaBuilder::new() @@ -91,9 +100,13 @@ fn build_discovery_configmap( .context(ObjectMetaSnafu)? .build(); - ConfigMapBuilder::new() - .metadata(metadata) - .add_data("OPA", url) - .build() - .context(BuildConfigMapSnafu) + let mut cm_builder = ConfigMapBuilder::new(); + + cm_builder.metadata(metadata).add_data("OPA", url); + + if let Some(tls) = tls_config { + cm_builder.add_data("OPA_SECRET_CLASS", &tls.server_secret_class); + } + + cm_builder.build().context(BuildConfigMapSnafu) } diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs index 6fb58f09..40705bfd 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/service.rs @@ -11,7 +11,9 @@ use stackable_operator::{ use crate::controller::build_recommended_labels; pub const APP_PORT: u16 = 8081; +pub const APP_TLS_PORT: u16 = 8443; pub const APP_PORT_NAME: &str = "http"; +pub const APP_TLS_PORT_NAME: &str = "https"; pub const METRICS_PORT_NAME: &str = "metrics"; #[derive(Snafu, Debug)] @@ -57,7 +59,7 @@ pub(crate) fn build_server_role_service( let service_spec = ServiceSpec { type_: Some(opa.spec.cluster_config.listener_class.k8s_service_type()), - ports: Some(data_service_ports()), + ports: Some(data_service_ports(opa.spec.cluster_config.tls.is_some())), selector: Some(service_selector_labels.into()), internal_traffic_policy: Some("Local".to_string()), ..ServiceSpec::default() @@ -102,7 +104,7 @@ pub(crate) fn build_rolegroup_headless_service( // options there are non-existent (mTLS still opens plain port) or suck (Kerberos). type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), - ports: Some(data_service_ports()), + ports: Some(data_service_ports(opa.spec.cluster_config.tls.is_some())), selector: Some(role_group_selector_labels(opa, rolegroup)?.into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() @@ -135,13 +137,17 @@ pub(crate) fn build_rolegroup_metrics_service( )) .context(ObjectMetaSnafu)? .with_labels(prometheus_labels()) - .with_annotations(prometheus_annotations()) + .with_annotations(prometheus_annotations( + opa.spec.cluster_config.tls.is_some(), + )) .build(); let service_spec = ServiceSpec { type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), - ports: Some(vec![metrics_service_port()]), + ports: Some(vec![metrics_service_port( + opa.spec.cluster_config.tls.is_some(), + )]), selector: Some(role_group_selector_labels(opa, rolegroup)?.into()), ..ServiceSpec::default() }; @@ -162,21 +168,28 @@ fn role_group_selector_labels( .context(BuildLabelSnafu) } -fn data_service_ports() -> Vec { - // Currently only HTTP is exposed +fn data_service_ports(tls_enabled: bool) -> Vec { + let (port_name, port) = if tls_enabled { + (APP_TLS_PORT_NAME, APP_TLS_PORT) + } else { + (APP_PORT_NAME, APP_PORT) + }; + vec![ServicePort { - name: Some(APP_PORT_NAME.to_string()), - port: APP_PORT.into(), + name: Some(port_name.to_string()), + port: port.into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }] } -fn metrics_service_port() -> ServicePort { +fn metrics_service_port(tls_enabled: bool) -> ServicePort { + let port = if tls_enabled { APP_TLS_PORT } else { APP_PORT }; + ServicePort { name: Some(METRICS_PORT_NAME.to_string()), - // The metrics are served on the same port as the HTTP traffic - port: APP_PORT.into(), + // The metrics are served on the same port as the HTTP/HTTPS traffic + port: port.into(), protocol: Some("TCP".to_string()), ..ServicePort::default() } @@ -192,11 +205,17 @@ fn prometheus_labels() -> Labels { /// These annotations can be used in a ServiceMonitor. /// /// see also -fn prometheus_annotations() -> Annotations { +fn prometheus_annotations(tls_enabled: bool) -> Annotations { + let (port, scheme) = if tls_enabled { + (APP_TLS_PORT, "https") + } else { + (APP_PORT, "http") + }; + Annotations::try_from([ ("prometheus.io/path".to_owned(), "/metrics".to_owned()), - ("prometheus.io/port".to_owned(), APP_PORT.to_string()), - ("prometheus.io/scheme".to_owned(), "http".to_owned()), + ("prometheus.io/port".to_owned(), port.to_string()), + ("prometheus.io/scheme".to_owned(), scheme.to_owned()), ("prometheus.io/scrape".to_owned(), "true".to_owned()), ]) .expect("should be valid annotations") diff --git a/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 b/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 new file mode 100644 index 00000000..83f37a19 --- /dev/null +++ b/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 @@ -0,0 +1,19 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - << EOF + --- + apiVersion: secrets.stackable.tech/v1alpha1 + kind: SecretClass + metadata: + name: opa-tls + spec: + backend: + autoTls: + ca: + autoGenerate: true + secret: + name: opa-tls-ca + namespace: $NAMESPACE diff --git a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 index 25d5aa57..fcbb48ec 100644 --- a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 @@ -30,8 +30,10 @@ spec: productVersion: "{{ test_scenario['values']['opa'] }}" {% endif %} pullPolicy: IfNotPresent -{% if lookup('env', 'VECTOR_AGGREGATOR') %} clusterConfig: + tls: + serverSecretClass: opa-tls +{% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} servers: diff --git a/tests/templates/kuttl/smoke/20-install-test-opa.yaml b/tests/templates/kuttl/smoke/20-install-test-opa.yaml index 05d33126..2c44e739 100644 --- a/tests/templates/kuttl/smoke/20-install-test-opa.yaml +++ b/tests/templates/kuttl/smoke/20-install-test-opa.yaml @@ -15,6 +15,8 @@ spec: labels: app: test-opa spec: + securityContext: + fsGroup: 1000 containers: - name: test-opa image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev @@ -27,3 +29,20 @@ spec: limits: memory: "128Mi" cpu: "1" + volumeMounts: + - name: tls + mountPath: /tls + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: opa-tls + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml index 9c83596e..51b31ea3 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml +++ b/tests/templates/kuttl/smoke/30-assert.yaml @@ -4,4 +4,4 @@ kind: TestAssert metadata: name: test-regorule commands: - - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-regorule.py -u 'http://test-opa-server:8081/v1/data/test' + - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-regorule.py -u "https://test-opa-server.$NAMESPACE.svc.cluster.local:8443/v1/data/test" diff --git a/tests/templates/kuttl/smoke/30_test-metrics.py b/tests/templates/kuttl/smoke/30_test-metrics.py index 54625f03..dac209fd 100644 --- a/tests/templates/kuttl/smoke/30_test-metrics.py +++ b/tests/templates/kuttl/smoke/30_test-metrics.py @@ -1,11 +1,21 @@ #!/usr/bin/env python import requests +import argparse +import os -metrics_url = "http://test-opa-server-default-metrics:8081/metrics" -response = requests.get(metrics_url) +if __name__ == "__main__": + all_args = argparse.ArgumentParser() + all_args.add_argument("-n", "--namespace", required=True, help="Kubernetes namespace") + args = vars(all_args.parse_args()) -assert response.status_code == 200, "Metrics endpoint must return a 200 status code" -assert "bundle_loaded_counter" in response.text, ( - f"Metric bundle_loaded_counter should exist in {metrics_url}" -) -print("Metrics test successful!") + namespace = args["namespace"] + metrics_url = f"https://test-opa-server-default-metrics.{namespace}.svc.cluster.local:8443/metrics" + + # Use the CA certificate for verification + response = requests.get(metrics_url, verify="/tls/ca.crt") + + assert response.status_code == 200, "Metrics endpoint must return a 200 status code" + assert "bundle_loaded_counter" in response.text, ( + f"Metric bundle_loaded_counter should exist in {metrics_url}" + ) + print("Metrics test successful!") diff --git a/tests/templates/kuttl/smoke/30_test-regorule.py b/tests/templates/kuttl/smoke/30_test-regorule.py index d3caedc2..db98e7db 100755 --- a/tests/templates/kuttl/smoke/30_test-regorule.py +++ b/tests/templates/kuttl/smoke/30_test-regorule.py @@ -20,14 +20,14 @@ # false # } # --- - # We need to query: http://:/v1/data//()+ - # In our case http://:8081/v1/data/test + # We need to query: https://:/v1/data//()+ + # In our case https://:8443/v1/data/test # --> {'result': {'hello': True}} - # or http://:8081/v1/data/test/hello + # or https://:8443/v1/data/test/hello # --> {'hello': True} - # url = 'http://test-opa-svc:8081/v1/data/test' - response = requests.post(args["url"], json={"input": {}}).json() + # url = 'https://test-opa-server..svc.cluster.local:8443/v1/data/test' + response = requests.post(args["url"], json={"input": {}}, verify="/tls/ca.crt").json() if ( "result" in response diff --git a/tests/templates/kuttl/smoke/31-assert.yaml b/tests/templates/kuttl/smoke/31-assert.yaml index 0084e371..d9d524b7 100644 --- a/tests/templates/kuttl/smoke/31-assert.yaml +++ b/tests/templates/kuttl/smoke/31-assert.yaml @@ -4,4 +4,4 @@ kind: TestAssert metadata: name: test-metrics commands: - - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-metrics.py + - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-metrics.py -n $NAMESPACE From 4357730914f8e0e04e36ae4b477fb4467a9a87e6 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 29 Oct 2025 21:36:40 +0100 Subject: [PATCH 02/13] fix: python formatting --- tests/templates/kuttl/smoke/30_test-metrics.py | 4 +++- tests/templates/kuttl/smoke/30_test-regorule.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/smoke/30_test-metrics.py b/tests/templates/kuttl/smoke/30_test-metrics.py index dac209fd..f78812f6 100644 --- a/tests/templates/kuttl/smoke/30_test-metrics.py +++ b/tests/templates/kuttl/smoke/30_test-metrics.py @@ -5,7 +5,9 @@ if __name__ == "__main__": all_args = argparse.ArgumentParser() - all_args.add_argument("-n", "--namespace", required=True, help="Kubernetes namespace") + all_args.add_argument( + "-n", "--namespace", required=True, help="Kubernetes namespace" + ) args = vars(all_args.parse_args()) namespace = args["namespace"] diff --git a/tests/templates/kuttl/smoke/30_test-regorule.py b/tests/templates/kuttl/smoke/30_test-regorule.py index db98e7db..d0085462 100755 --- a/tests/templates/kuttl/smoke/30_test-regorule.py +++ b/tests/templates/kuttl/smoke/30_test-regorule.py @@ -27,7 +27,9 @@ # --> {'hello': True} # url = 'https://test-opa-server..svc.cluster.local:8443/v1/data/test' - response = requests.post(args["url"], json={"input": {}}, verify="/tls/ca.crt").json() + response = requests.post( + args["url"], json={"input": {}}, verify="/tls/ca.crt" + ).json() if ( "result" in response From 94789fe852e85c6a12edef5e0f353f0e16dc2b60 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 30 Oct 2025 12:12:55 +0100 Subject: [PATCH 03/13] fix: pre-commit lint fix --- tests/templates/kuttl/aas-user-info/test-regorule.py | 12 ++++++------ .../kuttl/keycloak-user-info/test-regorule.py | 12 ++++++------ tests/templates/kuttl/smoke/20-install-test-opa.yaml | 2 +- tests/templates/kuttl/smoke/30_test-metrics.py | 1 - 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/templates/kuttl/aas-user-info/test-regorule.py b/tests/templates/kuttl/aas-user-info/test-regorule.py index e95d6070..b05dba1d 100755 --- a/tests/templates/kuttl/aas-user-info/test-regorule.py +++ b/tests/templates/kuttl/aas-user-info/test-regorule.py @@ -26,9 +26,9 @@ def assertions( # todo: split out customAttribute assertions print(f"Testing for {username} with customAttributes {expected_attributes}") custom_attributes = result[opa_attribute]["customAttributes"] - assert ( - custom_attributes == expected_attributes - ), f"got {custom_attributes}, expected: {expected_attributes}" + assert custom_attributes == expected_attributes, ( + f"got {custom_attributes}, expected: {expected_attributes}" + ) if __name__ == "__main__": @@ -40,9 +40,9 @@ def assertions( def make_request(payload): response = requests.post(args["url"], data=json.dumps(payload), params=params) expected_status_code = 200 - assert ( - response.status_code == expected_status_code - ), f"got {response.status_code}, expected: {expected_status_code}" + assert response.status_code == expected_status_code, ( + f"got {response.status_code}, expected: {expected_status_code}" + ) return response.json() for subject_id in ["alice", "bob"]: diff --git a/tests/templates/kuttl/keycloak-user-info/test-regorule.py b/tests/templates/kuttl/keycloak-user-info/test-regorule.py index d8f3e20b..cac2b7fa 100755 --- a/tests/templates/kuttl/keycloak-user-info/test-regorule.py +++ b/tests/templates/kuttl/keycloak-user-info/test-regorule.py @@ -32,9 +32,9 @@ def assertions( # todo: split out customAttribute assertions print(f"Testing for {username} with customAttributes {expected_attributes}") custom_attributes = result[opa_attribute]["customAttributes"] - assert ( - custom_attributes == expected_attributes - ), f"got {custom_attributes}, expected: {expected_attributes}" + assert custom_attributes == expected_attributes, ( + f"got {custom_attributes}, expected: {expected_attributes}" + ) if __name__ == "__main__": @@ -46,9 +46,9 @@ def assertions( def make_request(payload): response = requests.post(args["url"], data=json.dumps(payload), params=params) expected_status_code = 200 - assert ( - response.status_code == expected_status_code - ), f"got {response.status_code}, expected: {expected_status_code}" + assert response.status_code == expected_status_code, ( + f"got {response.status_code}, expected: {expected_status_code}" + ) return response.json() for username, groups in users_and_groups.items(): diff --git a/tests/templates/kuttl/smoke/20-install-test-opa.yaml b/tests/templates/kuttl/smoke/20-install-test-opa.yaml index 2c44e739..262a1840 100644 --- a/tests/templates/kuttl/smoke/20-install-test-opa.yaml +++ b/tests/templates/kuttl/smoke/20-install-test-opa.yaml @@ -42,7 +42,7 @@ spec: spec: storageClassName: secrets.stackable.tech accessModes: - - ReadWriteOnce + - ReadWriteOnce resources: requests: storage: "1" diff --git a/tests/templates/kuttl/smoke/30_test-metrics.py b/tests/templates/kuttl/smoke/30_test-metrics.py index f78812f6..b6b05b1a 100644 --- a/tests/templates/kuttl/smoke/30_test-metrics.py +++ b/tests/templates/kuttl/smoke/30_test-metrics.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import requests import argparse -import os if __name__ == "__main__": all_args = argparse.ArgumentParser() From d5a906d5ec85dc5cab10fcfec12c4ee8f93055f4 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Fri, 31 Oct 2025 14:05:42 +0100 Subject: [PATCH 04/13] Update docs/modules/opa/pages/usage-guide/tls.adoc Co-authored-by: Malte Sander --- docs/modules/opa/pages/usage-guide/tls.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/opa/pages/usage-guide/tls.adoc b/docs/modules/opa/pages/usage-guide/tls.adoc index 63ce31ca..889c1bf8 100644 --- a/docs/modules/opa/pages/usage-guide/tls.adoc +++ b/docs/modules/opa/pages/usage-guide/tls.adoc @@ -1,7 +1,7 @@ = Enabling TLS Encryption :description: Learn how to enable TLS encryption for your OPA cluster to secure client connections. -TLS encryption for securing connections between clients and the OPA server can be configured in the `OpaCluster` resource. When TLS is enabled, OPA will serve requests over HTTPS instead of HTTP. +TLS encryption for securing client connections to the OPA server can be configured in the `OpaCluster` resource. When enabled, OPA serves requests over HTTPS instead of HTTP. == Overview From 4da8d2b3f75acb3b3e126b6d7b7b560959ad04a4 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Fri, 31 Oct 2025 14:05:51 +0100 Subject: [PATCH 05/13] Update docs/modules/opa/pages/usage-guide/tls.adoc Co-authored-by: Malte Sander --- docs/modules/opa/pages/usage-guide/tls.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/opa/pages/usage-guide/tls.adoc b/docs/modules/opa/pages/usage-guide/tls.adoc index 889c1bf8..7a733209 100644 --- a/docs/modules/opa/pages/usage-guide/tls.adoc +++ b/docs/modules/opa/pages/usage-guide/tls.adoc @@ -8,7 +8,7 @@ TLS encryption for securing client connections to the OPA server can be configur TLS encryption in OPA is disabled by default. To enable it, you need to: 1. Create a `SecretClass` that provides TLS certificates -2. Reference the `SecretClass` in your `OpaCluster` specification +2. Reference the `SecretClass` in your `OpaCluster` custom resource The operator integrates with the xref:secret-operator:index.adoc[Secret Operator] to automatically provision and mount TLS certificates to the OPA pods. From 176096f78c912110030365f52f7072bb1a16f342 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Fri, 31 Oct 2025 14:06:00 +0100 Subject: [PATCH 06/13] Update docs/modules/opa/pages/usage-guide/tls.adoc Co-authored-by: Malte Sander --- docs/modules/opa/pages/usage-guide/tls.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/opa/pages/usage-guide/tls.adoc b/docs/modules/opa/pages/usage-guide/tls.adoc index 7a733209..14dcaa88 100644 --- a/docs/modules/opa/pages/usage-guide/tls.adoc +++ b/docs/modules/opa/pages/usage-guide/tls.adoc @@ -10,7 +10,7 @@ TLS encryption in OPA is disabled by default. To enable it, you need to: 1. Create a `SecretClass` that provides TLS certificates 2. Reference the `SecretClass` in your `OpaCluster` custom resource -The operator integrates with the xref:secret-operator:index.adoc[Secret Operator] to automatically provision and mount TLS certificates to the OPA pods. +The operator integrates with the xref:secret-operator:index.adoc[Secret Operator] to automatically provision and mount TLS certificates into the OPA pods. == Configuration From c3b732bf8959a3f64ecd9ef4661d6632c4956614 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Fri, 31 Oct 2025 14:06:46 +0100 Subject: [PATCH 07/13] Update tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 Co-authored-by: Malte Sander --- tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 b/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 index 83f37a19..319fa439 100644 --- a/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 +++ b/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 @@ -8,7 +8,7 @@ commands: apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: opa-tls + name: opa-tls-$NAMESPACE spec: backend: autoTls: From 700af813b60fd156545437136c5a299ebcf64362 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Fri, 31 Oct 2025 14:06:54 +0100 Subject: [PATCH 08/13] Update tests/templates/kuttl/smoke/10-install-opa.yaml.j2 Co-authored-by: Malte Sander --- tests/templates/kuttl/smoke/10-install-opa.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 index fcbb48ec..3e8f6923 100644 --- a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 @@ -32,7 +32,7 @@ spec: pullPolicy: IfNotPresent clusterConfig: tls: - serverSecretClass: opa-tls + serverSecretClass: opa-tls-$NAMESPACE {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} From 289787247bba4cba7ba43ee24de54d53d3f92da7 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 31 Oct 2025 14:25:42 +0100 Subject: [PATCH 09/13] refactor: streamline TLS configuration checks and add tls_enabled method --- rust/operator-binary/src/controller.rs | 14 ++++++-------- rust/operator-binary/src/crd/mod.rs | 7 +++++++ rust/operator-binary/src/discovery.rs | 5 ++--- rust/operator-binary/src/service.rs | 8 ++++---- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index c455897e..b5a6e934 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -830,8 +830,6 @@ fn build_server_rolegroup_daemonset( &v1alpha1::Container::BundleBuilder, ); - let opa_tls_config = opa.spec.cluster_config.tls.as_ref(); - cb_opa .image_from_product_image(resolved_product_image) .command(vec![ @@ -844,7 +842,7 @@ fn build_server_rolegroup_daemonset( .args(vec![build_opa_start_command( merged_config, &opa_container_name, - opa_tls_config, + opa.spec.cluster_config.tls_enabled(), )]) .add_env_vars(env) .add_env_var( @@ -858,7 +856,7 @@ fn build_server_rolegroup_daemonset( // .spec.template.spec.containers[name="opa"].ports: duplicate entries for key [containerPort=8081,protocol="TCP"] // // So we don't do that - if opa_tls_config.is_some() { + if opa.spec.cluster_config.tls_enabled() { cb_opa.add_container_port(service::APP_TLS_PORT_NAME, service::APP_TLS_PORT.into()); cb_opa .add_volume_mount(TLS_VOLUME_NAME, TLS_STORE_DIR) @@ -874,7 +872,7 @@ fn build_server_rolegroup_daemonset( .context(AddVolumeMountSnafu)? .resources(merged_config.resources.to_owned().into()); - let (probe_port_name, probe_scheme) = if opa_tls_config.is_some() { + let (probe_port_name, probe_scheme) = if opa.spec.cluster_config.tls_enabled() { (service::APP_TLS_PORT_NAME, Some("HTTPS".to_string())) } else { (APP_PORT_NAME, Some("HTTP".to_string())) @@ -949,7 +947,7 @@ fn build_server_rolegroup_daemonset( .service_account_name(service_account.name_any()) .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - if let Some(tls) = opa_tls_config { + if let Some(tls) = &opa.spec.cluster_config.tls { pb.add_volume( VolumeBuilder::new(TLS_VOLUME_NAME) .ephemeral( @@ -1196,7 +1194,7 @@ fn build_config_file(merged_config: &v1alpha1::OpaConfig) -> String { fn build_opa_start_command( merged_config: &v1alpha1::OpaConfig, container_name: &str, - tls_config: Option<&v1alpha1::OpaTls>, + tls_enabled: bool, ) -> String { let mut file_log_level = DEFAULT_FILE_LOG_LEVEL; let mut console_log_level = DEFAULT_CONSOLE_LOG_LEVEL; @@ -1238,7 +1236,7 @@ fn build_opa_start_command( } } - let (bind_port, tls_flags) = if tls_config.is_some() { + let (bind_port, tls_flags) = if tls_enabled { ( service::APP_TLS_PORT, format!( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index a6b08d34..8b782a20 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -251,6 +251,13 @@ impl v1alpha1::CurrentlySupportedListenerClasses { } } +impl v1alpha1::OpaClusterConfig { + /// Returns whether TLS encryption is enabled for the OPA server. + pub fn tls_enabled(&self) -> bool { + self.tls.is_some() + } +} + impl v1alpha1::OpaConfig { fn default_config() -> v1alpha1::OpaConfigFragment { v1alpha1::OpaConfigFragment { diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 05fb3805..3870c03c 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -66,8 +66,7 @@ fn build_discovery_configmap( svc: &Service, cluster_info: &KubernetesClusterInfo, ) -> Result { - let tls_config = opa.spec.cluster_config.tls.as_ref(); - let (scheme, port) = if tls_config.is_some() { + let (scheme, port) = if opa.spec.cluster_config.tls_enabled() { ("https", APP_TLS_PORT) } else { ("http", APP_PORT) @@ -104,7 +103,7 @@ fn build_discovery_configmap( cm_builder.metadata(metadata).add_data("OPA", url); - if let Some(tls) = tls_config { + if let Some(tls) = opa.spec.cluster_config.tls.as_ref() { cm_builder.add_data("OPA_SECRET_CLASS", &tls.server_secret_class); } diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs index 40705bfd..72cdd6ef 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/service.rs @@ -59,7 +59,7 @@ pub(crate) fn build_server_role_service( let service_spec = ServiceSpec { type_: Some(opa.spec.cluster_config.listener_class.k8s_service_type()), - ports: Some(data_service_ports(opa.spec.cluster_config.tls.is_some())), + ports: Some(data_service_ports(opa.spec.cluster_config.tls_enabled())), selector: Some(service_selector_labels.into()), internal_traffic_policy: Some("Local".to_string()), ..ServiceSpec::default() @@ -104,7 +104,7 @@ pub(crate) fn build_rolegroup_headless_service( // options there are non-existent (mTLS still opens plain port) or suck (Kerberos). type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), - ports: Some(data_service_ports(opa.spec.cluster_config.tls.is_some())), + ports: Some(data_service_ports(opa.spec.cluster_config.tls_enabled())), selector: Some(role_group_selector_labels(opa, rolegroup)?.into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() @@ -138,7 +138,7 @@ pub(crate) fn build_rolegroup_metrics_service( .context(ObjectMetaSnafu)? .with_labels(prometheus_labels()) .with_annotations(prometheus_annotations( - opa.spec.cluster_config.tls.is_some(), + opa.spec.cluster_config.tls_enabled(), )) .build(); @@ -146,7 +146,7 @@ pub(crate) fn build_rolegroup_metrics_service( type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), ports: Some(vec![metrics_service_port( - opa.spec.cluster_config.tls.is_some(), + opa.spec.cluster_config.tls_enabled(), )]), selector: Some(role_group_selector_labels(opa, rolegroup)?.into()), ..ServiceSpec::default() From ee643949b648eacef08e4eb8ecd4feb737f0d242 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 31 Oct 2025 14:34:18 +0100 Subject: [PATCH 10/13] feat: test OPA using HTTP in smoke test as well --- tests/templates/kuttl/smoke/10-assert.yaml.j2 | 34 +++++++++++++++++-- .../kuttl/smoke/10-install-opa.yaml.j2 | 30 +++++++++++++++- tests/templates/kuttl/smoke/11-assert.yaml | 5 ++- tests/templates/kuttl/smoke/20-assert.yaml | 2 +- tests/templates/kuttl/smoke/30-assert.yaml | 3 +- .../templates/kuttl/smoke/30_test-metrics.py | 20 ++++++----- .../templates/kuttl/smoke/30_test-regorule.py | 18 ++++++---- tests/templates/kuttl/smoke/31-assert.yaml | 3 +- tests/templates/kuttl/smoke/32-assert.yaml | 6 ++-- 9 files changed, 98 insertions(+), 23 deletions(-) diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 6b86f13f..6c5b516a 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -2,12 +2,42 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available opaclusters.opa.stackable.tech/test-opa --timeout 301s + - script: kubectl -n $NAMESPACE wait --for=condition=available opaclusters.opa.stackable.tech/test-opa-https --timeout 301s + - script: kubectl -n $NAMESPACE wait --for=condition=available opaclusters.opa.stackable.tech/test-opa-http --timeout 301s --- apiVersion: apps/v1 kind: DaemonSet metadata: - name: test-opa-server-default + name: test-opa-https-server-default +spec: + template: + spec: + containers: + - name: opa + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 250m + memory: 256Mi + - name: bundle-builder + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + - name: vector +{% endif %} + terminationGracePeriodSeconds: 125 # 2 minutes + 5s safety buffer +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: test-opa-http-server-default spec: template: spec: diff --git a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 index 3e8f6923..4fcd5491 100644 --- a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 @@ -20,7 +20,7 @@ data: apiVersion: opa.stackable.tech/v1alpha1 kind: OpaCluster metadata: - name: test-opa + name: test-opa-https spec: image: {% if test_scenario['values']['opa'].find(",") > 0 %} @@ -46,3 +46,31 @@ spec: default: envOverrides: SERVER_ROLE_GROUP_LEVEL_ENV_VAR: "SERVER_ROLE_GROUP_LEVEL_ENV_VAR" +--- +apiVersion: opa.stackable.tech/v1alpha1 +kind: OpaCluster +metadata: + name: test-opa-http +spec: + image: +{% if test_scenario['values']['opa'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opa'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opa'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opa'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + envOverrides: + SERVER_ROLE_LEVEL_ENV_VAR: "SERVER_ROLE_LEVEL_ENV_VAR" + roleGroups: + default: + envOverrides: + SERVER_ROLE_GROUP_LEVEL_ENV_VAR: "SERVER_ROLE_GROUP_LEVEL_ENV_VAR" diff --git a/tests/templates/kuttl/smoke/11-assert.yaml b/tests/templates/kuttl/smoke/11-assert.yaml index 3136c288..55689bdc 100644 --- a/tests/templates/kuttl/smoke/11-assert.yaml +++ b/tests/templates/kuttl/smoke/11-assert.yaml @@ -5,5 +5,8 @@ kind: TestAssert timeout: 600 commands: - script: | - FIRST_OPA_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector app.kubernetes.io/instance=test-opa -o jsonpath='{.items[0].metadata.name}') + FIRST_OPA_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector app.kubernetes.io/instance=test-opa-https -o jsonpath='{.items[0].metadata.name}') + kubectl exec -n $NAMESPACE --container opa $FIRST_OPA_POD -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' + - script: | + FIRST_OPA_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector app.kubernetes.io/instance=test-opa-http -o jsonpath='{.items[0].metadata.name}') kubectl exec -n $NAMESPACE --container opa $FIRST_OPA_POD -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml index 579dfd86..6882e481 100644 --- a/tests/templates/kuttl/smoke/20-assert.yaml +++ b/tests/templates/kuttl/smoke/20-assert.yaml @@ -9,4 +9,4 @@ metadata: name: test-opa status: readyReplicas: 1 - replicas: 1 + replicas: 1 \ No newline at end of file diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml index 51b31ea3..9635c30a 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml +++ b/tests/templates/kuttl/smoke/30-assert.yaml @@ -4,4 +4,5 @@ kind: TestAssert metadata: name: test-regorule commands: - - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-regorule.py -u "https://test-opa-server.$NAMESPACE.svc.cluster.local:8443/v1/data/test" + - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-regorule.py -u "https://test-opa-https-server.$NAMESPACE.svc.cluster.local:8443/v1/data/test" + - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-regorule.py -u "http://test-opa-http-server.$NAMESPACE.svc.cluster.local:8081/v1/data/test" diff --git a/tests/templates/kuttl/smoke/30_test-metrics.py b/tests/templates/kuttl/smoke/30_test-metrics.py index b6b05b1a..d3f0f436 100644 --- a/tests/templates/kuttl/smoke/30_test-metrics.py +++ b/tests/templates/kuttl/smoke/30_test-metrics.py @@ -4,19 +4,23 @@ if __name__ == "__main__": all_args = argparse.ArgumentParser() - all_args.add_argument( - "-n", "--namespace", required=True, help="Kubernetes namespace" - ) + all_args.add_argument("-u", "--url", required=True, help="OPA metrics url") args = vars(all_args.parse_args()) - namespace = args["namespace"] - metrics_url = f"https://test-opa-server-default-metrics.{namespace}.svc.cluster.local:8443/metrics" + metrics_url = args["url"] + + # Determine verification setting based on whether TLS is used + if metrics_url.startswith("http://"): + verify = False + protocol = "HTTP" + else: + verify = "/tls/ca.crt" + protocol = "HTTPS" - # Use the CA certificate for verification - response = requests.get(metrics_url, verify="/tls/ca.crt") + response = requests.get(metrics_url, verify=verify) assert response.status_code == 200, "Metrics endpoint must return a 200 status code" assert "bundle_loaded_counter" in response.text, ( f"Metric bundle_loaded_counter should exist in {metrics_url}" ) - print("Metrics test successful!") + print(f"Metrics test ({protocol}) successful!") diff --git a/tests/templates/kuttl/smoke/30_test-regorule.py b/tests/templates/kuttl/smoke/30_test-regorule.py index d0085462..c8550596 100755 --- a/tests/templates/kuttl/smoke/30_test-regorule.py +++ b/tests/templates/kuttl/smoke/30_test-regorule.py @@ -25,22 +25,28 @@ # --> {'result': {'hello': True}} # or https://:8443/v1/data/test/hello # --> {'hello': True} + # For HTTP: http://:8081/v1/data/test - # url = 'https://test-opa-server..svc.cluster.local:8443/v1/data/test' - response = requests.post( - args["url"], json={"input": {}}, verify="/tls/ca.crt" - ).json() + # Determine verification setting based on whether TLS is used + if args["url"].startswith("http://"): + verify = False + protocol = "HTTP" + else: + verify = "/tls/ca.crt" + protocol = "HTTPS" + + response = requests.post(args["url"], json={"input": {}}, verify=verify).json() if ( "result" in response and "hello" in response["result"] and response["result"]["hello"] ): - print("Regorule test successful!") + print(f"Regorule test ({protocol}) successful!") exit(0) else: print( - "Error: received " + f"Error ({protocol}): received " + str(response) + " - expected: {'result': {'hello': True}}" ) diff --git a/tests/templates/kuttl/smoke/31-assert.yaml b/tests/templates/kuttl/smoke/31-assert.yaml index d9d524b7..4f707911 100644 --- a/tests/templates/kuttl/smoke/31-assert.yaml +++ b/tests/templates/kuttl/smoke/31-assert.yaml @@ -4,4 +4,5 @@ kind: TestAssert metadata: name: test-metrics commands: - - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-metrics.py -n $NAMESPACE + - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-metrics.py -u "https://test-opa-https-server-default-metrics.$NAMESPACE.svc.cluster.local:8443/metrics" + - script: kubectl exec -n $NAMESPACE test-opa-0 -- python /tmp/30_test-metrics.py -u "http://test-opa-http-server-default-metrics.$NAMESPACE.svc.cluster.local:8081/metrics" diff --git a/tests/templates/kuttl/smoke/32-assert.yaml b/tests/templates/kuttl/smoke/32-assert.yaml index f6044238..5a9be682 100644 --- a/tests/templates/kuttl/smoke/32-assert.yaml +++ b/tests/templates/kuttl/smoke/32-assert.yaml @@ -5,6 +5,8 @@ metadata: name: test-env-overrides commands: # Role level env var - - script: kubectl exec -n $NAMESPACE -c opa svc/test-opa-server -- env | grep SERVER_ROLE_LEVEL_ENV_VAR + - script: kubectl exec -n $NAMESPACE -c opa svc/test-opa-https-server -- env | grep SERVER_ROLE_LEVEL_ENV_VAR + - script: kubectl exec -n $NAMESPACE -c opa svc/test-opa-http-server -- env | grep SERVER_ROLE_LEVEL_ENV_VAR # RoleGroup level env var - - script: kubectl exec -n $NAMESPACE -c opa svc/test-opa-server -- env | grep SERVER_ROLE_GROUP_LEVEL_ENV_VAR + - script: kubectl exec -n $NAMESPACE -c opa svc/test-opa-https-server -- env | grep SERVER_ROLE_GROUP_LEVEL_ENV_VAR + - script: kubectl exec -n $NAMESPACE -c opa svc/test-opa-http-server -- env | grep SERVER_ROLE_GROUP_LEVEL_ENV_VAR From 689165e0e7357dadcf75281a54411510f06f8c73 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 3 Nov 2025 15:20:15 +0100 Subject: [PATCH 11/13] chore: newline at end of file --- tests/templates/kuttl/smoke/20-assert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml index 6882e481..579dfd86 100644 --- a/tests/templates/kuttl/smoke/20-assert.yaml +++ b/tests/templates/kuttl/smoke/20-assert.yaml @@ -9,4 +9,4 @@ metadata: name: test-opa status: readyReplicas: 1 - replicas: 1 \ No newline at end of file + replicas: 1 From 5e04a5c0e05b953b96453312d52f421f98835a5b Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 3 Nov 2025 17:03:08 +0100 Subject: [PATCH 12/13] feat: make smoke test use tls dimension --- .../smoke/09-install-secretclass.yaml.j2 | 3 + tests/templates/kuttl/smoke/10-assert.yaml.j2 | 34 +----- .../kuttl/smoke/10-install-opa.yaml.j2 | 115 ++++++++---------- tests/templates/kuttl/smoke/11-assert.yaml | 5 +- .../kuttl/smoke/20-install-test-opa.yaml | 48 -------- .../kuttl/smoke/20-install-test-opa.yaml.j2 | 56 +++++++++ tests/templates/kuttl/smoke/30-assert.yaml | 8 -- tests/templates/kuttl/smoke/30-assert.yaml.j2 | 11 ++ tests/templates/kuttl/smoke/31-assert.yaml | 8 -- tests/templates/kuttl/smoke/31-assert.yaml.j2 | 11 ++ tests/templates/kuttl/smoke/32-assert.yaml | 6 +- tests/test-definition.yaml | 5 + 12 files changed, 139 insertions(+), 171 deletions(-) delete mode 100644 tests/templates/kuttl/smoke/20-install-test-opa.yaml create mode 100644 tests/templates/kuttl/smoke/20-install-test-opa.yaml.j2 delete mode 100644 tests/templates/kuttl/smoke/30-assert.yaml create mode 100644 tests/templates/kuttl/smoke/30-assert.yaml.j2 delete mode 100644 tests/templates/kuttl/smoke/31-assert.yaml create mode 100644 tests/templates/kuttl/smoke/31-assert.yaml.j2 diff --git a/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 b/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 index 319fa439..6e23dddb 100644 --- a/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 +++ b/tests/templates/kuttl/smoke/09-install-secretclass.yaml.j2 @@ -1,3 +1,4 @@ +{% if test_scenario['values']['use-tls'] == "true" %} --- apiVersion: kuttl.dev/v1beta1 kind: TestStep @@ -17,3 +18,5 @@ commands: secret: name: opa-tls-ca namespace: $NAMESPACE + EOF +{% endif %} diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 6c5b516a..6b86f13f 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -2,42 +2,12 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 300 commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available opaclusters.opa.stackable.tech/test-opa-https --timeout 301s - - script: kubectl -n $NAMESPACE wait --for=condition=available opaclusters.opa.stackable.tech/test-opa-http --timeout 301s + - script: kubectl -n $NAMESPACE wait --for=condition=available opaclusters.opa.stackable.tech/test-opa --timeout 301s --- apiVersion: apps/v1 kind: DaemonSet metadata: - name: test-opa-https-server-default -spec: - template: - spec: - containers: - - name: opa - resources: - limits: - cpu: 500m - memory: 256Mi - requests: - cpu: 250m - memory: 256Mi - - name: bundle-builder - resources: - limits: - cpu: 200m - memory: 128Mi - requests: - cpu: 100m - memory: 128Mi -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - - name: vector -{% endif %} - terminationGracePeriodSeconds: 125 # 2 minutes + 5s safety buffer ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: test-opa-http-server-default + name: test-opa-server-default spec: template: spec: diff --git a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 index 4fcd5491..64b512d5 100644 --- a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 @@ -1,76 +1,57 @@ --- -apiVersion: v1 -kind: ConfigMap -metadata: - name: test - labels: - opa.stackable.tech/bundle: "true" -data: - test.rego: | - package test +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 %} - custom: "{{ test_scenario['values']['opa'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['opa'].split(',')[0] }}" + custom: "{{ test_scenario['values']['opa'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opa'].split(',')[0] }}" {% else %} - productVersion: "{{ test_scenario['values']['opa'] }}" -{% endif %} - pullPolicy: IfNotPresent - clusterConfig: - tls: - serverSecretClass: opa-tls-$NAMESPACE -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery + productVersion: "{{ test_scenario['values']['opa'] }}" {% endif %} - servers: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - envOverrides: - SERVER_ROLE_LEVEL_ENV_VAR: "SERVER_ROLE_LEVEL_ENV_VAR" - roleGroups: - default: - envOverrides: - SERVER_ROLE_GROUP_LEVEL_ENV_VAR: "SERVER_ROLE_GROUP_LEVEL_ENV_VAR" ---- -apiVersion: opa.stackable.tech/v1alpha1 -kind: OpaCluster -metadata: - name: test-opa-http -spec: - image: -{% if test_scenario['values']['opa'].find(",") > 0 %} - custom: "{{ test_scenario['values']['opa'].split(',')[1] }}" - productVersion: "{{ test_scenario['values']['opa'].split(',')[0] }}" -{% else %} - productVersion: "{{ test_scenario['values']['opa'] }}" + pullPolicy: IfNotPresent + clusterConfig: +{% if test_scenario['values']['use-tls'] == "true" %} + tls: + serverSecretClass: opa-tls-$NAMESPACE {% endif %} - pullPolicy: IfNotPresent - clusterConfig: {% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery + vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} - servers: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - envOverrides: - SERVER_ROLE_LEVEL_ENV_VAR: "SERVER_ROLE_LEVEL_ENV_VAR" - roleGroups: - default: - envOverrides: - SERVER_ROLE_GROUP_LEVEL_ENV_VAR: "SERVER_ROLE_GROUP_LEVEL_ENV_VAR" + servers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + envOverrides: + SERVER_ROLE_LEVEL_ENV_VAR: "SERVER_ROLE_LEVEL_ENV_VAR" + roleGroups: + default: + envOverrides: + SERVER_ROLE_GROUP_LEVEL_ENV_VAR: "SERVER_ROLE_GROUP_LEVEL_ENV_VAR" + EOF \ No newline at end of file diff --git a/tests/templates/kuttl/smoke/11-assert.yaml b/tests/templates/kuttl/smoke/11-assert.yaml index 55689bdc..3136c288 100644 --- a/tests/templates/kuttl/smoke/11-assert.yaml +++ b/tests/templates/kuttl/smoke/11-assert.yaml @@ -5,8 +5,5 @@ kind: TestAssert timeout: 600 commands: - script: | - FIRST_OPA_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector app.kubernetes.io/instance=test-opa-https -o jsonpath='{.items[0].metadata.name}') - kubectl exec -n $NAMESPACE --container opa $FIRST_OPA_POD -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' - - script: | - FIRST_OPA_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector app.kubernetes.io/instance=test-opa-http -o jsonpath='{.items[0].metadata.name}') + FIRST_OPA_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector app.kubernetes.io/instance=test-opa -o jsonpath='{.items[0].metadata.name}') kubectl exec -n $NAMESPACE --container opa $FIRST_OPA_POD -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' diff --git a/tests/templates/kuttl/smoke/20-install-test-opa.yaml b/tests/templates/kuttl/smoke/20-install-test-opa.yaml deleted file mode 100644 index 262a1840..00000000 --- a/tests/templates/kuttl/smoke/20-install-test-opa.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test-opa - labels: - app: test-opa -spec: - replicas: 1 - selector: - matchLabels: - app: test-opa - template: - metadata: - labels: - app: test-opa - spec: - securityContext: - fsGroup: 1000 - containers: - - name: test-opa - image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - stdin: true - tty: true - resources: - requests: - memory: "128Mi" - cpu: "512m" - limits: - memory: "128Mi" - cpu: "1" - volumeMounts: - - name: tls - mountPath: /tls - volumes: - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: opa-tls - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" diff --git a/tests/templates/kuttl/smoke/20-install-test-opa.yaml.j2 b/tests/templates/kuttl/smoke/20-install-test-opa.yaml.j2 new file mode 100644 index 00000000..0af56858 --- /dev/null +++ b/tests/templates/kuttl/smoke/20-install-test-opa.yaml.j2 @@ -0,0 +1,56 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < Date: Mon, 3 Nov 2025 17:05:52 +0100 Subject: [PATCH 13/13] chore: newline at end of file --- tests/templates/kuttl/smoke/10-install-opa.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 index 64b512d5..6a77339e 100644 --- a/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opa.yaml.j2 @@ -54,4 +54,4 @@ commands: default: envOverrides: SERVER_ROLE_GROUP_LEVEL_ENV_VAR: "SERVER_ROLE_GROUP_LEVEL_ENV_VAR" - EOF \ No newline at end of file + EOF