From ba1c7ab097c3c96c7946265c986d467894704c59 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 19 Dec 2024 09:31:29 +0100 Subject: [PATCH 01/71] First skeleton of opa integration --- deploy/helm/superset-operator/crds/crds.yaml | 19 ++++++++++++++++ rust/crd/src/lib.rs | 22 +++++++++++++++++++ rust/operator-binary/src/authorization/mod.rs | 1 + rust/operator-binary/src/authorization/opa.rs | 21 ++++++++++++++++++ rust/operator-binary/src/main.rs | 1 + .../src/superset_controller.rs | 14 ++++++++++++ 6 files changed, 78 insertions(+) create mode 100644 rust/operator-binary/src/authorization/mod.rs create mode 100644 rust/operator-binary/src/authorization/opa.rs diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index 6bf48ca6..e1353b6c 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -71,6 +71,25 @@ spec: - authenticationClass type: object type: array + authorization: + description: 'Authorziation options for Superset. Currently only role mapping is enabled. This means if a user logs in and Opa authorization is enabled user roles got synced from opa into superset roles. Roles get created automated. Warning: This will discard all roles managed by the superset administrator.' + nullable: true + properties: + opa: + description: Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) and the name of the Rego package containing your authorization rules. Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) to learn how to deploy Rego authorization rules with OPA. + nullable: true + properties: + configMapName: + description: The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for the OPA stacklet that should be used for authorization requests. + type: string + package: + description: The name of the Rego package containing the Rego rules for the product. + nullable: true + type: string + required: + - configMapName + type: object + type: object clusterOperation: default: reconciliationPaused: false diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 0cbf3284..cac29c7e 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -8,6 +8,7 @@ use stackable_operator::{ commons::{ affinity::StackableAffinity, cluster_operation::ClusterOperation, + opa::OpaConfig, product_image_selection::ProductImage, resources::{ CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment, @@ -175,6 +176,13 @@ pub struct SupersetClusterConfig { #[serde(default)] pub authentication: Vec, + /// Authorziation options for Superset. + /// Currently only role mapping is enabled. This means if a user logs in and Opa authorization is enabled + /// user roles got synced from opa into superset roles. Roles get created automated. + /// Warning: This will discard all roles managed by the superset administrator. + #[serde(skip_serializing_if = "Option::is_none")] + pub authorization: Option, + /// The name of the Secret object containing the admin user credentials and database connection details. /// Read the /// [getting started guide first steps](DOCS_BASE_URL_PLACEHOLDER/superset/getting_started/first_steps) @@ -239,6 +247,12 @@ impl CurrentlySupportedListenerClasses { } } +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SupersetAuthorization { + pub opa: Option, +} + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct SupersetCredentials { @@ -472,6 +486,14 @@ impl SupersetCluster { } } + pub fn get_opa_config(&self) -> Option<&OpaConfig> { + self.spec + .cluster_config + .authorization + .as_ref() + .and_then(|a| a.opa.as_ref()) + } + /// Retrieve and merge resource configs for role and role groups pub fn merged_config( &self, diff --git a/rust/operator-binary/src/authorization/mod.rs b/rust/operator-binary/src/authorization/mod.rs new file mode 100644 index 00000000..932ba472 --- /dev/null +++ b/rust/operator-binary/src/authorization/mod.rs @@ -0,0 +1 @@ +pub mod opa; diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs new file mode 100644 index 00000000..689d37fe --- /dev/null +++ b/rust/operator-binary/src/authorization/opa.rs @@ -0,0 +1,21 @@ +use stackable_operator::{ + client::Client, + commons::opa::{OpaApiVersion, OpaConfig}, +}; +use stackable_superset_crd::SupersetCluster; + +pub struct SupersetOpaConfig { + opa_role_mapping: bool, +} + +impl SupersetOpaConfig { + pub async fn from_opa_config( + client: &Client, + superset: &SupersetCluster, + opa_config: &OpaConfig, + ) -> Result { + Ok(SupersetOpaConfig { + opa_role_mapping: true, + }) + } +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 88afd730..267f737d 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -23,6 +23,7 @@ use stackable_superset_crd::{druidconnection::DruidConnection, SupersetCluster, use crate::druid_connection_controller::DRUID_CONNECTION_CONTROLLER_NAME; use crate::superset_controller::SUPERSET_CONTROLLER_NAME; +mod authorization; mod commands; mod config; mod controller_commons; diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index cd02211b..7b565c96 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -75,6 +75,7 @@ use stackable_superset_crd::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ + authorization::opa::SupersetOpaConfig, commands::add_cert_to_python_certifi_command, config::{self, PYTHON_IMPORTS}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, @@ -283,6 +284,10 @@ pub enum Error { InvalidSupersetCluster { source: error_boundary::InvalidObject, }, + #[snafu(display("invalid OpaConfig"))] + InvalidOpaConfig { + source: stackable_operator::commons::opa::Error, + }, } type Result = std::result::Result; @@ -371,6 +376,15 @@ pub async fn reconcile_superset( ) .context(CreateClusterResourcesSnafu)?; + let superset_opa_config = match superset.get_opa_config() { + Some(opa_config) => Some( + SupersetOpaConfig::from_opa_config(client, superset, opa_config) + .await + .context(InvalidOpaConfigSnafu)?, + ), + None => None, + }; + let (rbac_sa, rbac_rolebinding) = build_rbac_resources( superset, APP_NAME, From d502af368d05715df47eae5683aefdfe69bc6cc9 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Dec 2024 13:14:59 +0100 Subject: [PATCH 02/71] WIP implementation opa-role-mapping --- rust/crd/src/lib.rs | 7 +++ rust/operator-binary/src/authorization/opa.rs | 43 ++++++++++++++++++- rust/operator-binary/src/config.rs | 4 ++ .../src/superset_controller.rs | 20 +++++++-- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index cac29c7e..12990317 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -86,6 +86,9 @@ pub enum SupersetConfigOptions { AuthLdapTlsCertfile, AuthLdapTlsKeyfile, AuthLdapTlsCacertfile, + CustomSecurityManager, + StackableOpaEndpoint, + OpaPackageName, } impl SupersetConfigOptions { @@ -133,6 +136,10 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::AuthLdapTlsCertfile => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapTlsKeyfile => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapTlsCacertfile => PythonType::StringLiteral, + SupersetConfigOptions::CustomSecurityManager => PythonType::StringLiteral, + SupersetConfigOptions::StackableOpaEndpoint => PythonType::StringLiteral, + SupersetConfigOptions::OpaPackageName => PythonType::StringLiteral, + // TODO: Set new options for OpaSecurityManager like: } } } diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 689d37fe..8b858963 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use stackable_operator::{ client::Client, commons::opa::{OpaApiVersion, OpaConfig}, @@ -5,7 +7,8 @@ use stackable_operator::{ use stackable_superset_crd::SupersetCluster; pub struct SupersetOpaConfig { - opa_role_mapping: bool, + opa_endpoint: String, + opa_package: Option, } impl SupersetOpaConfig { @@ -14,8 +17,44 @@ impl SupersetOpaConfig { superset: &SupersetCluster, opa_config: &OpaConfig, ) -> Result { + // Get opa_base_url for later use in CustomOpaSecurityManager + let opa_endpoint = opa_config + .full_document_url_from_config_map(client, superset, None, OpaApiVersion::V1) + .await?; + + let opa_package = opa_config.package.clone(); Ok(SupersetOpaConfig { - opa_role_mapping: true, + opa_endpoint, + opa_package, }) } + + // Adding necessary configurations. Imports are solved in config.rs + pub fn as_config(&self) -> BTreeMap> { + let config = BTreeMap::from([ + ( + "CUSTOM_SECURITY_MANAGER".to_string(), + Some("OpaSupersetSecurityManager".to_string()), + ), + ( + "AUTH_USER_REGISTRATION_ROLE".to_string(), + Some("os.getenv('AUTH_USER_REGISTRATION_ROLE', 'Public')".to_string()), + ), + // TODO: Figure out how to tell a what are the + // rule names used. + ( + "STACKABLE_OPA_RULE".to_string(), + Some("os.getenv('STACKABLE_OPA_RULE', 'user_roles')".to_string()), + ), + ( + "STACKABLE_OPA_ENDPOINT".to_string(), + Some(self.opa_endpoint.clone()), + ), + ( + "STACKABLE_OPA_PACKAGE".to_string(), + self.opa_package.clone(), + ), + ]); + config + } } diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index ee01a5f3..9d0481c0 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -35,6 +35,10 @@ pub const PYTHON_IMPORTS: &[&str] = &[ "from log_config import StackableLoggingConfigurator", ]; +// TODO: Eihter use or remove this +pub const OPA_IMPORTS: &[&str] = + &["from superset.security.manager import OpaSupersetSecurityManager"]; + pub fn add_superset_config( config: &mut BTreeMap, authentication_config: &SupersetClientAuthenticationDetailsResolved, diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 7b565c96..7f8d588e 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -77,7 +77,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ authorization::opa::SupersetOpaConfig, commands::add_cert_to_python_certifi_command, - config::{self, PYTHON_IMPORTS}, + config::{self, OPA_IMPORTS, PYTHON_IMPORTS}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{ @@ -426,6 +426,7 @@ pub async fn reconcile_superset( &rolegroup, rolegroup_config, &auth_config, + &superset_opa_config, &config.logging, vector_aggregator_address.as_deref(), )?; @@ -553,11 +554,12 @@ fn build_rolegroup_config_map( rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, authentication_config: &SupersetClientAuthenticationDetailsResolved, + superset_opa_config: &Option, logging: &Logging, vector_aggregator_address: Option<&str>, ) -> Result { let mut config_properties = BTreeMap::new(); - + let imports = PYTHON_IMPORTS; // TODO: this is true per default for versions 3.0.0 and up. // We deactivate it here to keep existing functionality. // However this is a security issue and should be configured properly @@ -567,6 +569,18 @@ fn build_rolegroup_config_map( config::add_superset_config(&mut config_properties, authentication_config) .context(AddSupersetConfigSnafu)?; + // Adding opa configuration properties to config_properties. + // This will be injected as key/value pair in superset_config.py + if let Some(opa_config) = superset_opa_config { + for (k, v) in opa_config.as_config() { + config_properties.insert(k, v.unwrap_or_default()); + } + // If opa role mapping is configured, insert CustomOpaSecurityManager import + for opa_import in OPA_IMPORTS { + imports.to_vec().push(&opa_import); + } + } + // The order here should be kept in order to preserve overrides. // No properties should be added after this extend. config_properties.extend( @@ -590,7 +604,7 @@ fn build_rolegroup_config_map( flask_app_config_writer::write::( &mut config_file, config_properties.iter(), - PYTHON_IMPORTS, + imports, ) .with_context(|_| BuildRoleGroupConfigFileSnafu { rolegroup: rolegroup.clone(), From b3c3d0f39752bb947642c614c50610edcff06c2f Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Dec 2024 14:53:07 +0100 Subject: [PATCH 03/71] Add SecurityManager dynamically --- rust/crd/src/lib.rs | 4 ++-- rust/operator-binary/src/config.rs | 1 - rust/operator-binary/src/superset_controller.rs | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 12990317..ca9b7d5c 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -88,7 +88,7 @@ pub enum SupersetConfigOptions { AuthLdapTlsCacertfile, CustomSecurityManager, StackableOpaEndpoint, - OpaPackageName, + StackableOpaPackage, } impl SupersetConfigOptions { @@ -138,7 +138,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::AuthLdapTlsCacertfile => PythonType::StringLiteral, SupersetConfigOptions::CustomSecurityManager => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaEndpoint => PythonType::StringLiteral, - SupersetConfigOptions::OpaPackageName => PythonType::StringLiteral, + SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, // TODO: Set new options for OpaSecurityManager like: } } diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 9d0481c0..912302a1 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -35,7 +35,6 @@ pub const PYTHON_IMPORTS: &[&str] = &[ "from log_config import StackableLoggingConfigurator", ]; -// TODO: Eihter use or remove this pub const OPA_IMPORTS: &[&str] = &["from superset.security.manager import OpaSupersetSecurityManager"]; diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 7f8d588e..01d3f234 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -559,7 +559,7 @@ fn build_rolegroup_config_map( vector_aggregator_address: Option<&str>, ) -> Result { let mut config_properties = BTreeMap::new(); - let imports = PYTHON_IMPORTS; + let mut imports = PYTHON_IMPORTS.to_vec(); // TODO: this is true per default for versions 3.0.0 and up. // We deactivate it here to keep existing functionality. // However this is a security issue and should be configured properly @@ -577,7 +577,7 @@ fn build_rolegroup_config_map( } // If opa role mapping is configured, insert CustomOpaSecurityManager import for opa_import in OPA_IMPORTS { - imports.to_vec().push(&opa_import); + imports.push(&opa_import); } } @@ -604,7 +604,7 @@ fn build_rolegroup_config_map( flask_app_config_writer::write::( &mut config_file, config_properties.iter(), - imports, + &imports, ) .with_context(|_| BuildRoleGroupConfigFileSnafu { rolegroup: rolegroup.clone(), From 25f143a2708821776e4502a4e46a3ab82e496b09 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Dec 2024 15:34:10 +0100 Subject: [PATCH 04/71] Better OPA_IMPORT --- rust/operator-binary/src/superset_controller.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 01d3f234..125fcdc9 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -576,9 +576,7 @@ fn build_rolegroup_config_map( config_properties.insert(k, v.unwrap_or_default()); } // If opa role mapping is configured, insert CustomOpaSecurityManager import - for opa_import in OPA_IMPORTS { - imports.push(&opa_import); - } + imports.append(&mut (*OPA_IMPORTS).to_vec()) } // The order here should be kept in order to preserve overrides. From a125456c362a7405027f97d479209faa9eb939a6 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Dec 2024 16:48:07 +0100 Subject: [PATCH 05/71] Security manager own file in Docker-Images. Fixing python expression --- rust/crd/src/lib.rs | 2 +- rust/operator-binary/src/authorization/opa.rs | 4 ++++ rust/operator-binary/src/config.rs | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index ca9b7d5c..2eec6910 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -136,7 +136,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::AuthLdapTlsCertfile => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapTlsKeyfile => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapTlsCacertfile => PythonType::StringLiteral, - SupersetConfigOptions::CustomSecurityManager => PythonType::StringLiteral, + SupersetConfigOptions::CustomSecurityManager => PythonType::Expression, SupersetConfigOptions::StackableOpaEndpoint => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, // TODO: Set new options for OpaSecurityManager like: diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 8b858963..68d4f3f9 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -30,12 +30,16 @@ impl SupersetOpaConfig { } // Adding necessary configurations. Imports are solved in config.rs + // TODO: Currently: .unwrap_or_default() which ends in e.g. : + // CUSTOM_SECURITY_MANAGER = None => CUSTOM_SECURITY_MANAGER = "" + // Could be better if not set. pub fn as_config(&self) -> BTreeMap> { let config = BTreeMap::from([ ( "CUSTOM_SECURITY_MANAGER".to_string(), Some("OpaSupersetSecurityManager".to_string()), ), + // TODO: Make this more smart. ( "AUTH_USER_REGISTRATION_ROLE".to_string(), Some("os.getenv('AUTH_USER_REGISTRATION_ROLE', 'Public')".to_string()), diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 912302a1..b1d10558 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -36,7 +36,7 @@ pub const PYTHON_IMPORTS: &[&str] = &[ ]; pub const OPA_IMPORTS: &[&str] = - &["from superset.security.manager import OpaSupersetSecurityManager"]; + &["from superset.security.OpaSupersetSecurityManager import OpaSupersetSecurityManager"]; pub fn add_superset_config( config: &mut BTreeMap, From 1e170093b5d3a15e9962d4f41d0b9122c3ca9c60 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 23 Dec 2024 16:49:25 +0100 Subject: [PATCH 06/71] making clippy happy for now --- rust/operator-binary/src/superset_controller.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 125fcdc9..ae42ba69 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -548,6 +548,7 @@ fn build_node_role_service( } /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +#[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( superset: &SupersetCluster, resolved_product_image: &ResolvedProductImage, From e237a7a4e19e9061b13d2d0963e87e812ff0b923 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 27 Dec 2024 13:38:31 +0100 Subject: [PATCH 07/71] Updating some approaches --- deploy/helm/superset-operator/Chart.yaml | 4 ++-- rust/crd/src/lib.rs | 3 ++- rust/operator-binary/src/authorization/opa.rs | 12 ++++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/deploy/helm/superset-operator/Chart.yaml b/deploy/helm/superset-operator/Chart.yaml index a1fe9bd7..4589dfdc 100644 --- a/deploy/helm/superset-operator/Chart.yaml +++ b/deploy/helm/superset-operator/Chart.yaml @@ -1,8 +1,8 @@ --- apiVersion: v2 name: superset-operator -version: "0.0.0-dev" -appVersion: "0.0.0-dev" +version: "" +appVersion: "" description: The Stackable Operator for Apache Superset home: https://github.com/stackabletech/superset-operator maintainers: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 2eec6910..1bbae050 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -119,7 +119,8 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::LoggingConfigurator => PythonType::Expression, SupersetConfigOptions::AuthType => PythonType::Expression, SupersetConfigOptions::AuthUserRegistration => PythonType::BoolLiteral, - SupersetConfigOptions::AuthUserRegistrationRole => PythonType::StringLiteral, + // Going to be an expression as we default it from env, if and only if opa is used + SupersetConfigOptions::AuthUserRegistrationRole => PythonType::Expression, SupersetConfigOptions::AuthRolesSyncAtLogin => PythonType::BoolLiteral, SupersetConfigOptions::AuthLdapServer => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapBindUser => PythonType::Expression, diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 68d4f3f9..81a70aab 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -20,9 +20,15 @@ impl SupersetOpaConfig { // Get opa_base_url for later use in CustomOpaSecurityManager let opa_endpoint = opa_config .full_document_url_from_config_map(client, superset, None, OpaApiVersion::V1) - .await?; + .await? + // Not pretty. + // Need to remove the resource name. Appended by default. + // TODO: Decide where to handle this + // could be better in security manager! + .replace("/v1/data/superset", ""); let opa_package = opa_config.package.clone(); + Ok(SupersetOpaConfig { opa_endpoint, opa_package, @@ -39,7 +45,9 @@ impl SupersetOpaConfig { "CUSTOM_SECURITY_MANAGER".to_string(), Some("OpaSupersetSecurityManager".to_string()), ), - // TODO: Make this more smart. + // This is now a PythonType::Expression. Makes it easy to find a default. + // only necessary when opa role mapping is activated, as the user + // has to have a role to be valid. ( "AUTH_USER_REGISTRATION_ROLE".to_string(), Some("os.getenv('AUTH_USER_REGISTRATION_ROLE', 'Public')".to_string()), From 8614577c9c2cc6d45541cb39598687d68fac89c7 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 27 Dec 2024 17:26:42 +0100 Subject: [PATCH 08/71] Adding more rules, more sophisticated handling of stuff --- rust/crd/src/lib.rs | 2 ++ rust/operator-binary/src/authorization/opa.rs | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 1bbae050..e6e20502 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -89,6 +89,7 @@ pub enum SupersetConfigOptions { CustomSecurityManager, StackableOpaEndpoint, StackableOpaPackage, + StackableOpaRule, } impl SupersetConfigOptions { @@ -140,6 +141,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::CustomSecurityManager => PythonType::Expression, SupersetConfigOptions::StackableOpaEndpoint => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, + SupersetConfigOptions::StackableOpaRule => PythonType::Expression, // TODO: Set new options for OpaSecurityManager like: } } diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 81a70aab..4b58a55a 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -52,11 +52,11 @@ impl SupersetOpaConfig { "AUTH_USER_REGISTRATION_ROLE".to_string(), Some("os.getenv('AUTH_USER_REGISTRATION_ROLE', 'Public')".to_string()), ), - // TODO: Figure out how to tell a what are the - // rule names used. + // There is no proper way to interfere this without changing e.g. CRD's. + // Thus, we go for an default and make it accessible through envoverrides. ( "STACKABLE_OPA_RULE".to_string(), - Some("os.getenv('STACKABLE_OPA_RULE', 'user_roles')".to_string()), + Some("os.getenv('OPA_RULE', 'user_roles')".to_string()), ), ( "STACKABLE_OPA_ENDPOINT".to_string(), From a4dda2a185220039c439605d9b32d545f5171de5 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 30 Dec 2024 11:29:38 +0100 Subject: [PATCH 09/71] Defaults are working --- rust/operator-binary/src/authorization/opa.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 4b58a55a..eb3e031e 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -56,7 +56,7 @@ impl SupersetOpaConfig { // Thus, we go for an default and make it accessible through envoverrides. ( "STACKABLE_OPA_RULE".to_string(), - Some("os.getenv('OPA_RULE', 'user_roles')".to_string()), + Some("os.getenv('STACKABLE_OPA_RULE', 'user_roles')".to_string()), ), ( "STACKABLE_OPA_ENDPOINT".to_string(), From 50050bced3a77f1980359dd276ea25cafdd0c61f Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 2 Jan 2025 12:19:11 +0100 Subject: [PATCH 10/71] Better interfering of package path --- rust/crd/src/lib.rs | 4 +-- rust/operator-binary/src/authorization/opa.rs | 28 ++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index e6e20502..32984961 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -87,7 +87,7 @@ pub enum SupersetConfigOptions { AuthLdapTlsKeyfile, AuthLdapTlsCacertfile, CustomSecurityManager, - StackableOpaEndpoint, + StackableOpaBaseUrl, StackableOpaPackage, StackableOpaRule, } @@ -139,7 +139,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::AuthLdapTlsKeyfile => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapTlsCacertfile => PythonType::StringLiteral, SupersetConfigOptions::CustomSecurityManager => PythonType::Expression, - SupersetConfigOptions::StackableOpaEndpoint => PythonType::StringLiteral, + SupersetConfigOptions::StackableOpaBaseUrl => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaRule => PythonType::Expression, // TODO: Set new options for OpaSecurityManager like: diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index eb3e031e..ba943f32 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -7,7 +7,7 @@ use stackable_operator::{ use stackable_superset_crd::SupersetCluster; pub struct SupersetOpaConfig { - opa_endpoint: String, + opa_base_url: String, opa_package: Option, } @@ -20,18 +20,20 @@ impl SupersetOpaConfig { // Get opa_base_url for later use in CustomOpaSecurityManager let opa_endpoint = opa_config .full_document_url_from_config_map(client, superset, None, OpaApiVersion::V1) - .await? - // Not pretty. - // Need to remove the resource name. Appended by default. - // TODO: Decide where to handle this - // could be better in security manager! - .replace("/v1/data/superset", ""); + .await?; - let opa_package = opa_config.package.clone(); + // striping package path from base url. Needed by CustomOpaSecurityManager. TODO: + let opa_base_url = match opa_config.package.clone() { + Some(opa_package_name) => { + let opa_path = format!("/v1/data/{opa_package_name}"); + opa_endpoint.replace(&opa_path, "") + } + None => opa_endpoint.replace("/v1/data/", ""), + }; Ok(SupersetOpaConfig { - opa_endpoint, - opa_package, + opa_base_url, + opa_package: opa_config.package.clone(), }) } @@ -53,14 +55,14 @@ impl SupersetOpaConfig { Some("os.getenv('AUTH_USER_REGISTRATION_ROLE', 'Public')".to_string()), ), // There is no proper way to interfere this without changing e.g. CRD's. - // Thus, we go for an default and make it accessible through envoverrides. + // Thus, we go for an default and make it accessible through envOverrides. ( "STACKABLE_OPA_RULE".to_string(), Some("os.getenv('STACKABLE_OPA_RULE', 'user_roles')".to_string()), ), ( - "STACKABLE_OPA_ENDPOINT".to_string(), - Some(self.opa_endpoint.clone()), + "STACKABLE_OPA_BASE_URL".to_string(), + Some(self.opa_base_url.clone()), ), ( "STACKABLE_OPA_PACKAGE".to_string(), From cc358027f59f46299ef2ada4e2bdf0da8387b3fa Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 2 Jan 2025 12:26:55 +0100 Subject: [PATCH 11/71] Happy Clippy --- rust/operator-binary/src/authorization/opa.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index ba943f32..0cae9d87 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -42,7 +42,7 @@ impl SupersetOpaConfig { // CUSTOM_SECURITY_MANAGER = None => CUSTOM_SECURITY_MANAGER = "" // Could be better if not set. pub fn as_config(&self) -> BTreeMap> { - let config = BTreeMap::from([ + BTreeMap::from([ ( "CUSTOM_SECURITY_MANAGER".to_string(), Some("OpaSupersetSecurityManager".to_string()), @@ -56,6 +56,7 @@ impl SupersetOpaConfig { ), // There is no proper way to interfere this without changing e.g. CRD's. // Thus, we go for an default and make it accessible through envOverrides. + // TODO: Documentation ( "STACKABLE_OPA_RULE".to_string(), Some("os.getenv('STACKABLE_OPA_RULE', 'user_roles')".to_string()), @@ -68,7 +69,6 @@ impl SupersetOpaConfig { "STACKABLE_OPA_PACKAGE".to_string(), self.opa_package.clone(), ), - ]); - config + ]) } } From 9ae3dca349f94067e27fb92d815f5445058f9c7f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 6 Jan 2025 16:33:07 +0100 Subject: [PATCH 12/71] update OpaSupersetSecurityManager import path --- rust/operator-binary/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index b1d10558..c11a9a6f 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -36,7 +36,7 @@ pub const PYTHON_IMPORTS: &[&str] = &[ ]; pub const OPA_IMPORTS: &[&str] = - &["from superset.security.OpaSupersetSecurityManager import OpaSupersetSecurityManager"]; + &["from superset.security.opa_manager import OpaSupersetSecurityManager"]; pub fn add_superset_config( config: &mut BTreeMap, From 40c5cd154a3a0c3c99f6f0b38556ed2078b03c17 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 10 Jan 2025 15:30:33 +0100 Subject: [PATCH 13/71] import new opa_authorizer module --- rust/operator-binary/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index c11a9a6f..540fbb7a 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -36,7 +36,7 @@ pub const PYTHON_IMPORTS: &[&str] = &[ ]; pub const OPA_IMPORTS: &[&str] = - &["from superset.security.opa_manager import OpaSupersetSecurityManager"]; + &["from opa_authorizer.opa_manager import OpaSupersetSecurityManager"]; pub fn add_superset_config( config: &mut BTreeMap, From a92dac5d572e6ac284385a84275dbedadf9af549 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 13 Jan 2025 14:25:43 +0100 Subject: [PATCH 14/71] Removing some ToDo's. Better comments --- rust/crd/src/lib.rs | 2 +- rust/operator-binary/src/authorization/opa.rs | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 32984961..20006b3a 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -138,11 +138,11 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::AuthLdapTlsCertfile => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapTlsKeyfile => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapTlsCacertfile => PythonType::StringLiteral, + // Configuration options used by CustomOpaSecurityManager SupersetConfigOptions::CustomSecurityManager => PythonType::Expression, SupersetConfigOptions::StackableOpaBaseUrl => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaRule => PythonType::Expression, - // TODO: Set new options for OpaSecurityManager like: } } } diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 0cae9d87..d000d0af 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -22,7 +22,7 @@ impl SupersetOpaConfig { .full_document_url_from_config_map(client, superset, None, OpaApiVersion::V1) .await?; - // striping package path from base url. Needed by CustomOpaSecurityManager. TODO: + // striping package path from base url. Needed by CustomOpaSecurityManager. let opa_base_url = match opa_config.package.clone() { Some(opa_package_name) => { let opa_path = format!("/v1/data/{opa_package_name}"); @@ -38,9 +38,6 @@ impl SupersetOpaConfig { } // Adding necessary configurations. Imports are solved in config.rs - // TODO: Currently: .unwrap_or_default() which ends in e.g. : - // CUSTOM_SECURITY_MANAGER = None => CUSTOM_SECURITY_MANAGER = "" - // Could be better if not set. pub fn as_config(&self) -> BTreeMap> { BTreeMap::from([ ( @@ -48,14 +45,13 @@ impl SupersetOpaConfig { Some("OpaSupersetSecurityManager".to_string()), ), // This is now a PythonType::Expression. Makes it easy to find a default. - // only necessary when opa role mapping is activated, as the user - // has to have a role to be valid. + // EnvOverrides are supported. ( "AUTH_USER_REGISTRATION_ROLE".to_string(), Some("os.getenv('AUTH_USER_REGISTRATION_ROLE', 'Public')".to_string()), ), // There is no proper way to interfere this without changing e.g. CRD's. - // Thus, we go for an default and make it accessible through envOverrides. + // EnvOverrides are supported. // TODO: Documentation ( "STACKABLE_OPA_RULE".to_string(), From a5c22ec818c42c7136244de5f4fade0723868c43 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 13 Jan 2025 14:41:46 +0100 Subject: [PATCH 15/71] Adding OpaRolesCache with 10 minutes default --- rust/crd/src/lib.rs | 2 ++ rust/operator-binary/src/authorization/opa.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 20006b3a..3268f595 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -90,6 +90,7 @@ pub enum SupersetConfigOptions { StackableOpaBaseUrl, StackableOpaPackage, StackableOpaRule, + OpaRolesCache, } impl SupersetConfigOptions { @@ -143,6 +144,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::StackableOpaBaseUrl => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaRule => PythonType::Expression, + SupersetConfigOptions::OpaRolesCache => PythonType::Expression, } } } diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index d000d0af..822d084f 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -65,6 +65,10 @@ impl SupersetOpaConfig { "STACKABLE_OPA_PACKAGE".to_string(), self.opa_package.clone(), ), + ( + "OPA_ROLES_CACHE".to_string(), + Some("os.getenv('OPA_ROLES_CACHE', '10')".to_string()), + ), ]) } } From 535e2ea80ade8de891759f4c61d01e83c0d2ef7c Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 13 Jan 2025 15:20:59 +0100 Subject: [PATCH 16/71] pre commit becomes happy --- deploy/helm/superset-operator/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/helm/superset-operator/Chart.yaml b/deploy/helm/superset-operator/Chart.yaml index 4589dfdc..a1fe9bd7 100644 --- a/deploy/helm/superset-operator/Chart.yaml +++ b/deploy/helm/superset-operator/Chart.yaml @@ -1,8 +1,8 @@ --- apiVersion: v2 name: superset-operator -version: "" -appVersion: "" +version: "0.0.0-dev" +appVersion: "0.0.0-dev" description: The Stackable Operator for Apache Superset home: https://github.com/stackabletech/superset-operator maintainers: From c2152505b33d7c6115d5e80255c47d525a70a930 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 14 Jan 2025 09:58:11 +0100 Subject: [PATCH 17/71] create opa test basics --- .../kuttl/oidc/10-install-postgresql.yaml | 2 +- ...10_helm-bitnami-postgresql-values.yaml.j2} | 0 ...install-keycloak.yaml => 30-keycloak.yaml} | 4 +- ...ak.yaml.j2 => 30_install-keycloak.yaml.j2} | 0 ...et.yaml.j2 => 40_install-superset.yaml.j2} | 0 .../kuttl/opa-authorization/10-assert.yaml | 14 ++++++ .../10-install-postgresql.yaml | 12 ++++++ .../kuttl/opa-authorization/20-assert.yaml | 14 ++++++ .../20-install-keycloak.yaml | 15 +++++++ .../kuttl/opa-authorization/30-assert.yaml | 6 +++ .../opa-authorization/30-install-opa.yaml.j2 | 43 +++++++++++++++++++ .../kuttl/opa-authorization/31-assert.yaml | 6 +++ .../kuttl/opa-authorization/31-opa-rego.yaml | 25 +++++++++++ .../kuttl/opa-authorization/40-assert.yaml | 16 +++++++ .../40-install-superset.yaml | 8 ++++ .../40_install-superset.yaml.j2 | 40 +++++++++++++++++ tests/test-definition.yaml | 3 ++ 17 files changed, 205 insertions(+), 3 deletions(-) rename tests/templates/kuttl/oidc/{helm-bitnami-postgresql-values.yaml.j2 => 10_helm-bitnami-postgresql-values.yaml.j2} (100%) rename tests/templates/kuttl/oidc/{30-install-keycloak.yaml => 30-keycloak.yaml} (79%) rename tests/templates/kuttl/oidc/{install-keycloak.yaml.j2 => 30_install-keycloak.yaml.j2} (100%) rename tests/templates/kuttl/oidc/{install-superset.yaml.j2 => 40_install-superset.yaml.j2} (100%) create mode 100644 tests/templates/kuttl/opa-authorization/10-assert.yaml create mode 100644 tests/templates/kuttl/opa-authorization/10-install-postgresql.yaml create mode 100644 tests/templates/kuttl/opa-authorization/20-assert.yaml create mode 100644 tests/templates/kuttl/opa-authorization/20-install-keycloak.yaml create mode 100644 tests/templates/kuttl/opa-authorization/30-assert.yaml create mode 100644 tests/templates/kuttl/opa-authorization/30-install-opa.yaml.j2 create mode 100644 tests/templates/kuttl/opa-authorization/31-assert.yaml create mode 100644 tests/templates/kuttl/opa-authorization/31-opa-rego.yaml create mode 100644 tests/templates/kuttl/opa-authorization/40-assert.yaml create mode 100644 tests/templates/kuttl/opa-authorization/40-install-superset.yaml create mode 100644 tests/templates/kuttl/opa-authorization/40_install-superset.yaml.j2 diff --git a/tests/templates/kuttl/oidc/10-install-postgresql.yaml b/tests/templates/kuttl/oidc/10-install-postgresql.yaml index 50c5ad67..a9fc0d36 100644 --- a/tests/templates/kuttl/oidc/10-install-postgresql.yaml +++ b/tests/templates/kuttl/oidc/10-install-postgresql.yaml @@ -6,7 +6,7 @@ commands: helm install superset-postgresql --namespace $NAMESPACE --version 12.5.6 - -f helm-bitnami-postgresql-values.yaml + -f 10_helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql --wait timeout: 600 diff --git a/tests/templates/kuttl/oidc/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/oidc/10_helm-bitnami-postgresql-values.yaml.j2 similarity index 100% rename from tests/templates/kuttl/oidc/helm-bitnami-postgresql-values.yaml.j2 rename to tests/templates/kuttl/oidc/10_helm-bitnami-postgresql-values.yaml.j2 diff --git a/tests/templates/kuttl/oidc/30-install-keycloak.yaml b/tests/templates/kuttl/oidc/30-keycloak.yaml similarity index 79% rename from tests/templates/kuttl/oidc/30-install-keycloak.yaml rename to tests/templates/kuttl/oidc/30-keycloak.yaml index 33104195..13edf988 100644 --- a/tests/templates/kuttl/oidc/30-install-keycloak.yaml +++ b/tests/templates/kuttl/oidc/30-keycloak.yaml @@ -12,7 +12,7 @@ commands: PASSWORD=T8mn72D9 \ CLIENT_ID=superset1 \ CLIENT_SECRET=R1bxHUD569vHeQdw \ - envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - + envsubst < 30_install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - INSTANCE_NAME=keycloak2 \ REALM=test2 \ @@ -23,4 +23,4 @@ commands: PASSWORD=NvfpU518 \ CLIENT_ID=superset2 \ CLIENT_SECRET=scWzh0D4v0GN8NrN \ - envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - + envsubst < 30_install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/oidc/install-keycloak.yaml.j2 b/tests/templates/kuttl/oidc/30_install-keycloak.yaml.j2 similarity index 100% rename from tests/templates/kuttl/oidc/install-keycloak.yaml.j2 rename to tests/templates/kuttl/oidc/30_install-keycloak.yaml.j2 diff --git a/tests/templates/kuttl/oidc/install-superset.yaml.j2 b/tests/templates/kuttl/oidc/40_install-superset.yaml.j2 similarity index 100% rename from tests/templates/kuttl/oidc/install-superset.yaml.j2 rename to tests/templates/kuttl/oidc/40_install-superset.yaml.j2 diff --git a/tests/templates/kuttl/opa-authorization/10-assert.yaml b/tests/templates/kuttl/opa-authorization/10-assert.yaml new file mode 100644 index 00000000..e9c60b15 --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-superset-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/opa-authorization/10-install-postgresql.yaml b/tests/templates/kuttl/opa-authorization/10-install-postgresql.yaml new file mode 100644 index 00000000..50c5ad67 --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/10-install-postgresql.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install superset-postgresql + --namespace $NAMESPACE + --version 12.5.6 + -f helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql + --wait + timeout: 600 diff --git a/tests/templates/kuttl/opa-authorization/20-assert.yaml b/tests/templates/kuttl/opa-authorization/20-assert.yaml new file mode 100644 index 00000000..02818f1f --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/20-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-keycloak +timeout: 480 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keycloak1 +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/opa-authorization/20-install-keycloak.yaml b/tests/templates/kuttl/opa-authorization/20-install-keycloak.yaml new file mode 100644 index 00000000..3159e3be --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/20-install-keycloak.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + INSTANCE_NAME=keycloak1 \ + REALM=test1 \ + USERNAME=jane.doe \ + FIRST_NAME=Jane \ + LAST_NAME=Doe \ + EMAIL=jane.doe@stackable.tech \ + PASSWORD=T8mn72D9 \ + CLIENT_ID=superset1 \ + CLIENT_SECRET=R1bxHUD569vHeQdw \ + envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/opa-authorization/30-assert.yaml b/tests/templates/kuttl/opa-authorization/30-assert.yaml new file mode 100644 index 00000000..e868cdaf --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/30-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE rollout status daemonset opa-server-default --timeout 300s diff --git a/tests/templates/kuttl/opa-authorization/30-install-opa.yaml.j2 b/tests/templates/kuttl/opa-authorization/30-install-opa.yaml.j2 new file mode 100644 index 00000000..93a9911d --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/30-install-opa.yaml.j2 @@ -0,0 +1,43 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 }} + containers: + opa: + loggers: + decision: + level: INFO + roleGroups: + default: {} diff --git a/tests/templates/kuttl/opa-authorization/31-assert.yaml b/tests/templates/kuttl/opa-authorization/31-assert.yaml new file mode 100644 index 00000000..e868cdaf --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/31-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE rollout status daemonset opa-server-default --timeout 300s diff --git a/tests/templates/kuttl/opa-authorization/31-opa-rego.yaml b/tests/templates/kuttl/opa-authorization/31-opa-rego.yaml new file mode 100644 index 00000000..1d85d5b9 --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/31-opa-rego.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test + labels: + opa.stackable.tech/bundle: "true" +data: + roles.rego: | + package superset + + import rego.v1 + + default user_roles := [] + + user_roles := roles if { + some user in users + roles := user.roles + user.username == input.username + } + + users := [ + {"username": "admin", "roles": ["Admin", "Test"]}, + {"username": "testuser", "roles": ["El_Testos", "Custom2"]} + ] diff --git a/tests/templates/kuttl/opa-authorization/40-assert.yaml b/tests/templates/kuttl/opa-authorization/40-assert.yaml new file mode 100644 index 00000000..3eb18400 --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/40-assert.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-superset +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true supersetclusters.superset.stackable.tech/superset --timeout 301s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: superset-node-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/opa-authorization/40-install-superset.yaml b/tests/templates/kuttl/opa-authorization/40-install-superset.yaml new file mode 100644 index 00000000..0cba3c7b --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/40-install-superset.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: > + envsubst '$NAMESPACE' < install-superset.yaml | + kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/opa-authorization/40_install-superset.yaml.j2 b/tests/templates/kuttl/opa-authorization/40_install-superset.yaml.j2 new file mode 100644 index 00000000..ffa2f88b --- /dev/null +++ b/tests/templates/kuttl/opa-authorization/40_install-superset.yaml.j2 @@ -0,0 +1,40 @@ +# $NAMESPACE will be replaced with the namespace of the test case. +--- +apiVersion: v1 +kind: Secret +metadata: + name: superset-credentials +type: Opaque +stringData: + adminUser.username: admin + adminUser.firstname: Superset + adminUser.lastname: Admin + adminUser.email: admin@superset.com + adminUser.password: admin + connections.secretKey: aQC11KVUJ3yTVcy2 + connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset +--- +apiVersion: superset.stackable.tech/v1alpha1 +kind: SupersetCluster +metadata: + name: superset +spec: + image: + productVersion: "{{ test_scenario['values']['superset'] }}" + pullPolicy: IfNotPresent + clusterConfig: + authorization: + opa: + configMapName: simple-opa + package: superset + credentialsSecret: superset-credentials +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 0d13fbf1..76d3ef82 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -40,6 +40,9 @@ tests: dimensions: - superset - openshift + - name: opa + - superset + - openshift - name: resources dimensions: - superset-latest From 06515aba7e11db5e1fc9408a485654ebf0702f31 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 14 Jan 2025 10:06:10 +0100 Subject: [PATCH 18/71] rename test directory --- tests/templates/kuttl/{opa-authorization => opa}/10-assert.yaml | 0 .../kuttl/{opa-authorization => opa}/10-install-postgresql.yaml | 0 tests/templates/kuttl/{opa-authorization => opa}/20-assert.yaml | 0 .../kuttl/{opa-authorization => opa}/20-install-keycloak.yaml | 0 tests/templates/kuttl/{opa-authorization => opa}/30-assert.yaml | 0 .../kuttl/{opa-authorization => opa}/30-install-opa.yaml.j2 | 0 tests/templates/kuttl/{opa-authorization => opa}/31-assert.yaml | 0 tests/templates/kuttl/{opa-authorization => opa}/31-opa-rego.yaml | 0 tests/templates/kuttl/{opa-authorization => opa}/40-assert.yaml | 0 .../kuttl/{opa-authorization => opa}/40-install-superset.yaml | 0 .../kuttl/{opa-authorization => opa}/40_install-superset.yaml.j2 | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename tests/templates/kuttl/{opa-authorization => opa}/10-assert.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/10-install-postgresql.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/20-assert.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/20-install-keycloak.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/30-assert.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/30-install-opa.yaml.j2 (100%) rename tests/templates/kuttl/{opa-authorization => opa}/31-assert.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/31-opa-rego.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/40-assert.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/40-install-superset.yaml (100%) rename tests/templates/kuttl/{opa-authorization => opa}/40_install-superset.yaml.j2 (100%) diff --git a/tests/templates/kuttl/opa-authorization/10-assert.yaml b/tests/templates/kuttl/opa/10-assert.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/10-assert.yaml rename to tests/templates/kuttl/opa/10-assert.yaml diff --git a/tests/templates/kuttl/opa-authorization/10-install-postgresql.yaml b/tests/templates/kuttl/opa/10-install-postgresql.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/10-install-postgresql.yaml rename to tests/templates/kuttl/opa/10-install-postgresql.yaml diff --git a/tests/templates/kuttl/opa-authorization/20-assert.yaml b/tests/templates/kuttl/opa/20-assert.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/20-assert.yaml rename to tests/templates/kuttl/opa/20-assert.yaml diff --git a/tests/templates/kuttl/opa-authorization/20-install-keycloak.yaml b/tests/templates/kuttl/opa/20-install-keycloak.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/20-install-keycloak.yaml rename to tests/templates/kuttl/opa/20-install-keycloak.yaml diff --git a/tests/templates/kuttl/opa-authorization/30-assert.yaml b/tests/templates/kuttl/opa/30-assert.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/30-assert.yaml rename to tests/templates/kuttl/opa/30-assert.yaml diff --git a/tests/templates/kuttl/opa-authorization/30-install-opa.yaml.j2 b/tests/templates/kuttl/opa/30-install-opa.yaml.j2 similarity index 100% rename from tests/templates/kuttl/opa-authorization/30-install-opa.yaml.j2 rename to tests/templates/kuttl/opa/30-install-opa.yaml.j2 diff --git a/tests/templates/kuttl/opa-authorization/31-assert.yaml b/tests/templates/kuttl/opa/31-assert.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/31-assert.yaml rename to tests/templates/kuttl/opa/31-assert.yaml diff --git a/tests/templates/kuttl/opa-authorization/31-opa-rego.yaml b/tests/templates/kuttl/opa/31-opa-rego.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/31-opa-rego.yaml rename to tests/templates/kuttl/opa/31-opa-rego.yaml diff --git a/tests/templates/kuttl/opa-authorization/40-assert.yaml b/tests/templates/kuttl/opa/40-assert.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/40-assert.yaml rename to tests/templates/kuttl/opa/40-assert.yaml diff --git a/tests/templates/kuttl/opa-authorization/40-install-superset.yaml b/tests/templates/kuttl/opa/40-install-superset.yaml similarity index 100% rename from tests/templates/kuttl/opa-authorization/40-install-superset.yaml rename to tests/templates/kuttl/opa/40-install-superset.yaml diff --git a/tests/templates/kuttl/opa-authorization/40_install-superset.yaml.j2 b/tests/templates/kuttl/opa/40_install-superset.yaml.j2 similarity index 100% rename from tests/templates/kuttl/opa-authorization/40_install-superset.yaml.j2 rename to tests/templates/kuttl/opa/40_install-superset.yaml.j2 From a53c78428b2f5b7135897f9e9fc505f2f0841f8d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 14 Jan 2025 10:11:59 +0100 Subject: [PATCH 19/71] fix test-definition --- tests/test-definition.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 76d3ef82..5cbc96ac 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -41,6 +41,7 @@ tests: - superset - openshift - name: opa + dimentions: - superset - openshift - name: resources From 5ea83bdb975751882361d00de28e57e2607c72f8 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 14 Jan 2025 10:42:32 +0100 Subject: [PATCH 20/71] fix typo --- tests/templates/kuttl/opa/31-opa-rego.yaml | 2 +- tests/test-definition.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/opa/31-opa-rego.yaml b/tests/templates/kuttl/opa/31-opa-rego.yaml index 1d85d5b9..f06d52ff 100644 --- a/tests/templates/kuttl/opa/31-opa-rego.yaml +++ b/tests/templates/kuttl/opa/31-opa-rego.yaml @@ -9,7 +9,7 @@ data: roles.rego: | package superset - import rego.v1 + import rego.v1 default user_roles := [] diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 5cbc96ac..8f6aa531 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -41,7 +41,7 @@ tests: - superset - openshift - name: opa - dimentions: + dimensions: - superset - openshift - name: resources From 748030f551a1d362c557fb9622387c596ce47b32 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 15 Jan 2025 09:09:34 +0100 Subject: [PATCH 21/71] fix opa test scaffold --- .../kuttl/opa/10-install-postgresql.yaml | 2 +- .../10_helm-bitnami-postgresql-values.yaml.j2 | 31 ++++ tests/templates/kuttl/opa/20-assert.yaml | 6 +- .../kuttl/opa/20-install-keycloak.yaml | 15 -- .../kuttl/opa/20-install-keycloak.yaml.j2 | 163 ++++++++++++++++++ .../kuttl/opa/20-keycloak-realm-cm.yaml | 91 ++++++++++ .../kuttl/opa/40-install-superset.yaml | 2 +- ...l-superset.yaml.j2 => 40_superset.yaml.j2} | 3 +- tests/test-definition.yaml | 4 + 9 files changed, 295 insertions(+), 22 deletions(-) create mode 100644 tests/templates/kuttl/opa/10_helm-bitnami-postgresql-values.yaml.j2 delete mode 100644 tests/templates/kuttl/opa/20-install-keycloak.yaml create mode 100644 tests/templates/kuttl/opa/20-install-keycloak.yaml.j2 create mode 100644 tests/templates/kuttl/opa/20-keycloak-realm-cm.yaml rename tests/templates/kuttl/opa/{40_install-superset.yaml.j2 => 40_superset.yaml.j2} (90%) diff --git a/tests/templates/kuttl/opa/10-install-postgresql.yaml b/tests/templates/kuttl/opa/10-install-postgresql.yaml index 50c5ad67..a9fc0d36 100644 --- a/tests/templates/kuttl/opa/10-install-postgresql.yaml +++ b/tests/templates/kuttl/opa/10-install-postgresql.yaml @@ -6,7 +6,7 @@ commands: helm install superset-postgresql --namespace $NAMESPACE --version 12.5.6 - -f helm-bitnami-postgresql-values.yaml + -f 10_helm-bitnami-postgresql-values.yaml --repo https://charts.bitnami.com/bitnami postgresql --wait timeout: 600 diff --git a/tests/templates/kuttl/opa/10_helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/opa/10_helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..7991d27e --- /dev/null +++ b/tests/templates/kuttl/opa/10_helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,31 @@ +--- +volumePermissions: + enabled: false + securityContext: + runAsUser: auto + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + +shmVolume: + chmod: + enabled: false + +auth: + username: superset + password: superset + database: superset diff --git a/tests/templates/kuttl/opa/20-assert.yaml b/tests/templates/kuttl/opa/20-assert.yaml index 02818f1f..943a1340 100644 --- a/tests/templates/kuttl/opa/20-assert.yaml +++ b/tests/templates/kuttl/opa/20-assert.yaml @@ -1,14 +1,12 @@ --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -metadata: - name: test-keycloak -timeout: 480 +timeout: 300 --- apiVersion: apps/v1 kind: Deployment metadata: - name: keycloak1 + name: keycloak status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/opa/20-install-keycloak.yaml b/tests/templates/kuttl/opa/20-install-keycloak.yaml deleted file mode 100644 index 3159e3be..00000000 --- a/tests/templates/kuttl/opa/20-install-keycloak.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - INSTANCE_NAME=keycloak1 \ - REALM=test1 \ - USERNAME=jane.doe \ - FIRST_NAME=Jane \ - LAST_NAME=Doe \ - EMAIL=jane.doe@stackable.tech \ - PASSWORD=T8mn72D9 \ - CLIENT_ID=superset1 \ - CLIENT_SECRET=R1bxHUD569vHeQdw \ - envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/opa/20-install-keycloak.yaml.j2 b/tests/templates/kuttl/opa/20-install-keycloak.yaml.j2 new file mode 100644 index 00000000..d8f3c96e --- /dev/null +++ b/tests/templates/kuttl/opa/20-install-keycloak.yaml.j2 @@ -0,0 +1,163 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: keycloak +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: keycloak +{% if test_scenario['values']['openshift'] == 'true' %} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: keycloak +subjects: + - kind: ServiceAccount + name: keycloak +roleRef: + kind: Role + name: keycloak + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - << EOF + --- + apiVersion: secrets.stackable.tech/v1alpha1 + kind: SecretClass + metadata: + name: keycloak-tls-$NAMESPACE + spec: + backend: + autoTls: + ca: + autoGenerate: true + secret: + name: tls + namespace: $NAMESPACE + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: keycloak + labels: + app: keycloak + spec: + replicas: 1 + selector: + matchLabels: + app: keycloak + template: + metadata: + labels: + app: keycloak + spec: + serviceAccountName: keycloak + containers: + - name: keycloak + image: quay.io/keycloak/keycloak:23.0.4 + args: + - start + - --hostname-strict=false + - --https-key-store-file=/tls/keystore.p12 + - --https-key-store-password=changeit + - --import-realm + env: + - name: KEYCLOAK_ADMIN + value: admin + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin-credentials + key: admin + - name: USER_INFO_FETCHER_CLIENT_ID + valueFrom: + secretKeyRef: + name: user-info-fetcher-client-credentials + key: clientId + - name: USER_INFO_FETCHER_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: user-info-fetcher-client-credentials + key: clientSecret + ports: + - name: https + containerPort: 8443 + readinessProbe: + httpGet: + scheme: HTTPS + path: /realms/master + port: https + resources: + limits: + cpu: 1 + memory: 1024Mi + requests: + cpu: 500m + memory: 1024Mi + volumeMounts: + - name: data + mountPath: /opt/keycloak/data/ + - name: tls + mountPath: /tls/ + - name: realm-volume + mountPath: /opt/keycloak/data/import + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsUser: 1000 + volumes: + - name: data + emptyDir: {} + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: keycloak-tls-$NAMESPACE + secrets.stackable.tech/format: tls-pkcs12 + secrets.stackable.tech/format.compatibility.tls-pkcs12.password: changeit + secrets.stackable.tech/scope: service=keycloak,node + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + - name: realm-volume + configMap: + name: keycloak-my-dataspace-realm + --- + apiVersion: v1 + kind: Secret + metadata: + name: keycloak-admin-credentials + stringData: + admin: "adminadmin" + --- + apiVersion: v1 + kind: Service + metadata: + name: keycloak + labels: + app: keycloak + spec: + ports: + - name: https + port: 8443 + targetPort: 8443 + selector: + app: keycloak + EOF diff --git a/tests/templates/kuttl/opa/20-keycloak-realm-cm.yaml b/tests/templates/kuttl/opa/20-keycloak-realm-cm.yaml new file mode 100644 index 00000000..7b669ef8 --- /dev/null +++ b/tests/templates/kuttl/opa/20-keycloak-realm-cm.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: user-info-fetcher-client-credentials +stringData: + clientId: user-info-fetcher + clientSecret: user-info-fetcher-client-secret +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: keycloak-my-dataspace-realm +data: + realm.json: | + { + "realm" : "my-dataspace", + "enabled" : true, + "groups" : [ { + "name" : "group-user", + "path" : "/group-user" + } ], + "users" : [ { + "username" : "service-account-user-info-fetcher", + "enabled" : true, + "totp" : false, + "emailVerified" : false, + "serviceAccountClientId" : "user-info-fetcher", + "credentials" : [ ], + "disableableCredentialTypes" : [ ], + "requiredActions" : [ ], + "realmRoles" : [ "default-roles-my-dataspace" ], + "clientRoles" : { + "realm-management" : [ + "view-users" + ] + }, + "notBefore" : 0, + "groups" : [ ] + }, + { + "enabled": true, + "username": "alice", + "email" : "alice@stackable.tech", + "credentials": [ + { + "type": "password", + "value": "aj238dSbs72k" + } + ], + "realmRoles": [ + "Test1", + "Test2" + ] + } + ], + "roles": { + "realm": [ + { + "name": "Test1", + "description": "Test1" + }, + { + "name": "Test2", + "description": "Test2" + } + ] + }, + "clients" : [ { + "clientId" : "${USER_INFO_FETCHER_CLIENT_ID}", + "surrogateAuthRequired" : false, + "enabled" : true, + "alwaysDisplayInConsole" : false, + "clientAuthenticatorType" : "client-secret", + "secret" : "${USER_INFO_FETCHER_CLIENT_SECRET}", + "redirectUris" : [ "/*" ], + "webOrigins" : [ "/*" ], + "notBefore" : 0, + "bearerOnly" : false, + "serviceAccountsEnabled" : true, + "publicClient" : false, + "frontchannelLogout" : true, + "protocol" : "openid-connect", + "attributes" : { + "oidc.ciba.grant.enabled" : "true", + "oauth2.device.authorization.grant.enabled" : "false" + }, + "authenticationFlowBindingOverrides" : { }, + "fullScopeAllowed" : true + } ] + } diff --git a/tests/templates/kuttl/opa/40-install-superset.yaml b/tests/templates/kuttl/opa/40-install-superset.yaml index 0cba3c7b..b525a1b8 100644 --- a/tests/templates/kuttl/opa/40-install-superset.yaml +++ b/tests/templates/kuttl/opa/40-install-superset.yaml @@ -4,5 +4,5 @@ kind: TestStep timeout: 300 commands: - script: > - envsubst '$NAMESPACE' < install-superset.yaml | + envsubst '$NAMESPACE' < 40_superset.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/opa/40_install-superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 similarity index 90% rename from tests/templates/kuttl/opa/40_install-superset.yaml.j2 rename to tests/templates/kuttl/opa/40_superset.yaml.j2 index ffa2f88b..b7778b45 100644 --- a/tests/templates/kuttl/opa/40_install-superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -20,12 +20,13 @@ metadata: name: superset spec: image: + custom: docker.stackable.tech/stackable/superset:4.0.2-stackable0.0.0-dev-opa productVersion: "{{ test_scenario['values']['superset'] }}" pullPolicy: IfNotPresent clusterConfig: authorization: opa: - configMapName: simple-opa + configMapName: opa package: superset credentialsSecret: superset-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 8f6aa531..5e30eb8f 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -15,6 +15,9 @@ dimensions: - no-tls - insecure-tls - server-verification-tls + - name: opa + values: + - 0.66.0 - name: openshift values: - "false" @@ -43,6 +46,7 @@ tests: - name: opa dimensions: - superset + - opa - openshift - name: resources dimensions: From c77f0bab84aee90af58b05fb43a57d54f6043b3e Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 15 Jan 2025 12:00:02 +0100 Subject: [PATCH 22/71] Adding rule_name to be defined by the user. defaults to empty string --- deploy/helm/superset-operator/crds/crds.yaml | 3 +++ rust/crd/src/lib.rs | 11 ++++++-- rust/operator-binary/src/authorization/opa.rs | 26 +++++++++---------- .../src/superset_controller.rs | 6 ++--- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index e1353b6c..89fbbd6f 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -86,6 +86,9 @@ spec: description: The name of the Rego package containing the Rego rules for the product. nullable: true type: string + rule_name: + default: '' + type: string required: - configMapName type: object diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 3268f595..cf585247 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -258,11 +258,18 @@ impl CurrentlySupportedListenerClasses { } } } +#[derive(Clone, Deserialize, Serialize, Eq, JsonSchema, Debug, PartialEq)] +pub struct SupersetOpaConfig { + #[serde(flatten)] + pub opa: OpaConfig, + #[serde(default)] + pub rule_name: String, +} #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct SupersetAuthorization { - pub opa: Option, + pub opa: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -498,7 +505,7 @@ impl SupersetCluster { } } - pub fn get_opa_config(&self) -> Option<&OpaConfig> { + pub fn get_opa_config(&self) -> Option<&SupersetOpaConfig> { self.spec .cluster_config .authorization diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 822d084f..2516339d 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -1,29 +1,28 @@ use std::collections::BTreeMap; -use stackable_operator::{ - client::Client, - commons::opa::{OpaApiVersion, OpaConfig}, -}; -use stackable_superset_crd::SupersetCluster; +use stackable_operator::{client::Client, commons::opa::OpaApiVersion}; +use stackable_superset_crd::{SupersetCluster, SupersetOpaConfig}; -pub struct SupersetOpaConfig { +pub struct SupersetOpaConfigResolved { opa_base_url: String, opa_package: Option, + rule_name: String, } -impl SupersetOpaConfig { +impl SupersetOpaConfigResolved { pub async fn from_opa_config( client: &Client, superset: &SupersetCluster, - opa_config: &OpaConfig, + opa_config: &SupersetOpaConfig, ) -> Result { // Get opa_base_url for later use in CustomOpaSecurityManager let opa_endpoint = opa_config + .opa .full_document_url_from_config_map(client, superset, None, OpaApiVersion::V1) .await?; // striping package path from base url. Needed by CustomOpaSecurityManager. - let opa_base_url = match opa_config.package.clone() { + let opa_base_url = match opa_config.opa.package.clone() { Some(opa_package_name) => { let opa_path = format!("/v1/data/{opa_package_name}"); opa_endpoint.replace(&opa_path, "") @@ -31,9 +30,10 @@ impl SupersetOpaConfig { None => opa_endpoint.replace("/v1/data/", ""), }; - Ok(SupersetOpaConfig { + Ok(SupersetOpaConfigResolved { opa_base_url, - opa_package: opa_config.package.clone(), + opa_package: opa_config.opa.package.to_owned(), + rule_name: opa_config.rule_name.to_owned(), }) } @@ -50,12 +50,10 @@ impl SupersetOpaConfig { "AUTH_USER_REGISTRATION_ROLE".to_string(), Some("os.getenv('AUTH_USER_REGISTRATION_ROLE', 'Public')".to_string()), ), - // There is no proper way to interfere this without changing e.g. CRD's. - // EnvOverrides are supported. // TODO: Documentation ( "STACKABLE_OPA_RULE".to_string(), - Some("os.getenv('STACKABLE_OPA_RULE', 'user_roles')".to_string()), + Some(self.rule_name.clone()), ), ( "STACKABLE_OPA_BASE_URL".to_string(), diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 56249b98..ce79f7f4 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -75,7 +75,7 @@ use stackable_superset_crd::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - authorization::opa::SupersetOpaConfig, + authorization::opa::SupersetOpaConfigResolved, commands::add_cert_to_python_certifi_command, config::{self, OPA_IMPORTS, PYTHON_IMPORTS}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, @@ -378,7 +378,7 @@ pub async fn reconcile_superset( let superset_opa_config = match superset.get_opa_config() { Some(opa_config) => Some( - SupersetOpaConfig::from_opa_config(client, superset, opa_config) + SupersetOpaConfigResolved::from_opa_config(client, superset, opa_config) .await .context(InvalidOpaConfigSnafu)?, ), @@ -555,7 +555,7 @@ fn build_rolegroup_config_map( rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, authentication_config: &SupersetClientAuthenticationDetailsResolved, - superset_opa_config: &Option, + superset_opa_config: &Option, logging: &Logging, vector_aggregator_address: Option<&str>, ) -> Result { From e19a39cfdff7c0c174197cb77926a8bbcc455b9f Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 15 Jan 2025 12:03:47 +0100 Subject: [PATCH 23/71] Adding default to rule_name --- deploy/helm/superset-operator/crds/crds.yaml | 2 +- rust/crd/src/lib.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index 89fbbd6f..dcfcb238 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -87,7 +87,7 @@ spec: nullable: true type: string rule_name: - default: '' + default: user_rules type: string required: - configMapName diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index cf585247..55135ae1 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -262,10 +262,14 @@ impl CurrentlySupportedListenerClasses { pub struct SupersetOpaConfig { #[serde(flatten)] pub opa: OpaConfig, - #[serde(default)] + #[serde(default = "opa_rule_name_default")] pub rule_name: String, } +fn opa_rule_name_default() -> String { + "user_rules".to_string() +} + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct SupersetAuthorization { From 5e0c32e165d1eae2f70a952e21c648546b5be9fb Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 15 Jan 2025 12:17:06 +0100 Subject: [PATCH 24/71] StackableOpaRule to string as we interfere from CRDs --- rust/crd/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 55135ae1..cfcb338f 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -143,7 +143,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::CustomSecurityManager => PythonType::Expression, SupersetConfigOptions::StackableOpaBaseUrl => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, - SupersetConfigOptions::StackableOpaRule => PythonType::Expression, + SupersetConfigOptions::StackableOpaRule => PythonType::StringLiteral, SupersetConfigOptions::OpaRolesCache => PythonType::Expression, } } From 33d3f3a6e4ce2782d545be98909a10195fedd35e Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 15 Jan 2025 13:53:20 +0100 Subject: [PATCH 25/71] Adding ttl to crds. Default to 10. --- deploy/helm/superset-operator/crds/crds.yaml | 4 ++++ rust/crd/src/lib.rs | 9 +++++++-- rust/operator-binary/src/authorization/opa.rs | 6 ++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index dcfcb238..e71f89bc 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -89,6 +89,10 @@ spec: rule_name: default: user_rules type: string + ttl: + default: 10 + format: int8 + type: integer required: - configMapName type: object diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index cfcb338f..d0f6b06d 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -90,7 +90,7 @@ pub enum SupersetConfigOptions { StackableOpaBaseUrl, StackableOpaPackage, StackableOpaRule, - OpaRolesCache, + OpaRolesCacheTTL, } impl SupersetConfigOptions { @@ -144,7 +144,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::StackableOpaBaseUrl => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaRule => PythonType::StringLiteral, - SupersetConfigOptions::OpaRolesCache => PythonType::Expression, + SupersetConfigOptions::OpaRolesCacheTTL => PythonType::IntLiteral, } } } @@ -264,8 +264,13 @@ pub struct SupersetOpaConfig { pub opa: OpaConfig, #[serde(default = "opa_rule_name_default")] pub rule_name: String, + #[serde(default = "ttl_default_time")] + pub ttl: i8, } +fn ttl_default_time() -> i8 { + 10 +} fn opa_rule_name_default() -> String { "user_rules".to_string() } diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 2516339d..eb297571 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -7,6 +7,7 @@ pub struct SupersetOpaConfigResolved { opa_base_url: String, opa_package: Option, rule_name: String, + ttl: i8, } impl SupersetOpaConfigResolved { @@ -34,6 +35,7 @@ impl SupersetOpaConfigResolved { opa_base_url, opa_package: opa_config.opa.package.to_owned(), rule_name: opa_config.rule_name.to_owned(), + ttl: opa_config.ttl.to_owned(), }) } @@ -64,8 +66,8 @@ impl SupersetOpaConfigResolved { self.opa_package.clone(), ), ( - "OPA_ROLES_CACHE".to_string(), - Some("os.getenv('OPA_ROLES_CACHE', '10')".to_string()), + "OPA_ROLES_CACHE_TTL".to_string(), + Some(self.ttl.to_string().clone()), ), ]) } From bd97146ed497395c87fff90f125327e890649d59 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 15 Jan 2025 15:04:41 +0100 Subject: [PATCH 26/71] cache_ttl now Duration type. Converted to seconds in superset_config.py --- deploy/helm/superset-operator/crds/crds.yaml | 11 ++++++----- rust/crd/src/lib.rs | 9 ++++++--- rust/operator-binary/src/authorization/opa.rs | 7 ++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index e71f89bc..33b64012 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -79,6 +79,10 @@ spec: description: Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) and the name of the Rego package containing your authorization rules. Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) to learn how to deploy Rego authorization rules with OPA. nullable: true properties: + cacheTtl: + default: 10s + description: Retention time of the opa roles. e.g. `30m`, `1h` or `2d` + type: string configMapName: description: The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for the OPA stacklet that should be used for authorization requests. type: string @@ -86,13 +90,10 @@ spec: description: The name of the Rego package containing the Rego rules for the product. nullable: true type: string - rule_name: + ruleName: default: user_rules + description: Name of the rule where roles should be mapped from type: string - ttl: - default: 10 - format: int8 - type: integer required: - configMapName type: object diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index d0f6b06d..016042df 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -259,17 +259,20 @@ impl CurrentlySupportedListenerClasses { } } #[derive(Clone, Deserialize, Serialize, Eq, JsonSchema, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] pub struct SupersetOpaConfig { #[serde(flatten)] pub opa: OpaConfig, + /// Name of the rule where roles should be mapped from #[serde(default = "opa_rule_name_default")] pub rule_name: String, + /// Retention time of the opa roles. e.g. `30m`, `1h` or `2d` #[serde(default = "ttl_default_time")] - pub ttl: i8, + pub cache_ttl: Duration, } -fn ttl_default_time() -> i8 { - 10 +fn ttl_default_time() -> Duration { + Duration::from_secs(10) } fn opa_rule_name_default() -> String { "user_rules".to_string() diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index eb297571..e07e0b41 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -1,3 +1,4 @@ +use stackable_operator::time::Duration; use std::collections::BTreeMap; use stackable_operator::{client::Client, commons::opa::OpaApiVersion}; @@ -7,7 +8,7 @@ pub struct SupersetOpaConfigResolved { opa_base_url: String, opa_package: Option, rule_name: String, - ttl: i8, + ttl: Duration, } impl SupersetOpaConfigResolved { @@ -35,7 +36,7 @@ impl SupersetOpaConfigResolved { opa_base_url, opa_package: opa_config.opa.package.to_owned(), rule_name: opa_config.rule_name.to_owned(), - ttl: opa_config.ttl.to_owned(), + ttl: opa_config.cache_ttl.to_owned(), }) } @@ -67,7 +68,7 @@ impl SupersetOpaConfigResolved { ), ( "OPA_ROLES_CACHE_TTL".to_string(), - Some(self.ttl.to_string().clone()), + Some(self.ttl.as_secs().to_string()), ), ]) } From 1d27837006d815242aae949ac090908ad8edf249 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 15 Jan 2025 15:17:51 +0100 Subject: [PATCH 27/71] wip: integration tests --- tests/templates/kuttl/opa/50-assert.yaml | 14 ++++ .../opa/50-install-test-container.yaml.j2 | 80 +++++++++++++++++++ .../kuttl/opa/51-create-superset-user.yaml | 15 ++++ 3 files changed, 109 insertions(+) create mode 100644 tests/templates/kuttl/opa/50-assert.yaml create mode 100644 tests/templates/kuttl/opa/50-install-test-container.yaml.j2 create mode 100644 tests/templates/kuttl/opa/51-create-superset-user.yaml diff --git a/tests/templates/kuttl/opa/50-assert.yaml b/tests/templates/kuttl/opa/50-assert.yaml new file mode 100644 index 00000000..58987778 --- /dev/null +++ b/tests/templates/kuttl/opa/50-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: python +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/opa/50-install-test-container.yaml.j2 b/tests/templates/kuttl/opa/50-install-test-container.yaml.j2 new file mode 100644 index 00000000..d1199711 --- /dev/null +++ b/tests/templates/kuttl/opa/50-install-test-container.yaml.j2 @@ -0,0 +1,80 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: python +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: python +{% if test_scenario['values']['openshift'] == 'true' %} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: python +subjects: + - kind: ServiceAccount + name: python +roleRef: + kind: Role + name: python + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: python + labels: + app: python +spec: + replicas: 1 + selector: + matchLabels: + app: python + template: + metadata: + labels: + app: python + spec: + serviceAccountName: python + securityContext: + fsGroup: 1000 + containers: + - name: python + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + stdin: true + tty: true + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + volumeMounts: + - name: tls + mountPath: /stackable/tls + env: + - name: REQUESTS_CA_BUNDLE + value: /stackable/tls/ca.crt + volumes: + - name: tls + csi: + driver: secrets.stackable.tech + volumeAttributes: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod diff --git a/tests/templates/kuttl/opa/51-create-superset-user.yaml b/tests/templates/kuttl/opa/51-create-superset-user.yaml new file mode 100644 index 00000000..bd0774c3 --- /dev/null +++ b/tests/templates/kuttl/opa/51-create-superset-user.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: login +timeout: 300 +commands: + - script: | + kubectl exec -n $NAMESPACE python-0 -- curl https://superset-node-default.default.svc.8088 --insecure -d '{ + "username": "newuser", + "first_name": "New", + "last_name": "User", + "email": "newuser@example.com", + "roles": ["Public"] + }' \ No newline at end of file From 441984327aa1ea9013f63752dc1a4fa9be9049d1 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 16 Jan 2025 14:18:23 +0100 Subject: [PATCH 28/71] integrate feedback --- deploy/helm/superset-operator/crds/crds.yaml | 23 +++++++---- rust/crd/src/lib.rs | 38 ++++++++++++------- rust/operator-binary/src/authorization/opa.rs | 29 +++++++------- tests/templates/kuttl/opa/40_superset.yaml.j2 | 1 + 4 files changed, 54 insertions(+), 37 deletions(-) diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index 33b64012..398790f0 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -79,10 +79,20 @@ spec: description: Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) and the name of the Rego package containing your authorization rules. Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) to learn how to deploy Rego authorization rules with OPA. nullable: true properties: - cacheTtl: - default: 10s - description: Retention time of the opa roles. e.g. `30m`, `1h` or `2d` - type: string + cache: + description: Configuration for an superset-internal cache for calls to OPA + properties: + entryTimeToLive: + default: 30s + description: Cache duration , e.g. `30m`, `1h` or `2d` + type: string + maxEntries: + default: 128 + description: Number of maximum user-role mappings cached concurrently + format: uint32 + minimum: 0.0 + type: integer + type: object configMapName: description: The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for the OPA stacklet that should be used for authorization requests. type: string @@ -90,11 +100,8 @@ spec: description: The name of the Rego package containing the Rego rules for the product. nullable: true type: string - ruleName: - default: user_rules - description: Name of the rule where roles should be mapped from - type: string required: + - cache - configMapName type: object type: object diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 016042df..e02a5158 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -90,7 +90,8 @@ pub enum SupersetConfigOptions { StackableOpaBaseUrl, StackableOpaPackage, StackableOpaRule, - OpaRolesCacheTTL, + StackableOpaCacheMaxEntries, + StackableOpaCacheEntryTTL, } impl SupersetConfigOptions { @@ -122,7 +123,7 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::AuthType => PythonType::Expression, SupersetConfigOptions::AuthUserRegistration => PythonType::BoolLiteral, // Going to be an expression as we default it from env, if and only if opa is used - SupersetConfigOptions::AuthUserRegistrationRole => PythonType::Expression, + SupersetConfigOptions::AuthUserRegistrationRole => PythonType::StringLiteral, SupersetConfigOptions::AuthRolesSyncAtLogin => PythonType::BoolLiteral, SupersetConfigOptions::AuthLdapServer => PythonType::StringLiteral, SupersetConfigOptions::AuthLdapBindUser => PythonType::Expression, @@ -144,7 +145,8 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::StackableOpaBaseUrl => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, SupersetConfigOptions::StackableOpaRule => PythonType::StringLiteral, - SupersetConfigOptions::OpaRolesCacheTTL => PythonType::IntLiteral, + SupersetConfigOptions::StackableOpaCacheMaxEntries => PythonType::IntLiteral, + SupersetConfigOptions::StackableOpaCacheEntryTTL => PythonType::IntLiteral, } } } @@ -263,19 +265,29 @@ impl CurrentlySupportedListenerClasses { pub struct SupersetOpaConfig { #[serde(flatten)] pub opa: OpaConfig, - /// Name of the rule where roles should be mapped from - #[serde(default = "opa_rule_name_default")] - pub rule_name: String, - /// Retention time of the opa roles. e.g. `30m`, `1h` or `2d` - #[serde(default = "ttl_default_time")] - pub cache_ttl: Duration, + + /// Configuration for an superset-internal cache for calls to OPA + pub cache: SupersetOpaCacheConfig, +} + +#[derive(Clone, Deserialize, Serialize, Eq, JsonSchema, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SupersetOpaCacheConfig { + /// Cache duration , e.g. `30m`, `1h` or `2d` + #[serde(default = "cache_ttl_default")] + pub entry_time_to_live: Duration, + + /// Number of maximum user-role mappings cached concurrently + #[serde(default = "cache_max_entries_default")] + pub max_entries: u32, } -fn ttl_default_time() -> Duration { - Duration::from_secs(10) +fn cache_max_entries_default() -> u32 { + 128 } -fn opa_rule_name_default() -> String { - "user_rules".to_string() + +fn cache_ttl_default() -> Duration { + Duration::from_secs(30) } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index e07e0b41..06f6b59b 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -7,8 +7,8 @@ use stackable_superset_crd::{SupersetCluster, SupersetOpaConfig}; pub struct SupersetOpaConfigResolved { opa_base_url: String, opa_package: Option, - rule_name: String, - ttl: Duration, + cache_max_entries: u32, + cache_ttl: Duration, } impl SupersetOpaConfigResolved { @@ -35,8 +35,8 @@ impl SupersetOpaConfigResolved { Ok(SupersetOpaConfigResolved { opa_base_url, opa_package: opa_config.opa.package.to_owned(), - rule_name: opa_config.rule_name.to_owned(), - ttl: opa_config.cache_ttl.to_owned(), + cache_max_entries: opa_config.cache.max_entries.to_owned(), + cache_ttl: opa_config.cache.entry_time_to_live.to_owned(), }) } @@ -47,29 +47,26 @@ impl SupersetOpaConfigResolved { "CUSTOM_SECURITY_MANAGER".to_string(), Some("OpaSupersetSecurityManager".to_string()), ), - // This is now a PythonType::Expression. Makes it easy to find a default. - // EnvOverrides are supported. - ( - "AUTH_USER_REGISTRATION_ROLE".to_string(), - Some("os.getenv('AUTH_USER_REGISTRATION_ROLE', 'Public')".to_string()), - ), - // TODO: Documentation ( "STACKABLE_OPA_RULE".to_string(), - Some(self.rule_name.clone()), + Some("user_roles".to_string()), ), ( "STACKABLE_OPA_BASE_URL".to_string(), - Some(self.opa_base_url.clone()), + Some(self.opa_base_url.to_owned()), ), ( "STACKABLE_OPA_PACKAGE".to_string(), - self.opa_package.clone(), + self.opa_package.to_owned(), ), ( - "OPA_ROLES_CACHE_TTL".to_string(), - Some(self.ttl.as_secs().to_string()), + "STACKABLE_OPA_CACHE_MAX_ENTRIES".to_string(), + Some(self.cache_max_entries.to_string()), ), + ( + "STACKABLE_OPA_CACHE_ENTRY_TTL".to_string(), + Some(self.cache_ttl.as_secs().to_string()) + ) ]) } } diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index b7778b45..47ee0881 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -28,6 +28,7 @@ spec: opa: configMapName: opa package: superset + cache: {} credentialsSecret: superset-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery From 6a8f8e7d55fd72699d7b638af5b0863133f151e7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 16 Jan 2025 22:45:18 +0100 Subject: [PATCH 29/71] create basic user role test --- tests/templates/kuttl/opa/get_user_roles.py | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/templates/kuttl/opa/get_user_roles.py diff --git a/tests/templates/kuttl/opa/get_user_roles.py b/tests/templates/kuttl/opa/get_user_roles.py new file mode 100644 index 00000000..be2cf70c --- /dev/null +++ b/tests/templates/kuttl/opa/get_user_roles.py @@ -0,0 +1,30 @@ +import logging +import os +import sys +import requests + +from bs4 import BeautifulSoup + +logging.basicConfig( + level='DEBUG', + format="%(asctime)s %(levelname)s: %(message)s", + stream=sys.stdout) + +superset_base_url = os.getenv('SUPERSET_BASE_URL', 'localhost:8088') + +session = requests.Session() + +# Click on "Login" in Superset +login_page = session.get(f'http://{superset_base_url}/login/') +assert login_page.status_code == 200 + +login_page_html = BeautifulSoup(login_page.text, 'html.parser') +csrf_token = login_page_html.find('input',{'id':'csrf_token'})['value'] + +# Login with CSRF token +welcome_page = session.post(f'http://{superset_base_url}/login/', data={'username': 'admin','password': 'admin', 'csrf_token': csrf_token}) +assert welcome_page.status_code == 200 + +# Get user roles +user_roles = set(session.get(f'http://{superset_base_url}/api/v1/me/roles/').json()['result']['roles'].keys()) +assert user_roles == {'Admin', 'Test'} From 203e3905516fd482979033cfdc21d4f80fa14b1d Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 17 Jan 2025 10:56:06 +0100 Subject: [PATCH 30/71] First documentation draft --- .../superset/pages/usage-guide/security.adoc | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/modules/superset/pages/usage-guide/security.adoc b/docs/modules/superset/pages/usage-guide/security.adoc index f4a3623c..904c1098 100644 --- a/docs/modules/superset/pages/usage-guide/security.adoc +++ b/docs/modules/superset/pages/usage-guide/security.adoc @@ -126,6 +126,66 @@ Further information for specifying an AuthenticationClass for an OIDC provider c Superset has a concept called `Roles` which allows you to grant user permissions based on roles. Have a look at the {superset-security}[Superset documentation on Security^{external-link-icon}^]. +=== [[Opa]] Opa Roles Mapping + +Superset can sync roles from open policy agent. Currently only mapping is enabled as a larger refactoring of the upstream superset concerning their security management is announced. + +In order to map roles from Opa into superset, we expect rego rules with a rule name `user_roles`. In the below example two users `admin` and `testuser` have roles defined as rego rule in Rego V1 to be complient with OPA v1.0.0 release. + +IMPORTANT: Only role mapping is enabled. Permissions can only be added through the Superset UI. RBAC through OPA is not implemented. + +[source,yaml] +---- +apiVersion: v1 +kind: ConfigMap +metadata: + name: superset-opa-regorules + labels: + opa.stackable.tech/bundle: "true" +data: + roles.rego: | + package superset + + import rego.v1 + + default user_roles := [] + + user_roles := roles if { + some user in users + roles := user.roles + user.username == input.username + } + + users := [ + {"username": "admin", "roles": ["Admin", "Test"]}, + {"username": "testuser", "roles": ["El_Testos", "Custom2"]} + ] +---- + +Mounting this `configMap` in superset as follows: + +[source,yaml] +---- +apiVersion: superset.stackable.tech/v1alpha1 +kind: SupersetCluster +metadata: + name: superset-with-opa-role-mapping +spec: + clusterConfig: + authorization: + roleMappingFromOpa: + configMapName: superset-opa-regorules # <1> + package: superset + cache: # <2> + entryTimeToLive: 10s # <3> + maxEntries: 5 # <4> +---- + +<1> ConfigMap name containing rego rules +<2> Mandatory Opa caching. Reduces calls to OPA API. +<3> Time for cached entries per user can live. Defaults to 30s. +<4> Number of maximum entries, defaults to 1000. Cache will be disabled for maxEntries: 0. + === Superset database You can view all the available roles in the web interface of Superset and can also assign users to these roles. From cd6a64cf39b34478bf286ebf86367bc83b16acab Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 17 Jan 2025 12:52:13 +0100 Subject: [PATCH 31/71] Adding reference to opa user-info-fetcher --- docs/modules/superset/pages/usage-guide/security.adoc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/modules/superset/pages/usage-guide/security.adoc b/docs/modules/superset/pages/usage-guide/security.adoc index 904c1098..effa2065 100644 --- a/docs/modules/superset/pages/usage-guide/security.adoc +++ b/docs/modules/superset/pages/usage-guide/security.adoc @@ -134,6 +134,8 @@ In order to map roles from Opa into superset, we expect rego rules with a rule n IMPORTANT: Only role mapping is enabled. Permissions can only be added through the Superset UI. RBAC through OPA is not implemented. +OPA rules can be synced using the xref:opa:usage-guide:user-info-fetcher[user-info-fetcher]. + [source,yaml] ---- apiVersion: v1 @@ -155,7 +157,7 @@ data: roles := user.roles user.username == input.username } - +# TODO: User opainfofetcher() users := [ {"username": "admin", "roles": ["Admin", "Test"]}, {"username": "testuser", "roles": ["El_Testos", "Custom2"]} @@ -182,7 +184,7 @@ spec: ---- <1> ConfigMap name containing rego rules -<2> Mandatory Opa caching. Reduces calls to OPA API. +<2> Mandatory Opa caching. If not set, default settings apply. <3> Time for cached entries per user can live. Defaults to 30s. <4> Number of maximum entries, defaults to 1000. Cache will be disabled for maxEntries: 0. From 49da77ae96de547b39ba9dbb2ed84c3715130751 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 20 Jan 2025 15:29:32 +0100 Subject: [PATCH 32/71] Updating changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c1d493a..4e3573bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Run a `containerdebug` process in the background of each Superset container to collect debugging information ([#578]). - Aggregate emitted Kubernetes events on the CustomResources ([#585]). +- Opa Role mapping as optional custom manager for superset ([#582]). ### Changed @@ -21,6 +22,7 @@ [#568]: https://github.com/stackabletech/superset-operator/pull/568 [#569]: https://github.com/stackabletech/superset-operator/pull/569 [#578]: https://github.com/stackabletech/superset-operator/pull/578 +[#582]: https://github.com/stackabletech/superset-operator/pull/582 [#585]: https://github.com/stackabletech/superset-operator/pull/585 ## [24.11.0] - 2024-11-18 From f76ba931522b0d9b6b0acb8d47abfdc5f045c587 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 20 Jan 2025 17:35:34 +0100 Subject: [PATCH 33/71] fix rego and first check in integration test --- tests/templates/kuttl/oidc/login.py | 2 +- .../kuttl/opa/20-keycloak-realm-cm.yaml | 57 +++++++++---------- tests/templates/kuttl/opa/31-assert.yaml | 6 -- tests/templates/kuttl/opa/31-opa-rego.yaml | 14 +---- .../templates/kuttl/opa/50-get-user-roles-cm | 7 +++ .../opa/{50-assert.yaml => 51-assert.yaml} | 0 .../kuttl/opa/51-create-superset-user.yaml | 15 ----- ...l.j2 => 51-install-test-container.yaml.j2} | 7 +++ .../kuttl/opa/52-test-user-roles.yaml.j2 | 8 +++ tests/templates/kuttl/opa/get_user_roles.py | 18 ++++-- 10 files changed, 64 insertions(+), 70 deletions(-) delete mode 100644 tests/templates/kuttl/opa/31-assert.yaml create mode 100644 tests/templates/kuttl/opa/50-get-user-roles-cm rename tests/templates/kuttl/opa/{50-assert.yaml => 51-assert.yaml} (100%) delete mode 100644 tests/templates/kuttl/opa/51-create-superset-user.yaml rename tests/templates/kuttl/opa/{50-install-test-container.yaml.j2 => 51-install-test-container.yaml.j2} (85%) create mode 100644 tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index 91e28eb8..bc80bd81 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -2,8 +2,8 @@ import json import logging -import requests import sys +import requests from bs4 import BeautifulSoup logging.basicConfig( diff --git a/tests/templates/kuttl/opa/20-keycloak-realm-cm.yaml b/tests/templates/kuttl/opa/20-keycloak-realm-cm.yaml index 7b669ef8..14cc8faf 100644 --- a/tests/templates/kuttl/opa/20-keycloak-realm-cm.yaml +++ b/tests/templates/kuttl/opa/20-keycloak-realm-cm.yaml @@ -16,11 +16,17 @@ data: { "realm" : "my-dataspace", "enabled" : true, - "groups" : [ { - "name" : "group-user", - "path" : "/group-user" - } ], - "users" : [ { + "groups" : [ + { + "name" : "Admin", + "path" : "/Admin" + }, + { + "name": "Test", + "path": "/Test" + }], + "users" : [ + { "username" : "service-account-user-info-fetcher", "enabled" : true, "totp" : false, @@ -39,33 +45,22 @@ data: "groups" : [ ] }, { - "enabled": true, - "username": "alice", - "email" : "alice@stackable.tech", - "credentials": [ - { - "type": "password", - "value": "aj238dSbs72k" - } - ], - "realmRoles": [ - "Test1", - "Test2" - ] - } + "username" : "admin", + "enabled" : true, + "emailVerified" : true, + "firstName" : "admin", + "lastName" : "admin", + "email" : "admin@example.com", + "credentials" : [ { + "type" : "password", + "userLabel" : "My password", + "secretData" : "{\"value\":\"JxIyEshkBUrhZX1BEN9JO8EM3ue5/SnGHDfuyTqOH6A=\",\"salt\":\"f6iCn2rWqZQaRnCCsKAoQQ==\",\"additionalParameters\":{}}", + "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}" + } ], + "realmRoles" : [ ], + "groups" : [ "/Admin", "/Test" ] + } ], - "roles": { - "realm": [ - { - "name": "Test1", - "description": "Test1" - }, - { - "name": "Test2", - "description": "Test2" - } - ] - }, "clients" : [ { "clientId" : "${USER_INFO_FETCHER_CLIENT_ID}", "surrogateAuthRequired" : false, diff --git a/tests/templates/kuttl/opa/31-assert.yaml b/tests/templates/kuttl/opa/31-assert.yaml deleted file mode 100644 index e868cdaf..00000000 --- a/tests/templates/kuttl/opa/31-assert.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 300 -commands: - - script: kubectl -n $NAMESPACE rollout status daemonset opa-server-default --timeout 300s diff --git a/tests/templates/kuttl/opa/31-opa-rego.yaml b/tests/templates/kuttl/opa/31-opa-rego.yaml index f06d52ff..5c8405a5 100644 --- a/tests/templates/kuttl/opa/31-opa-rego.yaml +++ b/tests/templates/kuttl/opa/31-opa-rego.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: test + name: opa-superset-uif-rego labels: opa.stackable.tech/bundle: "true" data: @@ -11,15 +11,7 @@ data: import rego.v1 - default user_roles := [] - user_roles := roles if { - some user in users - roles := user.roles - user.username == input.username + group_paths := data.stackable.opa.userinfo.v1.userInfoByUsername(input.username).groups + roles := [ trim(group,"/") | group := group_paths[_]] } - - users := [ - {"username": "admin", "roles": ["Admin", "Test"]}, - {"username": "testuser", "roles": ["El_Testos", "Custom2"]} - ] diff --git a/tests/templates/kuttl/opa/50-get-user-roles-cm b/tests/templates/kuttl/opa/50-get-user-roles-cm new file mode 100644 index 00000000..16a9ca20 --- /dev/null +++ b/tests/templates/kuttl/opa/50-get-user-roles-cm @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: > + kubectl create cm get-user-roles-script -n $NAMESPACE --from-file get_user_roles.py diff --git a/tests/templates/kuttl/opa/50-assert.yaml b/tests/templates/kuttl/opa/51-assert.yaml similarity index 100% rename from tests/templates/kuttl/opa/50-assert.yaml rename to tests/templates/kuttl/opa/51-assert.yaml diff --git a/tests/templates/kuttl/opa/51-create-superset-user.yaml b/tests/templates/kuttl/opa/51-create-superset-user.yaml deleted file mode 100644 index bd0774c3..00000000 --- a/tests/templates/kuttl/opa/51-create-superset-user.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -metadata: - name: login -timeout: 300 -commands: - - script: | - kubectl exec -n $NAMESPACE python-0 -- curl https://superset-node-default.default.svc.8088 --insecure -d '{ - "username": "newuser", - "first_name": "New", - "last_name": "User", - "email": "newuser@example.com", - "roles": ["Public"] - }' \ No newline at end of file diff --git a/tests/templates/kuttl/opa/50-install-test-container.yaml.j2 b/tests/templates/kuttl/opa/51-install-test-container.yaml.j2 similarity index 85% rename from tests/templates/kuttl/opa/50-install-test-container.yaml.j2 rename to tests/templates/kuttl/opa/51-install-test-container.yaml.j2 index d1199711..b6de3aef 100644 --- a/tests/templates/kuttl/opa/50-install-test-container.yaml.j2 +++ b/tests/templates/kuttl/opa/51-install-test-container.yaml.j2 @@ -68,9 +68,13 @@ spec: volumeMounts: - name: tls mountPath: /stackable/tls + - name: get-user-roles-script + mountPath: /tmp/scripts env: - name: REQUESTS_CA_BUNDLE value: /stackable/tls/ca.crt + - name: SUPERSET_BASE_URL + value: superset-external.$NAMESPACE.svc.cluster.local:8088 volumes: - name: tls csi: @@ -78,3 +82,6 @@ spec: volumeAttributes: secrets.stackable.tech/class: tls secrets.stackable.tech/scope: pod + - name: get-user-roles-script + configMap: + name: get-user-roles-script diff --git a/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 b/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 new file mode 100644 index 00000000..857a388f --- /dev/null +++ b/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # Need to wait until the superset API is ready before running the script + - script: | + sleep 30 + kubectl exec -n $NAMESPACE python-0 -i -- python /tmp/scripts/get_user_roles.py $NAMESPACE Admin,Test + diff --git a/tests/templates/kuttl/opa/get_user_roles.py b/tests/templates/kuttl/opa/get_user_roles.py index be2cf70c..58e6008a 100644 --- a/tests/templates/kuttl/opa/get_user_roles.py +++ b/tests/templates/kuttl/opa/get_user_roles.py @@ -1,5 +1,4 @@ import logging -import os import sys import requests @@ -10,21 +9,28 @@ format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout) -superset_base_url = os.getenv('SUPERSET_BASE_URL', 'localhost:8088') +namespace = sys.argv[1] +expected_roles = set(sys.argv[2].split(',')) session = requests.Session() # Click on "Login" in Superset -login_page = session.get(f'http://{superset_base_url}/login/') +login_page = session.get(f'http://superset-external.{namespace}.svc.cluster.local:8088/login/') assert login_page.status_code == 200 login_page_html = BeautifulSoup(login_page.text, 'html.parser') csrf_token = login_page_html.find('input',{'id':'csrf_token'})['value'] # Login with CSRF token -welcome_page = session.post(f'http://{superset_base_url}/login/', data={'username': 'admin','password': 'admin', 'csrf_token': csrf_token}) +welcome_page = session.post(f'http://superset-external.{namespace}.svc.cluster.local:8088/login/', data={'username': 'admin','password': 'admin', 'csrf_token': csrf_token}) assert welcome_page.status_code == 200 +logging.debug(welcome_page.url) + +# Call an API that will trigger an update of user roles +session.get(f'http://superset-external.{namespace}.svc.cluster.local:8088/api/v1/dashboard/') # Get user roles -user_roles = set(session.get(f'http://{superset_base_url}/api/v1/me/roles/').json()['result']['roles'].keys()) -assert user_roles == {'Admin', 'Test'} +user_roles = set(session.get(f'http://superset-external.{namespace}.svc.cluster.local:8088/api/v1/me/roles/').json()['result']['roles'].keys()) +logging.debug('User roles: %s', user_roles) +logging.debug('Expected roles: %s', expected_roles) +assert user_roles == expected_roles From deddd5a64388bf495cb16eef2fcedfd1a28c417d Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 21 Jan 2025 09:54:57 +0100 Subject: [PATCH 34/71] fix formatting issues --- tests/templates/kuttl/opa/31-opa-rego.yaml | 2 +- tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/templates/kuttl/opa/31-opa-rego.yaml b/tests/templates/kuttl/opa/31-opa-rego.yaml index 5c8405a5..c60324be 100644 --- a/tests/templates/kuttl/opa/31-opa-rego.yaml +++ b/tests/templates/kuttl/opa/31-opa-rego.yaml @@ -13,5 +13,5 @@ data: user_roles := roles if { group_paths := data.stackable.opa.userinfo.v1.userInfoByUsername(input.username).groups - roles := [ trim(group,"/") | group := group_paths[_]] + roles := [ trim(group,"/") | group := group_paths[_] ] } diff --git a/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 b/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 index 857a388f..bbbd3c8d 100644 --- a/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 +++ b/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 @@ -5,4 +5,3 @@ commands: - script: | sleep 30 kubectl exec -n $NAMESPACE python-0 -i -- python /tmp/scripts/get_user_roles.py $NAMESPACE Admin,Test - From 1093fa2730867c1a00d83a7f09856b921010e221 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 21 Jan 2025 10:21:23 +0100 Subject: [PATCH 35/71] Making rust fmt happy --- rust/operator-binary/src/authorization/opa.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 06f6b59b..6b5babde 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -65,8 +65,8 @@ impl SupersetOpaConfigResolved { ), ( "STACKABLE_OPA_CACHE_ENTRY_TTL".to_string(), - Some(self.cache_ttl.as_secs().to_string()) - ) + Some(self.cache_ttl.as_secs().to_string()), + ), ]) } } From 75f19d2767137e28e5fc6a6304afa3f5c778324e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 21 Jan 2025 14:27:02 +0100 Subject: [PATCH 36/71] lint with ruff formatter --- tests/templates/kuttl/ldap/login.py | 25 +++++++---- .../kuttl/logging/test_log_aggregation.py | 33 ++++++++------- tests/templates/kuttl/oidc/login.py | 42 ++++++++++--------- tests/templates/kuttl/opa/get_user_roles.py | 36 ++++++++++------ tests/templates/kuttl/smoke/login.py | 19 +++++---- tests/templates/kuttl/smoke/metrics.py | 3 +- 6 files changed, 94 insertions(+), 64 deletions(-) diff --git a/tests/templates/kuttl/ldap/login.py b/tests/templates/kuttl/ldap/login.py index d058d1c9..1099c41e 100644 --- a/tests/templates/kuttl/ldap/login.py +++ b/tests/templates/kuttl/ldap/login.py @@ -2,18 +2,25 @@ import sys import logging -if __name__ == '__main__': +if __name__ == "__main__": result = 0 - log_level = 'DEBUG' # if args.debug else 'INFO' - logging.basicConfig(level=log_level, format='%(asctime)s %(levelname)s: %(message)s', stream=sys.stdout) + log_level = "DEBUG" # if args.debug else 'INFO' + logging.basicConfig( + level=log_level, + format="%(asctime)s %(levelname)s: %(message)s", + stream=sys.stdout, + ) - http_code = requests.post('http://superset-with-ldap-node-default:8088/api/v1/security/login', json={ - 'username': 'integrationtest', - 'password': 'integrationtest', - 'provider': 'ldap', - 'refresh': 'true', - }).status_code + http_code = requests.post( + "http://superset-with-ldap-node-default:8088/api/v1/security/login", + json={ + "username": "integrationtest", + "password": "integrationtest", + "provider": "ldap", + "refresh": "true", + }, + ).status_code if http_code != 200: result = 1 diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py index 3fcc974e..48105202 100755 --- a/tests/templates/kuttl/logging/test_log_aggregation.py +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -4,9 +4,9 @@ def check_sent_events(): response = requests.post( - 'http://superset-vector-aggregator:8686/graphql', + "http://superset-vector-aggregator:8686/graphql", json={ - 'query': """ + "query": """ { transforms(first:100) { nodes { @@ -20,29 +20,30 @@ def check_sent_events(): } } """ - } + }, ) - assert response.status_code == 200, \ - 'Cannot access the API of the vector aggregator.' + assert response.status_code == 200, ( + "Cannot access the API of the vector aggregator." + ) result = response.json() - transforms = result['data']['transforms']['nodes'] + transforms = result["data"]["transforms"]["nodes"] for transform in transforms: - sentEvents = transform['metrics']['sentEventsTotal'] - componentId = transform['componentId'] + sentEvents = transform["metrics"]["sentEventsTotal"] + componentId = transform["componentId"] - if componentId == 'filteredInvalidEvents': - assert sentEvents is None or \ - sentEvents['sentEventsTotal'] == 0, \ - 'Invalid log events were sent.' + if componentId == "filteredInvalidEvents": + assert sentEvents is None or sentEvents["sentEventsTotal"] == 0, ( + "Invalid log events were sent." + ) else: - assert sentEvents is not None and \ - sentEvents['sentEventsTotal'] > 0, \ + assert sentEvents is not None and sentEvents["sentEventsTotal"] > 0, ( f'No events were sent in "{componentId}".' + ) -if __name__ == '__main__': +if __name__ == "__main__": check_sent_events() - print('Test successful!') + print("Test successful!") diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index bc80bd81..9b7cef67 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -7,9 +7,8 @@ from bs4 import BeautifulSoup logging.basicConfig( - level='DEBUG', - format="%(asctime)s %(levelname)s: %(message)s", - stream=sys.stdout) + level="DEBUG", format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout +) session = requests.Session() @@ -17,40 +16,45 @@ login_page = session.get("http://superset-external:8088/login/keycloak?next=") assert login_page.ok, "Redirection from Superset to Keycloak failed" -assert login_page.url.startswith("https://keycloak1.$NAMESPACE.svc.cluster.local:8443/realms/test1/protocol/openid-connect/auth?response_type=code&client_id=superset1"), \ - "Redirection to the Keycloak login page expected" +assert login_page.url.startswith( + "https://keycloak1.$NAMESPACE.svc.cluster.local:8443/realms/test1/protocol/openid-connect/auth?response_type=code&client_id=superset1" +), "Redirection to the Keycloak login page expected" # Enter username and password into the Keycloak login page and click on "Sign In" -login_page_html = BeautifulSoup(login_page.text, 'html.parser') -authenticate_url = login_page_html.form['action'] -welcome_page = session.post(authenticate_url, data={ - 'username': "jane.doe", - 'password': "T8mn72D9" -}) +login_page_html = BeautifulSoup(login_page.text, "html.parser") +authenticate_url = login_page_html.form["action"] +welcome_page = session.post( + authenticate_url, data={"username": "jane.doe", "password": "T8mn72D9"} +) assert welcome_page.ok, "Login failed" -assert welcome_page.url == "http://superset-external:8088/superset/welcome/", \ +assert welcome_page.url == "http://superset-external:8088/superset/welcome/", ( "Redirection to the Superset welcome page expected" +) # Open the user information page in Superset userinfo_page = session.get("http://superset-external:8088/users/userinfo/") assert userinfo_page.ok, "Retrieving user information failed" -assert userinfo_page.url == "http://superset-external:8088/superset/welcome/", \ +assert userinfo_page.url == "http://superset-external:8088/superset/welcome/", ( "Redirection to the Superset welcome page expected" +) # Expect the user data provided by Keycloak in Superset -userinfo_page_html = BeautifulSoup(userinfo_page.text, 'html.parser') -raw_data = userinfo_page_html.find(id='app')['data-bootstrap'] +userinfo_page_html = BeautifulSoup(userinfo_page.text, "html.parser") +raw_data = userinfo_page_html.find(id="app")["data-bootstrap"] data = json.loads(raw_data) -user_data = data['user'] +user_data = data["user"] -assert user_data['firstName'] == "Jane", \ +assert user_data["firstName"] == "Jane", ( "The first name of the user in Superset should match the one provided by Keycloak" -assert user_data['lastName'] == "Doe", \ +) +assert user_data["lastName"] == "Doe", ( "The last name of the user in Superset should match the one provided by Keycloak" -assert user_data['email'] == "jane.doe@stackable.tech", \ +) +assert user_data["email"] == "jane.doe@stackable.tech", ( "The email of the user in Superset should match the one provided by Keycloak" +) # TODO Use different OIDC providers (currently only Keycloak is # supported) diff --git a/tests/templates/kuttl/opa/get_user_roles.py b/tests/templates/kuttl/opa/get_user_roles.py index 58e6008a..c397eec3 100644 --- a/tests/templates/kuttl/opa/get_user_roles.py +++ b/tests/templates/kuttl/opa/get_user_roles.py @@ -5,32 +5,44 @@ from bs4 import BeautifulSoup logging.basicConfig( - level='DEBUG', - format="%(asctime)s %(levelname)s: %(message)s", - stream=sys.stdout) + level="DEBUG", format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout +) namespace = sys.argv[1] -expected_roles = set(sys.argv[2].split(',')) +expected_roles = set(sys.argv[2].split(",")) session = requests.Session() # Click on "Login" in Superset -login_page = session.get(f'http://superset-external.{namespace}.svc.cluster.local:8088/login/') +login_page = session.get( + f"http://superset-external.{namespace}.svc.cluster.local:8088/login/" +) assert login_page.status_code == 200 -login_page_html = BeautifulSoup(login_page.text, 'html.parser') -csrf_token = login_page_html.find('input',{'id':'csrf_token'})['value'] +login_page_html = BeautifulSoup(login_page.text, "html.parser") +csrf_token = login_page_html.find("input", {"id": "csrf_token"})["value"] # Login with CSRF token -welcome_page = session.post(f'http://superset-external.{namespace}.svc.cluster.local:8088/login/', data={'username': 'admin','password': 'admin', 'csrf_token': csrf_token}) +welcome_page = session.post( + f"http://superset-external.{namespace}.svc.cluster.local:8088/login/", + data={"username": "admin", "password": "admin", "csrf_token": csrf_token}, +) assert welcome_page.status_code == 200 logging.debug(welcome_page.url) # Call an API that will trigger an update of user roles -session.get(f'http://superset-external.{namespace}.svc.cluster.local:8088/api/v1/dashboard/') +session.get( + f"http://superset-external.{namespace}.svc.cluster.local:8088/api/v1/dashboard/" +) # Get user roles -user_roles = set(session.get(f'http://superset-external.{namespace}.svc.cluster.local:8088/api/v1/me/roles/').json()['result']['roles'].keys()) -logging.debug('User roles: %s', user_roles) -logging.debug('Expected roles: %s', expected_roles) +user_roles = set( + session.get( + f"http://superset-external.{namespace}.svc.cluster.local:8088/api/v1/me/roles/" + ) + .json()["result"]["roles"] + .keys() +) +logging.debug("User roles: %s", user_roles) +logging.debug("Expected roles: %s", expected_roles) assert user_roles == expected_roles diff --git a/tests/templates/kuttl/smoke/login.py b/tests/templates/kuttl/smoke/login.py index 9607fb2b..eca166b3 100644 --- a/tests/templates/kuttl/smoke/login.py +++ b/tests/templates/kuttl/smoke/login.py @@ -2,13 +2,18 @@ import sys import logging -logging.basicConfig(level='DEBUG', format='%(asctime)s %(levelname)s: %(message)s', stream=sys.stdout) +logging.basicConfig( + level="DEBUG", format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout +) -http_code = requests.post("http://superset-node-default:8088/api/v1/security/login", json={ - "password": "admin", - "provider": "db", - "refresh": "true", - "username": "admin", -}).status_code +http_code = requests.post( + "http://superset-node-default:8088/api/v1/security/login", + json={ + "password": "admin", + "provider": "db", + "refresh": "true", + "username": "admin", + }, +).status_code assert http_code == 200, "Login failed." diff --git a/tests/templates/kuttl/smoke/metrics.py b/tests/templates/kuttl/smoke/metrics.py index 682207fb..6c86f2f2 100644 --- a/tests/templates/kuttl/smoke/metrics.py +++ b/tests/templates/kuttl/smoke/metrics.py @@ -11,5 +11,6 @@ metrics_response = requests.get("http://superset-node-default:9102/metrics") assert metrics_response.status_code == 200, "Metrics could not be retrieved." -assert "superset_welcome" in metrics_response.text, \ +assert "superset_welcome" in metrics_response.text, ( "The metrics do not contain the superset_welcome counter." +) From f1de6ba1bf17c1da56d611d36ef98ab624b5236f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 21 Jan 2025 16:29:18 +0100 Subject: [PATCH 37/71] use TtlCache from operator-rs --- Cargo.lock | 6 +++--- Cargo.toml | 2 ++ rust/crd/src/lib.rs | 8 ++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57c3d4c6..6df57b4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2332,7 +2332,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stackable-operator" version = "0.84.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.0#af0d1f19d8770d346096a38c6dc82ba70e371039" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/cache#7af060b6f821d7aa82c8ecb0abf6e328a024106a" dependencies = [ "chrono", "clap", @@ -2370,7 +2370,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.0#af0d1f19d8770d346096a38c6dc82ba70e371039" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/cache#7af060b6f821d7aa82c8ecb0abf6e328a024106a" dependencies = [ "darling", "proc-macro2", @@ -2381,7 +2381,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.84.0#af0d1f19d8770d346096a38c6dc82ba70e371039" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/cache#7af060b6f821d7aa82c8ecb0abf6e328a024106a" dependencies = [ "kube", "semver", diff --git a/Cargo.toml b/Cargo.toml index db2b7ab6..ac816adc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,3 +31,5 @@ tracing = "0.1" # [patch."https://github.com/stackabletech/operator-rs.git"] # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/cache" } diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index e02a5158..f55237d2 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -7,6 +7,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::{ affinity::StackableAffinity, + cache::TtlCache, cluster_operation::ClusterOperation, opa::OpaConfig, product_image_selection::ProductImage, @@ -15,7 +16,10 @@ use stackable_operator::{ Resources, ResourcesFragment, }, }, - config::{fragment, fragment::Fragment, fragment::ValidationError, merge::Merge}, + config::{ + fragment::{self, Fragment, ValidationError}, + merge::Merge, + }, k8s_openapi::apimachinery::pkg::api::resource::Quantity, kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt}, memory::{BinaryMultiple, MemoryQuantity}, @@ -267,7 +271,7 @@ pub struct SupersetOpaConfig { pub opa: OpaConfig, /// Configuration for an superset-internal cache for calls to OPA - pub cache: SupersetOpaCacheConfig, + pub cache: TtlCache<30, 1000>, } #[derive(Clone, Deserialize, Serialize, Eq, JsonSchema, Debug, PartialEq)] From a09d2a5bd54bc3c064ba837eea53a300200dae5a Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 22 Jan 2025 09:09:00 +0100 Subject: [PATCH 38/71] Regenerate charts --- deploy/helm/superset-operator/crds/crds.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index 398790f0..db49180f 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -84,11 +84,11 @@ spec: properties: entryTimeToLive: default: 30s - description: Cache duration , e.g. `30m`, `1h` or `2d` + description: Time to live per entry; Entries which were not queried within the given duration, are removed. type: string maxEntries: - default: 128 - description: Number of maximum user-role mappings cached concurrently + default: 1000 + description: Maximum number of entries in the cache; If this threshold is reached then the least recently used item is removed. format: uint32 minimum: 0.0 type: integer From 98a7a55ed77c603762f8df1bbf518722efe3f60b Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 22 Jan 2025 14:10:16 +0100 Subject: [PATCH 39/71] Making pre-commit happy --- .../kuttl/logging/test_log_aggregation.py | 19 ++++++------ tests/templates/kuttl/oidc/login.py | 30 +++++++++---------- tests/templates/kuttl/smoke/metrics.py | 6 ++-- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py index 48105202..7e8c18b3 100755 --- a/tests/templates/kuttl/logging/test_log_aggregation.py +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -23,9 +23,9 @@ def check_sent_events(): }, ) - assert response.status_code == 200, ( - "Cannot access the API of the vector aggregator." - ) + assert ( + response.status_code == 200 + ), "Cannot access the API of the vector aggregator." result = response.json() @@ -35,13 +35,14 @@ def check_sent_events(): componentId = transform["componentId"] if componentId == "filteredInvalidEvents": - assert sentEvents is None or sentEvents["sentEventsTotal"] == 0, ( - "Invalid log events were sent." - ) + assert ( + sentEvents is None or sentEvents["sentEventsTotal"] == 0 + ), "Invalid log events were sent." + else: - assert sentEvents is not None and sentEvents["sentEventsTotal"] > 0, ( - f'No events were sent in "{componentId}".' - ) + assert ( + sentEvents is not None and sentEvents["sentEventsTotal"] > 0 + ), f'No events were sent in "{componentId}".' if __name__ == "__main__": diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index 9b7cef67..8ad0c966 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -28,17 +28,17 @@ ) assert welcome_page.ok, "Login failed" -assert welcome_page.url == "http://superset-external:8088/superset/welcome/", ( - "Redirection to the Superset welcome page expected" -) +assert ( + welcome_page.url == "http://superset-external:8088/superset/welcome/" +), "Redirection to the Superset welcome page expected" # Open the user information page in Superset userinfo_page = session.get("http://superset-external:8088/users/userinfo/") assert userinfo_page.ok, "Retrieving user information failed" -assert userinfo_page.url == "http://superset-external:8088/superset/welcome/", ( - "Redirection to the Superset welcome page expected" -) +assert ( + userinfo_page.url == "http://superset-external:8088/superset/welcome/" +), "Redirection to the Superset welcome page expected" # Expect the user data provided by Keycloak in Superset userinfo_page_html = BeautifulSoup(userinfo_page.text, "html.parser") @@ -46,15 +46,15 @@ data = json.loads(raw_data) user_data = data["user"] -assert user_data["firstName"] == "Jane", ( - "The first name of the user in Superset should match the one provided by Keycloak" -) -assert user_data["lastName"] == "Doe", ( - "The last name of the user in Superset should match the one provided by Keycloak" -) -assert user_data["email"] == "jane.doe@stackable.tech", ( - "The email of the user in Superset should match the one provided by Keycloak" -) +assert ( + user_data["firstName"] == "Jane" +), "The first name of the user in Superset should match the one provided by Keycloak" +assert ( + user_data["lastName"] == "Doe" +), "The last name of the user in Superset should match the one provided by Keycloak" +assert ( + user_data["email"] == "jane.doe@stackable.tech" +), "The email of the user in Superset should match the one provided by Keycloak" # TODO Use different OIDC providers (currently only Keycloak is # supported) diff --git a/tests/templates/kuttl/smoke/metrics.py b/tests/templates/kuttl/smoke/metrics.py index 6c86f2f2..cffaa151 100644 --- a/tests/templates/kuttl/smoke/metrics.py +++ b/tests/templates/kuttl/smoke/metrics.py @@ -11,6 +11,6 @@ metrics_response = requests.get("http://superset-node-default:9102/metrics") assert metrics_response.status_code == 200, "Metrics could not be retrieved." -assert "superset_welcome" in metrics_response.text, ( - "The metrics do not contain the superset_welcome counter." -) +assert ( + "superset_welcome" in metrics_response.text +), "The metrics do not contain the superset_welcome counter." From a7f1bb18ba5ee41d62986bb4196ccd55a5f8aedf Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Tue, 28 Jan 2025 16:00:23 +0100 Subject: [PATCH 40/71] apply typos and formatting corrections Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 2 +- .../superset/pages/usage-guide/security.adoc | 13 ++++++++----- rust/crd/src/lib.rs | 11 ++++++----- rust/operator-binary/src/superset_controller.rs | 1 + tests/templates/kuttl/opa/40_superset.yaml.j2 | 1 - 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e3573bd..32045b27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Run a `containerdebug` process in the background of each Superset container to collect debugging information ([#578]). - Aggregate emitted Kubernetes events on the CustomResources ([#585]). -- Opa Role mapping as optional custom manager for superset ([#582]). +- Support OPA role mapping as optional custom manager for Superset ([#582]). ### Changed diff --git a/docs/modules/superset/pages/usage-guide/security.adoc b/docs/modules/superset/pages/usage-guide/security.adoc index effa2065..a17a5104 100644 --- a/docs/modules/superset/pages/usage-guide/security.adoc +++ b/docs/modules/superset/pages/usage-guide/security.adoc @@ -126,13 +126,17 @@ Further information for specifying an AuthenticationClass for an OIDC provider c Superset has a concept called `Roles` which allows you to grant user permissions based on roles. Have a look at the {superset-security}[Superset documentation on Security^{external-link-icon}^]. -=== [[Opa]] Opa Roles Mapping +=== [[OPA]] OPA role mapping -Superset can sync roles from open policy agent. Currently only mapping is enabled as a larger refactoring of the upstream superset concerning their security management is announced. +Superset can sync roles from OpenPolicyAgent. +Currently only mapping is enabled as a larger refactoring of the upstream Superset concerning their security management is announced. -In order to map roles from Opa into superset, we expect rego rules with a rule name `user_roles`. In the below example two users `admin` and `testuser` have roles defined as rego rule in Rego V1 to be complient with OPA v1.0.0 release. +In order to map roles from OPA into Superset, we expect rego rules with a rule name `user_roles`. +In the below example two users `admin` and `testuser` have roles defined as rego rule in Rego v1 to be compliant with OPA v1.0.0 release. -IMPORTANT: Only role mapping is enabled. Permissions can only be added through the Superset UI. RBAC through OPA is not implemented. +IMPORTANT: Only role mapping is enabled! +Permissions of these roles can only be managed through the Superset UI. +RBAC through OPA is not implemented. OPA rules can be synced using the xref:opa:usage-guide:user-info-fetcher[user-info-fetcher]. @@ -157,7 +161,6 @@ data: roles := user.roles user.username == input.username } -# TODO: User opainfofetcher() users := [ {"username": "admin", "roles": ["Admin", "Test"]}, {"username": "testuser", "roles": ["El_Testos", "Custom2"]} diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index f55237d2..4c2de2b5 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -194,10 +194,11 @@ pub struct SupersetClusterConfig { #[serde(default)] pub authentication: Vec, - /// Authorziation options for Superset. - /// Currently only role mapping is enabled. This means if a user logs in and Opa authorization is enabled - /// user roles got synced from opa into superset roles. Roles get created automated. - /// Warning: This will discard all roles managed by the superset administrator. + /// Authorization options for Superset. + /// + /// Currently only role mapping is supported. This means if a user logs in and OPA role mapping is enabled, + /// user roles got synced from OPA into Superset. Roles in Superset get created automatically. + /// Warning: This will discard all roles previously assigned to the user. #[serde(skip_serializing_if = "Option::is_none")] pub authorization: Option, @@ -270,7 +271,7 @@ pub struct SupersetOpaConfig { #[serde(flatten)] pub opa: OpaConfig, - /// Configuration for an superset-internal cache for calls to OPA + /// Configuration for an Superset internal cache for calls to OPA pub cache: TtlCache<30, 1000>, } diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 223824bf..8b06dbb1 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -287,6 +287,7 @@ pub enum Error { InvalidSupersetCluster { source: error_boundary::InvalidObject, }, + #[snafu(display("invalid OpaConfig"))] InvalidOpaConfig { source: stackable_operator::commons::opa::Error, diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index 47ee0881..b7778b45 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -28,7 +28,6 @@ spec: opa: configMapName: opa package: superset - cache: {} credentialsSecret: superset-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery From 48c8a5294cd85edb45ba73639e3770bfe0c108fb Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 30 Jan 2025 10:34:41 +0100 Subject: [PATCH 41/71] update chart --- tests/templates/kuttl/oidc/40-install-superset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/oidc/40-install-superset.yaml b/tests/templates/kuttl/oidc/40-install-superset.yaml index 0cba3c7b..c97f66e8 100644 --- a/tests/templates/kuttl/oidc/40-install-superset.yaml +++ b/tests/templates/kuttl/oidc/40-install-superset.yaml @@ -4,5 +4,5 @@ kind: TestStep timeout: 300 commands: - script: > - envsubst '$NAMESPACE' < install-superset.yaml | + envsubst '$NAMESPACE' < 40_install-superset.yaml | kubectl apply -n $NAMESPACE -f - From 8ed619440c4aa483436ad243fc30c46fb7e86fe7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 31 Jan 2025 18:40:02 +0100 Subject: [PATCH 42/71] adress feedback in PR and rename envs --- CHANGELOG.md | 2 +- Cargo.lock | 8 ++--- Cargo.toml | 4 +-- deploy/helm/superset-operator/crds/crds.yaml | 15 ++++---- rust/crd/src/lib.rs | 24 ++++++------- rust/operator-binary/src/authorization/opa.rs | 35 +++++++++---------- .../src/superset_controller.rs | 7 ++-- tests/templates/kuttl/oidc/60-assert.yaml | 2 +- tests/templates/kuttl/oidc/60-login.yaml | 2 +- .../kuttl/oidc/{login.py => 60_login.py} | 0 tests/templates/kuttl/opa/40_superset.yaml.j2 | 1 + ...ser-roles-cm => 50-get-user-roles-cm.yaml} | 2 +- ...get_user_roles.py => 50_get_user_roles.py} | 0 .../kuttl/opa/52-test-user-roles.yaml.j2 | 2 +- 14 files changed, 52 insertions(+), 52 deletions(-) rename tests/templates/kuttl/oidc/{login.py => 60_login.py} (100%) rename tests/templates/kuttl/opa/{50-get-user-roles-cm => 50-get-user-roles-cm.yaml} (83%) rename tests/templates/kuttl/opa/{get_user_roles.py => 50_get_user_roles.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32045b27..6a944795 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Run a `containerdebug` process in the background of each Superset container to collect debugging information ([#578]). - Aggregate emitted Kubernetes events on the CustomResources ([#585]). -- Support OPA role mapping as optional custom manager for Superset ([#582]). +- Support OPA role mapping as optional custom security manager for Superset ([#582]). ### Changed diff --git a/Cargo.lock b/Cargo.lock index 6df57b4a..a9062843 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2331,8 +2331,8 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stackable-operator" -version = "0.84.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/cache#7af060b6f821d7aa82c8ecb0abf6e328a024106a" +version = "0.86.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.86.0#e5bfee596cc918b05f3e1d7e667c25951317cf31" dependencies = [ "chrono", "clap", @@ -2370,7 +2370,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/cache#7af060b6f821d7aa82c8ecb0abf6e328a024106a" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.86.0#e5bfee596cc918b05f3e1d7e667c25951317cf31" dependencies = [ "darling", "proc-macro2", @@ -2381,7 +2381,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat/cache#7af060b6f821d7aa82c8ecb0abf6e328a024106a" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.86.0#e5bfee596cc918b05f3e1d7e667c25951317cf31" dependencies = [ "kube", "semver", diff --git a/Cargo.toml b/Cargo.toml index ac816adc..3c7b54e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.84.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.86.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" @@ -31,5 +31,3 @@ tracing = "0.1" # [patch."https://github.com/stackabletech/operator-rs.git"] # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } -[patch."https://github.com/stackabletech/operator-rs.git"] -stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/cache" } diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index db49180f..fbfc7b1f 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -72,7 +72,10 @@ spec: type: object type: array authorization: - description: 'Authorziation options for Superset. Currently only role mapping is enabled. This means if a user logs in and Opa authorization is enabled user roles got synced from opa into superset roles. Roles get created automated. Warning: This will discard all roles managed by the superset administrator.' + description: |- + Authorization options for Superset. + + Currently only role mapping is supported. This means if a user logs in and OPA role mapping is enabled, user roles got synced from OPA into Superset. Roles in Superset get created automatically. Warning: This will discard all roles previously assigned to the user. nullable: true properties: opa: @@ -80,14 +83,14 @@ spec: nullable: true properties: cache: - description: Configuration for an superset-internal cache for calls to OPA + description: Configuration for an Superset internal cache for calls to OPA properties: entryTimeToLive: default: 30s - description: Time to live per entry; Entries which were not queried within the given duration, are removed. + description: Time to live per entry type: string maxEntries: - default: 1000 + default: 10000 description: Maximum number of entries in the cache; If this threshold is reached then the least recently used item is removed. format: uint32 minimum: 0.0 @@ -164,7 +167,7 @@ spec: Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details. properties: custom: - description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0` + description: Overwrite the docker image. Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` type: string productVersion: description: Version of the product, e.g. `1.4.1`. @@ -191,7 +194,7 @@ spec: nullable: true type: array repo: - description: Name of the docker repo, e.g. `docker.stackable.tech/stackable` + description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` nullable: true type: string stackableVersion: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 4c2de2b5..caa6916e 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -7,7 +7,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::{ affinity::StackableAffinity, - cache::TtlCache, + cache::UserInformationCache, cluster_operation::ClusterOperation, opa::OpaConfig, product_image_selection::ProductImage, @@ -91,11 +91,11 @@ pub enum SupersetConfigOptions { AuthLdapTlsKeyfile, AuthLdapTlsCacertfile, CustomSecurityManager, - StackableOpaBaseUrl, - StackableOpaPackage, - StackableOpaRule, - StackableOpaCacheMaxEntries, - StackableOpaCacheEntryTTL, + AuthOpaRequestUrl, + AuthOpaPackage, + AuthOpaRule, + AuthOpaCacheMaxEntries, + AuthOpaCacheTtlInSec, } impl SupersetConfigOptions { @@ -146,11 +146,11 @@ impl FlaskAppConfigOptions for SupersetConfigOptions { SupersetConfigOptions::AuthLdapTlsCacertfile => PythonType::StringLiteral, // Configuration options used by CustomOpaSecurityManager SupersetConfigOptions::CustomSecurityManager => PythonType::Expression, - SupersetConfigOptions::StackableOpaBaseUrl => PythonType::StringLiteral, - SupersetConfigOptions::StackableOpaPackage => PythonType::StringLiteral, - SupersetConfigOptions::StackableOpaRule => PythonType::StringLiteral, - SupersetConfigOptions::StackableOpaCacheMaxEntries => PythonType::IntLiteral, - SupersetConfigOptions::StackableOpaCacheEntryTTL => PythonType::IntLiteral, + SupersetConfigOptions::AuthOpaRequestUrl => PythonType::StringLiteral, + SupersetConfigOptions::AuthOpaPackage => PythonType::StringLiteral, + SupersetConfigOptions::AuthOpaRule => PythonType::StringLiteral, + SupersetConfigOptions::AuthOpaCacheMaxEntries => PythonType::IntLiteral, + SupersetConfigOptions::AuthOpaCacheTtlInSec => PythonType::IntLiteral, } } } @@ -272,7 +272,7 @@ pub struct SupersetOpaConfig { pub opa: OpaConfig, /// Configuration for an Superset internal cache for calls to OPA - pub cache: TtlCache<30, 1000>, + pub cache: UserInformationCache, } #[derive(Clone, Deserialize, Serialize, Eq, JsonSchema, Debug, PartialEq)] diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 6b5babde..6427b306 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -41,32 +41,31 @@ impl SupersetOpaConfigResolved { } // Adding necessary configurations. Imports are solved in config.rs - pub fn as_config(&self) -> BTreeMap> { - BTreeMap::from([ + pub fn as_config(&self) -> BTreeMap { + let mut config = BTreeMap::from([ ( "CUSTOM_SECURITY_MANAGER".to_string(), - Some("OpaSupersetSecurityManager".to_string()), + "OpaSupersetSecurityManager".to_string(), ), ( - "STACKABLE_OPA_RULE".to_string(), - Some("user_roles".to_string()), + "AUTH_OPA_REQUEST_URL".to_string(), + self.opa_base_url.to_owned(), ), ( - "STACKABLE_OPA_BASE_URL".to_string(), - Some(self.opa_base_url.to_owned()), + "AUTH_OPA_CACHE_MAX_ENTRIES".to_string(), + self.cache_max_entries.to_string(), ), ( - "STACKABLE_OPA_PACKAGE".to_string(), - self.opa_package.to_owned(), + "AUTH_OPA_CACHE_TTL_IN_SEC".to_string(), + self.cache_ttl.as_secs().to_string(), ), - ( - "STACKABLE_OPA_CACHE_MAX_ENTRIES".to_string(), - Some(self.cache_max_entries.to_string()), - ), - ( - "STACKABLE_OPA_CACHE_ENTRY_TTL".to_string(), - Some(self.cache_ttl.as_secs().to_string()), - ), - ]) + ("AUTH_OPA_RULE".to_string(), "user_roles".to_string()), + ]); + + if let Some(opa_package) = &self.opa_package { + config.insert("AUTH_OPA_PACKAGE".to_string(), opa_package.to_owned()); + } + + config } } diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 8b06dbb1..c72f17aa 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -577,11 +577,10 @@ fn build_rolegroup_config_map( // Adding opa configuration properties to config_properties. // This will be injected as key/value pair in superset_config.py if let Some(opa_config) = superset_opa_config { - for (k, v) in opa_config.as_config() { - config_properties.insert(k, v.unwrap_or_default()); - } // If opa role mapping is configured, insert CustomOpaSecurityManager import - imports.append(&mut (*OPA_IMPORTS).to_vec()) + imports.extend(OPA_IMPORTS); + + config_properties.extend(opa_config.as_config()); } // The order here should be kept in order to preserve overrides. diff --git a/tests/templates/kuttl/oidc/60-assert.yaml b/tests/templates/kuttl/oidc/60-assert.yaml index fee66ceb..df17232b 100644 --- a/tests/templates/kuttl/oidc/60-assert.yaml +++ b/tests/templates/kuttl/oidc/60-assert.yaml @@ -5,4 +5,4 @@ metadata: name: login timeout: 300 commands: - - script: kubectl exec -n $NAMESPACE python-0 -- python /stackable/login.py + - script: kubectl exec -n $NAMESPACE python-0 -- python /stackable/60_login.py diff --git a/tests/templates/kuttl/oidc/60-login.yaml b/tests/templates/kuttl/oidc/60-login.yaml index 0745bc4b..e5ca3052 100644 --- a/tests/templates/kuttl/oidc/60-login.yaml +++ b/tests/templates/kuttl/oidc/60-login.yaml @@ -6,4 +6,4 @@ metadata: commands: - script: > envsubst '$NAMESPACE' < login.py | - kubectl exec -n $NAMESPACE -i python-0 -- tee /stackable/login.py > /dev/null + kubectl exec -n $NAMESPACE -i python-0 -- tee /stackable/60_login.py > /dev/null diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/60_login.py similarity index 100% rename from tests/templates/kuttl/oidc/login.py rename to tests/templates/kuttl/oidc/60_login.py diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index b7778b45..47ee0881 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -28,6 +28,7 @@ spec: opa: configMapName: opa package: superset + cache: {} credentialsSecret: superset-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery diff --git a/tests/templates/kuttl/opa/50-get-user-roles-cm b/tests/templates/kuttl/opa/50-get-user-roles-cm.yaml similarity index 83% rename from tests/templates/kuttl/opa/50-get-user-roles-cm rename to tests/templates/kuttl/opa/50-get-user-roles-cm.yaml index 16a9ca20..9d00b1c7 100644 --- a/tests/templates/kuttl/opa/50-get-user-roles-cm +++ b/tests/templates/kuttl/opa/50-get-user-roles-cm.yaml @@ -4,4 +4,4 @@ kind: TestStep timeout: 300 commands: - script: > - kubectl create cm get-user-roles-script -n $NAMESPACE --from-file get_user_roles.py + kubectl create cm get-user-roles-script -n $NAMESPACE --from-file 50_get_user_roles.py diff --git a/tests/templates/kuttl/opa/get_user_roles.py b/tests/templates/kuttl/opa/50_get_user_roles.py similarity index 100% rename from tests/templates/kuttl/opa/get_user_roles.py rename to tests/templates/kuttl/opa/50_get_user_roles.py diff --git a/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 b/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 index bbbd3c8d..4ed895ba 100644 --- a/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 +++ b/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 @@ -4,4 +4,4 @@ commands: # Need to wait until the superset API is ready before running the script - script: | sleep 30 - kubectl exec -n $NAMESPACE python-0 -i -- python /tmp/scripts/get_user_roles.py $NAMESPACE Admin,Test + kubectl exec -n $NAMESPACE python-0 -i -- python /tmp/scripts/50_get_user_roles.py $NAMESPACE Admin,Test From 4b23391540efaed739ca202d93e1d6b7d97672f6 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 31 Jan 2025 18:59:34 +0100 Subject: [PATCH 43/71] fix changelog --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e19e6479..9cc7040b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,6 @@ - Default to OCI for image metadata and product image selection ([#586]). -[#578]: https://github.com/stackabletech/superset-operator/pull/578 -[#585]: https://github.com/stackabletech/superset-operator/pull/585 -[#586]: https://github.com/stackabletech/superset-operator/pull/586 - ## [24.11.1] - 2025-01-10 ### Fixed @@ -30,6 +26,7 @@ [#578]: https://github.com/stackabletech/superset-operator/pull/578 [#582]: https://github.com/stackabletech/superset-operator/pull/582 [#585]: https://github.com/stackabletech/superset-operator/pull/585 +[#586]: https://github.com/stackabletech/superset-operator/pull/586 ## [24.11.0] - 2024-11-18 From 161ac74dcd60dec24b9bba10c50ee36782d34bbf Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Feb 2025 10:30:26 +0100 Subject: [PATCH 44/71] Update opa tests. --- tests/release.yaml | 2 ++ tests/templates/kuttl/opa/31-opa-rego.yaml | 2 -- tests/test-definition.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/release.yaml b/tests/release.yaml index 76b389a8..f757f9fe 100644 --- a/tests/release.yaml +++ b/tests/release.yaml @@ -16,3 +16,5 @@ releases: operatorVersion: 0.0.0-dev superset: operatorVersion: 0.0.0-dev + opa: + operatorVersion: 0.0.0-dev diff --git a/tests/templates/kuttl/opa/31-opa-rego.yaml b/tests/templates/kuttl/opa/31-opa-rego.yaml index c60324be..18f1d3dd 100644 --- a/tests/templates/kuttl/opa/31-opa-rego.yaml +++ b/tests/templates/kuttl/opa/31-opa-rego.yaml @@ -9,8 +9,6 @@ data: roles.rego: | package superset - import rego.v1 - user_roles := roles if { group_paths := data.stackable.opa.userinfo.v1.userInfoByUsername(input.username).groups roles := [ trim(group,"/") | group := group_paths[_] ] diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 79bfe280..013af1cb 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -18,7 +18,7 @@ dimensions: - server-verification-tls - name: opa values: - - 0.66.0 + - 1.0.1 - name: openshift values: - "false" From ef001e7ede81623f21a4d0098c63e2ec36b46688 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 21 Feb 2025 10:12:35 +0100 Subject: [PATCH 45/71] support custom image for opa tests --- tests/templates/kuttl/opa/40_superset.yaml.j2 | 5 +++++ tests/test-definition.yaml | 1 + 2 files changed, 6 insertions(+) diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index 47ee0881..7de84db7 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -21,7 +21,12 @@ metadata: spec: image: custom: docker.stackable.tech/stackable/superset:4.0.2-stackable0.0.0-dev-opa +{% if test_scenario['values']['superset'].find(",") > 0 %} + custom: "{{ test_scenario['values']['superset'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['superset'].split(',')[0] }}" +{% else %} productVersion: "{{ test_scenario['values']['superset'] }}" +{% endif %} pullPolicy: IfNotPresent clusterConfig: authorization: diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 2c9e3a06..d51480bd 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -13,6 +13,7 @@ dimensions: - name: superset-latest values: - 4.1.1 + # - 4.1.1,oci.stackable.tech/razvan/superset:4.1.1-stackable0.0.0-dev - name: ldap-authentication values: - no-tls From ff0ee4646863d4f3b2c625f423e35f9f5241889c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:31:32 +0100 Subject: [PATCH 46/71] create and assign new role via API --- tests/templates/kuttl/opa/40_superset.yaml.j2 | 19 ++ .../templates/kuttl/opa/50_get_user_roles.py | 172 +++++++++++++----- .../kuttl/opa/52-test-user-roles.yaml.j2 | 2 +- 3 files changed, 150 insertions(+), 43 deletions(-) diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index 7de84db7..b2683d19 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -39,9 +39,28 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} nodes: + configOverrides: + superset_config.py: + # Enable the security API to be able to create roles from the test + FAB_ADD_SECURITY_API: "True" + # Enable FAB logging + SILENCE_FAB: "False" config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + containers: + superset: + console: + level: DEBUG + file: + level: DEBUG + loggers: + ROOT: + level: DEBUG + flask_appbuilder.security: + level: DEBUG + opa_authorizer: + level: DEBUG roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/opa/50_get_user_roles.py b/tests/templates/kuttl/opa/50_get_user_roles.py index c397eec3..1b6d9693 100644 --- a/tests/templates/kuttl/opa/50_get_user_roles.py +++ b/tests/templates/kuttl/opa/50_get_user_roles.py @@ -1,48 +1,136 @@ import logging import sys -import requests +import requests from bs4 import BeautifulSoup -logging.basicConfig( - level="DEBUG", format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout -) - -namespace = sys.argv[1] -expected_roles = set(sys.argv[2].split(",")) - -session = requests.Session() - -# Click on "Login" in Superset -login_page = session.get( - f"http://superset-external.{namespace}.svc.cluster.local:8088/login/" -) -assert login_page.status_code == 200 - -login_page_html = BeautifulSoup(login_page.text, "html.parser") -csrf_token = login_page_html.find("input", {"id": "csrf_token"})["value"] - -# Login with CSRF token -welcome_page = session.post( - f"http://superset-external.{namespace}.svc.cluster.local:8088/login/", - data={"username": "admin", "password": "admin", "csrf_token": csrf_token}, -) -assert welcome_page.status_code == 200 -logging.debug(welcome_page.url) - -# Call an API that will trigger an update of user roles -session.get( - f"http://superset-external.{namespace}.svc.cluster.local:8088/api/v1/dashboard/" -) - -# Get user roles -user_roles = set( - session.get( - f"http://superset-external.{namespace}.svc.cluster.local:8088/api/v1/me/roles/" +base_ui_url = "" +base_api_url = "" +bearer_token = "" +csrf_token = "" + + +def get_bearer_token(): + payload = {"password": "admin", "provider": "db", "username": "admin"} + headers = {"Content-Type": "application/json"} + response = requests.request( + "POST", f"{base_api_url}/security/login", json=payload, headers=headers + ) + return response.json()["access_token"] + + +def get_csrf_token(): + headers = {"Authorization": f"Bearer {bearer_token}"} + response = requests.request( + "GET", f"{base_api_url}/security/csrf_token/", data="", headers=headers + ) + return response.json()["result"] + + +def add_role(name: str): + headers = { + "X-CSRFToken": csrf_token, + "Authorization": f"Bearer {bearer_token}", + } + response = requests.request( + "POST", f"{base_api_url}/security/roles", json={"name": name}, headers=headers + ) + return response.json() + + +def add_permissions_to_role(role_id, permissions): + headers = { + "X-CSRFToken": csrf_token, + "Authorization": f"Bearer {bearer_token}", + } + response = requests.request( + "POST", + f"{base_api_url}/security/roles/{role_id}/permissions", + json={"permission_view_menu_ids": permissions}, + headers=headers, + ) + return response.json() + + +def get_roles(): + headers = { + "X-CSRFToken": csrf_token, + "Authorization": f"Bearer {bearer_token}", + } + + return requests.request( + "GET", f"{base_api_url}/security/users/1", headers=headers + ).json()["result"]["roles"] + + +def set_user_roles(roles: list[int]): + headers = { + "X-CSRFToken": csrf_token, + "Authorization": f"Bearer {bearer_token}", + } + result = requests.request( + "PUT", + f"{base_api_url}/security/users/1", + headers=headers, + json={"username": "admin", "password": "admin", "roles": roles}, + ).json() + + logging.info(f"Result of setting the roles {roles} to user: {result}") + + +def get_ui_roles() -> list[str]: + session = requests.Session() + + # Click on "Login" in Superset + login_page = session.get(f"{base_ui_url}/login/") + assert login_page.status_code == 200 + + login_page_html = BeautifulSoup(login_page.text, "html.parser") + csrf_token = login_page_html.find("input", {"id": "csrf_token"})["value"] + + # Login with CSRF token + welcome_page = session.post( + f"{base_ui_url}/login/", + data={"username": "admin", "password": "admin", "csrf_token": csrf_token}, + ) + assert welcome_page.status_code == 200 + logging.debug(welcome_page.url) + + return session.get(f"{base_api_url}/me/roles/").json()["result"]["roles"].keys() + + +def main(): + logging.basicConfig( + level="DEBUG", + format="%(asctime)s %(levelname)s: %(message)s", + stream=sys.stdout, ) - .json()["result"]["roles"] - .keys() -) -logging.debug("User roles: %s", user_roles) -logging.debug("Expected roles: %s", expected_roles) -assert user_roles == expected_roles + + namespace = sys.argv[1] + + global base_ui_url + global base_api_url + global bearer_token + global csrf_token + + base_ui_url = f"http://superset-external.{namespace}.svc.cluster.local:8088" + base_api_url = f"http://superset-external.{namespace}.svc.cluster.local:8088/api/v1" + bearer_token = get_bearer_token() + csrf_token = get_csrf_token() + + add_role("Test") + add_permissions_to_role(6, list(range(3))) + set_user_roles([1, 6]) + api_user_roles = [role["name"] for role in get_roles()] + ui_user_roles = get_ui_roles() + + expected_roles = ["Admin", "Test"] + logging.debug("Expected roles: {expected_roles}") + logging.debug("Got API user roles: {api_user_roles}") + logging.debug("Got UI user roles: {ui_user_roles}") + assert api_user_roles == ui_user_roles + assert expected_roles == ui_user_roles + + +if __name__ == "__main__": + main() diff --git a/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 b/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 index 4ed895ba..57f54032 100644 --- a/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 +++ b/tests/templates/kuttl/opa/52-test-user-roles.yaml.j2 @@ -4,4 +4,4 @@ commands: # Need to wait until the superset API is ready before running the script - script: | sleep 30 - kubectl exec -n $NAMESPACE python-0 -i -- python /tmp/scripts/50_get_user_roles.py $NAMESPACE Admin,Test + kubectl exec -n $NAMESPACE python-0 -i -- python /tmp/scripts/50_get_user_roles.py $NAMESPACE From c0f2368ac5564b1fe73105cebeb6f513c546d554 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:43:16 +0100 Subject: [PATCH 47/71] fix typos --- tests/templates/kuttl/opa/50_get_user_roles.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/opa/50_get_user_roles.py b/tests/templates/kuttl/opa/50_get_user_roles.py index 1b6d9693..a40f7f0b 100644 --- a/tests/templates/kuttl/opa/50_get_user_roles.py +++ b/tests/templates/kuttl/opa/50_get_user_roles.py @@ -96,7 +96,9 @@ def get_ui_roles() -> list[str]: assert welcome_page.status_code == 200 logging.debug(welcome_page.url) - return session.get(f"{base_api_url}/me/roles/").json()["result"]["roles"].keys() + return list( + session.get(f"{base_api_url}/me/roles/").json()["result"]["roles"].keys() + ) def main(): @@ -125,9 +127,9 @@ def main(): ui_user_roles = get_ui_roles() expected_roles = ["Admin", "Test"] - logging.debug("Expected roles: {expected_roles}") - logging.debug("Got API user roles: {api_user_roles}") - logging.debug("Got UI user roles: {ui_user_roles}") + logging.debug(f"Expected roles: {expected_roles}") + logging.debug(f"Got API user roles: {api_user_roles}") + logging.debug(f"Got UI user roles: {ui_user_roles}") assert api_user_roles == ui_user_roles assert expected_roles == ui_user_roles From d02e7bb8c00c77ed275bd6df741197430e9c21c7 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:03:19 +0100 Subject: [PATCH 48/71] add some comments --- tests/templates/kuttl/opa/50_get_user_roles.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/templates/kuttl/opa/50_get_user_roles.py b/tests/templates/kuttl/opa/50_get_user_roles.py index a40f7f0b..b7eedde9 100644 --- a/tests/templates/kuttl/opa/50_get_user_roles.py +++ b/tests/templates/kuttl/opa/50_get_user_roles.py @@ -1,3 +1,10 @@ +# +# Use the FAB security API to create a new role named "Test" and assign it to the "admin" user. +# +# Use the UI to login and fetch all roles assigned to the user that is logged in (admin). +# +# Compare that the FAB API roles and the UI roles (that are resolved from OPA) are the same. +# import logging import sys @@ -120,9 +127,15 @@ def main(): bearer_token = get_bearer_token() csrf_token = get_csrf_token() + # Create a new role and assign some permissions to it add_role("Test") add_permissions_to_role(6, list(range(3))) + + # Add the new role to the admin user. + # "1" is the existing "Admin" role id. + # "6" is the id of the new "Test" role. set_user_roles([1, 6]) + api_user_roles = [role["name"] for role in get_roles()] ui_user_roles = get_ui_roles() From 472817f99fa5afc0fb5fd52bc3928f6d09f642b4 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Sat, 22 Feb 2025 22:15:12 +0100 Subject: [PATCH 49/71] opa kuttl test is green (again) --- tests/templates/kuttl/opa/50_get_user_roles.py | 11 +++-------- .../kuttl/smoke/30-install-superset.yaml.j2 | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/templates/kuttl/opa/50_get_user_roles.py b/tests/templates/kuttl/opa/50_get_user_roles.py index b7eedde9..49980236 100644 --- a/tests/templates/kuttl/opa/50_get_user_roles.py +++ b/tests/templates/kuttl/opa/50_get_user_roles.py @@ -103,6 +103,9 @@ def get_ui_roles() -> list[str]: assert welcome_page.status_code == 200 logging.debug(welcome_page.url) + # Force roles to be loaded by the OPA security manager + session.get(f"{base_api_url}/dashboard/") + return list( session.get(f"{base_api_url}/me/roles/").json()["result"]["roles"].keys() ) @@ -131,19 +134,11 @@ def main(): add_role("Test") add_permissions_to_role(6, list(range(3))) - # Add the new role to the admin user. - # "1" is the existing "Admin" role id. - # "6" is the id of the new "Test" role. - set_user_roles([1, 6]) - - api_user_roles = [role["name"] for role in get_roles()] ui_user_roles = get_ui_roles() expected_roles = ["Admin", "Test"] logging.debug(f"Expected roles: {expected_roles}") - logging.debug(f"Got API user roles: {api_user_roles}") logging.debug(f"Got UI user roles: {ui_user_roles}") - assert api_user_roles == ui_user_roles assert expected_roles == ui_user_roles diff --git a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 index 872202e5..c9897f77 100644 --- a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 @@ -44,6 +44,17 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + containers: + superset: + console: + level: DEBUG + file: + level: DEBUG + loggers: + ROOT: + level: DEBUG + flask_appbuilder.security: + level: DEBUG configOverrides: superset_config.py: EXPERIMENTAL_FILE_HEADER: | @@ -51,6 +62,10 @@ spec: ROLE_HEADER_VAR = "role-value" EXPERIMENTAL_FILE_FOOTER: | ROLE_FOOTER_VAR = "role-value" + # Enable the security API to be able to create roles from the test + FAB_ADD_SECURITY_API: "True" + # Enable FAB logging + SILENCE_FAB: "False" roleGroups: default: replicas: 1 From 58856849c41f0a9e02ca1fc2ba536a2d9eee84e0 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Sun, 23 Feb 2025 10:33:16 +0100 Subject: [PATCH 50/71] silence most of Pyright errors and warnings --- .../templates/kuttl/opa/50_get_user_roles.py | 73 ++++++++----------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/tests/templates/kuttl/opa/50_get_user_roles.py b/tests/templates/kuttl/opa/50_get_user_roles.py index 49980236..7765d526 100644 --- a/tests/templates/kuttl/opa/50_get_user_roles.py +++ b/tests/templates/kuttl/opa/50_get_user_roles.py @@ -1,9 +1,7 @@ # -# Use the FAB security API to create a new role named "Test" and assign it to the "admin" user. -# -# Use the UI to login and fetch all roles assigned to the user that is logged in (admin). -# -# Compare that the FAB API roles and the UI roles (that are resolved from OPA) are the same. +# Use the FAB security API to create a new role named "Test". +# Use the UI to login and fetch the user roles (that are resolved from OPA). +# Check that that the user has both the "Admin" role as well as the newly created "Test" role. # import logging import sys @@ -17,21 +15,25 @@ csrf_token = "" -def get_bearer_token(): +def get_bearer_token() -> str: payload = {"password": "admin", "provider": "db", "username": "admin"} headers = {"Content-Type": "application/json"} response = requests.request( "POST", f"{base_api_url}/security/login", json=payload, headers=headers ) - return response.json()["access_token"] + json: dict[str, object] = response.json() + logging.info(f"get_bearer_token response {json}") + return str(json["access_token"]) -def get_csrf_token(): +def get_csrf_token() -> str: headers = {"Authorization": f"Bearer {bearer_token}"} response = requests.request( "GET", f"{base_api_url}/security/csrf_token/", data="", headers=headers ) - return response.json()["result"] + json: dict[str, object] = response.json() + logging.info(f"get_csrf_token response {json}") + return str(json["result"]) def add_role(name: str): @@ -42,10 +44,11 @@ def add_role(name: str): response = requests.request( "POST", f"{base_api_url}/security/roles", json={"name": name}, headers=headers ) - return response.json() + json: dict[str, object] = response.json() + logging.info(f"add_role response {json}") -def add_permissions_to_role(role_id, permissions): +def add_permissions_to_role(role_id: int, permissions: list[int]): headers = { "X-CSRFToken": csrf_token, "Authorization": f"Bearer {bearer_token}", @@ -56,33 +59,8 @@ def add_permissions_to_role(role_id, permissions): json={"permission_view_menu_ids": permissions}, headers=headers, ) - return response.json() - - -def get_roles(): - headers = { - "X-CSRFToken": csrf_token, - "Authorization": f"Bearer {bearer_token}", - } - - return requests.request( - "GET", f"{base_api_url}/security/users/1", headers=headers - ).json()["result"]["roles"] - - -def set_user_roles(roles: list[int]): - headers = { - "X-CSRFToken": csrf_token, - "Authorization": f"Bearer {bearer_token}", - } - result = requests.request( - "PUT", - f"{base_api_url}/security/users/1", - headers=headers, - json={"username": "admin", "password": "admin", "roles": roles}, - ).json() - - logging.info(f"Result of setting the roles {roles} to user: {result}") + json: dict[str, object] = response.json() + logging.info(f"add_permissions_to_role response {json}") def get_ui_roles() -> list[str]: @@ -93,7 +71,11 @@ def get_ui_roles() -> list[str]: assert login_page.status_code == 200 login_page_html = BeautifulSoup(login_page.text, "html.parser") - csrf_token = login_page_html.find("input", {"id": "csrf_token"})["value"] + csrf_token = login_page_html.find("input", id="csrf_token") + if csrf_token is None: + raise Exception("CSRF token not found in on the login page") + else: + csrf_token = csrf_token["value"] # Login with CSRF token welcome_page = session.post( @@ -104,11 +86,15 @@ def get_ui_roles() -> list[str]: logging.debug(welcome_page.url) # Force roles to be loaded by the OPA security manager - session.get(f"{base_api_url}/dashboard/") + # Assign to _ to shut up type checker + _ = session.get(f"{base_api_url}/dashboard/") - return list( - session.get(f"{base_api_url}/me/roles/").json()["result"]["roles"].keys() - ) + response = session.get(f"{base_api_url}/me/roles/") + json: dict[str, object] = response.json() + + logging.info(f"get_ui_roles response {json}") + + return list(json["result"]["roles"].keys()) def main(): @@ -132,6 +118,7 @@ def main(): # Create a new role and assign some permissions to it add_role("Test") + # "6" is the new role id (Superset has 5 builtin roles) add_permissions_to_role(6, list(range(3))) ui_user_roles = get_ui_roles() From 434232f93a90411c96f09611fb5fc8df107c4811 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 25 Feb 2025 12:16:45 +0100 Subject: [PATCH 51/71] Update rust/crd/src/lib.rs Co-authored-by: Sebastian Bernauer --- rust/crd/src/lib.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index dd349527..a37f4021 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -276,25 +276,6 @@ pub struct SupersetOpaConfig { pub cache: UserInformationCache, } -#[derive(Clone, Deserialize, Serialize, Eq, JsonSchema, Debug, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct SupersetOpaCacheConfig { - /// Cache duration , e.g. `30m`, `1h` or `2d` - #[serde(default = "cache_ttl_default")] - pub entry_time_to_live: Duration, - - /// Number of maximum user-role mappings cached concurrently - #[serde(default = "cache_max_entries_default")] - pub max_entries: u32, -} - -fn cache_max_entries_default() -> u32 { - 128 -} - -fn cache_ttl_default() -> Duration { - Duration::from_secs(30) -} #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] From 93819b5b8810b5177268fe2e74005c9cabe74e86 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 25 Feb 2025 11:38:37 +0100 Subject: [PATCH 52/71] Renaming fields and structs --- deploy/helm/superset-operator/crds/crds.yaml | 4 ++-- rust/crd/src/lib.rs | 17 ++++++++++------- rust/operator-binary/src/authorization/opa.rs | 4 ++-- tests/templates/kuttl/opa/40_superset.yaml.j2 | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index fbfc7b1f..243bb09d 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -75,10 +75,10 @@ spec: description: |- Authorization options for Superset. - Currently only role mapping is supported. This means if a user logs in and OPA role mapping is enabled, user roles got synced from OPA into Superset. Roles in Superset get created automatically. Warning: This will discard all roles previously assigned to the user. + Currently only role assignment is supported. This means that roles are assigned to users in OPA but, due to the way Superset is implemented, the database also needs to be updated to reflect these assignments. Therefore, user roles and permissions must already exist in the Superset database before they can be assigned to a user. Warning: Any user roles assigned with the Superset UI are discarded. nullable: true properties: - opa: + roleMappingFromOpa: description: Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) and the name of the Rego package containing your authorization rules. Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) to learn how to deploy Rego authorization rules with OPA. nullable: true properties: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index a37f4021..645cfc51 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -197,9 +197,12 @@ pub struct SupersetClusterConfig { /// Authorization options for Superset. /// - /// Currently only role mapping is supported. This means if a user logs in and OPA role mapping is enabled, - /// user roles got synced from OPA into Superset. Roles in Superset get created automatically. - /// Warning: This will discard all roles previously assigned to the user. + /// Currently only role assignment is supported. This means that roles are assigned to users in + /// OPA but, due to the way Superset is implemented, the database also needs to be updated + /// to reflect these assignments. + /// Therefore, user roles and permissions must already exist in the Superset database before + /// they can be assigned to a user. + /// Warning: Any user roles assigned with the Superset UI are discarded. #[serde(skip_serializing_if = "Option::is_none")] pub authorization: Option, @@ -268,7 +271,7 @@ impl CurrentlySupportedListenerClasses { } #[derive(Clone, Deserialize, Serialize, Eq, JsonSchema, Debug, PartialEq)] #[serde(rename_all = "camelCase")] -pub struct SupersetOpaConfig { +pub struct SupersetOpaRoleMappingConfig { #[serde(flatten)] pub opa: OpaConfig, @@ -280,7 +283,7 @@ pub struct SupersetOpaConfig { #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct SupersetAuthorization { - pub opa: Option, + pub role_mapping_from_opa: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -516,12 +519,12 @@ impl SupersetCluster { } } - pub fn get_opa_config(&self) -> Option<&SupersetOpaConfig> { + pub fn get_opa_config(&self) -> Option<&SupersetOpaRoleMappingConfig> { self.spec .cluster_config .authorization .as_ref() - .and_then(|a| a.opa.as_ref()) + .and_then(|a| a.role_mapping_from_opa.as_ref()) } /// Retrieve and merge resource configs for role and role groups diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 5e87199f..e12da47c 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use stackable_operator::{client::Client, commons::opa::OpaApiVersion, time::Duration}; -use stackable_superset_crd::{SupersetCluster, SupersetOpaConfig}; +use stackable_superset_crd::{SupersetCluster, SupersetOpaRoleMappingConfig}; pub struct SupersetOpaConfigResolved { opa_base_url: String, @@ -14,7 +14,7 @@ impl SupersetOpaConfigResolved { pub async fn from_opa_config( client: &Client, superset: &SupersetCluster, - opa_config: &SupersetOpaConfig, + opa_config: &SupersetOpaRoleMappingConfig, ) -> Result { // Get opa_base_url for later use in CustomOpaSecurityManager let opa_endpoint = opa_config diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index b2683d19..cb236049 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -30,7 +30,7 @@ spec: pullPolicy: IfNotPresent clusterConfig: authorization: - opa: + roleMappingFromOpa: configMapName: opa package: superset cache: {} From 1438a246f0d8215cea43511f3cd23e8652d14e4c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 25 Feb 2025 12:22:57 +0100 Subject: [PATCH 53/71] format code --- rust/crd/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 645cfc51..3d64ac6f 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -279,7 +279,6 @@ pub struct SupersetOpaRoleMappingConfig { pub cache: UserInformationCache, } - #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct SupersetAuthorization { From 5268e8e24f0744785caab98192915e40ec87aec3 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 25 Feb 2025 16:48:24 +0100 Subject: [PATCH 54/71] make field required --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 6 +++--- deploy/helm/superset-operator/crds/crds.yaml | 3 ++- rust/crd/src/lib.rs | 4 ++-- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 6072d45c..229631dd 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1558,7 +1558,7 @@ rec { "default" = [ "Debug" "Clone" "Copy" "PartialEq" "Eq" "PartialOrd" "Ord" "Hash" "Default" "Deref" "DerefMut" "Into" ]; "full" = [ "syn/full" ]; }; - resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Hash" "PartialEq" ]; + resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Eq" "Hash" "PartialEq" ]; }; "either" = rec { crateName = "either"; @@ -7229,13 +7229,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.85.0"; + version = "0.86.0"; edition = "2021"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "e5bfee596cc918b05f3e1d7e667c25951317cf31"; + sha256 = "04a866w46mbrsqv7iq9x6l2kh1bnykkmfnjwwfrqk6njn91arvf1"; }; libName = "stackable_operator"; authors = [ @@ -7268,7 +7268,7 @@ rec { name = "educe"; packageId = "educe"; usesDefaultFeatures = false; - features = [ "Clone" "Debug" "Default" "PartialEq" ]; + features = [ "Clone" "Debug" "Default" "PartialEq" "Eq" ]; } { name = "either"; @@ -7394,8 +7394,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "e5bfee596cc918b05f3e1d7e667c25951317cf31"; + sha256 = "04a866w46mbrsqv7iq9x6l2kh1bnykkmfnjwwfrqk6njn91arvf1"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -7429,8 +7429,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "e5bfee596cc918b05f3e1d7e667c25951317cf31"; + sha256 = "04a866w46mbrsqv7iq9x6l2kh1bnykkmfnjwwfrqk6njn91arvf1"; }; libName = "stackable_shared"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 290d87f2..7edbbe57 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,6 +1,6 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator-derive@0.3.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator@0.85.0": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-shared@0.0.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.86.0#stackable-operator-derive@0.3.1": "04a866w46mbrsqv7iq9x6l2kh1bnykkmfnjwwfrqk6njn91arvf1", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.86.0#stackable-operator@0.86.0": "04a866w46mbrsqv7iq9x6l2kh1bnykkmfnjwwfrqk6njn91arvf1", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.86.0#stackable-shared@0.0.1": "04a866w46mbrsqv7iq9x6l2kh1bnykkmfnjwwfrqk6njn91arvf1", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index 243bb09d..ee836af7 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -80,7 +80,6 @@ spec: properties: roleMappingFromOpa: description: Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) and the name of the Rego package containing your authorization rules. Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) to learn how to deploy Rego authorization rules with OPA. - nullable: true properties: cache: description: Configuration for an Superset internal cache for calls to OPA @@ -107,6 +106,8 @@ spec: - cache - configMapName type: object + required: + - roleMappingFromOpa type: object clusterOperation: default: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 3d64ac6f..0f2a5466 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -282,7 +282,7 @@ pub struct SupersetOpaRoleMappingConfig { #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct SupersetAuthorization { - pub role_mapping_from_opa: Option, + pub role_mapping_from_opa: SupersetOpaRoleMappingConfig, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -523,7 +523,7 @@ impl SupersetCluster { .cluster_config .authorization .as_ref() - .and_then(|a| a.role_mapping_from_opa.as_ref()) + .map(|a| &a.role_mapping_from_opa) } /// Retrieve and merge resource configs for role and role groups From 95c22b1fd730989f150ca5b12fcc2cfca263663d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 09:35:34 +0100 Subject: [PATCH 55/71] pass on opa endpoint instead of base url to the authorizer --- rust/operator-binary/src/authorization/opa.rs | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index e12da47c..9ded976f 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -4,8 +4,7 @@ use stackable_operator::{client::Client, commons::opa::OpaApiVersion, time::Dura use stackable_superset_crd::{SupersetCluster, SupersetOpaRoleMappingConfig}; pub struct SupersetOpaConfigResolved { - opa_base_url: String, - opa_package: Option, + opa_endpoint: String, cache_max_entries: u32, cache_ttl: Duration, } @@ -22,18 +21,8 @@ impl SupersetOpaConfigResolved { .full_document_url_from_config_map(client, superset, None, OpaApiVersion::V1) .await?; - // striping package path from base url. Needed by CustomOpaSecurityManager. - let opa_base_url = match opa_config.opa.package.clone() { - Some(opa_package_name) => { - let opa_path = format!("/v1/data/{opa_package_name}"); - opa_endpoint.replace(&opa_path, "") - } - None => opa_endpoint.replace("/v1/data/", ""), - }; - Ok(SupersetOpaConfigResolved { - opa_base_url, - opa_package: opa_config.opa.package.to_owned(), + opa_endpoint, cache_max_entries: opa_config.cache.max_entries.to_owned(), cache_ttl: opa_config.cache.entry_time_to_live.to_owned(), }) @@ -41,14 +30,14 @@ impl SupersetOpaConfigResolved { // Adding necessary configurations. Imports are solved in config.rs pub fn as_config(&self) -> BTreeMap { - let mut config = BTreeMap::from([ + BTreeMap::from([ ( "CUSTOM_SECURITY_MANAGER".to_string(), "OpaSupersetSecurityManager".to_string(), ), ( "AUTH_OPA_REQUEST_URL".to_string(), - self.opa_base_url.to_owned(), + self.opa_endpoint.to_owned(), ), ( "AUTH_OPA_CACHE_MAX_ENTRIES".to_string(), @@ -59,12 +48,6 @@ impl SupersetOpaConfigResolved { self.cache_ttl.as_secs().to_string(), ), ("AUTH_OPA_RULE".to_string(), "user_roles".to_string()), - ]); - - if let Some(opa_package) = &self.opa_package { - config.insert("AUTH_OPA_PACKAGE".to_string(), opa_package.to_owned()); - } - - config + ]) } } From 26e95f34297a082f2ab85d53e031d2aef60c30aa Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 11:20:00 +0100 Subject: [PATCH 56/71] Update docs/modules/superset/pages/usage-guide/security.adoc Co-authored-by: Sebastian Bernauer --- docs/modules/superset/pages/usage-guide/security.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/modules/superset/pages/usage-guide/security.adoc b/docs/modules/superset/pages/usage-guide/security.adoc index 5d94f4b5..b7229e7c 100644 --- a/docs/modules/superset/pages/usage-guide/security.adoc +++ b/docs/modules/superset/pages/usage-guide/security.adoc @@ -126,7 +126,8 @@ Further information for specifying an AuthenticationClass for an OIDC provider c Superset has a concept called `Roles` which allows you to grant user permissions based on roles. Have a look at the {superset-security}[Superset documentation on Security^{external-link-icon}^]. -=== [[OPA]] OPA role mapping +[opa] +=== OPA role mapping Superset can sync roles from OpenPolicyAgent. Currently only mapping is enabled as a larger refactoring of the upstream Superset concerning their security management is announced. From 52a2214c92bd61795cd737b15199eccfe2b139d6 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 11:20:17 +0100 Subject: [PATCH 57/71] Update docs/modules/superset/pages/usage-guide/security.adoc Co-authored-by: Sebastian Bernauer --- docs/modules/superset/pages/usage-guide/security.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/superset/pages/usage-guide/security.adoc b/docs/modules/superset/pages/usage-guide/security.adoc index b7229e7c..1c7b06ae 100644 --- a/docs/modules/superset/pages/usage-guide/security.adoc +++ b/docs/modules/superset/pages/usage-guide/security.adoc @@ -132,7 +132,7 @@ Have a look at the {superset-security}[Superset documentation on Security^{exter Superset can sync roles from OpenPolicyAgent. Currently only mapping is enabled as a larger refactoring of the upstream Superset concerning their security management is announced. -In order to map roles from OPA into Superset, we expect rego rules with a rule name `user_roles`. +In order to map roles from OPA into Superset, we expect rego rules with a rule named `user_roles`. In the below example two users `admin` and `testuser` have roles defined as rego rule in Rego v1 to be compliant with OPA v1.0.0 release. IMPORTANT: Only role mapping is enabled! From fdd8c6d75ba1a16efdbf91d9e01e47bafd5a94cb Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 11:20:37 +0100 Subject: [PATCH 58/71] Update docs/modules/superset/pages/usage-guide/security.adoc Co-authored-by: Sebastian Bernauer --- docs/modules/superset/pages/usage-guide/security.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/superset/pages/usage-guide/security.adoc b/docs/modules/superset/pages/usage-guide/security.adoc index 1c7b06ae..9e7b264e 100644 --- a/docs/modules/superset/pages/usage-guide/security.adoc +++ b/docs/modules/superset/pages/usage-guide/security.adoc @@ -133,7 +133,7 @@ Superset can sync roles from OpenPolicyAgent. Currently only mapping is enabled as a larger refactoring of the upstream Superset concerning their security management is announced. In order to map roles from OPA into Superset, we expect rego rules with a rule named `user_roles`. -In the below example two users `admin` and `testuser` have roles defined as rego rule in Rego v1 to be compliant with OPA v1.0.0 release. +In the below example two users `admin` and `testuser` have roles defined as rego rule. IMPORTANT: Only role mapping is enabled! Permissions of these roles can only be managed through the Superset UI. From 80136fe73a8a903bf5bfbdc8c7332f157427dba2 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 11:20:56 +0100 Subject: [PATCH 59/71] Update docs/modules/superset/pages/usage-guide/security.adoc Co-authored-by: Sebastian Bernauer --- docs/modules/superset/pages/usage-guide/security.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/superset/pages/usage-guide/security.adoc b/docs/modules/superset/pages/usage-guide/security.adoc index 9e7b264e..75f26688 100644 --- a/docs/modules/superset/pages/usage-guide/security.adoc +++ b/docs/modules/superset/pages/usage-guide/security.adoc @@ -168,7 +168,7 @@ data: ] ---- -Mounting this `configMap` in superset as follows: +You can instruct Superset to use the rego rule as follows: [source,yaml] ---- From d69ff8b2325a5b1984240bada64f89d006c18cf9 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:18:58 +0100 Subject: [PATCH 60/71] update security.adoc --- .../superset/pages/usage-guide/security.adoc | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/modules/superset/pages/usage-guide/security.adoc b/docs/modules/superset/pages/usage-guide/security.adoc index 75f26688..27ac1c75 100644 --- a/docs/modules/superset/pages/usage-guide/security.adoc +++ b/docs/modules/superset/pages/usage-guide/security.adoc @@ -129,17 +129,11 @@ Have a look at the {superset-security}[Superset documentation on Security^{exter [opa] === OPA role mapping -Superset can sync roles from OpenPolicyAgent. -Currently only mapping is enabled as a larger refactoring of the upstream Superset concerning their security management is announced. - -In order to map roles from OPA into Superset, we expect rego rules with a rule named `user_roles`. -In the below example two users `admin` and `testuser` have roles defined as rego rule. - -IMPORTANT: Only role mapping is enabled! -Permissions of these roles can only be managed through the Superset UI. -RBAC through OPA is not implemented. - -OPA rules can be synced using the xref:opa:usage-guide:user-info-fetcher[user-info-fetcher]. +Stackable ships a custom security manager that makes it possible to assign roles to users via the Open Policy Agent integration. +The roles must exist in the Superset database before they can be assigned to users. +If a role is not present in the Superset database, an error will be logged by the security manager and the user login will proceed without it. +Also the role names must match exactly the output of the Rego rule named `user_roles`. +In the following example, a rego package is defined that assigns roles to the users `admin` and `guest`. [source,yaml] ---- @@ -153,8 +147,6 @@ data: roles.rego: | package superset - import rego.v1 - default user_roles := [] user_roles := roles if { @@ -163,12 +155,17 @@ data: user.username == input.username } users := [ - {"username": "admin", "roles": ["Admin", "Test"]}, - {"username": "testuser", "roles": ["El_Testos", "Custom2"]} + {"username": "admin", "roles": ["Admin", "Test"]}, #<1> + {"username": "guest", "roles": ["Gamma"]} #<2> ] ---- -You can instruct Superset to use the rego rule as follows: +<1> Assign the roles `Admin` and `Test` to the `admin` user. The `Test` role is not a standard Superset role and must be created before the assignment. +<2> Assign the `Gamma` role to the `guest` user. + +OPA rules can make use of the xref:opa:usage-guide:user-info-fetcher[user-info-fetcher] integration. + +The following snippet shows how to use the OPA security manager in a Superset stacklet. [source,yaml] ---- @@ -192,6 +189,8 @@ spec: <3> Time for cached entries per user can live. Defaults to 30s. <4> Number of maximum entries, defaults to 1000. Cache will be disabled for maxEntries: 0. +IMPORTANT: Any role assignments done in the Superset UI are discarded and will be overridden by the OPA security manager. + === Superset database You can view all the available roles in the web interface of Superset and can also assign users to these roles. From 0d078b1f4c8932fee75ced030db6b50ed75c39a4 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:30:50 +0100 Subject: [PATCH 61/71] move constant --- rust/operator-binary/src/authorization/opa.rs | 3 +++ rust/operator-binary/src/config.rs | 3 --- rust/operator-binary/src/superset_controller.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index 9ded976f..c2253ac9 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -3,6 +3,9 @@ use std::collections::BTreeMap; use stackable_operator::{client::Client, commons::opa::OpaApiVersion, time::Duration}; use stackable_superset_crd::{SupersetCluster, SupersetOpaRoleMappingConfig}; +pub const OPA_IMPORTS: &[&str] = + &["from opa_authorizer.opa_manager import OpaSupersetSecurityManager"]; + pub struct SupersetOpaConfigResolved { opa_endpoint: String, cache_max_entries: u32, diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 091d5fdb..af2914e5 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -36,9 +36,6 @@ pub const PYTHON_IMPORTS: &[&str] = &[ "from log_config import StackableLoggingConfigurator", ]; -pub const OPA_IMPORTS: &[&str] = - &["from opa_authorizer.opa_manager import OpaSupersetSecurityManager"]; - pub fn add_superset_config( config: &mut BTreeMap, authentication_config: &SupersetClientAuthenticationDetailsResolved, diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index c72f17aa..77b86315 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -76,9 +76,9 @@ use stackable_superset_crd::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - authorization::opa::SupersetOpaConfigResolved, + authorization::opa::{SupersetOpaConfigResolved, OPA_IMPORTS}, commands::add_cert_to_python_certifi_command, - config::{self, OPA_IMPORTS, PYTHON_IMPORTS}, + config::{self, PYTHON_IMPORTS}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{ From bcd774f60c865b535f7e8088169e16a585439bf5 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:41:31 +0100 Subject: [PATCH 62/71] rename opa dimension --- tests/templates/kuttl/opa/30-install-opa.yaml.j2 | 2 +- tests/test-definition.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/templates/kuttl/opa/30-install-opa.yaml.j2 b/tests/templates/kuttl/opa/30-install-opa.yaml.j2 index 93a9911d..4ad30219 100644 --- a/tests/templates/kuttl/opa/30-install-opa.yaml.j2 +++ b/tests/templates/kuttl/opa/30-install-opa.yaml.j2 @@ -11,7 +11,7 @@ commands: name: opa spec: image: - productVersion: "{{ test_scenario['values']['opa'] }}" + productVersion: "{{ test_scenario['values']['opa-latest'] }}" pullPolicy: IfNotPresent clusterConfig: userInfo: diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index d51480bd..3ce36205 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -19,7 +19,7 @@ dimensions: - no-tls - insecure-tls - server-verification-tls - - name: opa + - name: opa-latest values: - 1.0.1 - name: openshift @@ -50,7 +50,7 @@ tests: - name: opa dimensions: - superset - - opa + - opa-latest - openshift - name: resources dimensions: From 1051900991c80cd36d51dfcae65691482c7f0a1b Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 13:49:02 +0100 Subject: [PATCH 63/71] revert changes to smoke test --- .../kuttl/smoke/30-install-superset.yaml.j2 | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 index c9897f77..872202e5 100644 --- a/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-superset.yaml.j2 @@ -44,17 +44,6 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - containers: - superset: - console: - level: DEBUG - file: - level: DEBUG - loggers: - ROOT: - level: DEBUG - flask_appbuilder.security: - level: DEBUG configOverrides: superset_config.py: EXPERIMENTAL_FILE_HEADER: | @@ -62,10 +51,6 @@ spec: ROLE_HEADER_VAR = "role-value" EXPERIMENTAL_FILE_FOOTER: | ROLE_FOOTER_VAR = "role-value" - # Enable the security API to be able to create roles from the test - FAB_ADD_SECURITY_API: "True" - # Enable FAB logging - SILENCE_FAB: "False" roleGroups: default: replicas: 1 From 63ac8cccd13c8c93234765dc54158dbc946a19f5 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:26:51 +0100 Subject: [PATCH 64/71] Update tests/templates/kuttl/opa/40_superset.yaml.j2 Co-authored-by: Sebastian Bernauer --- tests/templates/kuttl/opa/40_superset.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index cb236049..b4e0a6d0 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -33,7 +33,6 @@ spec: roleMappingFromOpa: configMapName: opa package: superset - cache: {} credentialsSecret: superset-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery From 499d02cd8077ef4e94cdcb976e32b3b430bd9f3e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 14:49:21 +0100 Subject: [PATCH 65/71] remove unused image field --- tests/templates/kuttl/opa/40_superset.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/templates/kuttl/opa/40_superset.yaml.j2 b/tests/templates/kuttl/opa/40_superset.yaml.j2 index b4e0a6d0..3d6a5d15 100644 --- a/tests/templates/kuttl/opa/40_superset.yaml.j2 +++ b/tests/templates/kuttl/opa/40_superset.yaml.j2 @@ -20,7 +20,6 @@ metadata: name: superset spec: image: - custom: docker.stackable.tech/stackable/superset:4.0.2-stackable0.0.0-dev-opa {% if test_scenario['values']['superset'].find(",") > 0 %} custom: "{{ test_scenario['values']['superset'].split(',')[1] }}" productVersion: "{{ test_scenario['values']['superset'].split(',')[0] }}" From f9f8c7ed817818483f81907d20dbe8b5a879b429 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 15:03:36 +0100 Subject: [PATCH 66/71] add serde cache defaults --- deploy/helm/superset-operator/crds/crds.yaml | 4 +++- rust/crd/src/lib.rs | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml index ee836af7..686550e4 100644 --- a/deploy/helm/superset-operator/crds/crds.yaml +++ b/deploy/helm/superset-operator/crds/crds.yaml @@ -82,6 +82,9 @@ spec: description: Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) and the name of the Rego package containing your authorization rules. Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) to learn how to deploy Rego authorization rules with OPA. properties: cache: + default: + entryTimeToLive: 30s + maxEntries: 10000 description: Configuration for an Superset internal cache for calls to OPA properties: entryTimeToLive: @@ -103,7 +106,6 @@ spec: nullable: true type: string required: - - cache - configMapName type: object required: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 0f2a5466..a33ea587 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -276,6 +276,7 @@ pub struct SupersetOpaRoleMappingConfig { pub opa: OpaConfig, /// Configuration for an Superset internal cache for calls to OPA + #[serde(default)] pub cache: UserInformationCache, } From bc9acf3badd93f13e9968ae0e03b240cf2170c7f Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 15:24:13 +0100 Subject: [PATCH 67/71] Update rust/operator-binary/src/authorization/opa.rs Co-authored-by: Sebastian Bernauer --- rust/operator-binary/src/authorization/opa.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/operator-binary/src/authorization/opa.rs b/rust/operator-binary/src/authorization/opa.rs index c2253ac9..c1898120 100644 --- a/rust/operator-binary/src/authorization/opa.rs +++ b/rust/operator-binary/src/authorization/opa.rs @@ -18,7 +18,6 @@ impl SupersetOpaConfigResolved { superset: &SupersetCluster, opa_config: &SupersetOpaRoleMappingConfig, ) -> Result { - // Get opa_base_url for later use in CustomOpaSecurityManager let opa_endpoint = opa_config .opa .full_document_url_from_config_map(client, superset, None, OpaApiVersion::V1) From 16eb8e52fedee39bb4ef47c0b057fe6a53766266 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 15:24:28 +0100 Subject: [PATCH 68/71] Update rust/operator-binary/src/superset_controller.rs Co-authored-by: Sebastian Bernauer --- rust/operator-binary/src/superset_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index 77b86315..f492b3ec 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -288,7 +288,7 @@ pub enum Error { source: error_boundary::InvalidObject, }, - #[snafu(display("invalid OpaConfig"))] + #[snafu(display("invalid OPA config"))] InvalidOpaConfig { source: stackable_operator::commons::opa::Error, }, From 9b7d6c9ac16f5763be09bd4675d52852a2bd0669 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 17:59:17 +0100 Subject: [PATCH 69/71] add missing EOF --- tests/templates/kuttl/opa/30-install-opa.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/templates/kuttl/opa/30-install-opa.yaml.j2 b/tests/templates/kuttl/opa/30-install-opa.yaml.j2 index 4ad30219..46b4599d 100644 --- a/tests/templates/kuttl/opa/30-install-opa.yaml.j2 +++ b/tests/templates/kuttl/opa/30-install-opa.yaml.j2 @@ -41,3 +41,4 @@ commands: level: INFO roleGroups: default: {} + EOF From 26f08f0ce30bdc462c6257f088a74014e7d9eeec Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 18:47:02 +0100 Subject: [PATCH 70/71] add vector aggregator config map --- tests/templates/kuttl/opa/05-assert.yaml.j2 | 10 ++++++++++ ...stall-vector-aggregator-discovery-configmap.yaml.j2 | 9 +++++++++ 2 files changed, 19 insertions(+) create mode 100644 tests/templates/kuttl/opa/05-assert.yaml.j2 create mode 100644 tests/templates/kuttl/opa/05-install-vector-aggregator-discovery-configmap.yaml.j2 diff --git a/tests/templates/kuttl/opa/05-assert.yaml.j2 b/tests/templates/kuttl/opa/05-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/opa/05-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/opa/05-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/opa/05-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/opa/05-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} From 416389b2c60544adaaeaeb06b794fe95db18f3ca Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 26 Feb 2025 19:18:10 +0100 Subject: [PATCH 71/71] add openshift ns patch --- tests/templates/kuttl/opa/00-patch-ns.yaml.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/templates/kuttl/opa/00-patch-ns.yaml.j2 diff --git a/tests/templates/kuttl/opa/00-patch-ns.yaml.j2 b/tests/templates/kuttl/opa/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/opa/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %}