diff --git a/CHANGELOG.md b/CHANGELOG.md index 495e8dbd..992e1460 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ - The `runAsUser` and `runAsGroup` fields will not be set anymore by the operator - The defaults from the docker images itself will now apply, which will be different from 1000/0 going forward - This is marked as breaking because tools and policies might exist, which require these fields to be set +- Changed listener class to be role-only ([#645]). ### Fixed @@ -48,6 +49,7 @@ [#625]: https://github.com/stackabletech/airflow-operator/pull/625 [#630]: https://github.com/stackabletech/airflow-operator/pull/630 [#636]: https://github.com/stackabletech/airflow-operator/pull/636 +[#645]: https://github.com/stackabletech/airflow-operator/pull/645 ## [25.3.0] - 2025-03-21 diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index b736bc02..b56fe56d 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -1332,10 +1332,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - nullable: true - type: string logging: default: containers: {} @@ -1480,11 +1476,16 @@ spec: x-kubernetes-preserve-unknown-fields: true roleConfig: default: + listenerClass: cluster-internal podDisruptionBudget: enabled: true maxUnavailable: null description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. properties: + listenerClass: + default: cluster-internal + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + type: string podDisruptionBudget: default: enabled: true @@ -1553,10 +1554,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - nullable: true - type: string logging: default: containers: {} diff --git a/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml b/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml index 9e092d7f..685d5e53 100644 --- a/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml +++ b/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml @@ -19,7 +19,7 @@ spec: mountPath: /dags/test_airflow_dag.py # <6> subPath: test_airflow_dag.py # <7> webservers: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/docs/modules/airflow/examples/example-airflow-incluster.yaml b/docs/modules/airflow/examples/example-airflow-incluster.yaml index 6e5e14fe..67b1ac96 100644 --- a/docs/modules/airflow/examples/example-airflow-incluster.yaml +++ b/docs/modules/airflow/examples/example-airflow-incluster.yaml @@ -11,7 +11,7 @@ spec: exposeConfig: false credentialsSecret: simple-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml b/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml index 9aea1a22..c4201e38 100644 --- a/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml +++ b/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml @@ -7,7 +7,7 @@ spec: productVersion: 2.10.5 clusterConfig: {} webservers: - config: + roleConfig: listenerClass: external-unstable envOverrides: &envOverrides AIRFLOW__LOGGING__REMOTE_LOGGING: "True" diff --git a/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml b/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml index 99778018..8d0a07e3 100644 --- a/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml +++ b/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml @@ -7,7 +7,7 @@ spec: productVersion: 2.10.5 clusterConfig: {} webservers: - config: + roleConfig: listenerClass: external-unstable envOverrides: &envOverrides AIRFLOW__CORE__XCOM_BACKEND: airflow.providers.common.io.xcom.backend.XComObjectStorageBackend diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml index 7fd72a9a..5248116b 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow.yaml @@ -11,7 +11,7 @@ spec: exposeConfig: false credentialsSecret: simple-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc index 9e4ee761..5700d86c 100644 --- a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc @@ -2,18 +2,20 @@ :description: Configure Airflow service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable. The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Webserver pod. -The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.config.listenerClass`: +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.roleConfig.listenerClass`: [source,yaml] ---- spec: webservers: - config: + roleConfig: listenerClass: external-unstable # <1> + config: + ... schedulers: ... celeryExecutors: ... ---- -<1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`). +<1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`) at role-level. This can be set only for the webservers role. diff --git a/examples/simple-airflow-cluster-dags-cmap.yaml b/examples/simple-airflow-cluster-dags-cmap.yaml index b5ed29cb..ae8145ca 100644 --- a/examples/simple-airflow-cluster-dags-cmap.yaml +++ b/examples/simple-airflow-cluster-dags-cmap.yaml @@ -96,7 +96,7 @@ spec: mountPath: /dags/test_airflow_dag.py subPath: test_airflow_dag.py webservers: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/examples/simple-airflow-cluster-ldap-insecure-tls.yaml b/examples/simple-airflow-cluster-ldap-insecure-tls.yaml index 196fe61c..8c0cbb0a 100644 --- a/examples/simple-airflow-cluster-ldap-insecure-tls.yaml +++ b/examples/simple-airflow-cluster-ldap-insecure-tls.yaml @@ -157,7 +157,7 @@ spec: - authenticationClass: airflow-with-ldap-insecure-tls-ldap userRegistrationRole: Admin webservers: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/examples/simple-airflow-cluster-ldap.yaml b/examples/simple-airflow-cluster-ldap.yaml index b34fd059..63389ee7 100644 --- a/examples/simple-airflow-cluster-ldap.yaml +++ b/examples/simple-airflow-cluster-ldap.yaml @@ -155,7 +155,7 @@ spec: - authenticationClass: airflow-with-ldap-server-veri-tls-ldap userRegistrationRole: Admin webservers: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/examples/simple-airflow-cluster.yaml b/examples/simple-airflow-cluster.yaml index d6844499..39aebe6c 100644 --- a/examples/simple-airflow-cluster.yaml +++ b/examples/simple-airflow-cluster.yaml @@ -28,7 +28,7 @@ spec: exposeConfig: false credentialsSecret: simple-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 3049e7ac..fb87ed08 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -53,7 +53,7 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, - kvp::{Annotation, Label, LabelError, Labels}, + kvp::{Annotation, Label, LabelError, Labels, ObjectLabels}, logging::controller::ReconcilerError, product_config_utils::{ CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, env_vars_from, @@ -512,26 +512,6 @@ pub async fn reconcile_airflow( &git_sync_resources, )?; - if let Some(listener_class) = - airflow.merged_listener_class(&airflow_role, &rolegroup.role_group) - { - if let Some(listener_group_name) = - airflow.group_listener_name(&airflow_role, &rolegroup) - { - let rg_group_listener = build_group_listener( - airflow, - &resolved_product_image, - &rolegroup, - listener_class.to_string(), - listener_group_name, - )?; - cluster_resources - .add(client, rg_group_listener) - .await - .context(ApplyGroupListenerSnafu)?; - } - } - ss_cond_builder.add( cluster_resources .add(client, rg_statefulset) @@ -568,6 +548,27 @@ pub async fn reconcile_airflow( .await .context(FailedToCreatePdbSnafu)?; } + + if let Some(listener_class) = airflow_role.listener_class_name(airflow) { + if let Some(listener_group_name) = airflow.group_listener_name(&airflow_role) { + let rg_group_listener = build_group_listener( + airflow, + build_recommended_labels( + airflow, + AIRFLOW_CONTROLLER_NAME, + &resolved_product_image.app_version_label, + role_name, + "none", + ), + listener_class.to_string(), + listener_group_name, + )?; + cluster_resources + .add(client, rg_group_listener) + .await + .context(ApplyGroupListenerSnafu)?; + } + } } cluster_resources @@ -840,8 +841,7 @@ fn build_rolegroup_metadata( pub fn build_group_listener( airflow: &v1alpha1::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, + object_labels: ObjectLabels, listener_class: String, listener_group_name: String, ) -> Result { @@ -851,13 +851,7 @@ pub fn build_group_listener( .name(listener_group_name) .ownerreference_from_resource(airflow, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) + .with_recommended_labels(object_labels) .context(ObjectMetaSnafu)? .build(), spec: listener::v1alpha1::ListenerSpec { @@ -1019,7 +1013,7 @@ fn build_server_rolegroup_statefulset( let mut pvcs: Option> = None; - if let Some(listener_group_name) = airflow.group_listener_name(airflow_role, rolegroup_ref) { + if let Some(listener_group_name) = airflow.group_listener_name(airflow_role) { // Listener endpoints for the Webserver role will use persistent volumes // so that load balancers can hard-code the target addresses. This will // be the case even when no class is set (and the value defaults to diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 86dfc9d9..046eb5c6 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -45,12 +45,16 @@ use stackable_operator::{ }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; -use crate::crd::{ - affinity::{get_affinity, get_executor_affinity}, - authentication::{ - AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetails, - AirflowClientAuthenticationDetailsResolved, +use crate::{ + crd::{ + affinity::{get_affinity, get_executor_affinity}, + authentication::{ + AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetails, + AirflowClientAuthenticationDetailsResolved, + }, + v1alpha1::WebserverRoleConfig, }, + util::role_service_name, }; pub mod affinity; @@ -163,6 +167,8 @@ impl FlaskAppConfigOptions for AirflowConfigOptions { #[versioned(version(name = "v1alpha1"))] pub mod versioned { + use crate::crd::v1alpha1::WebserverRoleConfig; + /// An Airflow cluster stacklet. This resource is managed by the Stackable operator for Apache Airflow. /// Find more information on how to use it and the resources that the operator generates in the /// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/). @@ -198,7 +204,7 @@ pub mod versioned { /// The `webserver` role provides the main UI for user interaction. #[serde(default, skip_serializing_if = "Option::is_none")] - pub webservers: Option>, + pub webservers: Option>, /// The `scheduler` is responsible for triggering jobs and persisting their metadata to the backend database. /// Jobs are scheduled on the workers/executors. @@ -258,6 +264,31 @@ pub mod versioned { #[schemars(schema_with = "raw_object_list_schema")] pub volume_mounts: Vec, } + + // TODO: move generic version to op-rs? + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct WebserverRoleConfig { + #[serde(flatten)] + pub common: GenericRoleConfig, + + /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + #[serde(default = "webserver_default_listener_class")] + pub listener_class: String, + } +} + +impl Default for v1alpha1::WebserverRoleConfig { + fn default() -> Self { + v1alpha1::WebserverRoleConfig { + listener_class: webserver_default_listener_class(), + common: Default::default(), + } + } +} + +fn webserver_default_listener_class() -> String { + "cluster-internal".to_string() } #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] @@ -277,16 +308,12 @@ impl HasStatusCondition for v1alpha1::AirflowCluster { } impl v1alpha1::AirflowCluster { - /// The name of the group-listener provided for a specific role-group. + /// The name of the group-listener provided for a specific role. /// Webservers will use this group listener so that only one load balancer - /// is needed (per role group). - pub fn group_listener_name( - &self, - role: &AirflowRole, - rolegroup: &RoleGroupRef, - ) -> Option { + /// is needed for that role. + pub fn group_listener_name(&self, role: &AirflowRole) -> Option { match role { - AirflowRole::Webserver => Some(rolegroup.object_name()), + AirflowRole::Webserver => Some(role_service_name(&self.name_any(), &role.to_string())), AirflowRole::Scheduler | AirflowRole::Worker => None, } } @@ -381,34 +408,6 @@ impl v1alpha1::AirflowCluster { fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu) } - pub fn merged_listener_class( - &self, - role: &AirflowRole, - rolegroup_name: &String, - ) -> Option { - if role == &AirflowRole::Webserver { - if let Some(webservers) = self.spec.webservers.as_ref() { - let conf_defaults = Some("cluster-internal".to_string()); - let mut conf_role = webservers.config.config.listener_class.to_owned(); - let mut conf_rolegroup = webservers - .role_groups - .get(rolegroup_name) - .map(|rg| rg.config.config.listener_class.clone()) - .unwrap_or_default(); - - conf_role.merge(&conf_defaults); - conf_rolegroup.merge(&conf_role); - - tracing::debug!("Merged listener-class: {:?}", conf_rolegroup); - conf_rolegroup - } else { - None - } - } else { - None - } - } - /// Retrieve and merge resource configs for the executor template pub fn merged_executor_config( &self, @@ -436,18 +435,18 @@ impl v1alpha1::AirflowCluster { } fn extract_role_from_webserver_config( - fragment: Role, + fragment: Role, ) -> Role { Role { config: CommonConfiguration { - config: fragment.config.config.airflow_config, + config: fragment.config.config, config_overrides: fragment.config.config_overrides, env_overrides: fragment.config.env_overrides, cli_overrides: fragment.config.cli_overrides, pod_overrides: fragment.config.pod_overrides, product_specific_common_config: fragment.config.product_specific_common_config, }, - role_config: fragment.role_config, + role_config: fragment.role_config.common, role_groups: fragment .role_groups .into_iter() @@ -456,7 +455,7 @@ fn extract_role_from_webserver_config( k, RoleGroup { config: CommonConfiguration { - config: v.config.config.airflow_config, + config: v.config.config, config_overrides: v.config.config_overrides, env_overrides: v.config.env_overrides, cli_overrides: v.config.cli_overrides, @@ -687,6 +686,17 @@ impl AirflowRole { } roles } + + pub fn listener_class_name(&self, airflow: &v1alpha1::AirflowCluster) -> Option { + match self { + Self::Webserver => airflow + .spec + .webservers + .to_owned() + .map(|webserver| webserver.role_config.listener_class), + Self::Worker | Self::Scheduler => None, + } + } } fn container_debug_command() -> String { @@ -810,29 +820,6 @@ pub struct AirflowConfig { pub graceful_shutdown_timeout: Option, } -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct WebserverConfig { - #[fragment_attrs(serde(default, flatten))] - pub airflow_config: AirflowConfig, - - /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. - #[serde(default)] - pub listener_class: String, -} - impl AirflowConfig { pub const CREDENTIALS_SECRET_PROPERTY: &'static str = "credentialsSecret"; pub const GIT_CREDENTIALS_SECRET_PROPERTY: &'static str = "gitCredentialsSecret"; diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index e36e5b87..9a10ad0b 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -23,7 +23,7 @@ use crate::{ authorization::AirflowAuthorizationResolved, v1alpha1, }, - util::env_var_from_secret, + util::{env_var_from_secret, role_service_name}, }; const AIRFLOW_CORE_AUTH_MANAGER: &str = "AIRFLOW__CORE__AUTH_MANAGER"; @@ -575,37 +575,26 @@ fn execution_server_env_vars(airflow: &v1alpha1::AirflowCluster) -> BTreeMap>() - .first_entry() - { - let webserver = format!( - "{name}-webserver-{rolegroup}", - name = name, - rolegroup = rolegroup.key() - ); - tracing::debug!("Webserver set [{webserver}]"); - // These settings are new in 3.x and will have no affect with earlier versions. - env.insert( - "AIRFLOW__CORE__EXECUTION_API_SERVER_URL".into(), - EnvVar { - name: "AIRFLOW__CORE__EXECUTION_API_SERVER_URL".into(), - value: Some(format!("http://{webserver}:8080/execution/")), - ..Default::default() - }, - ); - env.insert( - "AIRFLOW__CORE__BASE_URL".into(), - EnvVar { - name: "AIRFLOW__CORE__BASE_URL".into(), - value: Some(format!("http://{webserver}:8080/")), - ..Default::default() - }, - ); - } + if airflow.spec.webservers.as_ref().is_some() { + let webserver = role_service_name(name, &AirflowRole::Webserver.to_string()); + tracing::debug!("Webserver set [{webserver}]"); + // These settings are new in 3.x and will have no affect with earlier versions. + env.insert( + "AIRFLOW__CORE__EXECUTION_API_SERVER_URL".into(), + EnvVar { + name: "AIRFLOW__CORE__EXECUTION_API_SERVER_URL".into(), + value: Some(format!("http://{webserver}:8080/execution/")), + ..Default::default() + }, + ); + env.insert( + "AIRFLOW__CORE__BASE_URL".into(), + EnvVar { + name: "AIRFLOW__CORE__BASE_URL".into(), + value: Some(format!("http://{webserver}:8080/")), + ..Default::default() + }, + ); } } diff --git a/rust/operator-binary/src/util.rs b/rust/operator-binary/src/util.rs index 38e70f21..ffd49747 100644 --- a/rust/operator-binary/src/util.rs +++ b/rust/operator-binary/src/util.rs @@ -14,3 +14,7 @@ pub fn env_var_from_secret(var_name: &str, secret: &str, secret_key: &str) -> En ..Default::default() } } + +pub fn role_service_name(name: &str, role: &str) -> String { + format!("{name}-{role}") +} diff --git a/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 index bea56217..8410596e 100644 --- a/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/08-install-airflow.yaml.j2 @@ -39,8 +39,9 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2 index 6bcbc9ef..e84c9653 100644 --- a/tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/10-pause-airflow.yaml.j2 @@ -26,8 +26,9 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2 index 3f52a339..470f8d7d 100644 --- a/tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/20-stop-airflow.yaml.j2 @@ -26,8 +26,9 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2 index 56b2e3de..5af77035 100644 --- a/tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-restart-airflow.yaml.j2 @@ -26,8 +26,9 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/commons/health.py b/tests/templates/kuttl/commons/health.py index 385e3a29..de745052 100755 --- a/tests/templates/kuttl/commons/health.py +++ b/tests/templates/kuttl/commons/health.py @@ -14,13 +14,12 @@ ) parser = argparse.ArgumentParser(description="Health check script") - parser.add_argument("--role-group", type=str, default="default", help="Role group to check") parser.add_argument("--airflow-version", type=str, help="Airflow version") opts = parser.parse_args() - url = f"http://airflow-webserver-{opts.role_group}:8080/api/v1/health" + url = "http://airflow-webserver:8080/api/v1/health" if opts.airflow_version and opts.airflow_version.startswith("3"): - url = f"http://airflow-webserver-{opts.role_group}:8080/api/v2/monitor/health" + url = "http://airflow-webserver:8080/api/v2/monitor/health" count = 0 diff --git a/tests/templates/kuttl/commons/metrics.py b/tests/templates/kuttl/commons/metrics.py index e14efb59..fda935bf 100755 --- a/tests/templates/kuttl/commons/metrics.py +++ b/tests/templates/kuttl/commons/metrics.py @@ -38,8 +38,8 @@ def metrics_v3(role_group: str) -> None: # allow a few moments for the DAGs to be registered to all roles time.sleep(10) - rest_url = f"http://airflow-webserver-{role_group}:8080/api/v2" - token_url = f"http://airflow-webserver-{role_group}:8080/auth/token" + rest_url = "http://airflow-webserver:8080/api/v2" + token_url = "http://airflow-webserver:8080/auth/token" data = {"username": "airflow", "password": "airflow"} @@ -110,7 +110,7 @@ def metrics_v2(role_group: str) -> None: dag_id = "example_trigger_target_dag" dag_conf = {"message": "Hello World"} - rest_url = f"http://airflow-webserver-{role_group}:8080/api/v1" + rest_url = "http://airflow-webserver:8080/api/v1" auth = ("airflow", "airflow") # allow a few moments for the DAGs to be registered to all roles diff --git a/tests/templates/kuttl/external-access/40-assert.yaml.j2 b/tests/templates/kuttl/external-access/40-assert.yaml.j2 index dd58e251..27b1bbd4 100644 --- a/tests/templates/kuttl/external-access/40-assert.yaml.j2 +++ b/tests/templates/kuttl/external-access/40-assert.yaml.j2 @@ -27,22 +27,6 @@ status: --- apiVersion: apps/v1 kind: StatefulSet -metadata: - name: airflow-webserver-external-unstable -status: - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: airflow-webserver-cluster-internal -status: - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet metadata: name: airflow-scheduler-default spec: @@ -58,8 +42,8 @@ kind: PodDisruptionBudget metadata: name: airflow-webserver status: - expectedPods: 3 - currentHealthy: 3 + expectedPods: 1 + currentHealthy: 1 disruptionsAllowed: 1 --- apiVersion: policy/v1 @@ -74,23 +58,9 @@ status: apiVersion: v1 kind: Service metadata: - name: airflow-webserver-cluster-internal -spec: - type: ClusterIP # cluster-internal ---- -apiVersion: v1 -kind: Service -metadata: - name: airflow-webserver-default + name: airflow-webserver spec: type: NodePort # external-stable ---- -apiVersion: v1 -kind: Service -metadata: - name: airflow-webserver-external-unstable -spec: - type: NodePort # external-unstable {% if test_scenario['values']['executor'] == 'celery' %} --- diff --git a/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 index 487dc14e..0ed7716b 100644 --- a/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 @@ -32,8 +32,9 @@ spec: loadExamples: false credentialsSecret: test-airflow-credentials webservers: - config: + roleConfig: listenerClass: test-external-stable-$NAMESPACE + config: resources: cpu: min: 1000m @@ -43,14 +44,6 @@ spec: roleGroups: default: replicas: 1 - external-unstable: - replicas: 1 - config: - listenerClass: test-external-unstable-$NAMESPACE - cluster-internal: - replicas: 1 - config: - listenerClass: test-cluster-internal-$NAMESPACE {% if test_scenario['values']['executor'] == 'celery' %} celeryExecutors: roleGroups: diff --git a/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 index c5423238..b87f0d8e 100644 --- a/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/ldap/60-install-airflow-cluster.yaml.j2 @@ -65,8 +65,9 @@ commands: userRegistrationRole: Admin webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 index 35ebf3a6..7ebb7d78 100644 --- a/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/logging/41-install-airflow-cluster.yaml.j2 @@ -76,8 +76,9 @@ spec: - repo: https://github.com/stackabletech/example-dags gitFolder: dags webservers: - config: + roleConfig: listenerClass: external-unstable + config: resources: cpu: min: 1000m diff --git a/tests/templates/kuttl/logging/51-assert.yaml.j2 b/tests/templates/kuttl/logging/51-assert.yaml.j2 index 7621be7e..fa26b11c 100644 --- a/tests/templates/kuttl/logging/51-assert.yaml.j2 +++ b/tests/templates/kuttl/logging/51-assert.yaml.j2 @@ -11,6 +11,5 @@ commands: {% set airflow_version = test_scenario['values']['airflow'] %} {% endif %} - script: | - kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --role-group automatic-log-config --airflow-version "{{ airflow_version }}" - kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --role-group custom-log-config --airflow-version "{{ airflow_version }}" - + kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ airflow_version }}" + kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ airflow_version }}" diff --git a/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 index 1707cd26..324c0642 100644 --- a/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-configmap/30-install-airflow-cluster.yaml.j2 @@ -85,8 +85,9 @@ spec: mountPath: /dags/example_trigger_target_dag.py subPath: example_trigger_target_dag.py webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 6aa3f43c..361aee61 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -82,8 +82,9 @@ spec: configMap: name: test-cm-gitsync webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py b/tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py index acb38502..d0dae8cf 100755 --- a/tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py +++ b/tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py @@ -29,8 +29,8 @@ def assert_metric(role, metric): # allow a few moments for the DAGs to be registered to all roles time.sleep(10) -rest_url = "http://airflow-webserver-default:8080/api/v2" -token_url = "http://airflow-webserver-default:8080/auth/token" +rest_url = "http://airflow-webserver:8080/api/v2" +token_url = "http://airflow-webserver:8080/auth/token" data = {"username": "airflow", "password": "airflow"} diff --git a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 index f2e5c941..9fd843b0 100644 --- a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 +++ b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 @@ -58,8 +58,9 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index 69d3ce51..0b9da8f3 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -38,7 +38,7 @@ def userinfo_page(base_url: str, airflow_version: str) -> str: session = requests.Session() -url = "http://airflow-webserver-default:8080" +url = "http://airflow-webserver:8080" # Click on "Sign In with keycloak" in Airflow login_page = session.get(login_page(url, os.environ["AIRFLOW_VERSION"])) diff --git a/tests/templates/kuttl/opa/30-install-airflow.yaml.j2 b/tests/templates/kuttl/opa/30-install-airflow.yaml.j2 index e5e5f825..fcf057c1 100644 --- a/tests/templates/kuttl/opa/30-install-airflow.yaml.j2 +++ b/tests/templates/kuttl/opa/30-install-airflow.yaml.j2 @@ -46,8 +46,9 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} configOverrides: diff --git a/tests/templates/kuttl/opa/41_check-authorization_2.py b/tests/templates/kuttl/opa/41_check-authorization_2.py index bbb2414d..c38a8130 100644 --- a/tests/templates/kuttl/opa/41_check-authorization_2.py +++ b/tests/templates/kuttl/opa/41_check-authorization_2.py @@ -26,7 +26,7 @@ "password": "NvfpU518", } -url = "http://airflow-webserver-default:8080" +url = "http://airflow-webserver:8080" def check_api_authorization_for_user( diff --git a/tests/templates/kuttl/opa/41_check-authorization_3.py b/tests/templates/kuttl/opa/41_check-authorization_3.py index 7e0e00b9..55a20fe3 100644 --- a/tests/templates/kuttl/opa/41_check-authorization_3.py +++ b/tests/templates/kuttl/opa/41_check-authorization_3.py @@ -31,7 +31,7 @@ "password": "NvfpU518", } -url = "http://airflow-webserver-default:8080" +url = "http://airflow-webserver:8080" api = "api/v2" url_login = f"{url}/auth/login" diff --git a/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 index f9b16b29..7bf63a71 100644 --- a/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/30-install-airflow-cluster.yaml.j2 @@ -39,8 +39,9 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 index 9d25d48a..7c5324e6 100644 --- a/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 +++ b/tests/templates/kuttl/overrides/10-install-airflow.yaml.j2 @@ -49,7 +49,7 @@ spec: exposeConfig: false credentialsSecret: airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable envOverrides: COMMON_VAR: role-value # overridden by role group below diff --git a/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 b/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 index 1a905de6..da6e5d5a 100644 --- a/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 +++ b/tests/templates/kuttl/overrides/20-install-airflow2.yaml.j2 @@ -17,7 +17,7 @@ spec: exposeConfig: false credentialsSecret: airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 index c367089c..22a2b673 100644 --- a/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/resources/30-install-airflow-cluster.yaml.j2 @@ -39,8 +39,9 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 index 38e96a72..a4793606 100644 --- a/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-install-airflow-cluster.yaml.j2 @@ -42,8 +42,9 @@ spec: loadExamples: true credentialsSecret: test-airflow-credentials webservers: - config: + roleConfig: listenerClass: external-unstable + config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} configOverrides: