Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
- The `runAsUser` and `runAsGroup` fields will not be set anymore by the operator
- The defaults from the docker images itself will now apply, which will be different from 1000/0 going forward
- This is marked as breaking because tools and policies might exist, which require these fields to be set
- Changed listener class to be role-only ([#645]).

### Fixed

Expand All @@ -48,6 +49,7 @@
[#625]: https://github.com/stackabletech/airflow-operator/pull/625
[#630]: https://github.com/stackabletech/airflow-operator/pull/630
[#636]: https://github.com/stackabletech/airflow-operator/pull/636
[#645]: https://github.com/stackabletech/airflow-operator/pull/645

## [25.3.0] - 2025-03-21

Expand Down
13 changes: 5 additions & 8 deletions deploy/helm/airflow-operator/crds/crds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1332,10 +1332,6 @@ spec:
description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
nullable: true
type: string
listenerClass:
description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver.
nullable: true
type: string
logging:
default:
containers: {}
Expand Down Expand Up @@ -1480,11 +1476,16 @@ spec:
x-kubernetes-preserve-unknown-fields: true
roleConfig:
default:
listenerClass: cluster-internal
podDisruptionBudget:
enabled: true
maxUnavailable: null
description: This is a product-agnostic RoleConfig, which is sufficient for most of the products.
properties:
listenerClass:
default: cluster-internal
description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver.
type: string
podDisruptionBudget:
default:
enabled: true
Expand Down Expand Up @@ -1553,10 +1554,6 @@ spec:
description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
nullable: true
type: string
listenerClass:
description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver.
nullable: true
type: string
logging:
default:
containers: {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ spec:
mountPath: /dags/test_airflow_dag.py # <6>
subPath: test_airflow_dag.py # <7>
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ spec:
exposeConfig: false
credentialsSecret: simple-airflow-credentials
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ spec:
productVersion: 2.10.5
clusterConfig: {}
webservers:
config:
roleConfig:
listenerClass: external-unstable
envOverrides: &envOverrides
AIRFLOW__LOGGING__REMOTE_LOGGING: "True"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ spec:
productVersion: 2.10.5
clusterConfig: {}
webservers:
config:
roleConfig:
listenerClass: external-unstable
envOverrides: &envOverrides
AIRFLOW__CORE__XCOM_BACKEND: airflow.providers.common.io.xcom.backend.XComObjectStorageBackend
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ spec:
exposeConfig: false
credentialsSecret: simple-airflow-credentials
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
8 changes: 5 additions & 3 deletions docs/modules/airflow/pages/usage-guide/listenerclass.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,20 @@
:description: Configure Airflow service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable.

The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Webserver pod.
The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.config.listenerClass`:
The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.roleConfig.listenerClass`:

[source,yaml]
----
spec:
webservers:
config:
roleConfig:
listenerClass: external-unstable # <1>
config:
...
schedulers:
...
celeryExecutors:
...
----
<1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`).
<1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`) at role-level.
This can be set only for the webservers role.
2 changes: 1 addition & 1 deletion examples/simple-airflow-cluster-dags-cmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ spec:
mountPath: /dags/test_airflow_dag.py
subPath: test_airflow_dag.py
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
2 changes: 1 addition & 1 deletion examples/simple-airflow-cluster-ldap-insecure-tls.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ spec:
- authenticationClass: airflow-with-ldap-insecure-tls-ldap
userRegistrationRole: Admin
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
2 changes: 1 addition & 1 deletion examples/simple-airflow-cluster-ldap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ spec:
- authenticationClass: airflow-with-ldap-server-veri-tls-ldap
userRegistrationRole: Admin
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
2 changes: 1 addition & 1 deletion examples/simple-airflow-cluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ spec:
exposeConfig: false
credentialsSecret: simple-airflow-credentials
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
56 changes: 25 additions & 31 deletions rust/operator-binary/src/airflow_controller.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ use stackable_operator::{
core::{DeserializeGuard, error_boundary},
runtime::{controller::Action, reflector::ObjectRef},
},
kvp::{Annotation, Label, LabelError, Labels},
kvp::{Annotation, Label, LabelError, Labels, ObjectLabels},
logging::controller::ReconcilerError,
product_config_utils::{
CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, env_vars_from,
Expand Down Expand Up @@ -512,26 +512,6 @@ pub async fn reconcile_airflow(
&git_sync_resources,
)?;

if let Some(listener_class) =
airflow.merged_listener_class(&airflow_role, &rolegroup.role_group)
{
if let Some(listener_group_name) =
airflow.group_listener_name(&airflow_role, &rolegroup)
{
let rg_group_listener = build_group_listener(
airflow,
&resolved_product_image,
&rolegroup,
listener_class.to_string(),
listener_group_name,
)?;
cluster_resources
.add(client, rg_group_listener)
.await
.context(ApplyGroupListenerSnafu)?;
}
}

ss_cond_builder.add(
cluster_resources
.add(client, rg_statefulset)
Expand Down Expand Up @@ -568,6 +548,27 @@ pub async fn reconcile_airflow(
.await
.context(FailedToCreatePdbSnafu)?;
}

if let Some(listener_class) = airflow_role.listener_class_name(airflow) {
if let Some(listener_group_name) = airflow.group_listener_name(&airflow_role) {
let rg_group_listener = build_group_listener(
airflow,
build_recommended_labels(
airflow,
AIRFLOW_CONTROLLER_NAME,
&resolved_product_image.app_version_label,
role_name,
"none",
),
listener_class.to_string(),
listener_group_name,
)?;
cluster_resources
.add(client, rg_group_listener)
.await
.context(ApplyGroupListenerSnafu)?;
}
}
}

cluster_resources
Expand Down Expand Up @@ -840,8 +841,7 @@ fn build_rolegroup_metadata(

pub fn build_group_listener(
airflow: &v1alpha1::AirflowCluster,
resolved_product_image: &ResolvedProductImage,
rolegroup: &RoleGroupRef<v1alpha1::AirflowCluster>,
object_labels: ObjectLabels<v1alpha1::AirflowCluster>,
listener_class: String,
listener_group_name: String,
) -> Result<listener::v1alpha1::Listener> {
Expand All @@ -851,13 +851,7 @@ pub fn build_group_listener(
.name(listener_group_name)
.ownerreference_from_resource(airflow, None, Some(true))
.context(ObjectMissingMetadataForOwnerRefSnafu)?
.with_recommended_labels(build_recommended_labels(
airflow,
AIRFLOW_CONTROLLER_NAME,
&resolved_product_image.app_version_label,
&rolegroup.role,
&rolegroup.role_group,
))
.with_recommended_labels(object_labels)
.context(ObjectMetaSnafu)?
.build(),
spec: listener::v1alpha1::ListenerSpec {
Expand Down Expand Up @@ -1019,7 +1013,7 @@ fn build_server_rolegroup_statefulset(

let mut pvcs: Option<Vec<PersistentVolumeClaim>> = None;

if let Some(listener_group_name) = airflow.group_listener_name(airflow_role, rolegroup_ref) {
if let Some(listener_group_name) = airflow.group_listener_name(airflow_role) {
// Listener endpoints for the Webserver role will use persistent volumes
// so that load balancers can hard-code the target addresses. This will
// be the case even when no class is set (and the value defaults to
Expand Down
Loading