AppCred fixes and enhancing the AppCred kuttl tests

internal/openstack/applicationcredential.go

@@ -125,6 +125,11 @@ func EnsureApplicationCredentialForService(
 
 	// Check if AC CR exists and is ready
 	if acExists {
+		// We want to run reconcileApplicationCredential to update the AC CR if it exists and is ready and AC config fields changed
+		err = reconcileApplicationCredential(ctx, helper, instance, acName, serviceUser, secretName, passwordSelector, merged)
+		if err != nil {
+			return "", ctrl.Result{}, err
+		}
 		if acCR.IsReady() {
 			Log.Info("Application Credential is ready", "service", serviceName, "acName", acName, "secretName", acCR.Status.SecretName)
 			return acCR.Status.SecretName, ctrl.Result{}, nil
@@ -153,6 +158,31 @@ func EnsureApplicationCredentialForService(
 	return "", ctrl.Result{RequeueAfter: time.Second * 5}, nil
 }
 
+// CleanupApplicationCredential deletes the AC CR for a service if it exists.
+// Called when AC is disabled (globally or per-service) to clean up existing AC CRs.
+func CleanupApplicationCredential(
+	ctx context.Context,
+	helper *helper.Helper,
+	instance *corev1beta1.OpenStackControlPlane,
+	serviceName string,
+) error {
+	Log := GetLogger(ctx)
+	acName := keystonev1.GetACCRName(serviceName)
+	acCR := &keystonev1.KeystoneApplicationCredential{}
+	err := helper.GetClient().Get(ctx, types.NamespacedName{Name: acName, Namespace: instance.Namespace}, acCR)
+	if err != nil {
+		if k8s_errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	Log.Info("Application Credential disabled, deleting existing KeystoneApplicationCredential CR", "service", serviceName, "acName", acName)
+	if err := helper.GetClient().Delete(ctx, acCR); err != nil && !k8s_errors.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
 // reconcileApplicationCredential creates or updates a single ApplicationCredential CR
 func reconcileApplicationCredential(
 	ctx context.Context,

internal/openstack/barbican.go

@@ -101,6 +101,12 @@ func ReconcileBarbican(ctx context.Context, instance *corev1beta1.OpenStackContr
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Barbican.Template.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "barbican"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Barbican.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// preserve any previously set TLS certs, set CA cert

internal/openstack/cinder.go

@@ -124,6 +124,12 @@ func ReconcileCinder(ctx context.Context, instance *corev1beta1.OpenStackControl
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Cinder.Template.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "cinder"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Cinder.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// preserve any previously set TLS certs,set CA cert

internal/openstack/designate.go

@@ -113,6 +113,12 @@ func ReconcileDesignate(ctx context.Context, instance *corev1beta1.OpenStackCont
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Designate.Template.DesignateAPI.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "designate"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Designate.Template.DesignateAPI.Auth.ApplicationCredentialSecret = ""
 	}
 
 	svcs, err := service.GetServicesListWithLabel(

internal/openstack/glance.go

@@ -130,7 +130,6 @@ func ReconcileGlance(ctx context.Context, instance *corev1beta1.OpenStackControl
 
 	// Only call if AC enabled or currently configured
 	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Glance.ApplicationCredential) || hasACConfigured {
-
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(
 			ctx,
 			helper,
@@ -158,6 +157,15 @@ func ReconcileGlance(ctx context.Context, instance *corev1beta1.OpenStackControl
 			glanceAPI.Auth.ApplicationCredentialSecret = acSecretName
 			instance.Spec.Glance.Template.GlanceAPIs[name] = glanceAPI
 		}
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "glance"); err != nil {
+			return ctrl.Result{}, err
+		}
+		for name, glanceAPI := range instance.Spec.Glance.Template.GlanceAPIs {
+			glanceAPI.Auth.ApplicationCredentialSecret = ""
+			instance.Spec.Glance.Template.GlanceAPIs[name] = glanceAPI
+		}
 	}
 
 	// add selector to service overrides

internal/openstack/heat.go

@@ -144,6 +144,12 @@ func ReconcileHeat(ctx context.Context, instance *corev1beta1.OpenStackControlPl
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Heat.Template.Auth.ApplicationCredentialSecret = heatACSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "heat"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Heat.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// Heat API

internal/openstack/ironic.go

@@ -180,6 +180,16 @@ func ReconcileIronic(ctx context.Context, instance *corev1beta1.OpenStackControl
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Ironic.Template.IronicInspector.Auth.ApplicationCredentialSecret = inspectorACSecretName
+	} else {
+		// AC disabled - clean up any AC CRs
+		if err := CleanupApplicationCredential(ctx, helper, instance, "ironic"); err != nil {
+			return ctrl.Result{}, err
+		}
+		if err := CleanupApplicationCredential(ctx, helper, instance, "ironic-inspector"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Ironic.Template.Auth.ApplicationCredentialSecret = ""
+		instance.Spec.Ironic.Template.IronicInspector.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// Ironic API

internal/openstack/manila.go

@@ -103,6 +103,12 @@ func ReconcileManila(ctx context.Context, instance *corev1beta1.OpenStackControl
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Manila.Template.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "manila"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Manila.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// preserve any previously set TLS certs, set CA cert

internal/openstack/neutron.go

@@ -147,6 +147,12 @@ func ReconcileNeutron(ctx context.Context, instance *corev1beta1.OpenStackContro
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Neutron.Template.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "neutron"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Neutron.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	svcs, err := service.GetServicesListWithLabel(

internal/openstack/nova.go

@@ -219,6 +219,12 @@ func ReconcileNova(ctx context.Context, instance *corev1beta1.OpenStackControlPl
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Nova.Template.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "nova"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Nova.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// Nova API

internal/openstack/octavia.go

@@ -169,7 +169,7 @@ func ReconcileOctavia(ctx context.Context, instance *corev1beta1.OpenStackContro
 
 	// Only call if AC enabled or currently configured
 	if isACEnabled(instance.Spec.ApplicationCredential, instance.Spec.Octavia.ApplicationCredential) ||
-		instance.Spec.Octavia.Template.OctaviaAPI.Auth.ApplicationCredentialSecret != "" {
+		instance.Spec.Octavia.Template.Auth.ApplicationCredentialSecret != "" {
 
 		acSecretName, acResult, err := EnsureApplicationCredentialForService(
 			ctx,
@@ -194,7 +194,13 @@ func ReconcileOctavia(ctx context.Context, instance *corev1beta1.OpenStackContro
 		// Set ApplicationCredentialSecret based on what the helper returned:
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
-		instance.Spec.Octavia.Template.OctaviaAPI.Auth.ApplicationCredentialSecret = acSecretName
+		instance.Spec.Octavia.Template.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "octavia"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Octavia.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	svcs, err := service.GetServicesListWithLabel(

internal/openstack/placement.go

@@ -107,6 +107,12 @@ func ReconcilePlacementAPI(ctx context.Context, instance *corev1beta1.OpenStackC
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Placement.Template.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "placement"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Placement.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// set CA cert and preserve any previously set TLS certs

internal/openstack/swift.go

@@ -137,6 +137,12 @@ func ReconcileSwift(ctx context.Context, instance *corev1beta1.OpenStackControlP
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Swift.Template.SwiftProxy.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "swift"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Swift.Template.SwiftProxy.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// preserve any previously set TLS certs,set CA cert

internal/openstack/telemetry.go

@@ -161,9 +161,18 @@ func ReconcileTelemetry(ctx context.Context, instance *corev1beta1.OpenStackCont
 			// - If AC disabled: returns ""
 			// - If AC enabled and ready: returns the AC secret name
 			instance.Spec.Telemetry.Template.Autoscaling.Aodh.Auth.ApplicationCredentialSecret = aodhACSecretName
+		} else {
+			// AC disabled - clean up any AC CR
+			if err := CleanupApplicationCredential(ctx, helper, instance, "aodh"); err != nil {
+				return ctrl.Result{}, err
+			}
+			instance.Spec.Telemetry.Template.Autoscaling.Aodh.Auth.ApplicationCredentialSecret = ""
 		}
 	} else {
 		// Aodh service disabled, clear the field
+		if err := CleanupApplicationCredential(ctx, helper, instance, "aodh"); err != nil {
+			return ctrl.Result{}, err
+		}
 		instance.Spec.Telemetry.Template.Autoscaling.Aodh.Auth.ApplicationCredentialSecret = ""
 	}
 
@@ -203,9 +212,18 @@ func ReconcileTelemetry(ctx context.Context, instance *corev1beta1.OpenStackCont
 			// - If AC disabled: returns ""
 			// - If AC enabled and ready: returns the AC secret name
 			instance.Spec.Telemetry.Template.Ceilometer.Auth.ApplicationCredentialSecret = ceilometerACSecretName
+		} else {
+			// AC disabled - clean up any AC CR
+			if err := CleanupApplicationCredential(ctx, helper, instance, "ceilometer"); err != nil {
+				return ctrl.Result{}, err
+			}
+			instance.Spec.Telemetry.Template.Ceilometer.Auth.ApplicationCredentialSecret = ""
 		}
 	} else {
 		// Ceilometer service disabled, clear the field
+		if err := CleanupApplicationCredential(ctx, helper, instance, "ceilometer"); err != nil {
+			return ctrl.Result{}, err
+		}
 		instance.Spec.Telemetry.Template.Ceilometer.Auth.ApplicationCredentialSecret = ""
 	}
 
@@ -245,9 +263,18 @@ func ReconcileTelemetry(ctx context.Context, instance *corev1beta1.OpenStackCont
 			// - If AC disabled: returns ""
 			// - If AC enabled and ready: returns the AC secret name
 			instance.Spec.Telemetry.Template.CloudKitty.Auth.ApplicationCredentialSecret = cloudkittyACSecretName
+		} else {
+			// AC disabled - clean up any AC CR
+			if err := CleanupApplicationCredential(ctx, helper, instance, "cloudkitty"); err != nil {
+				return ctrl.Result{}, err
+			}
+			instance.Spec.Telemetry.Template.CloudKitty.Auth.ApplicationCredentialSecret = ""
 		}
 	} else {
 		// CloudKitty service disabled, clear the field
+		if err := CleanupApplicationCredential(ctx, helper, instance, "cloudkitty"); err != nil {
+			return ctrl.Result{}, err
+		}
 		instance.Spec.Telemetry.Template.CloudKitty.Auth.ApplicationCredentialSecret = ""
 	}
 

internal/openstack/watcher.go

@@ -116,6 +116,12 @@ func ReconcileWatcher(ctx context.Context, instance *corev1beta1.OpenStackContro
 		// - If AC disabled: returns ""
 		// - If AC enabled and ready: returns the AC secret name
 		instance.Spec.Watcher.Template.Auth.ApplicationCredentialSecret = acSecretName
+	} else {
+		// AC disabled - clean up any AC CR
+		if err := CleanupApplicationCredential(ctx, helper, instance, "watcher"); err != nil {
+			return ctrl.Result{}, err
+		}
+		instance.Spec.Watcher.Template.Auth.ApplicationCredentialSecret = ""
 	}
 
 	// preserve any previously set TLS certs, set CA cert

test/kuttl/tests/ctlplane-basic-deployment-with-appcred/03-assert-appcred-cleanup.yaml

@@ -0,0 +1,51 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |-
+      set -euo pipefail
+      NS="${NAMESPACE}"
+
+      SERVICES=(barbican cinder glance swift neutron placement nova ceilometer)
+
+      wait_deleted() {
+        local kind=$1 name=$2 timeout=${3:-180}
+        echo "Waiting for $kind/$name to be deleted..."
+        local end=$((SECONDS + timeout))
+        while [ $SECONDS -lt $end ]; do
+          if ! oc get "$kind" "$name" -n "$NS" &>/dev/null; then
+            echo "✓ $kind/$name deleted"
+            return 0
+          fi
+          sleep 5
+        done
+        echo "ERROR: $kind/$name still exists after ${timeout}s"
+        exit 1
+      }
+
+      echo "========================================="
+      echo "Testing Application Credential Cleanup"
+      echo "========================================="
+      echo
+
+      echo "=== Verifying global ApplicationCredential is disabled ==="
+      global_enabled=$(oc get openstackcontrolplane openstack -n "$NS" -o jsonpath='{.spec.applicationCredential.enabled}')
+      if [ "$global_enabled" != "false" ]; then
+        echo "ERROR: OpenStackControlPlane.spec.applicationCredential.enabled expected 'false', got '$global_enabled'"
+        exit 1
+      fi
+      echo "✓ OpenStackControlPlane.spec.applicationCredential.enabled = false"
+      echo
+
+      echo "=== Verifying AC CRs are deleted ==="
+      for svc in "${SERVICES[@]}"; do
+        wait_deleted appcred "ac-$svc" 180
+      done
+      echo
+
+      echo "=== Verifying AC Secrets are cleaned up ==="
+      for svc in "${SERVICES[@]}"; do
+        wait_deleted secret "ac-$svc-secret" 120
+      done
+      echo
+
+      echo "All ApplicationCredential CRs and Secrets cleaned up successfully"

test/kuttl/tests/ctlplane-basic-deployment-with-appcred/03-disable-appcred-config.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc patch openstackcontrolplane openstack -n $NAMESPACE --type merge -p '{"spec":{"applicationCredential":{"enabled":false}}}'