From c053f8794661006969e779b37c31e9bbb457b58d Mon Sep 17 00:00:00 2001 From: Predrag Knezevic Date: Wed, 4 Mar 2026 16:42:37 +0100 Subject: [PATCH 1/2] Migrate operator upgrade e2e tests to Godog/Cucumber BDD framework Replace testify-based upgrade tests with a Godog test runner and Gherkin feature file. Add step definitions for OLM install/upgrade, component readiness via leader election leases, and reconciliation verification. Simplify Makefile upgrade targets using a reusable install script macro and fix template path resolution so tests work from any directory. Co-Authored-By: Claude Opus 4.6 --- Makefile | 60 +-- hack/test/pre-upgrade-setup.sh | 172 --------- test/e2e/steps/hooks.go | 36 +- test/e2e/steps/steps.go | 56 ++- test/e2e/steps/testdata/rbac-template.yaml | 2 +- test/e2e/steps/upgrade_steps.go | 264 ++++++++++++++ .../features/operator-upgrade.feature | 45 +++ test/upgrade-e2e/post_upgrade_test.go | 341 ------------------ test/upgrade-e2e/upgrade_e2e_suite_test.go | 60 --- test/upgrade-e2e/upgrade_test.go | 42 +++ 10 files changed, 455 insertions(+), 623 deletions(-) delete mode 100755 hack/test/pre-upgrade-setup.sh create mode 100644 test/e2e/steps/upgrade_steps.go create mode 100644 test/upgrade-e2e/features/operator-upgrade.feature delete mode 100644 test/upgrade-e2e/post_upgrade_test.go delete mode 100644 test/upgrade-e2e/upgrade_e2e_suite_test.go create mode 100644 test/upgrade-e2e/upgrade_test.go diff --git a/Makefile b/Makefile index 41397e9cc4..9131c5af02 100644 --- a/Makefile +++ b/Makefile @@ -238,6 +238,17 @@ verify-crd-compatibility: $(CRD_DIFF) manifests #SECTION Test +define install-sh +.PHONY: $(1)/install.sh +$(1)/install.sh: manifests + @echo -e "\n\U1F4D8 Using $(1).yaml as source manifest\n" + sed "s/cert-git-version/cert-$$(VERSION)/g" manifests/$(1).yaml > $(2) + MANIFEST=$(2) INSTALL_DEFAULT_CATALOGS=false DEFAULT_CATALOG=$$(RELEASE_CATALOGS) envsubst '$$$$DEFAULT_CATALOG,$$$$CERT_MGR_VERSION,$$$$INSTALL_DEFAULT_CATALOGS,$$$$MANIFEST' < scripts/install.tpl.sh > $(1)-install.sh +endef + +$(eval $(call install-sh,experimental,operator-controller-experimental.yaml)) +$(eval $(call install-sh,standard,operator-controller-standard.yaml)) + .PHONY: test test: manifests generate fmt lint test-unit test-e2e test-regression #HELP Run all tests. @@ -335,52 +346,43 @@ run-latest-release: @echo -e "\n\U23EC Using $(RELEASE_INSTALL) as release installer\n" curl -L -s https://github.com/operator-framework/operator-controller/releases/latest/download/$(notdir $(RELEASE_INSTALL)) | bash -s -.PHONY: pre-upgrade-setup -pre-upgrade-setup: - ./hack/test/pre-upgrade-setup.sh $(CATALOG_IMG) $(TEST_CLUSTER_CATALOG_NAME) $(TEST_CLUSTER_EXTENSION_NAME) - -.PHONY: post-upgrade-checks -post-upgrade-checks: - go test -count=1 -v ./test/upgrade-e2e/... +.PHONY: test-upgrade-e2e +test-upgrade-e2e: + RELEASE_INSTALL=$(RELEASE_INSTALL) \ + RELEASE_UPGRADE=$(RELEASE_UPGRADE) \ + KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) \ + ROOT_DIR=$(ROOT_DIR) \ + go test -count=1 -v ./test/upgrade-e2e/upgrade_test.go -TEST_UPGRADE_E2E_TASKS := kind-cluster run-latest-release image-registry pre-upgrade-setup docker-build kind-load kind-deploy post-upgrade-checks kind-clean +TEST_UPGRADE_E2E_TASKS := kind-cluster docker-build kind-load test-upgrade-e2e kind-clean .PHONY: test-upgrade-st2st-e2e -test-upgrade-st2st-e2e: SOURCE_MANIFEST := $(STANDARD_MANIFEST) -test-upgrade-st2st-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL) +test-upgrade-st2st-e2e: RELEASE_INSTALL := https://github.com/operator-framework/operator-controller/releases/latest/download/install.sh +test-upgrade-st2st-e2e: RELEASE_UPGRADE := $(ROOT_DIR)/standard-install.sh test-upgrade-st2st-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-st2st-e2e -test-upgrade-st2st-e2e: export MANIFEST := $(STANDARD_RELEASE_MANIFEST) -test-upgrade-st2st-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog -test-upgrade-st2st-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package -test-upgrade-st2st-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> standard) e2e tests on a local kind cluster +test-upgrade-st2st-e2e: standard/install.sh $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> standard) e2e tests on a local kind cluster .PHONY: test-upgrade-ex2ex-e2e -test-upgrade-ex2ex-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_MANIFEST) -test-upgrade-ex2ex-e2e: RELEASE_INSTALL := $(EXPERIMENTAL_RELEASE_INSTALL) +test-upgrade-ex2ex-e2e: RELEASE_INSTALL := https://github.com/operator-framework/operator-controller/releases/latest/download/install-experimental.sh test-upgrade-ex2ex-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-ex2ex-e2e -test-upgrade-ex2ex-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) -test-upgrade-ex2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog -test-upgrade-ex2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package -test-upgrade-ex2ex-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (experimental -> experimental) e2e tests on a local kind cluster +test-upgrade-ex2ex-e2e: RELEASE_UPGRADE := $(ROOT_DIR)/experimental-install.sh +test-upgrade-ex2ex-e2e: experimental/install.sh $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (experimental -> experimental) e2e tests on a local kind cluster .PHONY: test-upgrade-st2ex-e2e -test-upgrade-st2ex-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_MANIFEST) -test-upgrade-st2ex-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL) +test-upgrade-st2ex-e2e: RELEASE_INSTALL := https://github.com/operator-framework/operator-controller/releases/latest/download/install.sh +test-upgrade-st2ex-e2e: RELEASE_UPGRADE := $(ROOT_DIR)/experimental-install.sh test-upgrade-st2ex-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-st2ex-e2e -test-upgrade-st2ex-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) -test-upgrade-st2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog -test-upgrade-st2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package -test-upgrade-st2ex-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> experimental) e2e tests on a local kind cluster +test-upgrade-st2ex-e2e: experimental/install.sh $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> experimental) e2e tests on a local kind cluster .PHONY: test-st2ex-e2e -test-st2ex-e2e: SOURCE_MANIFEST := $(STANDARD_MANIFEST) -test-st2ex-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL) +test-st2ex-e2e: RELEASE_INSTALL := $(ROOT_DIR)/standard-install.sh +test-st2ex-e2e: RELEASE_UPGRADE := $(ROOT_DIR)/experimental-install.sh test-st2ex-e2e: KIND_CLUSTER_NAME := operator-controller-st2ex-e2e test-st2ex-e2e: export MANIFEST := $(STANDARD_RELEASE_MANIFEST) test-st2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog test-st2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package -test-st2ex-e2e: run-internal image-registry pre-upgrade-setup kind-deploy-experimental post-upgrade-checks kind-clean #HELP Run swichover (standard -> experimental) e2e tests on a local kind cluster +test-st2ex-e2e: experimental/install.sh standard/install.sh $(TEST_UPGRADE_E2E_TASKS) #HELP Run swichover (standard -> experimental) e2e tests on a local kind cluster .PHONY: e2e-coverage e2e-coverage: diff --git a/hack/test/pre-upgrade-setup.sh b/hack/test/pre-upgrade-setup.sh deleted file mode 100755 index f2dc28761c..0000000000 --- a/hack/test/pre-upgrade-setup.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -help="pre-upgrade-setup.sh is used to create some basic resources -which will later be used in upgrade testing. - -Usage: - post-upgrade-checks.sh [TEST_CATALOG_IMG] [TEST_CATALOG_NAME] [TEST_CLUSTER_EXTENSION_NAME] -" - -if [[ "$#" -ne 3 ]]; then - echo "Illegal number of arguments passed" - echo "${help}" - exit 1 -fi - -TEST_CATALOG_IMG=$1 -TEST_CLUSTER_CATALOG_NAME=$2 -TEST_CLUSTER_EXTENSION_NAME=$3 - -kubectl apply -f - << EOF -apiVersion: olm.operatorframework.io/v1 -kind: ClusterCatalog -metadata: - name: ${TEST_CLUSTER_CATALOG_NAME} -spec: - source: - type: Image - image: - ref: ${TEST_CATALOG_IMG} - pollIntervalMinutes: 1440 -EOF - -kubectl apply -f - < leader pod name extensionObjects []client.Object } @@ -87,30 +89,37 @@ func RegisterHooks(sc *godog.ScenarioContext) { sc.After(ScenarioCleanup) } -func BeforeSuite() { - if devMode { - logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) - } else { - logger = textlogger.NewLogger(textlogger.NewConfig()) - } - +func detectOLMDeployment() (*appsv1.Deployment, error) { raw, err := k8sClient("get", "deployments", "-A", "-l", "app.kubernetes.io/part-of=olm", "-o", "jsonpath={.items}") if err != nil { - panic(fmt.Errorf("failed to get OLM deployments: %v", err)) + return nil, err } dl := []appsv1.Deployment{} if err := json.Unmarshal([]byte(raw), &dl); err != nil { - panic(fmt.Errorf("failed to unmarshal OLM deployments: %v", err)) + return nil, fmt.Errorf("failed to unmarshal OLM deployments: %v", err) } - var olm *appsv1.Deployment for _, d := range dl { if d.Name == olmDeploymentName { - olm = &d - olmNamespace = d.Namespace - break + return &d, nil } } + return nil, fmt.Errorf("failed to detect OLM Deployment") +} + +func BeforeSuite() { + if devMode { + logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1))) + } else { + logger = textlogger.NewLogger(textlogger.NewConfig()) + } + + olm, err := detectOLMDeployment() + if err != nil { + logger.Info("OLM deployments not found; skipping feature gate detection (upgrade scenarios will install OLM in Background)") + return + } + olmNamespace = olm.Namespace featureGatePattern := regexp.MustCompile(`--feature-gates=([[:alnum:]]+)=(true|false)`) for _, c := range olm.Spec.Template.Spec.Containers { @@ -144,6 +153,7 @@ func CreateScenarioContext(ctx context.Context, sc *godog.Scenario) (context.Con id: sc.Id, namespace: fmt.Sprintf("ns-%s", sc.Id), clusterExtensionName: fmt.Sprintf("ce-%s", sc.Id), + leaderPods: make(map[string]string), } return context.WithValue(ctx, scenarioContextKey, scCtx), nil } diff --git a/test/e2e/steps/steps.go b/test/e2e/steps/steps.go index abea3fcb22..6a827228a3 100644 --- a/test/e2e/steps/steps.go +++ b/test/e2e/steps/steps.go @@ -14,7 +14,9 @@ import ( "os/exec" "path/filepath" "reflect" + "runtime" "strings" + "sync" "time" "github.com/cucumber/godog" @@ -47,9 +49,24 @@ const ( ) var ( - olmNamespace = "olmv1-system" - kubeconfigPath string - k8sCli string + olmNamespace = "olmv1-system" + kubeconfigPath string + k8sCli string + deployImageRegistry = sync.OnceValue(func() error { + if os.Getenv("KIND_CLUSTER_NAME") == "" { + return nil + } + cmd := exec.Command("bash", "-c", "make image-registry") + dir, _ := os.LookupEnv("ROOT_DIR") + if dir == "" { + return fmt.Errorf("ROOT_DIR environment variable not set") + } + cmd.Dir = dir + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + }) ) func RegisterSteps(sc *godog.ScenarioContext) { @@ -89,6 +106,7 @@ func RegisterSteps(sc *godog.ScenarioContext) { sc.Step(`^(?i)resource "([^"]+)" is eventually restored$`, ResourceRestored) sc.Step(`^(?i)resource "([^"]+)" matches$`, ResourceMatches) + sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in "([^"]*)" namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInGivenNamespace) sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in test namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInNamespace) sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in \${TEST_NAMESPACE}$`, ServiceAccountWithNeededPermissionsIsAvailableInNamespace) sc.Step(`^(?i)ServiceAccount "([^"]*)" is available in \${TEST_NAMESPACE}$`, ServiceAccountIsAvailableInNamespace) @@ -108,6 +126,14 @@ func RegisterSteps(sc *godog.ScenarioContext) { sc.Step(`^(?i)Prometheus metrics are returned in the response$`, PrometheusMetricsAreReturned) sc.Step(`^(?i)min value for (ClusterExtension|ClusterExtensionRevision) ((?:\.[a-zA-Z]+)+) is set to (\d+)$`, SetCRDFieldMinValue) + + // Upgrade-specific steps + sc.Step(`^(?i)the latest stable OLM release is installed$`, LatestStableOLMReleaseIsInstalled) + sc.Step(`^(?i)OLM is upgraded$`, OLMIsUpgraded) + sc.Step(`^(?i)(catalogd|operator-controller) is ready to reconcile resources$`, ComponentIsReadyToReconcile) + sc.Step(`^(?i)all (ClusterCatalog|ClusterExtension) resources are reconciled$`, AllResourcesAreReconciled) + sc.Step(`^(?i)(ClusterCatalog|ClusterExtension) is reconciled$`, ResourceTypeIsReconciled) + sc.Step(`^(?i)ClusterCatalog reports ([[:alnum:]]+) as ([[:alnum:]]+) with Reason ([[:alnum:]]+)$`, ClusterCatalogReportsCondition) } func init() { @@ -768,7 +794,8 @@ func applyServiceAccount(ctx context.Context, serviceAccount string) error { "SERVICEACCOUNT_NAME": serviceAccount, }) - yaml, err := templateYaml(filepath.Join("steps", "testdata", "serviceaccount-template.yaml"), vars) + _, thisFile, _, _ := runtime.Caller(0) + yaml, err := templateYaml(filepath.Join(filepath.Dir(thisFile), "testdata", "serviceaccount-template.yaml"), vars) if err != nil { return fmt.Errorf("failed to template ServiceAccount yaml: %v", err) } @@ -795,7 +822,8 @@ func applyPermissionsToServiceAccount(ctx context.Context, serviceAccount, rbacT "CLUSTEREXTENSION_NAME": sc.clusterExtensionName, }, keyValue...) - yaml, err := templateYaml(filepath.Join("steps", "testdata", rbacTemplate), vars) + _, thisFile, _, _ := runtime.Caller(0) + yaml, err := templateYaml(filepath.Join(filepath.Dir(thisFile), "testdata", rbacTemplate), vars) if err != nil { return fmt.Errorf("failed to template RBAC yaml: %v", err) } @@ -814,6 +842,13 @@ func ServiceAccountIsAvailableInNamespace(ctx context.Context, serviceAccount st return applyServiceAccount(ctx, serviceAccount) } +// ServiceAccountWithNeededPermissionsIsAvailableInNamespace creates a ServiceAccount and applies standard RBAC permissions. +func ServiceAccountWithNeededPermissionsIsAvailableInGivenNamespace(ctx context.Context, serviceAccount string, ns string) error { + sc := scenarioCtx(ctx) + sc.namespace = ns + return applyPermissionsToServiceAccount(ctx, serviceAccount, "rbac-template.yaml") +} + // ServiceAccountWithNeededPermissionsIsAvailableInNamespace creates a ServiceAccount and applies standard RBAC permissions. func ServiceAccountWithNeededPermissionsIsAvailableInNamespace(ctx context.Context, serviceAccount string) error { return applyPermissionsToServiceAccount(ctx, serviceAccount, "rbac-template.yaml") @@ -972,12 +1007,19 @@ func CatalogIsUpdatedToVersion(name, version string) error { // CatalogServesBundles applies the ClusterCatalog YAML template to create a catalog that serves bundles. func CatalogServesBundles(ctx context.Context, catalogName string) error { - yamlContent, err := os.ReadFile(filepath.Join("steps", "testdata", fmt.Sprintf("%s-catalog-template.yaml", catalogName))) + if err := deployImageRegistry(); err != nil { + return err + } + sc := scenarioCtx(ctx) + sc.clusterCatalogName = fmt.Sprintf("%s-catalog", catalogName) + + _, thisFile, _, _ := runtime.Caller(0) + yamlContent, err := os.ReadFile(filepath.Join(filepath.Dir(thisFile), "testdata", fmt.Sprintf("%s-catalog-template.yaml", catalogName))) if err != nil { return fmt.Errorf("failed to read catalog yaml: %v", err) } - _, err = k8scliWithInput(substituteScenarioVars(string(yamlContent), scenarioCtx(ctx)), "apply", "-f", "-") + _, err = k8scliWithInput(substituteScenarioVars(string(yamlContent), sc), "apply", "-f", "-") if err != nil { return fmt.Errorf("failed to apply catalog: %v", err) } diff --git a/test/e2e/steps/testdata/rbac-template.yaml b/test/e2e/steps/testdata/rbac-template.yaml index 90138b9c69..b91cb1e220 100644 --- a/test/e2e/steps/testdata/rbac-template.yaml +++ b/test/e2e/steps/testdata/rbac-template.yaml @@ -11,7 +11,7 @@ metadata: rules: - apiGroups: [olm.operatorframework.io] resources: [clusterextensions, clusterextensions/finalizers] - resourceNames: ["${CLUSTEREXTENSION_NAME}"] + resourceNames: ["${CLUSTEREXTENSION_NAME}", "upgrade-ce"] verbs: [update] # Allow ClusterExtensionRevisions to set blockOwnerDeletion ownerReferences - apiGroups: [olm.operatorframework.io] diff --git a/test/e2e/steps/upgrade_steps.go b/test/e2e/steps/upgrade_steps.go new file mode 100644 index 0000000000..364a15f258 --- /dev/null +++ b/test/e2e/steps/upgrade_steps.go @@ -0,0 +1,264 @@ +package steps + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "os/exec" + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + catalogdv1 "github.com/operator-framework/operator-controller/api/v1" +) + +// leaseNames maps component labels to their leader election lease names. +var leaseNames = map[string]string{ + "catalogd": "catalogd-operator-lock", + "operator-controller": "9c4404e7.operatorframework.io", +} + +var ( + installOperator = sync.OnceValue(func() error { + err := runInstallScript("RELEASE_INSTALL") + if err != nil { + return err + } + olm, err := detectOLMDeployment() + if err != nil { + return err + } + olmNamespace = olm.Namespace + return nil + }) + upgradeOperator = sync.OnceValue(func() error { + return runInstallScript("RELEASE_UPGRADE") + }) +) + +// LatestStableOLMReleaseIsInstalled downloads and executes the latest stable OLM release install script. +// Uses sync.Once to ensure the install only happens once across multiple scenarios. +func LatestStableOLMReleaseIsInstalled(_ context.Context) error { + return installOperator() +} + +func runInstallScript(envVar string) error { + scriptPath, found := os.LookupEnv(envVar) + if !found { + return fmt.Errorf("missing %s env variable", envVar) + } + if scriptPath == "" { + return fmt.Errorf("%s environment variable must contain install script location", envVar) + } + var cmd *exec.Cmd + if u, err := url.Parse(scriptPath); err == nil && u.Scheme != "" { + cmd = exec.Command("bash", "-c", fmt.Sprintf("curl -L -s %s | bash -s", scriptPath)) //nolint:gosec // scriptPath is from a trusted env variable + } else { + cmd = exec.Command("bash", scriptPath) + } + dir, _ := os.LookupEnv("ROOT_DIR") + if dir == "" { + return fmt.Errorf("ROOT_DIR environment variable not set") + } + cmd.Dir = dir + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// OLMIsUpgraded applies the locally built OLM manifest to upgrade OLM. +// Uses sync.Once to ensure the upgrade only happens once across multiple scenarios. +func OLMIsUpgraded(ctx context.Context) error { + return upgradeOperator() +} + +// ComponentIsReadyToReconcile waits for the named component's deployment to be fully rolled out, +// then checks the leader election lease and stores the leader pod name in the scenario context. +func ComponentIsReadyToReconcile(ctx context.Context, component string) error { + sc := scenarioCtx(ctx) + + // Wait for deployment rollout to complete + depName, err := k8sClient("get", "deployments", "-n", olmNamespace, + "-l", fmt.Sprintf("app.kubernetes.io/name=%s", component), + "-o", "jsonpath={.items[0].metadata.name}") + if err != nil { + return fmt.Errorf("failed to find deployment for component %s: %w", component, err) + } + if depName == "" { + return fmt.Errorf("failed to find deployment for component %s: no matching deployments found", component) + } + if _, err := k8sClient("rollout", "status", fmt.Sprintf("deployment/%s", depName), + "-n", olmNamespace, fmt.Sprintf("--timeout=%s", timeout)); err != nil { + return fmt.Errorf("deployment rollout failed for %s: %w", component, err) + } + + // Check leader election lease + leaseName, ok := leaseNames[component] + if !ok { + return fmt.Errorf("unknown component: %s", component) + } + + // Leader election can take up to LeaseDuration (137s) + RetryPeriod (26s) ≈ 163s in the worst case + waitFor(ctx, func() bool { + output, err := k8sClient("get", "lease", leaseName, "-n", olmNamespace, + "-o", "jsonpath={.spec.holderIdentity}") + if err != nil || output == "" { + return false + } + // Extract pod name from lease holder identity (format: _) + podName := strings.Split(output, "_")[0] + sc.leaderPods[component] = podName + return true + }) + + logger.V(1).Info("Component is ready to reconcile", "component", component, "leaderPod", sc.leaderPods[component]) + return nil +} + +// resourceTypeToComponent maps resource type names to their controller component labels. +var resourceTypeToComponent = map[string]string{ + "ClusterCatalog": "catalogd", + "ClusterExtension": "operator-controller", +} + +// reconcileEndingCheck returns a function that checks whether the leader pod's logs +// contain a "reconcile ending" entry for the given resource name. +func reconcileEndingCheck(leaderPod, resourceName string) func() bool { + return func() bool { + logs, err := k8sClient("logs", leaderPod, "-n", olmNamespace, "--all-containers=true", "--tail=1000") + if err != nil { + return false + } + for _, line := range strings.Split(logs, "\n") { + if strings.Contains(line, "reconcile ending") && strings.Contains(line, resourceName) { + return true + } + } + return false + } +} + +// ResourceTypeIsReconciled waits for the component's deployment to be ready and +// then verifies that the scenario's resource of the given type has been reconciled +// by checking the leader pod's logs for a "reconcile ending" entry. +func ResourceTypeIsReconciled(ctx context.Context, resourceType string) error { + sc := scenarioCtx(ctx) + + component, ok := resourceTypeToComponent[resourceType] + if !ok { + return fmt.Errorf("unknown resource type: %s", resourceType) + } + + var resourceName string + switch resourceType { + case "ClusterCatalog": + resourceName = sc.clusterCatalogName + case "ClusterExtension": + resourceName = sc.clusterExtensionName + } + if resourceName == "" { + return fmt.Errorf("no %s found in scenario context", resourceType) + } + + if err := ComponentIsReadyToReconcile(ctx, component); err != nil { + return err + } + + leaderPod := sc.leaderPods[component] + waitFor(ctx, reconcileEndingCheck(leaderPod, resourceName)) + + // For ClusterCatalog, also verify that lastUnpacked is after the leader pod's creation. + // This mitigates flakiness caused by https://github.com/operator-framework/operator-controller/issues/1626 + if resourceType == "ClusterCatalog" { + waitFor(ctx, clusterCatalogUnpackedAfterPodCreation(resourceName, leaderPod)) + } + + return nil +} + +// clusterCatalogUnpackedAfterPodCreation returns a check function that verifies the +// ClusterCatalog is serving and its lastUnpacked timestamp is after the leader pod's creation. +func clusterCatalogUnpackedAfterPodCreation(resourceName, leaderPod string) func() bool { + return func() bool { + catalogJSON, err := k8sClient("get", "clustercatalog", resourceName, "-o", "json") + if err != nil { + return false + } + var catalog catalogdv1.ClusterCatalog + if err := json.Unmarshal([]byte(catalogJSON), &catalog); err != nil { + return false + } + + serving := apimeta.FindStatusCondition(catalog.Status.Conditions, catalogdv1.TypeServing) + if serving == nil || serving.Status != metav1.ConditionTrue || serving.Reason != catalogdv1.ReasonAvailable { + return false + } + progressing := apimeta.FindStatusCondition(catalog.Status.Conditions, catalogdv1.TypeProgressing) + if progressing == nil || progressing.Status != metav1.ConditionTrue || progressing.Reason != catalogdv1.ReasonSucceeded { + return false + } + + if catalog.Status.LastUnpacked == nil { + return false + } + + podJSON, err := k8sClient("get", "pod", leaderPod, "-n", olmNamespace, "-o", "json") + if err != nil { + return false + } + var pod corev1.Pod + if err := json.Unmarshal([]byte(podJSON), &pod); err != nil { + return false + } + + return catalog.Status.LastUnpacked.After(pod.CreationTimestamp.Time) + } +} + +// AllResourcesAreReconciled discovers all resources of the given type and verifies +// that the leader pod's logs contain a "reconcile ending" entry for each resource. +func AllResourcesAreReconciled(ctx context.Context, resourceType string) error { + sc := scenarioCtx(ctx) + + component, ok := resourceTypeToComponent[resourceType] + if !ok { + return fmt.Errorf("unknown resource type: %s", resourceType) + } + + leaderPod, ok := sc.leaderPods[component] + if !ok { + return fmt.Errorf("leader pod not found for component %s; run '%s is ready to reconcile resources' first", component, component) + } + + // Discover all resources + pluralType := strings.ToLower(resourceType) + "s" + output, err := k8sClient("get", pluralType, "-o", "jsonpath={.items[*].metadata.name}") + if err != nil { + return fmt.Errorf("failed to list %s resources: %w", resourceType, err) + } + resourceNames := strings.Fields(output) + if len(resourceNames) == 0 { + return fmt.Errorf("no %s resources found", resourceType) + } + + for _, name := range resourceNames { + waitFor(ctx, reconcileEndingCheck(leaderPod, name)) + } + + return nil +} + +// ClusterCatalogReportsCondition waits for the ClusterCatalog to have the specified condition type, status, and reason. +func ClusterCatalogReportsCondition(ctx context.Context, conditionType, conditionStatus, conditionReason string) error { + sc := scenarioCtx(ctx) + if sc.clusterCatalogName == "" { + return fmt.Errorf("cluster catalog name not set; run 'ClusterCatalog serves bundles' first") + } + return waitForCondition(ctx, "clustercatalog", sc.clusterCatalogName, conditionType, conditionStatus, &conditionReason, nil) +} diff --git a/test/upgrade-e2e/features/operator-upgrade.feature b/test/upgrade-e2e/features/operator-upgrade.feature new file mode 100644 index 0000000000..5a4b909491 --- /dev/null +++ b/test/upgrade-e2e/features/operator-upgrade.feature @@ -0,0 +1,45 @@ +@operator-upgrade +Feature: Operator upgrade verification + + As an OLM developer I would like to verify that after upgrading OLM itself, + pre-existing ClusterCatalogs and ClusterExtensions continue to function + and can be updated. + + Background: + Given the latest stable OLM release is installed + And ClusterCatalog "test" serves bundles + And ServiceAccount "olm-sa" with needed permissions is available in "upgrade-ns" namespace + And ClusterExtension is applied + """ + apiVersion: olm.operatorframework.io/v1 + kind: ClusterExtension + metadata: + name: upgrade-ce + spec: + namespace: upgrade-ns + serviceAccount: + name: olm-sa + source: + sourceType: Catalog + catalog: + packageName: test + selector: + matchLabels: + "olm.operatorframework.io/metadata.name": test-catalog + version: 1.0.0 + """ + And ClusterExtension is available + And OLM is upgraded + And catalogd is ready to reconcile resources + And operator-controller is ready to reconcile resources + + Scenario: ClusterCatalog continues unpacking after OLM upgrade + When ClusterCatalog is reconciled + Then ClusterCatalog reports Progressing as True with Reason Succeeded + And ClusterCatalog reports Serving as True with Reason Available + + Scenario: ClusterExtension remains functional after OLM upgrade + Given ClusterExtension is reconciled + When ClusterExtension is updated to version "1.0.1" + Then ClusterExtension is available + And bundle "test-operator.1.0.1" is installed in version "1.0.1" diff --git a/test/upgrade-e2e/post_upgrade_test.go b/test/upgrade-e2e/post_upgrade_test.go deleted file mode 100644 index 8293a6a493..0000000000 --- a/test/upgrade-e2e/post_upgrade_test.go +++ /dev/null @@ -1,341 +0,0 @@ -package upgradee2e - -import ( - "bufio" - "context" - "fmt" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - coordinationv1 "k8s.io/api/coordination/v1" - corev1 "k8s.io/api/core/v1" - apimeta "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - - ocv1 "github.com/operator-framework/operator-controller/api/v1" - testutil "github.com/operator-framework/operator-controller/internal/shared/util/test" -) - -const ( - artifactName = "operator-controller-upgrade-e2e" - container = "manager" -) - -func TestClusterCatalogUnpacking(t *testing.T) { - ctx := context.Background() - - t.Log("Checking that the controller-manager deployment is updated") - managerLabelSelector := labels.Set{"app.kubernetes.io/name": "catalogd"} - var managerDeployment appsv1.Deployment - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var managerDeployments appsv1.DeploymentList - err := c.List(ctx, &managerDeployments, client.MatchingLabels(managerLabelSelector), client.InNamespace("olmv1-system")) - require.NoError(ct, err) - require.Len(ct, managerDeployments.Items, 1) - managerDeployment = managerDeployments.Items[0] - require.Equal(ct, *managerDeployment.Spec.Replicas, managerDeployment.Status.UpdatedReplicas) - require.Equal(ct, *managerDeployment.Spec.Replicas, managerDeployment.Status.Replicas) - require.Equal(ct, *managerDeployment.Spec.Replicas, managerDeployment.Status.AvailableReplicas) - require.Equal(ct, *managerDeployment.Spec.Replicas, managerDeployment.Status.ReadyReplicas) - }, time.Minute, time.Second) - - var managerPod corev1.Pod - t.Log("Waiting for only one controller-manager pod to remain") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var managerPods corev1.PodList - err := c.List(ctx, &managerPods, client.MatchingLabels(managerLabelSelector)) - require.NoError(ct, err) - require.Len(ct, managerPods.Items, 1) - managerPod = managerPods.Items[0] - }, time.Minute, time.Second) - - t.Logf("Waiting for acquired leader election by checking lease") - // Instead of watching logs (which has timing issues in controller-runtime v0.23.1+), - // directly check the lease object to confirm leader election occurred - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var lease coordinationv1.Lease - err := c.Get(ctx, types.NamespacedName{Name: "catalogd-operator-lock", Namespace: "olmv1-system"}, &lease) - require.NoError(ct, err) - require.NotNil(ct, lease.Spec.HolderIdentity, "lease should have a holder") - require.NotEmpty(ct, *lease.Spec.HolderIdentity, "lease holder identity should not be empty") - }, 3*time.Minute, time.Second) - - t.Log("Reading logs to make sure that ClusterCatalog was reconciled by catalogdv1") - logCtx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - substrings := []string{ - "reconcile ending", - fmt.Sprintf(`ClusterCatalog=%q`, testClusterCatalogName), - } - found, err := watchPodLogsForSubstring(logCtx, &managerPod, substrings...) - require.NoError(t, err) - require.True(t, found) - - catalog := &ocv1.ClusterCatalog{} - t.Log("Ensuring ClusterCatalog has Status.Condition of Progressing with a status == True, reason == Succeeded") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - err := c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, catalog) - require.NoError(ct, err) - cond := apimeta.FindStatusCondition(catalog.Status.Conditions, ocv1.TypeProgressing) - require.NotNil(ct, cond) - require.Equal(ct, metav1.ConditionTrue, cond.Status) - require.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) - }, time.Minute, time.Second) - - t.Log("Ensuring ClusterCatalog has Status.Condition of Serving with a status == True, reason == Available") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - err := c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, catalog) - require.NoError(ct, err) - cond := apimeta.FindStatusCondition(catalog.Status.Conditions, ocv1.TypeServing) - require.NotNil(ct, cond) - require.Equal(ct, metav1.ConditionTrue, cond.Status) - require.Equal(ct, ocv1.ReasonAvailable, cond.Reason) - }, time.Minute, time.Second) -} - -func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { - t.Log("Starting checks after OLM upgrade") - ctx := context.Background() - defer testutil.CollectTestArtifacts(t, artifactName, c, cfg) - - // wait for catalogd deployment to finish - t.Log("Wait for catalogd deployment to be ready") - catalogdManagerPod := waitForDeployment(t, ctx, "catalogd") - - // wait for operator-controller deployment to finish - t.Log("Wait for operator-controller deployment to be ready") - managerPod := waitForDeployment(t, ctx, "operator-controller") - - t.Log("Wait for acquired leader election by checking lease") - // Instead of watching logs (which has timing issues in controller-runtime v0.23.1+), - // directly check the lease object to confirm leader election occurred - // Average case is under 1 minute but in the worst case: (previous leader crashed) - // we could have LeaseDuration (137s) + RetryPeriod (26s) +/- 163s - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var lease coordinationv1.Lease - err := c.Get(ctx, types.NamespacedName{Name: "9c4404e7.operatorframework.io", Namespace: "olmv1-system"}, &lease) - require.NoError(ct, err) - require.NotNil(ct, lease.Spec.HolderIdentity, "lease should have a holder") - require.NotEmpty(ct, *lease.Spec.HolderIdentity, "lease holder identity should not be empty") - }, 3*time.Minute, time.Second) - - t.Log("Reading logs to make sure that ClusterExtension was reconciled by operator-controller before we update it") - // Make sure that after we upgrade OLM itself we can still reconcile old objects without any changes - logCtx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - substrings := []string{ - "reconcile ending", - fmt.Sprintf(`ClusterExtension=%q`, testClusterExtensionName), - } - found, err := watchPodLogsForSubstring(logCtx, managerPod, substrings...) - require.NoError(t, err) - require.True(t, found) - - t.Log("Checking that the ClusterCatalog is unpacked") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var clusterCatalog ocv1.ClusterCatalog - require.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, &clusterCatalog)) - - // check serving condition - cond := apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, ocv1.TypeServing) - require.NotNil(ct, cond) - require.Equal(ct, metav1.ConditionTrue, cond.Status) - require.Equal(ct, ocv1.ReasonAvailable, cond.Reason) - - // mitigation for upgrade-e2e flakiness caused by the following bug - // https://github.com/operator-framework/operator-controller/issues/1626 - // wait until the unpack time > than the catalogd controller pod creation time - cond = apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, ocv1.TypeProgressing) - if cond == nil { - return - } - require.Equal(ct, metav1.ConditionTrue, cond.Status) - require.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) - - require.True(ct, clusterCatalog.Status.LastUnpacked.After(catalogdManagerPod.CreationTimestamp.Time)) - }, time.Minute, time.Second) - - // TODO: if we change the underlying revision storage mechanism, the new version - // will not detect any installed versions, we need to make sure that the upgrade - // test fails across revision storage mechanism changes that are not also accompanied - // by code that automatically migrates the revision storage. - - t.Log("Checking that the ClusterExtension is installed") - var clusterExtension ocv1.ClusterExtension - require.EventuallyWithT(t, func(ct *assert.CollectT) { - require.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) - cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeInstalled) - require.NotNil(ct, cond) - require.Equal(ct, metav1.ConditionTrue, cond.Status) - require.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) - require.Contains(ct, cond.Message, "Installed bundle") - require.NotNil(ct, clusterExtension.Status.Install) - require.NotEmpty(ct, clusterExtension.Status.Install.Bundle.Version) - }, time.Minute, time.Second) - - previousVersion := clusterExtension.Status.Install.Bundle.Version - - t.Log("Updating the ClusterExtension to change version") - // Make sure that after we upgrade OLM itself we can still reconcile old objects if we change them - clusterExtension.Spec.Source.Catalog.Version = "1.0.1" - require.NoError(t, c.Update(ctx, &clusterExtension)) - - t.Log("Checking that the ClusterExtension installs successfully") - require.EventuallyWithT(t, func(ct *assert.CollectT) { - require.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) - cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeInstalled) - require.NotNil(ct, cond) - require.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) - require.Contains(ct, cond.Message, "Installed bundle") - require.Equal(ct, ocv1.BundleMetadata{Name: "test-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.Install.Bundle) - require.NotEqual(ct, previousVersion, clusterExtension.Status.Install.Bundle.Version) - }, time.Minute, time.Second) -} - -// waitForDeployment checks that the updated deployment with the given app.kubernetes.io/name label -// has reached the desired number of replicas and that the number pods matches that number -// i.e. no old pods remain. It will return a pointer to the leader pod. This is only necessary -// to facilitate the mitigation put in place for https://github.com/operator-framework/operator-controller/issues/1626 -func waitForDeployment(t *testing.T, ctx context.Context, controlPlaneLabel string) *corev1.Pod { - deploymentLabelSelector := labels.Set{"app.kubernetes.io/name": controlPlaneLabel}.AsSelector() - - t.Log("Checking that the deployment is updated and available") - var desiredNumReplicas int32 - var deploymentNamespace string - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var managerDeployments appsv1.DeploymentList - require.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: deploymentLabelSelector})) - require.Len(ct, managerDeployments.Items, 1) - managerDeployment := managerDeployments.Items[0] - - require.True(ct, - managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas && - managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas, - ) - - // Check that the deployment has the Available condition set to True - var availableCond *appsv1.DeploymentCondition - for i := range managerDeployment.Status.Conditions { - if managerDeployment.Status.Conditions[i].Type == appsv1.DeploymentAvailable { - availableCond = &managerDeployment.Status.Conditions[i] - break - } - } - require.NotNil(ct, availableCond, "Available condition not found") - require.Equal(ct, corev1.ConditionTrue, availableCond.Status, "Deployment Available condition is not True") - - desiredNumReplicas = *managerDeployment.Spec.Replicas - deploymentNamespace = managerDeployment.Namespace - }, time.Minute, time.Second) - - var managerPods corev1.PodList - t.Logf("Ensure the number of remaining pods equal the desired number of replicas (%d)", desiredNumReplicas) - require.EventuallyWithT(t, func(ct *assert.CollectT) { - require.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: deploymentLabelSelector})) - require.Len(ct, managerPods.Items, int(desiredNumReplicas)) - }, time.Minute, time.Second) - - // Find the leader pod by checking the lease - t.Log("Finding the leader pod") - // Map component labels to their leader election lease names - leaseNames := map[string]string{ - "catalogd": "catalogd-operator-lock", - "operator-controller": "9c4404e7.operatorframework.io", - } - - leaseName, ok := leaseNames[controlPlaneLabel] - if !ok { - t.Fatalf("Unknown control plane component: %s", controlPlaneLabel) - } - - var leaderPod *corev1.Pod - require.EventuallyWithT(t, func(ct *assert.CollectT) { - var lease coordinationv1.Lease - require.NoError(ct, c.Get(ctx, types.NamespacedName{Name: leaseName, Namespace: deploymentNamespace}, &lease)) - require.NotNil(ct, lease.Spec.HolderIdentity) - - leaderIdentity := *lease.Spec.HolderIdentity - // The lease holder identity format is: _ - // Extract just the pod name by splitting on '_' - podName := strings.Split(leaderIdentity, "_")[0] - - // Find the pod with matching name - for i := range managerPods.Items { - if managerPods.Items[i].Name == podName { - leaderPod = &managerPods.Items[i] - break - } - } - require.NotNil(ct, leaderPod, "leader pod not found with identity: %s (pod name: %s)", leaderIdentity, podName) - }, time.Minute, time.Second) - - return leaderPod -} - -func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, substrings ...string) (bool, error) { - // Use a polling approach to periodically check pod logs for the substrings - // This handles controller-runtime v0.23.1+ where leader election happens faster - // and the message may be emitted before we start watching with Follow - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return false, ctx.Err() - case <-ticker.C: - // Get recent logs without Follow to check if message exists - logOpts := corev1.PodLogOptions{ - Container: container, - TailLines: ptr.To(int64(1000)), - } - - req := kclientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &logOpts) - // Use a per-poll timeout derived from ctx so reads don't outlive the caller's context - pollCtx, pollCancel := context.WithTimeout(ctx, 5*time.Second) - podLogs, err := req.Stream(pollCtx) - if err != nil { - pollCancel() - // Pod might not be ready yet, or the context timed out; continue polling - continue - } - - scanner := bufio.NewScanner(podLogs) - for scanner.Scan() { - line := scanner.Text() - - foundCount := 0 - for _, substring := range substrings { - if strings.Contains(line, substring) { - foundCount++ - } - } - if foundCount == len(substrings) { - podLogs.Close() - pollCancel() - return true, nil - } - } - - // Check for scanning errors before closing - if err := scanner.Err(); err != nil { - podLogs.Close() - pollCancel() - // Log the error but continue polling - might be transient - continue - } - podLogs.Close() - pollCancel() - } - } -} diff --git a/test/upgrade-e2e/upgrade_e2e_suite_test.go b/test/upgrade-e2e/upgrade_e2e_suite_test.go deleted file mode 100644 index a2acee4cda..0000000000 --- a/test/upgrade-e2e/upgrade_e2e_suite_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package upgradee2e - -import ( - "fmt" - "os" - "testing" - - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/operator-framework/operator-controller/internal/operator-controller/scheme" -) - -const ( - testClusterCatalogNameEnv = "TEST_CLUSTER_CATALOG_NAME" - testClusterExtensionNameEnv = "TEST_CLUSTER_EXTENSION_NAME" -) - -var ( - c client.Client - kclientset kubernetes.Interface - - cfg *rest.Config - testClusterCatalogName string - testClusterExtensionName string -) - -func TestMain(m *testing.M) { - var ok bool - cfg = ctrl.GetConfigOrDie() - testClusterCatalogName, ok = os.LookupEnv(testClusterCatalogNameEnv) - if !ok { - fmt.Printf("%q is not set", testClusterCatalogNameEnv) - os.Exit(1) - } - testClusterExtensionName, ok = os.LookupEnv(testClusterExtensionNameEnv) - if !ok { - fmt.Printf("%q is not set", testClusterExtensionNameEnv) - os.Exit(1) - } - - cfg := ctrl.GetConfigOrDie() - - var err error - c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - if err != nil { - fmt.Printf("failed to create client: %s\n", err) - os.Exit(1) - } - - kclientset, err = kubernetes.NewForConfig(cfg) - if err != nil { - fmt.Printf("failed to create kubernetes clientset: %s\n", err) - os.Exit(1) - } - - os.Exit(m.Run()) -} diff --git a/test/upgrade-e2e/upgrade_test.go b/test/upgrade-e2e/upgrade_test.go new file mode 100644 index 0000000000..17a24a0734 --- /dev/null +++ b/test/upgrade-e2e/upgrade_test.go @@ -0,0 +1,42 @@ +package upgradee2e + +import ( + "log" + "os" + "testing" + + "github.com/cucumber/godog" + "github.com/cucumber/godog/colors" + "github.com/spf13/pflag" + + "github.com/operator-framework/operator-controller/test/e2e/steps" +) + +var opts = godog.Options{ + Format: "pretty", + Paths: []string{"features"}, + Output: colors.Colored(os.Stdout), + Concurrency: 1, + NoColors: true, +} + +func init() { + godog.BindCommandLineFlags("godog.", &opts) +} + +func TestMain(m *testing.M) { + pflag.Parse() + opts.Paths = pflag.Args() + + sc := godog.TestSuite{ + ScenarioInitializer: func(sc *godog.ScenarioContext) { + sc.Before(steps.CreateScenarioContext) + steps.RegisterSteps(sc) + }, + Options: &opts, + }.Run() + + if sc != 0 { + log.Fatalf("non-zero status returned (%d), failed to run upgrade feature tests", sc) + } +} From 36ad16ec7fd695e761131fdb4051243cf5c2878c Mon Sep 17 00:00:00 2001 From: Predrag Knezevic Date: Thu, 5 Mar 2026 17:40:14 +0100 Subject: [PATCH 2/2] address reviewer comments --- test/e2e/steps/hooks.go | 3 +- test/e2e/steps/steps.go | 23 ++++--- .../rbac-template-for-all-extensions.yaml | 67 +++++++++++++++++++ test/e2e/steps/testdata/rbac-template.yaml | 2 +- .../features/operator-upgrade.feature | 2 +- 5 files changed, 84 insertions(+), 13 deletions(-) create mode 100644 test/e2e/steps/testdata/rbac-template-for-all-extensions.yaml diff --git a/test/e2e/steps/hooks.go b/test/e2e/steps/hooks.go index 861586a0a7..2b00367f95 100644 --- a/test/e2e/steps/hooks.go +++ b/test/e2e/steps/hooks.go @@ -31,6 +31,7 @@ type scenarioContext struct { namespace string clusterExtensionName string clusterCatalogName string + addedResources []resource removedResources []unstructured.Unstructured backGroundCmds []*exec.Cmd metricsResponse map[string]string @@ -181,7 +182,7 @@ func ScenarioCleanup(ctx context.Context, _ *godog.Scenario, err error) (context return ctx, err } - forDeletion := []resource{} + forDeletion := sc.addedResources if sc.clusterExtensionName != "" { forDeletion = append(forDeletion, resource{name: sc.clusterExtensionName, kind: "clusterextension"}) } diff --git a/test/e2e/steps/steps.go b/test/e2e/steps/steps.go index 6a827228a3..205106542d 100644 --- a/test/e2e/steps/steps.go +++ b/test/e2e/steps/steps.go @@ -106,9 +106,9 @@ func RegisterSteps(sc *godog.ScenarioContext) { sc.Step(`^(?i)resource "([^"]+)" is eventually restored$`, ResourceRestored) sc.Step(`^(?i)resource "([^"]+)" matches$`, ResourceMatches) - sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in "([^"]*)" namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInGivenNamespace) - sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in test namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInNamespace) - sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in \${TEST_NAMESPACE}$`, ServiceAccountWithNeededPermissionsIsAvailableInNamespace) + sc.Step(`^(?i)ServiceAccount "([^"]*)" with permissions to install extensions is available in "([^"]*)" namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInGivenNamespace) + sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in test namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInTestNamespace) + sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in \${TEST_NAMESPACE}$`, ServiceAccountWithNeededPermissionsIsAvailableInTestNamespace) sc.Step(`^(?i)ServiceAccount "([^"]*)" is available in \${TEST_NAMESPACE}$`, ServiceAccountIsAvailableInNamespace) sc.Step(`^(?i)ServiceAccount "([^"]*)" in test namespace is cluster admin$`, ServiceAccountWithClusterAdminPermissionsIsAvailableInNamespace) sc.Step(`^(?i)ServiceAccount "([^"]+)" in test namespace has permissions to fetch "([^"]+)" metrics$`, ServiceAccountWithFetchMetricsPermissions) @@ -786,13 +786,13 @@ func ResourceRestored(ctx context.Context, resource string) error { return nil } -func applyServiceAccount(ctx context.Context, serviceAccount string) error { +func applyServiceAccount(ctx context.Context, serviceAccount string, keyValue ...string) error { sc := scenarioCtx(ctx) vars := extendMap(map[string]string{ "TEST_NAMESPACE": sc.namespace, "SERVICE_ACCOUNT_NAME": serviceAccount, "SERVICEACCOUNT_NAME": serviceAccount, - }) + }, keyValue...) _, thisFile, _, _ := runtime.Caller(0) yaml, err := templateYaml(filepath.Join(filepath.Dir(thisFile), "testdata", "serviceaccount-template.yaml"), vars) @@ -842,15 +842,18 @@ func ServiceAccountIsAvailableInNamespace(ctx context.Context, serviceAccount st return applyServiceAccount(ctx, serviceAccount) } -// ServiceAccountWithNeededPermissionsIsAvailableInNamespace creates a ServiceAccount and applies standard RBAC permissions. +// ServiceAccountWithNeededPermissionsIsAvailableInTestNamespace creates a ServiceAccount and applies standard RBAC permissions. func ServiceAccountWithNeededPermissionsIsAvailableInGivenNamespace(ctx context.Context, serviceAccount string, ns string) error { sc := scenarioCtx(ctx) - sc.namespace = ns - return applyPermissionsToServiceAccount(ctx, serviceAccount, "rbac-template.yaml") + sc.addedResources = append(sc.addedResources, resource{name: ns, kind: "namespace"}) + if err := applyServiceAccount(ctx, serviceAccount, "TEST_NAMESPACE", ns); err != nil { + return err + } + return applyPermissionsToServiceAccount(ctx, serviceAccount, "rbac-template-for-all-extensions.yaml", "CLUSTEREXTENSION_NAME", "*", "TEST_NAMESPACE", ns) } -// ServiceAccountWithNeededPermissionsIsAvailableInNamespace creates a ServiceAccount and applies standard RBAC permissions. -func ServiceAccountWithNeededPermissionsIsAvailableInNamespace(ctx context.Context, serviceAccount string) error { +// ServiceAccountWithNeededPermissionsIsAvailableInTestNamespace creates a ServiceAccount and applies standard RBAC permissions. +func ServiceAccountWithNeededPermissionsIsAvailableInTestNamespace(ctx context.Context, serviceAccount string) error { return applyPermissionsToServiceAccount(ctx, serviceAccount, "rbac-template.yaml") } diff --git a/test/e2e/steps/testdata/rbac-template-for-all-extensions.yaml b/test/e2e/steps/testdata/rbac-template-for-all-extensions.yaml new file mode 100644 index 0000000000..fa27689828 --- /dev/null +++ b/test/e2e/steps/testdata/rbac-template-for-all-extensions.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ${TEST_NAMESPACE} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ${TEST_NAMESPACE}-${SERVICEACCOUNT_NAME}-olm-admin-clusterrole +rules: + - apiGroups: [olm.operatorframework.io] + resources: [clusterextensions, clusterextensions/finalizers] + verbs: [update] + # Allow ClusterExtensionRevisions to set blockOwnerDeletion ownerReferences + - apiGroups: [olm.operatorframework.io] + resources: [clusterextensionrevisions, clusterextensionrevisions/finalizers] + verbs: [update, create, list, watch, get, delete, patch] + - apiGroups: [apiextensions.k8s.io] + resources: [customresourcedefinitions] + verbs: [update, create, list, watch, get, delete, patch] + - apiGroups: [""] + resources: + - configmaps + - secrets + - services + - serviceaccounts + - events + - namespaces + verbs: [update, create, list, watch, get, delete, patch] + - apiGroups: ["apps"] + resources: + - deployments + verbs: [ update, create, list, watch, get, delete, patch ] + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: [ update, create, list, watch, get, delete, patch ] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: + - clusterroles + - roles + - clusterrolebindings + - rolebindings + verbs: [ update, create, list, watch, get, delete, patch ] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: [ update, create, list, watch, get, delete, patch ] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: [create] + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ${TEST_NAMESPACE}-${SERVICEACCOUNT_NAME}-install-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ${TEST_NAMESPACE}-${SERVICEACCOUNT_NAME}-olm-admin-clusterrole +subjects: + - kind: ServiceAccount + name: ${SERVICEACCOUNT_NAME} + namespace: ${TEST_NAMESPACE} diff --git a/test/e2e/steps/testdata/rbac-template.yaml b/test/e2e/steps/testdata/rbac-template.yaml index b91cb1e220..90138b9c69 100644 --- a/test/e2e/steps/testdata/rbac-template.yaml +++ b/test/e2e/steps/testdata/rbac-template.yaml @@ -11,7 +11,7 @@ metadata: rules: - apiGroups: [olm.operatorframework.io] resources: [clusterextensions, clusterextensions/finalizers] - resourceNames: ["${CLUSTEREXTENSION_NAME}", "upgrade-ce"] + resourceNames: ["${CLUSTEREXTENSION_NAME}"] verbs: [update] # Allow ClusterExtensionRevisions to set blockOwnerDeletion ownerReferences - apiGroups: [olm.operatorframework.io] diff --git a/test/upgrade-e2e/features/operator-upgrade.feature b/test/upgrade-e2e/features/operator-upgrade.feature index 5a4b909491..f80adac485 100644 --- a/test/upgrade-e2e/features/operator-upgrade.feature +++ b/test/upgrade-e2e/features/operator-upgrade.feature @@ -8,7 +8,7 @@ Feature: Operator upgrade verification Background: Given the latest stable OLM release is installed And ClusterCatalog "test" serves bundles - And ServiceAccount "olm-sa" with needed permissions is available in "upgrade-ns" namespace + And ServiceAccount "olm-sa" with permissions to install extensions is available in "upgrade-ns" namespace And ClusterExtension is applied """ apiVersion: olm.operatorframework.io/v1