Migrate operator upgrade tests to Godog/cucumber

Author: pedjak

Makefile

@@ -238,6 +238,17 @@ verify-crd-compatibility: $(CRD_DIFF) manifests
 
 #SECTION Test
 
+define install-sh
+.PHONY: $(1)/install.sh
+$(1)/install.sh: manifests
+	@echo -e "\n\U1F4D8 Using $(1).yaml as source manifest\n"
+	sed "s/cert-git-version/cert-$$(VERSION)/g" manifests/$(1).yaml > $(2)
+	MANIFEST=$(2) INSTALL_DEFAULT_CATALOGS=false DEFAULT_CATALOG=$$(RELEASE_CATALOGS) envsubst '$$$$DEFAULT_CATALOG,$$$$CERT_MGR_VERSION,$$$$INSTALL_DEFAULT_CATALOGS,$$$$MANIFEST' < scripts/install.tpl.sh > $(1)-install.sh
+endef
+
+$(eval $(call install-sh,experimental,operator-controller-experimental.yaml))
+$(eval $(call install-sh,standard,operator-controller-standard.yaml))
+
 .PHONY: test
 test: manifests generate fmt lint test-unit test-e2e test-regression #HELP Run all tests.
 
@@ -339,48 +350,43 @@ run-latest-release:
 pre-upgrade-setup:
 	./hack/test/pre-upgrade-setup.sh $(CATALOG_IMG) $(TEST_CLUSTER_CATALOG_NAME) $(TEST_CLUSTER_EXTENSION_NAME)
 
-.PHONY: post-upgrade-checks
-post-upgrade-checks:
-	go test -count=1 -v ./test/upgrade-e2e/...
+.PHONY: test-upgrade-e2e
+test-upgrade-e2e:
+	RELEASE_INSTALL=$(RELEASE_INSTALL) \
+	RELEASE_UPGRADE=$(RELEASE_UPGRADE) \
+	KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) \
+	ROOT_DIR=$(ROOT_DIR) \
+	go test -count=1 -v ./test/upgrade-e2e/upgrade_test.go
 
 
-TEST_UPGRADE_E2E_TASKS := kind-cluster run-latest-release image-registry pre-upgrade-setup docker-build kind-load kind-deploy post-upgrade-checks kind-clean
+TEST_UPGRADE_E2E_TASKS := kind-cluster docker-build kind-load test-upgrade-e2e kind-clean
 
 .PHONY: test-upgrade-st2st-e2e
-test-upgrade-st2st-e2e: SOURCE_MANIFEST := $(STANDARD_MANIFEST)
-test-upgrade-st2st-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL)
+test-upgrade-st2st-e2e: RELEASE_INSTALL := https://github.com/operator-framework/operator-controller/releases/latest/download/install.sh
+test-upgrade-st2st-e2e: RELEASE_UPGRADE := $(ROOT_DIR)/standard-install.sh
 test-upgrade-st2st-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-st2st-e2e
-test-upgrade-st2st-e2e: export MANIFEST := $(STANDARD_RELEASE_MANIFEST)
-test-upgrade-st2st-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog
-test-upgrade-st2st-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package
-test-upgrade-st2st-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> standard) e2e tests on a local kind cluster
+test-upgrade-st2st-e2e: standard/install.sh $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> standard) e2e tests on a local kind cluster
 
 .PHONY: test-upgrade-ex2ex-e2e
-test-upgrade-ex2ex-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_MANIFEST)
-test-upgrade-ex2ex-e2e: RELEASE_INSTALL := $(EXPERIMENTAL_RELEASE_INSTALL)
+test-upgrade-ex2ex-e2e: RELEASE_INSTALL := https://github.com/operator-framework/operator-controller/releases/latest/download/install-experimental.sh
 test-upgrade-ex2ex-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-ex2ex-e2e
-test-upgrade-ex2ex-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST)
-test-upgrade-ex2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog
-test-upgrade-ex2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package
-test-upgrade-ex2ex-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (experimental -> experimental) e2e tests on a local kind cluster
+test-upgrade-ex2ex-e2e: RELEASE_UPGRADE := $(ROOT_DIR)/experimental-install.sh
+test-upgrade-ex2ex-e2e: experimental/install.sh $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (experimental -> experimental) e2e tests on a local kind cluster
 
 .PHONY: test-upgrade-st2ex-e2e
-test-upgrade-st2ex-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_MANIFEST)
-test-upgrade-st2ex-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL)
+test-upgrade-st2ex-e2e: RELEASE_INSTALL := https://github.com/operator-framework/operator-controller/releases/latest/download/install.sh
+test-upgrade-st2ex-e2e: RELEASE_UPGRADE := $(ROOT_DIR)/experimental-install.sh
 test-upgrade-st2ex-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-st2ex-e2e
-test-upgrade-st2ex-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST)
-test-upgrade-st2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog
-test-upgrade-st2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package
-test-upgrade-st2ex-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> experimental) e2e tests on a local kind cluster
+test-upgrade-st2ex-e2e: experimental/install.sh $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> experimental) e2e tests on a local kind cluster
 
 .PHONY: test-st2ex-e2e
-test-st2ex-e2e: SOURCE_MANIFEST := $(STANDARD_MANIFEST)
-test-st2ex-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL)
+test-st2ex-e2e: RELEASE_INSTALL := $(ROOT_DIR)/standard-install.sh
+test-st2ex-e2e: RELEASE_UPGRADE := $(ROOT_DIR)/experimental-install.sh
 test-st2ex-e2e: KIND_CLUSTER_NAME := operator-controller-st2ex-e2e
 test-st2ex-e2e: export MANIFEST := $(STANDARD_RELEASE_MANIFEST)
 test-st2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog
 test-st2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package
-test-st2ex-e2e: run-internal image-registry pre-upgrade-setup kind-deploy-experimental post-upgrade-checks kind-clean #HELP Run swichover (standard -> experimental) e2e tests on a local kind cluster
+test-st2ex-e2e: experimental/install.sh standard/install.sh $(TEST_UPGRADE_E2E_TASKS) #HELP Run swichover (standard -> experimental) e2e tests on a local kind cluster
 
 .PHONY: e2e-coverage
 e2e-coverage:

test/e2e/steps/hooks.go

@@ -30,9 +30,11 @@ type scenarioContext struct {
 	id                   string
 	namespace            string
 	clusterExtensionName string
+	clusterCatalogName   string
 	removedResources     []unstructured.Unstructured
 	backGroundCmds       []*exec.Cmd
 	metricsResponse      map[string]string
+	leaderPods           map[string]string // component name -> leader pod name
 
 	extensionObjects []client.Object
 }
@@ -87,30 +89,37 @@ func RegisterHooks(sc *godog.ScenarioContext) {
 	sc.After(ScenarioCleanup)
 }
 
-func BeforeSuite() {
-	if devMode {
-		logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1)))
-	} else {
-		logger = textlogger.NewLogger(textlogger.NewConfig())
-	}
-
+func detectOLMDeployment() (*appsv1.Deployment, error) {
 	raw, err := k8sClient("get", "deployments", "-A", "-l", "app.kubernetes.io/part-of=olm", "-o", "jsonpath={.items}")
 	if err != nil {
-		panic(fmt.Errorf("failed to get OLM deployments: %v", err))
+		return nil, err
 	}
 	dl := []appsv1.Deployment{}
 	if err := json.Unmarshal([]byte(raw), &dl); err != nil {
 		panic(fmt.Errorf("failed to unmarshal OLM deployments: %v", err))
 	}
-	var olm *appsv1.Deployment
 
 	for _, d := range dl {
 		if d.Name == olmDeploymentName {
-			olm = &d
-			olmNamespace = d.Namespace
-			break
+			return &d, nil
 		}
 	}
+	return nil, fmt.Errorf("failed to detect OLM Deployment")
+}
+
+func BeforeSuite() {
+	if devMode {
+		logger = textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(1)))
+	} else {
+		logger = textlogger.NewLogger(textlogger.NewConfig())
+	}
+
+	olm, err := detectOLMDeployment()
+	if err != nil {
+		logger.Info("OLM deployments not found; skipping feature gate detection (upgrade scenarios will install OLM in Background)")
+		return
+	}
+	olmNamespace = olm.Namespace
 
 	featureGatePattern := regexp.MustCompile(`--feature-gates=([[:alnum:]]+)=(true|false)`)
 	for _, c := range olm.Spec.Template.Spec.Containers {
@@ -144,6 +153,7 @@ func CreateScenarioContext(ctx context.Context, sc *godog.Scenario) (context.Con
 		id:                   sc.Id,
 		namespace:            fmt.Sprintf("ns-%s", sc.Id),
 		clusterExtensionName: fmt.Sprintf("ce-%s", sc.Id),
+		leaderPods:           make(map[string]string),
 	}
 	return context.WithValue(ctx, scenarioContextKey, scCtx), nil
 }

test/e2e/steps/steps.go

@@ -14,7 +14,9 @@ import (
 	"os/exec"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/cucumber/godog"
@@ -47,9 +49,24 @@ const (
 )
 
 var (
-	olmNamespace   = "olmv1-system"
-	kubeconfigPath string
-	k8sCli         string
+	olmNamespace        = "olmv1-system"
+	kubeconfigPath      string
+	k8sCli              string
+	deployImageRegistry = sync.OnceValue(func() error {
+		if os.Getenv("KIND_CLUSTER_NAME") == "" {
+			return nil
+		}
+		cmd := exec.Command("bash", "-c", "make image-registry")
+		dir, _ := os.LookupEnv("ROOT_DIR")
+		if dir == "" {
+			return fmt.Errorf("ROOT_DIR environment variable not set")
+		}
+		cmd.Dir = dir
+		cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath))
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		return cmd.Run()
+	})
 )
 
 func RegisterSteps(sc *godog.ScenarioContext) {
@@ -89,6 +106,7 @@ func RegisterSteps(sc *godog.ScenarioContext) {
 	sc.Step(`^(?i)resource "([^"]+)" is eventually restored$`, ResourceRestored)
 	sc.Step(`^(?i)resource "([^"]+)" matches$`, ResourceMatches)
 
+	sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in "([^"]*)" namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInGivenNamespace)
 	sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in test namespace$`, ServiceAccountWithNeededPermissionsIsAvailableInNamespace)
 	sc.Step(`^(?i)ServiceAccount "([^"]*)" with needed permissions is available in \${TEST_NAMESPACE}$`, ServiceAccountWithNeededPermissionsIsAvailableInNamespace)
 	sc.Step(`^(?i)ServiceAccount "([^"]*)" is available in \${TEST_NAMESPACE}$`, ServiceAccountIsAvailableInNamespace)
@@ -108,6 +126,14 @@ func RegisterSteps(sc *godog.ScenarioContext) {
 	sc.Step(`^(?i)Prometheus metrics are returned in the response$`, PrometheusMetricsAreReturned)
 
 	sc.Step(`^(?i)min value for (ClusterExtension|ClusterExtensionRevision) ((?:\.[a-zA-Z]+)+) is set to (\d+)$`, SetCRDFieldMinValue)
+
+	// Upgrade-specific steps
+	sc.Step(`^(?i)the latest stable OLM release is installed$`, LatestStableOLMReleaseIsInstalled)
+	sc.Step(`^(?i)OLM is upgraded$`, OLMIsUpgraded)
+	sc.Step(`^(?i)(catalogd|operator-controller) is ready to reconcile resources$`, ComponentIsReadyToReconcile)
+	sc.Step(`^(?i)all (ClusterCatalog|ClusterExtension) resources are reconciled$`, AllResourcesAreReconciled)
+	sc.Step(`^(?i)(ClusterCatalog|ClusterExtension) is reconciled$`, ResourceTypeIsReconciled)
+	sc.Step(`^(?i)ClusterCatalog reports ([[:alnum:]]+) as ([[:alnum:]]+) with Reason ([[:alnum:]]+)$`, ClusterCatalogReportsCondition)
 }
 
 func init() {
@@ -768,7 +794,8 @@ func applyServiceAccount(ctx context.Context, serviceAccount string) error {
 		"SERVICEACCOUNT_NAME":  serviceAccount,
 	})
 
-	yaml, err := templateYaml(filepath.Join("steps", "testdata", "serviceaccount-template.yaml"), vars)
+	_, thisFile, _, _ := runtime.Caller(0)
+	yaml, err := templateYaml(filepath.Join(filepath.Dir(thisFile), "testdata", "serviceaccount-template.yaml"), vars)
 	if err != nil {
 		return fmt.Errorf("failed to template ServiceAccount yaml: %v", err)
 	}
@@ -795,7 +822,8 @@ func applyPermissionsToServiceAccount(ctx context.Context, serviceAccount, rbacT
 		"CLUSTEREXTENSION_NAME":  sc.clusterExtensionName,
 	}, keyValue...)
 
-	yaml, err := templateYaml(filepath.Join("steps", "testdata", rbacTemplate), vars)
+	_, thisFile, _, _ := runtime.Caller(0)
+	yaml, err := templateYaml(filepath.Join(filepath.Dir(thisFile), "testdata", rbacTemplate), vars)
 	if err != nil {
 		return fmt.Errorf("failed to template RBAC yaml: %v", err)
 	}
@@ -814,6 +842,13 @@ func ServiceAccountIsAvailableInNamespace(ctx context.Context, serviceAccount st
 	return applyServiceAccount(ctx, serviceAccount)
 }
 
+// ServiceAccountWithNeededPermissionsIsAvailableInNamespace creates a ServiceAccount and applies standard RBAC permissions.
+func ServiceAccountWithNeededPermissionsIsAvailableInGivenNamespace(ctx context.Context, serviceAccount string, ns string) error {
+	sc := scenarioCtx(ctx)
+	sc.namespace = ns
+	return applyPermissionsToServiceAccount(ctx, serviceAccount, "rbac-template.yaml")
+}
+
 // ServiceAccountWithNeededPermissionsIsAvailableInNamespace creates a ServiceAccount and applies standard RBAC permissions.
 func ServiceAccountWithNeededPermissionsIsAvailableInNamespace(ctx context.Context, serviceAccount string) error {
 	return applyPermissionsToServiceAccount(ctx, serviceAccount, "rbac-template.yaml")
@@ -972,12 +1007,19 @@ func CatalogIsUpdatedToVersion(name, version string) error {
 
 // CatalogServesBundles applies the ClusterCatalog YAML template to create a catalog that serves bundles.
 func CatalogServesBundles(ctx context.Context, catalogName string) error {
-	yamlContent, err := os.ReadFile(filepath.Join("steps", "testdata", fmt.Sprintf("%s-catalog-template.yaml", catalogName)))
+	if err := deployImageRegistry(); err != nil {
+		return err
+	}
+	sc := scenarioCtx(ctx)
+	sc.clusterCatalogName = fmt.Sprintf("%s-catalog", catalogName)
+
+	_, thisFile, _, _ := runtime.Caller(0)
+	yamlContent, err := os.ReadFile(filepath.Join(filepath.Dir(thisFile), "testdata", fmt.Sprintf("%s-catalog-template.yaml", catalogName)))
 	if err != nil {
 		return fmt.Errorf("failed to read catalog yaml: %v", err)
 	}
 
-	_, err = k8scliWithInput(substituteScenarioVars(string(yamlContent), scenarioCtx(ctx)), "apply", "-f", "-")
+	_, err = k8scliWithInput(substituteScenarioVars(string(yamlContent), sc), "apply", "-f", "-")
 	if err != nil {
 		return fmt.Errorf("failed to apply catalog: %v", err)
 	}

test/e2e/steps/testdata/rbac-template.yaml

@@ -11,7 +11,7 @@ metadata:
 rules:
   - apiGroups: [olm.operatorframework.io]
     resources: [clusterextensions, clusterextensions/finalizers]
-    resourceNames: ["${CLUSTEREXTENSION_NAME}"]
+    resourceNames: ["${CLUSTEREXTENSION_NAME}", "upgrade-ce"]
     verbs: [update]
   # Allow ClusterExtensionRevisions to set blockOwnerDeletion ownerReferences
   - apiGroups: [olm.operatorframework.io]