diff --git a/Makefile b/Makefile index a22f9a03a..80ca60164 100644 --- a/Makefile +++ b/Makefile @@ -258,7 +258,18 @@ E2E_TIMEOUT ?= 20m GODOG_ARGS ?= .PHONY: e2e e2e: #EXHELP Run the e2e tests. - go test -count=1 -v ./test/e2e/features_test.go -timeout=$(E2E_TIMEOUT) $(if $(GODOG_ARGS),-args $(GODOG_ARGS)) +ifeq ($(strip $(GODOG_ARGS)),) + go test -count=1 -v ./test/e2e/features_test.go -timeout=$(E2E_TIMEOUT) -args --godog.tags="~@Serial" --godog.concurrency=100; \ + parallelExitCode=$$?; \ + go test -count=1 -v ./test/e2e/features_test.go -timeout=$(E2E_TIMEOUT) -args --godog.tags="@Serial" --godog.concurrency=1; \ + serialExitCode=$$?; \ + if [ $$parallelExitCode -ne 0 ] || [ $$serialExitCode -ne 0 ]; then \ + echo "e2e tests failed: parallel=$$parallelExitCode serial=$$serialExitCode"; \ + exit 1; \ + fi +else + go test -count=1 -v ./test/e2e/features_test.go -timeout=$(E2E_TIMEOUT) -args $(GODOG_ARGS) +endif export CLUSTER_REGISTRY_HOST := docker-registry.operator-controller-e2e.svc:5000 .PHONY: extension-developer-e2e diff --git a/test/e2e/README.md b/test/e2e/README.md index 98b99b9ca..67adc2fa1 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -207,6 +207,10 @@ Use these variables in YAML templates: ### 5. Feature Tags +Tags can be used for different purposes in the test suite: + +#### Feature Gate Tags + Use tags to conditionally run scenarios based on feature gates: ```gherkin @@ -216,6 +220,28 @@ Scenario: Install operator having webhooks Scenarios are skipped if the feature gate is not enabled on the deployed controller. +#### Serial Execution Tag + +By default, scenarios run concurrently (up to 100 parallel scenarios). However, some tests must run serially, typically because they: +- Modify shared cluster resources (e.g., cluster-wide TLS configuration) +- Have resource constraints that prevent parallel execution +- Require exclusive access to a resource + +To mark a test for serial execution, add the `@Serial` tag: + +```gherkin +@Serial +Feature: TLS profile enforcement on metrics endpoints + + Scenario: Test TLS configuration + Given the "catalogd" deployment is configured with custom TLS settings + ... +``` + +The test runner automatically separates scenarios: +- Scenarios **without** `@Serial` run concurrently in the first test phase +- Scenarios **with** `@Serial` run sequentially in a separate serial test phase + ## Running Tests ### Run All Tests @@ -230,6 +256,15 @@ or make test-experimental-e2e ``` +Custom godog arguments can be modified by setting the following: +```bash +GODOG_ARGS=--godog.tags=@WebhookProviderCertManager make test-experimental-e2e +``` + +Note that when this is done the `make` target will no longer automatically split the test run into parallel and serial runs, and test execution time may increase. If you wish to add concurrency back into the arguments, it is recommended to also disable the `@Serial` tests: +```bash +GODOG_ARGS="--godog.tags=~@Serial --godog.concurrency=100" make test-experimental-e2e +``` ### Run Specific Feature diff --git a/test/e2e/features/ha.feature b/test/e2e/features/ha.feature index 6b889c99b..bb20cfafa 100644 --- a/test/e2e/features/ha.feature +++ b/test/e2e/features/ha.feature @@ -1,3 +1,4 @@ +@Serial Feature: HA failover for catalogd When catalogd is deployed with multiple replicas, the remaining pods must diff --git a/test/e2e/features/proxy.feature b/test/e2e/features/proxy.feature index c4ddcc091..9f30a47f0 100644 --- a/test/e2e/features/proxy.feature +++ b/test/e2e/features/proxy.feature @@ -1,3 +1,4 @@ +@Serial Feature: HTTPS proxy support for outbound catalog requests OLM's operator-controller fetches catalog data from catalogd over HTTPS. diff --git a/test/e2e/features/tls.feature b/test/e2e/features/tls.feature index 77293c12f..c13079633 100644 --- a/test/e2e/features/tls.feature +++ b/test/e2e/features/tls.feature @@ -1,3 +1,4 @@ +@Serial Feature: TLS profile enforcement on metrics endpoints Background: