Feature/quickstarter automatic tests improvements

.github/workflows/continuous-integration-workflow.yml

@@ -180,10 +180,10 @@ jobs:
         name: Checkout repository
         uses: actions/checkout@v4.2.2
       - 
-        name: Setup Go 1.18
+        name: Setup Go 1.24
         uses: actions/setup-go@v5
         with:
-          go-version: 1.18
+          go-version: 1.24
       -
         name: Verify all Go files are formatted with gofmt
         working-directory: tests
@@ -201,7 +201,7 @@ jobs:
         name: Verify all Go tests pass linting
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.49.0
+          version: v1.64.7
           working-directory: tests
           args: --timeout=10m
       -

.gitignore

@@ -21,3 +21,4 @@ output-vmware-iso
 prov-app-config.txt
 headers.txt
 response.txt
+.cache/

CHANGELOG.md

@@ -5,7 +5,7 @@
 ### Added
 
 ### Changed
-
+- Improved automatic tests for the quickstarters ([#1362](https://github.com/opendevstack/ods-core/pull/1362))
 ### Fixed
 
 ## [4.11.1] - 2025-12-05

jenkins/master/Dockerfile.ubi8

@@ -18,7 +18,7 @@ USER root
 COPY yum.repos.d/ubi.repo /etc/yum.repos.d/ubi.repo
 
 COPY ./scripts_for_usr-local-bin/* /usr/local/bin/
-RUN rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key \
+RUN rpm --import https://pkg.jenkins.io/rpm-stable/jenkins.io-2026.key \    
     && disable_yum_repository.sh /etc/yum.repos.d/ci-rpm-mirrors.repo \
         /etc/yum.repos.d/localdev-* /etc/yum.repos.d/epel.repo \
     && ensure_java_jre_is_adequate.sh master \

tests/Makefile

@@ -9,6 +9,7 @@ QS := ods-quickstarters/...
 
 ### By default we do not parallelize tests
 PARALLEL := 1
+PROJECT := unitt
 
 ## Full test of existing ODS core installation. Caution: Creates UNITT project and ODSVERIFY project.
 test: smoketest verify test-create-projects
@@ -35,12 +36,13 @@ test-create-projects:
 
 ## Run quickstarter tests within existing ODS installation. Depends on UNITT project.
 test-quickstarter:
-	@(./quickstarter-test.sh $(QS) $(PARALLEL))
+	@(./quickstarter-test.sh -p $(PROJECT) -q $(QS) -pa $(PARALLEL))
 .PHONY: test-quickstarter
 
 ## Install tools required for tests.
 prep-tools:
 	which go-junit-report || go get github.com/jstemmer/go-junit-report
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.50.0
 .PHONY: prep-tools
 
 ## Lint

tests/README.md

@@ -25,5 +25,42 @@ Run `make test` [in this directory](Makefile), which will execute project creati
 ## Running the quickstarter tests
 Run `make test-quickstarter` [in this directory](Makefile). By default, this will test all quickstarters in `ods-quickstarters` located next to `ods-core`. You can run just one specific quickstarter test with `make test-quickstarter QS=be-golang-plain` or run tests located in a custom directory like this: `make test-quickstarter QS=my-quickstarters/...` or `make test-quickstarter QS=my-quickstarters/foobar`. By default all tests run sequentially. To run some in parallel, use e.g. `make test-quickstarter PARALLEL=3`.
 
+### Keeping resources after tests
+By default, the quickstarter tests clean up all created resources (OpenShift resources, Helm releases, etc.) after each test completes. To keep the resources for debugging or inspection purposes, set the `KEEP_RESOURCES` environment variable:
+
+```bash
+KEEP_RESOURCES=true make test-quickstarter QS=be-python-flask
+```
+
+Or for the shell script:
+```bash
+KEEP_RESOURCES=true ./dev-test.sh be-python-flask e2etsqs
+```
+
+**Note:** Port-forward cleanup is handled separately and will still occur to prevent resource leaks.
+
 ## Authoring quickstarter tests
 Quickstarters must have a `testdata` directory, which needs to contain a `steps.yml` file describing which test steps to execute in YAML format. The allowed fields are defined by https://pkg.go.dev/github.com/opendevstack/ods-core/tests/quickstarter. Typically, the `testdata` directory will also contain a `golden` folder with JSON files describing the expected results. See https://github.com/opendevstack/ods-quickstarters/tree/master/be-golang-plain/testdata as an example.
+
+### Specifying namespace for OpenShift resource verification
+
+When verifying OpenShift resources, you can optionally specify a custom namespace. If not specified, the verification defaults to the `{{.ProjectID}}-dev` namespace.
+
+Example in `testdata/steps.yml`:
+
+```yaml
+- type: provision
+  provisionParams:
+    verify:
+      openShiftResources:
+        namespace: "test"  # Will check in {{.ProjectID}}-test
+        services:
+          - "{{.ComponentID}}"
+        deploymentConfigs:
+          - "{{.ComponentID}}"
+```
+
+You can also specify a full namespace (with project prefix):
+```yaml
+        namespace: "{{.ProjectID}}-cd"  # Explicit full namespace
+```

tests/create-projects/jenkinsfile_test.go

@@ -5,8 +5,9 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"os"
 	"path"
 	"runtime"
 	"strings"
@@ -101,7 +102,7 @@ func TestCreateProjectThruWebhookProxyJenkinsFile(t *testing.T) {
 
 	defer reponse.Body.Close()
 
-	bodyBytes, _ := ioutil.ReadAll(reponse.Body)
+	bodyBytes, _ := io.ReadAll(reponse.Body)
 
 	if reponse.StatusCode >= http.StatusAccepted {
 		t.Fatalf("Could not post request: %s", string(bodyBytes))
@@ -154,7 +155,7 @@ func TestCreateProjectThruWebhookProxyJenkinsFile(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	expected, err := ioutil.ReadFile("golden/jenkins-create-project-stages.json")
+	expected, err := os.ReadFile("golden/jenkins-create-project-stages.json")
 	if err != nil {
 		t.Fatal(err)
 	}

tests/dev-test.sh

@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+# Developer-friendly test runner for quickstarters
+# This script provides a simple interface to run quickstarter tests
+# with automatic environment detection and helpful output
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ODS_CORE_DIR="${SCRIPT_DIR}/.."
+
+# Default values
+QUICKSTARTER="${1:-be-python-flask}"
+PROJECT="${2:-devtest}"
+PARALLEL="${3:-1}"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+function print_header() {
+    echo -e "\n${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+    echo -e "${BLUE}  $1${NC}"
+    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"
+}
+
+function print_info() {
+    echo -e "${BLUE}ℹ️  $1${NC}"
+}
+
+function print_success() {
+    echo -e "${GREEN}✓ $1${NC}"
+}
+
+function print_warning() {
+    echo -e "${YELLOW}⚠️  $1${NC}"
+}
+
+function print_error() {
+    echo -e "${RED}✗ $1${NC}"
+}
+
+function check_prerequisites() {
+    local missing_tools=()
+
+    # Check for required tools
+    if ! command -v oc &> /dev/null; then
+        missing_tools+=("oc (OpenShift CLI)")
+    fi
+
+    if ! command -v go &> /dev/null; then
+        missing_tools+=("go")
+    fi
+
+    if ! command -v jq &> /dev/null; then
+        missing_tools+=("jq")
+    fi
+
+    if [ ${#missing_tools[@]} -ne 0 ]; then
+        print_error "Missing required tools:"
+        for tool in "${missing_tools[@]}"; do
+            echo "  - $tool"
+        done
+        echo ""
+        echo "Please install the missing tools and try again."
+        exit 1
+    fi
+
+    # Check if logged into OpenShift
+    if ! oc whoami &> /dev/null; then
+        print_error "Not logged into OpenShift"
+        echo "Please run 'oc login' first."
+        exit 1
+    fi
+
+    print_success "All prerequisites met"
+}
+
+function detect_environment() {
+    print_header "Environment Detection"
+
+    # Check if running in cluster
+    if [ -n "${KUBERNETES_SERVICE_HOST:-}" ]; then
+        print_info "Execution environment: Inside Kubernetes/OpenShift cluster"
+        print_info "Network strategy: Service DNS (no port-forwards needed)"
+    else
+        print_info "Execution environment: Local development machine"
+        print_info "Network strategy: Automatic (routes > port-forward > service DNS)"
+        print_warning "Port-forwards will be set up automatically as needed"
+    fi
+
+    # Show current OpenShift context
+    current_user=$(oc whoami 2>/dev/null || echo "unknown")
+    current_server=$(oc whoami --show-server 2>/dev/null || echo "unknown")
+
+    print_info "OpenShift user: $current_user"
+    print_info "OpenShift server: $current_server"
+}
+
+function show_test_info() {
+    print_header "Test Configuration"
+
+    echo "  Quickstarter: $QUICKSTARTER"
+    echo "  Project: $PROJECT"
+    echo "  Parallelism: $PARALLEL"
+    echo ""
+}
+
+function run_tests() {
+    print_header "Running Tests"
+
+    cd "$ODS_CORE_DIR/tests"
+
+    # Run the quickstarter test
+    print_info "Executing: make test-quickstarter QS=$QUICKSTARTER PROJECT=$PROJECT PARALLEL=$PARALLEL"
+    echo ""
+
+    if make test-quickstarter QS="$QUICKSTARTER" PROJECT="$PROJECT" PARALLEL="$PARALLEL"; then
+        print_success "Tests passed!"
+        return 0
+    else
+        print_error "Tests failed!"
+        return 1
+    fi
+}
+
+function show_usage() {
+    cat << EOF
+Usage: $0 [QUICKSTARTER] [PROJECT] [PARALLEL]
+
+Developer-friendly test runner for ODS quickstarters.
+
+Arguments:
+  QUICKSTARTER  Quickstarter to test (default: be-python-flask)
+                Examples:
+                  - be-python-flask          (single quickstarter)
+                  - ods-quickstarters/...    (all quickstarters)
+                  - be-golang-plain          (another single quickstarter)
+
+  PROJECT       OpenShift project name for testing (default: devtest)
+
+  PARALLEL      Number of tests to run in parallel (default: 1)
+
+Examples:
+  # Test be-python-flask in 'devtest' project
+  $0
+
+  # Test specific quickstarter in custom project
+  $0 be-golang-plain myproject
+
+  # Test all quickstarters with parallelism
+  $0 ods-quickstarters/... testproj 3
+
+Features:
+  ✓ Automatic environment detection (in-cluster vs local)
+  ✓ Smart URL resolution (routes > port-forward > service DNS)
+  ✓ Automatic port-forward setup for local development
+  ✓ Automatic cleanup on exit or interrupt (Ctrl+C)
+  ✓ Clear, colorful output
+
+Network Access:
+  When running locally, the test framework will automatically:
+  1. Try to use OpenShift routes if they exist (fastest, most reliable)
+  2. Set up port-forwards for services without routes
+  3. Fall back to service DNS if running inside the cluster
+
+  You don't need to manually set up port-forwards - it's all automatic!
+
+EOF
+}
+
+# Main execution
+main() {
+    # Show usage if help requested
+    if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+        show_usage
+        exit 0
+    fi
+
+    print_header "ODS Quickstarter Test Runner"
+
+    check_prerequisites
+    detect_environment
+    show_test_info
+
+    if run_tests; then
+        print_header "Test Summary"
+        print_success "All tests completed successfully!"
+        echo ""
+        exit 0
+    else
+        print_header "Test Summary"
+        print_error "Some tests failed. Check the output above for details."
+        exit 1
+    fi
+}
+
+main "$@"