fix(ci): fix RHDH OCP Orchestrator Helm e2e job failures

.ibm/pipelines/resources/rhdh-operator/rhdh-start-rbac.yaml

@@ -24,3 +24,18 @@ spec:
       configMaps:
         - name: rbac-policy
       mountPath: /opt/app-root/src/rbac
+  deployment:
+    patch:
+      spec:
+        template:
+          spec:
+            volumes:
+              - name: dynamic-plugins-root
+                ephemeral:
+                  volumeClaimTemplate:
+                    spec:
+                      accessModes:
+                        - ReadWriteOnce
+                      resources:
+                        requests:
+                          storage: 5Gi

.ibm/pipelines/resources/rhdh-operator/rhdh-start.yaml

@@ -24,3 +24,18 @@ spec:
       secrets:
         - name: rhdh-secrets
         - name: redis-secret
+  deployment:
+    patch:
+      spec:
+        template:
+          spec:
+            volumes:
+              - name: dynamic-plugins-root
+                ephemeral:
+                  volumeClaimTemplate:
+                    spec:
+                      accessModes:
+                        - ReadWriteOnce
+                      resources:
+                        requests:
+                          storage: 5Gi

.ibm/pipelines/utils.sh

@@ -818,6 +818,191 @@ perform_helm_install() {
     $(get_image_helm_set_params)
 }
 
+# Helper function to manually create sonataflow database with SSL support
+# This is a workaround for the helm chart's create-db job not including PGSSLMODE env var
+create_sonataflow_database_with_ssl() {
+  local namespace=$1
+
+  echo "Manually creating sonataflow database with SSL support..."
+
+  # Create a temporary pod to run psql with SSL
+  cat << EOF | oc apply -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: create-sonataflow-db-manual
+  namespace: ${namespace}
+spec:
+  restartPolicy: Never
+  containers:
+  - name: psql
+    image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.3-1
+    # Note: We intentionally do not set readOnlyRootFilesystem here because
+    # psql requires write access to /tmp for temporary files during SSL connections
+    command: ["sh", "-c"]
+    args:
+      - |
+        export PGSSLMODE=require
+        psql -h \${POSTGRES_HOST} -p \${POSTGRES_PORT} -U \${POSTGRES_USER} -d postgres -c 'CREATE DATABASE sonataflow;' && echo "Database created successfully" || echo "Database creation failed or database already exists"
+    env:
+    - name: POSTGRES_HOST
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: POSTGRES_HOST
+    - name: POSTGRES_USER
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: POSTGRES_USER
+    - name: POSTGRES_PORT
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: POSTGRES_PORT
+    - name: PGPASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: POSTGRES_PASSWORD
+    - name: PGSSLMODE
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: PGSSLMODE
+EOF
+
+  # Wait for the pod to start and complete
+  echo "Waiting for manual database creation pod to complete..."
+
+  # Wait up to 5 minutes for completion (accounts for image pull delays, rate limiting, etc.)
+  local timeout=300
+  local elapsed=0
+  local last_status=""
+  local status
+
+  while [ $elapsed -lt $timeout ]; do
+    status=$(oc get pod create-sonataflow-db-manual -n "${namespace}" -o jsonpath='{.status.phase}' 2> /dev/null || echo "NotFound")
+
+    # Print status changes
+    if [[ "$status" != "$last_status" ]]; then
+      echo "Pod status: $status (elapsed: ${elapsed}s)"
+      last_status="$status"
+    fi
+
+    if [[ "$status" == "Succeeded" ]]; then
+      echo "Database creation pod completed successfully"
+      break
+    elif [[ "$status" == "Failed" ]]; then
+      echo "ERROR: Database creation pod failed"
+      oc logs create-sonataflow-db-manual -n "${namespace}" 2> /dev/null || echo "Could not retrieve logs"
+      oc delete pod create-sonataflow-db-manual -n "${namespace}" --ignore-not-found=true
+      return 1
+    fi
+
+    sleep 5
+    elapsed=$((elapsed + 5))
+  done
+
+  if [ $elapsed -ge $timeout ]; then
+    echo "ERROR: Timeout waiting for database creation pod (${timeout}s elapsed)"
+    oc logs create-sonataflow-db-manual -n "${namespace}" 2> /dev/null || echo "Could not retrieve logs"
+    oc delete pod create-sonataflow-db-manual -n "${namespace}" --ignore-not-found=true
+    return 1
+  fi
+
+  # Check logs for successful completion
+  echo "Database creation output:"
+  oc logs create-sonataflow-db-manual -n "${namespace}" 2> /dev/null || echo "Could not retrieve logs"
+
+  # Clean up the pod
+  oc delete pod create-sonataflow-db-manual -n "${namespace}" --ignore-not-found=true
+
+  echo "Manual database creation completed successfully"
+}
+
+# Verify that the sonataflow database exists
+verify_sonataflow_database() {
+  local namespace=$1
+
+  echo "Verifying sonataflow database exists..."
+
+  # Create a verification pod
+  cat << EOF | oc apply -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: verify-sonataflow-db
+  namespace: ${namespace}
+spec:
+  restartPolicy: Never
+  containers:
+  - name: psql
+    image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.3-1
+    command: ["sh", "-c"]
+    args:
+      - |
+        export PGSSLMODE=require
+        psql -h \${POSTGRES_HOST} -p \${POSTGRES_PORT} -U \${POSTGRES_USER} -d postgres -c '\l' | grep sonataflow && echo "DATABASE EXISTS" || echo "DATABASE DOES NOT EXIST"
+    env:
+    - name: POSTGRES_HOST
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: POSTGRES_HOST
+    - name: POSTGRES_USER
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: POSTGRES_USER
+    - name: POSTGRES_PORT
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: POSTGRES_PORT
+    - name: PGPASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: POSTGRES_PASSWORD
+    - name: PGSSLMODE
+      valueFrom:
+        secretKeyRef:
+          name: postgres-cred
+          key: PGSSLMODE
+EOF
+
+  # Wait for completion (shorter timeout since image should be cached)
+  local timeout=60
+  local elapsed=0
+  local status
+  while [ $elapsed -lt $timeout ]; do
+    status=$(oc get pod verify-sonataflow-db -n "${namespace}" -o jsonpath='{.status.phase}' 2> /dev/null || echo "NotFound")
+    if [[ "$status" == "Succeeded" ]] || [[ "$status" == "Failed" ]]; then
+      break
+    fi
+    sleep 3
+    elapsed=$((elapsed + 3))
+  done
+
+  # Check the result
+  local verification_output
+  verification_output=$(oc logs verify-sonataflow-db -n "${namespace}" 2> /dev/null)
+  echo "$verification_output"
+
+  # Clean up
+  oc delete pod verify-sonataflow-db -n "${namespace}" --ignore-not-found=true
+
+  # Return success if database exists
+  if echo "$verification_output" | grep -q "DATABASE EXISTS"; then
+    echo "✓ Database verification successful"
+    return 0
+  else
+    echo "✗ Database verification failed"
+    return 1
+  fi
+}
+
 base_deployment() {
   configure_namespace ${NAME_SPACE}
 
@@ -843,16 +1028,35 @@ rbac_deployment() {
   echo "Deploying image from repository: ${QUAY_REPO}, TAG_NAME: ${TAG_NAME}, in NAME_SPACE: ${RELEASE_NAME_RBAC}"
   perform_helm_install "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}" "${HELM_CHART_RBAC_VALUE_FILE_NAME}"
 
-  # NOTE: This is a workaround to allow the sonataflow platform to connect to the external postgres db using ssl.
+  # NOTE: The helm chart's create-sonataflow-database job will fail because it doesn't include PGSSLMODE env var.
+  # We wait for the job to be created (indicating helm install is progressing), then manually create the database with SSL.
   until [[ $(oc get jobs -n "${NAME_SPACE_RBAC}" 2> /dev/null | grep "${RELEASE_NAME_RBAC}-create-sonataflow-database" | wc -l) -eq 1 ]]; do
     echo "Waiting for sf db creation job to be created. Retrying in 5 seconds..."
     sleep 5
   done
-  oc wait --for=condition=complete job/"${RELEASE_NAME_RBAC}-create-sonataflow-database" -n "${NAME_SPACE_RBAC}" --timeout=3m
+
+  # Don't wait for the helm job to complete - it will fail due to missing SSL configuration
+  # Instead, manually create the database with proper SSL support
+  create_sonataflow_database_with_ssl "${NAME_SPACE_RBAC}"
+
+  # Verify the database was created successfully
+  if ! verify_sonataflow_database "${NAME_SPACE_RBAC}"; then
+    echo "ERROR: Failed to verify sonataflow database creation. Workflows may fail to start."
+    echo "Attempting to continue anyway..."
+  fi
+
+  # Patch the sonataflow platform to configure SSL for the jobs service
+  echo "Patching SonataFlowPlatform with SSL configuration..."
   oc -n "${NAME_SPACE_RBAC}" patch sfp sonataflow-platform --type=merge \
     -p '{"spec":{"services":{"jobService":{"podTemplate":{"container":{"env":[{"name":"QUARKUS_DATASOURCE_REACTIVE_URL","value":"postgresql://postgress-external-db-primary.postgress-external-db.svc.cluster.local:5432/sonataflow?search_path=jobs-service&sslmode=require&ssl=true&trustAll=true"},{"name":"QUARKUS_DATASOURCE_REACTIVE_SSL_MODE","value":"require"},{"name":"QUARKUS_DATASOURCE_REACTIVE_TRUST_ALL","value":"true"}]}}}}}}'
+
+  echo "Restarting jobs-service deployment..."
   oc rollout restart deployment/sonataflow-platform-jobs-service -n "${NAME_SPACE_RBAC}"
 
+  # Wait for jobs-service to be ready before deploying workflows
+  echo "Waiting for jobs-service to be ready..."
+  oc rollout status deployment/sonataflow-platform-jobs-service -n "${NAME_SPACE_RBAC}" --timeout=3m || echo "WARNING: jobs-service rollout did not complete in time"
+
   # initiate orchestrator workflows deployment
   deploy_orchestrator_workflows "${NAME_SPACE_RBAC}"
 }
@@ -1263,7 +1467,7 @@ deploy_orchestrator_workflows() {
   oc apply -f "${WORKFLOW_MANIFESTS}"
 
   helm repo add orchestrator-workflows https://rhdhorchestrator.io/serverless-workflows
-  helm install greeting orchestrator-workflows/greeting -n "$namespace"
+  helm install greeting orchestrator-workflows/greeting -n "$namespace" --wait --timeout=5m
 
   until [[ $(oc get sf -n "$namespace" --no-headers 2> /dev/null | wc -l) -eq 2 ]]; do
     echo "No sf resources found. Retrying in 5 seconds..."

e2e-tests/playwright/support/pages/orchestrator.ts

@@ -31,6 +31,16 @@ export class Orchestrator {
     await this.page.getByRole("link", { name: "User Onboarding" }).click();
   }
 
+  async selectGreetingWorkflowItem() {
+    const workflowHeader = this.page.getByRole("heading", {
+      name: "Workflows",
+    });
+    await expect(workflowHeader).toBeVisible();
+    await expect(workflowHeader).toHaveText("Workflows");
+    await expect(Workflows.workflowsTable(this.page)).toBeVisible();
+    await this.page.getByRole("link", { name: "Greeting workflow" }).click();
+  }
+
   async getPageUrl() {
     return this.page.url();
   }

e2e-tests/playwright/support/pages/workflows.ts

@@ -1,10 +1,7 @@
 import { Page } from "@playwright/test";
 
 const workflowsTable = (page: Page) =>
-  page
-    .locator("#root div")
-    .filter({ hasText: "WorkflowsNameCategoryLast" })
-    .nth(2);
+  page.locator("#root div").filter({ hasText: "Workflows" }).nth(2);
 
 const WORKFLOWS = {
   workflowsTable,

e2e-tests/playwright/utils/common.ts

@@ -84,6 +84,10 @@ export class Common {
     await new Promise<void>((resolve) => {
       this.page.once("popup", async (popup) => {
         await popup.waitForLoadState();
+        // Wait for login button to be visible with longer timeout for slow Keycloak responses
+        await popup
+          .locator("#kc-login")
+          .waitFor({ state: "visible", timeout: 30000 });
         await popup.locator("#username").fill(userid);
         await popup.locator("#password").fill(password);
         await popup.locator("#kc-login").click();