Rocky Linux 9.7 containers

.github/workflows/stackhpc-all-in-one.yml

@@ -314,6 +314,26 @@ jobs:
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
+      - name: Pull container images
+        run: |
+          # Try up to 3 times to pull container images.
+          for attempt in $(seq 3); do
+              docker run -t --rm \
+                -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+                -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+                ${{ steps.kayobe_image.outputs.kayobe_image }} \
+                /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-container-image-pull.sh
+              if [ "$?" = "0" ]; then
+                  echo "Pulled container images on attempt $attempt"
+                  exit 0
+              fi
+              echo "Failed to pull container images on attempt $attempt"
+          done
+          echo "Failed to pull container imagers after $attempt attempts"
+          exit 1
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+
       - name: Service deploy
         run: |
           docker run -t --rm \

.github/workflows/stackhpc-container-image-build.yml

@@ -82,14 +82,14 @@ jobs:
         id: openstack_release
         run: |
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
-          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
+          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," | tee -a "$GITHUB_OUTPUT"
 
       # Generate a tag to apply to all built container images.
       # Without this, each kayobe * container image build command would use a different tag.
       - name: Generate container datetime tag
         id: datetime_tag
         run: |
-          echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+          echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" | tee -a "$GITHUB_OUTPUT"
 
       # Dynamically define job matrix.
       # We need a separate matrix entry for each distribution, when the relevant input is true.
@@ -110,7 +110,7 @@ jobs:
           # remove trailing comma
           output="${output%,}"
           output+="]}"
-          echo "matrix=$output" >> $GITHUB_OUTPUT
+          echo "matrix=$output" | tee -a "$GITHUB_OUTPUT"
 
       - name: Display container datetime tag
         run: |
@@ -184,7 +184,7 @@ jobs:
 
       - name: Get Kolla tag
         id: write-kolla-tag
-        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
+        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" | tee -a "$GITHUB_OUTPUT"
 
       - name: Configure localhost as a seed
         run: |
@@ -250,7 +250,7 @@ jobs:
         if: inputs.seed && matrix.distro.arch == 'amd64'
 
       - name: Get built container images
-        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" | tee "${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images"
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi
@@ -260,6 +260,7 @@ jobs:
 
       - name: Move image scan logs to output artifact
         run: mv image-scan-output image-build-logs/image-scan-output
+        if: ${{ !cancelled() }}
 
       - name: Fail if any images have critical vulnerabilities
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi

etc/kayobe/kolla-image-tags.yml

@@ -4,28 +4,19 @@
 # where the key is the OS distro and the value is the tag to deploy.
 kolla_image_tags:
   openstack:
-    rocky-9: 2025.1-rocky-9-20250730T105631
+    rocky-9: 2025.1-rocky-9-20260123T132550
     ubuntu-noble: 2025.1-ubuntu-noble-20250730T105631
   iscsid:
-    rocky-9: 2025.1-rocky-9-20251013T135156
     ubuntu-noble: 2025.1-ubuntu-noble-20251013T135156
   keystone:
-    rocky-9: 2025.1-rocky-9-20251031T094539
     ubuntu-noble: 2025.1-ubuntu-noble-20251031T094539
   kolla_toolbox:
-    rocky-9: 2025.1-rocky-9-20250905T151507
     ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507
   magnum:
-    rocky-9: 2025.1-rocky-9-20251016T073648
     ubuntu-noble: 2025.1-ubuntu-noble-20251016T073648
   neutron:
-    rocky-9: 2025.1-rocky-9-20260116T134029
     ubuntu-noble: 2025.1-ubuntu-noble-20260116T134029
-  ovn:
-    rocky-9: 2025.1-rocky-9-20250915T100822
   prometheus_libvirt_exporter:
-    rocky-9: 2025.1-rocky-9-20250822T151934
     ubuntu-noble: 2025.1-ubuntu-noble-20250822T151934
   rabbitmq:
-    rocky-9: 2025.1-rocky-9-20250905T151507
     ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507

etc/kayobe/kolla.yml

@@ -23,6 +23,11 @@ kolla_source_version: "{{ stackhpc_kolla_source_version }}"
 # Path in which to generate kolla configuration.
 #kolla_build_config_path:
 
+# View debug logs, like source downloads, during image build.
+kolla_build_extra_config: |
+  [DEFAULT]
+  debug = True
+
 ###############################################################################
 # Kolla-ansible installation.
 

etc/kayobe/kolla/kolla-build.conf

@@ -29,3 +29,8 @@ sha256 = amd64:1fd60b1fd59c239bed22719a5de402cb745d1f933540cb1ec196e2c03e6e8882,
 [magnum-conductor-plugin-helm]
 version = v3.18.2
 sha256 = amd64:c5deada86fe609deefdf40e9cbbe3da2f8cf3f6a4551a0ebe7886dc8fcf98bce,arm64:03181a494a0916b370a100a5b2536104963b095be53fb23d1e29b2afb1c7de8d
+
+# TODO: move to kolla_sources in kolla.yml once https://review.opendev.org/c/openstack/kayobe/+/970268 is available
+[prometheus-cadvisor]
+version = 0.54.1
+sha256 = amd64:21be8d2797433048474e676d37c215c28fb171509448ef9b1c4648a564e39595,arm64:21f7bac786f6c53a8091964b4d3ff2486a0c460e5a410000b59a9a565b4183a9

etc/kayobe/pulp-host-image-versions.yml

@@ -1,5 +1,5 @@
 ---
 # Overcloud host image versioning tags
 # These images must be in SMS, since they are used by our AIO CI runners
-stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260113T105634"
+stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260119T110913"
 stackhpc_ubuntu_noble_overcloud_host_image_version: "2025.1-20250930T144255"

etc/kayobe/trivy/allowed-vulnerabilities.yml

@@ -20,7 +20,10 @@ influxdb_allowed_vulnerabilities:
   - CVE-2024-45337
 magnum_conductor_allowed_vulnerabilities:
   - CVE-2024-45337
+opensearch_dashboards_allowed_vulnerabilities:
+  - CVE-2025-68428
 prometheus_blackbox_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
   - CVE-2024-45337
 prometheus_memcached_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
@@ -31,7 +34,10 @@ prometheus_elasticsearch_exporter_allowed_vulnerabilities:
 prometheus_node_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
 prometheus_openstack_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
   - CVE-2024-45337
+prometheus_ovn_exporter_allowed_vulnerabilities:
+  - CVE-2024-24790
 prometheus_libvirt_exporter_allowed_vulnerabilities:
   - CVE-2024-45337
 prometheus_cadvisor_allowed_vulnerabilities:

releasenotes/notes/rl9.7-c8d033f11097b76b.yaml

@@ -1,7 +1,8 @@
 ---
 features:
   - |
-    Added support for Rocky Linux 9.7.
+    Added support for Rocky Linux 9.7, including host packages and a full
+    container image refresh.
   - |
     9.7 is now the default release for Rocky Linux.
 

tools/scan-images.sh

@@ -58,24 +58,26 @@ get_images() {
 # Generate ignored vulnerabilities file
 generate_trivy_ignore() {
   local imagename=$1
-  local global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
-  local image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
+  local global_vulnerabilities
+  global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
+  local image_vulnerabilities
+  image_vulnerabilities=$(yq ."$imagename"'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
 
   touch .trivyignore
   for vulnerability in $global_vulnerabilities; do
-    echo $vulnerability >> .trivyignore
+    echo "$vulnerability" >> .trivyignore
   done
   for vulnerability in $image_vulnerabilities; do
-    echo $vulnerability >> .trivyignore
+    echo "$vulnerability" >> .trivyignore
   done
 }
 
 # Put results into CSV
 generate_summary_csv() {
-  local imagename=$1
-  local filename=$2
+  local scan="$1"
+  local summary="$2"
 
-  echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${imagename}/${filename}-summary.csv
+  echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > "$summary"
 
   jq -r '.Results[]
       | select(.Vulnerabilities)
@@ -94,16 +96,15 @@ generate_summary_csv() {
           ]
         )
       | .[]
-      | @csv' image-scan-output/${imagename}/${filename}-scan.json >> image-scan-output/${imagename}/${filename}-summary.csv
+      | @csv' "$scan" >> "$summary"
 }
 
 # Categorise images based on severity
 categorise_image() {
-  local imagename=$1
-  local filename=$2
-  local image=$3
+  local summary="$1"
+  local image="$2"
 
-  if [ $(grep "CRITICAL" image-scan-output/${imagename}/${filename}-summary.csv -c) -gt 0 ]; then
+  if [ "$(grep "CRITICAL" "$summary" -c)" -gt 0 ]; then
     echo "${image}" >> image-scan-output/critical-images.txt
   else
     echo "${image}" >> image-scan-output/high-images.txt
@@ -112,45 +113,78 @@ categorise_image() {
 
 # Generate SBOM, return correct scan command for SBOM
 generate_sbom() {
-  local imagename=$1
-  local filename=$2
-  local image=$3
+  local sbom="$1"
+  local scan="$2"
+  local image="$3"
   trivy image \
+        --debug \
         --format spdx-json \
-        --output image-scan-output/${imagename}/${filename}-sbom.json \
-        $image > /dev/null 2>&1
-  echo "trivy sbom $scan_common_args \
-        --output image-scan-output/${imagename}/${filename}-scan.json \
-        image-scan-output/${imagename}/${filename}-sbom.json"
+        --output "$sbom" \
+        "$image" &> "$sbom.log"
+  if [ ! -e "$sbom" ]; then
+    (
+      echo "ERROR: trivy image didn't produce the sbom file $sbom for $image" 1>&2
+      echo "==== trivy log ===="
+      cat "$sbom.log"
+    ) 1>&2
+    exit 1
+  elif grep -q FATAL "$sbom.log"; then
+    (
+      echo "ERROR: trivy image encountered a fatal error producing $sbom for $image"
+      echo "==== trivy log ===="
+      cat "$sbom.log"
+      echo "==== sbom.json ===="
+      cat "$sbom"
+    ) 1>&2
+    exit 1
+  else
+    echo "trivy sbom $scan_common_args --output $scan $sbom"
+  fi
 }
 
 # Scan images, generate SBOMs if requested
 scan_image() {
   local image=$1
-  local filename=$(basename $image | sed 's/:/\./g')
-  local imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g')
+  local filename
+  filename=$(basename "$image" | sed 's/:/\./g')
+  local imagename
+  imagename=$(echo "$filename" | cut -d "." -f 1 | sed 's/-/_/g')
+  local sbom="image-scan-output/${imagename}/${filename}-sbom.json"
+  local scan="image-scan-output/${imagename}/${filename}-scan.json"
+  local summary="image-scan-output/${imagename}/${filename}-summary.csv"
 
-  mkdir -p image-scan-output/$imagename
-  generate_trivy_ignore $imagename
+  mkdir -p "image-scan-output/$imagename"
+  generate_trivy_ignore "$imagename"
 
   # If SBOM is required, generate it first and scan the results, otherwise we
   # scan the image directly.
   if $generate_sbom; then
     echo "Generating SBOM for $imagename"
-    scan_command=$(generate_sbom $imagename $filename $image)
+    scan_command="$(generate_sbom "$sbom" "$scan" "$image")"
   else
-    scan_command="trivy image $scan_common_args \
-                  --output image-scan-output/${imagename}/${filename}-scan.json $image"
+    scan_command="trivy image $scan_common_args --output $scan $image"
   fi
 
   # Run scan against image or SBOM, format output. If no results, delete files.
   echo "Scanning $imagename for vulnerabilities"
-  if $scan_command > /dev/null 2>&1; then
-    rm -f image-scan-output/${imagename}/${filename}-scan.json
+  if $scan_command >& "$scan.log"; then
+    rm -f "$scan"
     echo "${image}" >> image-scan-output/clean-images.txt
+  elif [ ! -f "$scan" ]; then
+    (
+      echo "ERROR: trivy scan encountered an error producing $scan"
+      echo "Command: $scan_command"
+      echo "==== trivy log ===="
+      cat "$scan.log"
+      if $generate_sbom; then
+        echo "==== sbom.json ===="
+        cat "$sbom"
+      fi
+    ) 1>&2
+    exit 1
   else
-    generate_summary_csv $imagename $filename
-    categorise_image $imagename $filename $image
+    generate_summary_csv "$scan" "$summary"
+    categorise_image "$summary" "$image"
   fi
 }
 
@@ -170,9 +204,9 @@ main() {
   check_deps_installed
   file_prep
 
-  images=$(get_images $1 $2)
+  images=$(get_images "$1" "$2")
   for image in $images; do
-    scan_image $image
+    scan_image "$image"
   done
 }