From 86115e2cb160092aefa434375983807ce3d48790 Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Fri, 5 Dec 2025 17:37:23 +0100 Subject: [PATCH 01/11] bump cadvisor to 0.54.1 to fix multiple vulnerabilities --- etc/kayobe/kolla/kolla-build.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf index 6ff0aa287f..7457645702 100644 --- a/etc/kayobe/kolla/kolla-build.conf +++ b/etc/kayobe/kolla/kolla-build.conf @@ -29,3 +29,8 @@ sha256 = amd64:1fd60b1fd59c239bed22719a5de402cb745d1f933540cb1ec196e2c03e6e8882, [magnum-conductor-plugin-helm] version = v3.18.2 sha256 = amd64:c5deada86fe609deefdf40e9cbbe3da2f8cf3f6a4551a0ebe7886dc8fcf98bce,arm64:03181a494a0916b370a100a5b2536104963b095be53fb23d1e29b2afb1c7de8d + +# TODO: move to kolla_sources in kolla.yml once https://review.opendev.org/c/openstack/kayobe/+/970268 is available +[prometheus-cadvisor] +version = 0.54.1 +sha256 = amd64:21be8d2797433048474e676d37c215c28fb171509448ef9b1c4648a564e39595,arm64:21f7bac786f6c53a8091964b4d3ff2486a0c460e5a410000b59a9a565b4183a9 From 8f19226c783b4133c244522ab095d51948be8084 Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Mon, 8 Dec 2025 11:34:01 +0100 Subject: [PATCH 02/11] debug kolla container image build to see which sources are downloaded before docker build --- etc/kayobe/kolla.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 09392411fd..f0deb9b5dc 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -23,6 +23,11 @@ kolla_source_version: "{{ stackhpc_kolla_source_version }}" # Path in which to generate kolla configuration. #kolla_build_config_path: +# View debug logs, like source downloads, during image build. +kolla_build_extra_config: | + [DEFAULT] + debug = True + ############################################################################### # Kolla-ansible installation. From cecc980172cc16ffe8d62eb20b88adf9f0ff2164 Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Tue, 9 Dec 2025 10:44:57 +0100 Subject: [PATCH 03/11] aio workflow: add container image pull to accomodate temporary errors from ark (was getting a 500 error) --- .github/workflows/stackhpc-all-in-one.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 8cdd2e6381..a0f687e4e8 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -314,6 +314,26 @@ jobs: env: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + - name: Pull container images + run: | + # Try up to 3 times to pull container images. + for attempt in $(seq 3); do + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-container-image-pull.sh + if [ "$?" = "0" ]; then + echo "Pulled container images on attempt $attempt" + exit 0 + fi + echo "Failed to pull container images on attempt $attempt" + done + echo "Failed to pull container imagers after $attempt attempts" + exit 1 + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + - name: Service deploy run: | docker run -t --rm \ From b18d39c41cd24eebd1caf7730ffa92d59852afef Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Fri, 19 Dec 2025 15:44:29 +0100 Subject: [PATCH 04/11] release notes --- releasenotes/notes/rl9.7-c8d033f11097b76b.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml index dcdb765ec7..6b6fa315fa 100644 --- a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml +++ b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml @@ -1,7 +1,8 @@ --- features: - | - Added support for Rocky Linux 9.7. + Added support for Rocky Linux 9.7, including host packages and a full + container image refresh. - | 9.7 is now the default release for Rocky Linux. From aa1cc0a67604f06ccafcc2ada8a60b8e403c82bc Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Fri, 5 Dec 2025 17:40:05 +0100 Subject: [PATCH 05/11] Ignore CVE-2024-24790 in prometheus exporters control plane is trusted --- etc/kayobe/trivy/allowed-vulnerabilities.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/trivy/allowed-vulnerabilities.yml b/etc/kayobe/trivy/allowed-vulnerabilities.yml index 4759862058..e71f6d863f 100644 --- a/etc/kayobe/trivy/allowed-vulnerabilities.yml +++ b/etc/kayobe/trivy/allowed-vulnerabilities.yml @@ -21,6 +21,7 @@ influxdb_allowed_vulnerabilities: magnum_conductor_allowed_vulnerabilities: - CVE-2024-45337 prometheus_blackbox_exporter_allowed_vulnerabilities: + - CVE-2024-24790 - CVE-2024-45337 prometheus_memcached_exporter_allowed_vulnerabilities: - CVE-2024-45337 @@ -31,7 +32,10 @@ prometheus_elasticsearch_exporter_allowed_vulnerabilities: prometheus_node_exporter_allowed_vulnerabilities: - CVE-2024-45337 prometheus_openstack_exporter_allowed_vulnerabilities: + - CVE-2024-24790 - CVE-2024-45337 +prometheus_ovn_exporter_allowed_vulnerabilities: + - CVE-2024-24790 prometheus_libvirt_exporter_allowed_vulnerabilities: - CVE-2024-45337 prometheus_cadvisor_allowed_vulnerabilities: From 584832222604228d9f8e6b5d4eb63031ae3d8350 Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Tue, 6 Jan 2026 13:27:22 +0100 Subject: [PATCH 06/11] opensearch-dashboard: ignore CVE-2025-68428 CVE-2025-68428 is still present in opensearch-dashboards 2.19.4 because jspdf is still in version 3.0.1 --- etc/kayobe/trivy/allowed-vulnerabilities.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/trivy/allowed-vulnerabilities.yml b/etc/kayobe/trivy/allowed-vulnerabilities.yml index e71f6d863f..b85bff573e 100644 --- a/etc/kayobe/trivy/allowed-vulnerabilities.yml +++ b/etc/kayobe/trivy/allowed-vulnerabilities.yml @@ -20,6 +20,8 @@ influxdb_allowed_vulnerabilities: - CVE-2024-45337 magnum_conductor_allowed_vulnerabilities: - CVE-2024-45337 +opensearch_dashboards_allowed_vulnerabilities: + - CVE-2025-68428 prometheus_blackbox_exporter_allowed_vulnerabilities: - CVE-2024-24790 - CVE-2024-45337 From 5779103b309ec7f61f8a0fab5521c18d7ea00a51 Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Mon, 19 Jan 2026 16:52:28 +0100 Subject: [PATCH 07/11] Build kolla container images: show workflow values --- .github/workflows/stackhpc-container-image-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 67bd8e57ec..7b8bac6c6f 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -82,14 +82,14 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) - echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," | tee -a "$GITHUB_OUTPUT" # Generate a tag to apply to all built container images. # Without this, each kayobe * container image build command would use a different tag. - name: Generate container datetime tag id: datetime_tag run: | - echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" | tee -a "$GITHUB_OUTPUT" # Dynamically define job matrix. # We need a separate matrix entry for each distribution, when the relevant input is true. @@ -110,7 +110,7 @@ jobs: # remove trailing comma output="${output%,}" output+="]}" - echo "matrix=$output" >> $GITHUB_OUTPUT + echo "matrix=$output" | tee -a "$GITHUB_OUTPUT" - name: Display container datetime tag run: | @@ -184,7 +184,7 @@ jobs: - name: Get Kolla tag id: write-kolla-tag - run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" >> $GITHUB_OUTPUT + run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" | tee -a "$GITHUB_OUTPUT" - name: Configure localhost as a seed run: | @@ -250,7 +250,7 @@ jobs: if: inputs.seed && matrix.distro.arch == 'amd64' - name: Get built container images - run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images + run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" | tee "${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images" - name: Fail if no images have been built run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi From 4493522dbca7fa1a6cd0d9249ca91e159485a355 Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Wed, 28 Jan 2026 11:58:01 +0100 Subject: [PATCH 08/11] [skipci] Rebuilt rocky9 overcloud host image https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/21135194729/job/60775498739 --- etc/kayobe/pulp-host-image-versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml index 1e8aa8370c..4aacbb8e23 100644 --- a/etc/kayobe/pulp-host-image-versions.yml +++ b/etc/kayobe/pulp-host-image-versions.yml @@ -1,5 +1,5 @@ --- # Overcloud host image versioning tags # These images must be in SMS, since they are used by our AIO CI runners -stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260113T105634" +stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260119T110913" stackhpc_ubuntu_noble_overcloud_host_image_version: "2025.1-20250930T144255" From 7113f1c30934c41e537be612630e44d709979365 Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Mon, 19 Jan 2026 18:46:04 +0100 Subject: [PATCH 09/11] [skipci] Build kolla container images: always upload scan output directory --- .github/workflows/stackhpc-container-image-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 7b8bac6c6f..0191bb8cb0 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -260,6 +260,7 @@ jobs: - name: Move image scan logs to output artifact run: mv image-scan-output image-build-logs/image-scan-output + if: ${{ !cancelled() }} - name: Fail if any images have critical vulnerabilities run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi From 6fd4ee22239673b9b5b8d3087bda6ce60d7059ba Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Mon, 19 Jan 2026 18:49:49 +0100 Subject: [PATCH 10/11] [skipci] Show trivy errors when scanning images --- tools/scan-images.sh | 100 +++++++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 33 deletions(-) diff --git a/tools/scan-images.sh b/tools/scan-images.sh index fa792f4d7d..54c823ed1a 100755 --- a/tools/scan-images.sh +++ b/tools/scan-images.sh @@ -58,24 +58,26 @@ get_images() { # Generate ignored vulnerabilities file generate_trivy_ignore() { local imagename=$1 - local global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) - local image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + local global_vulnerabilities + global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + local image_vulnerabilities + image_vulnerabilities=$(yq ."$imagename"'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) touch .trivyignore for vulnerability in $global_vulnerabilities; do - echo $vulnerability >> .trivyignore + echo "$vulnerability" >> .trivyignore done for vulnerability in $image_vulnerabilities; do - echo $vulnerability >> .trivyignore + echo "$vulnerability" >> .trivyignore done } # Put results into CSV generate_summary_csv() { - local imagename=$1 - local filename=$2 + local scan="$1" + local summary="$2" - echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${imagename}/${filename}-summary.csv + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > "$summary" jq -r '.Results[] | select(.Vulnerabilities) @@ -94,16 +96,15 @@ generate_summary_csv() { ] ) | .[] - | @csv' image-scan-output/${imagename}/${filename}-scan.json >> image-scan-output/${imagename}/${filename}-summary.csv + | @csv' "$scan" >> "$summary" } # Categorise images based on severity categorise_image() { - local imagename=$1 - local filename=$2 - local image=$3 + local summary="$1" + local image="$2" - if [ $(grep "CRITICAL" image-scan-output/${imagename}/${filename}-summary.csv -c) -gt 0 ]; then + if [ "$(grep "CRITICAL" "$summary" -c)" -gt 0 ]; then echo "${image}" >> image-scan-output/critical-images.txt else echo "${image}" >> image-scan-output/high-images.txt @@ -112,45 +113,78 @@ categorise_image() { # Generate SBOM, return correct scan command for SBOM generate_sbom() { - local imagename=$1 - local filename=$2 - local image=$3 + local sbom="$1" + local scan="$2" + local image="$3" trivy image \ + --debug \ --format spdx-json \ - --output image-scan-output/${imagename}/${filename}-sbom.json \ - $image > /dev/null 2>&1 - echo "trivy sbom $scan_common_args \ - --output image-scan-output/${imagename}/${filename}-scan.json \ - image-scan-output/${imagename}/${filename}-sbom.json" + --output "$sbom" \ + "$image" &> "$sbom.log" + if [ ! -e "$sbom" ]; then + ( + echo "ERROR: trivy image didn't produce the sbom file $sbom for $image" 1>&2 + echo "==== trivy log ====" + cat "$sbom.log" + ) 1>&2 + exit 1 + elif grep -q FATAL "$sbom.log"; then + ( + echo "ERROR: trivy image encountered a fatal error producing $sbom for $image" + echo "==== trivy log ====" + cat "$sbom.log" + echo "==== sbom.json ====" + cat "$sbom" + ) 1>&2 + exit 1 + else + echo "trivy sbom $scan_common_args --output $scan $sbom" + fi } # Scan images, generate SBOMs if requested scan_image() { local image=$1 - local filename=$(basename $image | sed 's/:/\./g') - local imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g') + local filename + filename=$(basename "$image" | sed 's/:/\./g') + local imagename + imagename=$(echo "$filename" | cut -d "." -f 1 | sed 's/-/_/g') + local sbom="image-scan-output/${imagename}/${filename}-sbom.json" + local scan="image-scan-output/${imagename}/${filename}-scan.json" + local summary="image-scan-output/${imagename}/${filename}-summary.csv" - mkdir -p image-scan-output/$imagename - generate_trivy_ignore $imagename + mkdir -p "image-scan-output/$imagename" + generate_trivy_ignore "$imagename" # If SBOM is required, generate it first and scan the results, otherwise we # scan the image directly. if $generate_sbom; then echo "Generating SBOM for $imagename" - scan_command=$(generate_sbom $imagename $filename $image) + scan_command="$(generate_sbom "$sbom" "$scan" "$image")" else - scan_command="trivy image $scan_common_args \ - --output image-scan-output/${imagename}/${filename}-scan.json $image" + scan_command="trivy image $scan_common_args --output $scan $image" fi # Run scan against image or SBOM, format output. If no results, delete files. echo "Scanning $imagename for vulnerabilities" - if $scan_command > /dev/null 2>&1; then - rm -f image-scan-output/${imagename}/${filename}-scan.json + if $scan_command >& "$scan.log"; then + rm -f "$scan" echo "${image}" >> image-scan-output/clean-images.txt + elif [ ! -f "$scan" ]; then + ( + echo "ERROR: trivy scan encountered an error producing $scan" + echo "Command: $scan_command" + echo "==== trivy log ====" + cat "$scan.log" + if $generate_sbom; then + echo "==== sbom.json ====" + cat "$sbom" + fi + ) 1>&2 + exit 1 else - generate_summary_csv $imagename $filename - categorise_image $imagename $filename $image + generate_summary_csv "$scan" "$summary" + categorise_image "$summary" "$image" fi } @@ -170,9 +204,9 @@ main() { check_deps_installed file_prep - images=$(get_images $1 $2) + images=$(get_images "$1" "$2") for image in $images; do - scan_image $image + scan_image "$image" done } From e3743c533b82c3bad3502b6d05add6035b4903ba Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Thu, 22 Jan 2026 15:05:24 +0100 Subject: [PATCH 11/11] Rebuilt containers Workflow: https://github.com/stackhpc/stackhpc-kayobe-config/actions/runs/21287718238 --- etc/kayobe/kolla-image-tags.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 82a5234f11..377ca85585 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -4,28 +4,19 @@ # where the key is the OS distro and the value is the tag to deploy. kolla_image_tags: openstack: - rocky-9: 2025.1-rocky-9-20250730T105631 + rocky-9: 2025.1-rocky-9-20260123T132550 ubuntu-noble: 2025.1-ubuntu-noble-20250730T105631 iscsid: - rocky-9: 2025.1-rocky-9-20251013T135156 ubuntu-noble: 2025.1-ubuntu-noble-20251013T135156 keystone: - rocky-9: 2025.1-rocky-9-20251031T094539 ubuntu-noble: 2025.1-ubuntu-noble-20251031T094539 kolla_toolbox: - rocky-9: 2025.1-rocky-9-20250905T151507 ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507 magnum: - rocky-9: 2025.1-rocky-9-20251016T073648 ubuntu-noble: 2025.1-ubuntu-noble-20251016T073648 neutron: - rocky-9: 2025.1-rocky-9-20260116T134029 ubuntu-noble: 2025.1-ubuntu-noble-20260116T134029 - ovn: - rocky-9: 2025.1-rocky-9-20250915T100822 prometheus_libvirt_exporter: - rocky-9: 2025.1-rocky-9-20250822T151934 ubuntu-noble: 2025.1-ubuntu-noble-20250822T151934 rabbitmq: - rocky-9: 2025.1-rocky-9-20250905T151507 ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507