diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 8cdd2e6381..a0f687e4e8 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -314,6 +314,26 @@ jobs: env: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + - name: Pull container images + run: | + # Try up to 3 times to pull container images. + for attempt in $(seq 3); do + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-container-image-pull.sh + if [ "$?" = "0" ]; then + echo "Pulled container images on attempt $attempt" + exit 0 + fi + echo "Failed to pull container images on attempt $attempt" + done + echo "Failed to pull container imagers after $attempt attempts" + exit 1 + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + - name: Service deploy run: | docker run -t --rm \ diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 67bd8e57ec..0191bb8cb0 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -82,14 +82,14 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) - echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," | tee -a "$GITHUB_OUTPUT" # Generate a tag to apply to all built container images. # Without this, each kayobe * container image build command would use a different tag. - name: Generate container datetime tag id: datetime_tag run: | - echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" | tee -a "$GITHUB_OUTPUT" # Dynamically define job matrix. # We need a separate matrix entry for each distribution, when the relevant input is true. @@ -110,7 +110,7 @@ jobs: # remove trailing comma output="${output%,}" output+="]}" - echo "matrix=$output" >> $GITHUB_OUTPUT + echo "matrix=$output" | tee -a "$GITHUB_OUTPUT" - name: Display container datetime tag run: | @@ -184,7 +184,7 @@ jobs: - name: Get Kolla tag id: write-kolla-tag - run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" >> $GITHUB_OUTPUT + run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" | tee -a "$GITHUB_OUTPUT" - name: Configure localhost as a seed run: | @@ -250,7 +250,7 @@ jobs: if: inputs.seed && matrix.distro.arch == 'amd64' - name: Get built container images - run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images + run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" | tee "${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images" - name: Fail if no images have been built run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi @@ -260,6 +260,7 @@ jobs: - name: Move image scan logs to output artifact run: mv image-scan-output image-build-logs/image-scan-output + if: ${{ !cancelled() }} - name: Fail if any images have critical vulnerabilities run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 82a5234f11..377ca85585 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -4,28 +4,19 @@ # where the key is the OS distro and the value is the tag to deploy. kolla_image_tags: openstack: - rocky-9: 2025.1-rocky-9-20250730T105631 + rocky-9: 2025.1-rocky-9-20260123T132550 ubuntu-noble: 2025.1-ubuntu-noble-20250730T105631 iscsid: - rocky-9: 2025.1-rocky-9-20251013T135156 ubuntu-noble: 2025.1-ubuntu-noble-20251013T135156 keystone: - rocky-9: 2025.1-rocky-9-20251031T094539 ubuntu-noble: 2025.1-ubuntu-noble-20251031T094539 kolla_toolbox: - rocky-9: 2025.1-rocky-9-20250905T151507 ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507 magnum: - rocky-9: 2025.1-rocky-9-20251016T073648 ubuntu-noble: 2025.1-ubuntu-noble-20251016T073648 neutron: - rocky-9: 2025.1-rocky-9-20260116T134029 ubuntu-noble: 2025.1-ubuntu-noble-20260116T134029 - ovn: - rocky-9: 2025.1-rocky-9-20250915T100822 prometheus_libvirt_exporter: - rocky-9: 2025.1-rocky-9-20250822T151934 ubuntu-noble: 2025.1-ubuntu-noble-20250822T151934 rabbitmq: - rocky-9: 2025.1-rocky-9-20250905T151507 ubuntu-noble: 2025.1-ubuntu-noble-20250905T151507 diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 09392411fd..f0deb9b5dc 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -23,6 +23,11 @@ kolla_source_version: "{{ stackhpc_kolla_source_version }}" # Path in which to generate kolla configuration. #kolla_build_config_path: +# View debug logs, like source downloads, during image build. +kolla_build_extra_config: | + [DEFAULT] + debug = True + ############################################################################### # Kolla-ansible installation. diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf index 6ff0aa287f..7457645702 100644 --- a/etc/kayobe/kolla/kolla-build.conf +++ b/etc/kayobe/kolla/kolla-build.conf @@ -29,3 +29,8 @@ sha256 = amd64:1fd60b1fd59c239bed22719a5de402cb745d1f933540cb1ec196e2c03e6e8882, [magnum-conductor-plugin-helm] version = v3.18.2 sha256 = amd64:c5deada86fe609deefdf40e9cbbe3da2f8cf3f6a4551a0ebe7886dc8fcf98bce,arm64:03181a494a0916b370a100a5b2536104963b095be53fb23d1e29b2afb1c7de8d + +# TODO: move to kolla_sources in kolla.yml once https://review.opendev.org/c/openstack/kayobe/+/970268 is available +[prometheus-cadvisor] +version = 0.54.1 +sha256 = amd64:21be8d2797433048474e676d37c215c28fb171509448ef9b1c4648a564e39595,arm64:21f7bac786f6c53a8091964b4d3ff2486a0c460e5a410000b59a9a565b4183a9 diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml index 1e8aa8370c..4aacbb8e23 100644 --- a/etc/kayobe/pulp-host-image-versions.yml +++ b/etc/kayobe/pulp-host-image-versions.yml @@ -1,5 +1,5 @@ --- # Overcloud host image versioning tags # These images must be in SMS, since they are used by our AIO CI runners -stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260113T105634" +stackhpc_rocky_9_overcloud_host_image_version: "2025.1-20260119T110913" stackhpc_ubuntu_noble_overcloud_host_image_version: "2025.1-20250930T144255" diff --git a/etc/kayobe/trivy/allowed-vulnerabilities.yml b/etc/kayobe/trivy/allowed-vulnerabilities.yml index 4759862058..b85bff573e 100644 --- a/etc/kayobe/trivy/allowed-vulnerabilities.yml +++ b/etc/kayobe/trivy/allowed-vulnerabilities.yml @@ -20,7 +20,10 @@ influxdb_allowed_vulnerabilities: - CVE-2024-45337 magnum_conductor_allowed_vulnerabilities: - CVE-2024-45337 +opensearch_dashboards_allowed_vulnerabilities: + - CVE-2025-68428 prometheus_blackbox_exporter_allowed_vulnerabilities: + - CVE-2024-24790 - CVE-2024-45337 prometheus_memcached_exporter_allowed_vulnerabilities: - CVE-2024-45337 @@ -31,7 +34,10 @@ prometheus_elasticsearch_exporter_allowed_vulnerabilities: prometheus_node_exporter_allowed_vulnerabilities: - CVE-2024-45337 prometheus_openstack_exporter_allowed_vulnerabilities: + - CVE-2024-24790 - CVE-2024-45337 +prometheus_ovn_exporter_allowed_vulnerabilities: + - CVE-2024-24790 prometheus_libvirt_exporter_allowed_vulnerabilities: - CVE-2024-45337 prometheus_cadvisor_allowed_vulnerabilities: diff --git a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml index dcdb765ec7..6b6fa315fa 100644 --- a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml +++ b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml @@ -1,7 +1,8 @@ --- features: - | - Added support for Rocky Linux 9.7. + Added support for Rocky Linux 9.7, including host packages and a full + container image refresh. - | 9.7 is now the default release for Rocky Linux. diff --git a/tools/scan-images.sh b/tools/scan-images.sh index fa792f4d7d..54c823ed1a 100755 --- a/tools/scan-images.sh +++ b/tools/scan-images.sh @@ -58,24 +58,26 @@ get_images() { # Generate ignored vulnerabilities file generate_trivy_ignore() { local imagename=$1 - local global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) - local image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + local global_vulnerabilities + global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + local image_vulnerabilities + image_vulnerabilities=$(yq ."$imagename"'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) touch .trivyignore for vulnerability in $global_vulnerabilities; do - echo $vulnerability >> .trivyignore + echo "$vulnerability" >> .trivyignore done for vulnerability in $image_vulnerabilities; do - echo $vulnerability >> .trivyignore + echo "$vulnerability" >> .trivyignore done } # Put results into CSV generate_summary_csv() { - local imagename=$1 - local filename=$2 + local scan="$1" + local summary="$2" - echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${imagename}/${filename}-summary.csv + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > "$summary" jq -r '.Results[] | select(.Vulnerabilities) @@ -94,16 +96,15 @@ generate_summary_csv() { ] ) | .[] - | @csv' image-scan-output/${imagename}/${filename}-scan.json >> image-scan-output/${imagename}/${filename}-summary.csv + | @csv' "$scan" >> "$summary" } # Categorise images based on severity categorise_image() { - local imagename=$1 - local filename=$2 - local image=$3 + local summary="$1" + local image="$2" - if [ $(grep "CRITICAL" image-scan-output/${imagename}/${filename}-summary.csv -c) -gt 0 ]; then + if [ "$(grep "CRITICAL" "$summary" -c)" -gt 0 ]; then echo "${image}" >> image-scan-output/critical-images.txt else echo "${image}" >> image-scan-output/high-images.txt @@ -112,45 +113,78 @@ categorise_image() { # Generate SBOM, return correct scan command for SBOM generate_sbom() { - local imagename=$1 - local filename=$2 - local image=$3 + local sbom="$1" + local scan="$2" + local image="$3" trivy image \ + --debug \ --format spdx-json \ - --output image-scan-output/${imagename}/${filename}-sbom.json \ - $image > /dev/null 2>&1 - echo "trivy sbom $scan_common_args \ - --output image-scan-output/${imagename}/${filename}-scan.json \ - image-scan-output/${imagename}/${filename}-sbom.json" + --output "$sbom" \ + "$image" &> "$sbom.log" + if [ ! -e "$sbom" ]; then + ( + echo "ERROR: trivy image didn't produce the sbom file $sbom for $image" 1>&2 + echo "==== trivy log ====" + cat "$sbom.log" + ) 1>&2 + exit 1 + elif grep -q FATAL "$sbom.log"; then + ( + echo "ERROR: trivy image encountered a fatal error producing $sbom for $image" + echo "==== trivy log ====" + cat "$sbom.log" + echo "==== sbom.json ====" + cat "$sbom" + ) 1>&2 + exit 1 + else + echo "trivy sbom $scan_common_args --output $scan $sbom" + fi } # Scan images, generate SBOMs if requested scan_image() { local image=$1 - local filename=$(basename $image | sed 's/:/\./g') - local imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g') + local filename + filename=$(basename "$image" | sed 's/:/\./g') + local imagename + imagename=$(echo "$filename" | cut -d "." -f 1 | sed 's/-/_/g') + local sbom="image-scan-output/${imagename}/${filename}-sbom.json" + local scan="image-scan-output/${imagename}/${filename}-scan.json" + local summary="image-scan-output/${imagename}/${filename}-summary.csv" - mkdir -p image-scan-output/$imagename - generate_trivy_ignore $imagename + mkdir -p "image-scan-output/$imagename" + generate_trivy_ignore "$imagename" # If SBOM is required, generate it first and scan the results, otherwise we # scan the image directly. if $generate_sbom; then echo "Generating SBOM for $imagename" - scan_command=$(generate_sbom $imagename $filename $image) + scan_command="$(generate_sbom "$sbom" "$scan" "$image")" else - scan_command="trivy image $scan_common_args \ - --output image-scan-output/${imagename}/${filename}-scan.json $image" + scan_command="trivy image $scan_common_args --output $scan $image" fi # Run scan against image or SBOM, format output. If no results, delete files. echo "Scanning $imagename for vulnerabilities" - if $scan_command > /dev/null 2>&1; then - rm -f image-scan-output/${imagename}/${filename}-scan.json + if $scan_command >& "$scan.log"; then + rm -f "$scan" echo "${image}" >> image-scan-output/clean-images.txt + elif [ ! -f "$scan" ]; then + ( + echo "ERROR: trivy scan encountered an error producing $scan" + echo "Command: $scan_command" + echo "==== trivy log ====" + cat "$scan.log" + if $generate_sbom; then + echo "==== sbom.json ====" + cat "$sbom" + fi + ) 1>&2 + exit 1 else - generate_summary_csv $imagename $filename - categorise_image $imagename $filename $image + generate_summary_csv "$scan" "$summary" + categorise_image "$summary" "$image" fi } @@ -170,9 +204,9 @@ main() { check_deps_installed file_prep - images=$(get_images $1 $2) + images=$(get_images "$1" "$2") for image in $images; do - scan_image $image + scan_image "$image" done }