USHIFT-6423: Add release info images into the mirrored registry

docs/contributor/howto_quay_build.md

@@ -47,7 +47,7 @@ git checkout "${QUAY_VER}"
 Install the RPM dependencies required to build Quay images from sources.
 
 ```
-sudo dnf install -y podman podman-compose
+sudo dnf install -y podman podman-docker podman-compose
 sudo ln -s $(which podman-compose) /usr/bin/docker-compose
 ```
 

scripts/mirror-images.sh

@@ -22,6 +22,7 @@ function skopeo_retry() {
     for attempt in $(seq 3) ; do
         if ! skopeo "$@" ; then
             echo "WARNING: Failed to run skopeo, retry #${attempt}" >&2
+            echo "WARNING: skopeo $*" >&2
         else
             return 0
         fi
@@ -39,7 +40,8 @@ function skopeo_opts_for_image() {
     # Cannot copy a single platform for manifest image references
     local media_type
     media_type=$(skopeo_retry inspect --raw --authfile "${img_pull_file}" "${img_src_ref}" | jq -r .mediaType)
-    if [ "${media_type}" == "application/vnd.docker.distribution.manifest.list.v2+json" ] ; then
+    if [ "${media_type}" == "application/vnd.docker.distribution.manifest.list.v2+json" ] ||
+       [ "${media_type}" == "application/vnd.oci.image.index.v1+json" ] ; then
         echo -n "--all"
     fi
     echo -n ""

test/bin/ci_phase_boot_and_test.sh

@@ -52,7 +52,12 @@ cd "${ROOTDIR}/test"
 bash -x ./bin/manage_hypervisor_config.sh create
 
 # Setup a container registry and mirror images.
-bash -x ./bin/mirror_registry.sh
+# Release jobs need to also mirror the images from the brew RPMs.
+if [[ "${SCENARIO_SOURCES:-}" =~ .*releases.* ]]; then
+    bash -x ./bin/mirror_registry.sh -ri "${BREW_RPM_SOURCE}"
+else
+    bash -x ./bin/mirror_registry.sh
+fi
 
 # Prepare all the scenarios that need to run into an output directory
 # where all the relevant scenarios will be copied for execution

test/bin/mirror_registry.sh

@@ -321,8 +321,27 @@ finalize_registry() {
     rm -f "${QUAY_PULL_SECRET}"
 }
 
+get_release_info_images() {
+    local -r idir=$1
+    local -r ofile=$2
+
+    find "${idir}" -name "release-*$(uname -m).json" | while read -r json; do
+        for img in $(jq -r '.images[]' "${json}"); do
+            [ -z "${img}" ] && continue
+            # Skip OpenDataHub, llm-d and Red Hat AI images because they are too large to mirror
+            [[ "${img}" =~ "/opendatahub/" ]] && continue
+            [[ "${img}" =~ "/llm-d/" ]] && continue
+            [[ "${img}" =~ "/rhaiis/" ]] && continue
+            [[ "${img}" =~ "/modh/" ]] && continue
+
+            echo "${img}" >> "${ofile}"
+        done
+    done
+}
+
 mirror_images() {
     local -r ifile=$1
+    local -r ri_dir=$2
     local -r ffile=$(mktemp /tmp/from-list.XXXXXXXX)
     local -r ofile=$(mktemp /tmp/container-list.XXXXXXXX)
 
@@ -335,7 +354,24 @@ mirror_images() {
         echo "${src_img}" >> "${ffile}"
     done
 
-    # Add test assets images.
+    # Add images extracted from brew release info RPMs
+    if [ -n "${ri_dir}" ]; then
+        local -r brew_workdir=$(mktemp -d /tmp/brew-workdir.XXXXXXXX)
+        find "${ri_dir}" -name 'microshift-*release-info*.rpm' | while read -r rpm ; do
+            local rpm_dir
+            rpm_dir="${brew_workdir}/$(basename "${rpm}")"
+            mkdir -p "${rpm_dir}"
+
+            rpm2cpio "${rpm}" | cpio -idmu --quiet -D "${rpm_dir}"
+            get_release_info_images "${rpm_dir}" "${ffile}"
+        done
+        rm -rf "${brew_workdir}"
+    fi
+
+    # Add release info images from the source directory
+    get_release_info_images "${SCRIPTDIR}/../../assets/" "${ffile}"
+
+    # Add test assets images
     find "${SCRIPTDIR}/../assets/" -type f -exec grep -hPo "(?<=image: ).*\..*" {} \; | while read -r img; do
         echo "${img}" >> "${ffile}"
     done
@@ -350,6 +386,8 @@ usage() {
     echo "Usage: ${0} [-cf FILE]"
     echo "   -cf FILE    File containing the container image references to mirror."
     echo "               Defaults to '${CONTAINER_LIST}', skipped if does not exist."
+    echo "   -ri DIR     Directory containing the release info RPM packages to"
+    echo "               extract the container image references from."
     echo ""
     echo "The registry data is stored at '${MIRROR_REGISTRY_DIR}' on the host."
     exit 1
@@ -359,6 +397,7 @@ usage() {
 # Main
 #
 image_list_file="${CONTAINER_LIST}"
+release_info_dir=""
 
 while [ $# -gt 0 ]; do
     case $1 in
@@ -367,6 +406,11 @@ while [ $# -gt 0 ]; do
         [ -z "$1" ] && usage
         image_list_file=$1
         ;;
+    -ri)
+        shift
+        [ -z "$1" ] && usage
+        release_info_dir=$1
+        ;;
     *)
         usage
         ;;
@@ -379,8 +423,13 @@ if [ ! -f "${image_list_file}" ]; then
     exit 1
 fi
 
+if [ -n "${release_info_dir}" ] && [ ! -d "${release_info_dir}" ]; then
+    echo "ERROR: Directory '${release_info_dir}' does not exist"
+    exit 1
+fi
+
 setup_prereqs
 setup_registry
-mirror_images "${image_list_file}"
+mirror_images "${image_list_file}" "${release_info_dir}"
 finalize_registry
 echo "OK"