diff --git a/docs/contributor/howto_quay_build.md b/docs/contributor/howto_quay_build.md index 9e17a62f82..cf91cf844d 100644 --- a/docs/contributor/howto_quay_build.md +++ b/docs/contributor/howto_quay_build.md @@ -47,7 +47,7 @@ git checkout "${QUAY_VER}" Install the RPM dependencies required to build Quay images from sources. ``` -sudo dnf install -y podman podman-compose +sudo dnf install -y podman podman-docker podman-compose sudo ln -s $(which podman-compose) /usr/bin/docker-compose ``` diff --git a/scripts/mirror-images.sh b/scripts/mirror-images.sh index 5372d68792..d5c8ac7dee 100755 --- a/scripts/mirror-images.sh +++ b/scripts/mirror-images.sh @@ -22,6 +22,7 @@ function skopeo_retry() { for attempt in $(seq 3) ; do if ! skopeo "$@" ; then echo "WARNING: Failed to run skopeo, retry #${attempt}" >&2 + echo "WARNING: skopeo $*" >&2 else return 0 fi @@ -39,7 +40,8 @@ function skopeo_opts_for_image() { # Cannot copy a single platform for manifest image references local media_type media_type=$(skopeo_retry inspect --raw --authfile "${img_pull_file}" "${img_src_ref}" | jq -r .mediaType) - if [ "${media_type}" == "application/vnd.docker.distribution.manifest.list.v2+json" ] ; then + if [ "${media_type}" == "application/vnd.docker.distribution.manifest.list.v2+json" ] || + [ "${media_type}" == "application/vnd.oci.image.index.v1+json" ] ; then echo -n "--all" fi echo -n "" diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index fd87762fc4..a3b10b0dc6 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -52,7 +52,12 @@ cd "${ROOTDIR}/test" bash -x ./bin/manage_hypervisor_config.sh create # Setup a container registry and mirror images. -bash -x ./bin/mirror_registry.sh +# Release jobs need to also mirror the images from the brew RPMs. +if [[ "${SCENARIO_SOURCES:-}" =~ .*releases.* ]]; then + bash -x ./bin/mirror_registry.sh -ri "${BREW_RPM_SOURCE}" +else + bash -x ./bin/mirror_registry.sh +fi # Prepare all the scenarios that need to run into an output directory # where all the relevant scenarios will be copied for execution diff --git a/test/bin/mirror_registry.sh b/test/bin/mirror_registry.sh index 07dd6f72ac..987feeb53d 100755 --- a/test/bin/mirror_registry.sh +++ b/test/bin/mirror_registry.sh @@ -321,8 +321,27 @@ finalize_registry() { rm -f "${QUAY_PULL_SECRET}" } +get_release_info_images() { + local -r idir=$1 + local -r ofile=$2 + + find "${idir}" -name "release-*$(uname -m).json" | while read -r json; do + for img in $(jq -r '.images[]' "${json}"); do + [ -z "${img}" ] && continue + # Skip OpenDataHub, llm-d and Red Hat AI images because they are too large to mirror + [[ "${img}" =~ "/opendatahub/" ]] && continue + [[ "${img}" =~ "/llm-d/" ]] && continue + [[ "${img}" =~ "/rhaiis/" ]] && continue + [[ "${img}" =~ "/modh/" ]] && continue + + echo "${img}" >> "${ofile}" + done + done +} + mirror_images() { local -r ifile=$1 + local -r ri_dir=$2 local -r ffile=$(mktemp /tmp/from-list.XXXXXXXX) local -r ofile=$(mktemp /tmp/container-list.XXXXXXXX) @@ -335,7 +354,24 @@ mirror_images() { echo "${src_img}" >> "${ffile}" done - # Add test assets images. + # Add images extracted from brew release info RPMs + if [ -n "${ri_dir}" ]; then + local -r brew_workdir=$(mktemp -d /tmp/brew-workdir.XXXXXXXX) + find "${ri_dir}" -name 'microshift-*release-info*.rpm' | while read -r rpm ; do + local rpm_dir + rpm_dir="${brew_workdir}/$(basename "${rpm}")" + mkdir -p "${rpm_dir}" + + rpm2cpio "${rpm}" | cpio -idmu --quiet -D "${rpm_dir}" + get_release_info_images "${rpm_dir}" "${ffile}" + done + rm -rf "${brew_workdir}" + fi + + # Add release info images from the source directory + get_release_info_images "${SCRIPTDIR}/../../assets/" "${ffile}" + + # Add test assets images find "${SCRIPTDIR}/../assets/" -type f -exec grep -hPo "(?<=image: ).*\..*" {} \; | while read -r img; do echo "${img}" >> "${ffile}" done @@ -350,6 +386,8 @@ usage() { echo "Usage: ${0} [-cf FILE]" echo " -cf FILE File containing the container image references to mirror." echo " Defaults to '${CONTAINER_LIST}', skipped if does not exist." + echo " -ri DIR Directory containing the release info RPM packages to" + echo " extract the container image references from." echo "" echo "The registry data is stored at '${MIRROR_REGISTRY_DIR}' on the host." exit 1 @@ -359,6 +397,7 @@ usage() { # Main # image_list_file="${CONTAINER_LIST}" +release_info_dir="" while [ $# -gt 0 ]; do case $1 in @@ -367,6 +406,11 @@ while [ $# -gt 0 ]; do [ -z "$1" ] && usage image_list_file=$1 ;; + -ri) + shift + [ -z "$1" ] && usage + release_info_dir=$1 + ;; *) usage ;; @@ -379,8 +423,13 @@ if [ ! -f "${image_list_file}" ]; then exit 1 fi +if [ -n "${release_info_dir}" ] && [ ! -d "${release_info_dir}" ]; then + echo "ERROR: Directory '${release_info_dir}' does not exist" + exit 1 +fi + setup_prereqs setup_registry -mirror_images "${image_list_file}" +mirror_images "${image_list_file}" "${release_info_dir}" finalize_registry echo "OK"