Skip to content

KUBESAW-230: improve pairing mechanism #1113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rsoaresd wants to merge 3 commits into
base: master
Choose a base branch
from
Open

KUBESAW-230: improve pairing mechanism #1113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
de17b0a
improve pairing mechanism
rsoaresd Mar 11, 2025
ed4d161
Merge branch 'master' into test-pairing
mfrancisc Mar 13, 2025
b96cb8a
Merge branch 'master' into test-pairing
mfrancisc Apr 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 2 additions & 36 deletions make/ksctl.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,8 @@ endif
get-ksctl-and-install:
ifeq ($(strip $(KSCTL_REPO_PATH)),)
ifneq ($(CI_DISABLE_PAIRING),true)
ifeq ($(shell jq -r '.refs[0].org' <<< $${CLONEREFS_OPTIONS} 2>/dev/null || true | tr -d '[:space:]'),codeready-toolchain)
$(eval AUTHOR_LINK = $(shell jq -r '.refs[0].pulls[0].author_link' <<< $${CLONEREFS_OPTIONS} | tr -d '[:space:]'))
$(eval BRANCH_NAME := $(shell jq -r '.refs[0].pulls[0].head_ref' <<< $${CLONEREFS_OPTIONS} | tr -d '[:space:]'))
@echo "using author link ${AUTHOR_LINK}"
@echo "detected branch ${BRANCH_NAME}"
# check if a branch with the same ref exists in the user's fork of ksctl repo
@echo "branches of ${AUTHOR_LINK}/ksctl - checking if there is a branch ${BRANCH_NAME} we could pair with."
curl ${AUTHOR_LINK}/ksctl.git/info/refs?service=git-upload-pack --output -
$(eval REMOTE_KSCTL_BRANCH := $(shell curl ${AUTHOR_LINK}/ksctl.git/info/refs?service=git-upload-pack --output - 2>/dev/null | grep -a "refs/heads/${BRANCH_NAME}$$" | awk '{print $$2}'))
# check if the branch with the same name exists, if so then merge it with master and use the merge branch, if not then use master
@$(MAKE) clone-ksctl-and-pair REMOTE_KSCTL_BRANCH=${REMOTE_KSCTL_BRANCH} AUTHOR_LINK=${AUTHOR_LINK}
else
@echo "Either not running in openshift-ci or not in the codeready-toolchain org - no pairing"
endif
${PAIRING_EXEC} pair --clone-dir /tmp/ksctl --organization kubesaw --repository ksctl
$(eval KSCTL_REPO_PATH := /tmp/ksctl)
else
@echo "Pairing explicitly disabled"
endif
Expand All @@ -39,28 +27,6 @@ else
endif
@$(MAKE) install-ksctl KSCTL_REPO_PATH=${KSCTL_REPO_PATH} REMOTE_KSCTL_BRANCH=${REMOTE_KSCTL_BRANCH}

clone-ksctl-and-pair:
ifneq ($(strip $(REMOTE_KSCTL_BRANCH)),)
@echo "Branch ref of the user's fork to be used for pairing: \"${REMOTE_KSCTL_BRANCH}\""
# define temp dir
$(eval KSCTL_REPO_PATH := /tmp/ksctl)
# delete to have clear environment
rm -rf ${KSCTL_REPO_PATH}

git config --global user.email "devsandbox@redhat.com"
git config --global user.name "KubeSaw"
# clone
git clone --depth=1 https://github.com/kubesaw/ksctl.git ${KSCTL_REPO_PATH}
# add the user's fork as remote repo
git --git-dir=${KSCTL_REPO_PATH}/.git --work-tree=${KSCTL_REPO_PATH} remote add external ${AUTHOR_LINK}/ksctl.git
# fetch the branch
git --git-dir=${KSCTL_REPO_PATH}/.git --work-tree=${KSCTL_REPO_PATH} fetch external ${REMOTE_KSCTL_BRANCH}
# merge the branch with master
git --git-dir=${KSCTL_REPO_PATH}/.git --work-tree=${KSCTL_REPO_PATH} merge --allow-unrelated-histories --no-commit FETCH_HEAD
else
@echo "No branch for pairing found"
endif

install-ksctl:
ifneq ($(strip $(KSCTL_REPO_PATH)$(REMOTE_KSCTL_BRANCH)),)
@echo "Installing ksctl from directory $(or ${KSCTL_REPO_PATH}, /tmp/ksctl)"
Expand Down
27 changes: 21 additions & 6 deletions make/test.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,24 @@ endif

TESTS_RUN_FILTER_REGEXP ?= ""

PAIRING_DIR := "/tmp/pairing"
PAIRING_EXEC := "/tmp/pairing/pairing"
TOOLCHAIN_CICD := "/tmp/toolchain-cicd"

.PHONY: build-pairing
build-pairing:
# Clean the directory if it exists
if [ -d "${TOOLCHAIN_CICD}" ]; then rm -rf ${TOOLCHAIN_CICD}; fi
# Clone
git clone --branch add_pairing_go_script https://github.com/rsoaresd/toolchain-cicd.git ${TOOLCHAIN_CICD}
Copy link
Contributor Author

@rsoaresd rsoaresd Mar 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just temporary until codeready-toolchain/toolchain-cicd#135 is merged

# Build the Go binary into the specified directory
make -C ${TOOLCHAIN_CICD}/pairing build-pairing PAIRING_DIR=${PAIRING_DIR}


.PHONY: test-e2e
## Run the e2e tests
test-e2e: INSTALL_OPERATOR=true
test-e2e: prepare-e2e verify-migration-and-deploy-e2e e2e-run-parallel e2e-run e2e-run-metrics
test-e2e: build-pairing prepare-e2e verify-migration-and-deploy-e2e e2e-run-parallel e2e-run e2e-run-metrics
@echo "The tests successfully finished"
@echo "To clean the cluster run 'make clean-e2e-resources'"

Expand All @@ -57,7 +71,7 @@ test-e2e-sequential-only: prepare-e2e deploy-e2e e2e-run e2e-run-metrics
@echo "To clean the cluster run 'make clean-e2e-resources'"

.PHONY: prepare-and-deploy-e2e
## Prepare and Deploy e2e environment. Usefull to reset without having to run a test
## Prepare and Deploy e2e environment. Useful to reset without having to run a test
prepare-and-deploy-e2e: prepare-e2e deploy-e2e
@echo "To clean the cluster run 'make clean-e2e-resources'"

Expand Down Expand Up @@ -91,7 +105,7 @@ deploy-published-operators-e2e: clean-e2e-files deploy-e2e
.PHONY: deploy-e2e
deploy-e2e: INSTALL_OPERATOR=true
deploy-e2e: prepare-projects get-publish-install-and-register-operators
@echo "Operators are successfuly deployed using the ${ENVIRONMENT} environment."
@echo "Operators are successfully deployed using the ${ENVIRONMENT} environment."
@echo ""

label-olm-ns:
Expand Down Expand Up @@ -281,7 +295,8 @@ deploy-single-member-e2e-latest:

.PHONY: publish-current-bundles-for-e2e
## Target that is supposed to be called from CI - it builds & publishes the current operator bundles
publish-current-bundles-for-e2e: get-and-publish-operators
publish-current-bundles-for-e2e:
$(MAKE) build-pairing get-and-publish-operators

.PHONY: get-and-publish-operators
get-and-publish-operators: PUBLISH_OPERATOR=true
Expand Down Expand Up @@ -328,7 +343,7 @@ ifeq ($(DEPLOY_LATEST),true)
endif
else
@echo "Installing specific version of the member-operator"
scripts/ci/manage-member-operator.sh -po ${PUBLISH_OPERATOR} -io ${INSTALL_OPERATOR} -mn ${MEMBER_NS} ${MEMBER_REPO_PATH_PARAM} -qn ${QUAY_NAMESPACE} -ds ${DATE_SUFFIX} ${MEMBER_NS_2_PARAM} ${FORCED_TAG_PARAM}
scripts/ci/manage-member-operator.sh -po ${PUBLISH_OPERATOR} -io ${INSTALL_OPERATOR} -mn ${MEMBER_NS} ${MEMBER_REPO_PATH_PARAM} -qn ${QUAY_NAMESPACE} -ds ${DATE_SUFFIX} ${MEMBER_NS_2_PARAM} ${FORCED_TAG_PARAM} -pe ${PAIRING_EXEC}
endif

.PHONY: get-and-publish-host-operator
Expand All @@ -353,7 +368,7 @@ ifeq ($(DEPLOY_LATEST),true)
${KSCTL_BIN_DIR}ksctl adm install-operator host --kubeconfig "$(or ${KUBECONFIG}, ${HOME}/.kube/config)" --namespace ${HOST_NS} ${KSCTL_INSTALL_TIMEOUT_PARAM} -y
else
@echo "Installing specific version of the host-operator"
scripts/ci/manage-host-operator.sh -po ${PUBLISH_OPERATOR} -io ${INSTALL_OPERATOR} -hn ${HOST_NS} ${HOST_REPO_PATH_PARAM} -ds ${DATE_SUFFIX} -qn ${QUAY_NAMESPACE} ${REG_REPO_PATH_PARAM} ${FORCED_TAG_PARAM}
scripts/ci/manage-host-operator.sh -po ${PUBLISH_OPERATOR} -io ${INSTALL_OPERATOR} -hn ${HOST_NS} ${HOST_REPO_PATH_PARAM} -ds ${DATE_SUFFIX} -qn ${QUAY_NAMESPACE} ${REG_REPO_PATH_PARAM} ${FORCED_TAG_PARAM} -pe ${PAIRING_EXEC}
endif

###########################################################
Expand Down
8 changes: 7 additions & 1 deletion scripts/ci/manage-host-operator.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ user_help () {
echo "-io, --install-operator Installs the operator to an OpenShift cluster"
echo "-hn, --host-namespace Namespace the operator should be installed to"
echo "-hr, --host-repo-path Path to the host operator repo"
echo "-rr, --reg-repo-path Path to the registation service repo"
echo "-rr, --reg-repo-path Path to the registration service repo"
echo "-ds, --date-suffix Date suffix to be added to some resources that are created"
echo "-ft, --forced-tag Forces a tag to be set to all built images. In the case deployment the tag is used for index image in the created CatalogSource"
echo "-pe, --pairing-exec Binary path to execute the pairing"
echo "-h, --help To show this help text"
echo ""
exit 0
Expand Down Expand Up @@ -68,6 +69,11 @@ read_arguments() {
FORCED_TAG=$1
shift
;;
-pe|--pairing-exec)
shift
PAIRING_EXEC=$1
shift
;;
*)
echo "$1 is not a recognized flag!" >> /dev/stderr
user_help
Expand Down
6 changes: 6 additions & 0 deletions scripts/ci/manage-member-operator.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ user_help () {
echo "-mr, --member-repo-path Path to the member operator repo"
echo "-ds, --date-suffix Date suffix to be added to some resources that are created"
echo "-ft, --forced-tag Forces a tag to be set to all built images. In the case deployment the tag is used for index image in the created CatalogSource"
echo "-pe, --pairing-exec Binary path to execute the pairing"
echo "-h, --help To show this help text"
echo ""
exit 0
Expand Down Expand Up @@ -67,6 +68,11 @@ read_arguments() {
FORCED_TAG=$1
shift
;;
-pe|--pairing-exec)
shift
PAIRING_EXEC=$1
shift
;;
*)
echo "$1 is not a recognized flag!" >> /dev/stderr
user_help
Expand Down
121 changes: 5 additions & 116 deletions scripts/ci/manage-operator.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,13 @@ fi
WAS_ALREADY_PAIRED_FILE=/tmp/toolchain_e2e_already_paired

get_repo() {
PAIRED=false
if [[ -z ${PROVIDED_REPOSITORY_PATH} ]]; then
REPOSITORY_PATH="/tmp/codeready-toolchain/${REPOSITORY_NAME}"
rm -rf ${REPOSITORY_PATH}
# clone
git clone https://github.com/codeready-toolchain/${REPOSITORY_NAME}.git ${REPOSITORY_PATH}

pair_repo_if_needed

${PAIRING_EXEC} pair --clone-dir ${REPOSITORY_PATH} --organization codeready-toolchain --repository ${REPOSITORY_NAME}
else
REPOSITORY_PATH=${PROVIDED_REPOSITORY_PATH}
fi

}

set_tags() {
Expand All @@ -47,117 +42,11 @@ set_tags() {
push_image() {
GIT_COMMIT_ID=$(git --git-dir=${REPOSITORY_PATH}/.git --work-tree=${REPOSITORY_PATH} rev-parse --short HEAD)
IMAGE_LOC=quay.io/codeready-toolchain/${REPOSITORY_NAME}:${GIT_COMMIT_ID}
if is_provided_or_paired; then
IMAGE_BUILDER=${IMAGE_BUILDER:-"podman"}
make -C ${REPOSITORY_PATH} ${IMAGE_BUILDER}-push QUAY_NAMESPACE=${QUAY_NAMESPACE} IMAGE_TAG=${TAGS}
IMAGE_LOC=quay.io/${QUAY_NAMESPACE}/${REPOSITORY_NAME}:${TAGS}
fi
IMAGE_BUILDER=${IMAGE_BUILDER:-"podman"}
make -C ${REPOSITORY_PATH} ${IMAGE_BUILDER}-push QUAY_NAMESPACE=${QUAY_NAMESPACE} IMAGE_TAG=${TAGS}
IMAGE_LOC=quay.io/${QUAY_NAMESPACE}/${REPOSITORY_NAME}:${TAGS}
}

is_provided_or_paired() {
[[ -n ${PROVIDED_REPOSITORY_PATH} ]] || [[ ${PAIRED} == true ]]
}

pair_repo_if_needed() {
if [[ -n ${GITHUB_ACTIONS} ]]; then
PR_REPO_NAME=${GITHUB_REPOSITORY##*/}
else
PR_REPO_NAME=$(echo ${REPO_NAME} | sed 's/"//g')
fi


if [[ -n "${CI}${CLONEREFS_OPTIONS}" ]] && [[ ${PR_REPO_NAME} == "toolchain-e2e" ]]; then
if [[ -n ${CLONEREFS_OPTIONS} ]]; then
# get branch ref of the fork the PR was created from
AUTHOR_LINK=$(jq -r '.refs[0].pulls[0].author_link' <<< ${CLONEREFS_OPTIONS} | tr -d '[:space:]')
PULL_PULL_SHA=${PULL_PULL_SHA:-$(jq -r '.refs[0].pulls[0].sha' <<< ${CLONEREFS_OPTIONS} | tr -d '[:space:]')}
echo "using author link ${AUTHOR_LINK}"
echo "using pull sha ${PULL_PULL_SHA}"
# get branch ref of the fork the PR was created from
REPO_URL=${AUTHOR_LINK}/toolchain-e2e
echo "branches of ${REPO_URL} - trying to detect the branch name we should use for pairing."
curl ${REPO_URL}.git/info/refs?service=git-upload-pack --output -
GET_BRANCH_NAME=$(curl ${REPO_URL}.git/info/refs?service=git-upload-pack --output - 2>/dev/null | grep -a ${PULL_PULL_SHA} || true)
if [[ $(echo ${GET_BRANCH_NAME} | wc -l) > 1 ]]; then \
echo "################################### ERROR DURING THE E2E TEST SETUP ###################################
There were found more branches with the same latest commit '${PULL_PULL_SHA}' in the repo ${REPO_URL} - see:

${GET_BRANCH_NAME}

It's not possible to detect the correct branch this PR is made for.
Please delete the unrelated branch from your fork and rerun the e2e tests.
Note: If you have already deleted the unrelated branch from your fork, it can take a few hours before the
github api is updated so the e2e tests may still fail with the same error until then.
##########################################################################################################"
exit 1
fi
BRANCH_REF=$(echo ${GET_BRANCH_NAME} | awk '{print $2}')
echo "detected branch ref ${BRANCH_REF}"
# retrieve the branch name
BRANCH_NAME=$(echo ${BRANCH_REF} | awk -F'/' '{print $3}')
else
AUTHOR_LINK=https://github.com/${AUTHOR}
BRANCH_REF=refs/heads/${GITHUB_HEAD_REF}
BRANCH_NAME=${GITHUB_HEAD_REF}
REPO_URL=${AUTHOR_LINK}/toolchain-e2e
fi

if [[ -n "${BRANCH_REF}" ]]; then \
# check if a branch with the same ref exists in the user's fork of ${REPOSITORY_NAME} repo
echo "branches of ${AUTHOR_LINK}/${REPOSITORY_NAME} - checking if there is a branch ${BRANCH_REF} we could pair with."
curl ${AUTHOR_LINK}/${REPOSITORY_NAME}.git/info/refs?service=git-upload-pack --output -
REMOTE_E2E_BRANCH=$(curl ${AUTHOR_LINK}/${REPOSITORY_NAME}.git/info/refs?service=git-upload-pack --output - 2>/dev/null | grep -a "${BRANCH_REF}$" | awk '{print $2}')
echo "branch ref of the user's fork: \"${REMOTE_E2E_BRANCH}\" - if empty then not found"
# check if the branch with the same name exists, if so then merge it with master and use the merge branch, if not then use master \
if [[ -n "${REMOTE_E2E_BRANCH}" ]]; then \
if [[ -f ${WAS_ALREADY_PAIRED_FILE} ]]; then \
echo "#################################### ERROR WHILE TRYING TO PAIR PRs ####################################
There was an error while trying to pair this e2e PR with ${AUTHOR_LINK}/${REPOSITORY_NAME}@${BRANCH_REF}
The reason is that there was already detected a branch from another repo this PR could be paired with - see:

$(cat ${WAS_ALREADY_PAIRED_FILE})

It's not possible to pair a PR with multiple branches from other repositories.
Please delete one of the branches from your fork and rerun the e2e tests
Note: If you have already deleted one of the branches from your fork, it can take a few hours before the
github api is updated so the e2e tests may still fail with the same error until then.
##########################################################################################################"
exit 1
fi

git config --global user.email "devsandbox@redhat.com"
git config --global user.name "KubeSaw"

echo -e "repository: ${AUTHOR_LINK}/${REPOSITORY_NAME} \nbranch: ${BRANCH_NAME}" > ${WAS_ALREADY_PAIRED_FILE}
# add the user's fork as remote repo
git --git-dir=${REPOSITORY_PATH}/.git --work-tree=${REPOSITORY_PATH} remote add external ${AUTHOR_LINK}/${REPOSITORY_NAME}.git
# fetch the branch
git --git-dir=${REPOSITORY_PATH}/.git --work-tree=${REPOSITORY_PATH} fetch external ${BRANCH_REF}

echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo ""
echo "The following command will try to merge the paired PR using fast-forward way."
echo "If the command fails, then it means that the paired PR https://github.com/codeready-toolchain/${REPOSITORY_NAME}/ from branch ${BRANCH_NAME}"
echo "is not up-to-date with master and the fast-forward merge cannot be performed."
echo "If this happens, then rebase the PR with the latest changes from master and rerun this GH Actions build (or comment /retest in this PR)."
echo " https://github.com/codeready-toolchain/${REPOSITORY_NAME}/pulls?q=head%3A${BRANCH_NAME}"
echo ""
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
# merge the branch with master using fast-forward
git --git-dir=${REPOSITORY_PATH}/.git --work-tree=${REPOSITORY_PATH} merge --ff-only FETCH_HEAD
# print information about the last three commits, so we know what was merged plus some additional context/history
git --git-dir=${REPOSITORY_PATH}/.git --work-tree=${REPOSITORY_PATH} log --ancestry-path HEAD~3..HEAD

PAIRED=true
fi
fi
fi
}


install_operator() {
DISPLAYNAME=$(echo ${OPERATOR_NAME} | tr '-' ' ' | awk '{for (i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2)} 1')

Expand Down
2 changes: 0 additions & 2 deletions test/e2e/parallel/nstemplatetier_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,6 @@ func TestTierTemplateRevision(t *testing.T) {
Value: "100",
})
})

})

t.Run("when updating one tiertemplate the revisions field should be cleaned up from old entries", func(t *testing.T) {
Expand Down Expand Up @@ -517,7 +516,6 @@ func TestTierTemplateRevision(t *testing.T) {
// revisions values should be different compared to the previous ones
assert.NotEqual(t, revisionsBeforeUpdate, updatedTier.Status.Revisions)
})

}

func getTestCRQ(podsCount string) unstructured.Unstructured {
Expand Down
Loading