diff --git a/contrib/merge-prs.sh b/contrib/merge-prs.sh index 7e322a0f8b..00a28012d0 100755 --- a/contrib/merge-prs.sh +++ b/contrib/merge-prs.sh @@ -3,38 +3,58 @@ export LC_ALL=C set -eo pipefail +# Setup base branch and Bitcoin/Elements remote names. BASE_ORIG=merged-master BASE="${BASE_ORIG}" BITCOIN_UPSTREAM_REMOTE=bitcoin -BITCOIN_UPSTREAM="${BITCOIN_UPSTREAM_REMOTE}/master" -# ELEMENTS_UPSTREAM_REMOTE=upstream -# ELEMENTS_UPSTREAM="${ELEMENTS_UPSTREAM_REMOTE}/master" +export BITCOIN_UPSTREAM="${BITCOIN_UPSTREAM_REMOTE}/master" +ELEMENTS_UPSTREAM_REMOTE=upstream +export ELEMENTS_UPSTREAM="${ELEMENTS_UPSTREAM_REMOTE}/master" +# START USER CONFIG: +# Set your target upstream here +TARGET_UPSTREAM=$BITCOIN_UPSTREAM +TARGET_NAME="Bitcoin" +PR_PREFIX="bitcoin/bitcoin" +# TARGET_UPSTREAM=$ELEMENTS_UPSTREAM +# TARGET_NAME="Elements" +# PR_PREFIX="ElementsProject/elements" + +# Set your git worktree location here. This is where the merges will be done, and where you should checkout the merged-master branch. +WORKTREE="/home/byron/code/elements-worktree" + +# Set your parallellism during build/test. You probably want as many cores as possible. +# Parallel functional tests can somewhat exceed your core count, depends on the build machine CPU/RAM. +PARALLEL_BUILD=23 # passed to make -j +PARALLEL_TEST=46 # passed to test_runner.py --jobs +PARALLEL_FUZZ=12 # passed to test_runner.py -j when fuzzing + +# Setup a ccache dir if necessary. +#export CCACHE_DIR="/tmp/ccache" +#export CCACHE_MAXSIZE="20G" + +# Set and export a WEBHOOK environment variable to a Discord webhook URL outside of this script to get notifications of progress and failures. + +# We don't currently fuzz during merging. Check fuzzing and CI after a merge run. # Replace this with the location where we should put the fuzz test corpus -BITCOIN_QA_ASSETS="${HOME}/.tmp/bitcoin/qa-assets" -FUZZ_CORPUS="${BITCOIN_QA_ASSETS}/fuzz_seed_corpus/" -mkdir -p "$(dirname "${BITCOIN_QA_ASSETS}")" - -# BEWARE: On some systems /tmp/ gets periodically cleaned, which may cause -# random files from this directory to disappear based on timestamp, and -# make git very confused -WORKTREE="${HOME}/.tmp/elements-merge-worktree" -mkdir -p "${HOME}/.tmp" - -# These should be tuned to your machine; below values are for an 8-core -# 16-thread macbook pro -PARALLEL_BUILD=4 # passed to make -j -PARALLEL_TEST=12 # passed to test_runner.py --jobs -PARALLEL_FUZZ=8 # passed to test_runner.py -j when fuzzing +# BITCOIN_QA_ASSETS="${HOME}/code/bitcoin/qa-assets" +# FUZZ_CORPUS="${BITCOIN_QA_ASSETS}/fuzz_seed_corpus/" +# END USER CONFIG + +# Script SKIP_MERGE=0 DO_BUILD=1 KEEP_GOING=1 +DO_TEST=1 +DO_FUZZ=0 +NUM=15 +COUNT=0 if [[ "$1" == "setup" ]]; then echo "Setting up..." echo - git config remote.upstream.url >/dev/null || remote add upstream "https://github.com/ElementsProject/elements.git" + git config remote.upstream.url >/dev/null || git remote add upstream "https://github.com/ElementsProject/elements.git" git config remote.bitcoin.url >/dev/null || git remote add bitcoin "https://github.com/bitcoin/bitcoin.git" if git worktree list --porcelain | grep --silent prunable; then echo "You have stale git worktrees, please either fix them or run 'git worktree prune'." @@ -45,14 +65,7 @@ if [[ "$1" == "setup" ]]; then echo "Fetching all remotes..." echo git fetch --all - echo - #echo "Cloning fuzz test corpus..." - #echo - #if [[ ! -d "${BITCOIN_QA_ASSETS}" ]]; then - # cd "$(dirname ${BITCOIN_QA_ASSETS})" && git clone https://github.com/bitcoin-core/qa-assets.git - #fi - #echo - echo "Done! Remember to also check out merged-master, and push it back up when finished." + echo "Done! Remember to also checkout merged-master at ${WORKTREE}" exit 0 elif [[ "$1" == "continue" ]]; then SKIP_MERGE=1 @@ -62,17 +75,35 @@ elif [[ "$1" == "list-only" ]]; then DO_BUILD=0 elif [[ "$1" == "step" ]]; then KEEP_GOING=0 +elif [[ "$1" == "merge-only" ]]; then + SKIP_MERGE=0 + KEEP_GOING=0 + DO_BUILD=0 + DO_TEST=0 elif [[ "$1" == "step-continue" ]]; then SKIP_MERGE=1 KEEP_GOING=0 +elif [[ "$1" == "step-test" ]]; then + SKIP_MERGE=1 + KEEP_GOING=0 + DO_BUILD=0 +elif [[ "$1" == "step-fuzz" ]]; then + SKIP_MERGE=1 + KEEP_GOING=0 + DO_BUILD=0 + DO_TEST=0 +elif [[ "$1" == "analyze" ]]; then + DO_BUILD=0 + DO_TEST=0 else - echo "Usage: $0 " + echo "Usage: $0 " echo " setup will configure your repository for the first run of this script" echo " list-only will simply list all the PRs yet to be done" echo " go will try to merge every PR, building/testing each" echo " continue assumes the first git-merge has already happened, and starts with building" echo " step will try to merge/build/test a single PR" echo " step-continue assumes the first git-merge has already happened, and will try to build/test a single PR" + echo " analyze will analyze the next $NUM PRs and count conflicts NB: DO NOT CTRL+C THIS PROCESS" echo echo "Prior to use, please create a git worktree for the elements repo at:" echo " $WORKTREE" @@ -101,54 +132,95 @@ if [[ "$SKIP_MERGE" == "1" ]]; then fi ## Get full list of merges -# for elements -# COMMITS=$(git -C "$WORKTREE" log "$ELEMENTS_UPSTREAM" --not $BASE --merges --first-parent --pretty='format:%ct %cI %h Elements %s') -# for bitcoin -COMMITS=$(git -C "$WORKTREE" log "$BITCOIN_UPSTREAM" --not $BASE --merges --first-parent --pretty='format:%ct %cI %h Bitcoin %s') +COMMITS=$(git -C "$WORKTREE" log "$TARGET_UPSTREAM" --not $BASE --merges --first-parent --pretty="format:%ct %cI %h $TARGET_NAME %s") -cd "$WORKTREE" +cd "$WORKTREE" || exit 1 VERBOSE=1 +echo start > merge.log + quietly () { if [[ "$VERBOSE" == "1" ]]; then - "$@" + date | tee --append merge.log + time "$@" 2>&1 | tee --append merge.log else chronic "$@" fi } +notify () { + local MESSAGE="$1" + local JSON="{\"content\": \"$MESSAGE\"}" + if [ -n "$WEBHOOK" ]; then + curl -d "$JSON" -H "Content-Type: application/json" "$WEBHOOK" + else + echo "$MESSAGE" + fi + if [[ "$2" == "1" ]]; then + exit 1 + fi +} + ## Sort by unix timestamp and iterate over them -#echo "$ELT_COMMITS" "$BTC_COMMITS" | sort -n -k1 | while read line echo "$COMMITS" | tac | while read -r line do - echo - echo "=-=-=-=-=-=-=-=-=-=-=" - echo - - echo -e "$line" ## Extract data and output what we're doing - DATE=$(echo "$line" | cut -d ' ' -f 2) HASH=$(echo "$line" | cut -d ' ' -f 3) CHAIN=$(echo "$line" | cut -d ' ' -f 4) - PR_ID=$(echo "$line" | cut -d ' ' -f 6 | tr -d :) - PR_ID_ALT=$(echo "$line" | cut -d ' ' -f 8 | tr -d :) + PR_ID=$(echo "$line" | grep -o -P "#\d+") - if [[ "$PR_ID" == "pull" ]]; then - PR_ID="${PR_ID_ALT}" - fi - echo -e "$CHAIN PR \e[37m$PR_ID \e[33m$HASH\e[0m on \e[32m$DATE\e[0m " + GIT_HEAD=$(git rev-parse HEAD) ## Do it if [[ "$1" == "list-only" ]]; then + echo -e "$line" continue fi + if [[ "$1" == "analyze" ]]; then + COUNT=$((COUNT + 1)) + # todo: count conflicts in critical files + # CRITICAL_FILES=("src/wallet/spend.h", "src/wallet/spend.cpp") + MERGE_FILE="/tmp/$HASH.merge" + DIFF_FILE="/tmp/$HASH.diff" + git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_PREFIX$PR_ID)" > "$MERGE_FILE" || true + git -C "$WORKTREE" diff > "$DIFF_FILE" + git -C "$WORKTREE" reset --hard "$GIT_HEAD" > /dev/null + # FILES=$(grep "CONFLICT" "$MERGE_FILE") + NUM_FILES=$(grep -c "CONFLICT" "$MERGE_FILE") + NUM_CONFLICTS=$(grep -c "<<<<<<<" "$DIFF_FILE") + echo "$COUNT. Merge up to $PR_ID ($HASH) has $NUM_CONFLICTS conflicts in $NUM_FILES files." + if [[ "$COUNT" == "$NUM" ]]; then + exit 0 + else + continue + fi + fi + notify "starting merge of $PR_ID" + + # check for stoppers and halt if found + # a stopper is normally the PR after the version has been changed + # ie. the branch point we want to stop at for this version + STOPPERS=( + "#30716" # bitcoin v28 + "#32041" # bitcoin v29 + ) + for STOPPER in "${STOPPERS[@]}" + do + if [[ "$PR_ID" == *"$STOPPER"* ]]; then + echo "Found $STOPPER in $PR_ID! Exiting." + notify "hit stopper, exiting" + exit 1 + else + echo "Didn't find $STOPPER in $PR_ID. Continuing." + fi + done if [[ "$SKIP_MERGE" == "1" ]]; then echo -e "Continuing build of \e[37m$PR_ID\e[0m at $(date)" else echo -e "Start merge/build of \e[37m$PR_ID\e[0m at $(date)" - git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_ID)" + git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_PREFIX$PR_ID)" || notify "fail merge" 1 fi if [[ "$DO_BUILD" == "1" ]]; then @@ -163,35 +235,43 @@ do # The following is an expansion of `make check` that skips the libsecp # tests and also the benchmarks (though it does build them!) echo "Building" - quietly make -j"$PARALLEL_BUILD" -k -# quietly make -j1 check - echo "Linting" - quietly ./ci/lint/06_script.sh + quietly make -j"$PARALLEL_BUILD" -k || notify "fail build" 1 + # todo: fix linting step + # echo "Linting" + # quietly ./ci/lint/06_script.sh || notify "fail lint" + fi + + if [[ "$DO_TEST" == "1" ]]; then echo "Testing" - quietly ./src/qt/test/test_elements-qt - quietly ./src/test/test_bitcoin - quietly ./src/bench/bench_bitcoin - quietly ./test/util/bitcoin-util-test.py - quietly ./test/util/rpcauth-test.py - quietly make -C src/univalue/ check + quietly ./src/qt/test/test_elements-qt || notify "fail test qt" 1 + quietly ./src/test/test_bitcoin || notify "fail test bitcoin" 1 + quietly ./src/bench/bench_bitcoin || notify "fail test bench" 1 + quietly ./test/util/test_runner.py || notify "fail test util" 1 + quietly ./test/util/rpcauth-test.py || notify "fail test rpc" 1 echo "Functional testing" - quietly ./test/functional/test_runner.py --jobs="$PARALLEL_TEST" + quietly ./test/functional/test_runner.py --jobs="$PARALLEL_TEST" || notify "fail test runner" 1 + fi + + if [[ "$DO_FUZZ" == "1" ]]; then echo "Cleaning for fuzz" quietly make distclean || true quietly git -C "$WORKTREE" clean -xf echo "Building for fuzz" quietly ./autogen.sh # TODO turn on `,integer` after this rebase - quietly ./configure --with-incompatible-bdb --enable-fuzz --with-sanitizers=address,fuzzer,undefined CC=clang CXX=clang++ + quietly ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined CC="ccache clang" CXX="ccache clang++" quietly make -j"$PARALLEL_BUILD" -k echo "Fuzzing" - quietly ./test/fuzz/test_runner.py -j"$PARALLEL_FUZZ" "${FUZZ_CORPUS}" + quietly ./test/fuzz/test_runner.py -j"$PARALLEL_FUZZ" "${FUZZ_CORPUS}" || notify "fail fuzz" 1 fi if [[ "$KEEP_GOING" == "0" ]]; then - exit 1 + notify "$PR_ID done, exiting" + exit 0 + else + echo "$PR_ID done, continuing" fi -# bummer1.sh SKIP_MERGE=0 + echo "end" >> merge.log done diff --git a/doc/github-merge-script.md b/doc/github-merge-script.md index 87dba3abfe..f83f2f5f5d 100644 --- a/doc/github-merge-script.md +++ b/doc/github-merge-script.md @@ -24,8 +24,8 @@ github-merge.py For example for PR 1518 I see this output: ``` -ElementsProject/elements#1518 Avoid Simplicity header dependency propogation into master -* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propogation (Russell O'Connor) (upstream/simplicity, pull/1518/head) +ElementsProject/elements#1518 Avoid Simplicity header dependency propagation into master +* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propagation (Russell O'Connor) (upstream/simplicity, pull/1518/head) Dropping you on a shell so you can try building/testing the merged source. Run 'git diff HEAD~' to show the changes being merged. @@ -47,10 +47,10 @@ exit In our example this results in the following output: ``` -[pull/1518/local-merge 5e1a950e52] Merge ElementsProject/elements#1518: Avoid Simplicity header dependency propogation +[pull/1518/local-merge 5e1a950e52] Merge ElementsProject/elements#1518: Avoid Simplicity header dependency propagation Date: Thu Jan 22 14:49:26 2026 +0200 -ElementsProject/elements#1518 Avoid Simplicity header dependency propogation into master -* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propogation (Russell O'Connor) (upstream/simplicity, pull/1518/head) +ElementsProject/elements#1518 Avoid Simplicity header dependency propagation into master +* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propagation (Russell O'Connor) (upstream/simplicity, pull/1518/head) ACKs: * ACK 6c7788a; built and tested locally (delta1) * ACK 6c7788adf373cd8f0dc5c4fe2b7674625631721e; successfully ran local tests (apoelstra)