refactor(ci): unify node build logic and optimize space

.github/node_upgrade.sh

@@ -5,15 +5,20 @@
 # BASE_TAR_URL - URL of a tarball with binaries for base revision
 # BASE_REVISION - revision of cardano-node to upgrade from (alternative to BASE_TAR_URL)
 # UPGRADE_REVISION - revision of cardano-node to upgrade to
+# UPGRADE_CLI_REVISION - revision of cardano-cli to upgrade to (optional)
 
 set -Eeuo pipefail
 trap 'echo "Error at line $LINENO"' ERR
 
-if [[ -z "${BASE_TAR_URL:-""}" && -z "${BASE_REVISION:-""}" ]]; then
+if [[ -z "${BASE_TAR_URL:-}" && -z "${BASE_REVISION:-}" ]]; then
   echo "BASE_TAR_URL or BASE_REVISION must be set"
   exit 1
 fi
 
+if [ -n "${GITHUB_ACTIONS:-}" ]; then
+  CI_OPTIMIZE_SPACE="${CI_OPTIMIZE_SPACE:-"true"}"
+fi
+
 nix --version
 df -h .
 
@@ -69,31 +74,52 @@ echo "::group::Nix env setup step1"
 printf "start: %(%H:%M:%S)T\n" -1
 
 # shellcheck disable=SC1090,SC1091
-. .github/nix_override_cardano_node.sh
+. .github/source_cardano_node.sh
 
-# update cardano-node to specified revision
+# Prepare cardano-node for the base revision.
 # If BASE_TAR_URL is set, instead of using nix, download and extract binaries for base revision
-# from a published tarball to save disk space, and use the same nix env as will be used for
-# UPGRADE_REVISION. We are running out of space on GitHub Actions runners.
-if [[ -z "${BASE_TAR_URL:-""}" && -n "${BASE_REVISION:-""}" ]]; then
-  NODE_OVERRIDE=$(node_override "${BASE_REVISION}")
-elif [ -n "${UPGRADE_REVISION:-""}" ]; then
-  NODE_OVERRIDE=$(node_override "$UPGRADE_REVISION")
-else
-  NODE_OVERRIDE=$(node_override)
+# from a published tarball.
+PATH_PREPEND_BASE=""
+if [ -z "${BASE_TAR_URL:-}" ]; then
+  if [ -z "${BASE_REVISION:-}" ]; then
+    echo "Either BASE_TAR_URL or BASE_REVISION must be set"
+    exit 1
+  fi
+  cardano_bins_build_all "$BASE_REVISION" "" "_base"
+  PATH_PREPEND_BASE="$(cardano_bins_print_path_prepend "" "_base")"
+fi
+export PATH_PREPEND_BASE
+
+# Prepare cardano-node for the upgrade revision
+cardano_bins_build_all "${UPGRADE_REVISION:-"master"}" "${UPGRADE_CLI_REVISION:-}" "_upgrade"
+PATH_PREPEND_UPGRADE="$(cardano_bins_print_path_prepend "${UPGRADE_CLI_REVISION:-}" "_upgrade")"
+export PATH_PREPEND_UPGRADE
+
+# Prepare cardano-cli for the upgrade revision if UPGRADE_CLI_REVISION is set
+if [ -n "${UPGRADE_CLI_REVISION:-}" ]; then
+  export CARDANO_CLI_REV="$UPGRADE_CLI_REVISION"
+  # shellcheck disable=SC1090,SC1091
+  . .github/source_cardano_cli.sh
+  cardano_cli_build "$UPGRADE_CLI_REVISION" "_upgrade"
+  PATH_PREPEND_UPGRADE="$(cardano_cli_print_path_prepend "_upgrade")${PATH_PREPEND_UPGRADE}"
+  export PATH_PREPEND_UPGRADE
+fi
+
+# optimize nix store space if requested
+if [ "${CI_OPTIMIZE_SPACE:-"false"}" != "false" ]; then
+  nix store gc || :
 fi
 
-# shellcheck disable=SC2086
-nix flake update --accept-flake-config $NODE_OVERRIDE
 # shellcheck disable=SC2016
-nix develop --accept-flake-config .#venv --command bash -c '
+nix develop --accept-flake-config .#testenv --command bash -c '
   set -euo pipefail
-  : > "$WORKDIR/.nix_step1"
-  echo "::endgroup::"  # end group for "Nix env setup step1"
+  : > "$WORKDIR/.nix_setup"
+  echo "::endgroup::"  # end group for "Nix env setup"
 
   echo "::group::Python venv setup step1"
   printf "start: %(%H:%M:%S)T\n" -1
   . .github/setup_venv.sh clean
+  export PATH="${PATH_PREPEND_BASE}${PATH}"
   echo "::endgroup::"  # end group for "Python venv setup step1"
 
   echo "::group::🧪 Testrun Step1"
@@ -102,40 +128,14 @@ nix develop --accept-flake-config .#venv --command bash -c '
   # prepare scripts for stating cluster instance, start cluster instance, run smoke tests
   retval=0
   ./.github/node_upgrade_pytest.sh step1 || retval="$?"
-  exit "$retval"
-' || retval="$?"
-
-if [ ! -e "$WORKDIR/.nix_step1" ]; then
-  echo "Nix env setup failed, exiting"
-  exit 1
-fi
-
-# retval 0 == all tests passed; 1 == some tests failed; > 1 == some runtime error and we don't want to continue
-[ "$retval" -le 1 ] || exit "$retval"
-
-echo "::endgroup::"  # end group for "Testrun Step1"
-
-echo "::group::Nix env setup steps 2 & 3"
-printf "start: %(%H:%M:%S)T\n" -1
-
-# update cardano-node to specified branch and/or revision, or to the latest available revision
-if [ -n "${UPGRADE_REVISION:-""}" ]; then
-  NODE_OVERRIDE=$(node_override "$UPGRADE_REVISION")
-else
-  NODE_OVERRIDE=$(node_override)
-fi
-
-# shellcheck disable=SC2086
-nix flake update --accept-flake-config $NODE_OVERRIDE
-# shellcheck disable=SC2016
-nix develop --accept-flake-config .#venv --command bash -c '
-  set -euo pipefail
-  : > "$WORKDIR/.nix_step2"
-  echo "::endgroup::"  # end group for "Nix env setup steps 2 & 3"
+  # retval 0 == all tests passed; 1 == some tests failed; > 1 == some runtime error and we do not want to continue
+  [ "$retval" -le 1 ] || exit "$retval"
+  echo "::endgroup::"  # end group for "Testrun Step1"
 
   echo "::group::Python venv setup steps 2 & 3"
   printf "start: %(%H:%M:%S)T\n" -1
   . .github/setup_venv.sh clean
+  export PATH="${PATH_PREPEND_UPGRADE}${PATH}"
   echo "::endgroup::"  # end group for "Python venv setup steps 2 & 3"
 
   echo "::group::🧪 Testrun Step2"
@@ -144,27 +144,25 @@ nix develop --accept-flake-config .#venv --command bash -c '
   # update cluster nodes, run smoke tests
   retval=0
   ./.github/node_upgrade_pytest.sh step2 || retval="$?"
-  # retval 0 == all tests passed; 1 == some tests failed; > 1 == some runtime error and we dont want to continue
   [ "$retval" -le 1 ] || exit "$retval"
   echo "::endgroup::"  # end group for "Testrun Step2"
 
   echo "::group::🧪 Testrun Step3"
   printf "start: %(%H:%M:%S)T\n" -1
   df -h .
-  # update to Conway, run smoke tests
+  # update the rest of cluster nodes, run smoke tests
   retval=0
   ./.github/node_upgrade_pytest.sh step3 || retval="$?"
-  df -h .
   echo "::endgroup::"  # end group for "Testrun Step3"
 
   echo "::group::Teardown cluster & collect artifacts"
   printf "start: %(%H:%M:%S)T\n" -1
-  # teardown cluster
+  df -h .
   ./.github/node_upgrade_pytest.sh finish || :
   exit $retval
 ' || retval="$?"
 
-if [ ! -e "$WORKDIR/.nix_step2" ]; then
+if [ ! -e "$WORKDIR/.nix_setup" ]; then
   echo "Nix env setup failed, exiting"
   exit 1
 fi
@@ -175,7 +173,7 @@ fi
 _cleanup
 
 # prepare artifacts for upload in GitHub Actions
-if [ -n "${GITHUB_ACTIONS:-""}" ]; then
+if [ -n "${GITHUB_ACTIONS:-}" ]; then
   # save testing artifacts
   ./.github/save_artifacts.sh
 

.github/node_upgrade_pytest.sh

@@ -114,14 +114,6 @@ elif [ "$1" = "step2" ]; then
   NETWORK_MAGIC="$(jq '.networkMagic' "$STATE_CLUSTER/shelley/genesis.json")"
   export NETWORK_MAGIC
 
-  # Setup `cardano-cli` binary
-  if [ -n "${UPGRADE_CLI_REVISION:-""}" ]; then
-    export CARDANO_CLI_REV="$UPGRADE_CLI_REVISION"
-    # shellcheck disable=SC1090,SC1091
-    . .github/source_cardano_cli.sh
-    export PATH="$WORKDIR/cardano-cli-build/bin":"$PATH"
-  fi
-
   # add binaries saved in step1 to the PATH
   export PATH="${STEP1_BIN}:${PATH}"
 
@@ -278,17 +270,6 @@ elif [ "$1" = "step3" ]; then
   NETWORK_MAGIC="$(jq '.networkMagic' "$STATE_CLUSTER/shelley/genesis.json")"
   export NETWORK_MAGIC
 
-  # Setup `cardano-cli` binary
-  if [ -n "${UPGRADE_CLI_REVISION:-""}" ]; then
-    export CARDANO_CLI_REV="$UPGRADE_CLI_REVISION"
-    # the cardano-cli binary is already built in step2
-    if [ ! -e "$WORKDIR/cardano-cli-build/bin/cardano-cli" ]; then
-      echo "Failed to find the requested 'cardano-cli' binary" >&2
-      exit 6
-    fi
-    export PATH="$WORKDIR/cardano-cli-build/bin":"$PATH"
-  fi
-
   # generate config and topology files for p2p mode
   CARDANO_NODE_SOCKET_PATH="$WORKDIR/dry_p2p/state-cluster0/bft1.socket" \
     DRY_RUN=1 \