From a9ae28950b1c07651f2734a84b7cc38f75991962 Mon Sep 17 00:00:00 2001 From: Martin Kourim Date: Mon, 29 Dec 2025 14:37:32 +0100 Subject: [PATCH] refactor(ci): unify node build logic and optimize space - Remove `.github/nix_override_cardano_node.sh` and replace with `.github/source_cardano_node.sh` for consistent cardano-node binary sourcing and building. - Refactor `node_upgrade.sh` and `regression.sh` to use new node build functions, simplify environment variable handling, and optimize disk usage in CI. - Update dbsync build and path logic in `.github/source_dbsync.sh` for consistency and workspace cleanliness. - Add step to maximize build space in GitHub Actions workflows. - Switch from `venv` to `testenv` for test environments in flake.nix, and clarify shell environments for development and testing. - Update flake.lock to latest cardano-node revision. --- .github/nix_override_cardano_node.sh | 17 ---- .github/node_upgrade.sh | 102 ++++++++++----------- .github/node_upgrade_pytest.sh | 19 ---- .github/regression.sh | 65 +++++++------ .github/source_cardano_cli.sh | 61 +++++++----- .github/source_cardano_node.sh | 71 ++++++++++++++ .github/source_dbsync.sh | 46 +++++----- .github/workflows/regression_reusable.yaml | 2 + .github/workflows/upgrade_reusable.yaml | 2 + flake.lock | 18 ++-- flake.nix | 16 ++-- 11 files changed, 244 insertions(+), 175 deletions(-) delete mode 100644 .github/nix_override_cardano_node.sh create mode 100644 .github/source_cardano_node.sh diff --git a/.github/nix_override_cardano_node.sh b/.github/nix_override_cardano_node.sh deleted file mode 100644 index ca6b66113..000000000 --- a/.github/nix_override_cardano_node.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -node_override() { - local ref - # if argument is provided: - if [ -n "${1:-""}" ]; then - ref="/$1" - # else use specified branch and/or revision: - elif [ -n "${NODE_REV:-""}" ]; then - ref="/$NODE_REV" - else - # otherwise update to latest from default branch - ref="" - fi - - echo --override-input cardano-node "github:IntersectMBO/cardano-node$ref" -} diff --git a/.github/node_upgrade.sh b/.github/node_upgrade.sh index 6b2100aed..6d2fcc1a7 100755 --- a/.github/node_upgrade.sh +++ b/.github/node_upgrade.sh @@ -5,15 +5,20 @@ # BASE_TAR_URL - URL of a tarball with binaries for base revision # BASE_REVISION - revision of cardano-node to upgrade from (alternative to BASE_TAR_URL) # UPGRADE_REVISION - revision of cardano-node to upgrade to +# UPGRADE_CLI_REVISION - revision of cardano-cli to upgrade to (optional) set -Eeuo pipefail trap 'echo "Error at line $LINENO"' ERR -if [[ -z "${BASE_TAR_URL:-""}" && -z "${BASE_REVISION:-""}" ]]; then +if [[ -z "${BASE_TAR_URL:-}" && -z "${BASE_REVISION:-}" ]]; then echo "BASE_TAR_URL or BASE_REVISION must be set" exit 1 fi +if [ -n "${GITHUB_ACTIONS:-}" ]; then + CI_OPTIMIZE_SPACE="${CI_OPTIMIZE_SPACE:-"true"}" +fi + nix --version df -h . @@ -69,31 +74,52 @@ echo "::group::Nix env setup step1" printf "start: %(%H:%M:%S)T\n" -1 # shellcheck disable=SC1090,SC1091 -. .github/nix_override_cardano_node.sh +. .github/source_cardano_node.sh -# update cardano-node to specified revision +# Prepare cardano-node for the base revision. # If BASE_TAR_URL is set, instead of using nix, download and extract binaries for base revision -# from a published tarball to save disk space, and use the same nix env as will be used for -# UPGRADE_REVISION. We are running out of space on GitHub Actions runners. -if [[ -z "${BASE_TAR_URL:-""}" && -n "${BASE_REVISION:-""}" ]]; then - NODE_OVERRIDE=$(node_override "${BASE_REVISION}") -elif [ -n "${UPGRADE_REVISION:-""}" ]; then - NODE_OVERRIDE=$(node_override "$UPGRADE_REVISION") -else - NODE_OVERRIDE=$(node_override) +# from a published tarball. +PATH_PREPEND_BASE="" +if [ -z "${BASE_TAR_URL:-}" ]; then + if [ -z "${BASE_REVISION:-}" ]; then + echo "Either BASE_TAR_URL or BASE_REVISION must be set" + exit 1 + fi + cardano_bins_build_all "$BASE_REVISION" "" "_base" + PATH_PREPEND_BASE="$(cardano_bins_print_path_prepend "" "_base")" +fi +export PATH_PREPEND_BASE + +# Prepare cardano-node for the upgrade revision +cardano_bins_build_all "${UPGRADE_REVISION:-"master"}" "${UPGRADE_CLI_REVISION:-}" "_upgrade" +PATH_PREPEND_UPGRADE="$(cardano_bins_print_path_prepend "${UPGRADE_CLI_REVISION:-}" "_upgrade")" +export PATH_PREPEND_UPGRADE + +# Prepare cardano-cli for the upgrade revision if UPGRADE_CLI_REVISION is set +if [ -n "${UPGRADE_CLI_REVISION:-}" ]; then + export CARDANO_CLI_REV="$UPGRADE_CLI_REVISION" + # shellcheck disable=SC1090,SC1091 + . .github/source_cardano_cli.sh + cardano_cli_build "$UPGRADE_CLI_REVISION" "_upgrade" + PATH_PREPEND_UPGRADE="$(cardano_cli_print_path_prepend "_upgrade")${PATH_PREPEND_UPGRADE}" + export PATH_PREPEND_UPGRADE +fi + +# optimize nix store space if requested +if [ "${CI_OPTIMIZE_SPACE:-"false"}" != "false" ]; then + nix store gc || : fi -# shellcheck disable=SC2086 -nix flake update --accept-flake-config $NODE_OVERRIDE # shellcheck disable=SC2016 -nix develop --accept-flake-config .#venv --command bash -c ' +nix develop --accept-flake-config .#testenv --command bash -c ' set -euo pipefail - : > "$WORKDIR/.nix_step1" - echo "::endgroup::" # end group for "Nix env setup step1" + : > "$WORKDIR/.nix_setup" + echo "::endgroup::" # end group for "Nix env setup" echo "::group::Python venv setup step1" printf "start: %(%H:%M:%S)T\n" -1 . .github/setup_venv.sh clean + export PATH="${PATH_PREPEND_BASE}${PATH}" echo "::endgroup::" # end group for "Python venv setup step1" echo "::group::🧪 Testrun Step1" @@ -102,40 +128,14 @@ nix develop --accept-flake-config .#venv --command bash -c ' # prepare scripts for stating cluster instance, start cluster instance, run smoke tests retval=0 ./.github/node_upgrade_pytest.sh step1 || retval="$?" - exit "$retval" -' || retval="$?" - -if [ ! -e "$WORKDIR/.nix_step1" ]; then - echo "Nix env setup failed, exiting" - exit 1 -fi - -# retval 0 == all tests passed; 1 == some tests failed; > 1 == some runtime error and we don't want to continue -[ "$retval" -le 1 ] || exit "$retval" - -echo "::endgroup::" # end group for "Testrun Step1" - -echo "::group::Nix env setup steps 2 & 3" -printf "start: %(%H:%M:%S)T\n" -1 - -# update cardano-node to specified branch and/or revision, or to the latest available revision -if [ -n "${UPGRADE_REVISION:-""}" ]; then - NODE_OVERRIDE=$(node_override "$UPGRADE_REVISION") -else - NODE_OVERRIDE=$(node_override) -fi - -# shellcheck disable=SC2086 -nix flake update --accept-flake-config $NODE_OVERRIDE -# shellcheck disable=SC2016 -nix develop --accept-flake-config .#venv --command bash -c ' - set -euo pipefail - : > "$WORKDIR/.nix_step2" - echo "::endgroup::" # end group for "Nix env setup steps 2 & 3" + # retval 0 == all tests passed; 1 == some tests failed; > 1 == some runtime error and we do not want to continue + [ "$retval" -le 1 ] || exit "$retval" + echo "::endgroup::" # end group for "Testrun Step1" echo "::group::Python venv setup steps 2 & 3" printf "start: %(%H:%M:%S)T\n" -1 . .github/setup_venv.sh clean + export PATH="${PATH_PREPEND_UPGRADE}${PATH}" echo "::endgroup::" # end group for "Python venv setup steps 2 & 3" echo "::group::🧪 Testrun Step2" @@ -144,27 +144,25 @@ nix develop --accept-flake-config .#venv --command bash -c ' # update cluster nodes, run smoke tests retval=0 ./.github/node_upgrade_pytest.sh step2 || retval="$?" - # retval 0 == all tests passed; 1 == some tests failed; > 1 == some runtime error and we dont want to continue [ "$retval" -le 1 ] || exit "$retval" echo "::endgroup::" # end group for "Testrun Step2" echo "::group::🧪 Testrun Step3" printf "start: %(%H:%M:%S)T\n" -1 df -h . - # update to Conway, run smoke tests + # update the rest of cluster nodes, run smoke tests retval=0 ./.github/node_upgrade_pytest.sh step3 || retval="$?" - df -h . echo "::endgroup::" # end group for "Testrun Step3" echo "::group::Teardown cluster & collect artifacts" printf "start: %(%H:%M:%S)T\n" -1 - # teardown cluster + df -h . ./.github/node_upgrade_pytest.sh finish || : exit $retval ' || retval="$?" -if [ ! -e "$WORKDIR/.nix_step2" ]; then +if [ ! -e "$WORKDIR/.nix_setup" ]; then echo "Nix env setup failed, exiting" exit 1 fi @@ -175,7 +173,7 @@ fi _cleanup # prepare artifacts for upload in GitHub Actions -if [ -n "${GITHUB_ACTIONS:-""}" ]; then +if [ -n "${GITHUB_ACTIONS:-}" ]; then # save testing artifacts ./.github/save_artifacts.sh diff --git a/.github/node_upgrade_pytest.sh b/.github/node_upgrade_pytest.sh index b80be613d..06d64ea69 100755 --- a/.github/node_upgrade_pytest.sh +++ b/.github/node_upgrade_pytest.sh @@ -114,14 +114,6 @@ elif [ "$1" = "step2" ]; then NETWORK_MAGIC="$(jq '.networkMagic' "$STATE_CLUSTER/shelley/genesis.json")" export NETWORK_MAGIC - # Setup `cardano-cli` binary - if [ -n "${UPGRADE_CLI_REVISION:-""}" ]; then - export CARDANO_CLI_REV="$UPGRADE_CLI_REVISION" - # shellcheck disable=SC1090,SC1091 - . .github/source_cardano_cli.sh - export PATH="$WORKDIR/cardano-cli-build/bin":"$PATH" - fi - # add binaries saved in step1 to the PATH export PATH="${STEP1_BIN}:${PATH}" @@ -278,17 +270,6 @@ elif [ "$1" = "step3" ]; then NETWORK_MAGIC="$(jq '.networkMagic' "$STATE_CLUSTER/shelley/genesis.json")" export NETWORK_MAGIC - # Setup `cardano-cli` binary - if [ -n "${UPGRADE_CLI_REVISION:-""}" ]; then - export CARDANO_CLI_REV="$UPGRADE_CLI_REVISION" - # the cardano-cli binary is already built in step2 - if [ ! -e "$WORKDIR/cardano-cli-build/bin/cardano-cli" ]; then - echo "Failed to find the requested 'cardano-cli' binary" >&2 - exit 6 - fi - export PATH="$WORKDIR/cardano-cli-build/bin":"$PATH" - fi - # generate config and topology files for p2p mode CARDANO_NODE_SOCKET_PATH="$WORKDIR/dry_p2p/state-cluster0/bft1.socket" \ DRY_RUN=1 \ diff --git a/.github/regression.sh b/.github/regression.sh index e757cd1bd..ff96b6731 100755 --- a/.github/regression.sh +++ b/.github/regression.sh @@ -8,6 +8,10 @@ basic_err_string='echo "Error at line $LINENO"' # shellcheck disable=SC2064 trap "$basic_err_string" ERR +if [ -n "${GITHUB_ACTIONS:-}" ]; then + CI_OPTIMIZE_SPACE="${CI_OPTIMIZE_SPACE:-"true"}" +fi + nix --version df -h . @@ -34,9 +38,9 @@ mkdir -p "$WORKDIR" export TMPDIR="$WORKDIR/tmp" mkdir -p "$TMPDIR" -if [ "${CI_TOPOLOGY:-""}" = "legacy" ]; then +if [ "${CI_TOPOLOGY:-}" = "legacy" ]; then export ENABLE_LEGACY=1 -elif [ "${CI_TOPOLOGY:-""}" = "mixed" ]; then +elif [ "${CI_TOPOLOGY:-}" = "mixed" ]; then export MIXED_P2P=1 export NUM_POOLS="${NUM_POOLS:-4}" fi @@ -47,7 +51,7 @@ rm -rf "${ARTIFACTS_DIR:?}" export SCHEDULING_LOG=scheduling.log : > "$SCHEDULING_LOG" -MARKEXPR="${MARKEXPR:-""}" +MARKEXPR="${MARKEXPR:-}" if [ "$MARKEXPR" = "all" ]; then unset MARKEXPR elif [ "$MARKEXPR" = "conway only" ]; then @@ -58,7 +62,7 @@ elif [ "$MARKEXPR" = "dbsync config" ]; then export MARKEXPR="(dbsync and smoke) or dbsync_config" fi -if [ -n "${CLUSTERS_COUNT:-""}" ]; then +if [ -n "${CLUSTERS_COUNT:-}" ]; then export CLUSTERS_COUNT fi @@ -71,7 +75,7 @@ elif [ "$CLUSTER_ERA" = "conway 11" ]; then fi export CLUSTER_ERA -TX_ERA="${TX_ERA:-""}" +TX_ERA="${TX_ERA:-}" if [ "$TX_ERA" = "conway" ] || [ "$CLUSTER_ERA" = "conway" ]; then unset TX_ERA export COMMAND_ERA="conway" @@ -81,11 +85,11 @@ fi # Decrease the number of tests per cluster if we are using the "disk" (LMDB) UTxO backend to avoid # having too many concurrent readers. -if [ -z "${MAX_TESTS_PER_CLUSTER:-""}" ] && [[ "${UTXO_BACKEND:-""}" = "disk"* ]]; then +if [ -z "${MAX_TESTS_PER_CLUSTER:-}" ] && [[ "${UTXO_BACKEND:-}" = "disk"* ]]; then export MAX_TESTS_PER_CLUSTER=5 fi -if [ -n "${BOOTSTRAP_DIR:-""}" ]; then +if [ -n "${BOOTSTRAP_DIR:-}" ]; then : # don't touch `TESTNET_VARIANT` when running on testnet elif [ "${CI_BYRON_CLUSTER:-"false"}" != "false" ]; then export TESTNET_VARIANT="${TESTNET_VARIANT:-"${CLUSTER_ERA}_slow"}" @@ -96,7 +100,7 @@ fi export CARDANO_NODE_SOCKET_PATH_CI="$WORKDIR/state-cluster0/bft1.socket" # assume we run tests on testnet when `BOOTSTRAP_DIR` is set -if [ -n "${BOOTSTRAP_DIR:-""}" ]; then +if [ -n "${BOOTSTRAP_DIR:-}" ]; then export CARDANO_NODE_SOCKET_PATH_CI="$WORKDIR/state-cluster0/relay1.socket" export RUN_TARGET="${RUN_TARGET:-"testnets"}" fi @@ -104,7 +108,7 @@ fi echo "### Dependencies setup ###" # setup dbsync (disabled by default) -case "${DBSYNC_REV:-""}" in +case "${DBSYNC_REV:-}" in "" ) ;; "none" ) @@ -132,7 +136,7 @@ case "${PLUTUS_APPS_REV:="none"}" in esac # setup cardano-cli (use the built-in version by default) -case "${CARDANO_CLI_REV:-""}" in +case "${CARDANO_CLI_REV:-}" in "" ) ;; "none" ) @@ -141,9 +145,29 @@ case "${CARDANO_CLI_REV:-""}" in * ) # shellcheck disable=SC1090,SC1091 . .github/source_cardano_cli.sh + cardano_cli_build "$CARDANO_CLI_REV" + PATH_PREPEND="$(cardano_cli_print_path_prepend "")${PATH_PREPEND}" + export PATH_PREPEND ;; esac +# setup cardano-node binaries +case "${NODE_REV:-}" in + "" | "none" ) + NODE_REV=master + ;; +esac +# shellcheck disable=SC1090,SC1091 +. .github/source_cardano_node.sh +cardano_bins_build_all "$NODE_REV" "${CARDANO_CLI_REV:-}" +PATH_PREPEND="$(cardano_bins_print_path_prepend "${CARDANO_CLI_REV:-}")${PATH_PREPEND}" +export PATH_PREPEND + +# optimize nix store space if requested +if [ "${CI_OPTIMIZE_SPACE:-"false"}" != "false" ]; then + nix store gc || : +fi + echo "### Cleanup setup ###" _cleanup() { @@ -154,16 +178,11 @@ _cleanup() { if command -v stop_postgres >/dev/null 2>&1; then stop_postgres || : fi - - # cleanup dbsync repo if modified - if command -v cleanup_dbsync_repo >/dev/null 2>&1; then - cleanup_dbsync_repo || : - fi } # shellcheck disable=SC2329 _cleanup_testnet_on_interrupt() { - [ -z "${BOOTSTRAP_DIR:-""}" ] && return + [ -z "${BOOTSTRAP_DIR:-}" ] && return _PYTEST_CURRENT="$(find "$WORKDIR" -type l -name pytest-current)" [ -z "$_PYTEST_CURRENT" ] && return @@ -176,7 +195,7 @@ _cleanup_testnet_on_interrupt() { printf "start: %(%H:%M:%S)T\n" -1 # shellcheck disable=SC2016 - nix develop --accept-flake-config .#venv --command bash -c ' + nix develop --accept-flake-config .#testenv --command bash -c ' . .github/setup_venv.sh export PATH="$PATH_PREPEND":"$PATH" export CARDANO_NODE_SOCKET_PATH="$CARDANO_NODE_SOCKET_PATH_CI" @@ -211,10 +230,6 @@ echo "::endgroup::" # end group for "Script setup" echo "::group::Nix env setup" printf "start: %(%H:%M:%S)T\n" -1 -# function to update cardano-node to specified branch and/or revision, or to the latest available -# shellcheck disable=SC1090,SC1091 -. .github/nix_override_cardano_node.sh - if [ "$(echo "$PWD"/.bin/*)" != "${PWD}/.bin/*" ]; then echo echo "WARNING: using following binaries from ${PWD}/.bin:" @@ -252,10 +267,8 @@ trap "echo 'Stopping monitor'; kill ${MON_PID:-} 2>/dev/null || true" EXIT # Run tests and generate report -# shellcheck disable=SC2046,SC2119 -nix flake update --accept-flake-config $(node_override) # shellcheck disable=SC2016 -nix develop --accept-flake-config .#venv --command bash -c ' +nix develop --accept-flake-config .#testenv --command bash -c ' set -euo pipefail echo "::endgroup::" # end group for "Nix env setup" @@ -286,7 +299,7 @@ nix develop --accept-flake-config .#venv --command bash -c ' # Don't stop cluster instances just yet if KEEP_CLUSTERS_RUNNING is set to 1. # After any key is pressed, resume this script and stop all running cluster instances. -if [ "${KEEP_CLUSTERS_RUNNING:-""}" = 1 ]; then +if [ "${KEEP_CLUSTERS_RUNNING:-}" = 1 ]; then echo echo "KEEP_CLUSTERS_RUNNING is set, leaving clusters running until any key is pressed." echo "Press any key to continue..." @@ -302,7 +315,7 @@ _cleanup trap - SIGINT # prepare artifacts for upload in GitHub Actions -if [ -n "${GITHUB_ACTIONS:-""}" ]; then +if [ -n "${GITHUB_ACTIONS:-}" ]; then # move reports to root dir if [ -e .reports/testrun-report.html ]; then diff --git a/.github/source_cardano_cli.sh b/.github/source_cardano_cli.sh index b12fe2f56..34dbe997a 100644 --- a/.github/source_cardano_cli.sh +++ b/.github/source_cardano_cli.sh @@ -1,24 +1,41 @@ #!/bin/bash -_origpwd="$PWD" -cd "$WORKDIR" || exit 1 - -if [ -z "${CARDANO_CLI_REV:-""}" ]; then - echo "The value for CARDANO_CLI_REV cannot be empty" >&2 - exit 1 -fi - -# Build `cardano-cli` -nix build \ - --accept-flake-config \ - --no-write-lock-file \ - "github://github.com/IntersectMBO/cardano-cli?ref=${CARDANO_CLI_REV}#cardano-cli" \ - -o cardano-cli-build || exit 1 -[ -e cardano-cli-build/bin/cardano-cli ] || exit 1 - -# Add `cardano-cli` to PATH_PREPEND -PATH_PREPEND="${PATH_PREPEND:+"${PATH_PREPEND}:"}$(readlink -m cardano-cli-build/bin)" -export PATH_PREPEND - -cd "$_origpwd" || exit 1 -unset _origpwd +# Build cardano-cli from the standalone repo into $WORKDIR/cardano-cli-build +cardano_cli_build() { + local cli_rev="${1:?}" + local node_bindir_postfix="${2:-}" + local origpwd="$PWD" + + if [ -z "$cli_rev" ]; then + echo "The value for CARDANO_CLI_REV cannot be empty" >&2 + return 1 + fi + + cd "$WORKDIR" || return 1 + + local out="cardano-cli-build${node_bindir_postfix}" + nix build \ + --accept-flake-config \ + --no-write-lock-file \ + "github://github.com/IntersectMBO/cardano-cli?ref=${cli_rev}#cardano-cli" \ + -o "$out" || { cd "$origpwd" || true; return 1; } + + [ -e "${out}/bin/cardano-cli" ] || { cd "$origpwd" || true; return 1; } + + cd "$origpwd" || return 1 +} + +# Print PATH to prepend for the standalone cardano-cli build output. +cardano_cli_print_path_prepend() { + local node_bindir_postfix="${1:-}" + local origpwd="$PWD" + + cd "$WORKDIR" || return 1 + + local out="cardano-cli-build${node_bindir_postfix}" + local bin_dir + bin_dir="$(readlink -m "${out}/bin")" || { cd "$origpwd" || true; return 1; } + + cd "$origpwd" || return 1 + echo "${bin_dir:+"${bin_dir}:"}" +} diff --git a/.github/source_cardano_node.sh b/.github/source_cardano_node.sh new file mode 100644 index 000000000..dfe3ea504 --- /dev/null +++ b/.github/source_cardano_node.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Build all required binaries into $WORKDIR/*-build +cardano_bins_build_all() { + local node_rev="${1:?}" + local cli_rev="${2:-}" + local node_bindir_postfix="${3:-}" + local origpwd="$PWD" + + if [ -z "$node_rev" ]; then + echo "The value for NODE_REV cannot be empty" >&2 + return 1 + fi + + cd "$WORKDIR" || return 1 + + _cardano_bins_build_one() { + local flake_attr="$1" # e.g. cardano-node + local exe="$2" # e.g. cardano-node + local out="${flake_attr}-build${node_bindir_postfix}" + + nix build \ + --accept-flake-config \ + --no-write-lock-file \ + "github://github.com/IntersectMBO/cardano-node?ref=${node_rev}#${flake_attr}" \ + -o "$out" || return 1 + + [ -e "${out}/bin/${exe}" ] || return 1 + } + + _cardano_bins_build_one "cardano-node" "cardano-node" || { cd "$origpwd" || true; return 1; } + _cardano_bins_build_one "cardano-submit-api" "cardano-submit-api" || { cd "$origpwd" || true; return 1; } + _cardano_bins_build_one "bech32" "bech32" || { cd "$origpwd" || true; return 1; } + + if [ -z "$cli_rev" ]; then + _cardano_bins_build_one "cardano-cli" "cardano-cli" || { cd "$origpwd" || true; return 1; } + fi + + cd "$origpwd" || return 1 +} + +# Print PATH to prepend based on previously built outputs +cardano_bins_print_path_prepend() { + local cli_rev="${1:-}" + local node_bindir_postfix="${2:-}" + local origpwd="$PWD" + + cd "$WORKDIR" || return 1 + + local node_path_prepend="" + + _cardano_bins_add_bin_dir() { + local out_prefix="$1" + local out="${out_prefix}-build${node_bindir_postfix}" + local bin_dir + + bin_dir="$(readlink -m "${out}/bin")" || return 1 + node_path_prepend="${node_path_prepend:+"${node_path_prepend}:"}${bin_dir}" + } + + _cardano_bins_add_bin_dir "cardano-node" || { cd "$origpwd" || true; return 1; } + _cardano_bins_add_bin_dir "cardano-submit-api" || { cd "$origpwd" || true; return 1; } + _cardano_bins_add_bin_dir "bech32" || { cd "$origpwd" || true; return 1; } + + if [ -z "$cli_rev" ]; then + _cardano_bins_add_bin_dir "cardano-cli" || { cd "$origpwd" || true; return 1; } + fi + + cd "$origpwd" || return 1 + echo "${node_path_prepend:+"${node_path_prepend}:"}" +} diff --git a/.github/source_dbsync.sh b/.github/source_dbsync.sh index 96e2c6156..cdae664b0 100644 --- a/.github/source_dbsync.sh +++ b/.github/source_dbsync.sh @@ -45,7 +45,7 @@ file_is_available() { esac } -DBSYNC_TAR_URL="${DBSYNC_TAR_URL:-""}" +DBSYNC_TAR_URL="${DBSYNC_TAR_URL:-}" # Check if DBSYNC_TAR_URL is empty and DBSYNC_REV is a version number if [[ -z "$DBSYNC_TAR_URL" && "$DBSYNC_REV" =~ ^[0-9]+(\.[0-9]+)*$ ]]; then @@ -68,14 +68,13 @@ if [ -n "${DBSYNC_TAR_URL:-}" ]; then rm -f db-sync-node ln -s dbsync_download db-sync-node || exit 1 DBSYNC_SCHEMA_DIR="${WORKDIR}/db-sync-node/schema" - chmod -R u+w "$DBSYNC_SCHEMA_DIR" # Add write permissions export DBSYNC_SCHEMA_DIR rm -f smash-server || rm -f smash-server/bin/cardano-smash-server mkdir -p smash-server/bin ln -s "${WORKDIR}/dbsync_download/bin/cardano-smash-server" smash-server/bin/cardano-smash-server || exit 1 else # Build db-sync - case "${DBSYNC_REV:-""}" in + case "${DBSYNC_REV:-}" in "" ) echo "The value for DBSYNC_REV cannot be empty" >&2 exit 1 @@ -106,43 +105,44 @@ else git checkout "$DBSYNC_REV" git rev-parse HEAD - nix build --accept-flake-config .#cardano-db-sync -o db-sync-node \ - || nix build --accept-flake-config .#cardano-db-sync:exe:cardano-db-sync -o db-sync-node \ + # Build cardano-db-sync + nix build --accept-flake-config .#cardano-db-sync -o "${WORKDIR}/db-sync-node" \ + || nix build --accept-flake-config .#cardano-db-sync:exe:cardano-db-sync -o "${WORKDIR}/db-sync-node" \ || exit 1 + # Build cardano-smash-server if [ "${SMASH:-"false"}" != "false" ]; then - nix build --accept-flake-config .#cardano-smash-server -o smash-server || exit 1 + nix build --accept-flake-config .#cardano-smash-server -o "${WORKDIR}/smash-server" || exit 1 fi - DBSYNC_SCHEMA_DIR="$PWD/schema" + + mv "$PWD/schema" "${WORKDIR}/db-sync-schema" + DBSYNC_SCHEMA_DIR="${WORKDIR}/db-sync-schema" export DBSYNC_SCHEMA_DIR + + cd "$WORKDIR" || exit 1 + rm -rf cardano-db-sync # Save space by removing the source code fi -if [ ! -e db-sync-node/bin/cardano-db-sync ]; then +if [ ! -e "${WORKDIR}/db-sync-node/bin/cardano-db-sync" ]; then echo "The \`cardano-db-sync\` binary not found, line $LINENO in sourced db-sync setup" >&2 # assert exit 1 fi +if [ "${SMASH:-"false"}" != "false" ] && [ ! -e "${WORKDIR}/smash-server/bin/cardano-smash-server" ]; then + echo "The \`cardano-smash-server\` binary not found, line $LINENO in sourced db-sync setup" >&2 # assert + exit 1 +fi # Add `cardano-db-sync` and `cardano-smash-server` to PATH_PREPEND -PATH_PREPEND="${PATH_PREPEND:+"${PATH_PREPEND}:"}$(readlink -m db-sync-node/bin)" +PATH_PREPEND="${PATH_PREPEND:+"${PATH_PREPEND}:"}$(readlink -m "${WORKDIR}/db-sync-node/bin")" if [ -e smash-server/bin/cardano-smash-server ]; then - PATH_PREPEND="${PATH_PREPEND:+"${PATH_PREPEND}:"}$(readlink -m smash-server/bin)" + PATH_PREPEND="${PATH_PREPEND:+"${PATH_PREPEND}:"}$(readlink -m "${WORKDIR}/smash-server/bin")" fi export PATH_PREPEND -if [ -n "${DBSYNC_SKIP_INDEXES:-""}" ]; then - # Delete the indexes only after the binaries are ready, so the binaries can be retrieved from - # the nix binary cache if available. +# Remove migration files that create indexes +if [ -n "${DBSYNC_SKIP_INDEXES:-}" ]; then + chmod -R u+w "$DBSYNC_SCHEMA_DIR" rm -f "$DBSYNC_SCHEMA_DIR"/migration-4-000* - - if [ -z "$DBSYNC_TAR_URL" ]; then - cleanup_dbsync_repo() { - local _origpwd="$PWD" - cd "$DBSYNC_SCHEMA_DIR"/.. || exit 1 - # shellcheck disable=SC2015 - git stash && git stash drop || : - cd "$_origpwd" || exit 1 - } - fi fi cd "$REPODIR" || exit 1 diff --git a/.github/workflows/regression_reusable.yaml b/.github/workflows/regression_reusable.yaml index 201952056..4f21decb4 100644 --- a/.github/workflows/regression_reusable.yaml +++ b/.github/workflows/regression_reusable.yaml @@ -72,6 +72,8 @@ jobs: reusable_run: runs-on: ubuntu-latest steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v9 - name: Checkout code uses: actions/checkout@v6 - name: Install Nix diff --git a/.github/workflows/upgrade_reusable.yaml b/.github/workflows/upgrade_reusable.yaml index ea1f46277..4f841d449 100644 --- a/.github/workflows/upgrade_reusable.yaml +++ b/.github/workflows/upgrade_reusable.yaml @@ -23,6 +23,8 @@ jobs: reusable_run: runs-on: ubuntu-latest steps: + - name: Maximize build space + uses: ublue-os/remove-unwanted-software@v9 - name: Checkout code uses: actions/checkout@v6 - name: Install Nix diff --git a/flake.lock b/flake.lock index 8e98ffdc6..8731308dc 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "CHaP": { "flake": false, "locked": { - "lastModified": 1752755491, - "narHash": "sha256-LhTRY6kgvg5cGfoQ9FD2v15WucqO4C+VLMHa9JP/Zi4=", + "lastModified": 1763670101, + "narHash": "sha256-3S6OSnW0Nn+YBVmuV0XnYQRAuS3i0F9lRdH4KQiN1uI=", "owner": "intersectmbo", "repo": "cardano-haskell-packages", - "rev": "fe5f8c99284ca892efe46d91a9ccb00aa76f2525", + "rev": "d341a38325d5d65cde10fa92af125b226c51615f", "type": "github" }, "original": { @@ -147,11 +147,11 @@ "utils": "utils" }, "locked": { - "lastModified": 1752857436, - "narHash": "sha256-YAAwDfzMMTeEQa0zHin7yo2nMdxONJ983tJ3NrP7K6E=", + "lastModified": 1763736877, + "narHash": "sha256-c1a6DzDlm+wzwa85TWeOFrPEldsfjiZw7+DcMMW9nc4=", "owner": "IntersectMBO", "repo": "cardano-node", - "rev": "ca1ec278070baf4481564a6ba7b4a5b9e3d9f366", + "rev": "6c034ec038d8d276a3595e10e2d38643f09bd1f2", "type": "github" }, "original": { @@ -883,11 +883,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1754767907, - "narHash": "sha256-8OnUzRQZkqtUol9vuUuQC30hzpMreKptNyET2T9lB6g=", + "lastModified": 1766687554, + "narHash": "sha256-DegN7KD/EtFSKXf2jvqL6lvev6GlfAAatYBcRC8goEo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c5f08b62ed75415439d48152c2a784e36909b1bc", + "rev": "fd0ca39c92fdb4012ed8d60e1683c26fddadd136", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 97e9e8bf7..d29ee209a 100644 --- a/flake.nix +++ b/flake.nix @@ -28,21 +28,23 @@ postgres = pkgs.mkShell { nativeBuildInputs = with pkgs; [ glibcLocales postgresql lsof procps ]; }; - venv = pkgs.mkShell { + testenv = pkgs.mkShell { nativeBuildInputs = base.nativeBuildInputs ++ postgres.nativeBuildInputs ++ [ + pkgs.poetry + py3Full + py3Pkgs.virtualenv + ]; + }; + dev = pkgs.mkShell { + nativeBuildInputs = testenv.nativeBuildInputs ++ [ nodePkgs.cardano-cli nodePkgs.cardano-node nodePkgs.cardano-submit-api nodePkgs.bech32 pkgs.bashInteractive - pkgs.poetry - py3Full - py3Pkgs.virtualenv ]; }; - # Use 'venv' directly as 'default' and 'dev' - default = venv; - dev = venv; + default = dev; }; });