From 8a5ae800cd96ac46dd5218072c2cd1f76a4c0593 Mon Sep 17 00:00:00 2001 From: parithosh Date: Tue, 4 Nov 2025 14:34:54 +0100 Subject: [PATCH 1/2] update release tester --- .github/workflows/run-release-scheduled.yml | 177 ++++++++++++++++++++ .gitignore | 6 +- clients/latest-releases.template.yaml | 29 ++++ scripts/fetch-latest-releases.sh | 119 +++++++++++++ tests.yaml | 42 +++++ 5 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/run-release-scheduled.yml create mode 100644 clients/latest-releases.template.yaml create mode 100755 scripts/fetch-latest-releases.sh diff --git a/.github/workflows/run-release-scheduled.yml b/.github/workflows/run-release-scheduled.yml new file mode 100644 index 0000000..be29404 --- /dev/null +++ b/.github/workflows/run-release-scheduled.yml @@ -0,0 +1,177 @@ +name: Run weekly release test + +on: + workflow_dispatch: + inputs: + overrideClientPairs: + description: 'Override client pairs to run this test with (eg. lighthouse-geth,teku-besu)' + default: "" + type: string + overrideKurtosisConfig: + description: 'Override kurtosis config to run this test with (eg. kurtosis-config/default.yaml)' + default: "" + type: string + overrideKurtosisBranch: + description: 'Override kurtosis ethereum package branch to run this test with (eg. main)' + default: "" + type: string + overrideAssertoorImage: + description: 'Override assertoor image to use (eg. ethpandaops/assertoor:latest)' + default: "ethpandaops/assertoor:master" + type: string + sendNotification: + description: 'Send Discord notification on test failure' + default: "false" + type: string + schedule: + - cron: '0 0 * * 0' # Run every Sunday at midnight UTC + +concurrency: + group: "release-test" + cancel-in-progress: false + +jobs: + generate_client_config: + name: "Generate Latest Releases Config" + runs-on: ubuntu-latest + outputs: + config_generated: ${{ steps.generate.outputs.config_generated }} + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Fetch Latest Release Versions + id: fetch_versions + run: | + chmod +x scripts/fetch-latest-releases.sh + ./scripts/fetch-latest-releases.sh + + - name: Generate clients/latest-releases.yaml from template + id: generate + run: | + echo "Substituting versions into template..." + envsubst < clients/latest-releases.template.yaml > clients/latest-releases.yaml + + echo "Generated clients/latest-releases.yaml:" + cat clients/latest-releases.yaml + + echo "config_generated=true" >> $GITHUB_OUTPUT + + - name: Upload generated config + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: latest-releases-config + path: clients/latest-releases.yaml + retention-days: 7 + + get_tests: + name: "Load Tests" + needs: generate_client_config + runs-on: ubuntu-latest + outputs: + test_configs: ${{ steps.tests.outputs.test_configs }} + kurtosis_versions: ${{ steps.tests.outputs.kurtosis_versions }} + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Download generated config + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: latest-releases-config + path: clients/ + + - name: "Load test configurations from tests.yaml" + id: tests + shell: bash + run: | + tests_file="tests.yaml" + override_pairs="${{ inputs.overrideClientPairs }}" + override_kurtosis_config="${{ inputs.overrideKurtosisConfig }}" + override_kurtosis_branch="${{ inputs.overrideKurtosisBranch }}" + + # Filter for tests that use weekly-latest-releases client config + test_configs="$(cat $tests_file | yq -o json | jq '.tests' | jq -c 'map(select(.id == "weekly-latest-releases"))')" + + # Apply overrides if provided + if ! [ -z "$override_pairs" ]; then + test_configs="$(echo "$test_configs" | jq -c "map(.clientPairs = [\"$override_pairs\"])")" + fi + + if ! [ -z "$override_kurtosis_config" ]; then + test_configs="$(echo "$test_configs" | jq -c "map(.kurtosis = \"$override_kurtosis_config\")")" + fi + + if ! [ -z "$override_kurtosis_branch" ]; then + test_configs="$(echo "$test_configs" | jq -c "map(.kurtosis_branch = \"$override_kurtosis_branch\")")" + fi + + kurtosis_versions="$(echo "$test_configs" | jq -c "[.[] | select(.backend == \"docker\") | .kurtosis_version // \"latest\"] | unique")" + if [ $(echo "$kurtosis_versions" | jq -c ".[] | select(. == \"latest\")" | wc -l) -gt 0 ]; then + # get latest kurtosis version + echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt-get update -o Dir::Etc::sourcelist="sources.list.d/kurtosis.list" -o Dir::Etc::sourceparts="-" -o APT::Get::List-Cleanup="0" + latest_kurtosis_version=$(apt show kurtosis-cli | grep Version | awk '{print $2}') + + test_configs=$(echo "$test_configs" | jq -c --arg latest_kurtosis_version "$latest_kurtosis_version" \ + 'map(if ((.kurtosis_version == "latest") or (.kurtosis_version == null)) and .backend == "docker" then .kurtosis_version = $latest_kurtosis_version else . end)') + kurtosis_versions=$(echo "$kurtosis_versions" | jq -c ". + [\"$latest_kurtosis_version\"] | [.[] | select(. != \"latest\")] | unique") + fi + + echo "test_configs<> $GITHUB_OUTPUT + echo "$test_configs" >> $GITHUB_OUTPUT + echo "$(echo "$test_configs" | jq)" + echo "EOF" >> $GITHUB_OUTPUT + + echo "kurtosis_versions<> $GITHUB_OUTPUT + echo "$kurtosis_versions" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + prepare_cache: + needs: get_tests + name: "Warmup docker cache" + runs-on: ubuntu-latest + if: ${{ needs.get_tests.outputs.kurtosis_versions != '[]' }} + strategy: + fail-fast: false + matrix: + version: ${{ fromJson(needs.get_tests.outputs.kurtosis_versions) }} + steps: + - name: Warmup kurtosis docker cache (${{ matrix.version }}) + continue-on-error: true + id: cache + uses: ethpandaops/kurtosis-cache-github-action@v1 # v1 + with: + kurtosis_version: ${{ matrix.version }} + warmup_cache: true + warmup_only: true + cache_prefix: "kurtosis-docker" + s3_access_key: ${{ secrets.S3CACHE_ACCESS_KEY }} + s3_secret_key: ${{ secrets.S3CACHE_ACCESS_SECRET }} + s3_bucket: ${{ vars.S3CACHE_BUCKET }} + s3_endpoint: ${{ vars.S3CACHE_ENDPOINT }} + + run_tests: + needs: [get_tests, prepare_cache] + if: ${{ !cancelled() && needs.get_tests.outputs.test_configs != '[]' }} + uses: ./.github/workflows/_shared-run.yaml + name: "${{ matrix.config.name }}" + strategy: + fail-fast: false + matrix: + config: ${{ fromJson(needs.get_tests.outputs.test_configs) }} + with: + config: ${{ toJSON(matrix.config) }} + send_notification: ${{ inputs.sendNotification || 'true' }} + use_chatgpt: '{"url": "${{ vars.CHATGPT_URL }}", "model": "${{ vars.CHATGPT_MODEL }}", "extra_cfg": ${{ vars.CHATGPT_EXTRA_CFG }}}' + s3_bucket: ${{ vars.S3CACHE_BUCKET }} + s3_endpoint: ${{ vars.S3CACHE_ENDPOINT }} + assertoor_image: ${{ inputs.overrideAssertoorImage || 'ethpandaops/assertoor:master' }} + secrets: + RANCHER_URL: ${{ secrets.RANCHER_URL }} + RANCHER_TOKEN: ${{ secrets.RANCHER_TOKEN }} + DISCORD_HOOK: ${{ secrets.DISCORD_WEBHOOK }} + CHATGPT_KEY: ${{ secrets.CHATGPT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + S3_ACCESS_KEY: ${{ secrets.S3CACHE_ACCESS_KEY }} + S3_SECRET_KEY: ${{ secrets.S3CACHE_ACCESS_SECRET }} diff --git a/.gitignore b/.gitignore index cd78447..2d3fca5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -temp/ \ No newline at end of file +temp/ + +# Generated client configuration (dynamically created by workflow) +clients/latest-releases.yaml +CLAUDE.md \ No newline at end of file diff --git a/clients/latest-releases.template.yaml b/clients/latest-releases.template.yaml new file mode 100644 index 0000000..99bc332 --- /dev/null +++ b/clients/latest-releases.template.yaml @@ -0,0 +1,29 @@ + +consensus: # CL Clients + teku: + image: "consensys/teku:${TEKU_VERSION}" + prysm: + image: "offchainlabs/prysm-beacon-chain:${PRYSM_VERSION}" + vc_image: "offchainlabs/prysm-validator:${PRYSM_VERSION}" + lighthouse: + image: "sigp/lighthouse:${LIGHTHOUSE_VERSION}" + nimbus: + image: "statusim/nimbus-eth2:${NIMBUS_VERSION}" + vc_image: "statusim/nimbus-validator-client:${NIMBUS_VERSION}" + lodestar: + image: "chainsafe/lodestar:${LODESTAR_VERSION}" + grandine: + image: "sifrai/grandine:${GRANDINE_VERSION}" + +execution: # EL Clients + geth: + image: "ethereum/client-go:${GETH_VERSION}" + nethermind: + image: "nethermind/nethermind:${NETHERMIND_VERSION}" + erigon: + image: "erigontech/erigon:${ERIGON_VERSION}" + besu: + image: "hyperledger/besu:${BESU_VERSION}" + reth: + image: "ghcr.io/paradigmxyz/reth:${RETH_VERSION}" + diff --git a/scripts/fetch-latest-releases.sh b/scripts/fetch-latest-releases.sh new file mode 100755 index 0000000..7c3644b --- /dev/null +++ b/scripts/fetch-latest-releases.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script fetches the latest release versions for Ethereum clients +# and outputs them as GitHub Actions environment variables + +echo "Fetching latest release versions for Ethereum clients..." + +# Function to get latest GitHub release tag (excluding pre-releases) +get_latest_github_release() { + local repo=$1 + local tag + + # Get latest release (not pre-release) + tag=$(gh api "repos/${repo}/releases" --jq '[.[] | select(.prerelease == false)] | .[0].tag_name' 2>/dev/null || echo "") + + if [ -z "$tag" ]; then + echo "Warning: Could not fetch release for ${repo}, trying latest tag..." >&2 + tag=$(gh api "repos/${repo}/tags" --jq '.[0].name' 2>/dev/null || echo "latest") + fi + + echo "$tag" +} + +# Function to clean version tags (remove 'v' prefix if present) +clean_version() { + echo "$1" | sed 's/^v//' +} + +# Consensus Layer Clients +echo "Fetching consensus layer client versions..." + +LIGHTHOUSE_RAW=$(get_latest_github_release "sigp/lighthouse") +LIGHTHOUSE_VERSION=$(clean_version "$LIGHTHOUSE_RAW") +echo "Lighthouse: ${LIGHTHOUSE_VERSION}" + +PRYSM_RAW=$(get_latest_github_release "prysmaticlabs/prysm") +PRYSM_VERSION=$(clean_version "$PRYSM_RAW") +echo "Prysm: ${PRYSM_VERSION}" + +TEKU_RAW=$(get_latest_github_release "Consensys/teku") +TEKU_VERSION=$(clean_version "$TEKU_RAW") +echo "Teku: ${TEKU_VERSION}" + +NIMBUS_RAW=$(get_latest_github_release "status-im/nimbus-eth2") +NIMBUS_VERSION=$(clean_version "$NIMBUS_RAW") +echo "Nimbus: ${NIMBUS_VERSION}" + +LODESTAR_RAW=$(get_latest_github_release "ChainSafe/lodestar") +LODESTAR_VERSION=$(clean_version "$LODESTAR_RAW") +echo "Lodestar: ${LODESTAR_VERSION}" + +# Grandine doesn't have public GitHub releases, using Docker Hub +# Query Docker Hub API for latest tag +GRANDINE_VERSION=$(curl -s "https://hub.docker.com/v2/repositories/sifrai/grandine/tags/?page_size=100" | \ + jq -r '.results | map(select(.name | test("^v?[0-9]+\\.[0-9]+\\.[0-9]+$"))) | .[0].name' || echo "stable") +echo "Grandine: ${GRANDINE_VERSION}" + +# Execution Layer Clients +echo "Fetching execution layer client versions..." + +GETH_RAW=$(get_latest_github_release "ethereum/go-ethereum") +GETH_VERSION=$(clean_version "$GETH_RAW") +echo "Geth: ${GETH_VERSION}" + +NETHERMIND_RAW=$(get_latest_github_release "NethermindEth/nethermind") +NETHERMIND_VERSION=$(clean_version "$NETHERMIND_RAW") +echo "Nethermind: ${NETHERMIND_VERSION}" + +BESU_RAW=$(get_latest_github_release "hyperledger/besu") +BESU_VERSION=$(clean_version "$BESU_RAW") +echo "Besu: ${BESU_VERSION}" + +RETH_RAW=$(get_latest_github_release "paradigmxyz/reth") +RETH_VERSION=$(clean_version "$RETH_RAW") +echo "Reth: ${RETH_VERSION}" + +ERIGON_RAW=$(get_latest_github_release "erigontech/erigon") +ERIGON_VERSION=$(clean_version "$ERIGON_RAW") +echo "Erigon: ${ERIGON_VERSION}" + +# Output for GitHub Actions +if [ -n "${GITHUB_ENV:-}" ]; then + echo "Writing versions to GITHUB_ENV..." + { + echo "LIGHTHOUSE_VERSION=${LIGHTHOUSE_VERSION}" + echo "PRYSM_VERSION=${PRYSM_VERSION}" + echo "TEKU_VERSION=${TEKU_VERSION}" + echo "NIMBUS_VERSION=${NIMBUS_VERSION}" + echo "LODESTAR_VERSION=${LODESTAR_VERSION}" + echo "GRANDINE_VERSION=${GRANDINE_VERSION}" + echo "GETH_VERSION=${GETH_VERSION}" + echo "NETHERMIND_VERSION=${NETHERMIND_VERSION}" + echo "BESU_VERSION=${BESU_VERSION}" + echo "RETH_VERSION=${RETH_VERSION}" + echo "ERIGON_VERSION=${ERIGON_VERSION}" + } >> "$GITHUB_ENV" +fi + +# Also output for GITHUB_OUTPUT if available (for use in subsequent steps) +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "Writing versions to GITHUB_OUTPUT..." + { + echo "lighthouse_version=${LIGHTHOUSE_VERSION}" + echo "prysm_version=${PRYSM_VERSION}" + echo "teku_version=${TEKU_VERSION}" + echo "nimbus_version=${NIMBUS_VERSION}" + echo "lodestar_version=${LODESTAR_VERSION}" + echo "grandine_version=${GRANDINE_VERSION}" + echo "geth_version=${GETH_VERSION}" + echo "nethermind_version=${NETHERMIND_VERSION}" + echo "besu_version=${BESU_VERSION}" + echo "reth_version=${RETH_VERSION}" + echo "erigon_version=${ERIGON_VERSION}" + } >> "$GITHUB_OUTPUT" +fi + +echo "" +echo "✅ Successfully fetched all client versions" diff --git a/tests.yaml b/tests.yaml index 3308b75..8e9a5b3 100644 --- a/tests.yaml +++ b/tests.yaml @@ -43,6 +43,48 @@ tests: useExistingValidators: true validatorStartIndex: 680 +# Weekly test with dynamically fetched latest releases +- id: "weekly-latest-releases" + schedule: false + name: "Test latest GitHub releases (weekly)" + clients: clients/latest-releases.yaml + kurtosis: kurtosis-config/default.yaml + kurtosis_version: latest + kurtosis_branch: "main" + worker: [self-hosted-ghr-size-l-x64] + backend: docker + clientPairs: + # CL: [lighthouse,prysm,teku,lodestar,nimbus,grandine] + # EL: [geth,besu,reth,erigon,nethermind] + - lighthouse-reth,lighthouse-besu,grandine-nethermind + - prysm-reth,lighthouse-erigon,teku-besu + - prysm-besu,nimbus-nethermind,lodestar-besu + - lodestar-geth,nimbus-besu,prysm-erigon + - lighthouse-geth,nimbus-reth,grandine-besu + - prysm-geth,prysm-nethermind,lodestar-reth + - lighthouse-nethermind,nimbus-erigon,teku-geth + - lodestar-erigon,teku-nethermind,grandine-reth + - grandine-geth,teku-erigon,lodestar-nethermind + - teku-reth,nimbus-geth,grandine-erigon + assertoorTests: + - assertoor-tests/stability-check.yaml + - assertoor-tests/block-proposal-check.yaml + - assertoor-tests/eoa-transactions-test.yaml + - assertoor-tests/all-opcodes-test.yaml + - assertoor-tests/validator-withdrawal-test.yaml + - assertoor-tests/validator-exit-test.yaml + #- assertoor-tests/validator-slashing-test.yaml + - assertoor-tests/dencun-opcodes-test.yaml + - assertoor-tests/blob-transactions-test.yaml + - file: https://raw.githubusercontent.com/ethpandaops/assertoor/master/playbooks/pectra-dev/eip7002-all.yaml + config: + useExistingValidators: true + validatorStartIndex: 650 + - file: https://raw.githubusercontent.com/ethpandaops/assertoor/master/playbooks/pectra-dev/eip7251-all.yaml + config: + useExistingValidators: true + validatorStartIndex: 680 + # Check validator client combinations - id: "vc-compatibility" schedule: true From 3e86a387533891a01d2045f0c4984adda6b6e7ce Mon Sep 17 00:00:00 2001 From: parithosh Date: Tue, 4 Nov 2025 14:40:06 +0100 Subject: [PATCH 2/2] include prereleases --- scripts/fetch-latest-releases.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/fetch-latest-releases.sh b/scripts/fetch-latest-releases.sh index 7c3644b..00fc59f 100755 --- a/scripts/fetch-latest-releases.sh +++ b/scripts/fetch-latest-releases.sh @@ -3,16 +3,18 @@ set -euo pipefail # This script fetches the latest release versions for Ethereum clients # and outputs them as GitHub Actions environment variables +# +# Logic: Gets the most recent release (including pre-releases) based on published date. echo "Fetching latest release versions for Ethereum clients..." -# Function to get latest GitHub release tag (excluding pre-releases) +# Function to get latest GitHub release tag (includes pre-releases) get_latest_github_release() { local repo=$1 local tag - # Get latest release (not pre-release) - tag=$(gh api "repos/${repo}/releases" --jq '[.[] | select(.prerelease == false)] | .[0].tag_name' 2>/dev/null || echo "") + # Get the latest release (including pre-releases) + tag=$(gh api "repos/${repo}/releases" --jq '.[0].tag_name' 2>/dev/null || echo "") if [ -z "$tag" ]; then echo "Warning: Could not fetch release for ${repo}, trying latest tag..." >&2