From 7b33a72adbd1d68740e9642b85fb7a8ce7e69a08 Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 15 Oct 2025 11:30:13 +0200 Subject: [PATCH 1/3] update eest playbooks and add eest glamsterdam playbook --- .../execution-spec-tests-dependencies.yaml | 19 +- .../dev/execution-spec-tests-execute.yaml | 402 +++++++++--------- .../execution-sec-tests-sequential.yaml | 109 +++++ 3 files changed, 318 insertions(+), 212 deletions(-) create mode 100644 playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml diff --git a/playbooks/dev/execution-spec-tests-dependencies.yaml b/playbooks/dev/execution-spec-tests-dependencies.yaml index 97238f8c..eeac414e 100644 --- a/playbooks/dev/execution-spec-tests-dependencies.yaml +++ b/playbooks/dev/execution-spec-tests-dependencies.yaml @@ -2,13 +2,12 @@ id: execution-spec-tests-dependencies name: "Setup dependencies for execution spec tests" timeout: 1h tasks: - - name: run_shell - title: Install dependencies - config: - shell: bash - command: | - sudo dpkg --add-architecture amd64 - sudo apt-get update - sudo apt-get install -y build-essential python3 python3-pip libc6:amd64 - sudo rm /usr/lib/python3*/EXTERNALLY-MANAGED 2>/dev/null || true - + - name: run_shell + title: Install dependencies + config: + shell: bash + command: | + sudo dpkg --add-architecture amd64 + sudo apt-get update + sudo apt-get install -y build-essential python3 python3-pip python3-dev libc6:amd64 autoconf automake libffi-dev libtool pkg-config + sudo rm /usr/lib/python3*/EXTERNALLY-MANAGED 2>/dev/null || true diff --git a/playbooks/dev/execution-spec-tests-execute.yaml b/playbooks/dev/execution-spec-tests-execute.yaml index aa8f0160..138af8c0 100644 --- a/playbooks/dev/execution-spec-tests-execute.yaml +++ b/playbooks/dev/execution-spec-tests-execute.yaml @@ -2,206 +2,204 @@ id: execution-spec-tests-execute name: "Run 'execute' on execution spec tests" timeout: 1h config: - gitRepo: https://github.com/ethereum/execution-spec-tests.git - gitBranch: main - testPath: "" - chainID: "0" - rpcEndpoint: http://127.0.0.1:8545 - seedPrivateKey: "" - seedAmount: "1" # (In Wei). Amount used to seed child accounts for test execution. Can also use "1 ether" or "10000 gwei" as input - extraFlags: "" - runSetup: true - runTests: true - specTestsPath: "" - cleanupTestsPath: true - solcVersion: "0.8.24" + gitRepo: https://github.com/ethereum/execution-spec-tests.git + gitBranch: main + testPath: "" + chainID: "0" + rpcEndpoint: http://127.0.0.1:8545 + seedPrivateKey: "" + seedAmount: "1" # (In Wei). Amount used to seed child accounts for test execution. Can also use "1 ether" or "10000 gwei" as input + extraFlags: "" + runSetup: true + runTests: true + specTestsPath: "" + cleanupTestsPath: true + solcVersion: "0.8.24" tasks: - - name: run_shell - title: "Clone and prepare execution spec tests: ${gitRepo}@${gitBranch}" - id: setup - if: runSetup - config: - shell: bash - shellArgs: [--login] - envVars: - GIT_REPO: gitRepo - GIT_BRANCH: gitBranch - SOLC_VERSION: "solcVersion" - SPEC_TESTS_PATH: specTestsPath - command: | - set -e - - # Convert env vars. They are passed as RAW JSON values - - GIT_REPO=$(echo $GIT_REPO | jq -r) - GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) - SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) - SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) - - # Check if pip (python package manager) is installed - if ! command -v pip &> /dev/null - then - echo "pip could not be found. Please install python3-pip" - exit 1 - fi - - # Create dir for temp files - if [ -z "$SPEC_TESTS_PATH" ]; then - tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX) - - echo "============================" - echo "Temp dir created: ${tmp_dir}" - echo "============================" - - echo "::set-var specTestsPath ${tmp_dir}" - fi - - echo "============================" - echo "Clone git repo ${GIT_REPO} @ ${GIT_BRANCH}" - echo "============================" - git clone ${GIT_REPO} --branch ${GIT_BRANCH} --single-branch $SPEC_TESTS_PATH - - cd $SPEC_TESTS_PATH - export HOME=$SPEC_TESTS_PATH - export PATH=$PATH:$HOME/.local/bin - - echo "============================" - echo "Installing dependencies" - echo "============================" - pip install uv - uv sync --all-extras - uv run solc-select use "${SOLC_VERSION}" --always-install - - - name: generate_child_wallet - id: testwallet - if: runTests - title: "Generate main wallet for tests" - config: - randomSeed: true - configVars: - privateKey: "seedPrivateKey" - prefundMinBalance: "seedAmount" - - - name: run_shell - title: "Execute tests: [${testPath}]" - id: execute - if: runTests - config: - shell: bash - shellArgs: [--login] - envVars: - GIT_REPO: gitRepo - GIT_BRANCH: gitBranch - TEST_PATH: testPath - CHAIN_ID: chainID - RPC_ENDPOINT: rpcEndpoint - PRIVATE_KEY: tasks.testwallet.outputs.childWallet.privkey - SEED_AMOUNT: seedAmount - EXTRA_FLAGS: extraFlags - SOLC_VERSION: "solcVersion" - SPEC_TESTS_PATH: specTestsPath - CLEANUP_TESTS_PATH: cleanupTestsPath - command: | - set -e - - # Convert env vars. They are passed as RAW JSON values - - GIT_REPO=$(echo $GIT_REPO | jq -r) - GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) - TEST_PATH=$(echo $TEST_PATH | jq -r) - CHAIN_ID=$(echo $CHAIN_ID | jq -r) - RPC_ENDPOINT=$(echo $RPC_ENDPOINT | jq -r) - PRIVATE_KEY=$(echo $PRIVATE_KEY | jq -r) - SEED_AMOUNT=$(echo $SEED_AMOUNT | jq -r) - EXTRA_FLAGS=$(echo $EXTRA_FLAGS | jq -r) - SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) - SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) - CLEANUP_TESTS_PATH=$(echo $CLEANUP_TESTS_PATH | jq -r) - - echo "RPC_ENDPOINT: ${RPC_ENDPOINT}" - echo "CHAIN_ID: ${CHAIN_ID}" - - # Validate some inputs - if [ -z "$TEST_PATH" ]; then - echo - exit "You need to provide a test path" - fi - if [ -z "$PRIVATE_KEY" ]; then - echo - exit "You need to provide a private key to fund the tests" - fi - - # Check if pip (python package manager) is installed - if ! command -v pip &> /dev/null - then - echo "pip could not be found. Please install python3-pip" - exit 1 - fi - - # Create dir for temp files - cd $SPEC_TESTS_PATH - export HOME=$SPEC_TESTS_PATH - export PATH=$PATH:$HOME/.local/bin - - if [ "$CLEANUP_TESTS_PATH" == "true" ]; then - function cleanup { - rv=$? - rm -rf "$SPEC_TESTS_PATH" - echo "tmpdir removed" - exit $rv - } - - trap cleanup EXIT # always remove tempdir on exit - fi - - source .venv/bin/activate - - echo "============================" - echo "Running test: ${TEST_PATH}" - echo "============================" - uv run execute remote "${TEST_PATH}" \ - --rpc-chain-id=${CHAIN_ID} \ - --rpc-endpoint=${RPC_ENDPOINT} \ - --rpc-seed-key=${PRIVATE_KEY} \ - --seed-account-sweep-amount=${SEED_AMOUNT} \ - --json-report \ - --json-report-file=report.json \ - --html=report.html \ - ${EXTRA_FLAGS[@]} || true - - echo "============================" - echo "Exporting reports" - echo "============================" - - cp report.json ${ASSERTOOR_RESULT_DIR}/report.json - cp report.html ${ASSERTOOR_RESULT_DIR}/report.html - mkdir -p ${ASSERTOOR_RESULT_DIR}/assets - curl -o ${ASSERTOOR_RESULT_DIR}/assets/style.css https://raw.githubusercontent.com/ethpandaops/assertoor/refs/heads/master/res/execution-spec-tests-styles.css - - REPORT_JSON=$(cat report.json | jq -c '[.tests[] | {"nodeid": .nodeid, "outcome": .outcome, "setupDuration": .setup.duration, "callDuration": .call.duration, "teardownDuration": .teardown.duration}]') - TOTAL_TESTS=$(echo "$REPORT_JSON" | jq '. | length') - PASSED_TESTS=$(echo "$REPORT_JSON" | jq '[.[] | select(.outcome == "passed")] | length') - echo "Total tests: $TOTAL_TESTS" - echo "Passed tests: $PASSED_TESTS" - - # Check if tests passed - echo "::set-output passedTests ${PASSED_TESTS}" - echo "::set-output totalTests ${TOTAL_TESTS}" - - exit 0 - - - name: run_shell - title: "Check test results (${{tasks.execute.outputs.passedTests}} / ${{tasks.execute.outputs.totalTests}} passed)" - id: check - if: runTests - config: - shell: bash - envVars: - passedTests: "tasks.execute.outputs.passedTests" - totalTests: "tasks.execute.outputs.totalTests" - command: | - set -e - if [ "${passedTests}" != "${totalTests}" ]; then - echo "Some tests failed, see report.html in the task above for details" - exit 1 - fi \ No newline at end of file + - name: run_shell + title: "Clone and prepare execution spec tests: ${gitRepo}@${gitBranch}" + id: setup + if: runSetup + config: + shell: bash + shellArgs: [--login] + envVars: + GIT_REPO: gitRepo + GIT_BRANCH: gitBranch + SOLC_VERSION: "solcVersion" + SPEC_TESTS_PATH: specTestsPath + command: | + set -e + + # Convert env vars. They are passed as RAW JSON values + + GIT_REPO=$(echo $GIT_REPO | jq -r) + GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) + SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) + SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) + + # Check if pip (python package manager) is installed + if ! command -v pip &> /dev/null + then + echo "pip could not be found. Please install python3-pip" + exit 1 + fi + + # Create dir for temp files + if [ -z "$SPEC_TESTS_PATH" ]; then + tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX) + + echo "============================" + echo "Temp dir created: ${tmp_dir}" + echo "============================" + + echo "::set-var specTestsPath ${tmp_dir}" + fi + + echo "============================" + echo "Clone git repo ${GIT_REPO} @ ${GIT_BRANCH}" + echo "============================" + git clone ${GIT_REPO} --depth 1 --branch ${GIT_BRANCH} --single-branch $SPEC_TESTS_PATH + + cd $SPEC_TESTS_PATH + export HOME=$SPEC_TESTS_PATH + export PATH=$PATH:$HOME/.local/bin + + echo "============================" + echo "Installing dependencies" + echo "============================" + pip install uv + uv sync --all-extras + - name: generate_child_wallet + id: testwallet + if: runTests + title: "Generate main wallet for tests" + config: + randomSeed: true + configVars: + privateKey: "seedPrivateKey" + prefundMinBalance: "seedAmount" + + - name: run_shell + title: "Execute tests: [${testPath}]" + id: execute + if: runTests + config: + shell: bash + shellArgs: [--login] + envVars: + GIT_REPO: gitRepo + GIT_BRANCH: gitBranch + TEST_PATH: testPath + CHAIN_ID: chainID + RPC_ENDPOINT: rpcEndpoint + PRIVATE_KEY: tasks.testwallet.outputs.childWallet.privkey + SEED_AMOUNT: seedAmount + EXTRA_FLAGS: extraFlags + SOLC_VERSION: "solcVersion" + SPEC_TESTS_PATH: specTestsPath + CLEANUP_TESTS_PATH: cleanupTestsPath + command: | + set -e + + # Convert env vars. They are passed as RAW JSON values + + GIT_REPO=$(echo $GIT_REPO | jq -r) + GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) + TEST_PATH=$(echo $TEST_PATH | jq -r) + CHAIN_ID=$(echo $CHAIN_ID | jq -r) + RPC_ENDPOINT=$(echo $RPC_ENDPOINT | jq -r) + PRIVATE_KEY=$(echo $PRIVATE_KEY | jq -r) + SEED_AMOUNT=$(echo $SEED_AMOUNT | jq -r) + EXTRA_FLAGS=$(echo $EXTRA_FLAGS | jq -r) + SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) + SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) + CLEANUP_TESTS_PATH=$(echo $CLEANUP_TESTS_PATH | jq -r) + + echo "RPC_ENDPOINT: ${RPC_ENDPOINT}" + echo "CHAIN_ID: ${CHAIN_ID}" + + # Validate some inputs + if [ -z "$TEST_PATH" ]; then + echo + exit "You need to provide a test path" + fi + if [ -z "$PRIVATE_KEY" ]; then + echo + exit "You need to provide a private key to fund the tests" + fi + + # Check if pip (python package manager) is installed + if ! command -v pip &> /dev/null + then + echo "pip could not be found. Please install python3-pip" + exit 1 + fi + + # Create dir for temp files + cd $SPEC_TESTS_PATH + export HOME=$SPEC_TESTS_PATH + export PATH=$PATH:$HOME/.local/bin + + if [ "$CLEANUP_TESTS_PATH" == "true" ]; then + function cleanup { + rv=$? + rm -rf "$SPEC_TESTS_PATH" + echo "tmpdir removed" + exit $rv + } + + trap cleanup EXIT # always remove tempdir on exit + fi + + source .venv/bin/activate + + echo "============================" + echo "Running test: ${TEST_PATH}" + echo "============================" + uv run execute remote "${TEST_PATH}" \ + --rpc-chain-id=${CHAIN_ID} \ + --rpc-endpoint=${RPC_ENDPOINT} \ + --rpc-seed-key=${PRIVATE_KEY} \ + --seed-account-sweep-amount=${SEED_AMOUNT} \ + --json-report \ + --json-report-file=report.json \ + --html=report.html \ + ${EXTRA_FLAGS[@]} || true + + echo "============================" + echo "Exporting reports" + echo "============================" + + cp report.json ${ASSERTOOR_RESULT_DIR}/report.json + cp report.html ${ASSERTOOR_RESULT_DIR}/report.html + mkdir -p ${ASSERTOOR_RESULT_DIR}/assets + curl -o ${ASSERTOOR_RESULT_DIR}/assets/style.css https://raw.githubusercontent.com/ethpandaops/assertoor/refs/heads/master/res/execution-spec-tests-styles.css + + REPORT_JSON=$(cat report.json | jq -c '[.tests[] | {"nodeid": .nodeid, "outcome": .outcome, "setupDuration": .setup.duration, "callDuration": .call.duration, "teardownDuration": .teardown.duration}]') + TOTAL_TESTS=$(echo "$REPORT_JSON" | jq '. | length') + PASSED_TESTS=$(echo "$REPORT_JSON" | jq '[.[] | select(.outcome == "passed")] | length') + echo "Total tests: $TOTAL_TESTS" + echo "Passed tests: $PASSED_TESTS" + + # Check if tests passed + echo "::set-output passedTests ${PASSED_TESTS}" + echo "::set-output totalTests ${TOTAL_TESTS}" + + exit 0 + + - name: run_shell + title: "Check test results (${{tasks.execute.outputs.passedTests}} / ${{tasks.execute.outputs.totalTests}} passed)" + id: check + if: runTests + config: + shell: bash + envVars: + passedTests: "tasks.execute.outputs.passedTests" + totalTests: "tasks.execute.outputs.totalTests" + command: | + set -e + if [ "${passedTests}" != "${totalTests}" ]; then + echo "Some tests failed, see report.html in the task above for details" + exit 1 + fi diff --git a/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml b/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml new file mode 100644 index 00000000..8b302c43 --- /dev/null +++ b/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml @@ -0,0 +1,109 @@ +id: glamsterdam-execution-spec-tests-sequential-glamsterdam +name: "Run execution spec tests sequentially" +timeout: 4h +config: + walletPrivkey: "" + walletSeed: "spectests" + specTestsBranch: "main" +tasks: + # setup dependencies + - name: run_external_tasks + title: "Setup dependencies for execution spec tests" + timeout: 30m + config: + testFile: /tests/execution-spec-tests-dependencies.yaml + + - name: run_shell + title: "Create temp dir for execution spec tests" + id: tempdir + config: + shell: bash + command: | + set -e + tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX) + + echo "============================" + echo "Temp dir created: ${tmp_dir}" + echo "============================" + + echo "::set-output path ${tmp_dir}" + + echo "$tmp_dir" > $ASSERTOOR_SUMMARY + + - name: run_external_tasks + title: "Clone and prepare execution spec tests" + timeout: 10m + config: + testFile: /tests/execution-spec-tests-execute.yaml + testConfig: + gitRepo: https://github.com/ethereum/execution-spec-tests.git + runTests: false + testConfigVars: + specTestsPath: "tasks.tempdir.outputs.path" + gitBranch: "specTestsBranch" + + - name: check_clients_are_healthy + title: "Check if at least one client is ready" + id: clientCheck + timeout: 5m + config: + minClientCount: 1 + + - name: generate_child_wallet + id: specTestsWallet + title: "Generate main wallet for tests" + config: + prefundMinBalance: 1000000000000000000000000 # ensure a balance of 10000 ETH + configVars: + privateKey: "walletPrivkey" + walletSeed: "walletSeed" + + # wait for electra activation + - name: get_consensus_specs + id: consensusSpecs + title: "Get consensus chain specs" + - name: check_consensus_slot_range + title: "Wait for electra activation" + timeout: 1h + configVars: + minEpochNumber: "tasks.consensusSpecs.outputs.specs.GLOAS_FORK_EPOCH" + + # run execution spec tests + - name: run_tasks + title: "Run execution spec tests" + id: tests + config: + continueOnFailure: true + tasks: + - name: run_external_tasks + title: "Run execution spec tests: tests/amsterdam/eip7928_block_level_access_lists" + timeout: 2h + config: + testFile: /tests/execution-spec-tests-execute.yaml + testConfig: + gitRepo: https://github.com/ethereum/execution-spec-tests.git + testPath: ./tests/amsterdam/eip7928_block_level_access_lists + seedAmount: "10000000000000000000000" # 100 ETH + extraFlags: "--fork=Amsterdam -n 16" + runSetup: false + cleanupTestsPath: false + testConfigVars: + rpcEndpoint: "tasks.clientCheck.outputs.goodClients[0].elRpcUrl" + chainID: "tasks.consensusSpecs.outputs.specs.DEPOSIT_CHAIN_ID" + seedPrivateKey: "tasks.specTestsWallet.outputs.childWallet.privkey" + specTestsPath: "tasks.tempdir.outputs.path" + gitBranch: "specTestsBranch" +cleanupTasks: + - name: run_shell + title: "Cleanup temp dir for execution spec tests" + config: + shell: bash + envVars: + SPEC_TESTS_PATH: specTestsPath + command: | + set -e + SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) + + if [ ! -z "$SPEC_TESTS_PATH" ]; then + rm -rf $SPEC_TESTS_PATH + fi From 346081a1a04c1dccbb5f20264bc5b4069f37048f Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 15 Oct 2025 11:59:05 +0200 Subject: [PATCH 2/3] update tabsize --- .../execution-spec-tests-dependencies.yaml | 18 +- .../dev/execution-spec-tests-execute.yaml | 400 +++++++++--------- .../execution-sec-tests-sequential.yaml | 186 ++++---- 3 files changed, 302 insertions(+), 302 deletions(-) diff --git a/playbooks/dev/execution-spec-tests-dependencies.yaml b/playbooks/dev/execution-spec-tests-dependencies.yaml index eeac414e..9256b567 100644 --- a/playbooks/dev/execution-spec-tests-dependencies.yaml +++ b/playbooks/dev/execution-spec-tests-dependencies.yaml @@ -2,12 +2,12 @@ id: execution-spec-tests-dependencies name: "Setup dependencies for execution spec tests" timeout: 1h tasks: - - name: run_shell - title: Install dependencies - config: - shell: bash - command: | - sudo dpkg --add-architecture amd64 - sudo apt-get update - sudo apt-get install -y build-essential python3 python3-pip python3-dev libc6:amd64 autoconf automake libffi-dev libtool pkg-config - sudo rm /usr/lib/python3*/EXTERNALLY-MANAGED 2>/dev/null || true + - name: run_shell + title: Install dependencies + config: + shell: bash + command: | + sudo dpkg --add-architecture amd64 + sudo apt-get update + sudo apt-get install -y build-essential python3 python3-pip python3-dev libc6:amd64 autoconf automake libffi-dev libtool pkg-config + sudo rm /usr/lib/python3*/EXTERNALLY-MANAGED 2>/dev/null || true diff --git a/playbooks/dev/execution-spec-tests-execute.yaml b/playbooks/dev/execution-spec-tests-execute.yaml index 138af8c0..097626b6 100644 --- a/playbooks/dev/execution-spec-tests-execute.yaml +++ b/playbooks/dev/execution-spec-tests-execute.yaml @@ -2,204 +2,204 @@ id: execution-spec-tests-execute name: "Run 'execute' on execution spec tests" timeout: 1h config: - gitRepo: https://github.com/ethereum/execution-spec-tests.git - gitBranch: main - testPath: "" - chainID: "0" - rpcEndpoint: http://127.0.0.1:8545 - seedPrivateKey: "" - seedAmount: "1" # (In Wei). Amount used to seed child accounts for test execution. Can also use "1 ether" or "10000 gwei" as input - extraFlags: "" - runSetup: true - runTests: true - specTestsPath: "" - cleanupTestsPath: true - solcVersion: "0.8.24" + gitRepo: https://github.com/ethereum/execution-spec-tests.git + gitBranch: main + testPath: "" + chainID: "0" + rpcEndpoint: http://127.0.0.1:8545 + seedPrivateKey: "" + seedAmount: "1" # (In Wei). Amount used to seed child accounts for test execution. Can also use "1 ether" or "10000 gwei" as input + extraFlags: "" + runSetup: true + runTests: true + specTestsPath: "" + cleanupTestsPath: true + solcVersion: "0.8.24" tasks: - - name: run_shell - title: "Clone and prepare execution spec tests: ${gitRepo}@${gitBranch}" - id: setup - if: runSetup - config: - shell: bash - shellArgs: [--login] - envVars: - GIT_REPO: gitRepo - GIT_BRANCH: gitBranch - SOLC_VERSION: "solcVersion" - SPEC_TESTS_PATH: specTestsPath - command: | - set -e - - # Convert env vars. They are passed as RAW JSON values - - GIT_REPO=$(echo $GIT_REPO | jq -r) - GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) - SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) - SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) - - # Check if pip (python package manager) is installed - if ! command -v pip &> /dev/null - then - echo "pip could not be found. Please install python3-pip" - exit 1 - fi - - # Create dir for temp files - if [ -z "$SPEC_TESTS_PATH" ]; then - tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX) - - echo "============================" - echo "Temp dir created: ${tmp_dir}" - echo "============================" - - echo "::set-var specTestsPath ${tmp_dir}" - fi - - echo "============================" - echo "Clone git repo ${GIT_REPO} @ ${GIT_BRANCH}" - echo "============================" - git clone ${GIT_REPO} --depth 1 --branch ${GIT_BRANCH} --single-branch $SPEC_TESTS_PATH - - cd $SPEC_TESTS_PATH - export HOME=$SPEC_TESTS_PATH - export PATH=$PATH:$HOME/.local/bin - - echo "============================" - echo "Installing dependencies" - echo "============================" - pip install uv - uv sync --all-extras - - name: generate_child_wallet - id: testwallet - if: runTests - title: "Generate main wallet for tests" - config: - randomSeed: true - configVars: - privateKey: "seedPrivateKey" - prefundMinBalance: "seedAmount" - - - name: run_shell - title: "Execute tests: [${testPath}]" - id: execute - if: runTests - config: - shell: bash - shellArgs: [--login] - envVars: - GIT_REPO: gitRepo - GIT_BRANCH: gitBranch - TEST_PATH: testPath - CHAIN_ID: chainID - RPC_ENDPOINT: rpcEndpoint - PRIVATE_KEY: tasks.testwallet.outputs.childWallet.privkey - SEED_AMOUNT: seedAmount - EXTRA_FLAGS: extraFlags - SOLC_VERSION: "solcVersion" - SPEC_TESTS_PATH: specTestsPath - CLEANUP_TESTS_PATH: cleanupTestsPath - command: | - set -e - - # Convert env vars. They are passed as RAW JSON values - - GIT_REPO=$(echo $GIT_REPO | jq -r) - GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) - TEST_PATH=$(echo $TEST_PATH | jq -r) - CHAIN_ID=$(echo $CHAIN_ID | jq -r) - RPC_ENDPOINT=$(echo $RPC_ENDPOINT | jq -r) - PRIVATE_KEY=$(echo $PRIVATE_KEY | jq -r) - SEED_AMOUNT=$(echo $SEED_AMOUNT | jq -r) - EXTRA_FLAGS=$(echo $EXTRA_FLAGS | jq -r) - SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) - SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) - CLEANUP_TESTS_PATH=$(echo $CLEANUP_TESTS_PATH | jq -r) - - echo "RPC_ENDPOINT: ${RPC_ENDPOINT}" - echo "CHAIN_ID: ${CHAIN_ID}" - - # Validate some inputs - if [ -z "$TEST_PATH" ]; then - echo - exit "You need to provide a test path" - fi - if [ -z "$PRIVATE_KEY" ]; then - echo - exit "You need to provide a private key to fund the tests" - fi - - # Check if pip (python package manager) is installed - if ! command -v pip &> /dev/null - then - echo "pip could not be found. Please install python3-pip" - exit 1 - fi - - # Create dir for temp files - cd $SPEC_TESTS_PATH - export HOME=$SPEC_TESTS_PATH - export PATH=$PATH:$HOME/.local/bin - - if [ "$CLEANUP_TESTS_PATH" == "true" ]; then - function cleanup { - rv=$? - rm -rf "$SPEC_TESTS_PATH" - echo "tmpdir removed" - exit $rv - } - - trap cleanup EXIT # always remove tempdir on exit - fi - - source .venv/bin/activate - - echo "============================" - echo "Running test: ${TEST_PATH}" - echo "============================" - uv run execute remote "${TEST_PATH}" \ - --rpc-chain-id=${CHAIN_ID} \ - --rpc-endpoint=${RPC_ENDPOINT} \ - --rpc-seed-key=${PRIVATE_KEY} \ - --seed-account-sweep-amount=${SEED_AMOUNT} \ - --json-report \ - --json-report-file=report.json \ - --html=report.html \ - ${EXTRA_FLAGS[@]} || true - - echo "============================" - echo "Exporting reports" - echo "============================" - - cp report.json ${ASSERTOOR_RESULT_DIR}/report.json - cp report.html ${ASSERTOOR_RESULT_DIR}/report.html - mkdir -p ${ASSERTOOR_RESULT_DIR}/assets - curl -o ${ASSERTOOR_RESULT_DIR}/assets/style.css https://raw.githubusercontent.com/ethpandaops/assertoor/refs/heads/master/res/execution-spec-tests-styles.css - - REPORT_JSON=$(cat report.json | jq -c '[.tests[] | {"nodeid": .nodeid, "outcome": .outcome, "setupDuration": .setup.duration, "callDuration": .call.duration, "teardownDuration": .teardown.duration}]') - TOTAL_TESTS=$(echo "$REPORT_JSON" | jq '. | length') - PASSED_TESTS=$(echo "$REPORT_JSON" | jq '[.[] | select(.outcome == "passed")] | length') - echo "Total tests: $TOTAL_TESTS" - echo "Passed tests: $PASSED_TESTS" - - # Check if tests passed - echo "::set-output passedTests ${PASSED_TESTS}" - echo "::set-output totalTests ${TOTAL_TESTS}" - - exit 0 - - - name: run_shell - title: "Check test results (${{tasks.execute.outputs.passedTests}} / ${{tasks.execute.outputs.totalTests}} passed)" - id: check - if: runTests - config: - shell: bash - envVars: - passedTests: "tasks.execute.outputs.passedTests" - totalTests: "tasks.execute.outputs.totalTests" - command: | - set -e - if [ "${passedTests}" != "${totalTests}" ]; then - echo "Some tests failed, see report.html in the task above for details" - exit 1 - fi + - name: run_shell + title: "Clone and prepare execution spec tests: ${gitRepo}@${gitBranch}" + id: setup + if: runSetup + config: + shell: bash + shellArgs: [--login] + envVars: + GIT_REPO: gitRepo + GIT_BRANCH: gitBranch + SOLC_VERSION: "solcVersion" + SPEC_TESTS_PATH: specTestsPath + command: | + set -e + + # Convert env vars. They are passed as RAW JSON values + + GIT_REPO=$(echo $GIT_REPO | jq -r) + GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) + SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) + SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) + + # Check if pip (python package manager) is installed + if ! command -v pip &> /dev/null + then + echo "pip could not be found. Please install python3-pip" + exit 1 + fi + + # Create dir for temp files + if [ -z "$SPEC_TESTS_PATH" ]; then + tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX) + + echo "============================" + echo "Temp dir created: ${tmp_dir}" + echo "============================" + + echo "::set-var specTestsPath ${tmp_dir}" + fi + + echo "============================" + echo "Clone git repo ${GIT_REPO} @ ${GIT_BRANCH}" + echo "============================" + git clone ${GIT_REPO} --depth 1 --branch ${GIT_BRANCH} --single-branch $SPEC_TESTS_PATH + + cd $SPEC_TESTS_PATH + export HOME=$SPEC_TESTS_PATH + export PATH=$PATH:$HOME/.local/bin + + echo "============================" + echo "Installing dependencies" + echo "============================" + pip install uv + uv sync --all-extras + - name: generate_child_wallet + id: testwallet + if: runTests + title: "Generate main wallet for tests" + config: + randomSeed: true + configVars: + privateKey: "seedPrivateKey" + prefundMinBalance: "seedAmount" + + - name: run_shell + title: "Execute tests: [${testPath}]" + id: execute + if: runTests + config: + shell: bash + shellArgs: [--login] + envVars: + GIT_REPO: gitRepo + GIT_BRANCH: gitBranch + TEST_PATH: testPath + CHAIN_ID: chainID + RPC_ENDPOINT: rpcEndpoint + PRIVATE_KEY: tasks.testwallet.outputs.childWallet.privkey + SEED_AMOUNT: seedAmount + EXTRA_FLAGS: extraFlags + SOLC_VERSION: "solcVersion" + SPEC_TESTS_PATH: specTestsPath + CLEANUP_TESTS_PATH: cleanupTestsPath + command: | + set -e + + # Convert env vars. They are passed as RAW JSON values + + GIT_REPO=$(echo $GIT_REPO | jq -r) + GIT_BRANCH=$(echo $GIT_BRANCH | jq -r) + TEST_PATH=$(echo $TEST_PATH | jq -r) + CHAIN_ID=$(echo $CHAIN_ID | jq -r) + RPC_ENDPOINT=$(echo $RPC_ENDPOINT | jq -r) + PRIVATE_KEY=$(echo $PRIVATE_KEY | jq -r) + SEED_AMOUNT=$(echo $SEED_AMOUNT | jq -r) + EXTRA_FLAGS=$(echo $EXTRA_FLAGS | jq -r) + SOLC_VERSION=$(echo $SOLC_VERSION | jq -r) + SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) + CLEANUP_TESTS_PATH=$(echo $CLEANUP_TESTS_PATH | jq -r) + + echo "RPC_ENDPOINT: ${RPC_ENDPOINT}" + echo "CHAIN_ID: ${CHAIN_ID}" + + # Validate some inputs + if [ -z "$TEST_PATH" ]; then + echo + exit "You need to provide a test path" + fi + if [ -z "$PRIVATE_KEY" ]; then + echo + exit "You need to provide a private key to fund the tests" + fi + + # Check if pip (python package manager) is installed + if ! command -v pip &> /dev/null + then + echo "pip could not be found. Please install python3-pip" + exit 1 + fi + + # Create dir for temp files + cd $SPEC_TESTS_PATH + export HOME=$SPEC_TESTS_PATH + export PATH=$PATH:$HOME/.local/bin + + if [ "$CLEANUP_TESTS_PATH" == "true" ]; then + function cleanup { + rv=$? + rm -rf "$SPEC_TESTS_PATH" + echo "tmpdir removed" + exit $rv + } + + trap cleanup EXIT # always remove tempdir on exit + fi + + source .venv/bin/activate + + echo "============================" + echo "Running test: ${TEST_PATH}" + echo "============================" + uv run execute remote "${TEST_PATH}" \ + --rpc-chain-id=${CHAIN_ID} \ + --rpc-endpoint=${RPC_ENDPOINT} \ + --rpc-seed-key=${PRIVATE_KEY} \ + --seed-account-sweep-amount=${SEED_AMOUNT} \ + --json-report \ + --json-report-file=report.json \ + --html=report.html \ + ${EXTRA_FLAGS[@]} || true + + echo "============================" + echo "Exporting reports" + echo "============================" + + cp report.json ${ASSERTOOR_RESULT_DIR}/report.json + cp report.html ${ASSERTOOR_RESULT_DIR}/report.html + mkdir -p ${ASSERTOOR_RESULT_DIR}/assets + curl -o ${ASSERTOOR_RESULT_DIR}/assets/style.css https://raw.githubusercontent.com/ethpandaops/assertoor/refs/heads/master/res/execution-spec-tests-styles.css + + REPORT_JSON=$(cat report.json | jq -c '[.tests[] | {"nodeid": .nodeid, "outcome": .outcome, "setupDuration": .setup.duration, "callDuration": .call.duration, "teardownDuration": .teardown.duration}]') + TOTAL_TESTS=$(echo "$REPORT_JSON" | jq '. | length') + PASSED_TESTS=$(echo "$REPORT_JSON" | jq '[.[] | select(.outcome == "passed")] | length') + echo "Total tests: $TOTAL_TESTS" + echo "Passed tests: $PASSED_TESTS" + + # Check if tests passed + echo "::set-output passedTests ${PASSED_TESTS}" + echo "::set-output totalTests ${TOTAL_TESTS}" + + exit 0 + + - name: run_shell + title: "Check test results (${{tasks.execute.outputs.passedTests}} / ${{tasks.execute.outputs.totalTests}} passed)" + id: check + if: runTests + config: + shell: bash + envVars: + passedTests: "tasks.execute.outputs.passedTests" + totalTests: "tasks.execute.outputs.totalTests" + command: | + set -e + if [ "${passedTests}" != "${totalTests}" ]; then + echo "Some tests failed, see report.html in the task above for details" + exit 1 + fi diff --git a/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml b/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml index 8b302c43..6244be96 100644 --- a/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml +++ b/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml @@ -2,108 +2,108 @@ id: glamsterdam-execution-spec-tests-sequential-glamsterdam name: "Run execution spec tests sequentially" timeout: 4h config: - walletPrivkey: "" - walletSeed: "spectests" - specTestsBranch: "main" + walletPrivkey: "" + walletSeed: "spectests" + specTestsBranch: "main" tasks: - # setup dependencies - - name: run_external_tasks - title: "Setup dependencies for execution spec tests" - timeout: 30m - config: - testFile: /tests/execution-spec-tests-dependencies.yaml + # setup dependencies + - name: run_external_tasks + title: "Setup dependencies for execution spec tests" + timeout: 30m + config: + testFile: /tests/execution-spec-tests-dependencies.yaml - - name: run_shell - title: "Create temp dir for execution spec tests" - id: tempdir - config: - shell: bash - command: | - set -e - tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX) + - name: run_shell + title: "Create temp dir for execution spec tests" + id: tempdir + config: + shell: bash + command: | + set -e + tmp_dir=$(mktemp -d -t execution-spec-tests-XXXXXXXXXX) - echo "============================" - echo "Temp dir created: ${tmp_dir}" - echo "============================" + echo "============================" + echo "Temp dir created: ${tmp_dir}" + echo "============================" - echo "::set-output path ${tmp_dir}" + echo "::set-output path ${tmp_dir}" - echo "$tmp_dir" > $ASSERTOOR_SUMMARY + echo "$tmp_dir" > $ASSERTOOR_SUMMARY - - name: run_external_tasks - title: "Clone and prepare execution spec tests" - timeout: 10m - config: - testFile: /tests/execution-spec-tests-execute.yaml - testConfig: - gitRepo: https://github.com/ethereum/execution-spec-tests.git - runTests: false - testConfigVars: - specTestsPath: "tasks.tempdir.outputs.path" - gitBranch: "specTestsBranch" + - name: run_external_tasks + title: "Clone and prepare execution spec tests" + timeout: 10m + config: + testFile: /tests/execution-spec-tests-execute.yaml + testConfig: + gitRepo: https://github.com/ethereum/execution-spec-tests.git + runTests: false + testConfigVars: + specTestsPath: "tasks.tempdir.outputs.path" + gitBranch: "specTestsBranch" - - name: check_clients_are_healthy - title: "Check if at least one client is ready" - id: clientCheck - timeout: 5m - config: - minClientCount: 1 + - name: check_clients_are_healthy + title: "Check if at least one client is ready" + id: clientCheck + timeout: 5m + config: + minClientCount: 1 - - name: generate_child_wallet - id: specTestsWallet - title: "Generate main wallet for tests" - config: - prefundMinBalance: 1000000000000000000000000 # ensure a balance of 10000 ETH - configVars: - privateKey: "walletPrivkey" - walletSeed: "walletSeed" + - name: generate_child_wallet + id: specTestsWallet + title: "Generate main wallet for tests" + config: + prefundMinBalance: 1000000000000000000000000 # ensure a balance of 10000 ETH + configVars: + privateKey: "walletPrivkey" + walletSeed: "walletSeed" - # wait for electra activation - - name: get_consensus_specs - id: consensusSpecs - title: "Get consensus chain specs" - - name: check_consensus_slot_range - title: "Wait for electra activation" - timeout: 1h - configVars: - minEpochNumber: "tasks.consensusSpecs.outputs.specs.GLOAS_FORK_EPOCH" + # wait for electra activation + - name: get_consensus_specs + id: consensusSpecs + title: "Get consensus chain specs" + - name: check_consensus_slot_range + title: "Wait for electra activation" + timeout: 1h + configVars: + minEpochNumber: "tasks.consensusSpecs.outputs.specs.GLOAS_FORK_EPOCH" - # run execution spec tests - - name: run_tasks - title: "Run execution spec tests" - id: tests - config: - continueOnFailure: true - tasks: - - name: run_external_tasks - title: "Run execution spec tests: tests/amsterdam/eip7928_block_level_access_lists" - timeout: 2h - config: - testFile: /tests/execution-spec-tests-execute.yaml - testConfig: - gitRepo: https://github.com/ethereum/execution-spec-tests.git - testPath: ./tests/amsterdam/eip7928_block_level_access_lists - seedAmount: "10000000000000000000000" # 100 ETH - extraFlags: "--fork=Amsterdam -n 16" - runSetup: false - cleanupTestsPath: false - testConfigVars: - rpcEndpoint: "tasks.clientCheck.outputs.goodClients[0].elRpcUrl" - chainID: "tasks.consensusSpecs.outputs.specs.DEPOSIT_CHAIN_ID" - seedPrivateKey: "tasks.specTestsWallet.outputs.childWallet.privkey" - specTestsPath: "tasks.tempdir.outputs.path" - gitBranch: "specTestsBranch" + # run execution spec tests + - name: run_tasks + title: "Run execution spec tests" + id: tests + config: + continueOnFailure: true + tasks: + - name: run_external_tasks + title: "Run execution spec tests: tests/amsterdam/eip7928_block_level_access_lists" + timeout: 2h + config: + testFile: /tests/execution-spec-tests-execute.yaml + testConfig: + gitRepo: https://github.com/ethereum/execution-spec-tests.git + testPath: ./tests/amsterdam/eip7928_block_level_access_lists + seedAmount: "10000000000000000000000" # 100 ETH + extraFlags: "--fork=Amsterdam -n 16" + runSetup: false + cleanupTestsPath: false + testConfigVars: + rpcEndpoint: "tasks.clientCheck.outputs.goodClients[0].elRpcUrl" + chainID: "tasks.consensusSpecs.outputs.specs.DEPOSIT_CHAIN_ID" + seedPrivateKey: "tasks.specTestsWallet.outputs.childWallet.privkey" + specTestsPath: "tasks.tempdir.outputs.path" + gitBranch: "specTestsBranch" cleanupTasks: - - name: run_shell - title: "Cleanup temp dir for execution spec tests" - config: - shell: bash - envVars: - SPEC_TESTS_PATH: specTestsPath - command: | - set -e - SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) + - name: run_shell + title: "Cleanup temp dir for execution spec tests" + config: + shell: bash + envVars: + SPEC_TESTS_PATH: specTestsPath + command: | + set -e + SPEC_TESTS_PATH=$(echo $SPEC_TESTS_PATH | jq -r) - if [ ! -z "$SPEC_TESTS_PATH" ]; then - rm -rf $SPEC_TESTS_PATH - fi + if [ ! -z "$SPEC_TESTS_PATH" ]; then + rm -rf $SPEC_TESTS_PATH + fi From 7b8a510953c22f1118cab0d33cddc853b0a22da4 Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 15 Oct 2025 12:00:12 +0200 Subject: [PATCH 3/3] rename to wait for gloas --- playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml b/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml index 6244be96..6bc88103 100644 --- a/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml +++ b/playbooks/glamsterdam-dev/execution-sec-tests-sequential.yaml @@ -58,12 +58,12 @@ tasks: privateKey: "walletPrivkey" walletSeed: "walletSeed" - # wait for electra activation + # wait for gloas activation - name: get_consensus_specs id: consensusSpecs title: "Get consensus chain specs" - name: check_consensus_slot_range - title: "Wait for electra activation" + title: "Wait for gloas activation" timeout: 1h configVars: minEpochNumber: "tasks.consensusSpecs.outputs.specs.GLOAS_FORK_EPOCH"