From ce50b49aa2a56d46dd0658e8638b7c02d20eba4d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 17 Dec 2025 15:38:27 -0500 Subject: [PATCH 1/7] feat(ci): consolidate e2e tests into 6 CI suite scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses issue #882 by reducing CI job count from ~105 to ~18 jobs while maintaining all test coverage and developer workflows. Changes: - Create 6 CI suite scripts that run groups of related e2e tests - Add shared framework (ci_suite_framework.sh) for common functionality - Update GitHub Actions matrix to use CI suites instead of 35 individual tests - Update Mergify config to require new consolidated CI job checks - Enhance mergify_lint.py to validate all e2e tests are covered by CI suites ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .github/workflows/test.yaml | 43 ++-------- .mergify.yml | 124 +++++------------------------ e2e/ci_bootstrap_parallel_suite.sh | 19 +++++ e2e/ci_bootstrap_suite.sh | 33 ++++++++ e2e/ci_build_suite.sh | 23 ++++++ e2e/ci_config_suite.sh | 20 +++++ e2e/ci_specialized_suite.sh | 27 +++++++ e2e/ci_suite_framework.sh | 93 ++++++++++++++++++++++ e2e/ci_workflow_suite.sh | 31 ++++++++ e2e/mergify_lint.py | 43 +++++++++- 10 files changed, 311 insertions(+), 145 deletions(-) create mode 100755 e2e/ci_bootstrap_parallel_suite.sh create mode 100755 e2e/ci_bootstrap_suite.sh create mode 100755 e2e/ci_build_suite.sh create mode 100755 e2e/ci_config_suite.sh create mode 100755 e2e/ci_specialized_suite.sh create mode 100644 e2e/ci_suite_framework.sh create mode 100755 e2e/ci_workflow_suite.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 922ffc56..d29b2f59 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -75,41 +75,12 @@ jobs: # RHEL 9.4 has rustc 1.75 - "1.75" test-script: - - bootstrap - - bootstrap_extras - - bootstrap_build_tags - - bootstrap_prerelease - - bootstrap_conflicting_requirements - - bootstrap_constraints - - bootstrap_cache - - bootstrap_sdist_only - - bootstrap_git_url - - bootstrap_git_url_tag - - bootstrap_parallel - - bootstrap_parallel_git_url - - bootstrap_parallel_git_url_tag - - bootstrap_skip_constraints - - build - - build_order - - build_parallel - - build_sequence_git_url - - build_steps - - build_settings - - graph_to_constraints - - meson - - migrate_graph - - override - - pep517_build_sdist - - post_bootstrap_hook - - prebuilt_wheels_alt_server - - report_missing_dependency - - rust_vendor - - download_sequence - - optimize_build - - extra_metadata - - elfdeps - - prebuilt_wheel_hook - - lint_requirements + - ci_bootstrap_suite + - ci_bootstrap_parallel_suite + - ci_build_suite + - ci_config_suite + - ci_specialized_suite + - ci_workflow_suite os: - ubuntu-latest - macos-latest @@ -147,7 +118,7 @@ jobs: run: python -m pip install hatch 'click!=8.3.0' - name: Run tests - run: ./e2e/test_${{ matrix.test-script }}.sh + run: ./e2e/${{ matrix.test-script }}.sh - name: Upload logs for debugging if: ${{ failure() }} diff --git a/.mergify.yml b/.mergify.yml index b67756a8..0728bc78 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -37,111 +37,25 @@ pull_request_rules: # MyPy type checking - check-success=mypy - - check-success=e2e (3.11, 1.75, bootstrap, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_build_tags, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_cache, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_conflicting_requirements, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_constraints, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_extras, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_git_url, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_git_url_tag, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_parallel, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_parallel_git_url, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_parallel_git_url_tag, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_prerelease, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_sdist_only, ubuntu-latest) - - check-success=e2e (3.11, 1.75, bootstrap_skip_constraints, ubuntu-latest) - - check-success=e2e (3.11, 1.75, build, ubuntu-latest) - - check-success=e2e (3.11, 1.75, build_order, ubuntu-latest) - - check-success=e2e (3.11, 1.75, build_parallel, ubuntu-latest) - - check-success=e2e (3.11, 1.75, build_sequence_git_url, ubuntu-latest) - - check-success=e2e (3.11, 1.75, build_settings, ubuntu-latest) - - check-success=e2e (3.11, 1.75, build_steps, ubuntu-latest) - - check-success=e2e (3.11, 1.75, download_sequence, ubuntu-latest) - - check-success=e2e (3.11, 1.75, elfdeps, ubuntu-latest) - - check-success=e2e (3.11, 1.75, extra_metadata, ubuntu-latest) - - check-success=e2e (3.11, 1.75, graph_to_constraints, ubuntu-latest) - - check-success=e2e (3.11, 1.75, lint_requirements, ubuntu-latest) - - check-success=e2e (3.11, 1.75, meson, ubuntu-latest) - - check-success=e2e (3.11, 1.75, migrate_graph, ubuntu-latest) - - check-success=e2e (3.11, 1.75, optimize_build, ubuntu-latest) - - check-success=e2e (3.11, 1.75, override, ubuntu-latest) - - check-success=e2e (3.11, 1.75, pep517_build_sdist, ubuntu-latest) - - check-success=e2e (3.11, 1.75, post_bootstrap_hook, ubuntu-latest) - - check-success=e2e (3.11, 1.75, prebuilt_wheel_hook, ubuntu-latest) - - check-success=e2e (3.11, 1.75, prebuilt_wheels_alt_server, ubuntu-latest) - - check-success=e2e (3.11, 1.75, report_missing_dependency, ubuntu-latest) - - check-success=e2e (3.11, 1.75, rust_vendor, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_build_tags, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_build_tags, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_cache, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_cache, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_conflicting_requirements, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_conflicting_requirements, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_constraints, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_constraints, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_extras, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_extras, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_git_url, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_git_url, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_git_url_tag, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_parallel, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_parallel, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url_tag, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_parallel_git_url_tag, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_prerelease, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_prerelease, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_sdist_only, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_sdist_only, ubuntu-latest) - - check-success=e2e (3.12, 1.75, bootstrap_skip_constraints, macos-latest) - - check-success=e2e (3.12, 1.75, bootstrap_skip_constraints, ubuntu-latest) - - check-success=e2e (3.12, 1.75, build, macos-latest) - - check-success=e2e (3.12, 1.75, build, ubuntu-latest) - - check-success=e2e (3.12, 1.75, build_order, macos-latest) - - check-success=e2e (3.12, 1.75, build_order, ubuntu-latest) - - check-success=e2e (3.12, 1.75, build_parallel, macos-latest) - - check-success=e2e (3.12, 1.75, build_parallel, ubuntu-latest) - - check-success=e2e (3.12, 1.75, build_sequence_git_url, macos-latest) - - check-success=e2e (3.12, 1.75, build_sequence_git_url, ubuntu-latest) - - check-success=e2e (3.12, 1.75, build_settings, macos-latest) - - check-success=e2e (3.12, 1.75, build_settings, ubuntu-latest) - - check-success=e2e (3.12, 1.75, build_steps, macos-latest) - - check-success=e2e (3.12, 1.75, build_steps, ubuntu-latest) - - check-success=e2e (3.12, 1.75, download_sequence, macos-latest) - - check-success=e2e (3.12, 1.75, download_sequence, ubuntu-latest) - - check-success=e2e (3.12, 1.75, elfdeps, macos-latest) - - check-success=e2e (3.12, 1.75, elfdeps, ubuntu-latest) - - check-success=e2e (3.12, 1.75, extra_metadata, macos-latest) - - check-success=e2e (3.12, 1.75, extra_metadata, ubuntu-latest) - - check-success=e2e (3.12, 1.75, graph_to_constraints, macos-latest) - - check-success=e2e (3.12, 1.75, graph_to_constraints, ubuntu-latest) - - check-success=e2e (3.12, 1.75, lint_requirements, macos-latest) - - check-success=e2e (3.12, 1.75, lint_requirements, ubuntu-latest) - - check-success=e2e (3.12, 1.75, meson, macos-latest) - - check-success=e2e (3.12, 1.75, meson, ubuntu-latest) - - check-success=e2e (3.12, 1.75, migrate_graph, macos-latest) - - check-success=e2e (3.12, 1.75, migrate_graph, ubuntu-latest) - - check-success=e2e (3.12, 1.75, optimize_build, macos-latest) - - check-success=e2e (3.12, 1.75, optimize_build, ubuntu-latest) - - check-success=e2e (3.12, 1.75, override, macos-latest) - - check-success=e2e (3.12, 1.75, override, ubuntu-latest) - - check-success=e2e (3.12, 1.75, pep517_build_sdist, macos-latest) - - check-success=e2e (3.12, 1.75, pep517_build_sdist, ubuntu-latest) - - check-success=e2e (3.12, 1.75, post_bootstrap_hook, macos-latest) - - check-success=e2e (3.12, 1.75, post_bootstrap_hook, ubuntu-latest) - - check-success=e2e (3.12, 1.75, prebuilt_wheel_hook, macos-latest) - - check-success=e2e (3.12, 1.75, prebuilt_wheel_hook, ubuntu-latest) - - check-success=e2e (3.12, 1.75, prebuilt_wheels_alt_server, macos-latest) - - check-success=e2e (3.12, 1.75, prebuilt_wheels_alt_server, ubuntu-latest) - - check-success=e2e (3.12, 1.75, report_missing_dependency, macos-latest) - - check-success=e2e (3.12, 1.75, report_missing_dependency, ubuntu-latest) - - check-success=e2e (3.12, 1.75, rust_vendor, macos-latest) - - check-success=e2e (3.12, 1.75, rust_vendor, ubuntu-latest) + # E2E test suites (consolidated from individual tests) + - check-success=e2e (3.11, 1.75, ci_bootstrap_parallel_suite, ubuntu-latest) + - check-success=e2e (3.11, 1.75, ci_bootstrap_suite, ubuntu-latest) + - check-success=e2e (3.11, 1.75, ci_build_suite, ubuntu-latest) + - check-success=e2e (3.11, 1.75, ci_config_suite, ubuntu-latest) + - check-success=e2e (3.11, 1.75, ci_specialized_suite, ubuntu-latest) + - check-success=e2e (3.11, 1.75, ci_workflow_suite, ubuntu-latest) + - check-success=e2e (3.12, 1.75, ci_bootstrap_parallel_suite, macos-latest) + - check-success=e2e (3.12, 1.75, ci_bootstrap_parallel_suite, ubuntu-latest) + - check-success=e2e (3.12, 1.75, ci_bootstrap_suite, macos-latest) + - check-success=e2e (3.12, 1.75, ci_bootstrap_suite, ubuntu-latest) + - check-success=e2e (3.12, 1.75, ci_build_suite, macos-latest) + - check-success=e2e (3.12, 1.75, ci_build_suite, ubuntu-latest) + - check-success=e2e (3.12, 1.75, ci_config_suite, macos-latest) + - check-success=e2e (3.12, 1.75, ci_config_suite, ubuntu-latest) + - check-success=e2e (3.12, 1.75, ci_specialized_suite, macos-latest) + - check-success=e2e (3.12, 1.75, ci_specialized_suite, ubuntu-latest) + - check-success=e2e (3.12, 1.75, ci_workflow_suite, macos-latest) + - check-success=e2e (3.12, 1.75, ci_workflow_suite, ubuntu-latest) # At least 1 reviewer from maintainers - and: diff --git a/e2e/ci_bootstrap_parallel_suite.sh b/e2e/ci_bootstrap_parallel_suite.sh new file mode 100755 index 00000000..42051443 --- /dev/null +++ b/e2e/ci_bootstrap_parallel_suite.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# CI wrapper script for all bootstrap-parallel-related e2e tests +# This script runs multiple bootstrap parallel tests sequentially to reduce CI job count +# while preserving the ability to run individual tests during development. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/ci_suite_framework.sh" + +init_suite "Bootstrap Parallel Test Suite" "bootstrap-parallel tests" + +# Bootstrap parallel tests +test_section "bootstrap parallel tests" +run_test "bootstrap_parallel" +run_test "bootstrap_parallel_git_url" +run_test "bootstrap_parallel_git_url_tag" + +finish_suite diff --git a/e2e/ci_bootstrap_suite.sh b/e2e/ci_bootstrap_suite.sh new file mode 100755 index 00000000..deab8dd2 --- /dev/null +++ b/e2e/ci_bootstrap_suite.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# CI wrapper script for all bootstrap-related e2e tests +# This script runs multiple bootstrap tests sequentially to reduce CI job count +# while preserving the ability to run individual tests during development. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/ci_suite_framework.sh" + +init_suite "Bootstrap Test Suite" "standard bootstrap tests (parallel bootstrap tests are in separate suite)" + +# Bootstrap tests in logical order - basic first, then variations +test_section "basic bootstrap tests" +run_test "bootstrap" +run_test "bootstrap_extras" +run_test "bootstrap_build_tags" + +test_section "bootstrap constraint tests" +run_test "bootstrap_constraints" +run_test "bootstrap_skip_constraints" +run_test "bootstrap_conflicting_requirements" + +test_section "bootstrap configuration tests" +run_test "bootstrap_prerelease" +run_test "bootstrap_cache" +run_test "bootstrap_sdist_only" + +test_section "bootstrap git URL tests" +run_test "bootstrap_git_url" +run_test "bootstrap_git_url_tag" + +finish_suite diff --git a/e2e/ci_build_suite.sh b/e2e/ci_build_suite.sh new file mode 100755 index 00000000..344169c3 --- /dev/null +++ b/e2e/ci_build_suite.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# CI wrapper script for all build-related e2e tests +# This script runs multiple build tests sequentially to reduce CI job count +# while preserving the ability to run individual tests during development. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/ci_suite_framework.sh" + +init_suite "Build Test Suite" "build-related tests" + +# Build tests in logical order +test_section "core build tests" +run_test "build" +run_test "build_order" +run_test "build_steps" + +test_section "advanced build tests" +run_test "build_parallel" +run_test "build_sequence_git_url" + +finish_suite diff --git a/e2e/ci_config_suite.sh b/e2e/ci_config_suite.sh new file mode 100755 index 00000000..c880d913 --- /dev/null +++ b/e2e/ci_config_suite.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# CI wrapper script for all configuration/settings-related e2e tests +# This script runs multiple configuration tests sequentially to reduce CI job count +# while preserving the ability to run individual tests during development. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/ci_suite_framework.sh" + +init_suite "Configuration Test Suite" "configuration/settings tests" + +# Configuration and settings tests +test_section "configuration tests" +run_test "build_settings" +run_test "override" +run_test "extra_metadata" +run_test "lint_requirements" + +finish_suite diff --git a/e2e/ci_specialized_suite.sh b/e2e/ci_specialized_suite.sh new file mode 100755 index 00000000..1b558014 --- /dev/null +++ b/e2e/ci_specialized_suite.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# CI wrapper script for all specialized build-related e2e tests +# This script runs multiple specialized build tests sequentially to reduce CI job count +# while preserving the ability to run individual tests during development. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/ci_suite_framework.sh" + +init_suite "Specialized Build Test Suite" "specialized build tests" + +# Specialized build tests +test_section "specialized build system tests" +run_test "meson" +run_test "rust_vendor" + +test_section "build standard tests" +run_test "pep517_build_sdist" + +test_section "platform-specific tests" +run_test "elfdeps" + +test_section "hook tests" +run_test "prebuilt_wheel_hook" + +finish_suite diff --git a/e2e/ci_suite_framework.sh b/e2e/ci_suite_framework.sh new file mode 100644 index 00000000..aa866013 --- /dev/null +++ b/e2e/ci_suite_framework.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Common framework for CI test suites +# This provides shared functionality for running multiple e2e tests in sequence + +set -e +set -o pipefail + +# Global variables for tracking test results +FAILED_TESTS=() +TOTAL_TESTS=0 +SUITE_NAME="" +SUITE_DESCRIPTION="" + +# Initialize a test suite +# Usage: init_suite "Suite Name" "Description of what this suite tests" +init_suite() { + SUITE_NAME="$1" + SUITE_DESCRIPTION="$2" + FAILED_TESTS=() + TOTAL_TESTS=0 + + echo "==========================================" + echo "CI $SUITE_NAME" + echo "Running $SUITE_DESCRIPTION sequentially" + echo "==========================================" +} + +# Run a single test +# Usage: run_test "test_name_without_prefix" +run_test() { + local test_name="$1" + local test_script="$SCRIPTDIR/test_${test_name}.sh" + + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + echo "" + echo "==========================================" + echo "Running: $test_name" + echo "Script: $test_script" + echo "==========================================" + + if [ ! -f "$test_script" ]; then + echo "ERROR: Test script not found: $test_script" + FAILED_TESTS+=("$test_name (script not found)") + return 1 + fi + + if "$test_script"; then + echo "โœ“ PASSED: $test_name" + else + echo "โœ— FAILED: $test_name" + FAILED_TESTS+=("$test_name") + # Continue running other tests instead of exiting immediately + # This provides more comprehensive feedback in CI + fi +} + +# Print a section header for organizing related tests +# Usage: test_section "Section Description" +test_section() { + echo "" + echo "Running $1..." +} + +# Print final summary and exit with appropriate code +# Usage: finish_suite +finish_suite() { + echo "" + echo "==========================================" + echo "CI $SUITE_NAME Summary" + echo "==========================================" + echo "Total tests run: $TOTAL_TESTS" + echo "Passed: $((TOTAL_TESTS - ${#FAILED_TESTS[@]}))" + echo "Failed: ${#FAILED_TESTS[@]}" + + if [ ${#FAILED_TESTS[@]} -gt 0 ]; then + echo "" + echo "Failed tests:" + for test in "${FAILED_TESTS[@]}"; do + echo " - $test" + done + echo "" + echo "To debug individual failures, run the specific test script:" + for test in "${FAILED_TESTS[@]}"; do + test_name=$(echo "$test" | cut -d' ' -f1) + echo " ./e2e/test_${test_name}.sh" + done + exit 1 + else + echo "All tests in $SUITE_NAME passed! โœ“" + fi +} diff --git a/e2e/ci_workflow_suite.sh b/e2e/ci_workflow_suite.sh new file mode 100755 index 00000000..bb93fabf --- /dev/null +++ b/e2e/ci_workflow_suite.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# CI wrapper script for all workflow/pipeline-related e2e tests +# This script runs multiple workflow tests sequentially to reduce CI job count +# while preserving the ability to run individual tests during development. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/ci_suite_framework.sh" + +init_suite "Workflow Test Suite" "workflow/pipeline tests" + +# Workflow and pipeline tests +test_section "graph/constraints workflow tests" +run_test "graph_to_constraints" +run_test "migrate_graph" + +test_section "sequence/ordering tests" +run_test "download_sequence" +run_test "optimize_build" + +test_section "hook workflow tests" +run_test "post_bootstrap_hook" + +test_section "server/deployment tests" +run_test "prebuilt_wheels_alt_server" + +test_section "diagnostic tests" +run_test "report_missing_dependency" + +finish_suite diff --git a/e2e/mergify_lint.py b/e2e/mergify_lint.py index d745c04f..8fda6987 100644 --- a/e2e/mergify_lint.py +++ b/e2e/mergify_lint.py @@ -2,6 +2,7 @@ import itertools import pathlib +import re import sys import yaml @@ -38,20 +39,54 @@ os_versions.remove("macos-latest") e2e_dir = pathlib.Path("e2e") -e2e_jobs = set( +# Look for CI suite scripts instead of individual test scripts +ci_suite_jobs = set( + script.name[:-len(".sh")] for script in e2e_dir.glob("ci_*_suite.sh") +) +print("found CI suite scripts:\n ", "\n ".join(sorted(ci_suite_jobs)), sep="") + +# Also find all individual e2e test scripts to ensure they're all covered +individual_e2e_scripts = set( script.name[len("test_") : -len(".sh")] for script in e2e_dir.glob("test_*.sh") ) -print("found job scripts:\n ", "\n ".join(sorted(e2e_jobs)), sep="") +print("found individual e2e scripts:\n ", "\n ".join(sorted(individual_e2e_scripts)), sep="") # Remember if we should fail so we can apply all of the rules and then # exit with an error. RC = 0 -# Require test jobs for every script. -for script_name in sorted(e2e_jobs.difference(test_scripts)): +# Require test jobs for every CI suite script. +for script_name in sorted(ci_suite_jobs.difference(test_scripts)): print(f"ERROR: {script_name} not in the matrix in {github_actions_file}") RC = 1 +# Check that all individual e2e scripts are referenced in CI suite scripts +print("\nChecking that all individual e2e tests are covered by CI suites...") +referenced_scripts = set() +for ci_suite_file in e2e_dir.glob("ci_*_suite.sh"): + content = ci_suite_file.read_text(encoding="utf8") + # Look for run_test "script_name" calls (excluding commented lines) + for line in content.split('\n'): + # Skip lines that start with # (comments) + stripped = line.strip() + if stripped.startswith('#'): + continue + for match in re.finditer(r'run_test\s+"([^"]+)"', line): + referenced_scripts.add(match.group(1)) + +print("scripts referenced in CI suites:\n ", "\n ".join(sorted(referenced_scripts)), sep="") + +# Find any individual e2e scripts that aren't referenced in any CI suite +unreferenced_scripts = individual_e2e_scripts.difference(referenced_scripts) +if unreferenced_scripts: + print(f"\nERROR: The following e2e scripts are not referenced in any CI suite:") + for script in sorted(unreferenced_scripts): + print(f" - {script}") + print(f"Please add these scripts to the appropriate CI suite script.") + RC = 1 +else: + print("โœ“ All individual e2e scripts are covered by CI suites!") + # We expect a job for every combination of python version, rust # version, and test script. expected_jobs = set( From 430c5f9a447c6b17bddc223df1239fff7cdfbb55 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 17 Dec 2025 16:01:48 -0500 Subject: [PATCH 2/7] fix: force hatch python version in test jobs We have a matrix of jobs, let's make sure we force the python interpreter to be the one we want in case there are multiple installed in the test image. --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d29b2f59..9cfa6c8a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -118,7 +118,7 @@ jobs: run: python -m pip install hatch 'click!=8.3.0' - name: Run tests - run: ./e2e/${{ matrix.test-script }}.sh + run: HATCH_PYTHON=${{ matrix.python-version }} ./e2e/${{ matrix.test-script }}.sh - name: Upload logs for debugging if: ${{ failure() }} From dddfc61a396003d21f8e3fb114bce986051f968b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 17 Dec 2025 16:11:45 -0500 Subject: [PATCH 3/7] fix: ignore e2e build output --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5983ed78..c096f37f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ __pycache__/ /package.json /e2e-output/ /dist/ +/e2e/flit_core_override/build /e2e/fromager_hooks/build /e2e/fromager_hooks/dist /e2e/pyo3_test/.cargo From fc677fcc41fad38266d34e5aee99bd0db30f2104 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 17 Dec 2025 16:40:16 -0500 Subject: [PATCH 4/7] fix(ci): add test isolation to prevent CI suite flakiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clean e2e-output directory and hatch environment before each test in CI suites to prevent contamination between tests, particularly from plugin installations like in test_override.sh that were causing subsequent test failures. ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- e2e/ci_suite_framework.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/e2e/ci_suite_framework.sh b/e2e/ci_suite_framework.sh index aa866013..8b980710 100644 --- a/e2e/ci_suite_framework.sh +++ b/e2e/ci_suite_framework.sh @@ -27,7 +27,7 @@ init_suite() { echo "==========================================" } -# Run a single test +# Run a single test with clean environment # Usage: run_test "test_name_without_prefix" run_test() { local test_name="$1" @@ -46,6 +46,10 @@ run_test() { return 1 fi + # Clean environment before each test to prevent interference + clean_test_environment "$test_name" + + # Run the test if "$test_script"; then echo "โœ“ PASSED: $test_name" else @@ -56,6 +60,28 @@ run_test() { fi } +# Clean test environment before each test to prevent interference +# Usage: clean_test_environment "test_name" +clean_test_environment() { + local test_name="$1" + + echo "๐Ÿงน Cleaning environment before test: $test_name" + + # Remove e2e output directory + local outdir + outdir="$(dirname "$SCRIPTDIR")/e2e-output" + if [ -d "$outdir" ]; then + echo "Removing existing e2e-output directory..." + rm -rf "$outdir" || true + fi + + # Remove and recreate hatch e2e environment to ensure clean state + echo "Removing hatch e2e environment..." + hatch env remove e2e 2>/dev/null || true + + echo "โœจ Environment cleaned for test: $test_name" +} + # Print a section header for organizing related tests # Usage: test_section "Section Description" test_section() { From b4f9c6166c88068c886a0ddd78f908611877bc76 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 17 Dec 2025 17:00:55 -0500 Subject: [PATCH 5/7] fix: add more debug to override e2e test script --- e2e/test_override.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e/test_override.sh b/e2e/test_override.sh index 80f482be..b8580b7e 100755 --- a/e2e/test_override.sh +++ b/e2e/test_override.sh @@ -10,7 +10,11 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "$SCRIPTDIR/common.sh" -pip install e2e/flit_core_override +pip install e2e/flit_core_override entry-point-inspector + +python3 --version +epi group list +epi group show fromager.project_overrides fromager \ --verbose \ From 45d074d4bde546aeba70aefcf91c2de33f508f5c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 18 Dec 2025 14:38:17 -0500 Subject: [PATCH 6/7] docs: strengthen import guidelines for AI agents Enhance agent instruction import placement guidelines to be more explicit: - Add "ALWAYS FOLLOW" to section heading - Strengthen language: "MUST" instead of "should" - Add "ALL Python files" clarification - Emphasize "under any circumstances" for local imports Ensures future AI agents clearly understand PEP 8 import requirements. Co-Authored-By: Claude --- AGENTS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e7b2984b..8e8df31f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,10 +121,10 @@ These run automatically on commit if installed with `hatch run lint:install-hook ## Code Patterns -**Import Guidelines:** +**Import Guidelines (ALWAYS FOLLOW):** -- **PEP 8: imports should be at the top**: All import statements must be placed at the top of the file, after module docstrings and before other code -- **No local imports**: Do not place import statements inside functions, methods, or conditional blocks +- **PEP 8: imports should be at the top**: All import statements MUST be placed at the top of the file, after module docstrings and before other code. This applies to ALL Python files in the project. +- **No local imports**: Do not place import statements inside functions, methods, or conditional blocks under any circumstances ### Testing Pattern From eefcd49b3b8becbc880991407ffb5de32ff048e4 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 19 Dec 2025 08:48:51 -0500 Subject: [PATCH 7/7] fix: remove emoji from e2e framework scripts Avoid using emoji in log messages. --- e2e/ci_suite_framework.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e/ci_suite_framework.sh b/e2e/ci_suite_framework.sh index 8b980710..a119195d 100644 --- a/e2e/ci_suite_framework.sh +++ b/e2e/ci_suite_framework.sh @@ -51,9 +51,9 @@ run_test() { # Run the test if "$test_script"; then - echo "โœ“ PASSED: $test_name" + echo "PASSED: $test_name" else - echo "โœ— FAILED: $test_name" + echo "FAILED: $test_name" FAILED_TESTS+=("$test_name") # Continue running other tests instead of exiting immediately # This provides more comprehensive feedback in CI @@ -65,7 +65,7 @@ run_test() { clean_test_environment() { local test_name="$1" - echo "๐Ÿงน Cleaning environment before test: $test_name" + echo "Cleaning environment before test: $test_name" # Remove e2e output directory local outdir @@ -79,7 +79,7 @@ clean_test_environment() { echo "Removing hatch e2e environment..." hatch env remove e2e 2>/dev/null || true - echo "โœจ Environment cleaned for test: $test_name" + echo "Environment cleaned for test: $test_name" } # Print a section header for organizing related tests @@ -114,6 +114,6 @@ finish_suite() { done exit 1 else - echo "All tests in $SUITE_NAME passed! โœ“" + echo "All tests in $SUITE_NAME passed!" fi }