From e924c16e87e13246baca8838b9836901615d873f Mon Sep 17 00:00:00 2001 From: Zingo Andersen Date: Mon, 18 May 2026 16:53:26 +0200 Subject: [PATCH 1/2] Arm backed: Consolidate CI testing Testing is done via test_arm_backed.sh instead of different scripts. OOBT tests in CI is part of the test matrix instead of separate job descriptions doing the same thing. Github job names all start with test-arm-backend making it easier know what jobs belongs to and to filter in the HUD. Signed-off-by: Zingo Andersen Change-Id: Ic36bc41eb1a0581c68aa5bc3c15cdf4377ed981d --- .github/workflows/pull.yml | 4 +- .github/workflows/trunk.yml | 43 ++------------------- backends/arm/README.md | 4 ++ backends/arm/requirements-arm-ootb-test.txt | 7 ++++ backends/arm/test/test_arm_backend.sh | 27 +++++++++++++ backends/arm/test/test_arm_ootb.sh | 18 +++++++-- 6 files changed, 59 insertions(+), 44 deletions(-) create mode 100644 backends/arm/requirements-arm-ootb-test.txt diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index c56f9d16ddc..4b1ed06ff66 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -708,8 +708,8 @@ jobs: # Build QNN backend with Buck buck2 build //backends/qualcomm/... - unittest-arm-backend-with-no-deps: - name: unittest-arm-backend-with-no-deps + test-arm-backend-no-driver + name: test-arm-backend-no-driver uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main permissions: id-token: write diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 93a8fce7698..a57c4745a89 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -357,6 +357,9 @@ jobs: - test_arm_backend: test_run_ethos_u85 - test_arm_backend: test_smaller_stories_llama - test_arm_backend: test_memory_allocation + - test_arm_backend: test_ootb_tests_ethos_u + - test_arm_backend: test_ootb_tests_tosa + - test_arm_backend: test_deit_e2e_ethos_u fail-fast: false with: runner: linux.2xlarge.memory @@ -398,6 +401,7 @@ jobs: matrix: include: - test_arm_backend: test_pytest_ops_vkml + - test_arm_backend: test_ootb_tests_vgf fail-fast: false with: runner: linux.2xlarge.memory @@ -427,45 +431,6 @@ jobs: backends/arm/test/test_arm_backend.sh "${ARM_TEST}" - test-arm-ootb-linux: - name: test-arm-ootb-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - strategy: - matrix: - include: - - test_arm_ootb: run_ootb_tests_ethos_u - - test_arm_ootb: run_ootb_tests_tosa - - test_arm_ootb: run_deit_e2e_ethos_u - fail-fast: false - with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - - # Follow the steps required before running the notebooks - # Try to mirror these as closely as possible - source .ci/scripts/utils.sh - install_executorch "--use-pt-pinned-commit" - - .ci/scripts/setup-arm-baremetal-tools.sh - source examples/arm/arm-scratch/setup_path.sh - - # Install requirements for converting notebooks - pip install notebook - - # Run OOTB tests - OOTB_TEST=${{ matrix.test_arm_ootb }} - backends/arm/test/test_arm_ootb.sh $OOTB_TEST - test-coreml-delegate: name: test-coreml-delegate uses: pytorch/test-infra/.github/workflows/macos_job.yml@main diff --git a/backends/arm/README.md b/backends/arm/README.md index e9afa5a928d..f822077e170 100644 --- a/backends/arm/README.md +++ b/backends/arm/README.md @@ -252,6 +252,10 @@ Below is an overview of some of the testing options this script provides: | `test_arm_backend.sh test_pytest_models_vkml` | Runs model unit tests for VGF specific use-cases. | | `test_arm_backend.sh test_run_vkml` | Runs end-to-end unit tests for VGF specific use-cases. | | `test_arm_backend.sh test_model_smollm2_135M` | Runs some models with Corstone FVP. | +| `test_arm_backend.sh test_ootb_tests_ethos_u` | Runs out-of-the-box tests for Ethos-U. | +| `test_arm_backend.sh test_ootb_tests_tosa` | Runs out-of-the-box tests for TOSA. | +| `test_arm_backend.sh test_ootb_tests_vgf` | Runs out-of-the-box tests for VGF. | +| `test_arm_backend.sh test_deit_e2e_ethos_u` | Runs DEiT end-to-end tests on Ethos-U. | | `test_arm_backend.sh test_smaller_stories_llama` | Runs E2E model tests on Corstone FVP. | | `test_arm_backend.sh test_memory_allocation` | Runs memory allocation tests for Ethos-U specific targets | diff --git a/backends/arm/requirements-arm-ootb-test.txt b/backends/arm/requirements-arm-ootb-test.txt new file mode 100644 index 00000000000..a0d5ddb9b92 --- /dev/null +++ b/backends/arm/requirements-arm-ootb-test.txt @@ -0,0 +1,7 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +notebook +nbconvert diff --git a/backends/arm/test/test_arm_backend.sh b/backends/arm/test/test_arm_backend.sh index 3c13fb58cb4..691e7235437 100755 --- a/backends/arm/test/test_arm_backend.sh +++ b/backends/arm/test/test_arm_backend.sh @@ -266,6 +266,33 @@ test_run_vkml() { echo "${TEST_SUITE_NAME}: PASS" } +# -------------------------------------- +# -------- Out-of-the-box tests -------- +# -------------------------------------- +test_ootb_tests_ethos_u() { + echo "${TEST_SUITE_NAME}: Run out-of-the-box tests for Arm Ethos-U" + backends/arm/test/test_arm_ootb.sh run_ootb_tests_ethos_u + echo "${TEST_SUITE_NAME}: PASS" +} + +test_ootb_tests_tosa() { + echo "${TEST_SUITE_NAME}: Run out-of-the-box tests for TOSA" + backends/arm/test/test_arm_ootb.sh run_ootb_tests_tosa + echo "${TEST_SUITE_NAME}: PASS" +} + +test_ootb_tests_vgf() { + echo "${TEST_SUITE_NAME}: Run out-of-the-box tests for VGF" + backends/arm/test/test_arm_ootb.sh run_ootb_tests_vgf + echo "${TEST_SUITE_NAME}: PASS" +} + +test_deit_e2e_ethos_u() { + echo "${TEST_SUITE_NAME}: Run DEiT end-to-end test for Arm Ethos-U" + backends/arm/test/test_arm_ootb.sh run_deit_e2e_ethos_u + echo "${TEST_SUITE_NAME}: PASS" +} + # ------------------------------------ # -------- Miscellaneous tests ------- # ------------------------------------ diff --git a/backends/arm/test/test_arm_ootb.sh b/backends/arm/test/test_arm_ootb.sh index 078566445f9..f1e6fa17488 100755 --- a/backends/arm/test/test_arm_ootb.sh +++ b/backends/arm/test/test_arm_ootb.sh @@ -7,6 +7,10 @@ set -eo pipefail +script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +et_root_dir=$(cd "${script_dir}/../../.." && pwd) +ootb_requirements="${et_root_dir}/backends/arm/requirements-arm-ootb-test.txt" + help() { echo "Usage:" echo " $0 [TESTNAME]" @@ -28,8 +32,17 @@ else fi +install_ootb_test_requirements() { + python3 - <<'PY' || pip install -r "${ootb_requirements}" +import notebook # noqa: F401 +import nbconvert # noqa: F401 +PY +} + + run_ootb_tests_ethos_u() { echo "$FUNCNAME: Running out-of-the-box tests for Arm Ethos-U" + install_ootb_test_requirements jupyter nbconvert \ --to notebook \ --execute examples/arm/ethos_u_minimal_example.ipynb @@ -38,6 +51,7 @@ run_ootb_tests_ethos_u() { run_ootb_tests_tosa() { echo "$FUNCNAME: Running out-of-the-box tests for TOSA" + install_ootb_test_requirements jupyter nbconvert \ --to notebook \ --execute backends/arm/scripts/TOSA_minimal_example.ipynb @@ -46,6 +60,7 @@ run_ootb_tests_tosa() { run_ootb_tests_vgf() { echo "$FUNCNAME: Running out-of-the-box tests for VGF" + install_ootb_test_requirements jupyter nbconvert \ --to notebook \ --execute examples/arm/vgf_minimal_example.ipynb @@ -55,9 +70,6 @@ run_ootb_tests_vgf() { run_deit_e2e_ethos_u() { echo "$FUNCNAME: Fine-tune, export, build, and run the DEiT e2e test" - local script_dir - script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) - et_root_dir=$(cd "${script_dir}/../../.." && pwd) local example_dir="${et_root_dir}/examples/arm/image_classification_example_ethos_u" local work_root="${et_root_dir}/arm_test/deit_tiny_ootb_smoke" local model_dir="${work_root}/deit_tiny_finetuned" From 011d604151d59961375ae38fb2aff528aa38830a Mon Sep 17 00:00:00 2001 From: Zingo Andersen Date: Thu, 21 May 2026 11:58:53 +0200 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/pull.yml | 2 +- backends/arm/test/test_arm_ootb.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 4b1ed06ff66..b19bf5b7863 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -708,7 +708,7 @@ jobs: # Build QNN backend with Buck buck2 build //backends/qualcomm/... - test-arm-backend-no-driver + test-arm-backend-no-driver: name: test-arm-backend-no-driver uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main permissions: diff --git a/backends/arm/test/test_arm_ootb.sh b/backends/arm/test/test_arm_ootb.sh index f1e6fa17488..a96f47a7dbe 100755 --- a/backends/arm/test/test_arm_ootb.sh +++ b/backends/arm/test/test_arm_ootb.sh @@ -11,6 +11,8 @@ script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) et_root_dir=$(cd "${script_dir}/../../.." && pwd) ootb_requirements="${et_root_dir}/backends/arm/requirements-arm-ootb-test.txt" +cd "${et_root_dir}" + help() { echo "Usage:" echo " $0 [TESTNAME]"