diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 60d54d9489e..c2787681d4e 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -714,8 +714,8 @@ jobs: # Build QNN backend with Buck buck2 build //backends/qualcomm/... - unittest-arm-backend-with-no-deps: - name: unittest-arm-backend-with-no-deps + test-arm-backend-no-driver: + name: test-arm-backend-no-driver uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main permissions: id-token: write diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 3787410a813..5a6720cdfad 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -234,6 +234,9 @@ jobs: - test_arm_backend: test_run_ethos_u85 - test_arm_backend: test_smaller_stories_llama - test_arm_backend: test_memory_allocation + - test_arm_backend: test_ootb_tests_ethos_u + - test_arm_backend: test_ootb_tests_tosa + - test_arm_backend: test_deit_e2e_ethos_u fail-fast: false with: runner: linux.2xlarge.memory @@ -275,6 +278,7 @@ jobs: matrix: include: - test_arm_backend: test_pytest_ops_vkml + - test_arm_backend: test_ootb_tests_vgf fail-fast: false with: runner: linux.2xlarge.memory @@ -304,45 +308,6 @@ jobs: backends/arm/test/test_arm_backend.sh "${ARM_TEST}" - test-arm-ootb-linux: - name: test-arm-ootb-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - strategy: - matrix: - include: - - test_arm_ootb: run_ootb_tests_ethos_u - - test_arm_ootb: run_ootb_tests_tosa - - test_arm_ootb: run_deit_e2e_ethos_u - fail-fast: false - with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - - # Follow the steps required before running the notebooks - # Try to mirror these as closely as possible - source .ci/scripts/utils.sh - install_executorch "--use-pt-pinned-commit" - - .ci/scripts/setup-arm-baremetal-tools.sh - source examples/arm/arm-scratch/setup_path.sh - - # Install requirements for converting notebooks - pip install notebook - - # Run OOTB tests - OOTB_TEST=${{ matrix.test_arm_ootb }} - backends/arm/test/test_arm_ootb.sh $OOTB_TEST - test-coreml-delegate: needs: changed-files if: | diff --git a/backends/arm/README.md b/backends/arm/README.md index e9afa5a928d..f822077e170 100644 --- a/backends/arm/README.md +++ b/backends/arm/README.md @@ -252,6 +252,10 @@ Below is an overview of some of the testing options this script provides: | `test_arm_backend.sh test_pytest_models_vkml` | Runs model unit tests for VGF specific use-cases. | | `test_arm_backend.sh test_run_vkml` | Runs end-to-end unit tests for VGF specific use-cases. | | `test_arm_backend.sh test_model_smollm2_135M` | Runs some models with Corstone FVP. | +| `test_arm_backend.sh test_ootb_tests_ethos_u` | Runs out-of-the-box tests for Ethos-U. | +| `test_arm_backend.sh test_ootb_tests_tosa` | Runs out-of-the-box tests for TOSA. | +| `test_arm_backend.sh test_ootb_tests_vgf` | Runs out-of-the-box tests for VGF. | +| `test_arm_backend.sh test_deit_e2e_ethos_u` | Runs DEiT end-to-end tests on Ethos-U. | | `test_arm_backend.sh test_smaller_stories_llama` | Runs E2E model tests on Corstone FVP. | | `test_arm_backend.sh test_memory_allocation` | Runs memory allocation tests for Ethos-U specific targets | diff --git a/backends/arm/requirements-arm-ootb-test.txt b/backends/arm/requirements-arm-ootb-test.txt new file mode 100644 index 00000000000..a0d5ddb9b92 --- /dev/null +++ b/backends/arm/requirements-arm-ootb-test.txt @@ -0,0 +1,7 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +notebook +nbconvert diff --git a/backends/arm/test/test_arm_backend.sh b/backends/arm/test/test_arm_backend.sh index 6a69e94b147..be48d7ad234 100755 --- a/backends/arm/test/test_arm_backend.sh +++ b/backends/arm/test/test_arm_backend.sh @@ -269,6 +269,33 @@ test_run_vkml() { echo "${TEST_SUITE_NAME}: PASS" } +# -------------------------------------- +# -------- Out-of-the-box tests -------- +# -------------------------------------- +test_ootb_tests_ethos_u() { + echo "${TEST_SUITE_NAME}: Run out-of-the-box tests for Arm Ethos-U" + backends/arm/test/test_arm_ootb.sh run_ootb_tests_ethos_u + echo "${TEST_SUITE_NAME}: PASS" +} + +test_ootb_tests_tosa() { + echo "${TEST_SUITE_NAME}: Run out-of-the-box tests for TOSA" + backends/arm/test/test_arm_ootb.sh run_ootb_tests_tosa + echo "${TEST_SUITE_NAME}: PASS" +} + +test_ootb_tests_vgf() { + echo "${TEST_SUITE_NAME}: Run out-of-the-box tests for VGF" + backends/arm/test/test_arm_ootb.sh run_ootb_tests_vgf + echo "${TEST_SUITE_NAME}: PASS" +} + +test_deit_e2e_ethos_u() { + echo "${TEST_SUITE_NAME}: Run DEiT end-to-end test for Arm Ethos-U" + backends/arm/test/test_arm_ootb.sh run_deit_e2e_ethos_u + echo "${TEST_SUITE_NAME}: PASS" +} + # ------------------------------------ # -------- Miscellaneous tests ------- # ------------------------------------ diff --git a/backends/arm/test/test_arm_ootb.sh b/backends/arm/test/test_arm_ootb.sh index 078566445f9..a96f47a7dbe 100755 --- a/backends/arm/test/test_arm_ootb.sh +++ b/backends/arm/test/test_arm_ootb.sh @@ -7,6 +7,12 @@ set -eo pipefail +script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +et_root_dir=$(cd "${script_dir}/../../.." && pwd) +ootb_requirements="${et_root_dir}/backends/arm/requirements-arm-ootb-test.txt" + +cd "${et_root_dir}" + help() { echo "Usage:" echo " $0 [TESTNAME]" @@ -28,8 +34,17 @@ else fi +install_ootb_test_requirements() { + python3 - <<'PY' || pip install -r "${ootb_requirements}" +import notebook # noqa: F401 +import nbconvert # noqa: F401 +PY +} + + run_ootb_tests_ethos_u() { echo "$FUNCNAME: Running out-of-the-box tests for Arm Ethos-U" + install_ootb_test_requirements jupyter nbconvert \ --to notebook \ --execute examples/arm/ethos_u_minimal_example.ipynb @@ -38,6 +53,7 @@ run_ootb_tests_ethos_u() { run_ootb_tests_tosa() { echo "$FUNCNAME: Running out-of-the-box tests for TOSA" + install_ootb_test_requirements jupyter nbconvert \ --to notebook \ --execute backends/arm/scripts/TOSA_minimal_example.ipynb @@ -46,6 +62,7 @@ run_ootb_tests_tosa() { run_ootb_tests_vgf() { echo "$FUNCNAME: Running out-of-the-box tests for VGF" + install_ootb_test_requirements jupyter nbconvert \ --to notebook \ --execute examples/arm/vgf_minimal_example.ipynb @@ -55,9 +72,6 @@ run_ootb_tests_vgf() { run_deit_e2e_ethos_u() { echo "$FUNCNAME: Fine-tune, export, build, and run the DEiT e2e test" - local script_dir - script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) - et_root_dir=$(cd "${script_dir}/../../.." && pwd) local example_dir="${et_root_dir}/examples/arm/image_classification_example_ethos_u" local work_root="${et_root_dir}/arm_test/deit_tiny_ootb_smoke" local model_dir="${work_root}/deit_tiny_finetuned"