Run RISC-V tests with multiple RVV QEMU configurations

.ci/scripts/test_riscv_qemu.sh

@@ -17,16 +17,19 @@ et_root_dir=$(realpath "${script_dir}/../..")
 model="add"
 xnnpack=false
 quantize=false
-verbose=false
+verbose_xnnpack=false
+debug_xnnpack=false
 
 usage() {
     cat <<EOF
 Usage: $(basename "$0") [options]
 Options:
-  --model=<NAME>  Which model to export and run (default: add)
-  --xnnpack       Enable the XNNPACK backend (AOT partitioner + runtime)
-  --quantize      Produce an 8-bit quantized model
-  -h, --help      Show this help
+  --model=<NAME>     Which model to export and run (default: add)
+  --xnnpack          Enable the XNNPACK backend (AOT partitioner + runtime)
+  --quantize         Produce an 8-bit quantized model
+  --verbose-xnnpack  Build XNNPACK with XNN_LOG_LEVEL=4 to log microkernel dispatch
+  --debug-xnnpack    Enable XNNPACK partitioner DEBUG logging and dump the lowered graph
+  -h, --help         Show this help
 EOF
 }
 
@@ -35,7 +38,8 @@ for arg in "$@"; do
         --model=*) model="${arg#*=}" ;;
         --xnnpack) xnnpack=true ;;
         --quantize) quantize=true ;;
-        --verbose) verbose=true ;;
+        --debug-xnnpack) debug_xnnpack=true ;;
+        --verbose-xnnpack) verbose_xnnpack=true ;;
         -h|--help) usage; exit 0 ;;
         *) echo "Unknown option: $arg" >&2; usage; exit 1 ;;
     esac
@@ -48,8 +52,11 @@ fi
 if ${quantize}; then
     run_extra_args+=(--quantize)
 fi
-if ${verbose}; then
-    run_extra_args+=(--verbose)
+if ${debug_xnnpack}; then
+    run_extra_args+=(--debug-xnnpack)
+fi
+if ${verbose_xnnpack}; then
+    run_extra_args+=(--verbose-xnnpack)
 fi
 
 bash "${et_root_dir}/examples/riscv/setup.sh"

.github/workflows/_test_riscv.yml

@@ -27,10 +27,10 @@ on:
         required: false
         type: boolean
         default: false
-      gcc-version:
-        description: 'The version of GCC to use'
-        required: false
-        type: number
+      qemu-cpu:
+        description: 'Configuration(s) for the CPU to emulate with QEMU, expecting a JSON array'
+        required: true
+        type: string
       docker-image:
         description: 'The docker image to use for this job'
         required: false
@@ -41,7 +41,7 @@ jobs:
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     with:
       runner: linux.2xlarge
-      docker-image: ${{ inputs.docker-image || 'ci-image:executorch-ubuntu-22.04-gcc11' }}
+      docker-image: ci-image:executorch-ubuntu-24.04-gcc14
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: ${{ inputs.timeout }}
@@ -52,5 +52,37 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch "--use-pt-pinned-commit"
 
-        export GCC_VERSION=${{ inputs.gcc-version }}
-        bash .ci/scripts/test_riscv_qemu.sh --model="${{ inputs.model }}" ${{ inputs.xnnpack && '--xnnpack' || '' }} ${{ inputs.quantize && '--quantize' || '' }}
+        # Allows failure in `echo | jq | while read` pipeline to bubble up and fail the workflow
+        set -o pipefail
+
+        echo '${{ inputs.qemu-cpu }}' | jq -r '.[]' | while IFS= read -r qemu_cpu; do
+          export QEMU_CPU="${qemu_cpu}"
+          export GCC_VERSION=14
+          bash .ci/scripts/test_riscv_qemu.sh \
+            --model="${{ inputs.model }}" \
+            ${{ inputs.xnnpack && '--xnnpack --verbose-xnnpack' || '' }} \
+            ${{ inputs.quantize && '--quantize' || '' }}
+
+          # Generate markdown table from riscv_test/${{ inputs.model }}_riscv.etdump.json, sorted by sum_ms
+          (
+            etdump_json="riscv_test/${{ inputs.model }}_riscv.etdump.json"
+            echo "### Model=${{ inputs.model }} XNNPACK=${{ inputs.xnnpack }} Quantize=${{ inputs.quantize }} QEMU_CPU='${QEMU_CPU}'"
+            jq -r '
+              def r3: (. * 1000 | round) / 1000;
+              ["Section","Op","Count","Sum (ms)","Avg (ms)","Max (ms)","Microkernels"],
+              ["---","---","---","---","---","---","---"],
+              ( [ (.ops[]       | . + {section: "ops"}),
+                  (.framework[] | . + {section: "framework"}) ]
+                | sort_by(-.sum_ms) | .[]
+                | [.section, .op, .count, (.sum_ms|r3), (.avg_ms|r3), (.max_ms|r3), ((.kernels // []) | join(", "))] )
+              | "| " + (map(tostring) | join(" | ")) + " |"
+            ' "${etdump_json}"
+            echo
+            echo "<details><summary>Registered XNNPACK microkernels</summary>"
+            echo
+            jq -r '.registered_kernels[] | "- `" + . + "`"' "${etdump_json}"
+            echo
+            echo "</details>"
+            echo
+          ) >> $GITHUB_STEP_SUMMARY
+        done

.github/workflows/riscv64.yml

@@ -1,4 +1,4 @@
-name: RISC-V
+name: Test RISC-V Backend
 
 on:
   push:
@@ -50,6 +50,17 @@ jobs:
       model: ${{ matrix.model }}
       xnnpack: ${{ matrix.xnnpack }}
       quantize: ${{ matrix.quantize }}
-      # XNNPACK requires GCC 14+
-      gcc-version: ${{ matrix.xnnpack && 14 || 11 }}
-      docker-image: ${{ matrix.xnnpack && 'ci-image:executorch-ubuntu-24.04-gcc14' || 'ci-image:executorch-ubuntu-22.04-gcc11' }}
+      # If XNNPACK, test with multiple RVV length, disabled otherwise
+      qemu-cpu: >-
+        ${{
+          case(
+            matrix.xnnpack, '[
+              "rv64,zba=true,zbb=true,zbs=true,v=true,vlen=128,elen=64,vext_spec=v1.0",
+              "rv64,zba=true,zbb=true,zbs=true,v=true,vlen=256,elen=64,vext_spec=v1.0",
+              "rv64,zba=true,zbb=true,zbs=true,v=true,vlen=512,elen=64,vext_spec=v1.0"
+            ]',
+            '[
+              "rv64,zba=true,zbb=true,zbs=true,v=false"
+            ]'
+          )
+        }}

backends/xnnpack/CMakeLists.txt

@@ -169,6 +169,12 @@ install(
   EXPORT ExecuTorchTargets
   DESTINATION ${CMAKE_INSTALL_LIBDIR}
 )
+if(DEFINED EXECUTORCH_XNNPACK_LOG_LEVEL)
+  target_compile_definitions(
+    xnnpack-logging PUBLIC XNN_LOG_LEVEL=${EXECUTORCH_XNNPACK_LOG_LEVEL}
+  )
+endif()
+
 if(BUILD_TESTING)
   add_subdirectory(test)
 endif()

examples/riscv/aot_riscv.py

@@ -148,13 +148,13 @@ def main() -> None:
         help="Produce an 8-bit quantized model",
     )
     parser.add_argument(
-        "--verbose",
+        "--debug-xnnpack",
         action="store_true",
         help="Enable XNNPACK partitioner DEBUG logging and dump the lowered graph",
     )
     args = parser.parse_args()
 
-    if args.verbose:
+    if args.debug_xnnpack:
         logging.basicConfig(level=logging.DEBUG)
 
     if args.output is None:
@@ -181,7 +181,7 @@ def main() -> None:
             XnnpackPartitioner,
         )
 
-        partitioners.append(XnnpackPartitioner(verbose=args.verbose))
+        partitioners.append(XnnpackPartitioner(verbose=args.debug_xnnpack))
 
     compile_config = None
     if args.quantize:
@@ -202,7 +202,7 @@ def main() -> None:
         f"quantize={args.quantize} delegated_nodes={delegated}"
     )
 
-    if args.verbose:
+    if args.debug_xnnpack:
         from executorch.exir.backend.utils import print_delegated_graph
 
         print_delegated_graph(edge.exported_program().graph_module)