From 4d63e0085c19fe84abcdcff0060cbc7960b02fcb Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 10:23:11 -0600
Subject: [PATCH 01/17] test

---
 .github/workflows/pr_build.yml       | 552 ++++++++++++++-------------
 native/core/src/execution/planner.rs |   5 +
 2 files changed, 283 insertions(+), 274 deletions(-)

diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
index 1ae94f2741..207ce0a0d4 100644
--- a/.github/workflows/pr_build.yml
+++ b/.github/workflows/pr_build.yml
@@ -41,113 +41,113 @@ env:
 
 jobs:
 
-  # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release
-  linux-test:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        java_version: [11, 17]
-        test-target: [rust, java]
-        spark-version: ['3.5']
-        scala-version: ['2.12', '2.13']
-        is_push_event:
-          - ${{ github.event_name == 'push' }}
-        exclude: # exclude java 11 for pull_request event
-          - java_version: 11
-            is_push_event: false
-      fail-fast: false
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-    runs-on: ${{ matrix.os }}
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-      - if: matrix.test-target == 'rust'
-        name: Rust test steps
-        uses: ./.github/actions/rust-test
-      - if: matrix.test-target == 'java'
-        name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-          # upload test reports only for java 17
-          upload-test-reports: ${{ matrix.java_version == '17' }}
-
-  # run Comet tests for older 3.5.x releases with one java and scala versions
-  linux-test-35-minor-versions:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        java_version: [17]
-        test-target: [java]
-        spark-version: ['3.5']
-        spark-patch-version: ['3.5.2', '3.5.3', '3.5.4']
-        scala-version: ['2.12']
-        is_push_event:
-          - ${{ github.event_name == 'push' }}
-      fail-fast: false
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-    runs-on: ${{ matrix.os }}
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-      - if: matrix.test-target == 'rust'
-        name: Rust test steps
-        uses: ./.github/actions/rust-test
-      - if: matrix.test-target == 'java'
-        name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}"
-          # upload test reports only for java 17
-          upload-test-reports: ${{ matrix.java_version == '17' }}
-
-  linux-test-native-datafusion-scan:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        scan_impl: ['native_datafusion', 'native_iceberg_compat']
-        java_version: [17]
-        test-target: [rust, java]
-        spark-version: ['3.5']
-        scala-version: ['2.12']
-        is_push_event:
-          - ${{ github.event_name == 'push' }}
-      fail-fast: false
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }}
-    runs-on: ${{ matrix.os }}
-    env:
-      COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }}
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-      - if: matrix.test-target == 'rust'
-        name: Rust test steps
-        uses: ./.github/actions/rust-test
-      - if: matrix.test-target == 'java'
-        name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-          # upload test reports only for java 17
-          upload-test-reports: ${{ matrix.java_version == '17' }}
+#  # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release
+#  linux-test:
+#    strategy:
+#      matrix:
+#        os: [ubuntu-latest]
+#        java_version: [11, 17]
+#        test-target: [rust, java]
+#        spark-version: ['3.5']
+#        scala-version: ['2.12', '2.13']
+#        is_push_event:
+#          - ${{ github.event_name == 'push' }}
+#        exclude: # exclude java 11 for pull_request event
+#          - java_version: 11
+#            is_push_event: false
+#      fail-fast: false
+#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
+#    runs-on: ${{ matrix.os }}
+#    container:
+#      image: amd64/rust
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#      - if: matrix.test-target == 'rust'
+#        name: Rust test steps
+#        uses: ./.github/actions/rust-test
+#      - if: matrix.test-target == 'java'
+#        name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
+#          # upload test reports only for java 17
+#          upload-test-reports: ${{ matrix.java_version == '17' }}
+#
+#  # run Comet tests for older 3.5.x releases with one java and scala versions
+#  linux-test-35-minor-versions:
+#    strategy:
+#      matrix:
+#        os: [ubuntu-latest]
+#        java_version: [17]
+#        test-target: [java]
+#        spark-version: ['3.5']
+#        spark-patch-version: ['3.5.2', '3.5.3', '3.5.4']
+#        scala-version: ['2.12']
+#        is_push_event:
+#          - ${{ github.event_name == 'push' }}
+#      fail-fast: false
+#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
+#    runs-on: ${{ matrix.os }}
+#    container:
+#      image: amd64/rust
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#      - if: matrix.test-target == 'rust'
+#        name: Rust test steps
+#        uses: ./.github/actions/rust-test
+#      - if: matrix.test-target == 'java'
+#        name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}"
+#          # upload test reports only for java 17
+#          upload-test-reports: ${{ matrix.java_version == '17' }}
+#
+#  linux-test-native-datafusion-scan:
+#    strategy:
+#      matrix:
+#        os: [ubuntu-latest]
+#        scan_impl: ['native_datafusion', 'native_iceberg_compat']
+#        java_version: [17]
+#        test-target: [rust, java]
+#        spark-version: ['3.5']
+#        scala-version: ['2.12']
+#        is_push_event:
+#          - ${{ github.event_name == 'push' }}
+#      fail-fast: false
+#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }}
+#    runs-on: ${{ matrix.os }}
+#    env:
+#      COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }}
+#    container:
+#      image: amd64/rust
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#      - if: matrix.test-target == 'rust'
+#        name: Rust test steps
+#        uses: ./.github/actions/rust-test
+#      - if: matrix.test-target == 'java'
+#        name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
+#          # upload test reports only for java 17
+#          upload-test-reports: ${{ matrix.java_version == '17' }}
 
   linux-test-with-spark4_0:
     strategy:
@@ -156,6 +156,10 @@ jobs:
         java_version: [17]
         test-target: [java]
         spark-version: ['4.0']
+        suites: ['org.apache.comet.CometFuzzTestSuite',
+                 'org.apache.comet.exec.CometShuffleSuite,org.apache.comet.exec.CometAsyncShuffleSuite,org.apache.comet.exec.DisableAQECometShuffleSuite,org.apache.comet.exec.DisableAQECometAsyncShuffleSuite',
+                 'org.apache.comet.parquet.ParquetReadV1Suite,org.apache.comet.parquet.ParquetReadV2Suite',
+                 'org.apache.comet.exec.CometExecSuite,org.apache.comet.CometExpressionSuite']
         is_push_event:
           - ${{ github.event_name == 'push' }}
       fail-fast: false
@@ -173,172 +177,172 @@ jobs:
       - name: Java test steps
         uses: ./.github/actions/java-test
         with:
-          maven_opts: -Pspark-${{ matrix.spark-version }}
+          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=${{ matrix.suites }}"
           upload-test-reports: true
 
-  linux-test-with-old-spark:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        java_version: [11, 17]
-        test-target: [java]
-        spark-version: ['3.4']
-        scala-version: ['2.12', '2.13']
-      fail-fast: false
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-    runs-on: ${{ matrix.os }}
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-      - name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-
-  macos-test:
-    strategy:
-      matrix:
-        os: [macos-13]
-        java_version: [11, 17]
-        test-target: [rust, java]
-        spark-version: ['3.4', '3.5']
-        scala-version: ['2.12', '2.13']
-      fail-fast: false
-    if: github.event_name == 'push'
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-macos-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-      - if: matrix.test-target == 'rust'
-        name: Rust test steps
-        uses: ./.github/actions/rust-test
-      - if: matrix.test-target == 'java'
-        name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-
-  macos-aarch64-test:
-    strategy:
-      matrix:
-        java_version: [11, 17]
-        test-target: [rust, java]
-        spark-version: ['3.4', '3.5']
-        scala-version: ['2.12', '2.13']
-        is_push_event:
-          - ${{ github.event_name == 'push' }}
-        exclude: # exclude java 11 for pull_request event
-          - java_version: 11
-            is_push_event: false
-      fail-fast: false
-    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-    runs-on: macos-14
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-macos-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-          jdk-architecture: aarch64
-          protoc-architecture: aarch_64
-      - if: matrix.test-target == 'rust'
-        name: Rust test steps
-        uses: ./.github/actions/rust-test
-      - if: matrix.test-target == 'java'
-        name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-
-  macos-test-with-spark4_0:
-    strategy:
-      matrix:
-        os: [macos-13]
-        java_version: [17]
-        test-target: [java]
-        spark-version: ['4.0']
-      fail-fast: false
-    if: github.event_name == 'push'
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-macos-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-      - name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }}
-          upload-test-reports: true
-
-  macos-aarch64-test-with-spark4_0:
-    strategy:
-      matrix:
-        java_version: [17]
-        test-target: [java]
-        spark-version: ['4.0']
-        is_push_event:
-          - ${{ github.event_name == 'push' }}
-        exclude: # exclude java 11 for pull_request event
-          - java_version: 11
-            is_push_event: false
-      fail-fast: false
-    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
-    runs-on: macos-14
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-macos-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-          jdk-architecture: aarch64
-          protoc-architecture: aarch_64
-      - name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }}
-          upload-test-reports: true
-
-  macos-aarch64-test-with-old-spark:
-    strategy:
-      matrix:
-        java_version: [17]
-        test-target: [java]
-        spark-version: ['3.4']
-        scala-version: ['2.12', '2.13']
-        exclude:
-          - java_version: 8
-      fail-fast: false
-    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-    runs-on: macos-14
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-macos-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-          jdk-architecture: aarch64
-          protoc-architecture: aarch_64
-      - if: matrix.test-target == 'java'
-        name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-
+#  linux-test-with-old-spark:
+#    strategy:
+#      matrix:
+#        os: [ubuntu-latest]
+#        java_version: [11, 17]
+#        test-target: [java]
+#        spark-version: ['3.4']
+#        scala-version: ['2.12', '2.13']
+#      fail-fast: false
+#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
+#    runs-on: ${{ matrix.os }}
+#    container:
+#      image: amd64/rust
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#      - name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
+#
+#  macos-test:
+#    strategy:
+#      matrix:
+#        os: [macos-13]
+#        java_version: [11, 17]
+#        test-target: [rust, java]
+#        spark-version: ['3.4', '3.5']
+#        scala-version: ['2.12', '2.13']
+#      fail-fast: false
+#    if: github.event_name == 'push'
+#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
+#    runs-on: ${{ matrix.os }}
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-macos-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#      - if: matrix.test-target == 'rust'
+#        name: Rust test steps
+#        uses: ./.github/actions/rust-test
+#      - if: matrix.test-target == 'java'
+#        name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
+#
+#  macos-aarch64-test:
+#    strategy:
+#      matrix:
+#        java_version: [11, 17]
+#        test-target: [rust, java]
+#        spark-version: ['3.4', '3.5']
+#        scala-version: ['2.12', '2.13']
+#        is_push_event:
+#          - ${{ github.event_name == 'push' }}
+#        exclude: # exclude java 11 for pull_request event
+#          - java_version: 11
+#            is_push_event: false
+#      fail-fast: false
+#    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
+#    runs-on: macos-14
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-macos-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#          jdk-architecture: aarch64
+#          protoc-architecture: aarch_64
+#      - if: matrix.test-target == 'rust'
+#        name: Rust test steps
+#        uses: ./.github/actions/rust-test
+#      - if: matrix.test-target == 'java'
+#        name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
+#
+#  macos-test-with-spark4_0:
+#    strategy:
+#      matrix:
+#        os: [macos-13]
+#        java_version: [17]
+#        test-target: [java]
+#        spark-version: ['4.0']
+#      fail-fast: false
+#    if: github.event_name == 'push'
+#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
+#    runs-on: ${{ matrix.os }}
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-macos-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#      - name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: -Pspark-${{ matrix.spark-version }}
+#          upload-test-reports: true
+#
+#  macos-aarch64-test-with-spark4_0:
+#    strategy:
+#      matrix:
+#        java_version: [17]
+#        test-target: [java]
+#        spark-version: ['4.0']
+#        is_push_event:
+#          - ${{ github.event_name == 'push' }}
+#        exclude: # exclude java 11 for pull_request event
+#          - java_version: 11
+#            is_push_event: false
+#      fail-fast: false
+#    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
+#    runs-on: macos-14
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-macos-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#          jdk-architecture: aarch64
+#          protoc-architecture: aarch_64
+#      - name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: -Pspark-${{ matrix.spark-version }}
+#          upload-test-reports: true
+#
+#  macos-aarch64-test-with-old-spark:
+#    strategy:
+#      matrix:
+#        java_version: [17]
+#        test-target: [java]
+#        spark-version: ['3.4']
+#        scala-version: ['2.12', '2.13']
+#        exclude:
+#          - java_version: 8
+#      fail-fast: false
+#    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
+#    runs-on: macos-14
+#    steps:
+#      - uses: actions/checkout@v4
+#      - name: Setup Rust & Java toolchain
+#        uses: ./.github/actions/setup-macos-builder
+#        with:
+#          rust-version: ${{env.RUST_VERSION}}
+#          jdk-version: ${{ matrix.java_version }}
+#          jdk-architecture: aarch64
+#          protoc-architecture: aarch_64
+#      - if: matrix.test-target == 'java'
+#        name: Java test steps
+#        uses: ./.github/actions/java-test
+#        with:
+#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
+#
diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs
index 60587a6fb4..877245c358 100644
--- a/native/core/src/execution/planner.rs
+++ b/native/core/src/execution/planner.rs
@@ -956,6 +956,11 @@ impl PhysicalPlanner {
                     self.create_expr(filter.predicate.as_ref().unwrap(), child.schema())?;
 
                 let filter: Arc<dyn ExecutionPlan> = if filter.use_datafusion_filter {
+                    assert_eq!(
+                        scans.len(),
+                        0,
+                        "Must not use ScanExec as it may requires a copy"
+                    );
                     Arc::new(DataFusionFilterExec::try_new(
                         predicate,
                         Arc::clone(&child.native_plan),

From dd92518cc781bcb853ec7581d29dda1333e35702 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 10:30:09 -0600
Subject: [PATCH 02/17] test

---
 .github/workflows/pr_build.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
index 207ce0a0d4..8cacdaac78 100644
--- a/.github/workflows/pr_build.yml
+++ b/.github/workflows/pr_build.yml
@@ -156,14 +156,15 @@ jobs:
         java_version: [17]
         test-target: [java]
         spark-version: ['4.0']
-        suites: ['org.apache.comet.CometFuzzTestSuite',
-                 'org.apache.comet.exec.CometShuffleSuite,org.apache.comet.exec.CometAsyncShuffleSuite,org.apache.comet.exec.DisableAQECometShuffleSuite,org.apache.comet.exec.DisableAQECometAsyncShuffleSuite',
-                 'org.apache.comet.parquet.ParquetReadV1Suite,org.apache.comet.parquet.ParquetReadV2Suite',
-                 'org.apache.comet.exec.CometExecSuite,org.apache.comet.CometExpressionSuite']
+        suites:
+          - { name: "fuzz", value: "org.apache.comet.CometFuzzTestSuite" }
+          - { name: "shuffle", value: "org.apache.comet.exec.CometShuffleSuite,org.apache.comet.exec.CometAsyncShuffleSuite,org.apache.comet.exec.DisableAQECometShuffleSuite,org.apache.comet.exec.DisableAQECometAsyncShuffleSuite" }
+          - { name: "parquet", value: "org.apache.comet.parquet.ParquetReadV1Suite,org.apache.comet.parquet.ParquetReadV2Suite" }
+          - { name: "exec", value: "org.apache.comet.exec.CometExecSuite,org.apache.comet.CometExpressionSuite" }
         is_push_event:
           - ${{ github.event_name == 'push' }}
       fail-fast: false
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
+    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
     runs-on: ${{ matrix.os }}
     container:
       image: amd64/rust
@@ -177,7 +178,7 @@ jobs:
       - name: Java test steps
         uses: ./.github/actions/java-test
         with:
-          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=${{ matrix.suites }}"
+          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=${{ matrix.suites.value }}"
           upload-test-reports: true
 
 #  linux-test-with-old-spark:

From 440ca9159c65abcf9a8d119ea0731138b1fcd4c5 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 11:03:39 -0600
Subject: [PATCH 03/17] revert

---
 native/core/src/execution/planner.rs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs
index 877245c358..60587a6fb4 100644
--- a/native/core/src/execution/planner.rs
+++ b/native/core/src/execution/planner.rs
@@ -956,11 +956,6 @@ impl PhysicalPlanner {
                     self.create_expr(filter.predicate.as_ref().unwrap(), child.schema())?;
 
                 let filter: Arc<dyn ExecutionPlan> = if filter.use_datafusion_filter {
-                    assert_eq!(
-                        scans.len(),
-                        0,
-                        "Must not use ScanExec as it may requires a copy"
-                    );
                     Arc::new(DataFusionFilterExec::try_new(
                         predicate,
                         Arc::clone(&child.native_plan),

From 165ab14fef198c270a829fb40c789e1e1984ac58 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 12:36:28 -0600
Subject: [PATCH 04/17] test

---
 .github/workflows/pr_build.yml     | 349 -----------------------------
 .github/workflows/pr_build_new.yml | 108 +++++++++
 2 files changed, 108 insertions(+), 349 deletions(-)
 delete mode 100644 .github/workflows/pr_build.yml
 create mode 100644 .github/workflows/pr_build_new.yml

diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml
deleted file mode 100644
index 8cacdaac78..0000000000
--- a/.github/workflows/pr_build.yml
+++ /dev/null
@@ -1,349 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: PR Build
-
-concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
-  cancel-in-progress: true
-
-on:
-  push:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-  pull_request:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-  # manual trigger
-  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
-  workflow_dispatch:
-
-env:
-  RUST_VERSION: stable
-
-jobs:
-
-#  # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release
-#  linux-test:
-#    strategy:
-#      matrix:
-#        os: [ubuntu-latest]
-#        java_version: [11, 17]
-#        test-target: [rust, java]
-#        spark-version: ['3.5']
-#        scala-version: ['2.12', '2.13']
-#        is_push_event:
-#          - ${{ github.event_name == 'push' }}
-#        exclude: # exclude java 11 for pull_request event
-#          - java_version: 11
-#            is_push_event: false
-#      fail-fast: false
-#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-#    runs-on: ${{ matrix.os }}
-#    container:
-#      image: amd64/rust
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#      - if: matrix.test-target == 'rust'
-#        name: Rust test steps
-#        uses: ./.github/actions/rust-test
-#      - if: matrix.test-target == 'java'
-#        name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-#          # upload test reports only for java 17
-#          upload-test-reports: ${{ matrix.java_version == '17' }}
-#
-#  # run Comet tests for older 3.5.x releases with one java and scala versions
-#  linux-test-35-minor-versions:
-#    strategy:
-#      matrix:
-#        os: [ubuntu-latest]
-#        java_version: [17]
-#        test-target: [java]
-#        spark-version: ['3.5']
-#        spark-patch-version: ['3.5.2', '3.5.3', '3.5.4']
-#        scala-version: ['2.12']
-#        is_push_event:
-#          - ${{ github.event_name == 'push' }}
-#      fail-fast: false
-#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-#    runs-on: ${{ matrix.os }}
-#    container:
-#      image: amd64/rust
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#      - if: matrix.test-target == 'rust'
-#        name: Rust test steps
-#        uses: ./.github/actions/rust-test
-#      - if: matrix.test-target == 'java'
-#        name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}"
-#          # upload test reports only for java 17
-#          upload-test-reports: ${{ matrix.java_version == '17' }}
-#
-#  linux-test-native-datafusion-scan:
-#    strategy:
-#      matrix:
-#        os: [ubuntu-latest]
-#        scan_impl: ['native_datafusion', 'native_iceberg_compat']
-#        java_version: [17]
-#        test-target: [rust, java]
-#        spark-version: ['3.5']
-#        scala-version: ['2.12']
-#        is_push_event:
-#          - ${{ github.event_name == 'push' }}
-#      fail-fast: false
-#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }}
-#    runs-on: ${{ matrix.os }}
-#    env:
-#      COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }}
-#    container:
-#      image: amd64/rust
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#      - if: matrix.test-target == 'rust'
-#        name: Rust test steps
-#        uses: ./.github/actions/rust-test
-#      - if: matrix.test-target == 'java'
-#        name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-#          # upload test reports only for java 17
-#          upload-test-reports: ${{ matrix.java_version == '17' }}
-
-  linux-test-with-spark4_0:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        java_version: [17]
-        test-target: [java]
-        spark-version: ['4.0']
-        suites:
-          - { name: "fuzz", value: "org.apache.comet.CometFuzzTestSuite" }
-          - { name: "shuffle", value: "org.apache.comet.exec.CometShuffleSuite,org.apache.comet.exec.CometAsyncShuffleSuite,org.apache.comet.exec.DisableAQECometShuffleSuite,org.apache.comet.exec.DisableAQECometAsyncShuffleSuite" }
-          - { name: "parquet", value: "org.apache.comet.parquet.ParquetReadV1Suite,org.apache.comet.parquet.ParquetReadV2Suite" }
-          - { name: "exec", value: "org.apache.comet.exec.CometExecSuite,org.apache.comet.CometExpressionSuite" }
-        is_push_event:
-          - ${{ github.event_name == 'push' }}
-      fail-fast: false
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
-    runs-on: ${{ matrix.os }}
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-      - name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=${{ matrix.suites.value }}"
-          upload-test-reports: true
-
-#  linux-test-with-old-spark:
-#    strategy:
-#      matrix:
-#        os: [ubuntu-latest]
-#        java_version: [11, 17]
-#        test-target: [java]
-#        spark-version: ['3.4']
-#        scala-version: ['2.12', '2.13']
-#      fail-fast: false
-#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-#    runs-on: ${{ matrix.os }}
-#    container:
-#      image: amd64/rust
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#      - name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-#
-#  macos-test:
-#    strategy:
-#      matrix:
-#        os: [macos-13]
-#        java_version: [11, 17]
-#        test-target: [rust, java]
-#        spark-version: ['3.4', '3.5']
-#        scala-version: ['2.12', '2.13']
-#      fail-fast: false
-#    if: github.event_name == 'push'
-#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-#    runs-on: ${{ matrix.os }}
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-macos-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#      - if: matrix.test-target == 'rust'
-#        name: Rust test steps
-#        uses: ./.github/actions/rust-test
-#      - if: matrix.test-target == 'java'
-#        name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-#
-#  macos-aarch64-test:
-#    strategy:
-#      matrix:
-#        java_version: [11, 17]
-#        test-target: [rust, java]
-#        spark-version: ['3.4', '3.5']
-#        scala-version: ['2.12', '2.13']
-#        is_push_event:
-#          - ${{ github.event_name == 'push' }}
-#        exclude: # exclude java 11 for pull_request event
-#          - java_version: 11
-#            is_push_event: false
-#      fail-fast: false
-#    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-#    runs-on: macos-14
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-macos-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#          jdk-architecture: aarch64
-#          protoc-architecture: aarch_64
-#      - if: matrix.test-target == 'rust'
-#        name: Rust test steps
-#        uses: ./.github/actions/rust-test
-#      - if: matrix.test-target == 'java'
-#        name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-#
-#  macos-test-with-spark4_0:
-#    strategy:
-#      matrix:
-#        os: [macos-13]
-#        java_version: [17]
-#        test-target: [java]
-#        spark-version: ['4.0']
-#      fail-fast: false
-#    if: github.event_name == 'push'
-#    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
-#    runs-on: ${{ matrix.os }}
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-macos-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#      - name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: -Pspark-${{ matrix.spark-version }}
-#          upload-test-reports: true
-#
-#  macos-aarch64-test-with-spark4_0:
-#    strategy:
-#      matrix:
-#        java_version: [17]
-#        test-target: [java]
-#        spark-version: ['4.0']
-#        is_push_event:
-#          - ${{ github.event_name == 'push' }}
-#        exclude: # exclude java 11 for pull_request event
-#          - java_version: 11
-#            is_push_event: false
-#      fail-fast: false
-#    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
-#    runs-on: macos-14
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-macos-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#          jdk-architecture: aarch64
-#          protoc-architecture: aarch_64
-#      - name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: -Pspark-${{ matrix.spark-version }}
-#          upload-test-reports: true
-#
-#  macos-aarch64-test-with-old-spark:
-#    strategy:
-#      matrix:
-#        java_version: [17]
-#        test-target: [java]
-#        spark-version: ['3.4']
-#        scala-version: ['2.12', '2.13']
-#        exclude:
-#          - java_version: 8
-#      fail-fast: false
-#    name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}
-#    runs-on: macos-14
-#    steps:
-#      - uses: actions/checkout@v4
-#      - name: Setup Rust & Java toolchain
-#        uses: ./.github/actions/setup-macos-builder
-#        with:
-#          rust-version: ${{env.RUST_VERSION}}
-#          jdk-version: ${{ matrix.java_version }}
-#          jdk-architecture: aarch64
-#          protoc-architecture: aarch_64
-#      - if: matrix.test-target == 'java'
-#        name: Java test steps
-#        uses: ./.github/actions/java-test
-#        with:
-#          maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }}
-#
diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
new file mode 100644
index 0000000000..99d47d32e1
--- /dev/null
+++ b/.github/workflows/pr_build_new.yml
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PR Build
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  push:
+    paths-ignore:
+      - "doc/**"
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    paths-ignore:
+      - "doc/**"
+      - "docs/**"
+      - "**.md"
+  # manual trigger
+  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
+  workflow_dispatch:
+
+env:
+  RUST_VERSION: stable
+
+jobs:
+
+  linux-test-with-spark4_0:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        java_version: [17]
+        test-target: [java]
+        spark-version: ['4.0']
+    suites:
+      - name: "fuzz"
+        value: |
+          org.apache.comet.CometFuzzTestSuite
+          org.apache.comet.DataGeneratorSuite
+      - name: "shuffle"
+        value: |
+          org.apache.comet.exec.CometShuffleSuite
+          org.apache.comet.exec.CometShuffle4_0Suite
+          org.apache.comet.exec.CometNativeShuffleSuite
+          org.apache.comet.exec.CometShuffleEncryptionSuite
+          org.apache.comet.exec.CometShuffleManagerSuite
+          org.apache.comet.exec.CometAsyncShuffleSuite
+          org.apache.comet.exec.DisableAQECometShuffleSuite
+          org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
+      - name: "parquet"
+        value: |
+          org.apache.comet.parquet.ParquetReadV1Suite
+          org.apache.comet.parquet.ParquetReadV2Suite
+          org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
+          org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
+          org.apache.spark.sql.comet.ParquetEncryptionITCase
+          org.apache.comet.exec.CometNativeReaderSuite
+      - name: "exec"
+        value: |
+          org.apache.comet.exec.CometExecSuite
+          org.apache.comet.exec.CometExec3_4PlusSuite
+          org.apache.comet.exec.CometJoinSuite
+          org.apache.comet.exec.CometAggregateSuite
+          org.apache.comet.CometArrayExpressionSuite
+          org.apache.comet.CometCastSuite
+          org.apache.comet.CometExpressionSuite
+          org.apache.comet.CometExpressionCoverageSuite
+          org.apache.comet.CometNativeSuite
+          org.apache.comet.CometSparkSessionExtensionsSuite
+          org.apache.spark.CometPluginsSuite
+          org.apache.spark.sql.CometTPCDSQuerySuite
+          org.apache.spark.sql.CometTPCHQuerySuite
+          org.apache.spark.sql.comet.CometPlanStabilitySuite
+          org.apache.spark.sql.comet.CometTaskMetricsSuite
+
+    fail-fast: false
+    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
+    runs-on: ${{ matrix.os }}
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust & Java toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{env.RUST_VERSION}}
+          jdk-version: ${{ matrix.java_version }}
+      - name: Java test steps
+        uses: ./.github/actions/java-test
+        with:
+          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=${{ matrix.suites.value }}"
+          upload-test-reports: true

From 56dcdfb05bbf0032904c28db36b362fece7c60c9 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 12:40:17 -0600
Subject: [PATCH 05/17] test

---
 .github/workflows/pr_build_new.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
index 99d47d32e1..057649a9e2 100644
--- a/.github/workflows/pr_build_new.yml
+++ b/.github/workflows/pr_build_new.yml
@@ -104,5 +104,5 @@ jobs:
       - name: Java test steps
         uses: ./.github/actions/java-test
         with:
-          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=${{ matrix.suites.value }}"
+          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=$(echo "${{ matrix.suites.value }}" | paste -sd, -)"
           upload-test-reports: true

From ac79880390d707a43a7d67a40da53b0c2eb536ad Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 12:41:42 -0600
Subject: [PATCH 06/17] test

---
 .github/workflows/spark_sql_test.yml      | 20 ++++++++++----------
 .github/workflows/spark_sql_test_ansi.yml | 16 ++++++++--------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml
index 58defe6608..cb917062bb 100644
--- a/.github/workflows/spark_sql_test.yml
+++ b/.github/workflows/spark_sql_test.yml
@@ -22,16 +22,16 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  push:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-  pull_request:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
+#  push:
+#    paths-ignore:
+#      - "doc/**"
+#      - "docs/**"
+#      - "**.md"
+#  pull_request:
+#    paths-ignore:
+#      - "doc/**"
+#      - "docs/**"
+#      - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
diff --git a/.github/workflows/spark_sql_test_ansi.yml b/.github/workflows/spark_sql_test_ansi.yml
index 170c10644b..466d1f4eef 100644
--- a/.github/workflows/spark_sql_test_ansi.yml
+++ b/.github/workflows/spark_sql_test_ansi.yml
@@ -22,14 +22,14 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  push:
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-  pull_request:
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
+#  push:
+#    paths-ignore:
+#      - "docs/**"
+#      - "**.md"
+#  pull_request:
+#    paths-ignore:
+#      - "docs/**"
+#      - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:

From fab039162373b3231a1caa6c095a0cc1ba3c7aee Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 12:53:51 -0600
Subject: [PATCH 07/17] test

---
 .github/actions/java-test/action.yaml |  1 +
 .github/workflows/benchmark-tpch.yml  | 20 ++++++++++----------
 .github/workflows/miri.yml            | 20 ++++++++++----------
 3 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
index 53775da0df..0218259ec3 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -54,6 +54,7 @@ runs:
     - name: Run tests
       shell: bash
       run: |
+        echo "Running tests with ${{ inputs.maven_opts }}" 
         MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }}
     - name: Upload crash logs
       if: failure()
diff --git a/.github/workflows/benchmark-tpch.yml b/.github/workflows/benchmark-tpch.yml
index 3d4fbb750a..9cb5bc6512 100644
--- a/.github/workflows/benchmark-tpch.yml
+++ b/.github/workflows/benchmark-tpch.yml
@@ -22,16 +22,16 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  push:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-  pull_request:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
+#  push:
+#    paths-ignore:
+#      - "doc/**"
+#      - "docs/**"
+#      - "**.md"
+#  pull_request:
+#    paths-ignore:
+#      - "doc/**"
+#      - "docs/**"
+#      - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml
index 923598ee3b..aef91951dd 100644
--- a/.github/workflows/miri.yml
+++ b/.github/workflows/miri.yml
@@ -18,16 +18,16 @@
 name: Run Miri Safety Checks
 
 on:
-  push:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-  pull_request:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
+#  push:
+#    paths-ignore:
+#      - "doc/**"
+#      - "docs/**"
+#      - "**.md"
+#  pull_request:
+#    paths-ignore:
+#      - "doc/**"
+#      - "docs/**"
+#      - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:

From 64e3ab9f6485a92980b497d40cb33a68faecd894 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 12:56:27 -0600
Subject: [PATCH 08/17] test

---
 .github/workflows/pr_build_new.yml | 42 ++++--------------------------
 1 file changed, 5 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
index 057649a9e2..091bf139c5 100644
--- a/.github/workflows/pr_build_new.yml
+++ b/.github/workflows/pr_build_new.yml
@@ -50,45 +50,13 @@ jobs:
         spark-version: ['4.0']
     suites:
       - name: "fuzz"
-        value: |
-          org.apache.comet.CometFuzzTestSuite
-          org.apache.comet.DataGeneratorSuite
+        value: "org.apache.comet.CometFuzzTestSuite,org.apache.comet.DataGeneratorSuite"
       - name: "shuffle"
-        value: |
-          org.apache.comet.exec.CometShuffleSuite
-          org.apache.comet.exec.CometShuffle4_0Suite
-          org.apache.comet.exec.CometNativeShuffleSuite
-          org.apache.comet.exec.CometShuffleEncryptionSuite
-          org.apache.comet.exec.CometShuffleManagerSuite
-          org.apache.comet.exec.CometAsyncShuffleSuite
-          org.apache.comet.exec.DisableAQECometShuffleSuite
-          org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
+        value: "org.apache.comet.exec.CometShuffleSuite,org.apache.comet.exec.CometShuffle4_0Suite,org.apache.comet.exec.CometNativeShuffleSuite,org.apache.comet.exec.CometShuffleEncryptionSuite,org.apache.comet.exec.CometShuffleManagerSuite,org.apache.comet.exec.CometAsyncShuffleSuite,org.apache.comet.exec.DisableAQECometShuffleSuite,org.apache.comet.exec.DisableAQECometAsyncShuffleSuite"
       - name: "parquet"
-        value: |
-          org.apache.comet.parquet.ParquetReadV1Suite
-          org.apache.comet.parquet.ParquetReadV2Suite
-          org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
-          org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
-          org.apache.spark.sql.comet.ParquetEncryptionITCase
-          org.apache.comet.exec.CometNativeReaderSuite
+        value: "org.apache.comet.parquet.ParquetReadV1Suite,org.apache.comet.parquet.ParquetReadV2Suite,org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite,org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite,org.apache.spark.sql.comet.ParquetEncryptionITCase,org.apache.comet.exec.CometNativeReaderSuite"
       - name: "exec"
-        value: |
-          org.apache.comet.exec.CometExecSuite
-          org.apache.comet.exec.CometExec3_4PlusSuite
-          org.apache.comet.exec.CometJoinSuite
-          org.apache.comet.exec.CometAggregateSuite
-          org.apache.comet.CometArrayExpressionSuite
-          org.apache.comet.CometCastSuite
-          org.apache.comet.CometExpressionSuite
-          org.apache.comet.CometExpressionCoverageSuite
-          org.apache.comet.CometNativeSuite
-          org.apache.comet.CometSparkSessionExtensionsSuite
-          org.apache.spark.CometPluginsSuite
-          org.apache.spark.sql.CometTPCDSQuerySuite
-          org.apache.spark.sql.CometTPCHQuerySuite
-          org.apache.spark.sql.comet.CometPlanStabilitySuite
-          org.apache.spark.sql.comet.CometTaskMetricsSuite
-
+        value: "org.apache.comet.exec.CometExecSuite,org.apache.comet.exec.CometExec3_4PlusSuite,org.apache.comet.exec.CometJoinSuite,org.apache.comet.exec.CometAggregateSuite,org.apache.comet.CometArrayExpressionSuite,org.apache.comet.CometCastSuite,org.apache.comet.CometExpressionSuite,org.apache.comet.CometExpressionCoverageSuite,org.apache.comet.CometNativeSuite,org.apache.comet.CometSparkSessionExtensionsSuite,org.apache.spark.CometPluginsSuite,org.apache.spark.sql.CometTPCDSQuerySuite,org.apache.spark.sql.CometTPCHQuerySuite,org.apache.spark.sql.comet.CometPlanStabilitySuite,org.apache.spark.sql.comet.CometTaskMetricsSuite"
     fail-fast: false
     name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
     runs-on: ${{ matrix.os }}
@@ -104,5 +72,5 @@ jobs:
       - name: Java test steps
         uses: ./.github/actions/java-test
         with:
-          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=$(echo "${{ matrix.suites.value }}" | paste -sd, -)"
+          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=${{ matrix.suites.value }}"
           upload-test-reports: true

From 014aa790c7f980d3342ded4a542b08bab8825169 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 12:57:40 -0600
Subject: [PATCH 09/17] Revert "test"

This reverts commit 64e3ab9f6485a92980b497d40cb33a68faecd894.
---
 .github/workflows/pr_build_new.yml | 42 ++++++++++++++++++++++++++----
 1 file changed, 37 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
index 091bf139c5..057649a9e2 100644
--- a/.github/workflows/pr_build_new.yml
+++ b/.github/workflows/pr_build_new.yml
@@ -50,13 +50,45 @@ jobs:
         spark-version: ['4.0']
     suites:
       - name: "fuzz"
-        value: "org.apache.comet.CometFuzzTestSuite,org.apache.comet.DataGeneratorSuite"
+        value: |
+          org.apache.comet.CometFuzzTestSuite
+          org.apache.comet.DataGeneratorSuite
       - name: "shuffle"
-        value: "org.apache.comet.exec.CometShuffleSuite,org.apache.comet.exec.CometShuffle4_0Suite,org.apache.comet.exec.CometNativeShuffleSuite,org.apache.comet.exec.CometShuffleEncryptionSuite,org.apache.comet.exec.CometShuffleManagerSuite,org.apache.comet.exec.CometAsyncShuffleSuite,org.apache.comet.exec.DisableAQECometShuffleSuite,org.apache.comet.exec.DisableAQECometAsyncShuffleSuite"
+        value: |
+          org.apache.comet.exec.CometShuffleSuite
+          org.apache.comet.exec.CometShuffle4_0Suite
+          org.apache.comet.exec.CometNativeShuffleSuite
+          org.apache.comet.exec.CometShuffleEncryptionSuite
+          org.apache.comet.exec.CometShuffleManagerSuite
+          org.apache.comet.exec.CometAsyncShuffleSuite
+          org.apache.comet.exec.DisableAQECometShuffleSuite
+          org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
       - name: "parquet"
-        value: "org.apache.comet.parquet.ParquetReadV1Suite,org.apache.comet.parquet.ParquetReadV2Suite,org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite,org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite,org.apache.spark.sql.comet.ParquetEncryptionITCase,org.apache.comet.exec.CometNativeReaderSuite"
+        value: |
+          org.apache.comet.parquet.ParquetReadV1Suite
+          org.apache.comet.parquet.ParquetReadV2Suite
+          org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
+          org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
+          org.apache.spark.sql.comet.ParquetEncryptionITCase
+          org.apache.comet.exec.CometNativeReaderSuite
       - name: "exec"
-        value: "org.apache.comet.exec.CometExecSuite,org.apache.comet.exec.CometExec3_4PlusSuite,org.apache.comet.exec.CometJoinSuite,org.apache.comet.exec.CometAggregateSuite,org.apache.comet.CometArrayExpressionSuite,org.apache.comet.CometCastSuite,org.apache.comet.CometExpressionSuite,org.apache.comet.CometExpressionCoverageSuite,org.apache.comet.CometNativeSuite,org.apache.comet.CometSparkSessionExtensionsSuite,org.apache.spark.CometPluginsSuite,org.apache.spark.sql.CometTPCDSQuerySuite,org.apache.spark.sql.CometTPCHQuerySuite,org.apache.spark.sql.comet.CometPlanStabilitySuite,org.apache.spark.sql.comet.CometTaskMetricsSuite"
+        value: |
+          org.apache.comet.exec.CometExecSuite
+          org.apache.comet.exec.CometExec3_4PlusSuite
+          org.apache.comet.exec.CometJoinSuite
+          org.apache.comet.exec.CometAggregateSuite
+          org.apache.comet.CometArrayExpressionSuite
+          org.apache.comet.CometCastSuite
+          org.apache.comet.CometExpressionSuite
+          org.apache.comet.CometExpressionCoverageSuite
+          org.apache.comet.CometNativeSuite
+          org.apache.comet.CometSparkSessionExtensionsSuite
+          org.apache.spark.CometPluginsSuite
+          org.apache.spark.sql.CometTPCDSQuerySuite
+          org.apache.spark.sql.CometTPCHQuerySuite
+          org.apache.spark.sql.comet.CometPlanStabilitySuite
+          org.apache.spark.sql.comet.CometTaskMetricsSuite
+
     fail-fast: false
     name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
     runs-on: ${{ matrix.os }}
@@ -72,5 +104,5 @@ jobs:
       - name: Java test steps
         uses: ./.github/actions/java-test
         with:
-          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=${{ matrix.suites.value }}"
+          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=$(echo "${{ matrix.suites.value }}" | paste -sd, -)"
           upload-test-reports: true

From 04244ba7e1e0814825cad57b604de2a5ac2530f6 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 12:58:06 -0600
Subject: [PATCH 10/17] test

---
 .github/workflows/pr_build_new.yml | 81 +++++++++++++++---------------
 1 file changed, 40 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
index 057649a9e2..80d247bb5b 100644
--- a/.github/workflows/pr_build_new.yml
+++ b/.github/workflows/pr_build_new.yml
@@ -48,47 +48,46 @@ jobs:
         java_version: [17]
         test-target: [java]
         spark-version: ['4.0']
-    suites:
-      - name: "fuzz"
-        value: |
-          org.apache.comet.CometFuzzTestSuite
-          org.apache.comet.DataGeneratorSuite
-      - name: "shuffle"
-        value: |
-          org.apache.comet.exec.CometShuffleSuite
-          org.apache.comet.exec.CometShuffle4_0Suite
-          org.apache.comet.exec.CometNativeShuffleSuite
-          org.apache.comet.exec.CometShuffleEncryptionSuite
-          org.apache.comet.exec.CometShuffleManagerSuite
-          org.apache.comet.exec.CometAsyncShuffleSuite
-          org.apache.comet.exec.DisableAQECometShuffleSuite
-          org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
-      - name: "parquet"
-        value: |
-          org.apache.comet.parquet.ParquetReadV1Suite
-          org.apache.comet.parquet.ParquetReadV2Suite
-          org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
-          org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
-          org.apache.spark.sql.comet.ParquetEncryptionITCase
-          org.apache.comet.exec.CometNativeReaderSuite
-      - name: "exec"
-        value: |
-          org.apache.comet.exec.CometExecSuite
-          org.apache.comet.exec.CometExec3_4PlusSuite
-          org.apache.comet.exec.CometJoinSuite
-          org.apache.comet.exec.CometAggregateSuite
-          org.apache.comet.CometArrayExpressionSuite
-          org.apache.comet.CometCastSuite
-          org.apache.comet.CometExpressionSuite
-          org.apache.comet.CometExpressionCoverageSuite
-          org.apache.comet.CometNativeSuite
-          org.apache.comet.CometSparkSessionExtensionsSuite
-          org.apache.spark.CometPluginsSuite
-          org.apache.spark.sql.CometTPCDSQuerySuite
-          org.apache.spark.sql.CometTPCHQuerySuite
-          org.apache.spark.sql.comet.CometPlanStabilitySuite
-          org.apache.spark.sql.comet.CometTaskMetricsSuite
-
+        suites:
+          - name: "fuzz"
+            value: |
+              org.apache.comet.CometFuzzTestSuite
+              org.apache.comet.DataGeneratorSuite
+          - name: "shuffle"
+            value: |
+              org.apache.comet.exec.CometShuffleSuite
+              org.apache.comet.exec.CometShuffle4_0Suite
+              org.apache.comet.exec.CometNativeShuffleSuite
+              org.apache.comet.exec.CometShuffleEncryptionSuite
+              org.apache.comet.exec.CometShuffleManagerSuite
+              org.apache.comet.exec.CometAsyncShuffleSuite
+              org.apache.comet.exec.DisableAQECometShuffleSuite
+              org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
+          - name: "parquet"
+            value: |
+              org.apache.comet.parquet.ParquetReadV1Suite
+              org.apache.comet.parquet.ParquetReadV2Suite
+              org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
+              org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
+              org.apache.spark.sql.comet.ParquetEncryptionITCase
+              org.apache.comet.exec.CometNativeReaderSuite
+          - name: "exec"
+            value: |
+              org.apache.comet.exec.CometExecSuite
+              org.apache.comet.exec.CometExec3_4PlusSuite
+              org.apache.comet.exec.CometJoinSuite
+              org.apache.comet.exec.CometAggregateSuite
+              org.apache.comet.CometArrayExpressionSuite
+              org.apache.comet.CometCastSuite
+              org.apache.comet.CometExpressionSuite
+              org.apache.comet.CometExpressionCoverageSuite
+              org.apache.comet.CometNativeSuite
+              org.apache.comet.CometSparkSessionExtensionsSuite
+              org.apache.spark.CometPluginsSuite
+              org.apache.spark.sql.CometTPCDSQuerySuite
+              org.apache.spark.sql.CometTPCHQuerySuite
+              org.apache.spark.sql.comet.CometPlanStabilitySuite
+              org.apache.spark.sql.comet.CometTaskMetricsSuite
     fail-fast: false
     name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
     runs-on: ${{ matrix.os }}

From 1e6ede4fbaae47210546fcfec035a885a5c0cab0 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 13:01:28 -0600
Subject: [PATCH 11/17] test

---
 .github/actions/java-test/action.yaml | 5 +++--
 .github/workflows/pr_build_new.yml    | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
index 0218259ec3..092d56a441 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -54,8 +54,9 @@ runs:
     - name: Run tests
       shell: bash
       run: |
-        echo "Running tests with ${{ inputs.maven_opts }}" 
-        MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }}
+        MAVEN_SUITES="-DwildcardSuites=$(echo "${{ inputs.suites }}" | paste -sd, -)"
+        echo "Running with MAVEN_SUITES=$MAVEN_SUITES"
+        MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} ${{ env.MAVEN_SUITES }}
     - name: Upload crash logs
       if: failure()
       uses: actions/upload-artifact@v4
diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
index 80d247bb5b..9c087b0ba7 100644
--- a/.github/workflows/pr_build_new.yml
+++ b/.github/workflows/pr_build_new.yml
@@ -103,5 +103,6 @@ jobs:
       - name: Java test steps
         uses: ./.github/actions/java-test
         with:
-          maven_opts: "-Pspark-${{ matrix.spark-version }} -DwildcardSuites=$(echo "${{ matrix.suites.value }}" | paste -sd, -)"
+          suites: ${{ matrix.suites.value }}
+          maven_opts: "-Pspark-${{ matrix.spark-version }}"
           upload-test-reports: true

From 4e2276c91b1b89c94058d00880360e71504da806 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 13:02:26 -0600
Subject: [PATCH 12/17] test

---
 .github/workflows/pr_build_new.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
index 9c087b0ba7..b151a58a13 100644
--- a/.github/workflows/pr_build_new.yml
+++ b/.github/workflows/pr_build_new.yml
@@ -88,7 +88,7 @@ jobs:
               org.apache.spark.sql.CometTPCHQuerySuite
               org.apache.spark.sql.comet.CometPlanStabilitySuite
               org.apache.spark.sql.comet.CometTaskMetricsSuite
-    fail-fast: false
+      fail-fast: false
     name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
     runs-on: ${{ matrix.os }}
     container:

From 249f4ef3ad2e4e31f7b629eb66a38e8a64c4ff00 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 13:38:25 -0600
Subject: [PATCH 13/17] fix

---
 .github/actions/java-test/action.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
index 092d56a441..513f267e9b 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -18,6 +18,10 @@
 name: "Java Test"
 description: "Run Java tests"
 inputs:
+  suites:
+    description: 'List of suites to run'
+    required: true
+    default: ''
   maven_opts:
     description: 'Maven options passed to the mvn command'
     required: false

From 140ed3c2f72191851e0587d36062e8fff876f9d6 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 14:51:26 -0600
Subject: [PATCH 14/17] test

---
 .github/workflows/pr_build_linux_spark4.yml | 72 ---------------------
 .github/workflows/pr_build_new.yml          | 33 ----------
 2 files changed, 105 deletions(-)
 delete mode 100644 .github/workflows/pr_build_linux_spark4.yml

diff --git a/.github/workflows/pr_build_linux_spark4.yml b/.github/workflows/pr_build_linux_spark4.yml
deleted file mode 100644
index dc0a96c8d8..0000000000
--- a/.github/workflows/pr_build_linux_spark4.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: PR Build (Linux + Spark 4)
-
-concurrency:
-  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
-  cancel-in-progress: true
-
-on:
-
-# temporarily disabled on PRs due to https://github.com/apache/datafusion-comet/issues/1786
-#  push:
-#    paths-ignore:
-#      - "doc/**"
-#      - "docs/**"
-#      - "**.md"
-#  pull_request:
-#    paths-ignore:
-#      - "doc/**"
-#      - "docs/**"
-#      - "**.md"
-  # manual trigger
-  # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
-  workflow_dispatch:
-
-env:
-  RUST_VERSION: stable
-
-jobs:
-
-  linux-test-with-spark4_0:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        java_version: [17]
-        test-target: [java]
-        spark-version: ['4.0']
-        is_push_event:
-          - ${{ github.event_name == 'push' }}
-      fail-fast: false
-    name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }}
-    runs-on: ${{ matrix.os }}
-    container:
-      image: amd64/rust
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Rust & Java toolchain
-        uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{env.RUST_VERSION}}
-          jdk-version: ${{ matrix.java_version }}
-      - name: Java test steps
-        uses: ./.github/actions/java-test
-        with:
-          maven_opts: -Pspark-${{ matrix.spark-version }}
-          upload-test-reports: true
-
diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
index b151a58a13..6b99846574 100644
--- a/.github/workflows/pr_build_new.yml
+++ b/.github/workflows/pr_build_new.yml
@@ -52,42 +52,9 @@ jobs:
           - name: "fuzz"
             value: |
               org.apache.comet.CometFuzzTestSuite
-              org.apache.comet.DataGeneratorSuite
           - name: "shuffle"
             value: |
               org.apache.comet.exec.CometShuffleSuite
-              org.apache.comet.exec.CometShuffle4_0Suite
-              org.apache.comet.exec.CometNativeShuffleSuite
-              org.apache.comet.exec.CometShuffleEncryptionSuite
-              org.apache.comet.exec.CometShuffleManagerSuite
-              org.apache.comet.exec.CometAsyncShuffleSuite
-              org.apache.comet.exec.DisableAQECometShuffleSuite
-              org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
-          - name: "parquet"
-            value: |
-              org.apache.comet.parquet.ParquetReadV1Suite
-              org.apache.comet.parquet.ParquetReadV2Suite
-              org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
-              org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
-              org.apache.spark.sql.comet.ParquetEncryptionITCase
-              org.apache.comet.exec.CometNativeReaderSuite
-          - name: "exec"
-            value: |
-              org.apache.comet.exec.CometExecSuite
-              org.apache.comet.exec.CometExec3_4PlusSuite
-              org.apache.comet.exec.CometJoinSuite
-              org.apache.comet.exec.CometAggregateSuite
-              org.apache.comet.CometArrayExpressionSuite
-              org.apache.comet.CometCastSuite
-              org.apache.comet.CometExpressionSuite
-              org.apache.comet.CometExpressionCoverageSuite
-              org.apache.comet.CometNativeSuite
-              org.apache.comet.CometSparkSessionExtensionsSuite
-              org.apache.spark.CometPluginsSuite
-              org.apache.spark.sql.CometTPCDSQuerySuite
-              org.apache.spark.sql.CometTPCHQuerySuite
-              org.apache.spark.sql.comet.CometPlanStabilitySuite
-              org.apache.spark.sql.comet.CometTaskMetricsSuite
       fail-fast: false
     name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
     runs-on: ${{ matrix.os }}

From 5a92ddcce562f841a76f5b70f5a410b934206cf8 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 15:07:30 -0600
Subject: [PATCH 15/17] test

---
 .github/actions/java-test/action.yaml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
index 513f267e9b..a264bf20c1 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -18,10 +18,6 @@
 name: "Java Test"
 description: "Run Java tests"
 inputs:
-  suites:
-    description: 'List of suites to run'
-    required: true
-    default: ''
   maven_opts:
     description: 'Maven options passed to the mvn command'
     required: false
@@ -58,9 +54,9 @@ runs:
     - name: Run tests
       shell: bash
       run: |
-        MAVEN_SUITES="-DwildcardSuites=$(echo "${{ inputs.suites }}" | paste -sd, -)"
+        MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)"
         echo "Running with MAVEN_SUITES=$MAVEN_SUITES"
-        MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} ${{ env.MAVEN_SUITES }}
+        MAVEN_OPTS="-DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} ${{ env.MAVEN_SUITES }}
     - name: Upload crash logs
       if: failure()
       uses: actions/upload-artifact@v4

From 03ceeb946d8eaf9ec149e38d7380ac8539b9edbf Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 15:10:53 -0600
Subject: [PATCH 16/17] test

---
 .github/actions/java-test/action.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
index a264bf20c1..9f74fe39f8 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -18,6 +18,9 @@
 name: "Java Test"
 description: "Run Java tests"
 inputs:
+  suites:
+    description: 'suites'
+    default: ''
   maven_opts:
     description: 'Maven options passed to the mvn command'
     required: false

From 4bbe46a48a480de69b5b1fa89cfa5b1cca6a150f Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Tue, 27 May 2025 15:23:05 -0600
Subject: [PATCH 17/17] enable more tests

---
 .github/workflows/benchmark.yml    | 20 +++++++++---------
 .github/workflows/pr_build_new.yml | 33 ++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index de1ad57739..712a71582a 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -22,16 +22,16 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  push:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-  pull_request:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
+#  push:
+#    paths-ignore:
+#      - "doc/**"
+#      - "docs/**"
+#      - "**.md"
+#  pull_request:
+#    paths-ignore:
+#      - "doc/**"
+#      - "docs/**"
+#      - "**.md"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml
index 6b99846574..b151a58a13 100644
--- a/.github/workflows/pr_build_new.yml
+++ b/.github/workflows/pr_build_new.yml
@@ -52,9 +52,42 @@ jobs:
           - name: "fuzz"
             value: |
               org.apache.comet.CometFuzzTestSuite
+              org.apache.comet.DataGeneratorSuite
           - name: "shuffle"
             value: |
               org.apache.comet.exec.CometShuffleSuite
+              org.apache.comet.exec.CometShuffle4_0Suite
+              org.apache.comet.exec.CometNativeShuffleSuite
+              org.apache.comet.exec.CometShuffleEncryptionSuite
+              org.apache.comet.exec.CometShuffleManagerSuite
+              org.apache.comet.exec.CometAsyncShuffleSuite
+              org.apache.comet.exec.DisableAQECometShuffleSuite
+              org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
+          - name: "parquet"
+            value: |
+              org.apache.comet.parquet.ParquetReadV1Suite
+              org.apache.comet.parquet.ParquetReadV2Suite
+              org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
+              org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
+              org.apache.spark.sql.comet.ParquetEncryptionITCase
+              org.apache.comet.exec.CometNativeReaderSuite
+          - name: "exec"
+            value: |
+              org.apache.comet.exec.CometExecSuite
+              org.apache.comet.exec.CometExec3_4PlusSuite
+              org.apache.comet.exec.CometJoinSuite
+              org.apache.comet.exec.CometAggregateSuite
+              org.apache.comet.CometArrayExpressionSuite
+              org.apache.comet.CometCastSuite
+              org.apache.comet.CometExpressionSuite
+              org.apache.comet.CometExpressionCoverageSuite
+              org.apache.comet.CometNativeSuite
+              org.apache.comet.CometSparkSessionExtensionsSuite
+              org.apache.spark.CometPluginsSuite
+              org.apache.spark.sql.CometTPCDSQuerySuite
+              org.apache.spark.sql.CometTPCHQuerySuite
+              org.apache.spark.sql.comet.CometPlanStabilitySuite
+              org.apache.spark.sql.comet.CometTaskMetricsSuite
       fail-fast: false
     name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}]
     runs-on: ${{ matrix.os }}