From 9a9ff836539cc5be1acf290b7ea8b719b7d535fb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 25 May 2025 09:38:07 -0600 Subject: [PATCH 1/9] Add Docker build image --- Makefile | 4 +-- dev/Dockerfile | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 dev/Dockerfile diff --git a/Makefile b/Makefile index 9382d1aeee..c27927c540 100644 --- a/Makefile +++ b/Makefile @@ -30,13 +30,13 @@ core: cd native && cargo build test-rust: # We need to compile CometException so that the cargo test can pass - ./mvnw compile -pl common -DskipTests $(PROFILES) + ./mvnw compile -pl common -DskipTests $(PROFILES) -Dmaven.gitcommitid.skip=true cd native && cargo build && \ RUST_BACKTRACE=1 cargo test jvm: ./mvnw clean package -DskipTests $(PROFILES) test-jvm: core - SPARK_HOME=`pwd` COMET_CONF_DIR=$(shell pwd)/conf RUST_BACKTRACE=1 ./mvnw verify $(PROFILES) + SPARK_HOME=`pwd` COMET_CONF_DIR=$(shell pwd)/conf RUST_BACKTRACE=1 ./mvnw verify $(PROFILES) -Dmaven.gitcommitid.skip=true test: test-rust test-jvm clean: cd native && cargo clean diff --git a/dev/Dockerfile b/dev/Dockerfile new file mode 100644 index 0000000000..84ae844fc6 --- /dev/null +++ b/dev/Dockerfile @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM ubuntu:24.10 + +# Install dependencies +RUN apt-get update && apt-get install -y \ + curl \ + unzip \ + git \ + make \ + build-essential \ + ca-certificates \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install protoc +ENV PROTOC_VERSION=24.4 + +RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip \ + && unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local \ + && rm protoc-${PROTOC_VERSION}-linux-x86_64.zip + +# Install Rust +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" +RUN rustc --version && cargo --version + +# Install Java +RUN apt-get update && apt-get install -y \ + openjdk-17-jdk \ + && rm -rf /var/lib/apt/lists/* +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 +ENV PATH="$JAVA_HOME/bin:$PATH" +RUN java -version + +# Add Comet sources +RUN mkdir /comet +WORKDIR /comet +COPY ./.mvn /comet/.mvn +COPY ./common /comet/common +COPY ./dev /comet/dev +COPY ./fuzz-testing /comet/fuzz-testing +COPY ./spark /comet/spark +COPY ./spark-integration /comet/spark-integration +COPY ./native /comet/native +COPY ./pom.xml /comet +COPY ./Makefile /comet +COPY ./rust-toolchain.toml /comet +COPY ./mvnw /comet +COPY ./scalafmt.conf /comet +COPY ./.scalafix.conf /comet + +# Shallow clone of Apache Spark and apply diffs +#RUN git clone -b 'v3.5.5' --single-branch --depth 1 https://github.com/apache/spark.git apache-spark \ +# && cd apache-spark \ +# && git apply ../dev/diffs/3.5.5.diff + From 26e85667a1a30b700f1af6c97e121c5f4aaa7710 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 25 May 2025 10:35:10 -0600 Subject: [PATCH 2/9] experiment --- .github/workflows/pr_build.yml | 575 ++++++++++++++++----------------- dev/Dockerfile | 1 + 2 files changed, 286 insertions(+), 290 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 1ae94f2741..983cc66a34 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -41,304 +41,299 @@ env: jobs: - # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release - linux-test: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.5'] - scala-version: ['2.12', '2.13'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - - # run Comet tests for older 3.5.x releases with one java and scala versions - linux-test-35-minor-versions: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [17] - test-target: [java] - spark-version: ['3.5'] - spark-patch-version: ['3.5.2', '3.5.3', '3.5.4'] - scala-version: ['2.12'] - is_push_event: - - ${{ github.event_name == 'push' }} - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}" - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - - linux-test-native-datafusion-scan: - strategy: - matrix: - os: [ubuntu-latest] - scan_impl: ['native_datafusion', 'native_iceberg_compat'] - java_version: [17] - test-target: [rust, java] - spark-version: ['3.5'] - scala-version: ['2.12'] - is_push_event: - - ${{ github.event_name == 'push' }} - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }} - runs-on: ${{ matrix.os }} - env: - COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - +# # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release +# linux-test: +# strategy: +# matrix: +# os: [ubuntu-latest] +# java_version: [11, 17] +# test-target: [rust, java] +# spark-version: ['3.5'] +# scala-version: ['2.12', '2.13'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# exclude: # exclude java 11 for pull_request event +# - java_version: 11 +# is_push_event: false +# fail-fast: false +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# container: +# image: amd64/rust +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# # upload test reports only for java 17 +# upload-test-reports: ${{ matrix.java_version == '17' }} +# +# # run Comet tests for older 3.5.x releases with one java and scala versions +# linux-test-35-minor-versions: +# strategy: +# matrix: +# os: [ubuntu-latest] +# java_version: [17] +# test-target: [java] +# spark-version: ['3.5'] +# spark-patch-version: ['3.5.2', '3.5.3', '3.5.4'] +# scala-version: ['2.12'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# fail-fast: false +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# container: +# image: amd64/rust +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}" +# # upload test reports only for java 17 +# upload-test-reports: ${{ matrix.java_version == '17' }} +# +# linux-test-native-datafusion-scan: +# strategy: +# matrix: +# os: [ubuntu-latest] +# scan_impl: ['native_datafusion', 'native_iceberg_compat'] +# java_version: [17] +# test-target: [rust, java] +# spark-version: ['3.5'] +# scala-version: ['2.12'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# fail-fast: false +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }} +# runs-on: ${{ matrix.os }} +# env: +# COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }} +# container: +# image: amd64/rust +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# # upload test reports only for java 17 +# upload-test-reports: ${{ matrix.java_version == '17' }} +# linux-test-with-spark4_0: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [17] - test-target: [java] - spark-version: ['4.0'] - is_push_event: - - ${{ github.event_name == 'push' }} - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }} - upload-test-reports: true - - linux-test-with-old-spark: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [11, 17] - test-target: [java] - spark-version: ['3.4'] - scala-version: ['2.12', '2.13'] - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest container: image: amd64/rust steps: - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - - macos-test: - strategy: - matrix: - os: [macos-13] - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.4', '3.5'] - scala-version: ['2.12', '2.13'] - fail-fast: false - if: github.event_name == 'push' - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - macos-aarch64-test: - strategy: - matrix: - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.4', '3.5'] - scala-version: ['2.12', '2.13'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - macos-test-with-spark4_0: - strategy: - matrix: - os: [macos-13] - java_version: [17] - test-target: [java] - spark-version: ['4.0'] - fail-fast: false - if: github.event_name == 'push' - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - name: Java test steps - uses: ./.github/actions/java-test + - name: Build image locally + uses: docker/build-push-action@v5 with: - maven_opts: -Pspark-${{ matrix.spark-version }} - upload-test-reports: true + context: . + file: dev/Dockerfile + load: true + tags: comet-build:latest - macos-aarch64-test-with-spark4_0: - strategy: - matrix: - java_version: [17] - test-target: [java] - spark-version: ['4.0'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }} - upload-test-reports: true - - macos-aarch64-test-with-old-spark: - strategy: - matrix: - java_version: [17] - test-target: [java] - spark-version: ['3.4'] - scala-version: ['2.12', '2.13'] - exclude: - - java_version: 8 - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} + run: docker run -it comet-build:latest make test PROFILES="-Pspark-4.0" +# +# linux-test-with-old-spark: +# strategy: +# matrix: +# os: [ubuntu-latest] +# java_version: [11, 17] +# test-target: [java] +# spark-version: ['3.4'] +# scala-version: ['2.12', '2.13'] +# fail-fast: false +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# container: +# image: amd64/rust +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# +# macos-test: +# strategy: +# matrix: +# os: [macos-13] +# java_version: [11, 17] +# test-target: [rust, java] +# spark-version: ['3.4', '3.5'] +# scala-version: ['2.12', '2.13'] +# fail-fast: false +# if: github.event_name == 'push' +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# +# macos-aarch64-test: +# strategy: +# matrix: +# java_version: [11, 17] +# test-target: [rust, java] +# spark-version: ['3.4', '3.5'] +# scala-version: ['2.12', '2.13'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# exclude: # exclude java 11 for pull_request event +# - java_version: 11 +# is_push_event: false +# fail-fast: false +# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: macos-14 +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# jdk-architecture: aarch64 +# protoc-architecture: aarch_64 +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# +# macos-test-with-spark4_0: +# strategy: +# matrix: +# os: [macos-13] +# java_version: [17] +# test-target: [java] +# spark-version: ['4.0'] +# fail-fast: false +# if: github.event_name == 'push' +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }} +# upload-test-reports: true +# +# macos-aarch64-test-with-spark4_0: +# strategy: +# matrix: +# java_version: [17] +# test-target: [java] +# spark-version: ['4.0'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# exclude: # exclude java 11 for pull_request event +# - java_version: 11 +# is_push_event: false +# fail-fast: false +# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} +# runs-on: macos-14 +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# jdk-architecture: aarch64 +# protoc-architecture: aarch_64 +# - name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }} +# upload-test-reports: true +# +# macos-aarch64-test-with-old-spark: +# strategy: +# matrix: +# java_version: [17] +# test-target: [java] +# spark-version: ['3.4'] +# scala-version: ['2.12', '2.13'] +# exclude: +# - java_version: 8 +# fail-fast: false +# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: macos-14 +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# jdk-architecture: aarch64 +# protoc-architecture: aarch_64 +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# diff --git a/dev/Dockerfile b/dev/Dockerfile index 84ae844fc6..90ffe10662 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -55,6 +55,7 @@ WORKDIR /comet COPY ./.mvn /comet/.mvn COPY ./common /comet/common COPY ./dev /comet/dev +COPY ./docs /comet/docs COPY ./fuzz-testing /comet/fuzz-testing COPY ./spark /comet/spark COPY ./spark-integration /comet/spark-integration From 029b95e65ae1b7fd2333e4548930d4830342b67d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 25 May 2025 12:00:54 -0600 Subject: [PATCH 3/9] Test --- .github/workflows/pr_build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 983cc66a34..5d5c8c9e2b 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -151,8 +151,6 @@ jobs: # linux-test-with-spark4_0: runs-on: ubuntu-latest - container: - image: amd64/rust steps: - uses: actions/checkout@v4 From 19b1a32c59bf086d8750be75d551a5198d08941c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 25 May 2025 12:04:03 -0600 Subject: [PATCH 4/9] test --- .github/workflows/pr_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 5d5c8c9e2b..356d6850a5 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -166,7 +166,7 @@ jobs: tags: comet-build:latest - name: Java test steps - run: docker run -it comet-build:latest make test PROFILES="-Pspark-4.0" + run: docker run -t comet-build:latest make test PROFILES="-Pspark-4.0" # # linux-test-with-old-spark: From a6a05038b7c2bf23230d0f9ab00624a92ddb355b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 25 May 2025 12:13:23 -0600 Subject: [PATCH 5/9] test --- .github/workflows/pr_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 356d6850a5..abbf13cb0a 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -166,7 +166,7 @@ jobs: tags: comet-build:latest - name: Java test steps - run: docker run -t comet-build:latest make test PROFILES="-Pspark-4.0" + run: docker run comet-build:latest make test PROFILES="-Pspark-4.0" # # linux-test-with-old-spark: From f0f001010a27554dfbf57556817c4e75df699bcb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 25 May 2025 12:31:52 -0600 Subject: [PATCH 6/9] test --- .github/workflows/pr_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index abbf13cb0a..1e403100df 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -166,7 +166,7 @@ jobs: tags: comet-build:latest - name: Java test steps - run: docker run comet-build:latest make test PROFILES="-Pspark-4.0" + run: docker run comet-build:latest make test PROFILES="-Pspark-4.0 --batch-mode" # # linux-test-with-old-spark: From aed4872b1ae755eb606d2be0c24f40bb57905bf6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 25 May 2025 14:50:17 -0600 Subject: [PATCH 7/9] test --- .github/workflows/pr_build.yml | 321 +++++---------------------------- 1 file changed, 44 insertions(+), 277 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 1e403100df..478b1f024a 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -41,116 +41,20 @@ env: jobs: -# # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release -# linux-test: -# strategy: -# matrix: -# os: [ubuntu-latest] -# java_version: [11, 17] -# test-target: [rust, java] -# spark-version: ['3.5'] -# scala-version: ['2.12', '2.13'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# exclude: # exclude java 11 for pull_request event -# - java_version: 11 -# is_push_event: false -# fail-fast: false -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# container: -# image: amd64/rust -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# # upload test reports only for java 17 -# upload-test-reports: ${{ matrix.java_version == '17' }} -# -# # run Comet tests for older 3.5.x releases with one java and scala versions -# linux-test-35-minor-versions: -# strategy: -# matrix: -# os: [ubuntu-latest] -# java_version: [17] -# test-target: [java] -# spark-version: ['3.5'] -# spark-patch-version: ['3.5.2', '3.5.3', '3.5.4'] -# scala-version: ['2.12'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# fail-fast: false -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# container: -# image: amd64/rust -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}" -# # upload test reports only for java 17 -# upload-test-reports: ${{ matrix.java_version == '17' }} -# -# linux-test-native-datafusion-scan: -# strategy: -# matrix: -# os: [ubuntu-latest] -# scan_impl: ['native_datafusion', 'native_iceberg_compat'] -# java_version: [17] -# test-target: [rust, java] -# spark-version: ['3.5'] -# scala-version: ['2.12'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# fail-fast: false -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }} -# runs-on: ${{ matrix.os }} -# env: -# COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }} -# container: -# image: amd64/rust -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# # upload test reports only for java 17 -# upload-test-reports: ${{ matrix.java_version == '17' }} -# - linux-test-with-spark4_0: + linux-test-jdk-17: runs-on: ubuntu-latest + + strategy: + matrix: + spark_profile: [ + "-Pspark-3.4", + "-Pspark-3.5 -Pscala-2.12", + "-Pspark-3.5 -Pscala-2.13", + "-Pspark-4.0" + ] + + name: linux/java ${{ matrix.spark_profile }} + steps: - uses: actions/checkout@v4 @@ -166,172 +70,35 @@ jobs: tags: comet-build:latest - name: Java test steps - run: docker run comet-build:latest make test PROFILES="-Pspark-4.0 --batch-mode" + run: docker run comet-build:latest make test PROFILES="${{ matrix.spark_profile }} --batch-mode" -# -# linux-test-with-old-spark: -# strategy: -# matrix: -# os: [ubuntu-latest] -# java_version: [11, 17] -# test-target: [java] -# spark-version: ['3.4'] -# scala-version: ['2.12', '2.13'] -# fail-fast: false -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# container: -# image: amd64/rust -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# -# macos-test: -# strategy: -# matrix: -# os: [macos-13] -# java_version: [11, 17] -# test-target: [rust, java] -# spark-version: ['3.4', '3.5'] -# scala-version: ['2.12', '2.13'] -# fail-fast: false -# if: github.event_name == 'push' -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# -# macos-aarch64-test: -# strategy: -# matrix: -# java_version: [11, 17] -# test-target: [rust, java] -# spark-version: ['3.4', '3.5'] -# scala-version: ['2.12', '2.13'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# exclude: # exclude java 11 for pull_request event -# - java_version: 11 -# is_push_event: false -# fail-fast: false -# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: macos-14 -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# jdk-architecture: aarch64 -# protoc-architecture: aarch_64 -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# -# macos-test-with-spark4_0: -# strategy: -# matrix: -# os: [macos-13] -# java_version: [17] -# test-target: [java] -# spark-version: ['4.0'] -# fail-fast: false -# if: github.event_name == 'push' -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }} -# upload-test-reports: true -# -# macos-aarch64-test-with-spark4_0: -# strategy: -# matrix: -# java_version: [17] -# test-target: [java] -# spark-version: ['4.0'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# exclude: # exclude java 11 for pull_request event -# - java_version: 11 -# is_push_event: false -# fail-fast: false -# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} -# runs-on: macos-14 -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# jdk-architecture: aarch64 -# protoc-architecture: aarch_64 -# - name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }} -# upload-test-reports: true -# -# macos-aarch64-test-with-old-spark: -# strategy: -# matrix: -# java_version: [17] -# test-target: [java] -# spark-version: ['3.4'] -# scala-version: ['2.12', '2.13'] -# exclude: -# - java_version: 8 -# fail-fast: false -# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: macos-14 -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# jdk-architecture: aarch64 -# protoc-architecture: aarch_64 -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# + macos-test-jdk-17: + runs-on: macos-13 + + strategy: + matrix: + spark_profile: [ + "-Pspark-3.4", + "-Pspark-3.5 -Pscala-2.12", + "-Pspark-3.5 -Pscala-2.13", + "-Pspark-4.0" + ] + + name: macos/java ${{ matrix.spark_profile }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image locally + uses: docker/build-push-action@v5 + with: + context: . + file: dev/Dockerfile + load: true + tags: comet-build:latest + + - name: Java test steps + run: docker run comet-build:latest make test PROFILES="${{ matrix.spark_profile }} --batch-mode" From acc83ced978d662f06cdfd639fcb578812a71920 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 25 May 2025 14:55:18 -0600 Subject: [PATCH 8/9] Revert "test" This reverts commit aed4872b1ae755eb606d2be0c24f40bb57905bf6. --- .github/workflows/pr_build.yml | 321 ++++++++++++++++++++++++++++----- 1 file changed, 277 insertions(+), 44 deletions(-) diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 478b1f024a..1e403100df 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -41,20 +41,116 @@ env: jobs: - linux-test-jdk-17: +# # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release +# linux-test: +# strategy: +# matrix: +# os: [ubuntu-latest] +# java_version: [11, 17] +# test-target: [rust, java] +# spark-version: ['3.5'] +# scala-version: ['2.12', '2.13'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# exclude: # exclude java 11 for pull_request event +# - java_version: 11 +# is_push_event: false +# fail-fast: false +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# container: +# image: amd64/rust +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# # upload test reports only for java 17 +# upload-test-reports: ${{ matrix.java_version == '17' }} +# +# # run Comet tests for older 3.5.x releases with one java and scala versions +# linux-test-35-minor-versions: +# strategy: +# matrix: +# os: [ubuntu-latest] +# java_version: [17] +# test-target: [java] +# spark-version: ['3.5'] +# spark-patch-version: ['3.5.2', '3.5.3', '3.5.4'] +# scala-version: ['2.12'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# fail-fast: false +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# container: +# image: amd64/rust +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}" +# # upload test reports only for java 17 +# upload-test-reports: ${{ matrix.java_version == '17' }} +# +# linux-test-native-datafusion-scan: +# strategy: +# matrix: +# os: [ubuntu-latest] +# scan_impl: ['native_datafusion', 'native_iceberg_compat'] +# java_version: [17] +# test-target: [rust, java] +# spark-version: ['3.5'] +# scala-version: ['2.12'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# fail-fast: false +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }} +# runs-on: ${{ matrix.os }} +# env: +# COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }} +# container: +# image: amd64/rust +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# # upload test reports only for java 17 +# upload-test-reports: ${{ matrix.java_version == '17' }} +# + linux-test-with-spark4_0: runs-on: ubuntu-latest - - strategy: - matrix: - spark_profile: [ - "-Pspark-3.4", - "-Pspark-3.5 -Pscala-2.12", - "-Pspark-3.5 -Pscala-2.13", - "-Pspark-4.0" - ] - - name: linux/java ${{ matrix.spark_profile }} - steps: - uses: actions/checkout@v4 @@ -70,35 +166,172 @@ jobs: tags: comet-build:latest - name: Java test steps - run: docker run comet-build:latest make test PROFILES="${{ matrix.spark_profile }} --batch-mode" - - macos-test-jdk-17: - runs-on: macos-13 - - strategy: - matrix: - spark_profile: [ - "-Pspark-3.4", - "-Pspark-3.5 -Pscala-2.12", - "-Pspark-3.5 -Pscala-2.13", - "-Pspark-4.0" - ] - - name: macos/java ${{ matrix.spark_profile }} - - steps: - - uses: actions/checkout@v4 + run: docker run comet-build:latest make test PROFILES="-Pspark-4.0 --batch-mode" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build image locally - uses: docker/build-push-action@v5 - with: - context: . - file: dev/Dockerfile - load: true - tags: comet-build:latest - - - name: Java test steps - run: docker run comet-build:latest make test PROFILES="${{ matrix.spark_profile }} --batch-mode" +# +# linux-test-with-old-spark: +# strategy: +# matrix: +# os: [ubuntu-latest] +# java_version: [11, 17] +# test-target: [java] +# spark-version: ['3.4'] +# scala-version: ['2.12', '2.13'] +# fail-fast: false +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# container: +# image: amd64/rust +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# +# macos-test: +# strategy: +# matrix: +# os: [macos-13] +# java_version: [11, 17] +# test-target: [rust, java] +# spark-version: ['3.4', '3.5'] +# scala-version: ['2.12', '2.13'] +# fail-fast: false +# if: github.event_name == 'push' +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# +# macos-aarch64-test: +# strategy: +# matrix: +# java_version: [11, 17] +# test-target: [rust, java] +# spark-version: ['3.4', '3.5'] +# scala-version: ['2.12', '2.13'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# exclude: # exclude java 11 for pull_request event +# - java_version: 11 +# is_push_event: false +# fail-fast: false +# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: macos-14 +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# jdk-architecture: aarch64 +# protoc-architecture: aarch_64 +# - if: matrix.test-target == 'rust' +# name: Rust test steps +# uses: ./.github/actions/rust-test +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# +# macos-test-with-spark4_0: +# strategy: +# matrix: +# os: [macos-13] +# java_version: [17] +# test-target: [java] +# spark-version: ['4.0'] +# fail-fast: false +# if: github.event_name == 'push' +# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} +# runs-on: ${{ matrix.os }} +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# - name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }} +# upload-test-reports: true +# +# macos-aarch64-test-with-spark4_0: +# strategy: +# matrix: +# java_version: [17] +# test-target: [java] +# spark-version: ['4.0'] +# is_push_event: +# - ${{ github.event_name == 'push' }} +# exclude: # exclude java 11 for pull_request event +# - java_version: 11 +# is_push_event: false +# fail-fast: false +# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} +# runs-on: macos-14 +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# jdk-architecture: aarch64 +# protoc-architecture: aarch_64 +# - name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }} +# upload-test-reports: true +# +# macos-aarch64-test-with-old-spark: +# strategy: +# matrix: +# java_version: [17] +# test-target: [java] +# spark-version: ['3.4'] +# scala-version: ['2.12', '2.13'] +# exclude: +# - java_version: 8 +# fail-fast: false +# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} +# runs-on: macos-14 +# steps: +# - uses: actions/checkout@v4 +# - name: Setup Rust & Java toolchain +# uses: ./.github/actions/setup-macos-builder +# with: +# rust-version: ${{env.RUST_VERSION}} +# jdk-version: ${{ matrix.java_version }} +# jdk-architecture: aarch64 +# protoc-architecture: aarch_64 +# - if: matrix.test-target == 'java' +# name: Java test steps +# uses: ./.github/actions/java-test +# with: +# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} +# From a327235964d14c4ef285aafaf94d710beb42c03e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 27 May 2025 12:10:20 -0600 Subject: [PATCH 9/9] test --- .github/workflows/pr_build.yml | 337 ----------------------------- .github/workflows/pr_build_new.yml | 75 +++++++ dev/Dockerfile | 3 + 3 files changed, 78 insertions(+), 337 deletions(-) delete mode 100644 .github/workflows/pr_build.yml create mode 100644 .github/workflows/pr_build_new.yml diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml deleted file mode 100644 index 1e403100df..0000000000 --- a/.github/workflows/pr_build.yml +++ /dev/null @@ -1,337 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: PR Build - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -on: - push: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - pull_request: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - # manual trigger - # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow - workflow_dispatch: - -env: - RUST_VERSION: stable - -jobs: - -# # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release -# linux-test: -# strategy: -# matrix: -# os: [ubuntu-latest] -# java_version: [11, 17] -# test-target: [rust, java] -# spark-version: ['3.5'] -# scala-version: ['2.12', '2.13'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# exclude: # exclude java 11 for pull_request event -# - java_version: 11 -# is_push_event: false -# fail-fast: false -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# container: -# image: amd64/rust -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# # upload test reports only for java 17 -# upload-test-reports: ${{ matrix.java_version == '17' }} -# -# # run Comet tests for older 3.5.x releases with one java and scala versions -# linux-test-35-minor-versions: -# strategy: -# matrix: -# os: [ubuntu-latest] -# java_version: [17] -# test-target: [java] -# spark-version: ['3.5'] -# spark-patch-version: ['3.5.2', '3.5.3', '3.5.4'] -# scala-version: ['2.12'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# fail-fast: false -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# container: -# image: amd64/rust -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}" -# # upload test reports only for java 17 -# upload-test-reports: ${{ matrix.java_version == '17' }} -# -# linux-test-native-datafusion-scan: -# strategy: -# matrix: -# os: [ubuntu-latest] -# scan_impl: ['native_datafusion', 'native_iceberg_compat'] -# java_version: [17] -# test-target: [rust, java] -# spark-version: ['3.5'] -# scala-version: ['2.12'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# fail-fast: false -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }} -# runs-on: ${{ matrix.os }} -# env: -# COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }} -# container: -# image: amd64/rust -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# # upload test reports only for java 17 -# upload-test-reports: ${{ matrix.java_version == '17' }} -# - linux-test-with-spark4_0: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build image locally - uses: docker/build-push-action@v5 - with: - context: . - file: dev/Dockerfile - load: true - tags: comet-build:latest - - - name: Java test steps - run: docker run comet-build:latest make test PROFILES="-Pspark-4.0 --batch-mode" - -# -# linux-test-with-old-spark: -# strategy: -# matrix: -# os: [ubuntu-latest] -# java_version: [11, 17] -# test-target: [java] -# spark-version: ['3.4'] -# scala-version: ['2.12', '2.13'] -# fail-fast: false -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# container: -# image: amd64/rust -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# -# macos-test: -# strategy: -# matrix: -# os: [macos-13] -# java_version: [11, 17] -# test-target: [rust, java] -# spark-version: ['3.4', '3.5'] -# scala-version: ['2.12', '2.13'] -# fail-fast: false -# if: github.event_name == 'push' -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# -# macos-aarch64-test: -# strategy: -# matrix: -# java_version: [11, 17] -# test-target: [rust, java] -# spark-version: ['3.4', '3.5'] -# scala-version: ['2.12', '2.13'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# exclude: # exclude java 11 for pull_request event -# - java_version: 11 -# is_push_event: false -# fail-fast: false -# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: macos-14 -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# jdk-architecture: aarch64 -# protoc-architecture: aarch_64 -# - if: matrix.test-target == 'rust' -# name: Rust test steps -# uses: ./.github/actions/rust-test -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# -# macos-test-with-spark4_0: -# strategy: -# matrix: -# os: [macos-13] -# java_version: [17] -# test-target: [java] -# spark-version: ['4.0'] -# fail-fast: false -# if: github.event_name == 'push' -# name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# - name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }} -# upload-test-reports: true -# -# macos-aarch64-test-with-spark4_0: -# strategy: -# matrix: -# java_version: [17] -# test-target: [java] -# spark-version: ['4.0'] -# is_push_event: -# - ${{ github.event_name == 'push' }} -# exclude: # exclude java 11 for pull_request event -# - java_version: 11 -# is_push_event: false -# fail-fast: false -# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} -# runs-on: macos-14 -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# jdk-architecture: aarch64 -# protoc-architecture: aarch_64 -# - name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }} -# upload-test-reports: true -# -# macos-aarch64-test-with-old-spark: -# strategy: -# matrix: -# java_version: [17] -# test-target: [java] -# spark-version: ['3.4'] -# scala-version: ['2.12', '2.13'] -# exclude: -# - java_version: 8 -# fail-fast: false -# name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} -# runs-on: macos-14 -# steps: -# - uses: actions/checkout@v4 -# - name: Setup Rust & Java toolchain -# uses: ./.github/actions/setup-macos-builder -# with: -# rust-version: ${{env.RUST_VERSION}} -# jdk-version: ${{ matrix.java_version }} -# jdk-architecture: aarch64 -# protoc-architecture: aarch_64 -# - if: matrix.test-target == 'java' -# name: Java test steps -# uses: ./.github/actions/java-test -# with: -# maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -# diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml new file mode 100644 index 0000000000..802a6ef0c3 --- /dev/null +++ b/.github/workflows/pr_build_new.yml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PR Build (Linux + Spark 4) + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + paths-ignore: + - "doc/**" + - "docs/**" + - "**.md" + pull_request: + paths-ignore: + - "doc/**" + - "docs/**" + - "**.md" + # manual trigger + # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow + workflow_dispatch: + +jobs: + + create-docker-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image + uses: docker/build-push-action@v5 + with: + context: . + file: dev/Dockerfile + push: true + tags: comet-build:${{ github.sha }} + + + linux-test: + needs: create-docker-image + strategy: + matrix: + spark_version: ['3.5', '4.0'] + suites: + - { name: "fuzz", value: "org.apache.comet.CometFuzzTestSuite" } + - { name: "shuffle", value: "org.apache.comet.exec.CometShuffleSuite,org.apache.comet.exec.CometAsyncShuffleSuite,org.apache.comet.exec.DisableAQECometShuffleSuite,org.apache.comet.exec.DisableAQECometAsyncShuffleSuite" } + - { name: "parquet", value: "org.apache.comet.parquet.ParquetReadV1Suite,org.apache.comet.parquet.ParquetReadV2Suite" } + - { name: "exec", value: "org.apache.comet.exec.CometExecSuite,org.apache.comet.CometExpressionSuite" } + is_push_event: + - ${{ github.event_name == 'push' }} + fail-fast: false + name: ubuntu-spark-${{ matrix.spark_version }}-${{ matrix.suites.name }} + runs-on: ubuntu-latest + steps: + - name: Run Comet test suite + run: docker run comet-build:latest make test PROFILES="--batch-mode -Pspark-${{ matrix.spark_version }} -DwildcardSuites=${{ matrix.suites.value }}" diff --git a/dev/Dockerfile b/dev/Dockerfile index 90ffe10662..aeeda1b60c 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -67,6 +67,9 @@ COPY ./mvnw /comet COPY ./scalafmt.conf /comet COPY ./.scalafix.conf /comet +# run make to download and cache dependencies and build Rust code +RUN make + # Shallow clone of Apache Spark and apply diffs #RUN git clone -b 'v3.5.5' --single-branch --depth 1 https://github.com/apache/spark.git apache-spark \ # && cd apache-spark \