[vector] add faiss jni to support faiss in java

.github/actions/build-faiss-native/action.yml

@@ -0,0 +1,112 @@
+name: 'Build FAISS Native Library'
+description: 'Build FAISS native library for specified platform'
+inputs:
+  platform:
+    description: 'Target platform (linux-amd64, linux-aarch64, darwin-aarch64)'
+    required: true
+  faiss-version:
+    description: 'FAISS version to build (e.g., 1.7.4)'
+    required: false
+    default: '1.7.4'
+  use-homebrew:
+    description: 'Whether to use Homebrew for dependencies (macOS)'
+    required: false
+    default: 'false'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install native dependencies (Linux)
+      if: inputs.use-homebrew == 'false'
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          build-essential \
+          libopenblas-dev \
+          liblapack-dev \
+          patchelf \
+          libgomp1 \
+          wget
+
+    - name: Install GCC 9 (Linux)
+      if: inputs.use-homebrew == 'false'
+      shell: bash
+      run: |
+        sudo apt-get install -y gcc-9 g++-9 || sudo apt-get install -y gcc g++
+        if command -v gcc-9 &>/dev/null; then
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90
+        fi
+        gcc --version
+
+    - name: Install CMake 3.30.1 (Linux)
+      if: inputs.use-homebrew == 'false'
+      shell: bash
+      run: |
+        CMAKE_VERSION="3.30.1"
+        if [[ "${{ inputs.platform }}" == *"amd64"* ]]; then
+          CMAKE_ARCH="x86_64"
+        else
+          CMAKE_ARCH="aarch64"
+        fi
+        wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz
+        tar -xzf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz
+        sudo mv cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH} /opt/cmake
+        sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
+        cmake --version
+
+    - name: Install FAISS (Linux)
+      if: inputs.use-homebrew == 'false'
+      shell: bash
+      run: |
+        git clone --depth 1 --branch v1.7.4 https://github.com/facebookresearch/faiss.git /tmp/faiss
+        cd /tmp/faiss
+        cmake -B build \
+          -DFAISS_ENABLE_GPU=OFF \
+          -DFAISS_ENABLE_PYTHON=OFF \
+          -DBUILD_TESTING=OFF \
+          -DCMAKE_BUILD_TYPE=Release
+        cmake --build build -j $(nproc)
+        sudo cmake --install build
+
+    - name: Install dependencies (macOS)
+      if: inputs.use-homebrew == 'true'
+      shell: bash
+      run: |
+        brew install cmake libomp openblas faiss
+
+    - name: Build native library
+      shell: bash
+      run: |
+        ./paimon-faiss/paimon-faiss-jni/scripts/build-native.sh --clean --fat-lib --faiss-version ${{ inputs.faiss-version }}
+
+    - name: List built libraries (Linux)
+      if: inputs.use-homebrew == 'false' && inputs.platform == 'linux-amd64'
+      shell: bash
+      run: |
+        echo "=== Built libraries ==="
+        ls -la paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/
+        echo ""
+        echo "=== Library dependencies ==="
+        ldd paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/libpaimon_faiss_jni.so || true
+
+    - name: List built libraries (Linux AARCH64)
+      if: inputs.use-homebrew == 'false' && inputs.platform == 'linux-aarch64'
+      shell: bash
+      run: |
+        echo "=== Built libraries ==="
+        ls -la paimon-faiss/paimon-faiss-jni/src/main/resources/linux/aarch64/
+        echo ""
+        echo "=== Library dependencies ==="
+        ldd paimon-faiss/paimon-faiss-jni/src/main/resources/linux/aarch64/libpaimon_faiss_jni.so || true
+
+    - name: List built libraries (macOS)
+      if: inputs.use-homebrew == 'true'
+      shell: bash
+      run: |
+        echo "=== Built libraries ==="
+        ls -la paimon-faiss/paimon-faiss-jni/src/main/resources/darwin/aarch64/
+        echo ""
+        echo "=== Library dependencies ==="
+        otool -L paimon-faiss/paimon-faiss-jni/src/main/resources/darwin/aarch64/libpaimon_faiss_jni.dylib || true

.github/workflows/faiss-vector-index-tests.yml

@@ -0,0 +1,66 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: Faiss Vector Index Tests
+
+on:
+  push:
+    paths:
+      - 'paimon-faiss/**'
+      - '.github/workflows/faiss-vector-index-tests.yml'
+  pull_request:
+    paths:
+      - 'paimon-faiss/**'
+      - '.github/workflows/faiss-vector-index-tests.yml'
+
+env:
+  JDK_VERSION: 8
+  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build_test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'temurin'
+
+      - name: Build FAISS native library
+        uses: ./.github/actions/build-faiss-native
+        with:
+          platform: linux-amd64
+
+      - name: Build paimon-faiss-jni
+        shell: bash
+        run: mvn -B clean install -pl paimon-faiss/paimon-faiss-jni -am -DskipTests -Ppaimon-faiss
+
+      - name: Test paimon-faiss-jni
+        timeout-minutes: 10
+        run: mvn -T 1C  -B test -pl paimon-faiss/paimon-faiss-jni -DskipFaissTests=false -Ppaimon-faiss
+        env:
+          MAVEN_OPTS: -Xmx2048m

.github/workflows/utitcase.yml

@@ -27,6 +27,8 @@ on:
       - 'paimon-python/**'
       - '.github/workflows/paimon-python-checks.yml'
       - 'paimon-lucene/**'
+      - 'paimon-faiss/**'
+      - '.github/workflows/faiss-vector-index-tests.yml'
 
 env:
   JDK_VERSION: 8
@@ -62,7 +64,7 @@ jobs:
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
 
-          TEST_MODULES="!paimon-e2e-tests,"
+          TEST_MODULES="!paimon-e2e-tests,!paimon-faiss/paimon-faiss-jni,"
           for suffix in ut 3.5 3.4 3.3 3.2; do
             TEST_MODULES+="!org.apache.paimon:paimon-spark-${suffix}_2.12,"
           done

.gitignore

@@ -31,3 +31,9 @@ paimon-lucene/.idea/
 
 ### Mac OS ###
 .DS_Store
+
+### Native build artifacts ###
+paimon-faiss/paimon-faiss-jni/build/
+paimon-faiss/paimon-faiss-jni/src/main/resources/darwin*
+paimon-faiss/paimon-faiss-jni/src/main/resources/linux*
+paimon-faiss/paimon-faiss-jni/src/main/native/cmake-build-debug/

paimon-faiss/paimon-faiss-jni/NOTICE

@@ -0,0 +1,9 @@
+Apache Paimon Faiss JNI
+Copyright 2024-2026 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (https://www.apache.org/).
+
+This product includes Faiss (https://github.com/facebookresearch/faiss)
+developed by Facebook AI Research under the MIT License.
+

paimon-faiss/paimon-faiss-jni/pom.xml

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>paimon-faiss</artifactId>
+        <groupId>org.apache.paimon</groupId>
+        <version>1.4-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>paimon-faiss-jni</artifactId>
+    <name>Paimon : Faiss JNI</name>
+
+    <properties>
+        <faiss.version>1.7.4</faiss.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!-- Resources plugin to include native libraries -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <configuration>
+                    <nonFilteredFileExtensions>
+                        <nonFilteredFileExtension>so</nonFilteredFileExtension>
+                        <nonFilteredFileExtension>dylib</nonFilteredFileExtension>
+                    </nonFilteredFileExtensions>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <profiles>
+        <!-- Profile for deploying to Maven Central with all platform native libraries -->
+        <profile>
+            <id>release</id>
+            <build>
+                <plugins>
+                    <!-- Include all platform native libraries -->
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-resources-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>copy-native-libs</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>copy-resources</goal>
+                                </goals>
+                                <configuration>
+                                    <outputDirectory>${project.build.outputDirectory}</outputDirectory>
+                                    <resources>
+                                        <resource>
+                                            <directory>${project.basedir}/src/main/resources</directory>
+                                            <includes>
+                                                <include>**/*.so</include>
+                                                <include>**/*.dylib</include>
+                                            </includes>
+                                        </resource>
+                                    </resources>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>
+