diff --git a/.github/workflows/rebuild_native_libs.yml b/.github/workflows/rebuild_native_libs.yml deleted file mode 100644 index 026d3c3e..00000000 --- a/.github/workflows/rebuild_native_libs.yml +++ /dev/null @@ -1,296 +0,0 @@ -name: Build and Push Release CXX Libraries -on: - workflow_dispatch: -# This workflow is triggered manually from the Actions tab. -# It's meant to be run on a PR that changes the native C++ code. - -# It builds native libraries for all supported platforms and pushes them to the current branch. -# It splits the building process into 3 build jobs: -# 1. build-macos - Builds native libraries for MacOS: both ARM and x64. -# It uses native runners for each platform, because cross compilation on MacOS is complicated. -# 2. build-others - Builds native libraries for x64 Linux, ARM Linux and Windows. -# It uses cross-compilation for ARM Linux and Windows. -# -# Each build job saves the resulting binaries to the cache under a unique key -# When all build jobs are finished, the collect-commit-and-push job restores the binaries from the cache -# and pushes them to the current branch. - -jobs: - build-all-macos: - strategy: - matrix: - # macos-14 = ARM M1 - # macos-15-intel = x64 - # if you change OS definitions then you need to change conditions in cache-save steps below - os: [ macos-14, macos-15-intel ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install toolchains (CXX/NASM) - run: | - # https://github.com/actions/runner-images/issues/12912 - # Temporary fix (mtopolnik 2025-09-02): GH Action pre-installs a custom cmake version, install then fails - brew uninstall cmake - brew install make cmake gcc nasm - - name: Build CXX Library - run: | - cd core - export MACOSX_DEPLOYMENT_TARGET=13.0 - cmake -B build/release -DCMAKE_BUILD_TYPE=Release - cmake --build build/release --config Release - - name: Copy darwin-aarch64 CXX Library to the final directory - if: ${{ matrix.os == 'macos-14' }} - run: | - mkdir -p core/src/main/resources/io/questdb/client/bin/darwin-aarch64/ - mkdir -p core/src/main/bin/darwin-aarch64/ - cp core/target/classes/io/questdb/client/bin-local/libquestdb.dylib core/src/main/resources/io/questdb/client/bin/darwin-aarch64/ - - name: Copy darwin-x86-64 CXX Library to the final directory - if: ${{ matrix.os == 'macos-15-intel' }} - run: | - mkdir -p core/src/main/resources/io/questdb/client/bin/darwin-x86-64/ - mkdir -p core/src/main/bin/darwin-x86-64/ - cp core/target/classes/io/questdb/client/bin-local/libquestdb.dylib core/src/main/resources/io/questdb/client/bin/darwin-x86-64/ - - name: Save darwin-aarch64 Libraries to Cache - if: ${{ matrix.os == 'macos-14' }} - uses: actions/cache/save@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib - key: nativelibs-armosx-${{ github.sha }} - - name: Save darwin-x86-64 Libraries to Cache - if: ${{ matrix.os == 'macos-15-intel' }} - uses: actions/cache/save@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib - key: nativelibs-osx-${{ github.sha }} - build-all-linux-x86-64: - runs-on: ubuntu-latest - # manylinux2014 is a container with new-ish compilers and tools, but old glibc - 2.17 - # 2.17 is old enough to be compatible with most Linux distributions out there - container: - image: quay.io/pypa/manylinux2014_x86_64 - volumes: - - /node20217:/node20217 - - /node20217:/__e/node20 - steps: - - name: Install tools, most are needed to build nasm - run: | - ldd --version - yum update -y - yum install 'perl(Env)' perl-Font-TTF perl-Sort-Versions gcc wget perf asciidoc xmlto ghostscript adobe-source-sans-pro-fonts adobe-source-code-pro-fonts rpm-build zstd curl -y - - name: Build nasm - # we need nasm 2.14+ due to this bug https://bugzilla.nasm.us/show_bug.cgi?id=3392205 - # manylinux2014 distribution includes nasm 2.10 - # the nasm project itself provides RPMs, but they built against a newer glibc and other dependencies too - # thus we take src.rpm from nasm project and rebuild it in the manylinux2014 container - # this way we get a nasm binary that is compatible with the manylinux2014 environment - run: | - wget https://www.nasm.us/pub/nasm/releasebuilds/2.16.03/linux/nasm-2.16.03-0.fc39.src.rpm - rpmbuild --rebuild ./nasm-2.16.03-0.fc39.src.rpm - rpm -i ~/rpmbuild/RPMS/x86_64/nasm-2.16.03-0.el7.x86_64.rpm - - name: Install Node.js 20 glibc2.17 - # A hack to override default nodejs 20 to a build compatible with older glibc. - # Inspired by https://github.com/pytorch/test-infra/pull/5959 If it's good for pytorch, it's good for us too! :) - # Q: Why do we need this hack at all? A: Because many github actions, include action/checkout@v4, depend on nodejs 20. - # GitHub Actions runner provides a build of nodejs 20 that requires a newer glibc than manylinux2014 has. - # Thus we download a build of nodejs 20 that is compatible with manylinux2014 and override the default one. - run: | - curl -LO https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz - tar -xf node-v20.9.0-linux-x64-glibc-217.tar.xz --strip-components 1 -C /node20217 - ldd /__e/node20/bin/node - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install up-to-date CMake - run: | - wget -nv https://github.com/Kitware/CMake/releases/download/v3.29.2/cmake-3.29.2-linux-x86_64.tar.gz - tar -zxf cmake-3.29.2-linux-x86_64.tar.gz - echo "PATH=`pwd`/cmake-3.29.2-linux-x86_64/bin/:$PATH" >> "$GITHUB_ENV" - - name: Install GraalVM JDK 25 (for jni.h) - run: | - wget -nv -O graalvm.tar.gz https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_linux-x64_bin.tar.gz - mkdir graalvm - tar xfz graalvm.tar.gz -C graalvm --strip-components=1 - echo "JAVA_HOME=`pwd`/graalvm" >> "$GITHUB_ENV" - - name: Generate Makefiles - run: | - cd ./core - # git submodule update --init - cmake -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S. - - name: Build linux-x86-64 CXX Library - run: | - cd core - cmake --build cmake-build-release --config Release - mkdir -p src/main/resources/io/questdb/client/bin/linux-x86-64/ - mkdir -p src/main/bin/linux-x86-64/ - cp target/classes/io/questdb/client/bin-local/libquestdb.so src/main/resources/io/questdb/client/bin/linux-x86-64/ - - name: Save linux-x86-64 Libraries to Cache - uses: actions/cache/save@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so - key: nativelibs-linux-${{ github.sha }} - build-all-linux-aarch64: - runs-on: ubuntu-22.04-arm - container: - image: quay.io/pypa/manylinux_2_28_aarch64 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install tooling - run: | - yum update -y - yum install wget nasm zstd -y - - name: Install GraalVM JDK 25 (for jni.h) - run: | - wget -v --timeout=180 -O graalvm.tar.gz https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_linux-aarch64_bin.tar.gz - mkdir graalvm - tar xfz graalvm.tar.gz -C graalvm --strip-components=1 - echo "JAVA_HOME=`pwd`/graalvm" >> "$GITHUB_ENV" - - name: CMAKE linux-aarch64 - run: | - cd ./core - cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/linux-arm64.cmake -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-arm64 -S. - - name: Build linux-aarch64 CXX Library - run: | - cd core - cmake --build cmake-build-release-arm64 --config Release - mkdir -p src/main/resources/io/questdb/client/bin/linux-aarch64/ - mkdir -p src/main/bin/linux-aarch64/ - cp target/classes/io/questdb/client/bin-local/libquestdb.so src/main/resources/io/questdb/client/bin/linux-aarch64/ - - name: Save linux-aarch64 Libraries to Cache - uses: actions/cache/save@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so - key: nativelibs-armlinux-${{ github.sha }} - build-cxx-windows: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Increase file count and install tooling - run: | - sudo sysctl -w fs.file-max=500000 - sudo apt-get update -y - sudo apt-get install -y nasm gcc-mingw-w64 g++-mingw-w64 - - name: Install GraalVM JDK 25 (for jni.h) - run: | - wget -nv -O graalvm.tar.gz https://download.oracle.com/graalvm/25/latest/graalvm-jdk-25_linux-x64_bin.tar.gz - mkdir graalvm - tar xfz graalvm.tar.gz -C graalvm --strip-components=1 - echo "JAVA_HOME=`pwd`/graalvm" >> "$GITHUB_ENV" - - name: Download windows jni_md.h from JDK 25 - run: | - cd core - curl https://raw.githubusercontent.com/openjdk/jdk25u/master/src/java.base/windows/native/include/jni_md.h > $JAVA_HOME/include/jni_md.h - - name: CMake Windows - run: | - cd core - cmake -DCMAKE_TOOLCHAIN_FILE=./src/main/c/toolchains/windows-x86_64.cmake -DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -B cmake-build-release-win64 - - name: Build Windows CXX Library - run: | - cd core - cmake --build cmake-build-release-win64 --config Release - mkdir -p src/main/resources/io/questdb/client/bin/windows-x86-64/ - cp target/classes/io/questdb/client/bin-local/libquestdb.dll src/main/resources/io/questdb/client/bin/windows-x86-64/ - - name: Check CXX runtime dependency - run: | - cd ./core - if x86_64-w64-mingw32-objdump -p ./src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll | grep -q libstdc++; then - echo "Failure: CXX runtime dependency detected" - exit 1 - fi - - name: Check git status - run: | - git status - - name: Save Windows CXX Library to Cache - uses: actions/cache/save@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll - key: nativelibs-windows-${{ github.sha }} - collect-commit-and-push: - needs: - [ - build-all-macos, - build-cxx-windows, - build-all-linux-x86-64, - build-all-linux-aarch64, - ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Print file sizes before - run: | - mkdir -p ./core/src/main/resources/io/questdb/client/bin/ - find ./core/src/main/resources/io/questdb/client/bin/ -type f -exec ls -l {} \; || true - - name: Restore darwin-aarch64 Libraries from Cache - uses: actions/cache/restore@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib - key: nativelibs-armosx-${{ github.sha }} - - name: Restore darwin-x86-64 Libraries from Cache - uses: actions/cache/restore@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib - key: nativelibs-osx-${{ github.sha }} - - name: Restore linux-x86-64 Libraries from Cache - uses: actions/cache/restore@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so - key: nativelibs-linux-${{ github.sha }} - - name: Restore linux-aarch64 Libraries from Cache - uses: actions/cache/restore@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so - key: nativelibs-armlinux-${{ github.sha }} - - name: Restore Windows CXX Library from Cache - uses: actions/cache/restore@v3 - with: - path: | - core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll - key: nativelibs-windows-${{ github.sha }} - - name: Check git status before - run: | - git status - - name: Commit the files - run: | - git config --global user.name 'GitHub Actions - Rebuild Native Libraries' - git config --global user.email 'jaromir@questdb.io' - git add core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib - git add core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib - git add core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so - git add core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so - git add core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll - - echo "Removing exec permissions in Git index..." - git update-index --chmod=-x core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so - git update-index --chmod=-x core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so - git update-index --chmod=-x core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib - git update-index --chmod=-x core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib - - git commit -m "Rebuild CXX libraries" - - name: Check git status after - run: | - git status - - name: Print file sizes after - run: | - find ./core/src/main/resources/io/questdb/client/bin/ -type f -exec ls -l {} \; - - name: Push the files to the current branch - uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df - # Why do we use a commit hash instead of a tag for the github-push-action? - # ad-m/github-push-action is not as well-known repo as e.g. actions/checkout, and therefore we trust it less. - # d91a48109 is the same as the tag v0.8.0, but it's guaranteed to be immutable. - # So even if a bad actor takes over the repo, and rewrites tags to point to malicious commits, we will still be safe. - with: - branch: ${{ github.head_ref || github.ref_name }} diff --git a/.gitignore b/.gitignore index 9859a7c6..c886430c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ win64svc/cmake-build-debug win64svc/build win64svc/CMakeCache.txt win64svc/cmake-build-release -core/src/main/resources/io/questdb/client/bin/linux/libasm.a +core/src/main/resources/io/questdb/client/bin/ core/questdb/client/bin-local core/cmake-build-debug core/cmake-build-debug-coverage @@ -29,4 +29,4 @@ core/CMakeCache.txt **/build **/CMakeFiles .envrc -.vscode \ No newline at end of file +.vscode diff --git a/README.md b/README.md index ab127c6e..e764c76d 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,25 @@ Full release procedure, one-time setup, and failure handling: [artifacts/release ### Building Native Libraries -The client includes native libraries (C/C++ and assembly) for performance-critical operations. Pre-built binaries are included in the repository, but you can rebuild them locally if needed. +The client includes native libraries (C/C++ and assembly) for performance-critical operations. Native binaries are +built on demand by Maven and packaged into the client jar under `io/questdb/client/bin//`. + +For normal local development, initialize submodules once and run Maven: + +```bash +git submodule update --init --recursive +mvn -pl core package +``` + +Maven builds the current platform's native library during `generate-resources`. To skip the native build explicitly, +use `-Dquestdb.client.native.skip=true`. + +Release builds still use `-P include-native-artifacts`: CI builds each supported platform, stages the results under +`core/target/native-libs`, and Maven copies those staged binaries into the released jar. + +For IntelliJ, add `core/native/intellij_triggers.xml` as an Ant build file and configure the +`questdb-client-native-build` target to run before compilation. This mirrors the command-line Maven native build for +IDE test runs. #### Prerequisites @@ -279,7 +297,7 @@ brew install cmake nasm # Set deployment target export MACOSX_DEPLOYMENT_TARGET=13.0 -# Build native library +# Build native library directly with CMake cd core cmake -B cmake-build-release -DCMAKE_BUILD_TYPE=Release cmake --build cmake-build-release --config Release @@ -291,7 +309,7 @@ cmake --build cmake-build-release --config Release # Install build tools (Debian/Ubuntu) sudo apt-get install cmake nasm build-essential -# Build native library +# Build native library directly with CMake cd core cmake -DCMAKE_BUILD_TYPE=Release -B cmake-build-release -S. cmake --build cmake-build-release --config Release @@ -326,7 +344,7 @@ cmake --build cmake-build-release-win64 --config Release #### Native Library Output Locations -Built libraries are placed in the resources directory for each platform: +Direct CMake builds place libraries in the development resources directory: ``` core/target/classes/io/questdb/client/bin-local/ diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 3268313b..34fd3310 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -28,7 +28,7 @@ stages: - checkout: self fetchDepth: 0 lfs: false - submodules: false + submodules: recursive - bash: | git fetch origin $(System.PullRequest.SourceBranch) git checkout FETCH_HEAD @@ -58,7 +58,7 @@ stages: imageName: "macos-15" poolName: "Azure Pipelines" jdkArch: "x64" - windows-msvc-2022-x64: + windows-mingw-2022-x64: imageName: "windows-2022" poolName: "Azure Pipelines" jdkArch: "x64" @@ -70,7 +70,7 @@ stages: - checkout: self fetchDepth: 1 lfs: false - submodules: false + submodules: recursive - template: setup.yaml - bash: echo "##vso[task.setvariable variable=HOME]$USERPROFILE" displayName: "Set HOME on Windows" @@ -139,7 +139,7 @@ stages: - checkout: self fetchDepth: 1 lfs: false - submodules: false + submodules: recursive persistCredentials: true - template: setup.yaml - task: Cache@2 diff --git a/ci/setup.yaml b/ci/setup.yaml index 72c4a9dd..1c9818fd 100644 --- a/ci/setup.yaml +++ b/ci/setup.yaml @@ -1,4 +1,12 @@ steps: + - bash: git submodule update --init --recursive + displayName: "Initialize native submodules" + condition: ne(variables['Agent.OS'], 'Windows_NT') + + - powershell: git submodule update --init --recursive + displayName: "Initialize native submodules" + condition: eq(variables['Agent.OS'], 'Windows_NT') + - bash: | set -eux case "$(jdkArch)" in @@ -44,6 +52,43 @@ steps: displayName: "Install GraalVM JDK 25 (Windows)" condition: eq(variables['Agent.OS'], 'Windows_NT') + - bash: | + set -eux + sudo apt-get update + sudo apt-get install -y cmake nasm build-essential + displayName: "Install native build tools (Linux)" + condition: eq(variables['Agent.OS'], 'Linux') + + - bash: | + set -eux + for pkg in cmake nasm; do + brew list "$pkg" >/dev/null 2>&1 || brew install "$pkg" + done + displayName: "Install native build tools (macOS)" + condition: eq(variables['Agent.OS'], 'Darwin') + + - powershell: | + $ErrorActionPreference = "Stop" + + choco install msys2 --no-progress -y + + $msysRoot = @("C:\tools\msys64", "C:\msys64") | + Where-Object { Test-Path (Join-Path $_ "usr\bin\bash.exe") } | + Select-Object -First 1 + if (-not $msysRoot) { + throw "MSYS2 installation was not found" + } + + $bash = Join-Path $msysRoot "usr\bin\bash.exe" + & $bash -lc "pacman --noconfirm -Sy --needed mingw-w64-x86_64-binutils mingw-w64-x86_64-gcc mingw-w64-x86_64-nasm mingw-w64-x86_64-ninja" + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + Write-Host "##vso[task.prependpath]$msysRoot\mingw64\bin" + displayName: "Install native build tools (Windows)" + condition: eq(variables['Agent.OS'], 'Windows_NT') + - bash: sudo sysctl -w fs.file-max=500000 displayName: "Increase file count on Linux" condition: startsWith(variables['imageName'], 'ubuntu') diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 3538aa7f..91ab12e5 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -19,6 +19,9 @@ set(CMAKE_C_VISIBILITY_PRESET hidden) # use this is for debug purposes #set(CMAKE_VERBOSE_MAKEFILE ON) +set(QUESTDB_CLIENT_NATIVE_OUTPUT_DIR "" CACHE PATH "Native library output directory") +option(QUESTDB_CLIENT_NATIVE_COPY_BIN_LOCAL "Copy native build outputs into legacy bin-local development directories" ON) + # deal with windows slashes in JAVA_HOME if ($ENV{JAVA_HOME}) FILE(TO_CMAKE_PATH $ENV{JAVA_HOME} JAVA_HOME) @@ -146,11 +149,15 @@ else () ) endif () -set(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/target/classes/io/questdb/client/bin-local/) +if (QUESTDB_CLIENT_NATIVE_OUTPUT_DIR) + FILE(TO_CMAKE_PATH "${QUESTDB_CLIENT_NATIVE_OUTPUT_DIR}" OUTPUT) +else () + set(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/target/classes/io/questdb/client/bin-local/) +endif () -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${OUTPUT}) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${OUTPUT}) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OUTPUT}) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/archives") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${OUTPUT}") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${OUTPUT}") add_library(questdb SHARED ${SOURCE_FILES}) @@ -268,19 +275,20 @@ else () target_link_libraries(questdb rt -Wl,--gc-sections -Wl,--exclude-libs=ALL -static-libgcc) endif () -# Define the secondary output directory +if (QUESTDB_CLIENT_NATIVE_COPY_BIN_LOCAL) + # Define the secondary output directory + set(SECONDARY_OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/questdb/client/bin-local/) + set(SECONDARY_OUTPUT2 ${CMAKE_CURRENT_SOURCE_DIR}/target/classes/io/questdb/client/bin-local/) -set(SECONDARY_OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/questdb/client/bin-local/) -set(SECONDARY_OUTPUT2 ${CMAKE_CURRENT_SOURCE_DIR}/target/classes/io/questdb/client/bin-local/) - -add_custom_command(TARGET questdb POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${SECONDARY_OUTPUT} - COMMAND ${CMAKE_COMMAND} -E copy_directory ${OUTPUT} ${SECONDARY_OUTPUT} - COMMENT "Copying all build outputs from ${OUTPUT} to ${SECONDARY_OUTPUT}" -) + add_custom_command(TARGET questdb POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${SECONDARY_OUTPUT} + COMMAND ${CMAKE_COMMAND} -E copy_directory ${OUTPUT} ${SECONDARY_OUTPUT} + COMMENT "Copying all build outputs from ${OUTPUT} to ${SECONDARY_OUTPUT}" + ) -add_custom_command(TARGET questdb POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${SECONDARY_OUTPUT2} - COMMAND ${CMAKE_COMMAND} -E copy_directory ${OUTPUT} ${SECONDARY_OUTPUT2} - COMMENT "Copying all build outputs from ${OUTPUT} to ${SECONDARY_OUTPUT2}" -) + add_custom_command(TARGET questdb POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${SECONDARY_OUTPUT2} + COMMAND ${CMAKE_COMMAND} -E copy_directory ${OUTPUT} ${SECONDARY_OUTPUT2} + COMMENT "Copying all build outputs from ${OUTPUT} to ${SECONDARY_OUTPUT2}" + ) +endif () diff --git a/core/native/intellij_triggers.xml b/core/native/intellij_triggers.xml new file mode 100644 index 00000000..f3cef299 --- /dev/null +++ b/core/native/intellij_triggers.xml @@ -0,0 +1,36 @@ + + + IntelliJ integration to trigger the Maven steps that build the QuestDB Java client native library. + + To build the native library from IntelliJ before every test run: + * View -> Tool Windows -> Ant + * Add this file as an Ant build file + * Right click on "questdb-client-native-build" and select "Execute on" + * Select "Before Compilation" + + If the build fails, run "mvn package" from the command line to see the full Maven and CMake output. + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/core/pom.xml b/core/pom.xml index ad5391a2..c3190f78 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -37,6 +37,11 @@ None %regex[.*[^o].class] 1.37 + true + Release + ${project.build.directory}/cmake-build-${platform.dir.name} + ${project.build.outputDirectory}/io/questdb/client/bin/${platform.dir.name} + 1.3.4-SNAPSHOT @@ -169,6 +174,66 @@ false + + org.apache.maven.plugins + maven-clean-plugin + 3.4.1 + + + clean-native-resources + generate-resources + + clean + + + true + + + ${project.build.outputDirectory}/io/questdb/client/bin + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + + configure-native + generate-resources + + exec + + + ${questdb.client.native.skip} + ${project.basedir} + cmake + -S "${project.basedir}" -B "${questdb.client.native.build.dir}" -DCMAKE_BUILD_TYPE=${questdb.client.native.build.type} -DQUESTDB_CLIENT_NATIVE_OUTPUT_DIR="${questdb.client.native.output.dir}" -DQUESTDB_CLIENT_NATIVE_COPY_BIN_LOCAL=OFF ${questdb.client.native.configure.extra.args} + + + + build-native + generate-resources + + exec + + + ${questdb.client.native.skip} + ${project.basedir} + cmake + + --build + ${questdb.client.native.build.dir} + --config + ${questdb.client.native.build.type} + + + + + org.apache.maven.plugins maven-surefire-plugin @@ -228,6 +293,11 @@ windows + + windows-x86-64 + false + -G Ninja -DCMAKE_TOOLCHAIN_FILE="${project.basedir}/src/main/c/toolchains/windows-x86_64.cmake" + platform-linux-x86-64 @@ -239,6 +309,20 @@ linux-x86-64 + false + + + + platform-linux-x86_64 + + + unix + x86_64 + + + + linux-x86-64 + false @@ -249,6 +333,10 @@ amd64 + + freebsd-x86-64 + false + platform-linux-aarch64 @@ -260,6 +348,7 @@ linux-aarch64 + false @@ -272,6 +361,20 @@ darwin-x86-64 + false + + + + platform-osx-x86_64 + + + Mac + x86_64 + + + + darwin-x86-64 + false @@ -284,12 +387,73 @@ darwin-aarch64 + false include-native-artifacts + + true + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + validate-release-native-artifacts + validate + + run + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins maven-resources-plugin diff --git a/core/src/main/c/toolchains/windows-x86_64.cmake b/core/src/main/c/toolchains/windows-x86_64.cmake index 5cc44aa4..1fa5be33 100644 --- a/core/src/main/c/toolchains/windows-x86_64.cmake +++ b/core/src/main/c/toolchains/windows-x86_64.cmake @@ -1,16 +1,42 @@ set(CMAKE_SYSTEM_NAME Windows) set(CMAKE_SYSTEM_PROCESSOR x86_64) -set(CMAKE_AR ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-ar${CMAKE_EXECUTABLE_SUFFIX}) -set(CMAKE_ASM_COMPILER ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-gcc${CMAKE_EXECUTABLE_SUFFIX}) -set(CMAKE_C_COMPILER ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-gcc${CMAKE_EXECUTABLE_SUFFIX}) -set(CMAKE_CXX_COMPILER ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-g++${CMAKE_EXECUTABLE_SUFFIX}) -set(CMAKE_LINKER ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-ld${CMAKE_EXECUTABLE_SUFFIX}) -set(CMAKE_OBJCOPY ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-objcopy${CMAKE_EXECUTABLE_SUFFIX}) -set(CMAKE_RANLIB ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-ranlib${CMAKE_EXECUTABLE_SUFFIX}) -set(CMAKE_SIZE ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-size${CMAKE_EXECUTABLE_SUFFIX}) -set(CMAKE_STRIP ${WIN_TOOLCHAIN_PATH}x86_64-w64-mingw32-strip${CMAKE_EXECUTABLE_SUFFIX}) +set(QDB_WIN_TOOLCHAIN_PATH_ARGS) +if (WIN_TOOLCHAIN_PATH) + list(APPEND QDB_WIN_TOOLCHAIN_PATH_ARGS PATHS "${WIN_TOOLCHAIN_PATH}" NO_DEFAULT_PATH) +endif () + +function(qdb_find_win_tool out) + find_program(${out} NAMES ${ARGN} ${QDB_WIN_TOOLCHAIN_PATH_ARGS}) + if (NOT ${out}) + find_program(${out} NAMES ${ARGN}) + endif () + if (NOT ${out}) + list(GET ARGN 0 tool) + set(${out} "${WIN_TOOLCHAIN_PATH}${tool}${CMAKE_EXECUTABLE_SUFFIX}" CACHE FILEPATH "Windows toolchain ${tool}" FORCE) + endif () +endfunction() + +qdb_find_win_tool(QDB_WIN_AR x86_64-w64-mingw32-ar ar) +qdb_find_win_tool(QDB_WIN_ASM_COMPILER x86_64-w64-mingw32-gcc) +qdb_find_win_tool(QDB_WIN_C_COMPILER x86_64-w64-mingw32-gcc) +qdb_find_win_tool(QDB_WIN_CXX_COMPILER x86_64-w64-mingw32-g++) +qdb_find_win_tool(QDB_WIN_LINKER x86_64-w64-mingw32-ld ld) +qdb_find_win_tool(QDB_WIN_OBJCOPY x86_64-w64-mingw32-objcopy objcopy) +qdb_find_win_tool(QDB_WIN_RANLIB x86_64-w64-mingw32-ranlib ranlib) +qdb_find_win_tool(QDB_WIN_SIZE x86_64-w64-mingw32-size size) +qdb_find_win_tool(QDB_WIN_STRIP x86_64-w64-mingw32-strip strip) + +set(CMAKE_AR ${QDB_WIN_AR}) +set(CMAKE_ASM_COMPILER ${QDB_WIN_ASM_COMPILER}) +set(CMAKE_C_COMPILER ${QDB_WIN_C_COMPILER}) +set(CMAKE_CXX_COMPILER ${QDB_WIN_CXX_COMPILER}) +set(CMAKE_LINKER ${QDB_WIN_LINKER}) +set(CMAKE_OBJCOPY ${QDB_WIN_OBJCOPY}) +set(CMAKE_RANLIB ${QDB_WIN_RANLIB}) +set(CMAKE_SIZE ${QDB_WIN_SIZE}) +set(CMAKE_STRIP ${QDB_WIN_STRIP}) set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) \ No newline at end of file +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib b/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib deleted file mode 100644 index 82d21e59..00000000 Binary files a/core/src/main/resources/io/questdb/client/bin/darwin-aarch64/libquestdb.dylib and /dev/null differ diff --git a/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib b/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib deleted file mode 100644 index 647a12cb..00000000 Binary files a/core/src/main/resources/io/questdb/client/bin/darwin-x86-64/libquestdb.dylib and /dev/null differ diff --git a/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so b/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so deleted file mode 100644 index 94ad41c1..00000000 Binary files a/core/src/main/resources/io/questdb/client/bin/linux-aarch64/libquestdb.so and /dev/null differ diff --git a/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so b/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so deleted file mode 100644 index 15c0135d..00000000 Binary files a/core/src/main/resources/io/questdb/client/bin/linux-x86-64/libquestdb.so and /dev/null differ diff --git a/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll b/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll deleted file mode 100755 index e95dcecd..00000000 Binary files a/core/src/main/resources/io/questdb/client/bin/windows-x86-64/libquestdb.dll and /dev/null differ