From 43d986cc98cb4360fb5af62c5f30ec62530e8aaa Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Tue, 15 Apr 2025 23:20:44 +0100 Subject: [PATCH] Switch to manylinux_2_28 This change bring in the updated build platform for Linux added in duckdb/duckdb#16956. Additionally Linux AArch64 builds are switched to AArch64 GH runners to be built natively without cross-compilation. --- .../actions/manylinux_2014_setup/action.yml | 111 ------------------ .github/actions/ubuntu_18_setup/action.yml | 100 ---------------- .github/workflows/Java.yml | 108 ++++++++--------- scripts/setup_manylinux2014.sh | 84 ------------- 4 files changed, 52 insertions(+), 351 deletions(-) delete mode 100644 .github/actions/manylinux_2014_setup/action.yml delete mode 100644 .github/actions/ubuntu_18_setup/action.yml delete mode 100755 scripts/setup_manylinux2014.sh diff --git a/.github/actions/manylinux_2014_setup/action.yml b/.github/actions/manylinux_2014_setup/action.yml deleted file mode 100644 index 5efb89ae9..000000000 --- a/.github/actions/manylinux_2014_setup/action.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: "Setup manylinux2014 image" -description: "Installs/configures additional CI dependencies" -inputs: - aws-cli: - description: 'Setup aws-cli' - default: 0 - ninja-build: - description: 'Setup ninja-build' - default: 0 - vcpkg: - description: 'Setup vcpkg (installs to $GITHUB_WORKSPACE/vcpkg)' - default: 0 - openssl: - description: 'Setup OpenSSL (requires vcpkg to also be installed)' - default: 0 - ccache: - description: 'Setup Ccache' - default: 0 - jdk: - description: 'Setup JDK' - default: 0 - odbc: - description: 'Setup ODBC' - default: 0 - ssh: - description: 'Setup SSH' - default: 0 - glibc32: - description: 'Setup 32bit glibc' - default: 0 - nodejs: - description: 'Setup NodeJS' - default: 0 - gcc_4_8: - description: 'Setup GCC 4.8 (installs to /usr/bin/g++, default will still be GCC 10)' - default: 0 - python_alias: - description: 'Create an alias for python3 to python3.9' - default: 0 - -runs: - using: "composite" - steps: - - name: Setup general dependencies - shell: bash - run: scripts/setup_manylinux2014.sh general - - - name: Install AWS CLI - if: ${{ inputs.aws-cli == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh aws-cli - - - name: Setup dependencies for ODBC - if: ${{ inputs.odbc == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh odbc - - - name: Setup dependencies for ccache - if: ${{ inputs.ccache == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh ccache - - - name: Setup JDK - if: ${{ inputs.jdk == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh jdk - - - name: Setup SSH - if: ${{ inputs.ssh == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh ssh - - - name: Setup NodeJS - if: ${{ inputs.nodejs == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh nodejs - - - name: Setup 32bit compiler - if: ${{ inputs.glibc32 == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh glibc32 - - - name: Setup python3 as python3.9 - if: ${{ inputs.python_alias == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh python_alias - - - name: Setup old (GCC 4.8) compiler - if: ${{ inputs.gcc_4_8 == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh gcc_4_8 - - # Note instead of using scripts/setup_manylinux2014.sh vcpkg, we prefer to use - # lukka/run-vcpkg@v11.1 here as it configures vcpkg to cache to GH actions. - - name: Setup vcpkg - if: ${{ inputs.vcpkg == 1 }} - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 - - - name: Install OpenSSL - if: ${{ inputs.openssl == 1 }} - shell: bash - run: scripts/setup_manylinux2014.sh openssl - - - name: Setup Ccache - if: ${{ inputs.ccache == 1 }} - uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases - with: - key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }} diff --git a/.github/actions/ubuntu_18_setup/action.yml b/.github/actions/ubuntu_18_setup/action.yml deleted file mode 100644 index c01e8ce7d..000000000 --- a/.github/actions/ubuntu_18_setup/action.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: "Setup Ubuntu 18" -description: "Setup an Ubuntu 18 docker container with the required libraries" -inputs: - openssl: - description: 'OpenSSL' - default: 0 - python: - description: 'Python' - default: 1 - aarch64_cross_compile: - description: 'Install dependencies for aarch64 cross-compiling' - default: 0 - vcpkg: - description: 'Install vcpkg' - default: 0 - ccache: - description: 'Install ccache' - default: 0 - -runs: - using: "composite" - steps: - - name: Install - shell: bash - run: | - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config - - - name: Install - shell: bash - if: ${{ inputs.aarch64_cross_compile == 1 }} - run: | - apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Install Git 2.18.5 - shell: bash - run: | - wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz - tar xvf v2.18.5.tar.gz - cd git-2.18.5 - make - make prefix=/usr install - git --version - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Install CMake 3.21 - shell: bash - run: | - wget https://github.com/Kitware/CMake/releases/download/v3.21.3/cmake-3.21.3-linux-x86_64.sh - chmod +x cmake-3.21.3-linux-x86_64.sh - ./cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/usr/local - cmake --version - - - name: Install Python 3.8 - if: ${{ inputs.python }} == 1 - shell: bash - run: | - wget https://www.python.org/ftp/python/3.8.17/Python-3.8.17.tgz - tar xvf Python-3.8.17.tgz - cd Python-3.8.17 - mkdir -p pythonbin - ./configure --with-ensurepip=install - make -j - make install - python3.8 --version - python3.8 -m pip install pip - python3.8 -m pip install requests awscli - - - name: Version Check - shell: bash - run: | - ldd --version ldd - python3 --version - git --version - git log -1 --format=%h - - - name: Setup vcpkg - if: ${{ inputs.vcpkg == 1 }} - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 - - - name: Setup Ccache - if: ${{ inputs.ccache == 1 }} - uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases - with: - key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }} - - - name: Instal OpenSSL through vcpkg - if: ${{ inputs.openssl == 1 }} - shell: bash - run: | - cd $VCPKG_ROOT && ./vcpkg install openssl --triplet=${{ inputs.aarch64_cross_compile == 1 && 'arm64-linux' || 'x64-linux' }} \ No newline at end of file diff --git a/.github/workflows/Java.yml b/.github/workflows/Java.yml index 5a09d3bc3..315bf2e22 100644 --- a/.github/workflows/Java.yml +++ b/.github/workflows/Java.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.git_ref }} @@ -27,33 +27,35 @@ jobs: java-linux-amd64: name: Java Linux (amd64) runs-on: ubuntu-latest - container: - image: quay.io/pypa/manylinux_2_28_x86_64 env: - GEN: ninja - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + MANYLINUX_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 + MANYLINUX_PACKAGES: java-1.8.0-openjdk-devel ninja-build steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.git_ref }} - - uses: ./.github/actions/manylinux_2014_setup - with: - ninja-build: 1 - ccache: 1 - jdk: 1 - python_alias: 1 - aws-cli: 1 - - name: Build shell: bash - run: make release - - - name: Java Tests + run: | + docker run \ + -v.:/duckdb \ + -e GEN=ninja \ + -e JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk \ + ${{ env.MANYLINUX_IMAGE }} \ + bash -c 'dnf install ${{ env.MANYLINUX_PACKAGES }} -y && make -C /duckdb release' + + - name: JDBC Tests shell: bash if: ${{ inputs.skip_tests != 'true' }} - run: make test + run: | + docker run \ + -v.:/duckdb \ + -e GEN=ninja \ + -e JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk \ + ${{ env.MANYLINUX_IMAGE }} \ + bash -c 'dnf install ${{ env.MANYLINUX_PACKAGES }} -y && make -C /duckdb test' - name: Deploy shell: bash @@ -71,33 +73,37 @@ jobs: java-linux-aarch64: name: Java Linux (aarch64) - runs-on: ubuntu-latest - container: - image: ubuntu:20.04 + runs-on: ubuntu-24.04-arm needs: java-linux-amd64 env: - GEN: ninja - DUCKDB_PLATFORM: linux_arm64 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + MANYLINUX_IMAGE: quay.io/pypa/manylinux_2_28_aarch64 + MANYLINUX_PACKAGES: java-1.8.0-openjdk-devel ninja-build steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.git_ref }} - - uses: ./.github/actions/ubuntu_18_setup - with: - ccache: 1 - aarch64_cross_compile: 1 - - - name: Install Stuff - shell: bash - run: > - curl -L https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u345-b01/OpenJDK8U-jdk_x64_linux_hotspot_8u345b01.tar.gz | tar xvz - - name: Build shell: bash - run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ OVERRIDE_JDBC_OS_ARCH=arm64 make release + run: | + docker run \ + -v.:/duckdb \ + -e GEN=ninja \ + -e JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk \ + ${{ env.MANYLINUX_IMAGE }} \ + bash -c 'dnf install ${{ env.MANYLINUX_PACKAGES }} -y && make -C /duckdb release' + + - name: JDBC Tests + shell: bash + if: ${{ inputs.skip_tests != 'true' }} + run: | + docker run \ + -v.:/duckdb \ + -e GEN=ninja \ + -e JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk \ + ${{ env.MANYLINUX_IMAGE }} \ + bash -c 'dnf install ${{ env.MANYLINUX_PACKAGES }} -y && make -C /duckdb test' - name: Deploy shell: bash @@ -120,7 +126,7 @@ jobs: runs-on: windows-2019 needs: java-linux-amd64 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.git_ref }} @@ -162,7 +168,7 @@ jobs: runs-on: macos-14 needs: java-linux-amd64 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.git_ref }} @@ -210,7 +216,7 @@ jobs: - java-osx-universal steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.git_ref }} @@ -280,35 +286,25 @@ jobs: needs: java-linux-amd64 container: quay.io/pypa/manylinux_2_28_x86_64 env: - BUILD_JDBC: 1 GEN: ninja - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.git_ref }} - - uses: ./.github/actions/manylinux_2014_setup - with: - ninja-build: 1 - ccache: 1 - jdk: 1 - python_alias: 1 - aws-cli: 1 + - name: Install packages + shell: bash + run: | + dnf install java-1.8.0-openjdk-devel ninja-build -y - - name: Install + - name: Install CTS shell: bash run: | git clone https://github.com/cwida/jdbccts.git - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases - with: - key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }} - - name: Build shell: bash run: make release diff --git a/scripts/setup_manylinux2014.sh b/scripts/setup_manylinux2014.sh deleted file mode 100755 index 8e9a7d70e..000000000 --- a/scripts/setup_manylinux2014.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# This script is used to setup the additional dependencies required by duckdb -# for the manylinux2014 image. The reason for this to exist in a script is firstly to be able to -# easily reproduce the same environment locally. Secondly, these steps can be now used -# both in the x86 jobs as the aarch64 jobs on GH actions (aarch64 jobs need a special env). - -# Examples -# -### To install just basics: -# -# > scripts/setup_manylinux2014.sh general -# -### Install openssl (through vcpkg, which will be in /tmp/vcpkg), ccache and jdk -# -# > VCPKG_TARGET_DIR=/tmp scripts/setup_manylinux2014.sh general vcpkg openssl ccache jdk -# - -# Installs deps for a specific required dependency -install_deps() { - if [ "$1" = "general" ]; then - git config --global --add safe.directory '*' - yum install -y curl zip unzip tar - yum install -y ninja-build - - elif [ "$1" = "aws-cli" ]; then - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - ./aws/install - aws --version - - elif [ "$1" = "odbc" ]; then - yum install -y unixODBC-devel - - elif [ "$1" = "vcpkg" ]; then - # Note: it is preferred to use the lukka/run-vcpkg@v11 over this when running this - # in a Github Actions job. When running locally or in the aarch64, these can be used instead - # Also note that this action install. - # Note2: this installs vcpkg to $VCPKG_TARGET_DIR - ( - cd $VCPKG_TARGET_DIR ; - git clone https://github.com/Microsoft/vcpkg.git ; - git checkout a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 ; - cd vcpkg ; - ./bootstrap-vcpkg.sh - ) - export VCPKG_ROOT=$VCPKG_TARGET_DIR/vcpkg - - elif [ "$1" = "openssl" ]; then - yum install -y perl-IPC-Cmd - - elif [ "$1" = "ccache" ]; then - yum -y install ccache - - elif [ "$1" = "python_alias" ]; then - ln -fs /usr/local/bin/python3.9 /usr/local/bin/python3 - - elif [ "$1" = "jdk" ]; then - yum install -y java-11-openjdk-devel maven - - elif [ "$1" = "ssh" ]; then - yum install -y openssh-clients - - elif [ "$1" = "glibc32" ]; then - yum install -y libgcc*i686 libstdc++*i686 glibc*i686 libgfortran*i686 - - elif [ "$1" = "gcc_4_8" ]; then - yum install -y gcc-c++ - - elif [ "$1" = "nodejs" ]; then - yum install -y nodejs - - else - >&2 echo "unknown input for setup_manylinux2014.sh: '$1'" - exit $exit_code - fi -} - -for var in "$@" -do - install_deps $var -done - -