From a11a619985b30d04048e2d4cdac8ef260c2a4079 Mon Sep 17 00:00:00 2001 From: Zehua Zou <41586196+HuaHuaY@users.noreply.github.com> Date: Thu, 27 Nov 2025 15:12:13 +0800 Subject: [PATCH 1/3] refactor: use gcc-14 in ubuntu ci --- .devcontainer/Dockerfile.template | 5 +- .github/workflows/cpp-linter.yml | 5 ++ .github/workflows/docs.yml | 2 +- .github/workflows/license_check.yml | 2 +- .github/workflows/rc.yml | 11 +++- .github/workflows/sanitizer_test.yml | 3 ++ .github/workflows/test.yml | 11 ++++ .../IcebergThirdpartyToolchain.cmake | 1 + src/iceberg/table_metadata.cc | 52 ++++++++----------- 9 files changed, 57 insertions(+), 35 deletions(-) diff --git a/.devcontainer/Dockerfile.template b/.devcontainer/Dockerfile.template index 57333753c..bfde82584 100644 --- a/.devcontainer/Dockerfile.template +++ b/.devcontainer/Dockerfile.template @@ -26,10 +26,11 @@ RUN apt update && \ bash-completion \ build-essential \ ccache \ + clangd \ cmake \ curl \ - gcc \ - g++ \ + gcc-14 \ + g++-14 \ git \ htop \ libboost-all-dev \ diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 724950042..07fc9efb9 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -36,6 +36,9 @@ jobs: shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Run build + env: + CC: gcc-14 + CXX: g++-14 run: | mkdir build && cd build cmake .. -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON @@ -45,6 +48,8 @@ jobs: continue-on-error: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CC: gcc-14 + CXX: g++-14 with: style: file tidy-checks: '' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 396acacf4..23e4c4225 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,7 +17,7 @@ permissions: jobs: docs: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6.0.0 diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index 948cb22f6..d95491443 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -22,7 +22,7 @@ on: pull_request jobs: license-check: name: "License Check" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout repository uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 174c7c8df..a40989740 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -30,7 +30,7 @@ permissions: jobs: archive: name: Archive - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Checkout @@ -77,6 +77,13 @@ jobs: - macos-26 - ubuntu-24.04 # - windows-latest + include: + - os: ubuntu-24.04 + CC: gcc-14 + CXX: g++-14 + env: + CC: ${{ matrix.CC || 'cc' }} + CXX: ${{ matrix.CXX || 'c++' }} steps: - name: Checkout uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 @@ -109,7 +116,7 @@ jobs: if: github.ref_type == 'tag' needs: - verify - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write steps: diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 8ac8ca96b..eccdc20c6 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -46,6 +46,9 @@ jobs: shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Configure and Build with ASAN & UBSAN + env: + CC: gcc-14 + CXX: g++-14 run: | mkdir build && cd build cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b45594cd8..42fc4ddce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,9 +51,15 @@ jobs: run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Build Iceberg shell: bash + env: + CC: gcc-14 + CXX: g++-14 run: ci/scripts/build_iceberg.sh $(pwd) - name: Build Example shell: bash + env: + CC: gcc-14 + CXX: g++-14 run: ci/scripts/build_example.sh $(pwd)/example macos: name: AArch64 macOS 26 @@ -102,11 +108,16 @@ jobs: include: - title: AMD64 Ubuntu 24.04 runs-on: ubuntu-24.04 + CC: gcc-14 + CXX: g++-14 - title: AMD64 Windows 2025 runs-on: windows-2025 meson-setup-args: --vsenv - title: AArch64 macOS 26 runs-on: macos-26 + env: + CC: ${{ matrix.CC || 'cc' }} + CXX: ${{ matrix.CXX || 'c++' }} steps: - uses: actions/setup-python@v6 with: diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 04e01ed5a..39c85e2c3 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -97,6 +97,7 @@ function(resolve_arrow_dependency) set(ARROW_WITH_ZLIB ON) set(ZLIB_SOURCE "SYSTEM") set(ARROW_VERBOSE_THIRDPARTY_BUILD OFF) + set(CMAKE_CXX_STANDARD 17) fetchcontent_declare(VendoredArrow ${FC_DECLARE_COMMON_OPTIONS} diff --git a/src/iceberg/table_metadata.cc b/src/iceberg/table_metadata.cc index b8ddc958c..df23e2f92 100644 --- a/src/iceberg/table_metadata.cc +++ b/src/iceberg/table_metadata.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -184,51 +185,45 @@ Result TableMetadataCache::GetSnapshotsById Result TableMetadataCache::InitSchemasMap( const TableMetadata* metadata) { - SchemasMap schemas_map; - schemas_map.reserve(metadata->schemas.size()); - for (const auto& schema : metadata->schemas) { - if (schema->schema_id()) { - schemas_map.emplace(schema->schema_id().value(), schema); - } - } - return schemas_map; + return metadata->schemas | std::views::filter([](const auto& schema) { + return schema->schema_id().has_value(); + }) | + std::views::transform([](const auto& schema) { + return std::make_pair(schema->schema_id().value(), schema); + }) | + std::ranges::to(); } Result TableMetadataCache::InitPartitionSpecsMap( const TableMetadata* metadata) { - PartitionSpecsMap partition_specs_map; - partition_specs_map.reserve(metadata->partition_specs.size()); - for (const auto& spec : metadata->partition_specs) { - partition_specs_map.emplace(spec->spec_id(), spec); - } - return partition_specs_map; + return metadata->partition_specs | std::views::transform([](const auto& spec) { + return std::make_pair(spec->spec_id(), spec); + }) | + std::ranges::to(); } Result TableMetadataCache::InitSortOrdersMap( const TableMetadata* metadata) { - SortOrdersMap sort_orders_map; - sort_orders_map.reserve(metadata->sort_orders.size()); - for (const auto& order : metadata->sort_orders) { - sort_orders_map.emplace(order->order_id(), order); - } - return sort_orders_map; + return metadata->sort_orders | std::views::transform([](const auto& order) { + return std::make_pair(order->order_id(), order); + }) | + std::ranges::to(); } Result TableMetadataCache::InitSnapshotMap( const TableMetadata* metadata) { - SnapshotsMap snapshots_map; - snapshots_map.reserve(metadata->snapshots.size()); - for (const auto& snapshot : metadata->snapshots) { - snapshots_map.emplace(snapshot->snapshot_id, snapshot); - } - return snapshots_map; + return metadata->snapshots | std::views::transform([](const auto& snapshot) { + return std::make_pair(snapshot->snapshot_id, snapshot); + }) | + std::ranges::to(); } // TableMetadataUtil implementation Result TableMetadataUtil::CodecFromFileName( std::string_view file_name) { - if (file_name.find(".metadata.json") == std::string::npos) { + auto pos = file_name.find_last_of(".metadata.json"); + if (pos == std::string::npos) { return InvalidArgument("{} is not a valid metadata file", file_name); } @@ -237,8 +232,7 @@ Result TableMetadataUtil::CodecFromFileName( return MetadataFileCodecType::kGzip; } - std::string_view file_name_without_suffix = - file_name.substr(0, file_name.find_last_of(".metadata.json")); + std::string_view file_name_without_suffix = file_name.substr(0, pos); if (file_name_without_suffix.ends_with(".gz")) { return MetadataFileCodecType::kGzip; } From ac01e9895831734b5e83c623f201cd80545165ac Mon Sep 17 00:00:00 2001 From: Zehua Zou <41586196+HuaHuaY@users.noreply.github.com> Date: Thu, 27 Nov 2025 17:05:53 +0800 Subject: [PATCH 2/3] fix some ci --- .github/workflows/cpp-linter.yml | 2 -- .github/workflows/rc.yml | 9 ++++++--- .github/workflows/test.yml | 8 +++++--- cmake_modules/IcebergThirdpartyToolchain.cmake | 4 ++-- src/iceberg/iceberg-config.cmake.in | 13 ------------- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 07fc9efb9..438932d59 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -48,8 +48,6 @@ jobs: continue-on-error: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CC: gcc-14 - CXX: g++-14 with: style: file tidy-checks: '' diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index a40989740..55f50caa7 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -81,9 +81,6 @@ jobs: - os: ubuntu-24.04 CC: gcc-14 CXX: g++-14 - env: - CC: ${{ matrix.CC || 'cc' }} - CXX: ${{ matrix.CXX || 'c++' }} steps: - name: Checkout uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 @@ -92,6 +89,12 @@ jobs: with: name: archive + - name: Set Ubuntu Compilers + if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} + run: | + echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV + echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV + - name: Verify run: | tar_gz=$(echo apache-iceberg-cpp-*.tar.gz) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 42fc4ddce..3fafaa6f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,9 +115,6 @@ jobs: meson-setup-args: --vsenv - title: AArch64 macOS 26 runs-on: macos-26 - env: - CC: ${{ matrix.CC || 'cc' }} - CXX: ${{ matrix.CXX || 'c++' }} steps: - uses: actions/setup-python@v6 with: @@ -128,6 +125,11 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt + - name: Set Ubuntu Compilers + if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} + run: | + echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV + echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV - name: Build Iceberg run: | meson setup builddir ${{ matrix.meson-setup-args || '' }} diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 39c85e2c3..bb55ee575 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -324,7 +324,7 @@ function(resolve_croaring_dependency) set(CROARING_URL "$ENV{ICEBERG_CROARING_URL}") else() set(CROARING_URL - "https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v4.3.11.tar.gz") + "https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v4.4.3.tar.gz") endif() fetchcontent_declare(croaring @@ -543,7 +543,7 @@ function(resolve_cpr_dependency) RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL CURL) else() set(CPR_VENDORED FALSE) list(APPEND ICEBERG_SYSTEM_DEPENDENCIES cpr) diff --git a/src/iceberg/iceberg-config.cmake.in b/src/iceberg/iceberg-config.cmake.in index dd6e3a598..787fadcc6 100644 --- a/src/iceberg/iceberg-config.cmake.in +++ b/src/iceberg/iceberg-config.cmake.in @@ -72,19 +72,6 @@ endmacro() # Find system dependencies iceberg_find_dependencies("${ICEBERG_SYSTEM_DEPENDENCIES}") -# Create missing CRoaring targets if they don't exist -if(NOT TARGET roaring::roaring-headers) - add_library(roaring::roaring-headers INTERFACE IMPORTED) -endif() - -if(NOT TARGET roaring::roaring-headers-cpp) - add_library(roaring::roaring-headers-cpp INTERFACE IMPORTED) -endif() - -if(NOT TARGET CURL::libcurl) - add_library(CURL::libcurl INTERFACE IMPORTED) -endif() - include("${CMAKE_CURRENT_LIST_DIR}/iceberg-targets.cmake") if(TARGET iceberg::arrow_static) From 7ed84a227afa9636b69dbd9ad3062b2b8e38a39f Mon Sep 17 00:00:00 2001 From: Zehua Zou <41586196+HuaHuaY@users.noreply.github.com> Date: Thu, 27 Nov 2025 17:48:50 +0800 Subject: [PATCH 3/3] fix cpplint --- .github/workflows/cpp-linter.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 438932d59..5b511f2e0 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -57,7 +57,8 @@ jobs: thread-comments: true ignore: 'build|cmake_modules|ci' database: build - extra-args: '-I$PWD/src -I$PWD/build/src' + # need '-fno-builtin-std-forward_like', see https://github.com/llvm/llvm-project/issues/101614 + extra-args: '-I$PWD/src -I$PWD/build/src -fno-builtin-std-forward_like' - name: Fail fast?! if: steps.linter.outputs.checks-failed != 0 run: |