Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 6 additions & 93 deletions .github/workflows/cpp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -280,100 +280,13 @@ jobs:
ci/scripts/cpp_test.sh $(pwd) $(pwd)/build

windows:
name: ${{ matrix.title }}
runs-on: ${{ matrix.os }}
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
os:
- windows-2022
include:
- os: windows-2022
simd-level: AVX2
title: AMD64 Windows 2022 AVX2
env:
ARROW_BOOST_USE_SHARED: OFF
ARROW_BUILD_BENCHMARKS: ON
ARROW_BUILD_SHARED: ON
ARROW_BUILD_STATIC: OFF
ARROW_BUILD_TESTS: ON
ARROW_DATASET: ON
ARROW_FLIGHT: OFF
ARROW_HDFS: ON
ARROW_HOME: /usr
ARROW_JEMALLOC: OFF
ARROW_MIMALLOC: ON
ARROW_ORC: ON
ARROW_PARQUET: ON
ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
ARROW_SUBSTRAIT: ON
ARROW_USE_GLOG: OFF
ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
ARROW_WITH_BROTLI: OFF
ARROW_WITH_BZ2: OFF
ARROW_WITH_LZ4: OFF
ARROW_WITH_OPENTELEMETRY: OFF
ARROW_WITH_SNAPPY: ON
ARROW_WITH_ZLIB: ON
ARROW_WITH_ZSTD: ON
BOOST_SOURCE: BUNDLED
CMAKE_CXX_STANDARD: "17"
CMAKE_GENERATOR: Ninja
CMAKE_INSTALL_PREFIX: /usr
CMAKE_UNITY_BUILD: ON
steps:
- name: Disable Crash Dialogs
run: |
reg add `
"HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
/v DontShowUI `
/t REG_DWORD `
/d 1 `
/f
- name: Checkout Arrow
uses: actions/checkout@v5
with:
fetch-depth: 0
submodules: recursive
- name: Download Timezone Database
shell: bash
run: ci/scripts/download_tz_database.sh
- name: Install ccache
shell: bash
run: |
ci/scripts/install_ccache.sh 4.6.3 /usr
- name: Setup ccache
shell: bash
run: |
ci/scripts/ccache_setup.sh
- name: ccache info
id: ccache-info
shell: bash
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- name: Cache ccache
uses: actions/cache@v4
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-ccache-windows-${{ env.CACHE_VERSION }}-${{ hashFiles('cpp/**') }}
restore-keys: cpp-ccache-windows-${{ env.CACHE_VERSION }}-
env:
# We can invalidate the current cache by updating this.
CACHE_VERSION: "2022-09-13"
- name: Build
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
- name: Test
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
# For ORC
set TZDIR=C:\msys64\usr\share\zoneinfo
bash -c "ci/scripts/cpp_test.sh $(pwd) $(pwd)/build"
name: AMD64 Windows 2022 AVX2 C++
uses: ./.github/workflows/cpp_windows.yml
with:
arch: x64
os: windows-2022
simd-level: AVX2

windows-mingw:
name: AMD64 Windows MinGW ${{ matrix.msystem_upper }} C++
Expand Down
13 changes: 13 additions & 0 deletions .github/workflows/cpp_extra.yml
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,19 @@ jobs:
continue-on-error: true
run: archery docker push ${{ matrix.image }}

msvc-arm64:
needs: check-labels
if: >-
needs.check-labels.outputs.force == 'true' ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
name: ARM64 Windows 11 MSVC
uses: ./.github/workflows/cpp_windows.yml
with:
arch: arm64
os: windows-11-arm
simd-level: NONE

jni-linux:
needs: check-labels
name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }}
Expand Down
124 changes: 124 additions & 0 deletions .github/workflows/cpp_windows.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: C++ Windows MSVC Reusable

on:
workflow_call:
inputs:
arch:
description: "The architecture to build for"
required: true
type: string
os:
description: "The runner label to run the job on"
required: true
type: string
simd-level:
description: "The Arrow SIMD level to build with"
required: true
type: string

jobs:
windows:
runs-on: ${{ inputs.os }}
timeout-minutes: 60
env:
ARROW_BOOST_USE_SHARED: OFF
ARROW_BUILD_BENCHMARKS: ON
ARROW_BUILD_SHARED: ON
ARROW_BUILD_STATIC: OFF
ARROW_BUILD_TESTS: ON
ARROW_DATASET: ON
ARROW_FLIGHT: OFF
ARROW_HDFS: ON
ARROW_HOME: /usr
ARROW_JEMALLOC: OFF
ARROW_MIMALLOC: ON
ARROW_ORC: ON
ARROW_PARQUET: ON
ARROW_SIMD_LEVEL: ${{ inputs.simd-level }}
ARROW_SUBSTRAIT: ON
ARROW_USE_GLOG: OFF
ARROW_VERBOSE_THIRDPARTY_BUILD: OFF
ARROW_WITH_BROTLI: OFF
ARROW_WITH_BZ2: OFF
ARROW_WITH_LZ4: OFF
ARROW_WITH_OPENTELEMETRY: OFF
ARROW_WITH_SNAPPY: ON
ARROW_WITH_ZLIB: ON
ARROW_WITH_ZSTD: ON
BOOST_SOURCE: BUNDLED
CMAKE_CXX_STANDARD: "17"
CMAKE_GENERATOR: Ninja
CMAKE_INSTALL_PREFIX: /usr
CMAKE_UNITY_BUILD: ON
steps:
- name: Disable Crash Dialogs
run: |
reg add `
"HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
/v DontShowUI `
/t REG_DWORD `
/d 1 `
/f
- name: Checkout Arrow
uses: actions/checkout@v5
with:
fetch-depth: 0
submodules: recursive
- name: Download Timezone Database
shell: bash
run: ci/scripts/download_tz_database.sh
- name: Install msys2 (for tzdata for ORC tests)
uses: msys2/setup-msys2@v2
id: setup-msys2
- name: Install cmake
shell: bash
run: |
ci/scripts/install_cmake.sh 4.1.2 /usr
- name: Install ccache
shell: bash
run: |
ci/scripts/install_ccache.sh 4.12.1 /usr
- name: Setup ccache
shell: bash
run: |
ci/scripts/ccache_setup.sh
- name: ccache info
id: ccache-info
shell: bash
run: |
echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
- name: Cache ccache
uses: actions/cache@v4
with:
path: ${{ steps.ccache-info.outputs.cache-dir }}
key: cpp-ccache-windows-${{ inputs.arch }}-${{ hashFiles('cpp/**') }}
restore-keys: cpp-ccache-windows-${{ inputs.arch }}-
- name: Build
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ inputs.arch }}
bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
- name: Test
shell: cmd
run: |
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ inputs.arch }}
# For ORC
set TZDIR=${{ steps.setup-msys2.outputs.msys2-location }}\usr\share\zoneinfo
bash -c "ci/scripts/cpp_test.sh $(pwd) $(pwd)/build"
13 changes: 11 additions & 2 deletions ci/scripts/install_ccache.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,19 @@ prefix=$2
mkdir -p /tmp/ccache
case $(uname) in
MINGW64*)
url="https://github.com/ccache/ccache/releases/download/v${version}/ccache-${version}-windows-x86_64.zip"
declare -A archs
archs=([64-bit]="x86_64"
[ARM 64-bit Processor]="aarch64")
arch=$(powershell -Command "(Get-CimInstance Win32_OperatingSystem).OSArchitecture")
if [ -z "${archs[$arch]}" ]; then
echo "Unsupported architecture on Windows: ${arch}"
exit 0
fi
arch=${archs[$arch]}
url="https://github.com/ccache/ccache/releases/download/v${version}/ccache-${version}-windows-${arch}.zip"
pushd /tmp/ccache
curl --fail --location --remote-name "${url}"
unzip -j "ccache-${version}-windows-x86_64.zip"
unzip -j "ccache-${version}-windows-${arch}.zip"
chmod +x ccache.exe
mv ccache.exe "${prefix}/bin/"
popd
Expand Down
39 changes: 23 additions & 16 deletions ci/scripts/install_cmake.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,13 @@ if [ "$#" -ne 2 ]; then
exit 1
fi

declare -A archs
archs=([x86_64]=x86_64
[arm64]=aarch64
[aarch64]=aarch64)
declare -A linux_archs
linux_archs=([x86_64]="x86_64"
[aarch64]="aarch64")

arch=$(uname -m)
if [ -z "${archs[$arch]}" ]; then
echo "Unsupported architecture: ${arch}"
exit 0
fi
arch=${archs[$arch]}
declare -A windows_archs
windows_archs=([64-bit]="x86_64"
[ARM 64-bit Processor]="arm64")

version=$1
prefix=$2
Expand All @@ -43,12 +39,25 @@ platform=$(uname)
case ${platform} in
Linux)
platform=linux
arch=$(uname -m)
if [ -z "${linux_archs[$arch]}" ]; then
echo "Unsupported architecture on Linux: ${arch}"
exit 0
fi
arch=${linux_archs[$arch]}
;;
Darwin)
platform=macos
arch=universal
;;
MSYS_NT*|MINGW64_NT*)
platform=windows
arch=$(powershell -Command "(Get-CimInstance Win32_OperatingSystem).OSArchitecture")
if [ -z "${windows_archs[$arch]}" ]; then
echo "Unsupported architecture on Windows: ${arch}"
exit 0
fi
arch=${windows_archs[$arch]}
;;
*)
echo "Unsupported platform: ${platform}"
Expand All @@ -59,22 +68,20 @@ esac
mkdir -p "${prefix}"
url="https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${platform}-"
case ${platform} in
macos)
url+="universal.tar.gz"
curl -L "${url}" | tar -xzf - --directory "${prefix}" --strip-components=1
ln -s CMake.app/Contents/bin "${prefix}/bin"
;;
windows)
url+="${arch}.zip"
archive_name=$(basename "${url}")
curl -L -o "${archive_name}" "${url}"
unzip "${archive_name}"
base_name=$(basename "${archive_name}" .zip)
mv "${base_name}"/* "${prefix}"
cp -a "${base_name}"/* "${prefix}"
rm -rf "${base_name}" "${archive_name}"
;;
*)
url+="${arch}.tar.gz"
curl -L "${url}" | tar -xzf - --directory "${prefix}" --strip-components=1
if [ "${platform}" = "macos" ]; then
ln -s CMake.app/Contents/bin "${prefix}/bin"
fi
;;
esac
5 changes: 5 additions & 0 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -1118,6 +1118,11 @@ function(build_boost)
# This is for https://github.com/boostorg/container/issues/305
string(APPEND CMAKE_C_FLAGS " -Wno-strict-prototypes")
endif()
if(MSVC AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "ARM64")
set(BOOST_CONTEXT_IMPLEMENTATION
winfib
CACHE STRING "" FORCE)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to use CACHE variable here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, otherwise it will default to fcontext and the build will fail. If cmake is run a second time after the initial configuration failed, somehow it then works. I thus decided to use CACHE FORCE to make it more reliable.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm. Could you try CMP0126 https://cmake.org/cmake/help/latest/policy/CMP0126.html and normal (not cache) variable?

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index e805694f52..ca6b9e8746 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -71,6 +71,12 @@ cmake_policy(SET CMP0090 NEW)
 # MSVC runtime library flags are selected by an abstraction.
 cmake_policy(SET CMP0091 NEW)
 
+# https://cmake.org/cmake/help/latest/policy/CMP0126.html
+#
+# The set(CACHE) command does not remove any normal variable of the
+# same name from the current scope.
+cmake_policy(SET CMP0126 NEW)
+
 # https://cmake.org/cmake/help/latest/policy/CMP0135.html
 #
 # CMP0135 is for solving re-building and re-downloading.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sadly it didn't work :/

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh...

endif()
set(CMAKE_UNITY_BUILD OFF)

fetchcontent_makeavailable(boost)
Expand Down
Loading