diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 50c0be9a8a6aa..88c6da6c62d4e 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -35,6 +35,10 @@
/doc/dev/cephadm @ceph/orchestrators
/doc/man/8/cephadm.rst @ceph/orchestrators
+/qa/suites/orch/rook @ceph/rook
+/src/pybind/mgr/rook @ceph/rook
+/doc/mgr/rook.rst @ceph/rook
+
#ceph-volume
/src/ceph-volume @ceph/ceph-volume
/doc/ceph-volume @ceph/ceph-volume
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000000000..98b7d53d81197
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,22 @@
+---
+version: 2
+updates:
+ - package-ecosystem: "npm"
+ directory: "/src/pybind/mgr/dashboard/frontend"
+ schedule:
+ interval: "daily"
+ commit-message:
+ prefix: "mgr/dashboard:"
+ labels:
+ - "dashboard"
+ pull-request-branch-name:
+ separator: "-"
+
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "daily"
+ commit-message:
+ prefix: ".github/workflows:"
+ pull-request-branch-name:
+ separator: "-"
diff --git a/.github/labeler.yml b/.github/labeler.yml
index dfc5bb64d05de..bf8c4d316833c 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -54,6 +54,7 @@ mgr:
- src/pybind/mgr/ceph_module.pyi
- src/pybind/mgr/mgr_module.py
- src/pybind/mgr/mgr_util.py
+ - src/pybind/mgr/object_format.py
- src/pybind/mgr/requirements.txt
- src/pybind/mgr/tox.ini
- src/test/mgr/**
@@ -246,10 +247,9 @@ nfs:
- src/pybind/mgr/nfs/**
- src/pybind/mgr/cephadm/services/nfs.py
- src/pybind/mgr/cephadm/templates/services/nfs/ganesha.conf.j2
- - src/pybind/mgr/dashboard/services/ganesha.py
- - src/pybind/mgr/dashboard/tests/test_ganesha.py
+ - src/pybind/mgr/dashboard/controllers/nfs.py
+ - src/pybind/mgr/dashboard/tests/test_nfs.py
- qa/tasks/cephfs/test_nfs.py
- - qa/tasks/mgr/dashboard/test_ganesha.py
- doc/mgr/nfs.rst
- doc/cephfs/nfs.rst
- doc/cephadm/nfs.rst
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 46ad1113ddd19..3f1e312d14592 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,41 +1,48 @@
-
-## Checklist
-- [ ] References tracker ticket
-- [ ] Updates documentation if necessary
-- [ ] Includes tests for new functionality or reproducer for bug
----
+## Contribution Guidelines
+- To sign and title your commits, please refer to [Submitting Patches to Ceph](https://github.com/ceph/ceph/blob/master/SubmittingPatches.rst).
+
+- If you are submitting a fix for a stable branch (e.g. "pacific"), please refer to [Submitting Patches to Ceph - Backports](https://github.com/ceph/ceph/blob/master/SubmittingPatches-backports.rst) for the proper workflow.
+
+## Checklist
+- Tracker (select at least one)
+ - [ ] References tracker ticket
+ - [ ] Very recent bug; references commit where it was introduced
+ - [ ] New feature (ticket optional)
+ - [ ] Doc update (no ticket needed)
+ - [ ] Code cleanup (no ticket needed)
+- Component impact
+ - [ ] Affects [Dashboard](https://tracker.ceph.com/projects/dashboard/issues/new), opened tracker ticket
+ - [ ] Affects [Orchestrator](https://tracker.ceph.com/projects/orchestrator/issues/new), opened tracker ticket
+ - [ ] No impact that needs to be tracked
+- Documentation (select at least one)
+ - [ ] Updates relevant documentation
+ - [ ] No doc update is appropriate
+- Tests (select at least one)
+ - [ ] Includes [unit test(s)](https://docs.ceph.com/en/latest/dev/developer_guide/tests-unit-tests/)
+ - [ ] Includes [integration test(s)](https://docs.ceph.com/en/latest/dev/developer_guide/testing_integration_tests/)
+ - [ ] Includes bug reproducer
+ - [ ] No tests
Show available Jenkins commands
@@ -54,5 +61,5 @@ https://raw.githubusercontent.com/ceph/ceph/master/SubmittingPatches.rst
- `jenkins render docs`
- `jenkins test ceph-volume all`
- `jenkins test ceph-volume tox`
-
+- `jenkins test windows`
diff --git a/.github/stale.yml b/.github/stale.yml
deleted file mode 100644
index d375db30a1bb2..0000000000000
--- a/.github/stale.yml
+++ /dev/null
@@ -1,66 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 60
-
-# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: 90
-
-# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels:
- - pinned
- - security
- - "[Status] Maybe Later"
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: true
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: true
-
-# Label to use when marking as stale
-staleLabel: stale
-
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: >
- This pull request has been automatically marked as stale because it
- has not had any activity for 60 days. It will be closed if no further
- activity occurs for another 30 days.
-
- If you are a maintainer or core committer, please follow-up on this
- pull request to identify what steps should be taken by the author to move this
- proposed change forward.
-
- If you are the author of this pull request, thank you for your proposed
- contribution. If you believe this change is still appropriate, please
- ensure that any feedback has been addressed and ask for a code review.
-
-# Comment to post when removing the stale label.
-# unmarkComment: >
-# Your comment here.
-
-# Comment to post when closing a stale Issue or Pull Request.
-closeComment: >
- This pull request has been automatically closed because there has
- been no activity for 90 days. Please feel free to reopen this pull
- request (or open a new one) if the proposed change is still
- appropriate. Thank you for your contribution!
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-only: pulls
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-# daysUntilStale: 30
-# markComment: >
-# This pull request has been automatically marked as stale because it has not had
-# recent activity. It will be closed if no further activity occurs. Thank you
-# for your contributions.
-
-# issues:
-# exemptLabels:
-# - confirmed
diff --git a/.github/workflows/pr-checklist.yml b/.github/workflows/pr-checklist.yml
new file mode 100644
index 0000000000000..c916db33ec411
--- /dev/null
+++ b/.github/workflows/pr-checklist.yml
@@ -0,0 +1,16 @@
+---
+name: "Pull Request Checklist"
+on:
+ pull_request:
+ types:
+ - edited
+ - opened
+ - reopened
+jobs:
+ pr_checklist:
+ runs-on: ubuntu-latest
+ name: Verify
+ steps:
+ - name: Action
+ id: checklist
+ uses: ceph/ceph-pr-checklist-action@32e92d1a2a7c9991ed51de5fccb2296551373d60
diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml
index 77fcff462e382..481f75331afc0 100644
--- a/.github/workflows/pr-triage.yml
+++ b/.github/workflows/pr-triage.yml
@@ -12,13 +12,13 @@ jobs:
with:
sync-labels: ''
repo-token: "${{ secrets.GITHUB_TOKEN }}"
+ - name: Assign to Dashboard project
+ uses: srggrs/assign-one-project-github-action@65a8ddab497df42ef268001e67bbf976f8fd39e1
+ if: contains(github.event.pull_request.labels.*.name, 'dashboard')
+ with:
+ project: https://github.com/ceph/ceph/projects/6
- name: Assign milestone based on target brach name
uses: iyu/actions-milestone@dbf7e5348844c9ddc6b803a5721b85fa70fe3bb9
with:
configuration-path: .github/milestone.yml
repo-token: "${{ secrets.GITHUB_TOKEN }}"
- - name: Assign to Dashboard project
- uses: srggrs/assign-one-project-github-action@65a8ddab497df42ef268001e67bbf976f8fd39e1
- if: contains(github.event.pull_request.labels.*.name, 'dashboard')
- with:
- project: https://github.com/ceph/ceph/projects/6
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
new file mode 100644
index 0000000000000..96fdc9fe07e73
--- /dev/null
+++ b/.github/workflows/stale.yml
@@ -0,0 +1,54 @@
+# Configuration for stale action workflow - https://github.com/actions/stale
+
+name: 'Close stale issues and PRs'
+on:
+ schedule:
+ - cron: '30 1 * * *'
+
+jobs:
+ stale:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/stale@v5
+ with:
+ # PAT for GitHub API authentication
+ repo-token: "${{ secrets.GITHUB_TOKEN }}"
+
+ # Comment on the staled PRs
+ stale-pr-message: >
+ This pull request has been automatically marked as stale because it
+ has not had any activity for 60 days. It will be closed if no further
+ activity occurs for another 30 days.
+
+ If you are a maintainer or core committer, please follow-up on this
+ pull request to identify what steps should be taken by the author to move this
+ proposed change forward.
+
+ If you are the author of this pull request, thank you for your proposed
+ contribution. If you believe this change is still appropriate, please
+ ensure that any feedback has been addressed and ask for a code review.
+
+ # Comment on the staled PRs while closed
+ close-pr-message: >
+ This pull request has been automatically closed because there has
+ been no activity for 90 days. Please feel free to reopen this pull
+ request (or open a new one) if the proposed change is still
+ appropriate. Thank you for your contribution!
+
+ # Idle number of days before making PRs stale (exempts Issues)
+ days-before-pr-stale: 60
+
+ # Idle number of days before closing stale PRs (exempts Issues)
+ days-before-pr-close: 90
+
+ # Label to apply on staled PRs
+ stale-pr-label: 'stale'
+
+ # Labels on PRs exempted from stale
+ exempt-pr-labels: 'pinned,security'
+
+ # Exempt all PRs with milestones from stale (also exempts Issues)
+ exempt-all-pr-milestones: true
+
+ # Max number of operations per run
+ operations-per-run: 30
diff --git a/.githubmap b/.githubmap
index cde518fa6522c..82f1c0a396623 100644
--- a/.githubmap
+++ b/.githubmap
@@ -9,7 +9,7 @@
#
a2batic Kanika Murarka
aaSharma14 Aashish Sharma
-aclamk Adam Kupczyk
+aclamk Adam Kupczyk
adamemerson Adam C. Emerson
adk3798 Adam King
ajarr Ramana Raja
@@ -34,6 +34,7 @@ capri1989 Kai Wagner
cbodley Casey Bodley
chardan Jesse Williamson
chhabaramesh Ramesh Chander
+chrisphoffman Christopher Hoffman
CourtneyCCaldwell Courtney Caldwell
Daniel-Pivonka Daniel Pivonka
ddiss David Disseldorp
@@ -75,9 +76,11 @@ LenzGr Lenz Grimmer
leseb Sébastien Han
liewegas Sage Weil
liupan1111 Pan Liu
+ljflores Laura Flores
lxbsz Xiubo Li
majianpeng Jianpeng Ma
markhpc Mark Nelson
+Matan-B Matan Breizman
mattbenjamin Matt Benjamin
mchangir Milind Changire
mgfritch Michael Fritch
@@ -85,6 +88,7 @@ mikechristie Mike Christie
mogeb Mohamad Gebai
myoungwon Myoungwon Oh
neha-ojha Neha Ojha
+NitzanMordhai Nitzan Mordechai
nizamial09 Nizamudeen A
noahdesu Noah Watkins
oritwas Orit Wasserman
@@ -101,6 +105,7 @@ ronen-fr Ronen Friedman
runsisi luo runbing
rzarzynski Radoslaw Zarzynski
s0nea Tatjana Dehler
+Sarthak0702 Sarthak Gupta
saschagrunert Sascha Grunert
sebastian-philipp Sebastian Wagner
ShyamsundarR Shyamsundar R
@@ -140,3 +145,6 @@ yuvalif Yuval Lifshitz
yuyuyu101 Haomai Wang
zdover23 Zac Dover
Thingee Mike Perez
+cfsnyder Cory Snyder
+benhanokh Gabriel Benhanokh
+kamoltat Kamoltat Sirivadhna
diff --git a/.gitignore b/.gitignore
index 4c9ff3b26deb6..b01aef839bef6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,3 +82,4 @@ GTAGS
# Python building things where it shouldn't
/src/python-common/build/
+.cache
diff --git a/.gitmodules b/.gitmodules
index c9da017ee6cd3..f828bb3769ae8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -62,15 +62,13 @@
[submodule "s3select"]
path = src/s3select
url = https://github.com/ceph/s3select.git
-[submodule "src/jaegertracing/opentracing-cpp"]
- path = src/jaegertracing/opentracing-cpp
- url = https://github.com/opentracing/opentracing-cpp.git
- branch = v1.6.0
-[submodule "src/jaegertracing/jaeger-client-cpp"]
- path = src/jaegertracing/jaeger-client-cpp
- url = https://github.com/ceph/jaeger-client-cpp.git
- branch = hunter-disabled
[submodule "src/libkmip"]
path = src/libkmip
url = https://github.com/ceph/libkmip
branch = ceph-master
+[submodule "src/arrow"]
+ path = src/arrow
+ url = https://github.com/apache/arrow.git
+[submodule "src/utf8proc"]
+ path = src/utf8proc
+ url = https://github.com/JuliaStrings/utf8proc
diff --git a/.mailmap b/.mailmap
index 77a329d750634..17e770065c9c6 100644
--- a/.mailmap
+++ b/.mailmap
@@ -14,7 +14,7 @@ Abhishek Lekshmanan
Adam C. Emerson
Adam King
Adam Kupczyk
-Adam Kupczyk
+Adam Kupczyk
Adam Twardowski
Adir Lev
Ahoussi Armand
@@ -96,6 +96,7 @@ Chris Holcombe
Christian Brunner
Christian Marie
Christophe Courtaut
+Christopher Hoffman
Chu Hua-Rong
Chu Hua-Rong
Chuanhong Wang
@@ -333,6 +334,7 @@ Kévin Caradant
Lan De
Lan Liu
Laszlo Boszormenyi
+Laura Flores
Laurent Voullemier
Lei Liu
Lenz Grimmer
@@ -387,6 +389,7 @@ Marco Garcês
Mark Kogan
Mark Nelson
Mark Nelson
+Matan Breizman
Mathijs Smit <5450789+bk203@users.noreply.github.com>
Matt Benjamin
Matt Benjamin
@@ -440,6 +443,7 @@ Nick Fisk
Nicolas Yong
Ning Yao
Ning Yao
+Nitzan Mordechai
Nizamudeen A
Noah Watkins
Noah Watkins
@@ -536,6 +540,7 @@ Sandon Van Ness
Sandon Van Ness
Sangdi Xu
Sangdi Xu
+Sarthak Gupta
Scott A. Brandt
Sebastian Wagner
Sebastian Wagner
diff --git a/.organizationmap b/.organizationmap
index 317a52e232a92..dc0b5e8a23ba0 100644
--- a/.organizationmap
+++ b/.organizationmap
@@ -533,6 +533,7 @@ Red Hat Brad Hubbard
Red Hat Brett Niver
Red Hat Brian Andrus
Red Hat Casey Bodley
+Red Hat Christopher Hoffman
Red Hat Cleber Rosa
Red Hat Colin Walters
Red Hat Courtney Caldwell
@@ -580,12 +581,14 @@ Red Hat Karun Josy
Red Hat Kefu Chai
Red Hat Ken Dreyer
Red Hat Kotresh HR
+Red Hat Laura Flores
Red Hat Loic Dachary
Red Hat Loic Dachary
Red Hat Luis Pabón
Red Hat Marcus Watts
Red Hat Mark Kogan
Red Hat Mark Nelson
+Red Hat Matan Breizman
Red Hat Matt Benjamin
Red Hat Mehdi Abaakouk
Red Hat Mike Christie
@@ -597,6 +600,7 @@ Red Hat Nathan Weinberg
Red Hat Neha Ojha
Red Hat Neil Levine
Red Hat Nilamdyuti Goswami
+Red Hat Nitzan Mordechai
Red Hat Nizamudeen A
Red Hat Noah Watkins
Red Hat Or Friedmann
@@ -623,6 +627,7 @@ Red Hat Sahid Orentino Ferdjaoui Sam Lang
Red Hat Samuel Just
Red Hat Sandon Van Ness
+Red Hat Sarthak Gupta
Red Hat Scoots Hamilton
Red Hat Sebastian Wagner
Red Hat Servesha Dudhgaonkar
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0350131765570..f4d6e14fbbcf4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,9 +14,18 @@ cmake_policy(SET CMP0065 NEW)
cmake_policy(SET CMP0074 NEW)
cmake_policy(SET CMP0075 NEW)
cmake_policy(SET CMP0093 NEW)
+if(POLICY CMP0127)
+ cmake_policy(SET CMP0127 NEW)
+endif()
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
+if(NOT CMAKE_BUILD_TYPE AND EXISTS "${CMAKE_SOURCE_DIR}/.git")
+ set(default_build_type "Debug")
+ set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE
+ STRING "Default BUILD_TYPE is Debug, other options are: RelWithDebInfo, Release, and MinSizeRel." FORCE)
+endif()
+
if(CMAKE_SYSTEM_NAME MATCHES "Linux")
set(LINUX ON)
FIND_PACKAGE(Threads)
@@ -154,6 +163,7 @@ if(LINUX)
elseif(FREEBSD)
set(HAVE_UDEV OFF)
set(HAVE_LIBAIO OFF)
+ set(HAVE_LIBDML OFF)
set(HAVE_BLKID OFF)
set(HAVE_KEYUTILS OFF)
else()
@@ -175,7 +185,7 @@ endif()
option(WITH_FUSE "Fuse is here" ON)
if(WITH_FUSE)
- find_package(FUSE)
+ find_package(FUSE REQUIRED)
set(HAVE_LIBFUSE ${FUSE_FOUND})
endif()
@@ -228,6 +238,10 @@ CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_LIBURING "Require and build with system libur
CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF
"WITH_BLUESTORE" OFF)
+if(WITH_BLUESTORE_PMEM)
+ find_package(dml)
+ set(HAVE_LIBDML ${DML_FOUND})
+endif()
CMAKE_DEPENDENT_OPTION(WITH_RBD_MIGRATION_FORMAT_QCOW_V1
"Enable librbd QCOW v1 migration format support" ON
@@ -267,10 +281,16 @@ endif()
option(WITH_BLUEFS "libbluefs library" OFF)
+option(WITH_QAT "Enable Qat driver" OFF)
+if(WITH_QAT)
+ find_package(QatDrv REQUIRED COMPONENTS qat_s usdm_drv_s)
+ set(HAVE_QATDRV $(QatDrv_FOUND))
+endif()
+
option(WITH_QATZIP "Enable QATZIP" OFF)
if(WITH_QATZIP)
find_package(qatzip REQUIRED)
- set(HAVE_QATZIP ${QATZIP_FOUND})
+ set(HAVE_QATZIP ${qatzip_FOUND})
endif(WITH_QATZIP)
# needs mds and? XXX
@@ -297,16 +317,6 @@ if(LINUX)
endif()
endif()
-option(WITH_LEVELDB "LevelDB is here" ON)
-if(WITH_LEVELDB)
- if(LEVELDB_PREFIX)
- include_directories(SYSTEM ${LEVELDB_PREFIX}/include)
- link_directories(${LEVELDB_PREFIX}/lib)
- endif()
- find_package(leveldb REQUIRED)
- find_file(HAVE_LEVELDB_FILTER_POLICY leveldb/filter_policy.h PATHS ${LEVELDB_INCLUDE_DIR})
-endif(WITH_LEVELDB)
-
find_package(snappy REQUIRED)
option(WITH_BROTLI "Brotli compression support" OFF)
@@ -331,9 +341,11 @@ and then jemalloc. If neither of then is found. use the one in libc.")
if(ALLOCATOR)
if(${ALLOCATOR} MATCHES "tcmalloc(_minimal)?")
find_package(gperftools 2.6.2 REQUIRED)
+ set(ALLOC_LIBS gperftools::${ALLOCATOR})
set(HAVE_LIBTCMALLOC ON)
elseif(${ALLOCATOR} STREQUAL "jemalloc")
find_package(JeMalloc REQUIRED)
+ set(ALLOC_LIBS JeMalloc::JeMalloc)
set(HAVE_JEMALLOC 1)
elseif(NOT ALLOCATOR STREQUAL "libc")
message(FATAL_ERROR "Unsupported allocator selected: ${ALLOCATOR}")
@@ -346,8 +358,10 @@ else(ALLOCATOR)
endif()
if(gperftools_FOUND)
set(ALLOCATOR tcmalloc)
+ set(ALLOC_LIBS gperftools::tcmalloc)
elseif(JeMalloc_FOUND)
set(ALLOCATOR jemalloc)
+ set(ALLOC_LIBS JeMalloc::JeMalloc)
else()
if(NOT FREEBSD)
# FreeBSD already has jemalloc as its default allocator
@@ -356,6 +370,13 @@ else(ALLOCATOR)
set(ALLOCATOR "libc")
endif(gperftools_FOUND)
endif(ALLOCATOR)
+if(NOT ALLOCATOR STREQUAL "libc")
+ add_compile_options(
+ $<$:-fno-builtin-malloc>
+ $<$:-fno-builtin-calloc>
+ $<$:-fno-builtin-realloc>
+ $<$:-fno-builtin-free>)
+endif()
# Mingw generates incorrect entry points when using "-pie".
if(WIN32 OR (HAVE_LIBTCMALLOC AND WITH_STATIC_LIBSTDCXX))
@@ -364,10 +385,8 @@ else()
set(EXE_LINKER_USE_PIE ${ENABLE_SHARED})
endif()
-find_package(CURL REQUIRED)
-set(CMAKE_REQUIRED_INCLUDES ${CURL_INCLUDE_DIRS})
-set(CMAKE_REQUIRED_LIBRARIES ${CURL_LIBRARIES})
-CHECK_SYMBOL_EXISTS(curl_multi_wait curl/curl.h HAVE_CURL_MULTI_WAIT)
+# require libcurl with good curl_multi_wait(), see https://tracker.ceph.com/issues/15915
+find_package(CURL 7.32 REQUIRED)
find_package(OpenSSL REQUIRED)
set(CRYPTO_LIBS OpenSSL::Crypto)
@@ -401,6 +420,11 @@ option(WITH_RADOSGW_AMQP_ENDPOINT "Rados Gateway's pubsub support for AMQP push
option(WITH_RADOSGW_KAFKA_ENDPOINT "Rados Gateway's pubsub support for Kafka push endpoint" ON)
option(WITH_RADOSGW_LUA_PACKAGES "Rados Gateway's support for dynamically adding lua packagess" ON)
option(WITH_RADOSGW_DBSTORE "DBStore backend for Rados Gateway" ON)
+option(WITH_RADOSGW_MOTR "CORTX-Motr backend for Rados Gateway" OFF)
+option(WITH_RADOSGW_SELECT_PARQUET "Support for s3 select on parquet objects" ON)
+
+option(WITH_SYSTEM_ARROW "Use system-provided arrow" OFF)
+option(WITH_SYSTEM_UTF8PROC "Use system-provided utf8proc" OFF)
if(WITH_RADOSGW)
find_package(EXPAT REQUIRED)
@@ -461,12 +485,8 @@ option(WITH_CEPHFS "CephFS is enabled" ON)
if(NOT WIN32)
# Please specify 3.[0-7] if you want to build with a certain version of python3.
set(WITH_PYTHON3 "3" CACHE STRING "build with specified python3 version")
-if(NOT WITH_PYTHON3 STREQUAL "3")
- set(find_python3_exact "EXACT")
-endif()
-find_package(Python3 ${WITH_PYTHON3} ${find_python3_exact} REQUIRED
+find_package(Python3 ${WITH_PYTHON3} EXACT REQUIRED
COMPONENTS Interpreter Development)
-unset(find_python3_exact)
option(WITH_MGR "ceph-mgr is enabled" ON)
if(WITH_MGR)
@@ -668,7 +688,6 @@ option(WITH_SYSTEMD "build with systemd support" ON)
add_subdirectory(src)
add_subdirectory(qa)
-
add_subdirectory(doc)
if(WITH_MANPAGE)
add_subdirectory(man)
@@ -683,9 +702,7 @@ if(LINUX)
endif()
option(WITH_GRAFANA "install grafana dashboards" OFF)
-if(WITH_GRAFANA)
- add_subdirectory(monitoring/grafana/dashboards)
-endif()
+add_subdirectory(monitoring/ceph-mixin)
CMAKE_DEPENDENT_OPTION(WITH_BOOST_VALGRIND "Boost support for valgrind" OFF
"NOT WITH_SYSTEM_BOOST" OFF)
diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index ff1b2a1aa10db..19833faafcbf3 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -1,403 +1,45 @@
->=17.0.0
-
-* `ceph-mgr-modules-core` debian package does not recommend `ceph-mgr-rook`
- anymore. As the latter depends on `python3-numpy` which cannot be imported in
- different Python sub-interpreters multi-times if the version of
- `python3-numpy` is older than 1.19. Since `apt-get` installs the `Recommends`
- packages by default, `ceph-mgr-rook` was always installed along with
- `ceph-mgr` debian package as an indirect dependency. If your workflow depends
- on this behavior, you might want to install `ceph-mgr-rook` separately.
-
-* the "kvs" Ceph object class is not packaged anymore. "kvs" Ceph object class
- offers a distributed flat b-tree key-value store implemented on top of librados
- objects omap. Because we don't have existing internal users of this object
- class, it is not packaged anymore.
-
-* A new library is available, libcephsqlite. It provides a SQLite Virtual File
- System (VFS) on top of RADOS. The database and journals are striped over
- RADOS across multiple objects for virtually unlimited scaling and throughput
- only limited by the SQLite client. Applications using SQLite may change to
- the Ceph VFS with minimal changes, usually just by specifying the alternate
- VFS. We expect the library to be most impactful and useful for applications
- that were storing state in RADOS omap, especially without striping which
- limits scalability.
-
-* The ``device_health_metrics`` pool has been renamed ``.mgr``. It is now
- used as a common store for all ``ceph-mgr`` modules.
-
-* fs: A file system can be created with a specific ID ("fscid"). This is useful
- in certain recovery scenarios, e.g., monitor database lost and rebuilt, and
- the restored file system is expected to have the same ID as before.
-
-* fs: A file system can be renamed using the `fs rename` command. Any cephx
- credentials authorized for the old file system name will need to be
- reauthorized to the new file system name. Since the operations of the clients
- using these re-authorized IDs may be disrupted, this command requires the
- "--yes-i-really-mean-it" flag. Also, mirroring is expected to be disabled
- on the file system.
-* MDS upgrades no longer require stopping all standby MDS daemons before
- upgrading the sole active MDS for a file system.
-
-* RGW: `radosgw-admin realm delete` is now renamed to `radosgw-admin realm rm`. This
- is consistent with the help message.
-
-* OSD: Ceph now uses mclock_scheduler as its default osd_op_queue to provide QoS.
-
-* CephFS: Failure to replay the journal by a standby-replay daemon will now
- cause the rank to be marked damaged.
-
-* RGW: S3 bucket notification events now contain an `eTag` key instead of `etag`,
- and eventName values no longer carry the `s3:` prefix, fixing deviations from
- the message format observed on AWS.
-
-* RGW: It is possible to specify ssl options and ciphers for beast frontend now.
- The default ssl options setting is "no_sslv2:no_sslv3:no_tlsv1:no_tlsv1_1".
- If you want to return back the old behavior add 'ssl_options=' (empty) to
- ``rgw frontends`` configuration.
-
-* RGW: The behavior for Multipart Upload was modified so that only
- CompleteMultipartUpload notification is sent at the end of the multipart upload.
- The POST notification at the beginning of the upload, and PUT notifications that
- were sent on each part are not sent anymore.
-
-* MGR: The pg_autoscaler has a new default 'scale-down' profile which provides more
- performance from the start for new pools (for newly created clusters).
- Existing clusters will retain the old behavior, now called the 'scale-up' profile.
- For more details, see:
-
- https://docs.ceph.com/en/latest/rados/operations/placement-groups/
-
->=16.0.0
---------
-* mgr/nfs: ``nfs`` module is moved out of volumes plugin. Prior using the
- ``ceph nfs`` commands, ``nfs`` mgr module must be enabled.
-
-* volumes/nfs: The ``cephfs`` cluster type has been removed from the
- ``nfs cluster create`` subcommand. Clusters deployed by cephadm can
- support an NFS export of both ``rgw`` and ``cephfs`` from a single
- NFS cluster instance.
-
-* The ``nfs cluster update`` command has been removed. You can modify
- the placement of an existing NFS service (and/or its associated
- ingress service) using ``orch ls --export`` and ``orch apply -i
- ...``.
-
-* The ``orch apply nfs`` command no longer requires a pool or
- namespace argument. We strongly encourage users to use the defaults
- so that the ``nfs cluster ls`` and related commands will work
- properly.
-
-* The ``nfs cluster delete`` and ``nfs export delete`` commands are
- deprecated and will be removed in a future release. Please use
- ``nfs cluster rm`` and ``nfs export rm`` instead.
-
-* mgr-pg_autoscaler: Autoscaler will now start out by scaling each
- pool to have a full complements of pgs from the start and will only
- decrease it when other pools need more pgs due to increased usage.
- This improves out of the box performance of Ceph by allowing more PGs
- to be created for a given pool.
-
-* CephFS: Disabling allow_standby_replay on a file system will also stop all
- standby-replay daemons for that file system.
-
-* New bluestore_rocksdb_options_annex config parameter. Complements
- bluestore_rocksdb_options and allows setting rocksdb options without repeating
- the existing defaults.
-* The MDS in Pacific makes backwards-incompatible changes to the ON-RADOS
- metadata structures, which prevent a downgrade to older releases
- (to Octopus and older).
-
-* $pid expansion in config paths like `admin_socket` will now properly expand
- to the daemon pid for commands like `ceph-mds` or `ceph-osd`. Previously only
- `ceph-fuse`/`rbd-nbd` expanded `$pid` with the actual daemon pid.
-
-* The allowable options for some "radosgw-admin" commands have been changed.
-
- * "mdlog-list", "datalog-list", "sync-error-list" no longer accept
- start and end dates, but do accept a single optional start marker.
- * "mdlog-trim", "datalog-trim", "sync-error-trim" only accept a
- single marker giving the end of the trimmed range.
- * Similarly the date ranges and marker ranges have been removed on
- the RESTful DATALog and MDLog list and trim operations.
-
-* ceph-volume: The ``lvm batch`` subcommand received a major rewrite. This
- closed a number of bugs and improves usability in terms of size specification
- and calculation, as well as idempotency behaviour and disk replacement
- process. Please refer to
- https://docs.ceph.com/en/latest/ceph-volume/lvm/batch/ for more detailed
- information.
-
-* Configuration variables for permitted scrub times have changed. The legal
- values for ``osd_scrub_begin_hour`` and ``osd_scrub_end_hour`` are ``0`` -
- ``23``. The use of 24 is now illegal. Specifying ``0`` for both values
- causes every hour to be allowed. The legal vaues for
- ``osd_scrub_begin_week_day`` and ``osd_scrub_end_week_day`` are ``0`` -
- ``6``. The use of ``7`` is now illegal. Specifying ``0`` for both values
- causes every day of the week to be allowed.
-
-* Support for multiple file systems in a single Ceph cluster is now stable.
- New Ceph clusters enable support for multiple file systems by default.
- Existing clusters must still set the "enable_multiple" flag on the fs.
- See the CephFS documentation for more information.
-
-* volume/nfs: The "ganesha-" prefix from cluster id and nfs-ganesha common
- config object was removed to ensure a consistent namespace across different
- orchestrator backends. Delete any existing nfs-ganesha clusters prior
- to upgrading and redeploy new clusters after upgrading to Pacific.
-
-* A new health check, DAEMON_OLD_VERSION, warns if different versions of
- Ceph are running on daemons. It generates a health error if multiple
- versions are detected. This condition must exist for over
- ``mon_warn_older_version_delay`` (set to 1 week by default) in order for the
- health condition to be triggered. This allows most upgrades to proceed
- without falsely seeing the warning. If upgrade is paused for an extended
- time period, health mute can be used like this "ceph health mute
- DAEMON_OLD_VERSION --sticky". In this case after upgrade has finished use
- "ceph health unmute DAEMON_OLD_VERSION".
-
-* MGR: progress module can now be turned on/off, using these commands:
- ``ceph progress on`` and ``ceph progress off``.
-
-* The ceph_volume_client.py library used for manipulating legacy "volumes" in
- CephFS is removed. All remaining users should use the "fs volume" interface
- exposed by the ceph-mgr:
- https://docs.ceph.com/en/latest/cephfs/fs-volumes/
-
-* An AWS-compliant API: "GetTopicAttributes" was added to replace the existing
- "GetTopic" API. The new API should be used to fetch information about topics
- used for bucket notifications.
-
-* librbd: The shared, read-only parent cache's config option
- ``immutable_object_cache_watermark`` has now been updated to properly reflect
- the upper cache utilization before space is reclaimed. The default
- ``immutable_object_cache_watermark`` is now ``0.9``. If the capacity reaches
- 90% the daemon will delete cold cache.
-
-* OSD: the option ``osd_fast_shutdown_notify_mon`` has been introduced to allow
- the OSD to notify the monitor it is shutting down even if ``osd_fast_shutdown``
- is enabled. This helps with the monitor logs on larger clusters, that may get
- many 'osd.X reported immediately failed by osd.Y' messages, and confuse tools.
-* rgw/kms/vault: the transit logic has been revamped to better use
- the transit engine in vault. To take advantage of this new
- functionality configuration changes are required. See the current
- documentation (radosgw/vault) for more details.
-
-* Scubs are more aggressive in trying to find more simultaneous possible PGs within osd_max_scrubs limitation.
- It is possible that increasing osd_scrub_sleep may be necessary to maintain client responsiveness.
-
-* Version 2 of the cephx authentication protocol (``CEPHX_V2`` feature bit) is
- now required by default. It was introduced in 2018, adding replay attack
- protection for authorizers and making msgr v1 message signatures stronger
- (CVE-2018-1128 and CVE-2018-1129). Support is present in Jewel 10.2.11,
- Luminous 12.2.6, Mimic 13.2.1, Nautilus 14.2.0 and later; upstream kernels
- 4.9.150, 4.14.86, 4.19 and later; various distribution kernels, in particular
- CentOS 7.6 and later. To enable older clients, set ``cephx_require_version``
- and ``cephx_service_require_version`` config options to 1.
-
->=15.0.0
---------
-
-* MON: The cluster log now logs health detail every ``mon_health_to_clog_interval``,
- which has been changed from 1hr to 10min. Logging of health detail will be
- skipped if there is no change in health summary since last known.
-
-* The ``ceph df`` command now lists the number of pgs in each pool.
-
-* Monitors now have config option ``mon_allow_pool_size_one``, which is disabled
- by default. However, if enabled, user now have to pass the
- ``--yes-i-really-mean-it`` flag to ``osd pool set size 1``, if they are really
- sure of configuring pool size 1.
-
-* librbd now inherits the stripe unit and count from its parent image upon creation.
- This can be overridden by specifying different stripe settings during clone creation.
-
-* The balancer is now on by default in upmap mode. Since upmap mode requires
- ``require_min_compat_client`` luminous, new clusters will only support luminous
- and newer clients by default. Existing clusters can enable upmap support by running
- ``ceph osd set-require-min-compat-client luminous``. It is still possible to turn
- the balancer off using the ``ceph balancer off`` command. In earlier versions,
- the balancer was included in the ``always_on_modules`` list, but needed to be
- turned on explicitly using the ``ceph balancer on`` command.
-
-* MGR: the "cloud" mode of the diskprediction module is not supported anymore
- and the ``ceph-mgr-diskprediction-cloud`` manager module has been removed. This
- is because the external cloud service run by ProphetStor is no longer accessible
- and there is no immediate replacement for it at this time. The "local" prediction
- mode will continue to be supported.
-
-* Cephadm: There were a lot of small usability improvements and bug fixes:
-
- * Grafana when deployed by Cephadm now binds to all network interfaces.
- * ``cephadm check-host`` now prints all detected problems at once.
- * Cephadm now calls ``ceph dashboard set-grafana-api-ssl-verify false``
- when generating an SSL certificate for Grafana.
- * The Alertmanager is now correctly pointed to the Ceph Dashboard
- * ``cephadm adopt`` now supports adopting an Alertmanager
- * ``ceph orch ps`` now supports filtering by service name
- * ``ceph orch host ls`` now marks hosts as offline, if they are not
- accessible.
-
-* Cephadm can now deploy NFS Ganesha services. For example, to deploy NFS with
- a service id of mynfs, that will use the RADOS pool nfs-ganesha and namespace
- nfs-ns::
-
- ceph orch apply nfs mynfs nfs-ganesha nfs-ns
-
-* Cephadm: ``ceph orch ls --export`` now returns all service specifications in
- yaml representation that is consumable by ``ceph orch apply``. In addition,
- the commands ``orch ps`` and ``orch ls`` now support ``--format yaml`` and
- ``--format json-pretty``.
-
-* CephFS: Automatic static subtree partitioning policies may now be configured
- using the new distributed and random ephemeral pinning extended attributes on
- directories. See the documentation for more information:
- https://docs.ceph.com/docs/master/cephfs/multimds/
-
-* Cephadm: ``ceph orch apply osd`` supports a ``--preview`` flag that prints a preview of
- the OSD specification before deploying OSDs. This makes it possible to
- verify that the specification is correct, before applying it.
-
-* RGW: The ``radosgw-admin`` sub-commands dealing with orphans --
- ``radosgw-admin orphans find``, ``radosgw-admin orphans finish``, and
- ``radosgw-admin orphans list-jobs`` -- have been deprecated. They have
- not been actively maintained and they store intermediate results on
- the cluster, which could fill a nearly-full cluster. They have been
- replaced by a tool, currently considered experimental,
- ``rgw-orphan-list``.
-
-* RBD: The name of the rbd pool object that is used to store
- rbd trash purge schedule is changed from "rbd_trash_trash_purge_schedule"
- to "rbd_trash_purge_schedule". Users that have already started using
- ``rbd trash purge schedule`` functionality and have per pool or namespace
- schedules configured should copy "rbd_trash_trash_purge_schedule"
- object to "rbd_trash_purge_schedule" before the upgrade and remove
- "rbd_trash_purge_schedule" using the following commands in every RBD
- pool and namespace where a trash purge schedule was previously
- configured::
-
- rados -p [-N namespace] cp rbd_trash_trash_purge_schedule rbd_trash_purge_schedule
- rados -p [-N namespace] rm rbd_trash_trash_purge_schedule
-
- or use any other convenient way to restore the schedule after the
- upgrade.
-
-* librbd: The shared, read-only parent cache has been moved to a separate librbd
- plugin. If the parent cache was previously in-use, you must also instruct
- librbd to load the plugin by adding the following to your configuration::
-
- rbd_plugins = parent_cache
-
-* Monitors now have a config option ``mon_osd_warn_num_repaired``, 10 by default.
- If any OSD has repaired more than this many I/O errors in stored data a
- ``OSD_TOO_MANY_REPAIRS`` health warning is generated.
-
-* Introduce commands that manipulate required client features of a file system::
-
- ceph fs required_client_features add
- ceph fs required_client_features rm
- ceph fs feature ls
-
-* OSD: A new configuration option ``osd_compact_on_start`` has been added which triggers
- an OSD compaction on start. Setting this option to ``true`` and restarting an OSD
- will result in an offline compaction of the OSD prior to booting.
-
-* OSD: the option named ``bdev_nvme_retry_count`` has been removed. Because
- in SPDK v20.07, there is no easy access to bdev_nvme options, and this
- option is hardly used, so it was removed.
-
-* Now when noscrub and/or nodeep-scrub flags are set globally or per pool,
- scheduled scrubs of the type disabled will be aborted. All user initiated
- scrubs are NOT interrupted.
-
-* Alpine build related script, documentation and test have been removed since
- the most updated APKBUILD script of Ceph is already included by Alpine Linux's
- aports repository.
-
-* fs: Names of new FSs, volumes, subvolumes and subvolume groups can only
- contain alphanumeric and ``-``, ``_`` and ``.`` characters. Some commands
- or CephX credentials may not work with old FSs with non-conformant names.
-
-* It is now possible to specify the initial monitor to contact for Ceph tools
- and daemons using the ``mon_host_override`` config option or
- ``--mon-host-override `` command-line switch. This generally should only
- be used for debugging and only affects initial communication with Ceph's
- monitor cluster.
-
-* `blacklist` has been replaced with `blocklist` throughout. The following commands have changed:
-
- - ``ceph osd blacklist ...`` are now ``ceph osd blocklist ...``
- - ``ceph osd. dump_blacklist`` is now ``ceph osd. dump_blocklist``
-
-* The following config options have changed:
-
- - ``mon osd blacklist default expire`` is now ``mon osd blocklist default expire``
- - ``mon mds blacklist interval`` is now ``mon mds blocklist interval``
- - ``mon mgr blacklist interval`` is now ''mon mgr blocklist interval``
- - ``rbd blacklist on break lock`` is now ``rbd blocklist on break lock``
- - ``rbd blacklist expire seconds`` is now ``rbd blocklist expire seconds``
- - ``mds session blacklist on timeout`` is now ``mds session blocklist on timeout``
- - ``mds session blacklist on evict`` is now ``mds session blocklist on evict``
-
-* CephFS: Compatibility code for old on-disk format of snapshot has been removed.
- Current on-disk format of snapshot was introduced by Mimic release. If there
- are any snapshots created by Ceph release older than Mimic. Before upgrading,
- either delete them all or scrub the whole filesystem:
-
- ceph daemon scrub_path / force recursive repair
- ceph daemon scrub_path '~mdsdir' force recursive repair
-
-* CephFS: Scrub is supported in multiple active mds setup. MDS rank 0 handles
- scrub commands, and forward scrub to other mds if necessary.
-
-* The following librados API calls have changed:
-
- - ``rados_blacklist_add`` is now ``rados_blocklist_add``; the former will issue a deprecation warning and be removed in a future release.
- - ``rados.blacklist_add`` is now ``rados.blocklist_add`` in the C++ API.
-
-* The JSON output for the following commands now shows ``blocklist`` instead of ``blacklist``:
-
- - ``ceph osd dump``
- - ``ceph osd. dump_blocklist``
-
-* caps: MON and MDS caps can now be used to restrict client's ability to view
- and operate on specific Ceph file systems. The FS can be specificed using
- ``fsname`` in caps. This also affects subcommand ``fs authorize``, the caps
- produce by it will be specific to the FS name passed in its arguments.
-
-* fs: root_squash flag can be set in MDS caps. It disallows file system
- operations that need write access for clients with uid=0 or gid=0. This
- feature should prevent accidents such as an inadvertent `sudo rm -rf /`.
-
-* fs: "fs authorize" now sets MON cap to "allow fsname="
- instead of setting it to "allow r" all the time.
-
-* ``ceph pg #.# list_unfound`` output has been enhanced to provide
- might_have_unfound information which indicates which OSDs may
- contain the unfound objects.
-
-* The ``ceph orch apply rgw`` syntax and behavior have changed. RGW
- services can now be arbitrarily named (it is no longer forced to be
- `realm.zone`). The ``--rgw-realm=...`` and ``--rgw-zone=...``
- arguments are now optional, which means that if they are omitted, a
- vanilla single-cluster RGW will be deployed. When the realm and
- zone are provided, the user is now responsible for setting up the
- multisite configuration beforehand--cephadm no longer attempts to
- create missing realms or zones.
-
-* The ``min_size`` and ``max_size`` CRUSH rule properties have been removed. Older
- CRUSH maps will still compile but Ceph will issue a warning that these fields are
- ignored.
-* The cephadm NFS support has been simplified to no longer allow the
- pool and namespace where configuration is stored to be customized.
- As a result, the ``ceph orch apply nfs`` command no longer has
- ``--pool`` or ``--namespace`` arguments.
-
- Existing cephadm NFS deployments (from earlier version of Pacific or
- from Octopus) will be automatically migrated when the cluster is
- upgraded. Note that the NFS ganesha daemons will be redeployed and
- it is possible that their IPs will change.
-
-* RGW now requires a secure connection to the monitor by default
- (``auth_client_required=cephx`` and ``ms_mon_client_mode=secure``).
- If you have cephx authentication disabled on your cluster, you may
- need to adjust these settings for RGW.
+>=18.0.0
+
+* RGW's default backend for `rgw_enable_ops_log` changed from RADOS to file.
+ The default value of `rgw_ops_log_rados` is now false, and `rgw_ops_log_file_path`
+ defaults to "/var/log/ceph/ops-log-$cluster-$name.log".
+* The SPDK backend for BlueStore is now able to connect to an NVMeoF target.
+ Please note that this is not an officially supported feature.
+* RGW's pubsub interface now returns boolean fields using bool. Before this change,
+ `/topics/` returns "stored_secret" and "persistent" using a string
+ of "true" or "false" with quotes around them. After this change, these fields
+ are returned without quotes so they can be decoded as boolean values in JSON.
+ The same applies to the `is_truncated` field returned by `/subscriptions/`.
+* RGW's response of `Action=GetTopicAttributes&TopicArn=` REST API now
+ returns `HasStoredSecret` and `Persistent` as boolean in the JSON string
+ encoded in `Attributes/EndPoint`.
+* All boolean fields previously rendered as string by `rgw-admin` command when
+ the JSON format is used are now rendered as boolean. If your scripts/tools
+ relies on this behavior, please update them accordingly. The impacted field names
+ are:
+ * absolute
+ * add
+ * admin
+ * appendable
+ * bucket_key_enabled
+ * delete_marker
+ * exists
+ * has_bucket_info
+ * high_precision_time
+ * index
+ * is_master
+ * is_prefix
+ * is_truncated
+ * linked
+ * log_meta
+ * log_op
+ * pending_removal
+ * read_only
+ * retain_head_object
+ * rule_exist
+ * start_with_full_sync
+ * sync_from_all
+ * syncstopped
+ * system
+ * truncated
+ * user_stats_sync
diff --git a/README.md b/README.md
index a9fe954636294..96f2c6aff28d1 100644
--- a/README.md
+++ b/README.md
@@ -98,6 +98,14 @@ for a couple of external dependencies:
-DCMAKE_INSTALL_PREFIX=/opt/ceph -DCMAKE_C_FLAGS="-Og -g3 -gdwarf-4" \
..
+Ceph has several bundled dependencies such as Boost, RocksDB and Arrow. By
+default, cmake will build these bundled dependencies from source instead of
+using libraries that are already installed on the system. You can opt-in to
+using these system libraries, provided they meet the minimum version required
+by Ceph, with cmake options like `WITH_SYSTEM_BOOST`:
+
+ cmake -DWITH_SYSTEM_BOOST=ON [...]
+
To view an exhaustive list of -D options, you can invoke `cmake` with:
cmake -LH
@@ -213,3 +221,6 @@ To build the documentation, ensure that you are in the top-level
admin/build-doc
+## Reporting Issues
+
+To report an issue and view existing issues, please visit https://tracker.ceph.com/projects/ceph.
diff --git a/SubmittingPatches.rst b/SubmittingPatches.rst
index 4e01082fd9eaa..f708b3c004f3f 100644
--- a/SubmittingPatches.rst
+++ b/SubmittingPatches.rst
@@ -173,7 +173,7 @@ able to refer to the bug tracker easily.
Here is an example showing a properly-formed commit message::
- doc: add "--foo" option to bar
+ doc/rados/mumble: Add "--foo" option to bar
This commit updates the man page for bar with the newly added "--foo"
option.
diff --git a/admin/doc-read-the-docs.txt b/admin/doc-read-the-docs.txt
index b65cc4638491a..7e0ecc86d75b0 100644
--- a/admin/doc-read-the-docs.txt
+++ b/admin/doc-read-the-docs.txt
@@ -1,2 +1,2 @@
plantweb
-git+https://github.com/readthedocs/readthedocs-sphinx-search@master
+git+https://github.com/readthedocs/readthedocs-sphinx-search@main
diff --git a/admin/doc-requirements.txt b/admin/doc-requirements.txt
index dd5994d51ae5c..8d7739b7dec00 100644
--- a/admin/doc-requirements.txt
+++ b/admin/doc-requirements.txt
@@ -1,7 +1,7 @@
-Sphinx == 3.5.4
+Sphinx == 4.4.0
git+https://github.com/ceph/sphinx-ditaa.git@py3#egg=sphinx-ditaa
git+https://github.com/vlasovskikh/funcparserlib.git
-breathe >= 4.20.0
+breathe >= 4.20.0,!=4.33
cryptography
Jinja2
pyyaml >= 5.1.2
@@ -10,8 +10,10 @@ pcpp
prettytable
sphinx-autodoc-typehints
sphinx-prompt
-sphinx_rtd_theme == 0.5.2
+sphinx_rtd_theme == 1.0.0
Sphinx-Substitution-Extensions
typed-ast
sphinxcontrib-openapi
sphinxcontrib-seqdiag
+mistune < 2.0.0
+natsort
diff --git a/ceph.spec.in b/ceph.spec.in
index 3579d7e628eaf..f2bd82b3026d4 100644
--- a/ceph.spec.in
+++ b/ceph.spec.in
@@ -17,8 +17,8 @@
#################################################################################
# conditional build section
#
-# please read http://rpm.org/user_doc/conditional_builds.html for explanation of
-# bcond syntax!
+# please read this for explanation of bcond syntax:
+# https://rpm-software-management.github.io/rpm/manual/conditionalbuilds.html
#################################################################################
%bcond_with make_check
%bcond_with zbd
@@ -36,7 +36,11 @@
%bcond_with rbd_rwl_cache
%endif
%if 0%{?fedora} || 0%{?rhel}
+%if 0%{?rhel} < 9
%bcond_with system_pmdk
+%else
+%bcond_without system_pmdk
+%endif
%bcond_without selinux
%if 0%{?rhel} >= 8
%bcond_with cephfs_java
@@ -94,6 +98,8 @@
# distros that do _not_ ship cmd2/colorama
%bcond_with cephfs_shell
%endif
+%bcond_with system_arrow
+%bcond_with system_utf8proc
%if 0%{?fedora} || 0%{?suse_version} || 0%{?rhel} >= 8
%global weak_deps 1
%endif
@@ -139,6 +145,13 @@
%endif
%endif
+%if 0%{with seastar}
+# disable -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1, as gcc-toolset-{9,10}-annobin
+# do not provide gcc-annobin.so anymore, despite that they provide annobin.so. but
+# redhat-rpm-config still passes -fplugin=gcc-annobin to the compiler.
+%undefine _annotated_build
+%endif
+
#################################################################################
# main package definition
#################################################################################
@@ -184,10 +197,14 @@ BuildRequires: gperf
BuildRequires: cmake > 3.5
BuildRequires: fuse-devel
%if 0%{with seastar} && 0%{?rhel}
-BuildRequires: gcc-toolset-9-gcc-c++ >= 9.2.1-2.3
+BuildRequires: gcc-toolset-10-gcc-c++ >= 10.3.1-1.2
+%else
+%if 0%{?suse_version}
+BuildRequires: gcc11-c++
%else
BuildRequires: gcc-c++
%endif
+%endif
%if 0%{with tcmalloc}
# libprofiler did not build on ppc64le until 2.7.90
%if 0%{?fedora} || 0%{?rhel} >= 8
@@ -200,7 +217,6 @@ BuildRequires: gperftools-devel >= 2.6.1
BuildRequires: gperftools-devel >= 2.4
%endif
%endif
-BuildRequires: leveldb-devel > 1.2
BuildRequires: libaio-devel
BuildRequires: libblkid-devel >= 2.17
BuildRequires: cryptsetup-devel
@@ -250,7 +266,6 @@ BuildRequires: hostname
BuildRequires: jq
BuildRequires: libuuid-devel
BuildRequires: python%{python3_pkgversion}-bcrypt
-BuildRequires: python%{python3_pkgversion}-nose
BuildRequires: python%{python3_pkgversion}-pecan
BuildRequires: python%{python3_pkgversion}-requests
BuildRequires: python%{python3_pkgversion}-dateutil
@@ -261,10 +276,15 @@ BuildRequires: socat
%if 0%{with zbd}
BuildRequires: libzbd-devel
%endif
+%if 0%{?suse_version}
+BuildRequires: libthrift-devel >= 0.13.0
+%else
+BuildRequires: thrift-devel >= 0.13.0
+%endif
+BuildRequires: re2-devel
%if 0%{with jaeger}
BuildRequires: bison
BuildRequires: flex
-BuildRequires: thrift-devel >= 0.13.0
%if 0%{?fedora} || 0%{?rhel}
BuildRequires: json-devel
%endif
@@ -277,6 +297,13 @@ BuildRequires: libevent-devel
BuildRequires: libpmem-devel
BuildRequires: libpmemobj-devel
%endif
+%if 0%{with system_arrow}
+BuildRequires: arrow-devel
+BuildRequires: parquet-devel
+%endif
+%if 0%{with system_utf8proc}
+BuildRequires: utf8proc-devel
+%endif
%if 0%{with seastar}
BuildRequires: c-ares-devel
BuildRequires: gnutls-devel
@@ -291,10 +318,10 @@ BuildRequires: libasan
BuildRequires: libatomic
%endif
%if 0%{?rhel}
-BuildRequires: gcc-toolset-9-annobin
-BuildRequires: gcc-toolset-9-libubsan-devel
-BuildRequires: gcc-toolset-9-libasan-devel
-BuildRequires: gcc-toolset-9-libatomic-devel
+BuildRequires: gcc-toolset-10-annobin
+BuildRequires: gcc-toolset-10-libubsan-devel
+BuildRequires: gcc-toolset-10-libasan-devel
+BuildRequires: gcc-toolset-10-libatomic-devel
%endif
%endif
#################################################################################
@@ -312,6 +339,7 @@ BuildRequires: libbz2-devel
BuildRequires: mozilla-nss-devel
BuildRequires: keyutils-devel
BuildRequires: libopenssl-devel
+BuildRequires: ninja
BuildRequires: openldap2-devel
#BuildRequires: krb5
#BuildRequires: krb5-devel
@@ -331,6 +359,7 @@ BuildRequires: nss-devel
BuildRequires: keyutils-libs-devel
BuildRequires: libibverbs-devel
BuildRequires: librdmacm-devel
+BuildRequires: ninja-build
BuildRequires: openldap-devel
#BuildRequires: krb5-devel
BuildRequires: openssl-devel
@@ -343,11 +372,10 @@ BuildRequires: lz4-devel >= 1.7
%endif
# distro-conditional make check dependencies
%if 0%{with make_check}
+BuildRequires: golang
%if 0%{?fedora} || 0%{?rhel}
BuildRequires: golang-github-prometheus
-BuildRequires: jsonnet
BuildRequires: libtool-ltdl-devel
-BuildRequires: ninja-build
BuildRequires: xmlsec1
BuildRequires: xmlsec1-devel
%ifarch x86_64
@@ -364,11 +392,9 @@ BuildRequires: python%{python3_pkgversion}-pyOpenSSL
%endif
%if 0%{?suse_version}
BuildRequires: golang-github-prometheus-prometheus
-BuildRequires: jsonnet
BuildRequires: libxmlsec1-1
BuildRequires: libxmlsec1-nss1
BuildRequires: libxmlsec1-openssl1
-BuildRequires: ninja
BuildRequires: python%{python3_pkgversion}-CherryPy
BuildRequires: python%{python3_pkgversion}-PyJWT
BuildRequires: python%{python3_pkgversion}-Routes
@@ -465,6 +491,8 @@ Summary: Utility to bootstrap Ceph clusters
BuildArch: noarch
Requires: lvm2
Requires: python%{python3_pkgversion}
+Requires: openssh-server
+Requires: which
%if 0%{?weak_deps}
Recommends: podman >= 2.0.2
%endif
@@ -486,9 +514,6 @@ Requires: python%{python3_pkgversion}-cephfs = %{_epoch_prefix}%{version}-%{rele
Requires: python%{python3_pkgversion}-rgw = %{_epoch_prefix}%{version}-%{release}
Requires: python%{python3_pkgversion}-ceph-argparse = %{_epoch_prefix}%{version}-%{release}
Requires: python%{python3_pkgversion}-ceph-common = %{_epoch_prefix}%{version}-%{release}
-%if 0%{with jaeger}
-Requires: libjaeger = %{_epoch_prefix}%{version}-%{release}
-%endif
%if 0%{?fedora} || 0%{?rhel}
Requires: python%{python3_pkgversion}-prettytable
%endif
@@ -524,9 +549,6 @@ Group: System/Filesystems
%endif
Provides: ceph-test:/usr/bin/ceph-monstore-tool
Requires: ceph-base = %{_epoch_prefix}%{version}-%{release}
-%if 0%{with jaeger}
-Requires: libjaeger = %{_epoch_prefix}%{version}-%{release}
-%endif
%description mon
ceph-mon is the cluster monitor daemon for the Ceph distributed file
system. One or more instances of ceph-mon form a Paxos part-time
@@ -662,14 +684,16 @@ Group: System/Filesystems
%endif
Requires: ceph-mgr = %{_epoch_prefix}%{version}-%{release}
Requires: python%{python3_pkgversion}-asyncssh
-Requires: python%{python3_pkgversion}-cherrypy
+Requires: python%{python3_pkgversion}-natsort
Requires: cephadm = %{_epoch_prefix}%{version}-%{release}
%if 0%{?suse_version}
Requires: openssh
+Requires: python%{python3_pkgversion}-CherryPy
Requires: python%{python3_pkgversion}-Jinja2
%endif
%if 0%{?rhel} || 0%{?fedora}
Requires: openssh-clients
+Requires: python%{python3_pkgversion}-cherrypy
Requires: python%{python3_pkgversion}-jinja2
%endif
%description mgr-cephadm
@@ -1056,19 +1080,6 @@ Obsoletes: libcephfs2-devel < %{_epoch_prefix}%{version}-%{release}
This package contains libraries and headers needed to develop programs
that use Ceph distributed file system.
-%if 0%{with jaeger}
-%package -n libjaeger
-Summary: Ceph distributed file system tracing library
-%if 0%{?suse_version}
-Group: System/Libraries
-%endif
-Provides: libjaegertracing.so.0()(64bit)
-Provides: libopentracing.so.1()(64bit)
-%description -n libjaeger
-This package contains libraries needed to provide distributed
-tracing for Ceph.
-%endif
-
%package -n python%{python3_pkgversion}-cephfs
Summary: Python 3 libraries for Ceph distributed file system
%if 0%{?suse_version}
@@ -1242,7 +1253,7 @@ This package provides Ceph default alerts for Prometheus.
%endif
%if 0%{with seastar} && 0%{?rhel}
-. /opt/rh/gcc-toolset-9/enable
+. /opt/rh/gcc-toolset-10/enable
%endif
%if 0%{with cephfs_java}
@@ -1264,6 +1275,9 @@ export LDFLAGS="$RPM_LD_FLAGS"
%if 0%{with seastar}
# seastar uses longjmp() to implement coroutine. and this annoys longjmp_chk()
export CXXFLAGS=$(echo $RPM_OPT_FLAGS | sed -e 's/-Wp,-D_FORTIFY_SOURCE=2//g')
+# remove from CFLAGS too because it causes the arrow submodule to fail with:
+# warning _FORTIFY_SOURCE requires compiling with optimization (-O)
+export CFLAGS=$(echo $RPM_OPT_FLAGS | sed -e 's/-Wp,-D_FORTIFY_SOURCE=2//g')
%endif
env | sort
@@ -1274,6 +1288,10 @@ env | sort
mkdir -p %{_vpath_builddir}
pushd %{_vpath_builddir}
cmake .. \
+%if 0%{?suse_version}
+ -DCMAKE_C_COMPILER=gcc-11 \
+ -DCMAKE_CXX_COMPILER=g++-11 \
+%endif
-DCMAKE_INSTALL_PREFIX=%{_prefix} \
-DCMAKE_INSTALL_LIBDIR:PATH=%{_libdir} \
-DCMAKE_INSTALL_LIBEXECDIR:PATH=%{_libexecdir} \
@@ -1282,10 +1300,13 @@ cmake .. \
-DCMAKE_INSTALL_MANDIR:PATH=%{_mandir} \
-DCMAKE_INSTALL_DOCDIR:PATH=%{_docdir}/ceph \
-DCMAKE_INSTALL_INCLUDEDIR:PATH=%{_includedir} \
- -DCMAKE_INSTALL_SYSTEMD_SERVICEDIR:PATH=%{_unitdir} \
+ -DSYSTEMD_SYSTEM_UNIT_DIR:PATH=%{_unitdir} \
-DWITH_MANPAGE:BOOL=ON \
-DWITH_PYTHON3:STRING=%{python3_version} \
-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF \
+%if 0%{?suse_version}
+ -DWITH_RADOSGW_SELECT_PARQUET:BOOL=OFF \
+%endif
%if 0%{without ceph_test_package}
-DWITH_TESTS:BOOL=OFF \
%endif
@@ -1352,6 +1373,12 @@ cmake .. \
%endif
%if 0%{?rhel}
-DWITH_FMT_HEADER_ONLY:BOOL=ON \
+%endif
+%if 0%{with system_arrow}
+ -DWITH_SYSTEM_ARROW:BOOL=ON \
+%endif
+%if 0%{with system_utf8proc}
+ -DWITH_SYSTEM_UTF8PROC:BOOL=ON \
%endif
-DWITH_GRAFANA:BOOL=ON
@@ -1450,7 +1477,7 @@ mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-rbd
mkdir -p %{buildroot}%{_localstatedir}/lib/ceph/bootstrap-rbd-mirror
# prometheus alerts
-install -m 644 -D monitoring/prometheus/alerts/ceph_default_alerts.yml %{buildroot}/etc/prometheus/ceph/ceph_default_alerts.yml
+install -m 644 -D monitoring/ceph-mixin/prometheus_alerts.yml %{buildroot}/etc/prometheus/ceph/ceph_default_alerts.yml
%if 0%{?suse_version}
# create __pycache__ directories and their contents
@@ -1465,6 +1492,9 @@ install -m 644 -D monitoring/prometheus/alerts/ceph_default_alerts.yml %{buildro
%clean
rm -rf %{buildroot}
+# built binaries are no longer necessary at this point,
+# but are consuming ~17GB of disk in the build environment
+rm -rf %{_vpath_builddir}
#################################################################################
# files and systemd scriptlets
@@ -1565,7 +1595,7 @@ exit 0
%{_mandir}/man8/cephadm.8*
%attr(0700,cephadm,cephadm) %dir %{_sharedstatedir}/cephadm
%attr(0700,cephadm,cephadm) %dir %{_sharedstatedir}/cephadm/.ssh
-%attr(0600,cephadm,cephadm) %{_sharedstatedir}/cephadm/.ssh/authorized_keys
+%config(noreplace) %attr(0600,cephadm,cephadm) %{_sharedstatedir}/cephadm/.ssh/authorized_keys
%files common
%dir %{_docdir}/ceph
@@ -1715,6 +1745,7 @@ fi
%dir %{_datadir}/ceph/mgr
%{_datadir}/ceph/mgr/mgr_module.*
%{_datadir}/ceph/mgr/mgr_util.*
+%{_datadir}/ceph/mgr/object_format.*
%{_unitdir}/ceph-mgr@.service
%{_unitdir}/ceph-mgr.target
%attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/mgr
@@ -2330,14 +2361,6 @@ fi
%{_includedir}/cephfs/metrics/Types.h
%{_libdir}/libcephfs.so
-%if %{with jaeger}
-%files -n libjaeger
-%{_libdir}/libopentracing.so.*
-%{_libdir}/libjaegertracing.so.*
-%post -n libjaeger -p /sbin/ldconfig
-%postun -n libjaeger -p /sbin/ldconfig
-%endif
-
%files -n python%{python3_pkgversion}-cephfs
%{python3_sitearch}/cephfs.cpython*.so
%{python3_sitearch}/cephfs-*.egg-info
@@ -2527,8 +2550,6 @@ exit 0
%endif
%attr(0755,root,root) %dir %{_sysconfdir}/grafana/dashboards/ceph-dashboard
%config %{_sysconfdir}/grafana/dashboards/ceph-dashboard/*
-%doc monitoring/grafana/dashboards/README
-%doc monitoring/grafana/README.md
%files prometheus-alerts
%if 0%{?suse_version}
diff --git a/cmake/modules/AddCephTest.cmake b/cmake/modules/AddCephTest.cmake
index 0df7125b50833..46d3a1b4cb813 100644
--- a/cmake/modules/AddCephTest.cmake
+++ b/cmake/modules/AddCephTest.cmake
@@ -38,7 +38,6 @@ if(WITH_GTEST_PARALLEL)
BUILD_COMMAND ""
INSTALL_COMMAND "")
add_dependencies(tests gtest-parallel_ext)
- find_package(Python3 QUIET REQUIRED)
set(GTEST_PARALLEL_COMMAND
${Python3_EXECUTABLE} ${gtest_parallel_source_dir}/gtest-parallel)
endif()
@@ -70,7 +69,6 @@ function(add_tox_test name)
list(APPEND tox_envs py3)
endif()
string(REPLACE ";" "," tox_envs "${tox_envs}")
- find_package(Python3 QUIET REQUIRED)
add_test(
NAME setup-venv-for-${name}
COMMAND ${CMAKE_SOURCE_DIR}/src/tools/setup-virtualenv.sh --python=${Python3_EXECUTABLE} ${venv_path}
diff --git a/cmake/modules/BuildArrow.cmake b/cmake/modules/BuildArrow.cmake
new file mode 100644
index 0000000000000..45ebb697446dc
--- /dev/null
+++ b/cmake/modules/BuildArrow.cmake
@@ -0,0 +1,143 @@
+# apache arrow and its parquet library are used in radosgw for s3 select
+
+function(build_arrow)
+ # only enable the parquet component
+ set(arrow_CMAKE_ARGS -DARROW_PARQUET=ON)
+
+ # only use preinstalled dependencies for arrow, don't fetch/build any
+ list(APPEND arrow_CMAKE_ARGS -DARROW_DEPENDENCY_SOURCE=SYSTEM)
+
+ # only build static version of arrow and parquet
+ list(APPEND arrow_CMAKE_ARGS -DARROW_BUILD_SHARED=OFF)
+ list(APPEND arrow_CMAKE_ARGS -DARROW_BUILD_STATIC=ON)
+
+ # arrow only supports its own bundled version of jemalloc, so can't
+ # share the version ceph is using
+ list(APPEND arrow_CMAKE_ARGS -DARROW_JEMALLOC=OFF)
+
+ # transitive dependencies
+ list(APPEND arrow_INTERFACE_LINK_LIBRARIES thrift)
+
+ if (NOT WITH_SYSTEM_UTF8PROC)
+ # forward utf8proc_ROOT from build_utf8proc()
+ list(APPEND arrow_CMAKE_ARGS -Dutf8proc_ROOT=${utf8proc_ROOT})
+ # non-system utf8proc is bundled as a static library
+ list(APPEND arrow_CMAKE_ARGS -DARROW_UTF8PROC_USE_SHARED=OFF)
+ # make sure utf8proc submodule builds first, so arrow can find its byproducts
+ list(APPEND arrow_DEPENDS utf8proc::utf8proc)
+ endif()
+
+ list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_BROTLI=${HAVE_BROTLI})
+ if (HAVE_BROTLI) # optional, off by default
+ list(APPEND arrow_INTERFACE_LINK_LIBRARIES ${brotli_libs})
+ endif (HAVE_BROTLI)
+
+ list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_BZ2=OFF)
+
+ list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_LZ4=${HAVE_LZ4})
+ if (HAVE_LZ4) # optional, on by default
+ list(APPEND arrow_INTERFACE_LINK_LIBRARIES LZ4::LZ4)
+ endif (HAVE_LZ4)
+
+ list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_SNAPPY=ON) # required
+ list(APPEND arrow_INTERFACE_LINK_LIBRARIES snappy::snappy)
+
+ list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_ZLIB=ON) # required
+ list(APPEND arrow_INTERFACE_LINK_LIBRARIES ZLIB::ZLIB)
+
+ list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_ZSTD=${WITH_SYSTEM_ZSTD})
+ if (WITH_SYSTEM_ZSTD)
+ find_package(Zstd 1.4.4 REQUIRED)
+ list(APPEND arrow_INTERFACE_LINK_LIBRARIES Zstd::Zstd)
+ endif (WITH_SYSTEM_ZSTD)
+
+ list(APPEND arrow_CMAKE_ARGS -DBOOST_ROOT=${BOOST_ROOT})
+ list(APPEND arrow_CMAKE_ARGS -DBOOST_INCLUDEDIR=${Boost_INCLUDE_DIR})
+ list(APPEND arrow_CMAKE_ARGS -DBOOST_LIBRARYDIR=${BOOST_LIBRARYDIR})
+
+ if (NOT WITH_SYSTEM_BOOST)
+ # make sure boost submodule builds first, so arrow can find its byproducts
+ list(APPEND arrow_DEPENDS Boost)
+ endif()
+
+ # cmake doesn't properly handle arguments containing ";", such as
+ # CMAKE_PREFIX_PATH, for which reason we'll have to use some other separator.
+ string(REPLACE ";" "!" CMAKE_PREFIX_PATH_ALT_SEP "${CMAKE_PREFIX_PATH}")
+ list(APPEND arrow_CMAKE_ARGS -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH_ALT_SEP})
+ if(CMAKE_TOOLCHAIN_FILE)
+ list(APPEND arrow_CMAKE_ARGS
+ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
+ endif()
+
+ list(APPEND arrow_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER})
+ list(APPEND arrow_CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
+ list(APPEND arrow_CMAKE_ARGS -DCMAKE_AR=${CMAKE_AR})
+ if(CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE STREQUAL "None")
+ list(APPEND arrow_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
+ else()
+ list(APPEND arrow_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release)
+ endif()
+
+ # we use an external project and copy the sources to bin directory to ensure
+ # that object files are built outside of the source tree.
+ include(ExternalProject)
+ set(arrow_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/arrow/cpp")
+ set(arrow_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/arrow/cpp")
+
+ set(arrow_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow")
+ list(APPEND arrow_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${arrow_INSTALL_PREFIX})
+
+ set(arrow_INSTALL_LIBDIR "lib") # force lib so we don't have to guess between lib/lib64
+ list(APPEND arrow_CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${arrow_INSTALL_LIBDIR})
+ set(arrow_LIBRARY_DIR "${arrow_INSTALL_PREFIX}/${arrow_INSTALL_LIBDIR}")
+
+ set(arrow_LIBRARY "${arrow_LIBRARY_DIR}/libarrow.a")
+ set(parquet_LIBRARY "${arrow_LIBRARY_DIR}/libparquet.a")
+
+ set(arrow_INCLUDE_DIR "${arrow_INSTALL_PREFIX}/include")
+ # this include directory won't exist until the install step, but the
+ # imported targets need it early for INTERFACE_INCLUDE_DIRECTORIES
+ file(MAKE_DIRECTORY "${arrow_INCLUDE_DIR}")
+
+ set(arrow_BYPRODUCTS ${arrow_LIBRARY})
+ list(APPEND arrow_BYPRODUCTS ${parquet_LIBRARY})
+
+ if(CMAKE_MAKE_PROGRAM MATCHES "make")
+ # try to inherit command line arguments passed by parent "make" job
+ set(make_cmd $(MAKE))
+ set(install_cmd $(MAKE) install)
+ else()
+ set(make_cmd ${CMAKE_COMMAND} --build )
+ set(install_cmd ${CMAKE_COMMAND} --build --target install)
+ endif()
+
+ # clear the DESTDIR environment variable from debian/rules,
+ # because it messes with the internal install paths of arrow's bundled deps
+ set(NO_DESTDIR_COMMAND ${CMAKE_COMMAND} -E env --unset=DESTDIR)
+
+ ExternalProject_Add(arrow_ext
+ SOURCE_DIR "${arrow_SOURCE_DIR}"
+ CMAKE_ARGS ${arrow_CMAKE_ARGS}
+ BINARY_DIR "${arrow_BINARY_DIR}"
+ BUILD_COMMAND ${NO_DESTDIR_COMMAND} ${make_cmd}
+ BUILD_BYPRODUCTS "${arrow_BYPRODUCTS}"
+ INSTALL_COMMAND ${NO_DESTDIR_COMMAND} ${install_cmd}
+ INSTALL_DIR "${arrow_INSTALL_PREFIX}"
+ DEPENDS "${arrow_DEPENDS}"
+ LIST_SEPARATOR !)
+
+ add_library(Arrow::Arrow STATIC IMPORTED)
+ add_dependencies(Arrow::Arrow arrow_ext)
+ set_target_properties(Arrow::Arrow PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${arrow_INCLUDE_DIR}"
+ INTERFACE_LINK_LIBRARIES "${arrow_INTERFACE_LINK_LIBRARIES}"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+ IMPORTED_LOCATION "${arrow_LIBRARY}")
+
+ add_library(Arrow::Parquet STATIC IMPORTED)
+ add_dependencies(Arrow::Parquet arrow_ext)
+ target_link_libraries(Arrow::Parquet INTERFACE Arrow::Arrow)
+ set_target_properties(Arrow::Parquet PROPERTIES
+ IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+ IMPORTED_LOCATION "${parquet_LIBRARY}")
+endfunction()
diff --git a/cmake/modules/BuildBoost.cmake b/cmake/modules/BuildBoost.cmake
index dc8999e3b6342..5fdb6a38ef4cf 100644
--- a/cmake/modules/BuildBoost.cmake
+++ b/cmake/modules/BuildBoost.cmake
@@ -1,8 +1,9 @@
-# This module builds Boost
-# executables are. It sets the following variables:
+# This module builds Boost. It sets the following variables:
#
# Boost_FOUND : boolean - system has Boost
+# BOOST_ROOT : path
# Boost_LIBRARIES : list(filepath) - the libraries needed to use Boost
+# Boost_LIBRARY_DIR_RELEASE : path - the library path
# Boost_INCLUDE_DIRS : list(path) - the Boost include directories
#
# Following hints are respected
@@ -70,7 +71,7 @@ function(do_build_boost root_dir version)
if(c MATCHES "^python([0-9])\$")
set(with_python_version "${CMAKE_MATCH_1}")
list(APPEND boost_with_libs "python")
- elseif(c MATCHES "^python([0-9])\\.?([0-9])\$")
+ elseif(c MATCHES "^python([0-9])\\.?([0-9]+)\$")
set(with_python_version "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}")
list(APPEND boost_with_libs "python")
else()
@@ -79,10 +80,20 @@ function(do_build_boost root_dir version)
endforeach()
list_replace(boost_with_libs "unit_test_framework" "test")
string(REPLACE ";" "," boost_with_libs "${boost_with_libs}")
+
+ if(CMAKE_CXX_COMPILER_ID STREQUAL GNU)
+ set(toolset gcc)
+ elseif(CMAKE_CXX_COMPILER_ID STREQUAL Clang)
+ set(toolset clang)
+ else()
+ message(SEND_ERROR "unknown compiler: ${CMAKE_CXX_COMPILER_ID}")
+ endif()
+
# build b2 and prepare the project-config.jam for boost
set(configure_command
./bootstrap.sh --prefix=
- --with-libraries=${boost_with_libs})
+ --with-libraries=${boost_with_libs}
+ --with-toolset=${toolset})
set(b2 ./b2)
if(BOOST_J)
@@ -92,14 +103,6 @@ function(do_build_boost root_dir version)
# suppress all debugging levels for b2
list(APPEND b2 -d0)
- if(CMAKE_CXX_COMPILER_ID STREQUAL GNU)
- set(toolset gcc)
- elseif(CMAKE_CXX_COMPILER_ID STREQUAL Clang)
- set(toolset clang)
- else()
- message(SEND_ERROR "unknown compiler: ${CMAKE_CXX_COMPILER_ID}")
- endif()
-
set(user_config ${CMAKE_BINARY_DIR}/user-config.jam)
# edit the user-config.jam so b2 will be able to use the specified
# toolset and python
@@ -153,14 +156,10 @@ function(do_build_boost root_dir version)
set(boost_version 1.75.0)
set(boost_sha256 953db31e016db7bb207f11432bef7df100516eeb746843fa0486a222e3fd49cb)
string(REPLACE "." "_" boost_version_underscore ${boost_version} )
- set(boost_url
- https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version_underscore}.tar.bz2)
- if(CMAKE_VERSION VERSION_GREATER 3.7)
- set(boost_url
- "${boost_url} http://downloads.sourceforge.net/project/boost/boost/${boost_version}/boost_${boost_version_underscore}.tar.bz2")
- set(boost_url
- "${boost_url} https://download.ceph.com/qa/boost_${boost_version_underscore}.tar.bz2")
- endif()
+ string(JOIN " " boost_url
+ https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version_underscore}.tar.bz2
+ https://downloads.sourceforge.net/project/boost/boost/${boost_version}/boost_${boost_version_underscore}.tar.bz2
+ https://download.ceph.com/qa/boost_${boost_version_underscore}.tar.bz2)
set(source_dir
URL ${boost_url}
URL_HASH SHA256=${boost_sha256}
@@ -190,8 +189,10 @@ macro(build_boost version)
# target, so we can collect "Boost_LIBRARIES" which is then used by
# ExternalProject_Add(Boost ...)
set(install_dir "${CMAKE_BINARY_DIR}/boost")
+ set(BOOST_ROOT ${install_dir})
set(Boost_INCLUDE_DIRS ${install_dir}/include)
set(Boost_INCLUDE_DIR ${install_dir}/include)
+ set(Boost_LIBRARY_DIR_RELEASE ${install_dir}/lib)
set(Boost_VERSION ${version})
# create the directory so cmake won't complain when looking at the imported
# target
diff --git a/cmake/modules/BuildDPDK.cmake b/cmake/modules/BuildDPDK.cmake
index bfa8ff0c285ac..f2e3d2186ab83 100644
--- a/cmake/modules/BuildDPDK.cmake
+++ b/cmake/modules/BuildDPDK.cmake
@@ -94,7 +94,7 @@ function(do_build_dpdk dpdk_dir)
ExternalProject_Add(dpdk-ext
SOURCE_DIR ${dpdk_source_dir}
CONFIGURE_COMMAND ${make_cmd} config O=${dpdk_dir} T=${target}
- BUILD_COMMAND ${make_cmd} O=${dpdk_dir} CC=${CMAKE_C_COMPILER} EXTRA_CFLAGS=${extra_cflags}
+ BUILD_COMMAND ${make_cmd} O=${dpdk_dir} CC=${CMAKE_C_COMPILER} EXTRA_CFLAGS=${extra_cflags} RTE_DEVEL_BUILD=n
BUILD_IN_SOURCE 1
INSTALL_COMMAND ""
LOG_CONFIGURE ON
diff --git a/cmake/modules/BuildFIO.cmake b/cmake/modules/BuildFIO.cmake
index 481f3edf09d22..3a0694b543eed 100644
--- a/cmake/modules/BuildFIO.cmake
+++ b/cmake/modules/BuildFIO.cmake
@@ -2,8 +2,14 @@ function(build_fio)
# we use an external project and copy the sources to bin directory to ensure
# that object files are built outside of the source tree.
include(ExternalProject)
- if(ALLOCATOR)
- set(FIO_EXTLIBS EXTLIBS=-l${ALLOCATOR})
+ if(ALLOC_LIBS)
+ get_target_property(alloc_lib_path
+ ${ALLOC_LIBS} IMPORTED_LOCATION)
+ get_filename_component(alloc_lib_dir
+ ${alloc_lib_path} DIRECTORY)
+ get_filename_component(alloc_lib_name
+ ${alloc_lib_path} NAME)
+ set(FIO_EXTLIBS "EXTLIBS='-L${alloc_lib_dir} -l:${alloc_lib_name}'")
endif()
include(FindMake)
@@ -20,7 +26,7 @@ function(build_fio)
SOURCE_DIR ${source_dir}
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND /configure
- BUILD_COMMAND ${make_cmd} fio EXTFLAGS=-Wno-format-truncation ${FIO_EXTLIBS}
+ BUILD_COMMAND ${make_cmd} fio EXTFLAGS=-Wno-format-truncation "${FIO_EXTLIBS}"
INSTALL_COMMAND cp /fio ${CMAKE_BINARY_DIR}/bin
LOG_CONFIGURE ON
LOG_BUILD ON
diff --git a/cmake/modules/BuildJaeger.cmake b/cmake/modules/BuildJaeger.cmake
deleted file mode 100644
index 6a65a6bb42bd3..0000000000000
--- a/cmake/modules/BuildJaeger.cmake
+++ /dev/null
@@ -1,88 +0,0 @@
-# This module builds Jaeger after it's dependencies are installed and discovered
-# opentracing: is built using cmake/modules/Buildopentracing.cmake
-# Thrift: found using cmake/modules/Findthrift.cmake (not by default)
-# yaml-cpp, nlhomann-json: are installed locally and then discovered using
-# Find.cmake
-# Boost Libraries: uses ceph build boost cmake/modules/BuildBoost.cmake
-
-include(BuildOpenTracing)
-
-# will do all linking and path setting
-function(set_library_properties_for_external_project _target _lib)
- # Manually create the directory, it will be created as part of the build,
- # but this runs in the configuration phase, and CMake generates an error if
- # we add an include directory that does not exist yet.
- set(_libfullname "${CMAKE_SHARED_LIBRARY_PREFIX}${_lib}${CMAKE_SHARED_LIBRARY_SUFFIX}")
- set(_libpath "${CMAKE_BINARY_DIR}/external/lib/${_libfullname}")
- set(_includepath "${CMAKE_BINARY_DIR}/external/include")
- message(STATUS "Configuring ${_target} with ${_libpath}")
- add_library(${_target} SHARED IMPORTED)
- add_dependencies(${_target} ${_lib})
-
- file(MAKE_DIRECTORY "${_includepath}")
- set_target_properties(${_target} PROPERTIES
- INTERFACE_LINK_LIBRARIES "${_libpath}"
- IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
- IMPORTED_LOCATION "${_libpath}"
- INTERFACE_INCLUDE_DIRECTORIES "${_includepath}")
-endfunction()
-
-function(build_jaeger)
- set(Jaeger_SOURCE_DIR "${CMAKE_SOURCE_DIR}/src/jaegertracing/jaeger-client-cpp")
- set(Jaeger_INSTALL_DIR "${CMAKE_BINARY_DIR}/external")
- set(Jaeger_BINARY_DIR "${CMAKE_BINARY_DIR}/external/jaegertracing")
-
- file(MAKE_DIRECTORY "${Jaeger_INSTALL_DIR}")
- set(Jaeger_CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON
- -DBUILD_SHARED_LIBS=ON
- -DHUNTER_ENABLED=OFF
- -DBUILD_TESTING=OFF
- -DJAEGERTRACING_BUILD_EXAMPLES=OFF
- -DCMAKE_PREFIX_PATH="${CMAKE_BINARY_DIR}/external;${CMAKE_BINARY_DIR}/boost"
- -DCMAKE_INSTALL_RPATH=${CMAKE_BINARY_DIR}/external
- -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE
- -DOpenTracing_DIR=${CMAKE_SOURCE_DIR}/src/jaegertracing/opentracing-cpp
- -Dnlohmann_json_DIR=/usr/lib
- -DCMAKE_FIND_ROOT_PATH=${CMAKE_BINARY_DIR}/external
- -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/external
- -DCMAKE_INSTALL_LIBDIR=${CMAKE_BINARY_DIR}/external/lib
- -DBoost_INCLUDE_DIRS=${CMAKE_BINARY_DIR}/boost/include
- -DBOOST_ROOT=${CMAKE_BINARY_DIR}/boost
- -Dthrift_HOME=${CMAKE_BINARY_DIR}/external
- -DOpenTracing_HOME=${CMAKE_BINARY_DIR}/external)
-
- # build these libraries along with jaeger
- set(dependencies opentracing)
- if(NOT WITH_SYSTEM_BOOST)
- list(APPEND dependencies Boost)
- endif()
- include(BuildOpenTracing)
- build_opentracing()
-
- if(CMAKE_MAKE_PROGRAM MATCHES "make")
- # try to inherit command line arguments passed by parent "make" job
- set(make_cmd $(MAKE) jaegertracing)
- else()
- set(make_cmd ${CMAKE_COMMAND} --build --target jaegertracing)
- endif()
- set(install_cmd DESTDIR= ${CMAKE_MAKE_PROGRAM} install)
-
- include(ExternalProject)
- ExternalProject_Add(jaegertracing
- SOURCE_DIR ${Jaeger_SOURCE_DIR}
- UPDATE_COMMAND ""
- INSTALL_DIR "external"
- PREFIX ${Jaeger_INSTALL_DIR}
- CMAKE_ARGS ${Jaeger_CMAKE_ARGS}
- BINARY_DIR ${Jaeger_BINARY_DIR}
- BUILD_COMMAND ${make_cmd}
- INSTALL_COMMAND ${install_cmd}
- DEPENDS ${dependencies}
- BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/external/lib/libjaegertracing.so
- LOG_BUILD ON)
-
- set_library_properties_for_external_project(opentracing::libopentracing
- opentracing)
- set_library_properties_for_external_project(jaegertracing::libjaegertracing
- jaegertracing)
-endfunction()
diff --git a/cmake/modules/BuildOpenTracing.cmake b/cmake/modules/BuildOpenTracing.cmake
deleted file mode 100644
index 07a1ea44d1c05..0000000000000
--- a/cmake/modules/BuildOpenTracing.cmake
+++ /dev/null
@@ -1,36 +0,0 @@
-function(build_opentracing)
- set(opentracing_SOURCE_DIR "${CMAKE_SOURCE_DIR}/src/jaegertracing/opentracing-cpp")
- set(opentracing_BINARY_DIR "${CMAKE_BINARY_DIR}/external/opentracing-cpp")
-
- set(opentracing_CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON
- -DBUILD_MOCKTRACER=OFF
- -DENABLE_LINTING=OFF
- -DBUILD_STATIC_LIBS=OFF
- -DBUILD_TESTING=OFF
- -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/external
- -DCMAKE_INSTALL_RPATH=${CMAKE_BINARY_DIR}/external
- -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE
- -DCMAKE_INSTALL_LIBDIR=${CMAKE_BINARY_DIR}/external/lib
- -DCMAKE_PREFIX_PATH=${CMAKE_BINARY_DIR}/external)
-
- if(CMAKE_MAKE_PROGRAM MATCHES "make")
- # try to inherit command line arguments passed by parent "make" job
- set(make_cmd $(MAKE) opentracing)
- else()
- set(make_cmd ${CMAKE_COMMAND} --build --target opentracing)
- endif()
- set(install_cmd DESTDIR= ${CMAKE_MAKE_PROGRAM} install)
-
- include(ExternalProject)
- ExternalProject_Add(opentracing
- SOURCE_DIR ${opentracing_SOURCE_DIR}
- UPDATE_COMMAND ""
- INSTALL_DIR "external"
- PREFIX "external/opentracing-cpp"
- CMAKE_ARGS ${opentracing_CMAKE_ARGS}
- BUILD_IN_SOURCE 1
- BUILD_COMMAND ${make_cmd}
- INSTALL_COMMAND ${install_cmd}
- BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/external/lib/libopentracing.so
- LOG_BUILD ON)
-endfunction()
diff --git a/cmake/modules/BuildOpentelemetry.cmake b/cmake/modules/BuildOpentelemetry.cmake
new file mode 100644
index 0000000000000..6bd2380627192
--- /dev/null
+++ b/cmake/modules/BuildOpentelemetry.cmake
@@ -0,0 +1,87 @@
+function(target_create _target _lib)
+ add_library(${_target} STATIC IMPORTED)
+ set_target_properties(
+ ${_target} PROPERTIES IMPORTED_LOCATION
+ "${opentelemetry_BINARY_DIR}/${_lib}")
+endfunction()
+
+function(build_opentelemetry)
+ set(opentelemetry_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/opentelemetry-cpp")
+ set(opentelemetry_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/opentelemetry-cpp")
+ set(opentelemetry_cpp_targets opentelemetry_trace opentelemetry_exporter_jaeger_trace)
+ set(opentelemetry_CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+ -DWITH_JAEGER=ON
+ -DBUILD_TESTING=OFF
+ -DWITH_EXAMPLES=OFF
+ -DBoost_INCLUDE_DIR=${CMAKE_BINARY_DIR}/boost/include)
+
+ set(opentelemetry_libs
+ ${opentelemetry_BINARY_DIR}/sdk/src/trace/libopentelemetry_trace.a
+ ${opentelemetry_BINARY_DIR}/sdk/src/resource/libopentelemetry_resources.a
+ ${opentelemetry_BINARY_DIR}/sdk/src/common/libopentelemetry_common.a
+ ${opentelemetry_BINARY_DIR}/exporters/jaeger/libopentelemetry_exporter_jaeger_trace.a
+ ${opentelemetry_BINARY_DIR}/ext/src/http/client/curl/libhttp_client_curl.a
+ ${CURL_LIBRARIES}
+ )
+ set(opentelemetry_include_dir ${opentelemetry_SOURCE_DIR}/api/include/
+ ${opentelemetry_SOURCE_DIR}/exporters/jaeger/include/
+ ${opentelemetry_SOURCE_DIR}/ext/include/
+ ${opentelemetry_SOURCE_DIR}/sdk/include/)
+ include_directories(SYSTEM ${opentelemetry_include_dir})
+ # TODO: add target based propogation
+ set(opentelemetry_deps opentelemetry_trace opentelemetry_resources opentelemetry_common
+ opentelemetry_exporter_jaeger_trace http_client_curl
+ ${CURL_LIBRARIES})
+
+ if(CMAKE_MAKE_PROGRAM MATCHES "make")
+ # try to inherit command line arguments passed by parent "make" job
+ set(make_cmd $(MAKE) ${opentelemetry_cpp_targets})
+ else()
+ set(make_cmd ${CMAKE_COMMAND} --build --target
+ ${opentelemetry_cpp_targets})
+ endif()
+
+ if(NOT WITH_SYSTEM_BOOST)
+ list(APPEND dependencies Boost)
+ endif()
+
+ include(ExternalProject)
+ ExternalProject_Add(
+ opentelemetry-cpp
+ GIT_REPOSITORY https://github.com/ideepika/opentelemetry-cpp.git
+ GIT_TAG wip-ceph
+ GIT_SHALLOW 1
+ SOURCE_DIR ${opentelemetry_SOURCE_DIR}
+ PREFIX "opentelemetry-cpp"
+ CMAKE_ARGS ${opentelemetry_CMAKE_ARGS}
+ BUILD_COMMAND ${make_cmd}
+ BINARY_DIR ${opentelemetry_BINARY_DIR}
+ INSTALL_COMMAND ""
+ BUILD_BYPRODUCTS ${opentelemetry_libs}
+ DEPENDS ${dependencies}
+ LOG_BUILD ON)
+
+ # CMake doesn't allow to add a list of libraries to the import property, hence
+ # we create individual targets and link their libraries which finally
+ # interfaces to opentelemetry target
+ target_create("opentelemetry_trace" "sdk/src/trace/libopentelemetry_trace.a")
+ target_create("opentelemetry_resources"
+ "sdk/src/resource/libopentelemetry_resources.a")
+ target_create("opentelemetry_common"
+ "sdk/src/common/libopentelemetry_common.a")
+ target_create("opentelemetry_exporter_jaeger_trace"
+ "exporters/jaeger/libopentelemetry_exporter_jaeger_trace.a")
+ target_create("http_client_curl"
+ "ext/src/http/client/curl/libhttp_client_curl.a")
+
+ # will do all linking and path setting fake include path for
+ # interface_include_directories since this happens at build time
+ file(MAKE_DIRECTORY
+ "${opentelemetry_BINARY_DIR}/opentelemetry-cpp/exporters/jaeger/include")
+ add_library(opentelemetry::libopentelemetry INTERFACE IMPORTED)
+ add_dependencies(opentelemetry::libopentelemetry opentelemetry-cpp)
+ set_target_properties(
+ opentelemetry::libopentelemetry
+ PROPERTIES
+ INTERFACE_LINK_LIBRARIES "${opentelemetry_deps}")
+endfunction()
diff --git a/cmake/modules/BuildQatDrv.cmake b/cmake/modules/BuildQatDrv.cmake
deleted file mode 100644
index ebf33d6464c36..0000000000000
--- a/cmake/modules/BuildQatDrv.cmake
+++ /dev/null
@@ -1,33 +0,0 @@
-##
-# Make file for QAT linux driver project
-##
-
-set(qatdrv_root_dir "${CMAKE_BINARY_DIR}/qatdrv")
-set(qatdrv_url "https://01.org/sites/default/files/downloads/intelr-quickassist-technology/qat1.7.l.4.2.0-00012.tar.gz")
-set(qatdrv_url_hash "SHA256=47990b3283ded748799dba42d4b0e1bdc0be3cf3978bd587533cd12788b03856")
-set(qatdrv_config_args "--enable-qat-uio")
-
-include(ExternalProject)
-ExternalProject_Add(QatDrv
- URL ${qatdrv_url}
- URL_HASH ${qatdrv_url_hash}
- CONFIGURE_COMMAND ${qatdrv_env} ./configure ${qatdrv_config_args}
-
-# Temporarily forcing single thread as multi-threaded make is causing build
-# failues.
- BUILD_COMMAND make -j1 quickassist-all
- BUILD_IN_SOURCE 1
- INSTALL_COMMAND ""
- TEST_COMMAND ""
- PREFIX ${qatdrv_root_dir})
-
-set(QatDrv_INCLUDE_DIRS
- ${qatdrv_root_dir}/src/QatDrv/quickassist/include
- ${qatdrv_root_dir}/src/QatDrv/quickassist/lookaside/access_layer/include
- ${qatdrv_root_dir}/src/QatDrv/quickassist/include/lac
- ${qatdrv_root_dir}/src/QatDrv/quickassist/utilities/libusdm_drv
- ${qatdrv_root_dir}/src/QatDrv/quickassist/utilities/libusdm_drv/linux/include)
-
-set(QatDrv_LIBRARIES
- ${qatdrv_root_dir}/src/QatDrv/build/libqat_s.so
- ${qatdrv_root_dir}/src/QatDrv/build/libusdm_drv_s.so)
diff --git a/cmake/modules/BuildSPDK.cmake b/cmake/modules/BuildSPDK.cmake
index 0a6acc8d05997..d6ce97e1d2e87 100644
--- a/cmake/modules/BuildSPDK.cmake
+++ b/cmake/modules/BuildSPDK.cmake
@@ -68,12 +68,14 @@ macro(build_spdk)
add_dependencies(${spdk_lib} spdk-ext)
endforeach()
- set_target_properties(spdk::env_dpdk PROPERTIES
- INTERFACE_LINK_LIBRARIES "dpdk::dpdk;rt")
- set_target_properties(spdk::lvol PROPERTIES
- INTERFACE_LINK_LIBRARIES spdk::util)
- set_target_properties(spdk::util PROPERTIES
- INTERFACE_LINK_LIBRARIES ${UUID_LIBRARIES})
set(SPDK_INCLUDE_DIR "${source_dir}/include")
+ add_library(spdk::spdk INTERFACE IMPORTED)
+ add_dependencies(spdk::spdk
+ ${SPDK_LIBRARIES})
+ # workaround for https://review.spdk.io/gerrit/c/spdk/spdk/+/6798
+ set_target_properties(spdk::spdk PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES ${SPDK_INCLUDE_DIR}
+ INTERFACE_LINK_LIBRARIES
+ "-Wl,--whole-archive $ -Wl,--no-whole-archive;dpdk::dpdk;rt;${UUID_LIBRARIES}")
unset(source_dir)
endmacro()
diff --git a/cmake/modules/BuildUtf8proc.cmake b/cmake/modules/BuildUtf8proc.cmake
new file mode 100644
index 0000000000000..3f74a9b46f9dd
--- /dev/null
+++ b/cmake/modules/BuildUtf8proc.cmake
@@ -0,0 +1,69 @@
+# utf8proc is a dependency of the arrow submodule
+
+function(build_utf8proc)
+ # only build static version
+ list(APPEND utf8proc_CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF)
+
+ # cmake doesn't properly handle arguments containing ";", such as
+ # CMAKE_PREFIX_PATH, for which reason we'll have to use some other separator.
+ string(REPLACE ";" "!" CMAKE_PREFIX_PATH_ALT_SEP "${CMAKE_PREFIX_PATH}")
+ list(APPEND utf8proc_CMAKE_ARGS -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH_ALT_SEP})
+ if(CMAKE_TOOLCHAIN_FILE)
+ list(APPEND utf8proc_CMAKE_ARGS
+ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
+ endif()
+
+ list(APPEND utf8proc_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER})
+ list(APPEND utf8proc_CMAKE_ARGS -DCMAKE_AR=${CMAKE_AR})
+ list(APPEND utf8proc_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
+
+ set(utf8proc_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/utf8proc")
+ set(utf8proc_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/utf8proc")
+
+ set(utf8proc_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/utf8proc/install")
+ list(APPEND utf8proc_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${utf8proc_INSTALL_PREFIX})
+
+ # expose the install path as utf8proc_ROOT for find_package()
+ set(utf8proc_ROOT "${utf8proc_INSTALL_PREFIX}" PARENT_SCOPE)
+
+ set(utf8proc_INSTALL_LIBDIR "lib") # force lib so we don't have to guess between lib/lib64
+ list(APPEND utf8proc_CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${utf8proc_INSTALL_LIBDIR})
+ set(utf8proc_LIBRARY_DIR "${utf8proc_INSTALL_PREFIX}/${utf8proc_INSTALL_LIBDIR}")
+
+ set(utf8proc_LIBRARY "${utf8proc_LIBRARY_DIR}/libutf8proc.a")
+
+ set(utf8proc_INCLUDE_DIR "${utf8proc_INSTALL_PREFIX}/include")
+ # this include directory won't exist until the install step, but the
+ # imported target needs it early for INTERFACE_INCLUDE_DIRECTORIES
+ file(MAKE_DIRECTORY "${utf8proc_INCLUDE_DIR}")
+
+ set(utf8proc_BYPRODUCTS ${utf8proc_LIBRARY})
+
+ if(CMAKE_MAKE_PROGRAM MATCHES "make")
+ # try to inherit command line arguments passed by parent "make" job
+ set(make_cmd $(MAKE))
+ else()
+ set(make_cmd ${CMAKE_COMMAND} --build )
+ endif()
+
+ # we use an external project and copy the sources to bin directory to ensure
+ # that object files are built outside of the source tree.
+ include(ExternalProject)
+ ExternalProject_Add(utf8proc_ext
+ SOURCE_DIR "${utf8proc_SOURCE_DIR}"
+ CMAKE_ARGS ${utf8proc_CMAKE_ARGS}
+ BINARY_DIR "${utf8proc_BINARY_DIR}"
+ BUILD_COMMAND "${make_cmd}"
+ BUILD_BYPRODUCTS "${utf8proc_BYPRODUCTS}"
+ INSTALL_DIR "${utf8proc_INSTALL_PREFIX}"
+ DEPENDS "${utf8proc_DEPENDS}"
+ LIST_SEPARATOR !)
+
+ add_library(utf8proc::utf8proc STATIC IMPORTED)
+ add_dependencies(utf8proc::utf8proc utf8proc_ext)
+ set_target_properties(utf8proc::utf8proc PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${utf8proc_INCLUDE_DIR}"
+ INTERFACE_LINK_LIBRARIES "${utf8proc_INTERFACE_LINK_LIBRARIES}"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+ IMPORTED_LOCATION "${utf8proc_LIBRARY}")
+endfunction()
diff --git a/cmake/modules/CheckCxxAtomic.cmake b/cmake/modules/CheckCxxAtomic.cmake
index f2d89cf3e0beb..02d55612e818e 100644
--- a/cmake/modules/CheckCxxAtomic.cmake
+++ b/cmake/modules/CheckCxxAtomic.cmake
@@ -10,8 +10,9 @@ function(check_cxx_atomics var)
check_cxx_source_compiles("
#include
#include
+#include
-#if __s390x__
+#if defined(__SIZEOF_INT128__)
// Boost needs 16-byte atomics for tagged pointers.
// These are implemented via inline instructions on the platform
// if 16-byte alignment can be proven, and are delegated to libatomic
@@ -21,13 +22,26 @@ function(check_cxx_atomics var)
// We specifically test access via an otherwise unknown pointer here
// to ensure we get the most complex case. If this access can be
// done without libatomic, then all accesses can be done.
-bool atomic16(std::atomic *ptr)
+struct tagged_ptr {
+ int* ptr;
+ std::size_t tag;
+};
+
+void atomic16(std::atomic *ptr)
{
- return *ptr != 0;
+ tagged_ptr p{nullptr, 1};
+ ptr->store(p);
+ tagged_ptr f = ptr->load();
+ tagged_ptr new_tag{nullptr, 0};
+ ptr->compare_exchange_strong(f, new_tag);
}
#endif
int main() {
+#if defined(__SIZEOF_INT128__)
+ std::atomic ptr;
+ atomic16(&ptr);
+#endif
std::atomic w1;
std::atomic w2;
std::atomic w4;
diff --git a/cmake/modules/Distutils.cmake b/cmake/modules/Distutils.cmake
index 8dc69f0af5185..192c8c3efe2c6 100644
--- a/cmake/modules/Distutils.cmake
+++ b/cmake/modules/Distutils.cmake
@@ -1,5 +1,16 @@
include(CMakeParseArguments)
+# ensure that we are using the exact python version specified by
+# 'WITH_PYTHON3', in case some included 3rd party libraries call
+# 'find_package(Python3 ...) without specifying the exact version number. if
+# the building host happens to have a higher version of python3, that version
+# would be picked up instead by find_package(Python3). and that is not want we
+# expect.
+find_package(Python3 ${WITH_PYTHON3} EXACT
+ QUIET
+ REQUIRED
+ COMPONENTS Interpreter)
+
function(distutils_install_module name)
set(py_srcs setup.py README.rst requirements.txt test-requirements.txt bin ${name})
foreach(src ${py_srcs})
@@ -63,9 +74,8 @@ function(distutils_add_cython_module target name src)
set(PY_CXX ${compiler_launcher} ${CMAKE_CXX_COMPILER} ${cxx_compiler_arg1})
set(PY_LDSHARED ${link_launcher} ${CMAKE_C_COMPILER} ${c_compiler_arg1} "-shared")
- set(suffix_var "EXT_SUFFIX")
execute_process(COMMAND "${Python3_EXECUTABLE}" -c
- "from distutils import sysconfig; print(sysconfig.get_config_var('${suffix_var}'))"
+ "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))"
RESULT_VARIABLE result
OUTPUT_VARIABLE ext_suffix
ERROR_VARIABLE error
@@ -75,6 +85,9 @@ function(distutils_add_cython_module target name src)
endif()
set(output_dir "${CYTHON_MODULE_DIR}/lib.3")
set(setup_py ${CMAKE_CURRENT_SOURCE_DIR}/setup.py)
+ if(DEFINED ENV{VERBOSE})
+ set(maybe_verbose --verbose)
+ endif()
add_custom_command(
OUTPUT ${output_dir}/${name}${ext_suffix}
COMMAND
@@ -89,7 +102,7 @@ function(distutils_add_cython_module target name src)
CYTHON_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR}
CEPH_LIBDIR=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
${Python3_EXECUTABLE} ${setup_py}
- build --verbose --build-base ${CYTHON_MODULE_DIR}
+ build ${maybe_verbose} --build-base ${CYTHON_MODULE_DIR}
--build-platlib ${output_dir}
MAIN_DEPENDENCY ${src}
DEPENDS ${setup_py}
@@ -107,6 +120,9 @@ function(distutils_install_cython_module name)
if(DU_DISABLE_VTA AND HAS_VTA)
set(CFLAG_DISABLE_VTA -fno-var-tracking-assignments)
endif()
+ if(DEFINED ENV{VERBOSE})
+ set(maybe_verbose --verbose)
+ endif()
install(CODE "
set(ENV{CC} \"${PY_CC}\")
set(ENV{LDSHARED} \"${PY_LDSHARED}\")
@@ -130,12 +146,12 @@ function(distutils_install_cython_module name)
execute_process(
COMMAND
${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/setup.py
- build --verbose --build-base ${CYTHON_MODULE_DIR}
+ build ${maybe_verbose} --build-base ${CYTHON_MODULE_DIR}
--build-platlib ${CYTHON_MODULE_DIR}/lib.3
build_ext --cython-c-in-temp --build-temp ${CMAKE_CURRENT_BINARY_DIR} --cython-include-dirs ${PROJECT_SOURCE_DIR}/src/pybind/rados
install \${options} --single-version-externally-managed --record /dev/null
egg_info --egg-base ${CMAKE_CURRENT_BINARY_DIR}
- --verbose
+ ${maybe_verbose}
WORKING_DIRECTORY \"${CMAKE_CURRENT_SOURCE_DIR}\"
RESULT_VARIABLE install_res)
if(NOT \"\${install_res}\" STREQUAL 0)
diff --git a/cmake/modules/FindArrow.cmake b/cmake/modules/FindArrow.cmake
new file mode 100644
index 0000000000000..39beed4b0515b
--- /dev/null
+++ b/cmake/modules/FindArrow.cmake
@@ -0,0 +1,477 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so)
+# This module defines
+# ARROW_FOUND, whether Arrow has been found
+# ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0"
+# ARROW_IMPORT_LIB, path to libarrow's import library (Windows only)
+# ARROW_INCLUDE_DIR, directory containing headers
+# ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead
+# ARROW_LIB_DIR, directory containing Arrow libraries
+# ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead
+# ARROW_SHARED_LIB, path to libarrow's shared library
+# ARROW_SO_VERSION, shared object version of found Arrow such as "100"
+# ARROW_STATIC_LIB, path to libarrow.a
+# ARROW_VERSION, version of found Arrow
+# ARROW_VERSION_MAJOR, major version of found Arrow
+# ARROW_VERSION_MINOR, minor version of found Arrow
+# ARROW_VERSION_PATCH, patch version of found Arrow
+
+# cbodley copied this from the arrow submodule at v6.0.1
+# cbodley added the import target Arrow::Arrow to match build_arrow()
+
+if(DEFINED ARROW_FOUND)
+ return()
+endif()
+
+include(FindPkgConfig)
+include(FindPackageHandleStandardArgs)
+
+if(WIN32 AND NOT MINGW)
+ # This is used to handle builds using e.g. clang in an MSVC setting.
+ set(MSVC_TOOLCHAIN TRUE)
+else()
+ set(MSVC_TOOLCHAIN FALSE)
+endif()
+
+set(ARROW_SEARCH_LIB_PATH_SUFFIXES)
+if(CMAKE_LIBRARY_ARCHITECTURE)
+ list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
+endif()
+list(APPEND
+ ARROW_SEARCH_LIB_PATH_SUFFIXES
+ "lib64"
+ "lib32"
+ "lib"
+ "bin")
+set(ARROW_CONFIG_SUFFIXES
+ "_RELEASE"
+ "_RELWITHDEBINFO"
+ "_MINSIZEREL"
+ "_DEBUG"
+ "")
+if(CMAKE_BUILD_TYPE)
+ string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED)
+ set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}")
+ list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}")
+endif()
+
+if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
+ if(MSVC_TOOLCHAIN)
+ set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
+ else()
+ set(ARROW_MSVC_STATIC_LIB_SUFFIX "")
+ endif()
+endif()
+
+# Internal function.
+#
+# Set shared library name for ${base_name} to ${output_variable}.
+#
+# Example:
+# arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow)
+# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux
+# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS
+# # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows
+# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows
+function(arrow_build_shared_library_name output_variable base_name)
+ set(${output_variable}
+ "${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}"
+ PARENT_SCOPE)
+endfunction()
+
+# Internal function.
+#
+# Set import library name for ${base_name} to ${output_variable}.
+# This is useful only for MSVC build. Import library is used only
+# with MSVC build.
+#
+# Example:
+# arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow)
+# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless)
+# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless)
+# # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows
+# # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
+function(arrow_build_import_library_name output_variable base_name)
+ set(${output_variable}
+ "${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
+ PARENT_SCOPE)
+endfunction()
+
+# Internal function.
+#
+# Set static library name for ${base_name} to ${output_variable}.
+#
+# Example:
+# arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow)
+# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux
+# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS
+# # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows
+# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
+function(arrow_build_static_library_name output_variable base_name)
+ set(${output_variable}
+ "${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}"
+ PARENT_SCOPE)
+endfunction()
+
+# Internal function.
+#
+# Set macro value for ${macro_name} in ${header_content} to ${output_variable}.
+#
+# Example:
+# arrow_extract_macro_value(version_major
+# "ARROW_VERSION_MAJOR"
+# "#define ARROW_VERSION_MAJOR 1.0.0")
+# # -> version_major=1.0.0
+function(arrow_extract_macro_value output_variable macro_name header_content)
+ string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition
+ "${header_content}")
+ string(REGEX REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value
+ "${macro_definition}")
+ set(${output_variable}
+ "${macro_value}"
+ PARENT_SCOPE)
+endfunction()
+
+# Internal macro only for arrow_find_package.
+#
+# Find package in HOME.
+macro(arrow_find_package_home)
+ find_path(${prefix}_include_dir "${header_path}"
+ PATHS "${home}"
+ PATH_SUFFIXES "include"
+ NO_DEFAULT_PATH)
+ set(include_dir "${${prefix}_include_dir}")
+ set(${prefix}_INCLUDE_DIR
+ "${include_dir}"
+ PARENT_SCOPE)
+
+ if(MSVC_TOOLCHAIN)
+ set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES})
+ # .dll isn't found by find_library with MSVC because .dll isn't included in
+ # CMAKE_FIND_LIBRARY_SUFFIXES.
+ list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ endif()
+ find_library(${prefix}_shared_lib
+ NAMES "${shared_lib_name}"
+ PATHS "${home}"
+ PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
+ NO_DEFAULT_PATH)
+ if(MSVC_TOOLCHAIN)
+ set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL})
+ endif()
+ set(shared_lib "${${prefix}_shared_lib}")
+ set(${prefix}_SHARED_LIB
+ "${shared_lib}"
+ PARENT_SCOPE)
+ if(shared_lib)
+ add_library(${target_shared} SHARED IMPORTED)
+ set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}")
+ if(include_dir)
+ set_target_properties(${target_shared} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+ "${include_dir}")
+ endif()
+ find_library(${prefix}_import_lib
+ NAMES "${import_lib_name}"
+ PATHS "${home}"
+ PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
+ NO_DEFAULT_PATH)
+ set(import_lib "${${prefix}_import_lib}")
+ set(${prefix}_IMPORT_LIB
+ "${import_lib}"
+ PARENT_SCOPE)
+ if(import_lib)
+ set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}")
+ endif()
+ endif()
+
+ find_library(${prefix}_static_lib
+ NAMES "${static_lib_name}"
+ PATHS "${home}"
+ PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
+ NO_DEFAULT_PATH)
+ set(static_lib "${${prefix}_static_lib}")
+ set(${prefix}_STATIC_LIB
+ "${static_lib}"
+ PARENT_SCOPE)
+ if(static_lib)
+ add_library(${target_static} STATIC IMPORTED)
+ set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}")
+ if(include_dir)
+ set_target_properties(${target_static} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+ "${include_dir}")
+ endif()
+ endif()
+endmacro()
+
+# Internal macro only for arrow_find_package.
+#
+# Find package by CMake package configuration.
+macro(arrow_find_package_cmake_package_configuration)
+ find_package(${cmake_package_name} CONFIG)
+ if(${cmake_package_name}_FOUND)
+ set(${prefix}_USE_CMAKE_PACKAGE_CONFIG
+ TRUE
+ PARENT_SCOPE)
+ if(TARGET ${target_shared})
+ foreach(suffix ${ARROW_CONFIG_SUFFIXES})
+ get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix})
+ if(shared_lib)
+ # Remove shared library version:
+ # libarrow.so.100.0.0 -> libarrow.so
+ # Because ARROW_HOME and pkg-config approaches don't add
+ # shared library version.
+ string(REGEX REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1"
+ shared_lib "${shared_lib}")
+ set(${prefix}_SHARED_LIB
+ "${shared_lib}"
+ PARENT_SCOPE)
+ break()
+ endif()
+ endforeach()
+ endif()
+ if(TARGET ${target_static})
+ foreach(suffix ${ARROW_CONFIG_SUFFIXES})
+ get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix})
+ if(static_lib)
+ set(${prefix}_STATIC_LIB
+ "${static_lib}"
+ PARENT_SCOPE)
+ break()
+ endif()
+ endforeach()
+ endif()
+ endif()
+endmacro()
+
+# Internal macro only for arrow_find_package.
+#
+# Find package by pkg-config.
+macro(arrow_find_package_pkg_config)
+ pkg_check_modules(${prefix}_PC ${pkg_config_name})
+ if(${prefix}_PC_FOUND)
+ set(${prefix}_USE_PKG_CONFIG
+ TRUE
+ PARENT_SCOPE)
+
+ set(include_dir "${${prefix}_PC_INCLUDEDIR}")
+ set(lib_dir "${${prefix}_PC_LIBDIR}")
+ set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}")
+ # Use the first shared library path as the IMPORTED_LOCATION
+ # for ${target_shared}. This assumes that the first shared library
+ # path is the shared library path for this module.
+ list(GET shared_lib_paths 0 first_shared_lib_path)
+ # Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES
+ # for ${target_shared}. This assumes that the rest shared library
+ # paths are dependency library paths for this module.
+ list(LENGTH shared_lib_paths n_shared_lib_paths)
+ if(n_shared_lib_paths LESS_EQUAL 1)
+ set(rest_shared_lib_paths)
+ else()
+ list(SUBLIST
+ shared_lib_paths
+ 1
+ -1
+ rest_shared_lib_paths)
+ endif()
+
+ set(${prefix}_VERSION
+ "${${prefix}_PC_VERSION}"
+ PARENT_SCOPE)
+ set(${prefix}_INCLUDE_DIR
+ "${include_dir}"
+ PARENT_SCOPE)
+ set(${prefix}_SHARED_LIB
+ "${first_shared_lib_path}"
+ PARENT_SCOPE)
+
+ add_library(${target_shared} SHARED IMPORTED)
+ set_target_properties(${target_shared}
+ PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}"
+ INTERFACE_LINK_LIBRARIES "${rest_shared_lib_paths}"
+ IMPORTED_LOCATION "${first_shared_lib_path}")
+ get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION)
+
+ find_library(${prefix}_static_lib
+ NAMES "${static_lib_name}"
+ PATHS "${lib_dir}"
+ NO_DEFAULT_PATH)
+ set(static_lib "${${prefix}_static_lib}")
+ set(${prefix}_STATIC_LIB
+ "${static_lib}"
+ PARENT_SCOPE)
+ if(static_lib)
+ add_library(${target_static} STATIC IMPORTED)
+ set_target_properties(${target_static}
+ PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}"
+ IMPORTED_LOCATION "${static_lib}")
+ endif()
+ endif()
+endmacro()
+
+function(arrow_find_package
+ prefix
+ home
+ base_name
+ header_path
+ cmake_package_name
+ pkg_config_name)
+ arrow_build_shared_library_name(shared_lib_name ${base_name})
+ arrow_build_import_library_name(import_lib_name ${base_name})
+ arrow_build_static_library_name(static_lib_name ${base_name})
+
+ set(target_shared ${base_name}_shared)
+ set(target_static ${base_name}_static)
+
+ if(home)
+ arrow_find_package_home()
+ set(${prefix}_FIND_APPROACH
+ "HOME: ${home}"
+ PARENT_SCOPE)
+ else()
+ arrow_find_package_cmake_package_configuration()
+ if(${cmake_package_name}_FOUND)
+ set(${prefix}_FIND_APPROACH
+ "CMake package configuration: ${cmake_package_name}"
+ PARENT_SCOPE)
+ else()
+ arrow_find_package_pkg_config()
+ set(${prefix}_FIND_APPROACH
+ "pkg-config: ${pkg_config_name}"
+ PARENT_SCOPE)
+ endif()
+ endif()
+
+ if(NOT include_dir)
+ if(TARGET ${target_shared})
+ get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES)
+ elseif(TARGET ${target_static})
+ get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES)
+ endif()
+ endif()
+ if(include_dir)
+ set(${prefix}_INCLUDE_DIR
+ "${include_dir}"
+ PARENT_SCOPE)
+ endif()
+
+ if(shared_lib)
+ get_filename_component(lib_dir "${shared_lib}" DIRECTORY)
+ elseif(static_lib)
+ get_filename_component(lib_dir "${static_lib}" DIRECTORY)
+ else()
+ set(lib_dir NOTFOUND)
+ endif()
+ set(${prefix}_LIB_DIR
+ "${lib_dir}"
+ PARENT_SCOPE)
+ # For backward compatibility
+ set(${prefix}_LIBS
+ "${lib_dir}"
+ PARENT_SCOPE)
+endfunction()
+
+if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
+ file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME)
+endif()
+arrow_find_package(ARROW
+ "${ARROW_HOME}"
+ arrow
+ arrow/api.h
+ Arrow
+ arrow)
+
+if(ARROW_HOME)
+ if(ARROW_INCLUDE_DIR)
+ file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT)
+ arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR"
+ "${ARROW_CONFIG_H_CONTENT}")
+ arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR"
+ "${ARROW_CONFIG_H_CONTENT}")
+ arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH"
+ "${ARROW_CONFIG_H_CONTENT}")
+ if("${ARROW_VERSION_MAJOR}" STREQUAL ""
+ OR "${ARROW_VERSION_MINOR}" STREQUAL ""
+ OR "${ARROW_VERSION_PATCH}" STREQUAL "")
+ set(ARROW_VERSION "0.0.0")
+ else()
+ set(ARROW_VERSION
+ "${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}")
+ endif()
+
+ arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION"
+ "${ARROW_CONFIG_H_CONTENT}")
+ string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}")
+ arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION"
+ "${ARROW_CONFIG_H_CONTENT}")
+ string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION
+ "${ARROW_FULL_SO_VERSION_QUOTED}")
+ endif()
+else()
+ if(ARROW_USE_CMAKE_PACKAGE_CONFIG)
+ find_package(Arrow CONFIG)
+ elseif(ARROW_USE_PKG_CONFIG)
+ pkg_get_variable(ARROW_SO_VERSION arrow so_version)
+ pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version)
+ endif()
+endif()
+
+set(ARROW_ABI_VERSION ${ARROW_SO_VERSION})
+
+mark_as_advanced(ARROW_ABI_VERSION
+ ARROW_CONFIG_SUFFIXES
+ ARROW_FULL_SO_VERSION
+ ARROW_IMPORT_LIB
+ ARROW_INCLUDE_DIR
+ ARROW_LIBS
+ ARROW_LIB_DIR
+ ARROW_SEARCH_LIB_PATH_SUFFIXES
+ ARROW_SHARED_IMP_LIB
+ ARROW_SHARED_LIB
+ ARROW_SO_VERSION
+ ARROW_STATIC_LIB
+ ARROW_VERSION
+ ARROW_VERSION_MAJOR
+ ARROW_VERSION_MINOR
+ ARROW_VERSION_PATCH)
+
+find_package_handle_standard_args(
+ Arrow
+ REQUIRED_VARS # The first required variable is shown
+ # in the found message. So this list is
+ # not sorted alphabetically.
+ ARROW_INCLUDE_DIR ARROW_LIB_DIR ARROW_FULL_SO_VERSION ARROW_SO_VERSION
+ VERSION_VAR ARROW_VERSION)
+set(ARROW_FOUND ${Arrow_FOUND})
+
+if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY)
+ message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})")
+ message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}")
+ message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}")
+ message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}")
+ message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}")
+ message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}")
+endif()
+
+if(Arrow_FOUND AND NOT TARGET Arrow::Arrow)
+ add_library(Arrow::Arrow SHARED IMPORTED)
+ set_target_properties(Arrow::Arrow PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+ IMPORTED_LOCATION "${ARROW_SHARED_LIB}")
+endif()
diff --git a/cmake/modules/FindLinuxZNS.cmake b/cmake/modules/FindLinuxZNS.cmake
new file mode 100644
index 0000000000000..e0c3bba03161f
--- /dev/null
+++ b/cmake/modules/FindLinuxZNS.cmake
@@ -0,0 +1,18 @@
+# Try to find linux/blkzoned.h
+
+find_path(LinuxZNS_INCLUDE_DIR NAMES
+ "linux/blkzoned.h")
+
+find_package_handle_standard_args(LinuxZNS
+ REQUIRED_VARS
+ LinuxZNS_INCLUDE_DIR)
+
+mark_as_advanced(
+ LinuxZNS_INCLUDE_DIR)
+
+if(LinuxZNS_FOUND AND NOT (TARGET Linux::ZNS))
+ add_library(Linux::ZNS INTERFACE IMPORTED)
+ set_target_properties(Linux::ZNS PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${LinuxZNS_INCLUDE_DIR}"
+ INTERFACE_COMPILE_DEFINITIONS HAVE_ZNS=1)
+endif()
diff --git a/cmake/modules/FindParquet.cmake b/cmake/modules/FindParquet.cmake
new file mode 100644
index 0000000000000..df5894774a6cc
--- /dev/null
+++ b/cmake/modules/FindParquet.cmake
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find Parquet (parquet/api/reader.h, libparquet.a, libparquet.so)
+#
+# This module requires Arrow from which it uses
+# arrow_find_package()
+#
+# This module defines
+# PARQUET_FOUND, whether Parquet has been found
+# PARQUET_IMPORT_LIB, path to libparquet's import library (Windows only)
+# PARQUET_INCLUDE_DIR, directory containing headers
+# PARQUET_LIBS, deprecated. Use PARQUET_LIB_DIR instead
+# PARQUET_LIB_DIR, directory containing Parquet libraries
+# PARQUET_SHARED_IMP_LIB, deprecated. Use PARQUET_IMPORT_LIB instead
+# PARQUET_SHARED_LIB, path to libparquet's shared library
+# PARQUET_SO_VERSION, shared object version of found Parquet such as "100"
+# PARQUET_STATIC_LIB, path to libparquet.a
+
+# cbodley copied this from the arrow submodule at v6.0.1
+# cbodley added the import target Arrow::Parquet to match build_arrow()
+
+if(DEFINED PARQUET_FOUND)
+ return()
+endif()
+
+set(find_package_arguments)
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION)
+ list(APPEND find_package_arguments "${${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION}")
+endif()
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED)
+ list(APPEND find_package_arguments REQUIRED)
+endif()
+if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
+ list(APPEND find_package_arguments QUIET)
+endif()
+find_package(Arrow ${find_package_arguments})
+
+if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
+ file(TO_CMAKE_PATH "$ENV{PARQUET_HOME}" PARQUET_HOME)
+endif()
+
+if((NOT PARQUET_HOME) AND ARROW_HOME)
+ set(PARQUET_HOME ${ARROW_HOME})
+endif()
+
+if(ARROW_FOUND)
+ arrow_find_package(PARQUET
+ "${PARQUET_HOME}"
+ parquet
+ parquet/api/reader.h
+ Parquet
+ parquet)
+ if(PARQUET_HOME)
+ if(PARQUET_INCLUDE_DIR)
+ file(READ "${PARQUET_INCLUDE_DIR}/parquet/parquet_version.h"
+ PARQUET_VERSION_H_CONTENT)
+ arrow_extract_macro_value(PARQUET_VERSION_MAJOR "PARQUET_VERSION_MAJOR"
+ "${PARQUET_VERSION_H_CONTENT}")
+ arrow_extract_macro_value(PARQUET_VERSION_MINOR "PARQUET_VERSION_MINOR"
+ "${PARQUET_VERSION_H_CONTENT}")
+ arrow_extract_macro_value(PARQUET_VERSION_PATCH "PARQUET_VERSION_PATCH"
+ "${PARQUET_VERSION_H_CONTENT}")
+ if("${PARQUET_VERSION_MAJOR}" STREQUAL ""
+ OR "${PARQUET_VERSION_MINOR}" STREQUAL ""
+ OR "${PARQUET_VERSION_PATCH}" STREQUAL "")
+ set(PARQUET_VERSION "0.0.0")
+ else()
+ set(PARQUET_VERSION
+ "${PARQUET_VERSION_MAJOR}.${PARQUET_VERSION_MINOR}.${PARQUET_VERSION_PATCH}")
+ endif()
+
+ arrow_extract_macro_value(PARQUET_SO_VERSION_QUOTED "PARQUET_SO_VERSION"
+ "${PARQUET_VERSION_H_CONTENT}")
+ string(REGEX REPLACE "^\"(.+)\"$" "\\1" PARQUET_SO_VERSION
+ "${PARQUET_SO_VERSION_QUOTED}")
+ arrow_extract_macro_value(PARQUET_FULL_SO_VERSION_QUOTED "PARQUET_FULL_SO_VERSION"
+ "${PARQUET_VERSION_H_CONTENT}")
+ string(REGEX REPLACE "^\"(.+)\"$" "\\1" PARQUET_FULL_SO_VERSION
+ "${PARQUET_FULL_SO_VERSION_QUOTED}")
+ endif()
+ else()
+ if(PARQUET_USE_CMAKE_PACKAGE_CONFIG)
+ find_package(Parquet CONFIG)
+ elseif(PARQUET_USE_PKG_CONFIG)
+ pkg_get_variable(PARQUET_SO_VERSION parquet so_version)
+ pkg_get_variable(PARQUET_FULL_SO_VERSION parquet full_so_version)
+ endif()
+ endif()
+ set(PARQUET_ABI_VERSION "${PARQUET_SO_VERSION}")
+endif()
+
+mark_as_advanced(PARQUET_ABI_VERSION
+ PARQUET_IMPORT_LIB
+ PARQUET_INCLUDE_DIR
+ PARQUET_LIBS
+ PARQUET_LIB_DIR
+ PARQUET_SHARED_IMP_LIB
+ PARQUET_SHARED_LIB
+ PARQUET_SO_VERSION
+ PARQUET_STATIC_LIB
+ PARQUET_VERSION)
+
+find_package_handle_standard_args(
+ Parquet
+ REQUIRED_VARS PARQUET_INCLUDE_DIR PARQUET_LIB_DIR PARQUET_SO_VERSION
+ VERSION_VAR PARQUET_VERSION)
+set(PARQUET_FOUND ${Parquet_FOUND})
+
+if(Parquet_FOUND AND NOT Parquet_FIND_QUIETLY)
+ message(STATUS "Parquet version: ${PARQUET_VERSION} (${PARQUET_FIND_APPROACH})")
+ message(STATUS "Found the Parquet shared library: ${PARQUET_SHARED_LIB}")
+ message(STATUS "Found the Parquet import library: ${PARQUET_IMPORT_LIB}")
+ message(STATUS "Found the Parquet static library: ${PARQUET_STATIC_LIB}")
+endif()
+
+if(Parquet_FOUND AND NOT TARGET Arrow::Parquet)
+ add_library(Arrow::Parquet SHARED IMPORTED)
+ set_target_properties(Arrow::Parquet PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${PARQUET_INCLUDE_DIR}"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+ IMPORTED_LOCATION "${PARQUET_SHARED_LIB}")
+endif()
diff --git a/cmake/modules/FindPython/Support.cmake b/cmake/modules/FindPython/Support.cmake
index c05bbe33062d9..fb362bfe21cf3 100644
--- a/cmake/modules/FindPython/Support.cmake
+++ b/cmake/modules/FindPython/Support.cmake
@@ -17,7 +17,7 @@ if (NOT DEFINED _${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR)
message (FATAL_ERROR "FindPython: INTERNAL ERROR")
endif()
if (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR EQUAL 3)
- set(_${_PYTHON_PREFIX}_VERSIONS 3.9 3.8 3.7 3.6 3.5 3.4 3.3 3.2 3.1 3.0)
+ set(_${_PYTHON_PREFIX}_VERSIONS 3.10 3.9 3.8 3.7 3.6 3.5 3.4 3.3 3.2 3.1 3.0)
elseif (_${_PYTHON_PREFIX}_REQUIRED_VERSION_MAJOR EQUAL 2)
set(_${_PYTHON_PREFIX}_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0)
else()
diff --git a/cmake/modules/FindQatDrv.cmake b/cmake/modules/FindQatDrv.cmake
new file mode 100644
index 0000000000000..3305a38c06cb9
--- /dev/null
+++ b/cmake/modules/FindQatDrv.cmake
@@ -0,0 +1,80 @@
+# - Find_QatDrv
+# Find the qat driver library and includes
+#
+# QatDrv_INCLUDE_DIRS = where to find cap.h, qae_mem.h, etc
+# QatDrv_LIBRARIES - List of libraries when using qat driver
+# QatDrv_FOUND - True if qat driver found
+
+set(expected_version "v1.7.L.4.14.0-00031")
+set(expected_version_url "https://www.intel.com/content/www/us/en/download/19081/intel-quickassist-technology-intel-qat-driver-for-linux-for-intel-server-boards-and-systems-based-on-intel-62x-chipset.html")
+
+function(get_qatdrv_version versionfile)
+ file(STRINGS "${versionfile}" QAT_VERSION_LINE
+ REGEX "^PACKAGE_VERSION_MAJOR_NUMBER=[0-9]+$")
+ string(REGEX REPLACE "^PACKAGE_VERSION_MAJOR_NUMBER=([0-9]+)$"
+ "\\1" QAT_VERSION1 "${QAT_VERSION_LINE}")
+ unset(QAT_VERSION_LINE)
+ file(STRINGS "${versionfile}" QAT_VERSION_LINE
+ REGEX "^PACKAGE_VERSION_MINOR_NUMBER=[0-9]+$")
+ string(REGEX REPLACE "^PACKAGE_VERSION_MINOR_NUMBER=([0-9]+)$"
+ "\\1" QAT_VERSION2 "${QAT_VERSION_LINE}")
+ unset(QAT_VERSION_LINE)
+ file(STRINGS "${versionfile}" QAT_VERSION_LINE
+ REGEX "^PACKAGE_VERSION_PATCH_NUMBER=[0-9]+$")
+ string(REGEX REPLACE "^PACKAGE_VERSION_PATCH_NUMBER=([0-9]+)$"
+ "\\1" QAT_VERSION3 "${QAT_VERSION_LINE}")
+ unset(QAT_VERSION_LINE)
+ file(STRINGS "${versionfile}" QAT_VERSION_LINE
+ REGEX "^PACKAGE_VERSION_BUILD_NUMBER=[0-9]+$")
+ string(REGEX REPLACE "^PACKAGE_VERSION_BUILD_NUMBER=([0-9]+)$"
+ "\\1" QAT_VERSION4 "${QAT_VERSION_LINE}")
+
+ set(QatDrv_VERSION_MAJOR ${QAT_VERSION1} PARENT_SCOPE)
+ set(QatDrv_VERSION_MINOR ${QAT_VERSION2} PARENT_SCOPE)
+ set(QatDrv_VERSION_PATCH ${QAT_VERSION3} PARENT_SCOPE)
+ set(QatDrv_VERSION_TWEAK ${QAT_VERSION4} PARENT_SCOPE)
+ set(QatDrv_VERSION_COUNT 4 PARENT_SCOPE)
+ set(QatDrv_VERSION ${QAT_VERSION1}.${QAT_VERSION2}.${QAT_VERSION3}.${QAT_VERSION4}
+ PARENT_SCOPE)
+endfunction()
+
+find_path(QATDRV_INCLUDE_DIR
+ name quickassist/include/cpa.h
+ HINTS $ENV{ICP_ROOT} /opt/APP/driver/QAT
+ NO_DEFAULT_PATH)
+if(QATDRV_INCLUDE_DIR)
+ get_qatdrv_version(${QATDRV_INCLUDE_DIR}/versionfile)
+ set(QatDrv_INCLUDE_DIRS
+ ${QATDRV_INCLUDE_DIR}/quickassist/include
+ ${QATDRV_INCLUDE_DIR}/quickassist/include/dc
+ ${QATDRV_INCLUDE_DIR}/quickassist/lookaside/access_layer/include
+ ${QATDRV_INCLUDE_DIR}/quickassist/include/lac
+ ${QATDRV_INCLUDE_DIR}/quickassist/utilities/libusdm_drv
+ ${QATDRV_INCLUDE_DIR}/quickassist/utilities/libusdm_drv/include)
+endif()
+foreach(component ${QatDrv_FIND_COMPONENTS})
+ find_library(QatDrv_${component}_LIBRARIES
+ NAMES ${component}
+ HINTS ${QATDRV_INCLUDE_DIR}/build/)
+ mark_as_advanced(QatDrv_INCLUDE_DIRS
+ QatDrv_${component}_LIBRARIES)
+ list(APPEND QatDrv_LIBRARIES "${QatDrv_${component}_LIBRARIES}")
+endforeach()
+
+set(failure_message "Please ensure QAT driver has been installed with version no less than ${expected_version}. And set the environment variable \"ICP_ROOT\" for the QAT driver package root directory, i.e. \"export ICP_ROOT=/the/directory/of/QAT\". Or download and install QAT driver ${expected_version} manually at the link ${expected_version_url}. Remember to set the environment variable \"ICP_ROOT\" for the package root directory.")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(QatDrv
+ REQUIRED_VARS QatDrv_LIBRARIES QatDrv_INCLUDE_DIRS
+ VERSION_VAR QatDrv_VERSION
+ FAIL_MESSAGE ${failure_message})
+
+foreach(component ${QatDrv_FIND_COMPONENTS})
+ if(NOT TARGET QatDrv::${component})
+ add_library(QatDrv::${component} STATIC IMPORTED GLOBAL)
+ set_target_properties(QatDrv::${component} PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${QatDrv_INCLUDE_DIRS}"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+ IMPORTED_LOCATION "${QatDrv_${component}_LIBRARIES}")
+ endif()
+endforeach()
diff --git a/cmake/modules/Finddml.cmake b/cmake/modules/Finddml.cmake
new file mode 100644
index 0000000000000..8e94ad26d6c36
--- /dev/null
+++ b/cmake/modules/Finddml.cmake
@@ -0,0 +1,58 @@
+# - Find libdml
+# Find the dml and dmlhl libraries and includes
+#
+# DML_INCLUDE_DIR - where to find dml.hpp etc.
+# DML_LIBRARIES - List of libraries when using dml.
+# DML_HL_LIBRARIES - List of libraries when using dmlhl.
+# DML_FOUND - True if DML found.
+
+
+find_path(DML_INCLUDE_DIR
+ dml/dml.hpp
+ PATHS
+ /usr/include
+ /usr/local/include)
+
+find_library(DML_LIBRARIES NAMES dml libdml PATHS
+ /usr/local/
+ /usr/local/lib64
+ /usr/lib64
+ /usr/lib)
+
+find_library(DML_HL_LIBRARIES NAMES dmlhl libdmlhl PATHS
+ /usr/local/
+ /usr/local/lib64
+ /usr/lib64
+ /usr/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(dml DEFAULT_MSG
+ DML_LIBRARIES
+ DML_INCLUDE_DIR
+ DML_HL_LIBRARIES)
+
+mark_as_advanced(
+ DML_LIBRARIES
+ DML_INCLUDE_DIR
+ DML_HL_LIBRARIES)
+
+if(DML_FOUND)
+ if(NOT (TARGET dml::dml))
+ add_library(dml::dml UNKNOWN IMPORTED)
+ set_target_properties(dml::dml PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+ IMPORTED_LOCATION "${DML_LIBRARIES}")
+ endif()
+
+ if(NOT (TARGET dml::dmlhl))
+ add_library(dml::dmlhl UNKNOWN IMPORTED)
+ set_target_properties(dml::dmlhl PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${DML_INCLUDE_DIR}"
+ INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}
+ INTERFACE_COMPILE_FEATURES cxx_std_17
+ INTERFACE_COMPILE_DEFINITIONS "DML_HW"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+ IMPORTED_LOCATION "${DML_HL_LIBRARIES}")
+ endif()
+endif()
diff --git a/cmake/modules/Finddpdk.cmake b/cmake/modules/Finddpdk.cmake
index fae4691a8865d..ad2ec2548c17e 100644
--- a/cmake/modules/Finddpdk.cmake
+++ b/cmake/modules/Finddpdk.cmake
@@ -26,12 +26,12 @@ else()
include)
find_path(dpdk_common_INCLUDE_DIR rte_common.h
HINTS
- ENC DPDK_DIR
+ ENV DPDK_DIR
PATH_SUFFIXES
dpdk
include)
set(dpdk_INCLUDE_DIRS "${dpdk_config_INCLUDE_DIR}")
- if(NOT dpdk_config_INCLUDE_DIR EQUAL dpdk_common_INCLUDE_DIR)
+ if(dpdk_common_INCLUDE_DIR AND NOT dpdk_config_INCLUDE_DIR STREQUAL dpdk_common_INCLUDE_DIR)
list(APPEND dpdk_INCLUDE_DIRS "${dpdk_common_INCLUDE_DIR}")
endif()
endif()
@@ -48,7 +48,6 @@ set(components
mbuf
mempool
mempool_ring
- mempool_stack
net
pci
pmd_af_packet
@@ -66,6 +65,8 @@ set(components
pmd_ring
pmd_sfc_efx
pmd_vmxnet3_uio
+ pmd_hns3
+ pmd_hinic
ring
timer)
@@ -79,11 +80,7 @@ foreach(c ${components})
get_target_property(DPDK_rte_${c}_LIBRARY
${dpdk_lib} IMPORTED_LOCATION)
else()
- find_library(DPDK_rte_${c}_LIBRARY
- NAMES
- # use static library
- ${CMAKE_STATIC_LIBRARY_PREFIX}rte_${c}${CMAKE_STATIC_LIBRARY_SUFFIX}
- rte_${c}
+ find_library(DPDK_rte_${c}_LIBRARY rte_${c}
HINTS
ENV DPDK_DIR
${dpdk_LIBRARY_DIRS}
@@ -135,7 +132,8 @@ if(dpdk_FOUND)
find_package(Threads QUIET)
list(APPEND _dpdk_libs
Threads::Threads
- dpdk::cflags)
+ dpdk::cflags
+ numa)
set_target_properties(dpdk::dpdk PROPERTIES
INTERFACE_LINK_LIBRARIES "${_dpdk_libs}"
INTERFACE_INCLUDE_DIRECTORIES "${dpdk_INCLUDE_DIRS}")
diff --git a/cmake/modules/Findleveldb.cmake b/cmake/modules/Findleveldb.cmake
deleted file mode 100644
index 32786e9799451..0000000000000
--- a/cmake/modules/Findleveldb.cmake
+++ /dev/null
@@ -1,18 +0,0 @@
-# - Find LevelDB
-#
-# LEVELDB_INCLUDE_DIR - Where to find leveldb/db.h
-# LEVELDB_LIBRARIES - List of libraries when using LevelDB.
-# LEVELDB_FOUND - True if LevelDB found.
-
-find_path(LEVELDB_INCLUDE_DIR leveldb/db.h
- HINTS $ENV{LEVELDB_ROOT}/include
- DOC "Path in which the file leveldb/db.h is located." )
-
-find_library(LEVELDB_LIBRARIES leveldb
- HINTS $ENV{LEVELDB_ROOT}/lib
- DOC "Path to leveldb library." )
-
-mark_as_advanced(LEVELDB_INCLUDE_DIR LEVELDB_LIBRARIES)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(leveldb DEFAULT_MSG LEVELDB_LIBRARIES LEVELDB_INCLUDE_DIR)
diff --git a/cmake/modules/Findqatzip.cmake b/cmake/modules/Findqatzip.cmake
index 3a593228dd010..2d0f2ace38875 100644
--- a/cmake/modules/Findqatzip.cmake
+++ b/cmake/modules/Findqatzip.cmake
@@ -1,17 +1,24 @@
-# - Find Qatzip
+# - Find qatzip
# Find the qatzip compression library and includes
#
-# QATZIP_INCLUDE_DIR - where to find qatzip.h, etc.
-# QATZIP_LIBRARIES - List of libraries when using qatzip.
-# QATZIP_FOUND - True if qatzip found.
+# qatzip_INCLUDE_DIR - where to find qatzip.h, etc.
+# qatzip_LIBRARIES - List of libraries when using qatzip.
+# qatzip_FOUND - True if qatzip found.
-find_path(QATZIP_INCLUDE_DIR NAMES qatzip.h)
-
-find_library(QATZIP_LIBRARIES NAMES qatzip)
+find_path(qatzip_INCLUDE_DIR NAMES qatzip.h)
+find_library(qatzip_LIBRARIES NAMES qatzip HINTS /usr/local/lib64/)
include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(qatzip DEFAULT_MSG QATZIP_LIBRARIES QATZIP_INCLUDE_DIR)
+find_package_handle_standard_args(qatzip DEFAULT_MSG qatzip_LIBRARIES qatzip_INCLUDE_DIR)
mark_as_advanced(
- QATZIP_LIBRARIES
- QATZIP_INCLUDE_DIR)
+ qatzip_LIBRARIES
+ qatzip_INCLUDE_DIR)
+
+if(qatzip_FOUND AND NOT TARGET qatzip::qatzip)
+ add_library(qatzip::qatzip SHARED IMPORTED)
+ set_target_properties(qatzip::qatzip PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${qatzip_INCLUDE_DIR}"
+ IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+ IMPORTED_LOCATION "${qatzip_LIBRARIES}")
+endif()
diff --git a/cmake/modules/Findutf8proc.cmake b/cmake/modules/Findutf8proc.cmake
new file mode 100644
index 0000000000000..2d390ba9915e1
--- /dev/null
+++ b/cmake/modules/Findutf8proc.cmake
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# note: cbodley copied this from the Arrow repo and removed ARROW variables
+
+function(extract_utf8proc_version)
+ if(utf8proc_INCLUDE_DIR)
+ file(READ "${utf8proc_INCLUDE_DIR}/utf8proc.h" UTF8PROC_H_CONTENT)
+
+ string(REGEX MATCH "#define UTF8PROC_VERSION_MAJOR [0-9]+"
+ UTF8PROC_MAJOR_VERSION_DEFINITION "${UTF8PROC_H_CONTENT}")
+ string(REGEX MATCH "#define UTF8PROC_VERSION_MINOR [0-9]+"
+ UTF8PROC_MINOR_VERSION_DEFINITION "${UTF8PROC_H_CONTENT}")
+ string(REGEX MATCH "#define UTF8PROC_VERSION_PATCH [0-9]+"
+ UTF8PROC_PATCH_VERSION_DEFINITION "${UTF8PROC_H_CONTENT}")
+
+ string(REGEX MATCH "[0-9]+$" UTF8PROC_MAJOR_VERSION
+ "${UTF8PROC_MAJOR_VERSION_DEFINITION}")
+ string(REGEX MATCH "[0-9]+$" UTF8PROC_MINOR_VERSION
+ "${UTF8PROC_MINOR_VERSION_DEFINITION}")
+ string(REGEX MATCH "[0-9]+$" UTF8PROC_PATCH_VERSION
+ "${UTF8PROC_PATCH_VERSION_DEFINITION}")
+ set(utf8proc_VERSION
+ "${UTF8PROC_MAJOR_VERSION}.${UTF8PROC_MINOR_VERSION}.${UTF8PROC_PATCH_VERSION}"
+ PARENT_SCOPE)
+ else()
+ set(utf8proc_VERSION
+ ""
+ PARENT_SCOPE)
+ endif()
+endfunction(extract_utf8proc_version)
+
+if(NOT utf8proc_USE_STATIC_LIB)
+ set(utf8proc_LIB_NAMES)
+ if(CMAKE_IMPORT_LIBRARY_SUFFIX)
+ list(APPEND utf8proc_LIB_NAMES
+ "${CMAKE_IMPORT_LIBRARY_PREFIX}utf8proc${CMAKE_IMPORT_LIBRARY_SUFFIX}")
+ endif()
+ list(APPEND utf8proc_LIB_NAMES
+ "${CMAKE_SHARED_LIBRARY_PREFIX}utf8proc${CMAKE_SHARED_LIBRARY_SUFFIX}")
+else()
+ if(MSVC AND NOT DEFINED utf8proc_MSVC_STATIC_LIB_SUFFIX)
+ set(utf8proc_MSVC_STATIC_LIB_SUFFIX "_static")
+ endif()
+ set(utf8proc_STATIC_LIB_SUFFIX
+ "${utf8proc_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(utf8proc_LIB_NAMES
+ "${CMAKE_STATIC_LIBRARY_PREFIX}utf8proc${utf8proc_STATIC_LIB_SUFFIX}")
+endif()
+
+if(utf8proc_ROOT)
+ find_library(utf8proc_LIB
+ NAMES ${utf8proc_LIB_NAMES}
+ PATHS ${utf8proc_ROOT}
+ PATH_SUFFIXES lib lib64
+ NO_DEFAULT_PATH)
+ find_path(utf8proc_INCLUDE_DIR
+ NAMES utf8proc.h
+ PATHS ${utf8proc_ROOT}
+ NO_DEFAULT_PATH
+ PATH_SUFFIXES include)
+ extract_utf8proc_version()
+else()
+ find_library(utf8proc_LIB
+ NAMES ${utf8proc_LIB_NAMES}
+ PATH_SUFFIXES lib lib64)
+ find_path(utf8proc_INCLUDE_DIR
+ NAMES utf8proc.h
+ PATH_SUFFIXES include)
+ extract_utf8proc_version()
+endif()
+
+find_package_handle_standard_args(
+ utf8proc
+ REQUIRED_VARS utf8proc_LIB utf8proc_INCLUDE_DIR
+ VERSION_VAR utf8proc_VERSION)
+
+if(utf8proc_FOUND)
+ set(utf8proc_FOUND TRUE)
+ add_library(utf8proc::utf8proc UNKNOWN IMPORTED)
+ set_target_properties(utf8proc::utf8proc
+ PROPERTIES IMPORTED_LOCATION "${utf8proc_LIB}"
+ INTERFACE_INCLUDE_DIRECTORIES
+ "${utf8proc_INCLUDE_DIR}")
+endif()
diff --git a/debian/control b/debian/control
index 1800b000eeebd..f64664e4319ae 100644
--- a/debian/control
+++ b/debian/control
@@ -18,13 +18,15 @@ Build-Depends: automake,
dh-python,
flex ,
git,
+ golang,
gperf,
g++ (>= 7),
hostname ,
javahelper,
jq ,
- jsonnet ,
junit4,
+ libarrow-dev ,
+ libparquet-dev ,
libaio-dev,
libbabeltrace-ctf-dev,
libbabeltrace-dev,
@@ -48,7 +50,6 @@ Build-Depends: automake,
librdmacm-dev,
libkeyutils-dev,
libldap2-dev,
- libleveldb-dev,
liblttng-ust-dev,
liblua5.3-dev,
liblz4-dev (>= 0.0~r131),
@@ -66,24 +67,26 @@ Build-Depends: automake,
libnl-genl-3-dev,
libxml2-dev,
librabbitmq-dev,
+ libre2-dev,
+ libutf8proc-dev (>= 2.2.0),
librdkafka-dev,
luarocks,
- libthrift-dev (>= 0.13.0) ,
+ libthrift-dev (>= 0.13.0),
libyaml-cpp-dev (>= 0.6)