From cd1e6a7bb2b4de375d6b534183b68d2ed2f7542f Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Sat, 29 Nov 2025 20:05:35 +0800 Subject: [PATCH 1/2] refactor: add a manifest dir to hold all manifest related files Improve code organization by collecting manifest-related files. writer helpers are renamed with _internal.h suffix to prevent installation, keeping the public API clean. --- src/iceberg/CMakeLists.txt | 21 ++++++++------- src/iceberg/inheritable_metadata.cc | 4 +-- src/iceberg/manifest/CMakeLists.txt | 18 +++++++++++++ .../{ => manifest}/manifest_adapter.cc | 7 +++-- .../manifest_adapter_internal.h} | 2 +- src/iceberg/{ => manifest}/manifest_entry.cc | 2 +- src/iceberg/{ => manifest}/manifest_entry.h | 2 ++ src/iceberg/{ => manifest}/manifest_list.cc | 2 +- src/iceberg/{ => manifest}/manifest_list.h | 2 +- src/iceberg/{ => manifest}/manifest_reader.cc | 8 +++--- src/iceberg/{ => manifest}/manifest_reader.h | 2 +- .../manifest_reader_internal.cc | 4 +-- .../{ => manifest}/manifest_reader_internal.h | 4 +-- src/iceberg/{ => manifest}/manifest_writer.cc | 18 ++++++++----- src/iceberg/{ => manifest}/manifest_writer.h | 8 +++--- src/iceberg/manifest/meson.build | 26 ++++++++++++++++++ src/iceberg/{ => manifest}/v1_metadata.cc | 7 +++-- .../v1_metadata_internal.h} | 4 +-- src/iceberg/{ => manifest}/v2_metadata.cc | 7 +++-- .../v2_metadata_internal.h} | 4 +-- src/iceberg/{ => manifest}/v3_metadata.cc | 7 +++-- .../v3_metadata_internal.h} | 4 +-- src/iceberg/meson.build | 27 +++++++------------ src/iceberg/partition_summary.cc | 2 +- src/iceberg/row/manifest_wrapper.cc | 2 +- src/iceberg/row/manifest_wrapper.h | 2 +- src/iceberg/table_scan.cc | 6 ++--- src/iceberg/table_scan.h | 2 +- src/iceberg/test/file_scan_task_test.cc | 2 +- .../test/manifest_list_reader_writer_test.cc | 6 ++--- .../test/manifest_list_versions_test.cc | 8 +++--- .../test/manifest_reader_writer_test.cc | 8 +++--- .../test/manifest_writer_versions_test.cc | 8 +++--- src/iceberg/test/struct_like_test.cc | 4 +-- src/iceberg/type_fwd.h | 2 ++ 35 files changed, 142 insertions(+), 100 deletions(-) create mode 100644 src/iceberg/manifest/CMakeLists.txt rename src/iceberg/{ => manifest}/manifest_adapter.cc (99%) rename src/iceberg/{manifest_adapter.h => manifest/manifest_adapter_internal.h} (98%) rename src/iceberg/{ => manifest}/manifest_entry.cc (98%) rename src/iceberg/{ => manifest}/manifest_entry.h (99%) rename src/iceberg/{ => manifest}/manifest_list.cc (97%) rename src/iceberg/{ => manifest}/manifest_list.h (99%) rename src/iceberg/{ => manifest}/manifest_reader.cc (95%) rename src/iceberg/{ => manifest}/manifest_reader.h (98%) rename src/iceberg/{ => manifest}/manifest_reader_internal.cc (99%) rename src/iceberg/{ => manifest}/manifest_reader_internal.h (96%) rename src/iceberg/{ => manifest}/manifest_writer.cc (96%) rename src/iceberg/{ => manifest}/manifest_writer.h (98%) create mode 100644 src/iceberg/manifest/meson.build rename src/iceberg/{ => manifest}/v1_metadata.cc (97%) rename src/iceberg/{v1_metadata.h => manifest/v1_metadata_internal.h} (95%) rename src/iceberg/{ => manifest}/v2_metadata.cc (98%) rename src/iceberg/{v2_metadata.h => manifest/v2_metadata_internal.h} (96%) rename src/iceberg/{ => manifest}/v3_metadata.cc (98%) rename src/iceberg/{v3_metadata.h => manifest/v3_metadata_internal.h} (96%) diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index fb0c6ad70..9b3e3b892 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -33,12 +33,15 @@ set(ICEBERG_SOURCES file_writer.cc inheritable_metadata.cc json_internal.cc - manifest_adapter.cc - manifest_entry.cc - manifest_list.cc - manifest_reader.cc - manifest_reader_internal.cc - manifest_writer.cc + manifest/manifest_adapter.cc + manifest/manifest_entry.cc + manifest/manifest_list.cc + manifest/manifest_reader.cc + manifest/manifest_reader_internal.cc + manifest/manifest_writer.cc + manifest/v1_metadata.cc + manifest/v2_metadata.cc + manifest/v3_metadata.cc metadata_columns.cc name_mapping.cc partition_field.cc @@ -74,10 +77,7 @@ set(ICEBERG_SOURCES util/temporal_util.cc util/timepoint.cc util/truncate_util.cc - util/uuid.cc - v1_metadata.cc - v2_metadata.cc - v3_metadata.cc) + util/uuid.cc) set(ICEBERG_STATIC_BUILD_INTERFACE_LIBS) set(ICEBERG_SHARED_BUILD_INTERFACE_LIBS) @@ -131,6 +131,7 @@ iceberg_install_all_headers(iceberg) add_subdirectory(catalog) add_subdirectory(expression) +add_subdirectory(manifest) add_subdirectory(row) add_subdirectory(util) diff --git a/src/iceberg/inheritable_metadata.cc b/src/iceberg/inheritable_metadata.cc index 58eb28345..081b90ac5 100644 --- a/src/iceberg/inheritable_metadata.cc +++ b/src/iceberg/inheritable_metadata.cc @@ -23,8 +23,8 @@ #include -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" #include "iceberg/snapshot.h" namespace iceberg { diff --git a/src/iceberg/manifest/CMakeLists.txt b/src/iceberg/manifest/CMakeLists.txt new file mode 100644 index 000000000..cb77f73eb --- /dev/null +++ b/src/iceberg/manifest/CMakeLists.txt @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +iceberg_install_all_headers(iceberg/manifest) diff --git a/src/iceberg/manifest_adapter.cc b/src/iceberg/manifest/manifest_adapter.cc similarity index 99% rename from src/iceberg/manifest_adapter.cc rename to src/iceberg/manifest/manifest_adapter.cc index 94bedced0..da75a4683 100644 --- a/src/iceberg/manifest_adapter.cc +++ b/src/iceberg/manifest/manifest_adapter.cc @@ -17,16 +17,15 @@ * under the License. */ -#include "iceberg/manifest_adapter.h" - #include #include #include #include "iceberg/arrow/nanoarrow_status_internal.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_adapter_internal.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" #include "iceberg/result.h" #include "iceberg/schema.h" #include "iceberg/util/checked_cast.h" diff --git a/src/iceberg/manifest_adapter.h b/src/iceberg/manifest/manifest_adapter_internal.h similarity index 98% rename from src/iceberg/manifest_adapter.h rename to src/iceberg/manifest/manifest_adapter_internal.h index be76c284e..3d8383a9e 100644 --- a/src/iceberg/manifest_adapter.h +++ b/src/iceberg/manifest/manifest_adapter_internal.h @@ -19,7 +19,7 @@ #pragma once -/// \file iceberg/manifest_adapter.h +/// \file iceberg/manifest/manifest_adapter_internal.h /// Base class for adapters handling v1/v2/v3/v4 manifest metadata. #include diff --git a/src/iceberg/manifest_entry.cc b/src/iceberg/manifest/manifest_entry.cc similarity index 98% rename from src/iceberg/manifest_entry.cc rename to src/iceberg/manifest/manifest_entry.cc index cf8ec0772..91fba88d4 100644 --- a/src/iceberg/manifest_entry.cc +++ b/src/iceberg/manifest/manifest_entry.cc @@ -17,7 +17,7 @@ * under the License. */ -#include "iceberg/manifest_entry.h" +#include "iceberg/manifest/manifest_entry.h" #include #include diff --git a/src/iceberg/manifest_entry.h b/src/iceberg/manifest/manifest_entry.h similarity index 99% rename from src/iceberg/manifest_entry.h rename to src/iceberg/manifest/manifest_entry.h index 9702a4871..8e3183a43 100644 --- a/src/iceberg/manifest_entry.h +++ b/src/iceberg/manifest/manifest_entry.h @@ -19,6 +19,8 @@ #pragma once +/// \file iceberg/manifest/manifest_entry.h + #include #include #include diff --git a/src/iceberg/manifest_list.cc b/src/iceberg/manifest/manifest_list.cc similarity index 97% rename from src/iceberg/manifest_list.cc rename to src/iceberg/manifest/manifest_list.cc index 853a0182b..4351081ca 100644 --- a/src/iceberg/manifest_list.cc +++ b/src/iceberg/manifest/manifest_list.cc @@ -17,7 +17,7 @@ * under the License. */ -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_list.h" #include "iceberg/schema.h" diff --git a/src/iceberg/manifest_list.h b/src/iceberg/manifest/manifest_list.h similarity index 99% rename from src/iceberg/manifest_list.h rename to src/iceberg/manifest/manifest_list.h index fe1b246c2..85c0b89e4 100644 --- a/src/iceberg/manifest_list.h +++ b/src/iceberg/manifest/manifest_list.h @@ -19,7 +19,7 @@ #pragma once -/// \file iceberg/manifest_list.h +/// \file iceberg/manifest/manifest_list.h #include #include diff --git a/src/iceberg/manifest_reader.cc b/src/iceberg/manifest/manifest_reader.cc similarity index 95% rename from src/iceberg/manifest_reader.cc rename to src/iceberg/manifest/manifest_reader.cc index 94532c674..1750151ef 100644 --- a/src/iceberg/manifest_reader.cc +++ b/src/iceberg/manifest/manifest_reader.cc @@ -17,11 +17,11 @@ * under the License. */ -#include "iceberg/manifest_reader.h" +#include "iceberg/manifest/manifest_reader.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" -#include "iceberg/manifest_reader_internal.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader_internal.h" #include "iceberg/schema.h" #include "iceberg/schema_internal.h" #include "iceberg/type.h" diff --git a/src/iceberg/manifest_reader.h b/src/iceberg/manifest/manifest_reader.h similarity index 98% rename from src/iceberg/manifest_reader.h rename to src/iceberg/manifest/manifest_reader.h index e16282889..e7d6044dc 100644 --- a/src/iceberg/manifest_reader.h +++ b/src/iceberg/manifest/manifest_reader.h @@ -19,7 +19,7 @@ #pragma once -/// \file iceberg/manifest_reader.h +/// \file iceberg/manifest/manifest_reader.h /// Data reader interface for manifest files. #include diff --git a/src/iceberg/manifest_reader_internal.cc b/src/iceberg/manifest/manifest_reader_internal.cc similarity index 99% rename from src/iceberg/manifest_reader_internal.cc rename to src/iceberg/manifest/manifest_reader_internal.cc index 9898e6c97..f7e8d428a 100644 --- a/src/iceberg/manifest_reader_internal.cc +++ b/src/iceberg/manifest/manifest_reader_internal.cc @@ -24,8 +24,8 @@ #include "iceberg/arrow/nanoarrow_status_internal.h" #include "iceberg/arrow_c_data_guard_internal.h" #include "iceberg/file_format.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" #include "iceberg/schema.h" #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" diff --git a/src/iceberg/manifest_reader_internal.h b/src/iceberg/manifest/manifest_reader_internal.h similarity index 96% rename from src/iceberg/manifest_reader_internal.h rename to src/iceberg/manifest/manifest_reader_internal.h index b65f4cd81..513f40507 100644 --- a/src/iceberg/manifest_reader_internal.h +++ b/src/iceberg/manifest/manifest_reader_internal.h @@ -19,12 +19,12 @@ #pragma once -/// \file iceberg/internal/manifest_reader_internal.h +/// \file iceberg/manifest/manifest_reader_internal.h /// Reader implementation for manifest list files and manifest files. #include "iceberg/file_reader.h" #include "iceberg/inheritable_metadata.h" -#include "iceberg/manifest_reader.h" +#include "iceberg/manifest/manifest_reader.h" namespace iceberg { diff --git a/src/iceberg/manifest_writer.cc b/src/iceberg/manifest/manifest_writer.cc similarity index 96% rename from src/iceberg/manifest_writer.cc rename to src/iceberg/manifest/manifest_writer.cc index 3209a0c43..c0c95b25e 100644 --- a/src/iceberg/manifest_writer.cc +++ b/src/iceberg/manifest/manifest_writer.cc @@ -17,20 +17,20 @@ * under the License. */ -#include "iceberg/manifest_writer.h" +#include "iceberg/manifest/manifest_writer.h" #include -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/v1_metadata_internal.h" +#include "iceberg/manifest/v2_metadata_internal.h" +#include "iceberg/manifest/v3_metadata_internal.h" #include "iceberg/partition_summary_internal.h" #include "iceberg/result.h" #include "iceberg/schema.h" #include "iceberg/table_metadata.h" #include "iceberg/util/macros.h" -#include "iceberg/v1_metadata.h" -#include "iceberg/v2_metadata.h" -#include "iceberg/v3_metadata.h" namespace iceberg { @@ -360,6 +360,12 @@ Result> ManifestWriter::MakeV3Writer( manifest_location, first_row_id); } +ManifestListWriter::ManifestListWriter(std::unique_ptr writer, + std::unique_ptr adapter) + : writer_(std::move(writer)), adapter_(std::move(adapter)) {} + +ManifestListWriter::~ManifestListWriter() = default; + Status ManifestListWriter::Add(const ManifestFile& file) { if (adapter_->size() >= kBatchSize) { ICEBERG_ASSIGN_OR_RAISE(auto array, adapter_->FinishAppending()); diff --git a/src/iceberg/manifest_writer.h b/src/iceberg/manifest/manifest_writer.h similarity index 98% rename from src/iceberg/manifest_writer.h rename to src/iceberg/manifest/manifest_writer.h index 8889ae28f..aaa395afc 100644 --- a/src/iceberg/manifest_writer.h +++ b/src/iceberg/manifest/manifest_writer.h @@ -19,7 +19,7 @@ #pragma once -/// \file iceberg/manifest_writer.h +/// \file iceberg/manifest/manifest_writer.h /// Data writer interface for manifest files and manifest list files. #include @@ -28,7 +28,6 @@ #include "iceberg/file_writer.h" #include "iceberg/iceberg_export.h" -#include "iceberg/manifest_adapter.h" #include "iceberg/metrics.h" #include "iceberg/result.h" #include "iceberg/type_fwd.h" @@ -183,10 +182,9 @@ class ICEBERG_EXPORT ManifestWriter { class ICEBERG_EXPORT ManifestListWriter { public: ManifestListWriter(std::unique_ptr writer, - std::unique_ptr adapter) - : writer_(std::move(writer)), adapter_(std::move(adapter)) {} + std::unique_ptr adapter); - ~ManifestListWriter() = default; + ~ManifestListWriter(); /// \brief Write manifest file to manifest list file. /// \param file Manifest file to write. diff --git a/src/iceberg/manifest/meson.build b/src/iceberg/manifest/meson.build new file mode 100644 index 000000000..f49c5a5f5 --- /dev/null +++ b/src/iceberg/manifest/meson.build @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +install_headers( + [ + 'manifest_entry.h', + 'manifest_list.h', + 'manifest_reader.h', + 'manifest_writer.h', + ], + subdir: 'iceberg/manifest', +) diff --git a/src/iceberg/v1_metadata.cc b/src/iceberg/manifest/v1_metadata.cc similarity index 97% rename from src/iceberg/v1_metadata.cc rename to src/iceberg/manifest/v1_metadata.cc index 101ef479c..f4960c207 100644 --- a/src/iceberg/v1_metadata.cc +++ b/src/iceberg/manifest/v1_metadata.cc @@ -17,13 +17,12 @@ * under the License. */ -#include "iceberg/v1_metadata.h" - #include #include "iceberg/json_internal.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/v1_metadata_internal.h" #include "iceberg/schema.h" #include "iceberg/schema_internal.h" #include "iceberg/type.h" diff --git a/src/iceberg/v1_metadata.h b/src/iceberg/manifest/v1_metadata_internal.h similarity index 95% rename from src/iceberg/v1_metadata.h rename to src/iceberg/manifest/v1_metadata_internal.h index a6d7ab47b..dc88f6455 100644 --- a/src/iceberg/v1_metadata.h +++ b/src/iceberg/manifest/v1_metadata_internal.h @@ -19,9 +19,9 @@ #pragma once -#include "iceberg/manifest_adapter.h" +/// \file iceberg/manifest/v1_metadata_internal.h -/// \file iceberg/v1_metadata.h +#include "iceberg/manifest/manifest_adapter_internal.h" namespace iceberg { diff --git a/src/iceberg/v2_metadata.cc b/src/iceberg/manifest/v2_metadata.cc similarity index 98% rename from src/iceberg/v2_metadata.cc rename to src/iceberg/manifest/v2_metadata.cc index 486acb62d..737fa62fe 100644 --- a/src/iceberg/v2_metadata.cc +++ b/src/iceberg/manifest/v2_metadata.cc @@ -17,11 +17,10 @@ * under the License. */ -#include "iceberg/v2_metadata.h" - #include "iceberg/json_internal.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/v2_metadata_internal.h" #include "iceberg/schema.h" #include "iceberg/schema_internal.h" #include "iceberg/util/macros.h" diff --git a/src/iceberg/v2_metadata.h b/src/iceberg/manifest/v2_metadata_internal.h similarity index 96% rename from src/iceberg/v2_metadata.h rename to src/iceberg/manifest/v2_metadata_internal.h index 3097085ef..e23dca6c5 100644 --- a/src/iceberg/v2_metadata.h +++ b/src/iceberg/manifest/v2_metadata_internal.h @@ -19,9 +19,9 @@ #pragma once -/// \file iceberg/v2_metadata.h +/// \file iceberg/manifest/v2_metadata_internal.h -#include "iceberg/manifest_adapter.h" +#include "iceberg/manifest/manifest_adapter_internal.h" namespace iceberg { diff --git a/src/iceberg/v3_metadata.cc b/src/iceberg/manifest/v3_metadata.cc similarity index 98% rename from src/iceberg/v3_metadata.cc rename to src/iceberg/manifest/v3_metadata.cc index ae309edcd..87d4d77d9 100644 --- a/src/iceberg/v3_metadata.cc +++ b/src/iceberg/manifest/v3_metadata.cc @@ -17,14 +17,13 @@ * under the License. */ -#include "iceberg/v3_metadata.h" - #include #include #include "iceberg/json_internal.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/v3_metadata_internal.h" #include "iceberg/result.h" #include "iceberg/schema.h" #include "iceberg/schema_internal.h" diff --git a/src/iceberg/v3_metadata.h b/src/iceberg/manifest/v3_metadata_internal.h similarity index 96% rename from src/iceberg/v3_metadata.h rename to src/iceberg/manifest/v3_metadata_internal.h index 163f6edff..88f2a4f32 100644 --- a/src/iceberg/v3_metadata.h +++ b/src/iceberg/manifest/v3_metadata_internal.h @@ -19,9 +19,9 @@ #pragma once -/// \file iceberg/v3_metadata.h +/// \file iceberg/manifest/v3_metadata_internal.h -#include "iceberg/manifest_adapter.h" +#include "iceberg/manifest/manifest_adapter_internal.h" #include "iceberg/result.h" namespace iceberg { diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index bdb5dbf3e..16324c310 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -55,12 +55,15 @@ iceberg_sources = files( 'file_writer.cc', 'inheritable_metadata.cc', 'json_internal.cc', - 'manifest_adapter.cc', - 'manifest_entry.cc', - 'manifest_list.cc', - 'manifest_reader.cc', - 'manifest_reader_internal.cc', - 'manifest_writer.cc', + 'manifest/manifest_adapter.cc', + 'manifest/manifest_entry.cc', + 'manifest/manifest_list.cc', + 'manifest/manifest_reader.cc', + 'manifest/manifest_reader_internal.cc', + 'manifest/manifest_writer.cc', + 'manifest/v1_metadata.cc', + 'manifest/v2_metadata.cc', + 'manifest/v3_metadata.cc', 'metadata_columns.cc', 'name_mapping.cc', 'partition_field.cc', @@ -97,9 +100,6 @@ iceberg_sources = files( 'util/timepoint.cc', 'util/truncate_util.cc', 'util/uuid.cc', - 'v1_metadata.cc', - 'v2_metadata.cc', - 'v3_metadata.cc', ) # CRoaring does not export symbols, so on Windows it must @@ -161,11 +161,6 @@ install_headers( 'iceberg_export.h', 'inheritable_metadata.h', 'location_provider.h', - 'manifest_adapter.h', - 'manifest_entry.h', - 'manifest_list.h', - 'manifest_reader.h', - 'manifest_writer.h', 'metadata_columns.h', 'metrics.h', 'name_mapping.h', @@ -191,15 +186,13 @@ install_headers( 'transform.h', 'type_fwd.h', 'type.h', - 'v1_metadata.h', - 'v2_metadata.h', - 'v3_metadata.h', ], subdir: 'iceberg', ) subdir('catalog') subdir('expression') +subdir('manifest') subdir('row') subdir('util') diff --git a/src/iceberg/partition_summary.cc b/src/iceberg/partition_summary.cc index d0ed4a951..2937ae2d0 100644 --- a/src/iceberg/partition_summary.cc +++ b/src/iceberg/partition_summary.cc @@ -20,7 +20,7 @@ #include #include "iceberg/expression/literal.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_list.h" #include "iceberg/partition_summary_internal.h" #include "iceberg/result.h" #include "iceberg/row/partition_values.h" diff --git a/src/iceberg/row/manifest_wrapper.cc b/src/iceberg/row/manifest_wrapper.cc index 4c8708e43..851f9e72f 100644 --- a/src/iceberg/row/manifest_wrapper.cc +++ b/src/iceberg/row/manifest_wrapper.cc @@ -19,7 +19,7 @@ #include "iceberg/row/manifest_wrapper.h" -#include "iceberg/manifest_reader_internal.h" +#include "iceberg/manifest/manifest_reader_internal.h" #include "iceberg/util/macros.h" namespace iceberg { diff --git a/src/iceberg/row/manifest_wrapper.h b/src/iceberg/row/manifest_wrapper.h index 3271eac48..bc04c1e8b 100644 --- a/src/iceberg/row/manifest_wrapper.h +++ b/src/iceberg/row/manifest_wrapper.h @@ -26,7 +26,7 @@ #include #include "iceberg/iceberg_export.h" -#include "iceberg/manifest_list.h" +#include "iceberg/manifest/manifest_list.h" #include "iceberg/row/struct_like.h" namespace iceberg { diff --git a/src/iceberg/table_scan.cc b/src/iceberg/table_scan.cc index bc52a1fad..56c6ee2c7 100644 --- a/src/iceberg/table_scan.cc +++ b/src/iceberg/table_scan.cc @@ -24,9 +24,9 @@ #include "iceberg/arrow_c_data.h" #include "iceberg/file_reader.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" -#include "iceberg/manifest_reader.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" #include "iceberg/snapshot.h" diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index 9e7f313cc..c3b1b28f4 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -23,7 +23,7 @@ #include #include "iceberg/arrow_c_data.h" -#include "iceberg/manifest_entry.h" +#include "iceberg/manifest/manifest_entry.h" #include "iceberg/type_fwd.h" namespace iceberg { diff --git a/src/iceberg/test/file_scan_task_test.cc b/src/iceberg/test/file_scan_task_test.cc index 51af7c0cb..b72528507 100644 --- a/src/iceberg/test/file_scan_task_test.cc +++ b/src/iceberg/test/file_scan_task_test.cc @@ -29,7 +29,7 @@ #include "iceberg/arrow/arrow_fs_file_io_internal.h" #include "iceberg/file_format.h" -#include "iceberg/manifest_entry.h" +#include "iceberg/manifest/manifest_entry.h" #include "iceberg/parquet/parquet_register.h" #include "iceberg/schema.h" #include "iceberg/table_scan.h" diff --git a/src/iceberg/test/manifest_list_reader_writer_test.cc b/src/iceberg/test/manifest_list_reader_writer_test.cc index a6985f766..ee6c7d9fe 100644 --- a/src/iceberg/test/manifest_list_reader_writer_test.cc +++ b/src/iceberg/test/manifest_list_reader_writer_test.cc @@ -23,9 +23,9 @@ #include "iceberg/arrow/arrow_fs_file_io_internal.h" #include "iceberg/avro/avro_register.h" #include "iceberg/expression/literal.h" -#include "iceberg/manifest_list.h" -#include "iceberg/manifest_reader.h" -#include "iceberg/manifest_writer.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/manifest/manifest_writer.h" #include "iceberg/test/matchers.h" #include "iceberg/test/temp_file_test_base.h" #include "iceberg/test/test_common.h" diff --git a/src/iceberg/test/manifest_list_versions_test.cc b/src/iceberg/test/manifest_list_versions_test.cc index bf83e8c56..9c32a59d5 100644 --- a/src/iceberg/test/manifest_list_versions_test.cc +++ b/src/iceberg/test/manifest_list_versions_test.cc @@ -29,13 +29,13 @@ #include "iceberg/avro/avro_register.h" #include "iceberg/file_reader.h" #include "iceberg/file_writer.h" -#include "iceberg/manifest_list.h" -#include "iceberg/manifest_reader.h" -#include "iceberg/manifest_writer.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/manifest/manifest_writer.h" +#include "iceberg/manifest/v1_metadata_internal.h" #include "iceberg/schema.h" #include "iceberg/schema_internal.h" #include "iceberg/test/matchers.h" -#include "iceberg/v1_metadata.h" namespace iceberg { diff --git a/src/iceberg/test/manifest_reader_writer_test.cc b/src/iceberg/test/manifest_reader_writer_test.cc index 820576966..b3daaf94e 100644 --- a/src/iceberg/test/manifest_reader_writer_test.cc +++ b/src/iceberg/test/manifest_reader_writer_test.cc @@ -24,10 +24,10 @@ #include "iceberg/arrow/arrow_fs_file_io_internal.h" #include "iceberg/avro/avro_register.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" -#include "iceberg/manifest_reader.h" -#include "iceberg/manifest_writer.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/manifest/manifest_writer.h" #include "iceberg/partition_spec.h" #include "iceberg/schema.h" #include "iceberg/test/matchers.h" diff --git a/src/iceberg/test/manifest_writer_versions_test.cc b/src/iceberg/test/manifest_writer_versions_test.cc index 6e3dfde1e..0bbdbcc52 100644 --- a/src/iceberg/test/manifest_writer_versions_test.cc +++ b/src/iceberg/test/manifest_writer_versions_test.cc @@ -28,10 +28,10 @@ #include "iceberg/arrow/arrow_file_io.h" #include "iceberg/avro/avro_register.h" #include "iceberg/file_format.h" -#include "iceberg/manifest_entry.h" -#include "iceberg/manifest_list.h" -#include "iceberg/manifest_reader.h" -#include "iceberg/manifest_writer.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/manifest/manifest_writer.h" #include "iceberg/metrics.h" #include "iceberg/partition_spec.h" #include "iceberg/row/partition_values.h" diff --git a/src/iceberg/test/struct_like_test.cc b/src/iceberg/test/struct_like_test.cc index d64e219d4..97df61b8d 100644 --- a/src/iceberg/test/struct_like_test.cc +++ b/src/iceberg/test/struct_like_test.cc @@ -24,8 +24,8 @@ #include #include "iceberg/arrow_c_data_guard_internal.h" -#include "iceberg/manifest_list.h" -#include "iceberg/manifest_reader_internal.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader_internal.h" #include "iceberg/row/arrow_array_wrapper.h" #include "iceberg/row/manifest_wrapper.h" #include "iceberg/schema.h" diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 68a4543fb..9ca9ccebd 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -141,6 +141,8 @@ class ManifestListReader; class ManifestListWriter; class ManifestReader; class ManifestWriter; +class ManifestEntryAdapter; +class ManifestFileAdapter; struct ReaderOptions; struct WriterOptions; From 87b2a1df86acbf8723598782142e5a9413112b5b Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Sun, 30 Nov 2025 16:21:59 +0800 Subject: [PATCH 2/2] fix: review comments & make the ctors in manifest_writer.h private --- src/iceberg/manifest/manifest_writer.cc | 21 ++++++++++++--------- src/iceberg/manifest/manifest_writer.h | 22 +++++++++++++--------- src/iceberg/type_fwd.h | 2 -- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/iceberg/manifest/manifest_writer.cc b/src/iceberg/manifest/manifest_writer.cc index c0c95b25e..8cd940d56 100644 --- a/src/iceberg/manifest/manifest_writer.cc +++ b/src/iceberg/manifest/manifest_writer.cc @@ -294,8 +294,8 @@ Result> ManifestWriter::MakeV1Writer( auto writer, OpenFileWriter(manifest_location, std::move(schema), std::move(file_io), adapter->metadata(), "manifest_entry")); - return std::make_unique(std::move(writer), std::move(adapter), - manifest_location, std::nullopt); + return std::unique_ptr(new ManifestWriter( + std::move(writer), std::move(adapter), manifest_location, std::nullopt)); } Result> ManifestWriter::MakeV2Writer( @@ -324,8 +324,8 @@ Result> ManifestWriter::MakeV2Writer( auto writer, OpenFileWriter(manifest_location, std::move(schema), std::move(file_io), adapter->metadata(), "manifest_entry")); - return std::make_unique(std::move(writer), std::move(adapter), - manifest_location, std::nullopt); + return std::unique_ptr(new ManifestWriter( + std::move(writer), std::move(adapter), manifest_location, std::nullopt)); } Result> ManifestWriter::MakeV3Writer( @@ -356,8 +356,8 @@ Result> ManifestWriter::MakeV3Writer( auto writer, OpenFileWriter(manifest_location, std::move(schema), std::move(file_io), adapter->metadata(), "manifest_entry")); - return std::make_unique(std::move(writer), std::move(adapter), - manifest_location, first_row_id); + return std::unique_ptr(new ManifestWriter( + std::move(writer), std::move(adapter), manifest_location, first_row_id)); } ManifestListWriter::ManifestListWriter(std::unique_ptr writer, @@ -406,7 +406,8 @@ Result> ManifestListWriter::MakeV1Writer( auto writer, OpenFileWriter(manifest_list_location, std::move(schema), std::move(file_io), adapter->metadata(), "manifest_file")); - return std::make_unique(std::move(writer), std::move(adapter)); + return std::unique_ptr( + new ManifestListWriter(std::move(writer), std::move(adapter))); } Result> ManifestListWriter::MakeV2Writer( @@ -424,7 +425,8 @@ Result> ManifestListWriter::MakeV2Writer( OpenFileWriter(manifest_list_location, std::move(schema), std::move(file_io), adapter->metadata(), "manifest_file")); - return std::make_unique(std::move(writer), std::move(adapter)); + return std::unique_ptr( + new ManifestListWriter(std::move(writer), std::move(adapter))); } Result> ManifestListWriter::MakeV3Writer( @@ -441,7 +443,8 @@ Result> ManifestListWriter::MakeV3Writer( auto writer, OpenFileWriter(manifest_list_location, std::move(schema), std::move(file_io), adapter->metadata(), "manifest_file")); - return std::make_unique(std::move(writer), std::move(adapter)); + return std::unique_ptr( + new ManifestListWriter(std::move(writer), std::move(adapter))); } } // namespace iceberg diff --git a/src/iceberg/manifest/manifest_writer.h b/src/iceberg/manifest/manifest_writer.h index aaa395afc..a708e6e3b 100644 --- a/src/iceberg/manifest/manifest_writer.h +++ b/src/iceberg/manifest/manifest_writer.h @@ -37,10 +37,6 @@ namespace iceberg { /// \brief Write manifest entries to a manifest file. class ICEBERG_EXPORT ManifestWriter { public: - ManifestWriter(std::unique_ptr writer, - std::unique_ptr adapter, - std::string_view manifest_location, std::optional first_row_id); - ~ManifestWriter(); /// \brief Write the entry that all its fields are populated correctly. @@ -159,11 +155,17 @@ class ICEBERG_EXPORT ManifestWriter { std::shared_ptr current_schema, ManifestContent content); private: + // Private constructor for internal use only, use the static Make*Writer methods + // instead. + ManifestWriter(std::unique_ptr writer, + std::unique_ptr adapter, + std::string_view manifest_location, std::optional first_row_id); + Status CheckDataFile(const DataFile& file) const; static constexpr int64_t kBatchSize = 1024; std::unique_ptr writer_; - std::unique_ptr adapter_; + std::unique_ptr adapter_; bool closed_{false}; std::string manifest_location_; std::optional first_row_id_; @@ -181,9 +183,6 @@ class ICEBERG_EXPORT ManifestWriter { /// \brief Write manifest files to a manifest list file. class ICEBERG_EXPORT ManifestListWriter { public: - ManifestListWriter(std::unique_ptr writer, - std::unique_ptr adapter); - ~ManifestListWriter(); /// \brief Write manifest file to manifest list file. @@ -238,9 +237,14 @@ class ICEBERG_EXPORT ManifestListWriter { std::string_view manifest_list_location, std::shared_ptr file_io); private: + // Private constructor for internal use only, use the static Make*Writer methods + // instead. + ManifestListWriter(std::unique_ptr writer, + std::unique_ptr adapter); + static constexpr int64_t kBatchSize = 1024; std::unique_ptr writer_; - std::unique_ptr adapter_; + std::unique_ptr adapter_; }; } // namespace iceberg diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 9ca9ccebd..68a4543fb 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -141,8 +141,6 @@ class ManifestListReader; class ManifestListWriter; class ManifestReader; class ManifestWriter; -class ManifestEntryAdapter; -class ManifestFileAdapter; struct ReaderOptions; struct WriterOptions;