From 21940c209e8b452341106aa290e51ed739cd2e0c Mon Sep 17 00:00:00 2001 From: "shuxu.li" Date: Tue, 28 Oct 2025 20:32:19 +0800 Subject: [PATCH 1/7] feat: add table create/replace/update interface to catalog --- src/iceberg/catalog.h | 34 +++++++++++++++++++ .../catalog/memory/in_memory_catalog.cc | 13 +++++++ .../catalog/memory/in_memory_catalog.h | 8 +++++ src/iceberg/table.cc | 4 +++ src/iceberg/table.h | 5 +++ src/iceberg/test/mock_catalog.h | 9 +++++ src/iceberg/transaction.h | 24 +++++++++---- 7 files changed, 91 insertions(+), 6 deletions(-) diff --git a/src/iceberg/catalog.h b/src/iceberg/catalog.h index 83ea677ab..8985ac3fc 100644 --- a/src/iceberg/catalog.h +++ b/src/iceberg/catalog.h @@ -140,6 +140,21 @@ class ICEBERG_EXPORT Catalog { const std::string& location, const std::unordered_map& properties) = 0; + /// \brief Start a transaction to replace a table + /// + /// \param identifier a table identifier + /// \param schema a schema + /// \param spec a partition spec + /// \param location a location for the table; leave empty if unspecified + /// \param properties a string map of table properties + /// \param orCreate whether to create the table if not exists + /// \return a Transaction to replace the table or ErrorKind::kNotFound if the table + /// doesn't exist and orCreate is false + virtual Result> StageReplaceTable( + const TableIdentifier& identifier, const Schema& schema, const PartitionSpec& spec, + const std::string& location, + const std::unordered_map& properties, bool orCreate) = 0; + /// \brief Check whether table exists /// /// \param identifier a table identifier @@ -161,6 +176,15 @@ class ICEBERG_EXPORT Catalog { /// - On failure, contains error information. virtual Status DropTable(const TableIdentifier& identifier, bool purge) = 0; + /// \brief Rename a table + /// + /// \param from the current table identifier + /// \param to the new table identifier + /// \return Status indicating the outcome of the operation. + /// - On success, the table was renamed. + /// - On failure, contains error information. + virtual Status RenameTable(const TableIdentifier& from, const TableIdentifier& to) = 0; + /// \brief Load a table /// /// \param identifier a table identifier @@ -223,6 +247,16 @@ class ICEBERG_EXPORT Catalog { /// /// \return the Transaction to create the table virtual std::unique_ptr StageCreate() = 0; + + /// \brief Starts a transaction to replace the table + /// + /// \return the Transaction to replace the table + virtual std::unique_ptr StageReplace() = 0; + + /// \brief Starts a transaction to create or replace the table + /// + /// \breturn the Transaction to create or replace the table + virtual std::unique_ptr StageCreateOrReplace() = 0; }; /// \brief Instantiate a builder to either create a table or start a create/replace diff --git a/src/iceberg/catalog/memory/in_memory_catalog.cc b/src/iceberg/catalog/memory/in_memory_catalog.cc index c024aac20..cceed1206 100644 --- a/src/iceberg/catalog/memory/in_memory_catalog.cc +++ b/src/iceberg/catalog/memory/in_memory_catalog.cc @@ -404,6 +404,13 @@ Result> InMemoryCatalog::StageCreateTable( return NotImplemented("stage create table"); } +Result> InMemoryCatalog::StageReplaceTable( + const TableIdentifier& identifier, const Schema& schema, const PartitionSpec& spec, + const std::string& location, + const std::unordered_map& properties, bool orCreate) { + return NotImplemented("stage replace table"); +} + Result InMemoryCatalog::TableExists(const TableIdentifier& identifier) const { std::unique_lock lock(mutex_); return root_namespace_->TableExists(identifier); @@ -415,6 +422,12 @@ Status InMemoryCatalog::DropTable(const TableIdentifier& identifier, bool purge) return root_namespace_->UnregisterTable(identifier); } +Status InMemoryCatalog::RenameTable(const TableIdentifier& from, + const TableIdentifier& to) { + std::unique_lock lock(mutex_); + return NotImplemented("rename table"); +} + Result> InMemoryCatalog::LoadTable( const TableIdentifier& identifier) { if (!file_io_) [[unlikely]] { diff --git a/src/iceberg/catalog/memory/in_memory_catalog.h b/src/iceberg/catalog/memory/in_memory_catalog.h index 59c6d3ad8..0d3910ab1 100644 --- a/src/iceberg/catalog/memory/in_memory_catalog.h +++ b/src/iceberg/catalog/memory/in_memory_catalog.h @@ -85,10 +85,18 @@ class ICEBERG_EXPORT InMemoryCatalog const std::string& location, const std::unordered_map& properties) override; + Result> StageReplaceTable( + const TableIdentifier& identifier, const Schema& schema, const PartitionSpec& spec, + const std::string& location, + const std::unordered_map& properties, + bool orCreate) override; + Result TableExists(const TableIdentifier& identifier) const override; Status DropTable(const TableIdentifier& identifier, bool purge) override; + Status RenameTable(const TableIdentifier& from, const TableIdentifier& to) override; + Result> LoadTable(const TableIdentifier& identifier) override; Result> RegisterTable( diff --git a/src/iceberg/table.cc b/src/iceberg/table.cc index cdcb6a95b..e584cbe03 100644 --- a/src/iceberg/table.cc +++ b/src/iceberg/table.cc @@ -133,6 +133,10 @@ const std::vector& Table::history() const { return metadata_->snapshot_log; } +std::shared_ptr Table::NewTransaction() { + throw NotImplemented("Table::NewTransaction is not implemented"); +} + const std::shared_ptr& Table::io() const { return io_; } std::unique_ptr Table::NewScan() const { diff --git a/src/iceberg/table.h b/src/iceberg/table.h index f249f3d69..f868279d4 100644 --- a/src/iceberg/table.h +++ b/src/iceberg/table.h @@ -109,6 +109,11 @@ class ICEBERG_EXPORT Table { /// filter data. virtual std::unique_ptr NewScan() const; + /// \brief Create a new transaction for this table + /// + /// \return a shared pointer to the new Transaction + virtual std::shared_ptr NewTransaction(); + /// \brief Returns a FileIO to read and write table data and metadata files const std::shared_ptr& io() const; diff --git a/src/iceberg/test/mock_catalog.h b/src/iceberg/test/mock_catalog.h index f54982bb2..a6f397b9c 100644 --- a/src/iceberg/test/mock_catalog.h +++ b/src/iceberg/test/mock_catalog.h @@ -71,10 +71,19 @@ class MockCatalog : public Catalog { const std::string&, (const std::unordered_map&)), (override)); + MOCK_METHOD((Result>), StageReplaceTable, + (const TableIdentifier&, const Schema&, const PartitionSpec&, + const std::string&, (const std::unordered_map&), + bool), + (override)); + MOCK_METHOD(Result, TableExists, (const TableIdentifier&), (const, override)); MOCK_METHOD(Status, DropTable, (const TableIdentifier&, bool), (override)); + MOCK_METHOD(Status, RenameTable, (const TableIdentifier&, const TableIdentifier&), + (override)); + MOCK_METHOD((Result>), LoadTable, (const TableIdentifier&), (override)); diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 0149f329d..73b8dc30a 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -21,6 +21,7 @@ #pragma once #include +#include #include "iceberg/iceberg_export.h" #include "iceberg/type_fwd.h" @@ -37,17 +38,28 @@ class ICEBERG_EXPORT Transaction { /// \return this transaction's table virtual const std::shared_ptr& table() const = 0; - /// \brief Create a new append API to add files to this table + /// \brief Create a new append API to add data files to this table /// - /// \return a new AppendFiles + /// \return a new AppendFiles instance virtual std::shared_ptr NewAppend() = 0; + /// \brief Apply multiple metadata updates to this transaction + /// + /// \param requirements the table requirements to validate + /// \param updates the table updates to apply + /// \return Status::OK if all updates were queued successfully + virtual Status UpdateTable( + const std::vector>& requirements, + std::vector> updates) = 0; + /// \brief Apply the pending changes from all actions and commit /// - /// May throw ValidationException if any update cannot be applied to the current table - /// metadata. May throw CommitFailedException if the updates cannot be committed due to - /// conflicts. - virtual void CommitTransaction() = 0; + /// This method applies all pending data operations and metadata updates in the + /// transaction and commits them to the table in a single atomic operation. + /// + /// \return Status::OK if the transaction was committed successfully, or an error + /// status if validation failed or the commit encountered conflicts + virtual Status CommitTransaction() = 0; }; } // namespace iceberg From 0b0239d63b3ac6ece257562e17693d177efadeb3 Mon Sep 17 00:00:00 2001 From: "shuxu.li" Date: Tue, 28 Oct 2025 20:37:07 +0800 Subject: [PATCH 2/7] feat: add table create/replace/update interface to catalog --- src/iceberg/transaction.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 73b8dc30a..05a15dae9 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -38,9 +38,9 @@ class ICEBERG_EXPORT Transaction { /// \return this transaction's table virtual const std::shared_ptr
& table() const = 0; - /// \brief Create a new append API to add data files to this table + /// \brief Create a new append API to add files to this table /// - /// \return a new AppendFiles instance + /// \return a new AppendFiles virtual std::shared_ptr NewAppend() = 0; /// \brief Apply multiple metadata updates to this transaction From befeff2876286b02af7f819c992becf975d2b44a Mon Sep 17 00:00:00 2001 From: "shuxu.li" Date: Wed, 29 Oct 2025 09:34:02 +0800 Subject: [PATCH 3/7] feat: add table create/replace/update interface to catalog --- src/iceberg/transaction.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 05a15dae9..f54edcb58 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -24,6 +24,7 @@ #include #include "iceberg/iceberg_export.h" +#include "iceberg/result.h" #include "iceberg/type_fwd.h" namespace iceberg { From 564ca06f9312dc85e7b1f6e08808c5c45e5ae83e Mon Sep 17 00:00:00 2001 From: "shuxu.li" Date: Sat, 8 Nov 2025 20:18:40 +0800 Subject: [PATCH 4/7] feat: add table create/replace/update interface to catalog --- src/iceberg/catalog.h | 2 +- src/iceberg/table.cc | 2 +- src/iceberg/table.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/iceberg/catalog.h b/src/iceberg/catalog.h index 8985ac3fc..c71d5cc9e 100644 --- a/src/iceberg/catalog.h +++ b/src/iceberg/catalog.h @@ -255,7 +255,7 @@ class ICEBERG_EXPORT Catalog { /// \brief Starts a transaction to create or replace the table /// - /// \breturn the Transaction to create or replace the table + /// \return the Transaction to create or replace the table virtual std::unique_ptr StageCreateOrReplace() = 0; }; diff --git a/src/iceberg/table.cc b/src/iceberg/table.cc index e584cbe03..2b0b4c87e 100644 --- a/src/iceberg/table.cc +++ b/src/iceberg/table.cc @@ -133,7 +133,7 @@ const std::vector& Table::history() const { return metadata_->snapshot_log; } -std::shared_ptr Table::NewTransaction() { +std::shared_ptr Table::NewTransaction() const { throw NotImplemented("Table::NewTransaction is not implemented"); } diff --git a/src/iceberg/table.h b/src/iceberg/table.h index f868279d4..4cfcc5300 100644 --- a/src/iceberg/table.h +++ b/src/iceberg/table.h @@ -112,7 +112,7 @@ class ICEBERG_EXPORT Table { /// \brief Create a new transaction for this table /// /// \return a shared pointer to the new Transaction - virtual std::shared_ptr NewTransaction(); + virtual std::shared_ptr NewTransaction() const; /// \brief Returns a FileIO to read and write table data and metadata files const std::shared_ptr& io() const; From 6697df9dd329f9c3be4ea0645834fbdc25f046d1 Mon Sep 17 00:00:00 2001 From: "shuxu.li" Date: Tue, 11 Nov 2025 09:29:17 +0800 Subject: [PATCH 5/7] feat: add table create/replace/update interface to catalog --- src/iceberg/catalog.h | 25 ------------------- .../catalog/memory/in_memory_catalog.cc | 7 ------ .../catalog/memory/in_memory_catalog.h | 6 ----- src/iceberg/table.cc | 2 +- src/iceberg/table.h | 2 +- src/iceberg/test/mock_catalog.h | 6 ----- src/iceberg/transaction.h | 2 +- 7 files changed, 3 insertions(+), 47 deletions(-) diff --git a/src/iceberg/catalog.h b/src/iceberg/catalog.h index c71d5cc9e..d033199ca 100644 --- a/src/iceberg/catalog.h +++ b/src/iceberg/catalog.h @@ -140,21 +140,6 @@ class ICEBERG_EXPORT Catalog { const std::string& location, const std::unordered_map& properties) = 0; - /// \brief Start a transaction to replace a table - /// - /// \param identifier a table identifier - /// \param schema a schema - /// \param spec a partition spec - /// \param location a location for the table; leave empty if unspecified - /// \param properties a string map of table properties - /// \param orCreate whether to create the table if not exists - /// \return a Transaction to replace the table or ErrorKind::kNotFound if the table - /// doesn't exist and orCreate is false - virtual Result> StageReplaceTable( - const TableIdentifier& identifier, const Schema& schema, const PartitionSpec& spec, - const std::string& location, - const std::unordered_map& properties, bool orCreate) = 0; - /// \brief Check whether table exists /// /// \param identifier a table identifier @@ -247,16 +232,6 @@ class ICEBERG_EXPORT Catalog { /// /// \return the Transaction to create the table virtual std::unique_ptr StageCreate() = 0; - - /// \brief Starts a transaction to replace the table - /// - /// \return the Transaction to replace the table - virtual std::unique_ptr StageReplace() = 0; - - /// \brief Starts a transaction to create or replace the table - /// - /// \return the Transaction to create or replace the table - virtual std::unique_ptr StageCreateOrReplace() = 0; }; /// \brief Instantiate a builder to either create a table or start a create/replace diff --git a/src/iceberg/catalog/memory/in_memory_catalog.cc b/src/iceberg/catalog/memory/in_memory_catalog.cc index cceed1206..753c3358a 100644 --- a/src/iceberg/catalog/memory/in_memory_catalog.cc +++ b/src/iceberg/catalog/memory/in_memory_catalog.cc @@ -404,13 +404,6 @@ Result> InMemoryCatalog::StageCreateTable( return NotImplemented("stage create table"); } -Result> InMemoryCatalog::StageReplaceTable( - const TableIdentifier& identifier, const Schema& schema, const PartitionSpec& spec, - const std::string& location, - const std::unordered_map& properties, bool orCreate) { - return NotImplemented("stage replace table"); -} - Result InMemoryCatalog::TableExists(const TableIdentifier& identifier) const { std::unique_lock lock(mutex_); return root_namespace_->TableExists(identifier); diff --git a/src/iceberg/catalog/memory/in_memory_catalog.h b/src/iceberg/catalog/memory/in_memory_catalog.h index 0d3910ab1..069a1d004 100644 --- a/src/iceberg/catalog/memory/in_memory_catalog.h +++ b/src/iceberg/catalog/memory/in_memory_catalog.h @@ -85,12 +85,6 @@ class ICEBERG_EXPORT InMemoryCatalog const std::string& location, const std::unordered_map& properties) override; - Result> StageReplaceTable( - const TableIdentifier& identifier, const Schema& schema, const PartitionSpec& spec, - const std::string& location, - const std::unordered_map& properties, - bool orCreate) override; - Result TableExists(const TableIdentifier& identifier) const override; Status DropTable(const TableIdentifier& identifier, bool purge) override; diff --git a/src/iceberg/table.cc b/src/iceberg/table.cc index 2b0b4c87e..7c7df7c6f 100644 --- a/src/iceberg/table.cc +++ b/src/iceberg/table.cc @@ -133,7 +133,7 @@ const std::vector& Table::history() const { return metadata_->snapshot_log; } -std::shared_ptr Table::NewTransaction() const { +std::unique_ptr Table::NewTransaction() const { throw NotImplemented("Table::NewTransaction is not implemented"); } diff --git a/src/iceberg/table.h b/src/iceberg/table.h index 4cfcc5300..898061e53 100644 --- a/src/iceberg/table.h +++ b/src/iceberg/table.h @@ -112,7 +112,7 @@ class ICEBERG_EXPORT Table { /// \brief Create a new transaction for this table /// /// \return a shared pointer to the new Transaction - virtual std::shared_ptr NewTransaction() const; + virtual std::unique_ptr NewTransaction() const; /// \brief Returns a FileIO to read and write table data and metadata files const std::shared_ptr& io() const; diff --git a/src/iceberg/test/mock_catalog.h b/src/iceberg/test/mock_catalog.h index a6f397b9c..7c54ebacb 100644 --- a/src/iceberg/test/mock_catalog.h +++ b/src/iceberg/test/mock_catalog.h @@ -71,12 +71,6 @@ class MockCatalog : public Catalog { const std::string&, (const std::unordered_map&)), (override)); - MOCK_METHOD((Result>), StageReplaceTable, - (const TableIdentifier&, const Schema&, const PartitionSpec&, - const std::string&, (const std::unordered_map&), - bool), - (override)); - MOCK_METHOD(Result, TableExists, (const TableIdentifier&), (const, override)); MOCK_METHOD(Status, DropTable, (const TableIdentifier&, bool), (override)); diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index f54edcb58..f51b91ac0 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -51,7 +51,7 @@ class ICEBERG_EXPORT Transaction { /// \return Status::OK if all updates were queued successfully virtual Status UpdateTable( const std::vector>& requirements, - std::vector> updates) = 0; + const std::vector>& updates) = 0; /// \brief Apply the pending changes from all actions and commit /// From 547033e1c7932a9461b75b73076c958d5080d668 Mon Sep 17 00:00:00 2001 From: "shuxu.li" Date: Tue, 11 Nov 2025 09:34:03 +0800 Subject: [PATCH 6/7] feat: add table create/replace/update interface to catalog --- src/iceberg/table.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iceberg/table.h b/src/iceberg/table.h index 898061e53..672cac754 100644 --- a/src/iceberg/table.h +++ b/src/iceberg/table.h @@ -111,7 +111,7 @@ class ICEBERG_EXPORT Table { /// \brief Create a new transaction for this table /// - /// \return a shared pointer to the new Transaction + /// \return a pointer to the new Transaction virtual std::unique_ptr NewTransaction() const; /// \brief Returns a FileIO to read and write table data and metadata files From 9d49902ac09f750ecb091c2d92e74727516fbd74 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Tue, 11 Nov 2025 14:00:00 +0800 Subject: [PATCH 7/7] Update src/iceberg/transaction.h --- src/iceberg/transaction.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index f51b91ac0..0bcedd6d8 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -44,15 +44,6 @@ class ICEBERG_EXPORT Transaction { /// \return a new AppendFiles virtual std::shared_ptr NewAppend() = 0; - /// \brief Apply multiple metadata updates to this transaction - /// - /// \param requirements the table requirements to validate - /// \param updates the table updates to apply - /// \return Status::OK if all updates were queued successfully - virtual Status UpdateTable( - const std::vector>& requirements, - const std::vector>& updates) = 0; - /// \brief Apply the pending changes from all actions and commit /// /// This method applies all pending data operations and metadata updates in the