diff --git a/src/iceberg/manage_snapshots.h b/src/iceberg/manage_snapshots.h new file mode 100644 index 000000000..6e685f2db --- /dev/null +++ b/src/iceberg/manage_snapshots.h @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/manage_snapshots.h +/// API for managing snapshots, branches, and tags in a table + +#include +#include + +#include "iceberg/iceberg_export.h" +#include "iceberg/pending_update.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +/// \brief API for managing snapshots, branches, and tags in a table +/// +/// ManageSnapshots provides operations for: +/// - Rolling table data back to a state at an older snapshot +/// - Cherry-picking orphan snapshots in audit workflows +/// - Creating, updating, and managing branches and tags +/// - Setting retention policies for snapshots and references +/// +/// Behavioral notes: +/// - Does NOT allow conflicting calls to SetCurrentSnapshot() and +/// RollbackToTime() in the same operation +/// - Commit conflicts will NOT be resolved automatically and will result in +/// CommitFailed error +/// - Changes are applied to current table metadata when committed +/// +/// Example usage: +/// \code +/// table.ManageSnapshots() +/// .CreateBranch("experiment", snapshot_id) +/// .SetMinSnapshotsToKeep("experiment", 5) +/// .Commit(); +/// \endcode +class ICEBERG_EXPORT ManageSnapshots : public PendingUpdateTyped { + public: + ~ManageSnapshots() override = default; + + // ======================================================================== + // SNAPSHOT ROLLBACK OPERATIONS + // ======================================================================== + + /// \brief Roll table's data back to a specific snapshot + /// + /// This will be set as the current snapshot. Unlike RollbackTo(), the + /// snapshot is NOT required to be an ancestor of the current table state. + /// + /// \param snapshot_id ID of the snapshot to roll back table data to + /// \return Reference to this for method chaining + virtual ManageSnapshots& SetCurrentSnapshot(int64_t snapshot_id) = 0; + + /// \brief Roll table's data back to the last snapshot before given timestamp + /// + /// Cannot be used in the same operation as SetCurrentSnapshot(). + /// + /// \param timestamp_millis Timestamp in milliseconds since epoch + /// \return Reference to this for method chaining + virtual ManageSnapshots& RollbackToTime(int64_t timestamp_millis) = 0; + + /// \brief Rollback table's state to a specific snapshot (must be ancestor) + /// + /// The snapshot MUST be an ancestor of the current snapshot. + /// + /// \param snapshot_id ID of snapshot to roll back to (must be ancestor) + /// \return Reference to this for method chaining + virtual ManageSnapshots& RollbackTo(int64_t snapshot_id) = 0; + + /// \brief Apply changes from a snapshot and create new current snapshot + /// + /// Used in audit workflows where data is written to an orphan snapshot, + /// audited, and then applied to the current state. + /// + /// \param snapshot_id Snapshot ID whose changes to apply + /// \return Reference to this for method chaining + virtual ManageSnapshots& Cherrypick(int64_t snapshot_id) = 0; + + // ======================================================================== + // BRANCH OPERATIONS + // ======================================================================== + + /// \brief Create a new branch pointing to the current snapshot + /// + /// The branch will point to the current snapshot if it exists. + /// + /// \param name Branch name + /// \return Reference to this for method chaining + virtual ManageSnapshots& CreateBranch(std::string_view name) = 0; + + /// \brief Create a new branch pointing to a specific snapshot + /// + /// \param name Branch name + /// \param snapshot_id Snapshot ID to be the branch head + /// \return Reference to this for method chaining + virtual ManageSnapshots& CreateBranch(std::string_view name, int64_t snapshot_id) = 0; + + /// \brief Remove a branch by name + /// + /// \param name Branch name to remove + /// \return Reference to this for method chaining + virtual ManageSnapshots& RemoveBranch(std::string_view name) = 0; + + /// \brief Rename an existing branch + /// + /// \param name Current branch name + /// \param new_name Desired new branch name + /// \return Reference to this for method chaining + virtual ManageSnapshots& RenameBranch(std::string_view name, + std::string_view new_name) = 0; + + /// \brief Replace a branch to point to a specified snapshot + /// + /// \param name Branch to replace + /// \param snapshot_id New snapshot ID for the branch + /// \return Reference to this for method chaining + virtual ManageSnapshots& ReplaceBranch(std::string_view name, int64_t snapshot_id) = 0; + + /// \brief Replace one branch with another branch's snapshot + /// + /// The `to` branch remains unchanged. The `from` branch retains its + /// retention properties. If `from` doesn't exist, it's created with + /// default properties. + /// + /// \param from Branch to replace + /// \param to Branch to replace with + /// \return Reference to this for method chaining + virtual ManageSnapshots& ReplaceBranch(std::string_view from, std::string_view to) = 0; + + /// \brief Fast-forward one branch to another if from is ancestor of to + /// + /// Moves the `from` branch to the `to` branch's snapshot. The `to` branch + /// remains unchanged. The `from` branch retains its retention properties. + /// Creates `from` with default properties if it doesn't exist. Only works + /// if `from` is an ancestor of `to`. + /// + /// \param from Branch to fast-forward + /// \param to Reference branch to fast-forward to + /// \return Reference to this for method chaining + virtual ManageSnapshots& FastForwardBranch(std::string_view from, + std::string_view to) = 0; + + // ======================================================================== + // TAG OPERATIONS + // ======================================================================== + + /// \brief Create a new tag pointing to a snapshot + /// + /// \param name Tag name + /// \param snapshot_id Snapshot ID for the tag head + /// \return Reference to this for method chaining + virtual ManageSnapshots& CreateTag(std::string_view name, int64_t snapshot_id) = 0; + + /// \brief Remove a tag by name + /// + /// \param name Tag name to remove + /// \return Reference to this for method chaining + virtual ManageSnapshots& RemoveTag(std::string_view name) = 0; + + /// \brief Replace an existing tag to point to a new snapshot + /// + /// \param name Tag to replace + /// \param snapshot_id New snapshot ID for the tag + /// \return Reference to this for method chaining + virtual ManageSnapshots& ReplaceTag(std::string_view name, int64_t snapshot_id) = 0; + + // ======================================================================== + // RETENTION POLICY OPERATIONS + // ======================================================================== + + /// \brief Update minimum number of snapshots to keep for a branch + /// + /// \param branch_name Name of the branch + /// \param min_snapshots_to_keep Minimum number of snapshots to retain + /// \return Reference to this for method chaining + virtual ManageSnapshots& SetMinSnapshotsToKeep(std::string_view branch_name, + int min_snapshots_to_keep) = 0; + + /// \brief Update maximum snapshot age for a branch + /// + /// \param branch_name Name of the branch + /// \param max_snapshot_age_ms Maximum snapshot age in milliseconds + /// \return Reference to this for method chaining + virtual ManageSnapshots& SetMaxSnapshotAgeMs(std::string_view branch_name, + int64_t max_snapshot_age_ms) = 0; + + /// \brief Update retention policy for a reference (branch or tag) + /// + /// \param name Branch or tag name + /// \param max_ref_age_ms Retention age in milliseconds of the reference + /// \return Reference to this for method chaining + virtual ManageSnapshots& SetMaxRefAgeMs(std::string_view name, + int64_t max_ref_age_ms) = 0; + + // Non-copyable, movable (inherited from PendingUpdate) + ManageSnapshots(const ManageSnapshots&) = delete; + ManageSnapshots& operator=(const ManageSnapshots&) = delete; + + protected: + ManageSnapshots() = default; +}; + +} // namespace iceberg diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 25a03932d..eeeb9b1bf 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -80,6 +80,7 @@ add_iceberg_test(schema_test add_iceberg_test(table_test SOURCES json_internal_test.cc + manage_snapshots_test.cc pending_update_test.cc schema_json_test.cc table_test.cc diff --git a/src/iceberg/test/manage_snapshots_test.cc b/src/iceberg/test/manage_snapshots_test.cc new file mode 100644 index 000000000..70e8c47e8 --- /dev/null +++ b/src/iceberg/test/manage_snapshots_test.cc @@ -0,0 +1,422 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/manage_snapshots.h" + +#include +#include + +#include + +#include "iceberg/result.h" +#include "iceberg/snapshot.h" +#include "iceberg/test/matchers.h" + +namespace iceberg { + +// Mock implementation of ManageSnapshots for testing +// This mock tracks which methods were called to verify behavior +class MockManageSnapshots : public ManageSnapshots { + public: + MockManageSnapshots() = default; + + Result Apply() override { + if (should_fail_) { + return ValidationFailed("Mock validation failed"); + } + apply_called_ = true; + + // Return a snapshot reflecting the configuration + return Snapshot{ + .snapshot_id = current_snapshot_id_, + .parent_snapshot_id = std::nullopt, + .sequence_number = 1, + .timestamp_ms = TimePointMs{std::chrono::milliseconds{1000}}, + .manifest_list = "s3://bucket/metadata/snap-manifest-list.avro", + .summary = {}, + .schema_id = std::nullopt, + }; + } + + Status Commit() override { + if (should_fail_commit_) { + return CommitFailed("Mock commit failed"); + } + commit_called_ = true; + return {}; + } + + ManageSnapshots& SetCurrentSnapshot(int64_t snapshot_id) override { + current_snapshot_id_ = snapshot_id; + return *this; + } + + ManageSnapshots& RollbackToTime(int64_t timestamp_millis) override { + rollback_timestamp_ = timestamp_millis; + return *this; + } + + ManageSnapshots& RollbackTo(int64_t snapshot_id) override { + rollback_snapshot_id_ = snapshot_id; + return *this; + } + + ManageSnapshots& Cherrypick(int64_t snapshot_id) override { + cherrypick_snapshot_id_ = snapshot_id; + return *this; + } + + ManageSnapshots& CreateBranch(std::string_view name) override { + branches_created_.emplace_back(name); + return *this; + } + + ManageSnapshots& CreateBranch(std::string_view name, int64_t snapshot_id) override { + branches_created_.emplace_back(name); + branch_snapshots_[std::string(name)] = snapshot_id; + return *this; + } + + ManageSnapshots& RemoveBranch(std::string_view name) override { + branches_removed_.emplace_back(name); + return *this; + } + + ManageSnapshots& RenameBranch(std::string_view name, + std::string_view new_name) override { + branches_renamed_[std::string(name)] = std::string(new_name); + return *this; + } + + ManageSnapshots& ReplaceBranch(std::string_view name, int64_t snapshot_id) override { + branches_replaced_[std::string(name)] = snapshot_id; + return *this; + } + + ManageSnapshots& ReplaceBranch(std::string_view from, std::string_view to) override { + branch_replacements_[std::string(from)] = std::string(to); + return *this; + } + + ManageSnapshots& FastForwardBranch(std::string_view from, + std::string_view to) override { + branch_fast_forwards_[std::string(from)] = std::string(to); + return *this; + } + + ManageSnapshots& CreateTag(std::string_view name, int64_t snapshot_id) override { + tags_created_[std::string(name)] = snapshot_id; + return *this; + } + + ManageSnapshots& RemoveTag(std::string_view name) override { + tags_removed_.emplace_back(name); + return *this; + } + + ManageSnapshots& ReplaceTag(std::string_view name, int64_t snapshot_id) override { + tags_replaced_[std::string(name)] = snapshot_id; + return *this; + } + + ManageSnapshots& SetMinSnapshotsToKeep(std::string_view branch_name, + int min_snapshots_to_keep) override { + min_snapshots_[std::string(branch_name)] = min_snapshots_to_keep; + return *this; + } + + ManageSnapshots& SetMaxSnapshotAgeMs(std::string_view branch_name, + int64_t max_snapshot_age_ms) override { + max_snapshot_age_[std::string(branch_name)] = max_snapshot_age_ms; + return *this; + } + + ManageSnapshots& SetMaxRefAgeMs(std::string_view name, + int64_t max_ref_age_ms) override { + max_ref_age_[std::string(name)] = max_ref_age_ms; + return *this; + } + + void SetShouldFail(bool fail) { should_fail_ = fail; } + void SetShouldFailCommit(bool fail) { should_fail_commit_ = fail; } + bool ApplyCalled() const { return apply_called_; } + bool CommitCalled() const { return commit_called_; } + + private: + bool should_fail_ = false; + bool should_fail_commit_ = false; + bool apply_called_ = false; + bool commit_called_ = false; + + int64_t current_snapshot_id_ = 1; + std::optional rollback_timestamp_; + std::optional rollback_snapshot_id_; + std::optional cherrypick_snapshot_id_; + + std::vector branches_created_; + std::unordered_map branch_snapshots_; + std::vector branches_removed_; + std::unordered_map branches_renamed_; + std::unordered_map branches_replaced_; + std::unordered_map branch_replacements_; + std::unordered_map branch_fast_forwards_; + + std::unordered_map tags_created_; + std::vector tags_removed_; + std::unordered_map tags_replaced_; + + std::unordered_map min_snapshots_; + std::unordered_map max_snapshot_age_; + std::unordered_map max_ref_age_; +}; + +// ======================================================================== +// SNAPSHOT ROLLBACK OPERATIONS TESTS +// ======================================================================== + +TEST(ManageSnapshotsTest, SetCurrentSnapshot) { + MockManageSnapshots manage; + manage.SetCurrentSnapshot(100); + + auto result = manage.Apply(); + ASSERT_THAT(result, IsOk()); + EXPECT_EQ(result.value().snapshot_id, 100); +} + +TEST(ManageSnapshotsTest, RollbackToTime) { + MockManageSnapshots manage; + manage.RollbackToTime(1609459200000); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, RollbackTo) { + MockManageSnapshots manage; + manage.RollbackTo(50); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, Cherrypick) { + MockManageSnapshots manage; + manage.Cherrypick(75); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +// ======================================================================== +// BRANCH OPERATIONS TESTS +// ======================================================================== + +TEST(ManageSnapshotsTest, CreateBranchCurrent) { + MockManageSnapshots manage; + manage.CreateBranch("experiment"); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, CreateBranchWithSnapshot) { + MockManageSnapshots manage; + manage.CreateBranch("feature-branch", 100); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, RemoveBranch) { + MockManageSnapshots manage; + manage.RemoveBranch("old-branch"); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, RenameBranch) { + MockManageSnapshots manage; + manage.RenameBranch("old-name", "new-name"); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, ReplaceBranchWithSnapshot) { + MockManageSnapshots manage; + manage.ReplaceBranch("my-branch", 200); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, ReplaceBranchWithBranch) { + MockManageSnapshots manage; + manage.ReplaceBranch("from-branch", "to-branch"); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, FastForwardBranch) { + MockManageSnapshots manage; + manage.FastForwardBranch("my-branch", "main"); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +// ======================================================================== +// TAG OPERATIONS TESTS +// ======================================================================== + +TEST(ManageSnapshotsTest, CreateTag) { + MockManageSnapshots manage; + manage.CreateTag("v1.0", 100); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, RemoveTag) { + MockManageSnapshots manage; + manage.RemoveTag("old-tag"); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, ReplaceTag) { + MockManageSnapshots manage; + manage.ReplaceTag("v1.0", 150); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +// ======================================================================== +// RETENTION POLICY TESTS +// ======================================================================== + +TEST(ManageSnapshotsTest, SetMinSnapshotsToKeep) { + MockManageSnapshots manage; + manage.SetMinSnapshotsToKeep("main", 5); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, SetMaxSnapshotAgeMs) { + MockManageSnapshots manage; + manage.SetMaxSnapshotAgeMs("main", 86400000); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, SetMaxRefAgeMs) { + MockManageSnapshots manage; + manage.SetMaxRefAgeMs("experiment", 604800000); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +// ======================================================================== +// METHOD CHAINING TESTS +// ======================================================================== + +TEST(ManageSnapshotsTest, MethodChaining) { + MockManageSnapshots manage; + + manage.CreateBranch("experiment", 100) + .SetMinSnapshotsToKeep("experiment", 5) + .SetMaxSnapshotAgeMs("experiment", 86400000) + .CreateTag("v1.0", 100); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +TEST(ManageSnapshotsTest, ComplexMethodChaining) { + MockManageSnapshots manage; + + manage.SetCurrentSnapshot(100) + .CreateBranch("feature", 100) + .CreateTag("release-1.0", 100) + .SetMinSnapshotsToKeep("feature", 3) + .SetMaxSnapshotAgeMs("feature", 3600000) + .SetMaxRefAgeMs("release-1.0", 2592000000); + + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); +} + +// ======================================================================== +// GENERAL TESTS +// ======================================================================== + +TEST(ManageSnapshotsTest, ApplySuccess) { + MockManageSnapshots manage; + auto result = manage.Apply(); + EXPECT_THAT(result, IsOk()); + EXPECT_TRUE(manage.ApplyCalled()); +} + +TEST(ManageSnapshotsTest, ApplyValidationFailed) { + MockManageSnapshots manage; + manage.SetShouldFail(true); + auto result = manage.Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Mock validation failed")); +} + +TEST(ManageSnapshotsTest, CommitSuccess) { + MockManageSnapshots manage; + auto status = manage.Commit(); + EXPECT_THAT(status, IsOk()); + EXPECT_TRUE(manage.CommitCalled()); +} + +TEST(ManageSnapshotsTest, CommitFailed) { + MockManageSnapshots manage; + manage.SetShouldFailCommit(true); + auto status = manage.Commit(); + EXPECT_THAT(status, IsError(ErrorKind::kCommitFailed)); + EXPECT_THAT(status, HasErrorMessage("Mock commit failed")); +} + +TEST(ManageSnapshotsTest, InheritanceFromPendingUpdate) { + std::unique_ptr base_ptr = std::make_unique(); + auto status = base_ptr->Commit(); + EXPECT_THAT(status, IsOk()); +} + +TEST(ManageSnapshotsTest, InheritanceFromPendingUpdateTyped) { + std::unique_ptr> typed_ptr = + std::make_unique(); + auto status = typed_ptr->Commit(); + EXPECT_THAT(status, IsOk()); + + auto result = typed_ptr->Apply(); + EXPECT_THAT(result, IsOk()); +} + +} // namespace iceberg diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 81681ebcd..10db59c31 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -160,6 +160,8 @@ class PendingUpdate; template class PendingUpdateTyped; +class ManageSnapshots; + /// ---------------------------------------------------------------------------- /// TODO: Forward declarations below are not added yet. /// ----------------------------------------------------------------------------