support clean metrics of OrphanFilesCleaner (#118)

Author: chongchongxiao

include/paimon/orphan_files_cleaner.h

@@ -184,6 +184,11 @@ class PAIMON_EXPORT OrphanFilesCleaner {
     /// files.
     virtual Result<std::set<std::string>> Clean() = 0;
 
+    /// Retrieve metrics related to orphan files cleaning operations.
+    ///
+    /// @return A shared pointer to a `Metrics` object containing cleaning metrics.
+    virtual std::shared_ptr<Metrics> GetMetrics() const = 0;
+
  protected:
     OrphanFilesCleaner() = default;
 };

src/paimon/core/operation/file_store_commit_impl.cpp

@@ -62,6 +62,7 @@
 #include "paimon/core/schema/schema_manager.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/sink/commit_message_impl.h"
+#include "paimon/core/utils/duration.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/fs/file_system.h"
@@ -379,7 +380,7 @@ Status FileStoreCommitImpl::Commit(const std::shared_ptr<ManifestCommittable>& c
 
     int32_t attempt = 0;
     int32_t generated_snapshot = 0;
-    const auto started = std::chrono::high_resolution_clock::now();
+    Duration duration;
     if (!ignore_empty_commit_ || !append_table_files.empty() || !append_table_index_files.empty()) {
         PAIMON_ASSIGN_OR_RAISE(int32_t cnt,
                                TryCommit(append_table_files, append_table_index_files,
@@ -403,10 +404,7 @@ Status FileStoreCommitImpl::Commit(const std::shared_ptr<ManifestCommittable>& c
             compaction_input_file_size += entry.File()->file_size;
         }
     }
-    metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION,
-                         std::chrono::duration_cast<std::chrono::milliseconds>(
-                             std::chrono::high_resolution_clock::now() - started)
-                             .count());
+    metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, duration.Get());
     metrics_->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, attempt);
     metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, table_files_added);
     metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED, table_files_deleted);

src/paimon/core/operation/file_store_scan.cpp

@@ -39,6 +39,7 @@
 #include "paimon/core/operation/metrics/scan_metrics.h"
 #include "paimon/core/partition/partition_info.h"
 #include "paimon/core/stats/simple_stats.h"
+#include "paimon/core/utils/duration.h"
 #include "paimon/core/utils/field_mapping.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/predicate/literal.h"
@@ -94,7 +95,7 @@ Result<std::vector<PartitionEntry>> FileStoreScan::ReadPartitionEntries() const
 }
 
 Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() const {
-    const auto started = std::chrono::high_resolution_clock::now();
+    Duration duration;
     std::optional<Snapshot> snapshot;
     std::vector<ManifestFileMeta> all_manifest_file_metas;
     std::vector<ManifestFileMeta> filtered_manifest_file_metas;
@@ -133,10 +134,7 @@ Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() cons
         [](const int64_t sum, const ManifestFileMeta& manifest_file_meta) {
             return sum + manifest_file_meta.NumAddedFiles() - manifest_file_meta.NumDeletedFiles();
         });
-    metrics_->SetCounter(ScanMetrics::LAST_SCAN_DURATION,
-                         std::chrono::duration_cast<std::chrono::milliseconds>(
-                             std::chrono::high_resolution_clock::now() - started)
-                             .count());
+    metrics_->SetCounter(ScanMetrics::LAST_SCAN_DURATION, duration.Get());
     metrics_->SetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID,
                          snapshot.has_value() ? snapshot.value().Id() : int64_t{0});
     metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS, filtered_manifest_file_metas.size());

src/paimon/core/operation/metrics/clean_metrics.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2026-present Alibaba Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace paimon {
+
+/// Metrics to measure clean operation.
+class CleanMetrics {
+ public:
+    static constexpr char CLEAN_DURATION[] = "cleanDuration";
+    static constexpr char CLEAN_LIST_DIRECTORIES_DURATION[] = "listDirectoriesDuration";
+    static constexpr char CLEAN_LIST_DIRECTORIES[] = "listDirectories";
+    static constexpr char CLEAN_LIST_FILE_STATUS_DURATION[] = "listFileStatusDuration";
+    static constexpr char CLEAN_LIST_FILE_STATUS_TASKS[] = "listFileStatusTasks";
+    static constexpr char CLEAN_LIST_USED_FILES_DURATION[] = "listUsedFilesDuration";
+    static constexpr char CLEAN_USED_FILES[] = "usedFiles";
+    static constexpr char CLEAN_SCAN_ORPHAN_FILES_DURATION[] = "scanOrphanFilesDuration";
+    static constexpr char CLEAN_ORPHAN_FILES[] = "orphanFiles";
+};
+
+}  // namespace paimon

src/paimon/core/operation/orphan_files_cleaner_impl.cpp

@@ -25,14 +25,17 @@
 
 #include "fmt/format.h"
 #include "paimon/common/executor/future.h"
+#include "paimon/common/metrics/metrics_impl.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/core/manifest/manifest_entry.h"
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_file_meta.h"
 #include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/operation/metrics/clean_metrics.h"
 #include "paimon/core/snapshot.h"
+#include "paimon/core/utils/duration.h"
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/snapshot_manager.h"
 #include "paimon/status.h"
@@ -61,7 +64,8 @@ OrphanFilesCleanerImpl::OrphanFilesCleanerImpl(
       manifest_file_(manifest_file),
       manifest_list_(manifest_list),
       older_than_ms_(older_than_ms),
-      should_be_retained_(should_be_retained) {}
+      should_be_retained_(should_be_retained),
+      metrics_(std::make_shared<MetricsImpl>()) {}
 
 bool OrphanFilesCleanerImpl::SupportToClean(const std::string& file_name) {
     static std::vector<std::pair<std::string, std::string>> supported_pattern = {
@@ -83,6 +87,7 @@ bool OrphanFilesCleanerImpl::SupportToClean(const std::string& file_name) {
 }
 
 Result<std::set<std::string>> OrphanFilesCleanerImpl::Clean() {
+    Duration main_duration;
     if (!MinimalTryBestListingDirs(PathUtil::JoinPath(root_path_, "tag")).empty()) {
         return Status::NotImplemented("OrphanFilesCleaner do not support cleaning table with tag");
     }
@@ -98,9 +103,11 @@ Result<std::set<std::string>> OrphanFilesCleanerImpl::Clean() {
     }
     PAIMON_ASSIGN_OR_RAISE(std::set<std::string> used_file_names, GetUsedFiles());
 
+    Duration duration;
     std::set<std::string> need_to_deletes;
     std::vector<std::future<void>> futures;
     ScopeGuard guard([&futures]() { Wait(futures); });
+    uint64_t file_statuses_duration = duration.Reset();
     for (const auto& file_statuses : CollectAll(file_statuses_futures)) {
         for (const auto& file_status : file_statuses) {
             if (file_status->IsDir()) {
@@ -129,10 +136,18 @@ Result<std::set<std::string>> OrphanFilesCleanerImpl::Clean() {
             }
         }
     }
+    metrics_->SetCounter(CleanMetrics::CLEAN_DURATION, main_duration.Get());
+    metrics_->SetCounter(CleanMetrics::CLEAN_SCAN_ORPHAN_FILES_DURATION, duration.Get());
+    metrics_->SetCounter(CleanMetrics::CLEAN_LIST_FILE_STATUS_DURATION, file_statuses_duration);
+    metrics_->SetCounter(CleanMetrics::CLEAN_LIST_FILE_STATUS_TASKS,
+                         static_cast<uint64_t>(file_statuses_futures.size()));
+    metrics_->SetCounter(CleanMetrics::CLEAN_ORPHAN_FILES,
+                         static_cast<uint64_t>(need_to_deletes.size()));
     return need_to_deletes;
 }
 
 Result<std::set<std::string>> OrphanFilesCleanerImpl::ListPaimonFileDirs() const {
+    Duration duration;
     std::set<std::string> paimon_file_dirs;
     paimon_file_dirs.insert(snapshot_manager_->SnapshotDirectory());
     paimon_file_dirs.insert(FileStorePathFactory::ManifestPath(root_path_));
@@ -156,6 +171,9 @@ Result<std::set<std::string>> OrphanFilesCleanerImpl::ListPaimonFileDirs() const
     //         ListFileDirs(external_path, partition_keys_.size());
     //     paimon_file_dirs.insert(external_file_dirs.begin(), external_file_dirs.end());
     // }
+    metrics_->SetCounter(CleanMetrics::CLEAN_LIST_DIRECTORIES_DURATION, duration.Get());
+    metrics_->SetCounter(CleanMetrics::CLEAN_LIST_DIRECTORIES,
+                         static_cast<uint64_t>(paimon_file_dirs.size()));
     return paimon_file_dirs;
 }
 
@@ -225,6 +243,7 @@ Result<std::set<std::string>> OrphanFilesCleanerImpl::GetUsedFiles() const {
     // TODO(jinli.zjw): consider changelog(add tests), stats
     used_files.insert(SnapshotManager::EARLIEST);
     used_files.insert(SnapshotManager::LATEST);
+    Duration duration;
     PAIMON_ASSIGN_OR_RAISE(std::vector<Snapshot> snapshots, snapshot_manager_->GetAllSnapshots());
     for (const auto& snapshot : snapshots) {
         used_files.insert(SnapshotManager::SNAPSHOT_PREFIX + std::to_string(snapshot.Id()));
@@ -257,6 +276,8 @@ Result<std::set<std::string>> OrphanFilesCleanerImpl::GetUsedFiles() const {
             }
         }
     }
+    metrics_->SetCounter(CleanMetrics::CLEAN_LIST_USED_FILES_DURATION, duration.Get());
+    metrics_->SetCounter(CleanMetrics::CLEAN_USED_FILES, static_cast<uint64_t>(used_files.size()));
     return used_files;
 }
 

src/paimon/core/operation/orphan_files_cleaner_impl.h

@@ -28,6 +28,7 @@
 #include "paimon/core/snapshot.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/metrics.h"
 #include "paimon/orphan_files_cleaner.h"
 #include "paimon/result.h"
 
@@ -62,6 +63,10 @@ class OrphanFilesCleanerImpl : public OrphanFilesCleaner {
 
     Result<std::set<std::string>> Clean() override;
 
+    std::shared_ptr<Metrics> GetMetrics() const override {
+        return metrics_;
+    }
+
  private:
     Result<std::set<std::string>> ListPaimonFileDirs() const;
     std::vector<std::unique_ptr<FileStatus>> TryBestListingDirs(const std::string& path) const;
@@ -86,5 +91,7 @@ class OrphanFilesCleanerImpl : public OrphanFilesCleaner {
     std::shared_ptr<ManifestList> manifest_list_;
     int64_t older_than_ms_;
     std::function<bool(const std::string&)> should_be_retained_;
+
+    std::shared_ptr<Metrics> metrics_;
 };
 }  // namespace paimon

src/paimon/core/utils/duration.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2026-present Alibaba Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <chrono>
+
+namespace paimon {
+
+// Calculate operation duration.
+class Duration {
+ public:
+    Duration() : start_(std::chrono::high_resolution_clock::now()) {}
+
+    uint64_t Get() {
+        return std::chrono::duration_cast<std::chrono::milliseconds>(
+                   std::chrono::high_resolution_clock::now() - start_)
+            .count();
+    }
+
+    uint64_t Reset() {
+        uint64_t dura = Get();
+        start_ = std::chrono::high_resolution_clock::now();
+        return dura;
+    }
+
+ private:
+    std::chrono::high_resolution_clock::time_point start_;
+};
+
+}  // namespace paimon