diff --git a/include/paimon/orphan_files_cleaner.h b/include/paimon/orphan_files_cleaner.h index 411314ad7..e9c6859fb 100644 --- a/include/paimon/orphan_files_cleaner.h +++ b/include/paimon/orphan_files_cleaner.h @@ -184,6 +184,11 @@ class PAIMON_EXPORT OrphanFilesCleaner { /// files. virtual Result> Clean() = 0; + /// Retrieve metrics related to orphan files cleaning operations. + /// + /// @return A shared pointer to a `Metrics` object containing cleaning metrics. + virtual std::shared_ptr GetMetrics() const = 0; + protected: OrphanFilesCleaner() = default; }; diff --git a/src/paimon/core/operation/orphan_files_cleaner_impl.cpp b/src/paimon/core/operation/orphan_files_cleaner_impl.cpp index 740a629b6..d2832a573 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_impl.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner_impl.cpp @@ -25,6 +25,7 @@ #include "fmt/format.h" #include "paimon/common/executor/future.h" +#include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" @@ -32,7 +33,9 @@ #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/operation/metrics/clean_metrics.h" #include "paimon/core/snapshot.h" +#include "paimon/core/utils/duration.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/status.h" @@ -61,7 +64,8 @@ OrphanFilesCleanerImpl::OrphanFilesCleanerImpl( manifest_file_(manifest_file), manifest_list_(manifest_list), older_than_ms_(older_than_ms), - should_be_retained_(should_be_retained) {} + should_be_retained_(should_be_retained), + metrics_(std::make_shared()) {} bool OrphanFilesCleanerImpl::SupportToClean(const std::string& file_name) { static std::vector> supported_pattern = { @@ -91,16 +95,18 @@ Result> OrphanFilesCleanerImpl::Clean() { "OrphanFilesCleaner do not support cleaning table with branch"); } PAIMON_ASSIGN_OR_RAISE(std::set all_dirs, ListPaimonFileDirs()); + PAIMON_ASSIGN_OR_RAISE(std::set used_file_names, GetUsedFiles()); + Duration duration; std::vector>>> file_statuses_futures; for (const auto& dir : all_dirs) { file_statuses_futures.push_back( Via(executor_.get(), [this, dir] { return TryBestListingDirs(dir); })); } - PAIMON_ASSIGN_OR_RAISE(std::set used_file_names, GetUsedFiles()); std::set need_to_deletes; std::vector> futures; ScopeGuard guard([&futures]() { Wait(futures); }); + uint64_t file_statuses_duration = duration.Reset(); for (const auto& file_statuses : CollectAll(file_statuses_futures)) { for (const auto& file_status : file_statuses) { if (file_status->IsDir()) { @@ -129,10 +135,17 @@ Result> OrphanFilesCleanerImpl::Clean() { } } } + metrics_->SetCounter(CleanMetrics::CLEAN_SCAN_ORPHAN_FILES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_FILE_STATUS_DURATION, file_statuses_duration); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_FILE_STATUS_TASKS, + static_cast(file_statuses_futures.size())); + metrics_->SetCounter(CleanMetrics::CLEAN_ORPHAN_FILES, + static_cast(need_to_deletes.size())); return need_to_deletes; } Result> OrphanFilesCleanerImpl::ListPaimonFileDirs() const { + Duration duration; std::set paimon_file_dirs; paimon_file_dirs.insert(snapshot_manager_->SnapshotDirectory()); paimon_file_dirs.insert(FileStorePathFactory::ManifestPath(root_path_)); @@ -156,6 +169,9 @@ Result> OrphanFilesCleanerImpl::ListPaimonFileDirs() const // ListFileDirs(external_path, partition_keys_.size()); // paimon_file_dirs.insert(external_file_dirs.begin(), external_file_dirs.end()); // } + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_DIRECTORIES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_DIRECTORIES, + static_cast(paimon_file_dirs.size())); return paimon_file_dirs; } @@ -225,6 +241,7 @@ Result> OrphanFilesCleanerImpl::GetUsedFiles() const { // TODO(jinli.zjw): consider changelog(add tests), stats used_files.insert(SnapshotManager::EARLIEST); used_files.insert(SnapshotManager::LATEST); + Duration duration; PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, snapshot_manager_->GetAllSnapshots()); for (const auto& snapshot : snapshots) { used_files.insert(SnapshotManager::SNAPSHOT_PREFIX + std::to_string(snapshot.Id())); @@ -257,6 +274,8 @@ Result> OrphanFilesCleanerImpl::GetUsedFiles() const { } } } + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_USED_FILES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_USED_FILES, static_cast(used_files.size())); return used_files; } diff --git a/src/paimon/core/operation/orphan_files_cleaner_impl.h b/src/paimon/core/operation/orphan_files_cleaner_impl.h index 14be18daf..f49918270 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_impl.h +++ b/src/paimon/core/operation/orphan_files_cleaner_impl.h @@ -28,6 +28,7 @@ #include "paimon/core/snapshot.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/metrics.h" #include "paimon/orphan_files_cleaner.h" #include "paimon/result.h" @@ -62,6 +63,10 @@ class OrphanFilesCleanerImpl : public OrphanFilesCleaner { Result> Clean() override; + std::shared_ptr GetMetrics() const override { + return metrics_; + } + private: Result> ListPaimonFileDirs() const; std::vector> TryBestListingDirs(const std::string& path) const; @@ -86,5 +91,7 @@ class OrphanFilesCleanerImpl : public OrphanFilesCleaner { std::shared_ptr manifest_list_; int64_t older_than_ms_; std::function should_be_retained_; + + std::shared_ptr metrics_; }; } // namespace paimon