diff --git a/include/paimon/orphan_files_cleaner.h b/include/paimon/orphan_files_cleaner.h index 411314ad7..e9c6859fb 100644 --- a/include/paimon/orphan_files_cleaner.h +++ b/include/paimon/orphan_files_cleaner.h @@ -184,6 +184,11 @@ class PAIMON_EXPORT OrphanFilesCleaner { /// files. virtual Result> Clean() = 0; + /// Retrieve metrics related to orphan files cleaning operations. + /// + /// @return A shared pointer to a `Metrics` object containing cleaning metrics. + virtual std::shared_ptr GetMetrics() const = 0; + protected: OrphanFilesCleaner() = default; }; diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index 109bc37f7..a9c4a4d05 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -62,6 +62,7 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/duration.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/fs/file_system.h" @@ -379,7 +380,7 @@ Status FileStoreCommitImpl::Commit(const std::shared_ptr& c int32_t attempt = 0; int32_t generated_snapshot = 0; - const auto started = std::chrono::high_resolution_clock::now(); + Duration duration; if (!ignore_empty_commit_ || !append_table_files.empty() || !append_table_index_files.empty()) { PAIMON_ASSIGN_OR_RAISE(int32_t cnt, TryCommit(append_table_files, append_table_index_files, @@ -403,10 +404,7 @@ Status FileStoreCommitImpl::Commit(const std::shared_ptr& c compaction_input_file_size += entry.File()->file_size; } } - metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, - std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - started) - .count()); + metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, duration.Get()); metrics_->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, attempt); metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, table_files_added); metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED, table_files_deleted); diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index 34987d103..9fb9ed3be 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -39,6 +39,7 @@ #include "paimon/core/operation/metrics/scan_metrics.h" #include "paimon/core/partition/partition_info.h" #include "paimon/core/stats/simple_stats.h" +#include "paimon/core/utils/duration.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/predicate/literal.h" @@ -94,7 +95,7 @@ Result> FileStoreScan::ReadPartitionEntries() const } Result> FileStoreScan::CreatePlan() const { - const auto started = std::chrono::high_resolution_clock::now(); + Duration duration; std::optional snapshot; std::vector all_manifest_file_metas; std::vector filtered_manifest_file_metas; @@ -133,10 +134,7 @@ Result> FileStoreScan::CreatePlan() cons [](const int64_t sum, const ManifestFileMeta& manifest_file_meta) { return sum + manifest_file_meta.NumAddedFiles() - manifest_file_meta.NumDeletedFiles(); }); - metrics_->SetCounter(ScanMetrics::LAST_SCAN_DURATION, - std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - started) - .count()); + metrics_->SetCounter(ScanMetrics::LAST_SCAN_DURATION, duration.Get()); metrics_->SetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID, snapshot.has_value() ? snapshot.value().Id() : int64_t{0}); metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS, filtered_manifest_file_metas.size()); diff --git a/src/paimon/core/operation/metrics/clean_metrics.h b/src/paimon/core/operation/metrics/clean_metrics.h new file mode 100644 index 000000000..e898783eb --- /dev/null +++ b/src/paimon/core/operation/metrics/clean_metrics.h @@ -0,0 +1,35 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace paimon { + +/// Metrics to measure clean operation. +class CleanMetrics { + public: + static constexpr char CLEAN_DURATION[] = "cleanDuration"; + static constexpr char CLEAN_LIST_DIRECTORIES_DURATION[] = "listDirectoriesDuration"; + static constexpr char CLEAN_LIST_DIRECTORIES[] = "listDirectories"; + static constexpr char CLEAN_LIST_FILE_STATUS_DURATION[] = "listFileStatusDuration"; + static constexpr char CLEAN_LIST_FILE_STATUS_TASKS[] = "listFileStatusTasks"; + static constexpr char CLEAN_LIST_USED_FILES_DURATION[] = "listUsedFilesDuration"; + static constexpr char CLEAN_USED_FILES[] = "usedFiles"; + static constexpr char CLEAN_SCAN_ORPHAN_FILES_DURATION[] = "scanOrphanFilesDuration"; + static constexpr char CLEAN_ORPHAN_FILES[] = "orphanFiles"; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/orphan_files_cleaner_impl.cpp b/src/paimon/core/operation/orphan_files_cleaner_impl.cpp index 740a629b6..874a8c5ce 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_impl.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner_impl.cpp @@ -25,6 +25,7 @@ #include "fmt/format.h" #include "paimon/common/executor/future.h" +#include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" @@ -32,7 +33,9 @@ #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/operation/metrics/clean_metrics.h" #include "paimon/core/snapshot.h" +#include "paimon/core/utils/duration.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/status.h" @@ -61,7 +64,8 @@ OrphanFilesCleanerImpl::OrphanFilesCleanerImpl( manifest_file_(manifest_file), manifest_list_(manifest_list), older_than_ms_(older_than_ms), - should_be_retained_(should_be_retained) {} + should_be_retained_(should_be_retained), + metrics_(std::make_shared()) {} bool OrphanFilesCleanerImpl::SupportToClean(const std::string& file_name) { static std::vector> supported_pattern = { @@ -83,6 +87,7 @@ bool OrphanFilesCleanerImpl::SupportToClean(const std::string& file_name) { } Result> OrphanFilesCleanerImpl::Clean() { + Duration main_duration; if (!MinimalTryBestListingDirs(PathUtil::JoinPath(root_path_, "tag")).empty()) { return Status::NotImplemented("OrphanFilesCleaner do not support cleaning table with tag"); } @@ -98,9 +103,11 @@ Result> OrphanFilesCleanerImpl::Clean() { } PAIMON_ASSIGN_OR_RAISE(std::set used_file_names, GetUsedFiles()); + Duration duration; std::set need_to_deletes; std::vector> futures; ScopeGuard guard([&futures]() { Wait(futures); }); + uint64_t file_statuses_duration = duration.Reset(); for (const auto& file_statuses : CollectAll(file_statuses_futures)) { for (const auto& file_status : file_statuses) { if (file_status->IsDir()) { @@ -129,10 +136,18 @@ Result> OrphanFilesCleanerImpl::Clean() { } } } + metrics_->SetCounter(CleanMetrics::CLEAN_DURATION, main_duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_SCAN_ORPHAN_FILES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_FILE_STATUS_DURATION, file_statuses_duration); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_FILE_STATUS_TASKS, + static_cast(file_statuses_futures.size())); + metrics_->SetCounter(CleanMetrics::CLEAN_ORPHAN_FILES, + static_cast(need_to_deletes.size())); return need_to_deletes; } Result> OrphanFilesCleanerImpl::ListPaimonFileDirs() const { + Duration duration; std::set paimon_file_dirs; paimon_file_dirs.insert(snapshot_manager_->SnapshotDirectory()); paimon_file_dirs.insert(FileStorePathFactory::ManifestPath(root_path_)); @@ -156,6 +171,9 @@ Result> OrphanFilesCleanerImpl::ListPaimonFileDirs() const // ListFileDirs(external_path, partition_keys_.size()); // paimon_file_dirs.insert(external_file_dirs.begin(), external_file_dirs.end()); // } + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_DIRECTORIES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_DIRECTORIES, + static_cast(paimon_file_dirs.size())); return paimon_file_dirs; } @@ -225,6 +243,7 @@ Result> OrphanFilesCleanerImpl::GetUsedFiles() const { // TODO(jinli.zjw): consider changelog(add tests), stats used_files.insert(SnapshotManager::EARLIEST); used_files.insert(SnapshotManager::LATEST); + Duration duration; PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, snapshot_manager_->GetAllSnapshots()); for (const auto& snapshot : snapshots) { used_files.insert(SnapshotManager::SNAPSHOT_PREFIX + std::to_string(snapshot.Id())); @@ -257,6 +276,8 @@ Result> OrphanFilesCleanerImpl::GetUsedFiles() const { } } } + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_USED_FILES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_USED_FILES, static_cast(used_files.size())); return used_files; } diff --git a/src/paimon/core/operation/orphan_files_cleaner_impl.h b/src/paimon/core/operation/orphan_files_cleaner_impl.h index 14be18daf..f49918270 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_impl.h +++ b/src/paimon/core/operation/orphan_files_cleaner_impl.h @@ -28,6 +28,7 @@ #include "paimon/core/snapshot.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/metrics.h" #include "paimon/orphan_files_cleaner.h" #include "paimon/result.h" @@ -62,6 +63,10 @@ class OrphanFilesCleanerImpl : public OrphanFilesCleaner { Result> Clean() override; + std::shared_ptr GetMetrics() const override { + return metrics_; + } + private: Result> ListPaimonFileDirs() const; std::vector> TryBestListingDirs(const std::string& path) const; @@ -86,5 +91,7 @@ class OrphanFilesCleanerImpl : public OrphanFilesCleaner { std::shared_ptr manifest_list_; int64_t older_than_ms_; std::function should_be_retained_; + + std::shared_ptr metrics_; }; } // namespace paimon diff --git a/src/paimon/core/utils/duration.h b/src/paimon/core/utils/duration.h new file mode 100644 index 000000000..d5b64dbbf --- /dev/null +++ b/src/paimon/core/utils/duration.h @@ -0,0 +1,44 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace paimon { + +// Calculate operation duration. +class Duration { + public: + Duration() : start_(std::chrono::high_resolution_clock::now()) {} + + uint64_t Get() { + return std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - start_) + .count(); + } + + uint64_t Reset() { + uint64_t dura = Get(); + start_ = std::chrono::high_resolution_clock::now(); + return dura; + } + + private: + std::chrono::high_resolution_clock::time_point start_; +}; + +} // namespace paimon