Skip to content

Commit d8582c6

Browse files
committed
feat: support scan metrics of FileStoreScan to align with ScanMetrics
1 parent 5dc22e7 commit d8582c6

2 files changed

Lines changed: 20 additions & 4 deletions

File tree

src/paimon/core/operation/file_store_scan.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() cons
129129
}
130130
}
131131
const int64_t all_data_files = std::accumulate(
132-
all_manifest_file_metas.begin(), all_manifest_file_metas.end(), 0,
132+
all_manifest_file_metas.begin(), all_manifest_file_metas.end(), int64_t{0},
133133
[](const int64_t sum, const ManifestFileMeta& manifest_file_meta) {
134134
return sum + manifest_file_meta.NumAddedFiles() - manifest_file_meta.NumDeletedFiles();
135135
});
@@ -138,7 +138,7 @@ Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() cons
138138
std::chrono::high_resolution_clock::now() - started)
139139
.count());
140140
metrics_->SetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID,
141-
snapshot.has_value() ? snapshot.value().Id() : 0);
141+
snapshot.has_value() ? snapshot.value().Id() : int64_t{0});
142142
metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS, filtered_manifest_file_metas.size());
143143
metrics_->SetCounter(ScanMetrics::LAST_SCAN_SKIPPED_TABLE_FILES,
144144
all_data_files - manifest_entries.size());
@@ -151,14 +151,15 @@ Status FileStoreScan::ReadManifests(std::optional<Snapshot>* snapshot_ptr,
151151
std::vector<ManifestFileMeta>* all_manifests_ptr,
152152
std::vector<ManifestFileMeta>* filter_manifests_ptr) const {
153153
auto& snapshot = *snapshot_ptr;
154-
auto& filtered_manifests = *filter_manifests_ptr;
155154
auto& all_manifests = *all_manifests_ptr;
155+
auto& filtered_manifests = *filter_manifests_ptr;
156156
if (specified_snapshot_ != std::nullopt) {
157157
snapshot = specified_snapshot_;
158158
} else {
159159
PAIMON_ASSIGN_OR_RAISE(snapshot, snapshot_manager_->LatestSnapshot());
160160
}
161161
if (snapshot == std::nullopt) {
162+
all_manifests = std::vector<ManifestFileMeta>();
162163
filtered_manifests = std::vector<ManifestFileMeta>();
163164
return Status::OK();
164165
}

src/paimon/core/operation/key_value_file_store_scan_test.cpp

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,12 +144,27 @@ TEST_F(KeyValueFileStoreScanTest, TestMaxSequenceNumber) {
144144
CreateFileStoreScan(table_path, scan_filter,
145145
/*table_schema_id=*/0, /*snapshot_id=*/2));
146146

147+
const auto started = std::chrono::high_resolution_clock::now();
147148
ASSERT_OK_AND_ASSIGN(std::shared_ptr<FileStoreScan::RawPlan> raw_plan, scan->CreatePlan());
148149
std::shared_ptr<Metrics> metrics = scan->GetScanMetrics();
149150
ASSERT_TRUE(metrics);
151+
ASSERT_OK_AND_ASSIGN(uint64_t last_scan_duration,
152+
metrics->GetCounter(ScanMetrics::LAST_SCAN_DURATION));
153+
ASSERT_LE(last_scan_duration, std::chrono::duration_cast<std::chrono::milliseconds>(
154+
std::chrono::high_resolution_clock::now() - started)
155+
.count());
150156
ASSERT_OK_AND_ASSIGN(uint64_t last_scanned_snapshot_id,
151157
metrics->GetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID));
152-
ASSERT_EQ(2u, last_scanned_snapshot_id);
158+
ASSERT_EQ(last_scanned_snapshot_id, 2u);
159+
ASSERT_OK_AND_ASSIGN(uint64_t last_scanned_manifests,
160+
metrics->GetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS));
161+
ASSERT_EQ(last_scanned_manifests, 2u);
162+
ASSERT_OK_AND_ASSIGN(uint64_t last_scan_skipped_table_files,
163+
metrics->GetCounter(ScanMetrics::LAST_SCAN_SKIPPED_TABLE_FILES));
164+
ASSERT_EQ(last_scan_skipped_table_files, 0u);
165+
ASSERT_OK_AND_ASSIGN(uint64_t last_scan_resulted_table_files,
166+
metrics->GetCounter(ScanMetrics::LAST_SCAN_RESULTED_TABLE_FILES));
167+
ASSERT_EQ(last_scan_resulted_table_files, 2u);
153168
int64_t max_sequence_num = GetMaxSequenceNumberOfRawPlan(raw_plan);
154169
ASSERT_EQ(max_sequence_num, 1);
155170
// test multiple scan

0 commit comments

Comments
 (0)