Skip to content

Commit 5f54636

Browse files
committed
[feature](be) Add OLAP scan filter profile counters
### What problem does this PR solve? Issue Number: N/A Related PR: N/A Problem Summary: Existing OLAP scan profiles only expose aggregate filter counters, making it hard to understand the selectivity of each conjunct, runtime filter, key range, and index-related filtering step. This change adds scan-local filter IDs and materializes detailed ScanFilterInfo and KeyRangeInfo profile sections. Per-filter counters now show input rows, output rows, filtered rows, stage participation, source expression or runtime filter debug text, and runtime filter wait/always-true data where applicable. Empty runtime filter fields are omitted for scans without runtime filters, and runtime filter IDs remain distinct from scan filter IDs. ### Release note Add detailed OLAP scan filter profile counters in query profile. ### Check List (For Author) - Test: Manual test - ./build.sh --be - Ran local profile_level=2 scan and runtime-filter join queries on scan_filter_profile_db and checked generated profiles - Behavior changed: Yes. Query profile includes detailed OLAP ScanFilterInfo and KeyRangeInfo counters. - Does this need documentation: No
1 parent e5bf60e commit 5f54636

9 files changed

Lines changed: 467 additions & 139 deletions

be/src/exec/operator/olap_scan_operator.cpp

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,50 @@
5555

5656
namespace doris {
5757

58+
namespace {
59+
60+
constexpr int64_t MAX_PROFILE_KEY_RANGES = 32;
61+
62+
int64_t key_range_count(const std::vector<std::unique_ptr<OlapScanRange>>& ranges) {
63+
int64_t count = 0;
64+
for (const auto& range : ranges) {
65+
if (range->has_lower_bound) {
66+
++count;
67+
}
68+
}
69+
return count;
70+
}
71+
72+
std::string scan_keys_profile_string(const std::vector<std::unique_ptr<OlapScanRange>>& ranges) {
73+
fmt::memory_buffer scan_keys_buffer;
74+
int64_t range_count = 0;
75+
int64_t printed = 0;
76+
for (const auto& range : ranges) {
77+
if (!range->has_lower_bound) {
78+
continue;
79+
}
80+
if (printed < MAX_PROFILE_KEY_RANGES) {
81+
if (printed > 0) {
82+
fmt::format_to(scan_keys_buffer, "; ");
83+
}
84+
fmt::format_to(scan_keys_buffer, "{}{} : {}{}", range->begin_include ? "[" : "(",
85+
range->begin_scan_range.debug_string(),
86+
range->end_scan_range.debug_string(), range->end_include ? "]" : ")");
87+
++printed;
88+
}
89+
++range_count;
90+
}
91+
if (range_count > printed) {
92+
if (printed > 0) {
93+
fmt::format_to(scan_keys_buffer, "; ");
94+
}
95+
fmt::format_to(scan_keys_buffer, "... {} more", range_count - printed);
96+
}
97+
return fmt::to_string(scan_keys_buffer);
98+
}
99+
100+
} // namespace
101+
58102
Status OlapScanLocalState::init(RuntimeState* state, LocalStateInfo& info) {
59103
const TOlapScanNode& olap_scan_node = _parent->cast<OlapScanOperatorX>()._olap_scan_node;
60104

@@ -598,9 +642,9 @@ void OlapScanLocalState::_register_key_range_scan_filter() {
598642

599643
ScanFilterDesc desc;
600644
desc.kind = ScanFilterKind::KEY_RANGE;
601-
desc.compact_info = _scan_keys.debug_string();
602-
desc.debug_string = desc.compact_info;
645+
desc.compact_info = scan_keys_profile_string(_cond_ranges);
603646
desc.source_filter_ids = std::move(source_filter_ids);
647+
desc.range_count = key_range_count(_cond_ranges);
604648
_key_range_scan_filter = _scan_filter_profile->register_filter(std::move(desc));
605649
}
606650

@@ -1217,6 +1261,14 @@ Status OlapScanLocalState::_build_key_ranges_and_filters() {
12171261
for (const auto& it : _slot_id_to_predicates[*key_to_erase]) {
12181262
if (!can_erase_predicate(*it)) {
12191263
new_predicates.push_back(it);
1264+
} else if (_scan_filter_profile != nullptr) {
1265+
const auto& handle = it->scan_filter_handle();
1266+
DORIS_CHECK(handle);
1267+
auto& source_ids = _slot_id_to_scan_filter_ids_for_key_range[*key_to_erase];
1268+
if (std::find(source_ids.begin(), source_ids.end(), handle.filter_id) ==
1269+
source_ids.end()) {
1270+
source_ids.push_back(handle.filter_id);
1271+
}
12201272
}
12211273
}
12221274
if (new_predicates.empty()) {
@@ -1237,7 +1289,9 @@ Status OlapScanLocalState::_build_key_ranges_and_filters() {
12371289
}
12381290

12391291
if (state()->enable_profile()) {
1240-
custom_profile()->add_info_string("KeyRanges", _scan_keys.debug_string());
1292+
if (_scan_filter_profile == nullptr) {
1293+
custom_profile()->add_info_string("KeyRanges", _scan_keys.debug_string());
1294+
}
12411295
custom_profile()->add_info_string("TabletIds", tablets_id_to_string(_scan_ranges));
12421296
}
12431297
VLOG_CRITICAL << _scan_keys.debug_string();

be/src/exec/operator/scan_operator.cpp

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ Status ScanLocalState<Derived>::open(RuntimeState* state) {
287287

288288
RETURN_IF_ERROR(_process_conjuncts(state));
289289

290-
if (state->enable_profile()) {
290+
if (state->enable_profile() && _scan_filter_profile == nullptr) {
291291
custom_profile()->add_info_string("PushDownPredicates",
292292
predicates_to_string(_slot_id_to_predicates));
293293
}
@@ -408,7 +408,7 @@ Status ScanLocalState<Derived>::_normalize_conjuncts(RuntimeState* state) {
408408
++it;
409409
}
410410

411-
if (state->enable_profile()) {
411+
if (state->enable_profile() && _scan_filter_profile == nullptr) {
412412
std::string message;
413413
for (auto& conjunct : _conjuncts) {
414414
if (conjunct->root()) {
@@ -537,13 +537,6 @@ Status ScanLocalState<Derived>::_normalize_predicate(VExprContext* context, cons
537537
for (size_t i = predicates_before; i < slot_predicates.size(); ++i) {
538538
slot_predicates[i]->attach_scan_filter(handle);
539539
}
540-
if (_is_key_column(slot->col_name())) {
541-
auto& source_ids = _slot_id_to_scan_filter_ids_for_key_range[slot->id()];
542-
if (std::find(source_ids.begin(), source_ids.end(), handle.filter_id) ==
543-
source_ids.end()) {
544-
source_ids.push_back(handle.filter_id);
545-
}
546-
}
547540
}
548541
}
549542
}
@@ -1401,7 +1394,7 @@ Status ScanLocalState<Derived>::close(RuntimeState* state) {
14011394
std::list<std::shared_ptr<ScannerDelegate>> {}.swap(_scanners);
14021395
COUNTER_SET(_wait_for_dependency_timer, _scan_dependency->watcher_elapse_time());
14031396
COUNTER_SET(_wait_for_rf_timer, rf_time);
1404-
_helper.collect_realtime_profile(custom_profile());
1397+
_helper.collect_realtime_profile(custom_profile(), _scan_filter_profile.get());
14051398
if (_scan_filter_profile != nullptr) {
14061399
_scan_filter_profile->materialize(custom_profile(), state->profile_level());
14071400
}

be/src/exec/runtime_filter/runtime_filter_consumer.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "exprs/vbloom_predicate.h"
2424
#include "exprs/vdirect_in_predicate.h"
2525
#include "runtime/runtime_profile.h"
26+
#include "runtime/scan_filter_profile.h"
2627

2728
namespace doris {
2829
Status RuntimeFilterConsumer::_apply_ready_expr(std::vector<RuntimeFilterExprPtr>& push_exprs) {
@@ -258,4 +259,14 @@ void RuntimeFilterConsumer::collect_realtime_profile(RuntimeProfile* parent_oper
258259
c->update(_always_true_counter->value());
259260
}
260261

262+
void RuntimeFilterConsumer::collect_scan_filter_profile(ScanFilterProfile* scan_filter_profile) {
263+
std::unique_lock<std::recursive_mutex> l(_rmtx);
264+
DCHECK(scan_filter_profile != nullptr);
265+
scan_filter_profile->set_runtime_filter_profile_stats(
266+
{.runtime_filter_id = _wrapper->filter_id(),
267+
.wait_time_ns = _wait_timer->value(),
268+
.always_true_filter_rows = _always_true_counter->value(),
269+
.debug_string = debug_string()});
270+
}
271+
261272
} // namespace doris

be/src/exec/runtime_filter/runtime_filter_consumer.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
#include "runtime/runtime_profile.h"
2929

3030
namespace doris {
31+
class ScanFilterProfile;
32+
3133
// Work on ScanNode or MultiCastDataStreamSource, RuntimeFilterConsumerHelper will manage all RuntimeFilterConsumer
3234
// Used to create RuntimeFilterExpr to filter data
3335
class RuntimeFilterConsumer : public RuntimeFilter {
@@ -68,6 +70,7 @@ class RuntimeFilterConsumer : public RuntimeFilter {
6870

6971
// Called by RuntimeFilterConsumerHelper
7072
void collect_realtime_profile(RuntimeProfile* parent_operator_profile);
73+
void collect_scan_filter_profile(ScanFilterProfile* scan_filter_profile);
7174

7275
static std::string to_string(const State& state) {
7376
switch (state) {

be/src/exec/runtime_filter/runtime_filter_consumer_helper.cpp

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include "exec/runtime_filter/runtime_filter_consumer.h"
2121
#include "runtime/runtime_profile.h"
22+
#include "runtime/scan_filter_profile.h"
2223

2324
namespace doris {
2425
RuntimeFilterConsumerHelper::RuntimeFilterConsumerHelper(
@@ -129,8 +130,21 @@ Status RuntimeFilterConsumerHelper::try_append_late_arrival_runtime_filter(
129130
return Status::OK();
130131
}
131132

132-
void RuntimeFilterConsumerHelper::collect_realtime_profile(
133-
RuntimeProfile* parent_operator_profile) {
133+
void RuntimeFilterConsumerHelper::collect_realtime_profile(RuntimeProfile* parent_operator_profile,
134+
ScanFilterProfile* scan_filter_profile) {
135+
if (_consumers.empty()) {
136+
return;
137+
}
138+
139+
if (scan_filter_profile != nullptr) {
140+
scan_filter_profile->set_runtime_filter_acquire_time(
141+
_acquire_runtime_filter_timer->value());
142+
for (const auto& consumer : _consumers) {
143+
consumer->collect_scan_filter_profile(scan_filter_profile);
144+
}
145+
return;
146+
}
147+
134148
std::ignore = parent_operator_profile->add_counter("RuntimeFilterInfo", TUnit::NONE,
135149
RuntimeProfile::ROOT_COUNTER, 1);
136150
RuntimeProfile::Counter* c = parent_operator_profile->add_counter(
@@ -142,4 +156,4 @@ void RuntimeFilterConsumerHelper::collect_realtime_profile(
142156
}
143157
}
144158

145-
} // namespace doris
159+
} // namespace doris

be/src/exec/runtime_filter/runtime_filter_consumer_helper.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
#include "runtime/runtime_profile.h"
2525

2626
namespace doris {
27+
class ScanFilterProfile;
28+
2729
// this class used in ScanNode or MultiCastDataStreamSource
2830
/**
2931
* init -> acquire_runtime_filter -> try_append_late_arrival_runtime_filter
@@ -49,7 +51,8 @@ class RuntimeFilterConsumerHelper {
4951

5052
// Called by XXXLocalState::close()
5153
// parent_operator_profile is owned by LocalState so update it is safe at here.
52-
void collect_realtime_profile(RuntimeProfile* parent_operator_profile);
54+
void collect_realtime_profile(RuntimeProfile* parent_operator_profile,
55+
ScanFilterProfile* scan_filter_profile = nullptr);
5356

5457
size_t runtime_filter_nums() const { return _runtime_filter_descs.size(); }
5558

@@ -71,4 +74,4 @@ class RuntimeFilterConsumerHelper {
7174
std::unique_ptr<RuntimeProfile::Counter> _acquire_runtime_filter_timer =
7275
std::make_unique<RuntimeProfile::Counter>(TUnit::TIME_NS, 0);
7376
};
74-
} // namespace doris
77+
} // namespace doris

0 commit comments

Comments
 (0)