Skip to content

Commit b382ecd

Browse files
authored
Optimize metric label cloning (apache#22406)
## Which issue does this PR close? - Part of apache#22189. ## Rationale for this change `ParquetFileMetrics::new` registers many per-file metrics with the same `filename` label. Before this PR, each metric built its own owned filename label with `filename.to_string()`, which repeatedly copied the same dynamic string during parquet scan setup. This PR keeps parquet metrics eagerly registered, so `ExecutionPlan::metrics()` visibility during execution is unchanged, while reducing repeated label string allocation and copying. ## What changes are included in this PR? - Store owned `Label` name/value strings behind `Arc<str>` internally, while keeping borrowed static label strings allocation-free. - Reuse one cloned `filename` label across the per-file parquet metrics in `ParquetFileMetrics::new`. - Add a metrics test confirming borrowed and owned label values remain equal and display the same way. ## Are these changes tested? Yes. ## Are there any user-facing changes? No. Metric registration timing and displayed label values are unchanged.
1 parent 936844a commit b382ecd

3 files changed

Lines changed: 156 additions & 49 deletions

File tree

datafusion/datasource-parquet/src/metrics.rs

Lines changed: 42 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
use std::sync::Arc;
19+
1820
use datafusion_physical_plan::metrics::{
19-
Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricCategory, MetricType,
20-
PruningMetrics, RatioMergeStrategy, RatioMetrics, Time,
21+
Count, ExecutionPlanMetricsSet, Gauge, Label, MetricBuilder, MetricCategory,
22+
MetricType, PruningMetrics, RatioMergeStrategy, RatioMetrics, Time,
2123
};
2224

2325
/// Stores metrics about the parquet execution for a particular parquet file.
@@ -100,46 +102,51 @@ impl ParquetFileMetrics {
100102
filename: &str,
101103
metrics: &ExecutionPlanMetricsSet,
102104
) -> Self {
105+
// Share the filename label across all per-file metrics to avoid
106+
// allocating the same filename string for each metric.
107+
let filename_label = Label::new("filename", Arc::<str>::from(filename));
108+
let builder = MetricBuilder::new(metrics).with_label(filename_label);
109+
103110
// -----------------------
104111
// 'summary' level metrics
105112
// -----------------------
106-
let row_groups_pruned_bloom_filter = MetricBuilder::new(metrics)
107-
.with_new_label("filename", filename.to_string())
113+
let row_groups_pruned_bloom_filter = builder
114+
.clone()
108115
.with_type(MetricType::Summary)
109116
.pruning_metrics("row_groups_pruned_bloom_filter", partition);
110117

111-
let limit_pruned_row_groups = MetricBuilder::new(metrics)
112-
.with_new_label("filename", filename.to_string())
118+
let limit_pruned_row_groups = builder
119+
.clone()
113120
.with_type(MetricType::Summary)
114121
.pruning_metrics("limit_pruned_row_groups", partition);
115122

116-
let row_groups_pruned_statistics = MetricBuilder::new(metrics)
117-
.with_new_label("filename", filename.to_string())
123+
let row_groups_pruned_statistics = builder
124+
.clone()
118125
.with_type(MetricType::Summary)
119126
.pruning_metrics("row_groups_pruned_statistics", partition);
120127

121-
let page_index_pages_pruned = MetricBuilder::new(metrics)
122-
.with_new_label("filename", filename.to_string())
128+
let page_index_pages_pruned = builder
129+
.clone()
123130
.with_type(MetricType::Summary)
124131
.pruning_metrics("page_index_pages_pruned", partition);
125132

126-
let bytes_scanned = MetricBuilder::new(metrics)
127-
.with_new_label("filename", filename.to_string())
133+
let bytes_scanned = builder
134+
.clone()
128135
.with_type(MetricType::Summary)
129136
.with_category(MetricCategory::Bytes)
130137
.counter("bytes_scanned", partition);
131138

132-
let metadata_load_time = MetricBuilder::new(metrics)
133-
.with_new_label("filename", filename.to_string())
139+
let metadata_load_time = builder
140+
.clone()
134141
.with_type(MetricType::Summary)
135142
.subset_time("metadata_load_time", partition);
136143

137144
let files_ranges_pruned_statistics = MetricBuilder::new(metrics)
138145
.with_type(MetricType::Summary)
139146
.pruning_metrics("files_ranges_pruned_statistics", partition);
140147

141-
let scan_efficiency_ratio = MetricBuilder::new(metrics)
142-
.with_new_label("filename", filename.to_string())
148+
let scan_efficiency_ratio = builder
149+
.clone()
143150
.with_type(MetricType::Summary)
144151
.ratio_metrics_with_strategy(
145152
"scan_efficiency_ratio",
@@ -150,45 +157,44 @@ impl ParquetFileMetrics {
150157
// -----------------------
151158
// 'dev' level metrics
152159
// -----------------------
153-
let predicate_evaluation_errors = MetricBuilder::new(metrics)
154-
.with_new_label("filename", filename.to_string())
160+
let predicate_evaluation_errors = builder
161+
.clone()
155162
.with_category(MetricCategory::Rows)
156163
.counter("predicate_evaluation_errors", partition);
157164

158-
let pushdown_rows_pruned = MetricBuilder::new(metrics)
159-
.with_new_label("filename", filename.to_string())
165+
let pushdown_rows_pruned = builder
166+
.clone()
160167
.with_category(MetricCategory::Rows)
161168
.counter("pushdown_rows_pruned", partition);
162-
let pushdown_rows_matched = MetricBuilder::new(metrics)
163-
.with_new_label("filename", filename.to_string())
169+
let pushdown_rows_matched = builder
170+
.clone()
164171
.with_category(MetricCategory::Rows)
165172
.counter("pushdown_rows_matched", partition);
166173

167-
let row_pushdown_eval_time = MetricBuilder::new(metrics)
168-
.with_new_label("filename", filename.to_string())
174+
let row_pushdown_eval_time = builder
175+
.clone()
169176
.subset_time("row_pushdown_eval_time", partition);
170-
let statistics_eval_time = MetricBuilder::new(metrics)
171-
.with_new_label("filename", filename.to_string())
177+
let statistics_eval_time = builder
178+
.clone()
172179
.subset_time("statistics_eval_time", partition);
173-
let bloom_filter_eval_time = MetricBuilder::new(metrics)
174-
.with_new_label("filename", filename.to_string())
180+
let bloom_filter_eval_time = builder
181+
.clone()
175182
.subset_time("bloom_filter_eval_time", partition);
176183

177-
let page_index_eval_time = MetricBuilder::new(metrics)
178-
.with_new_label("filename", filename.to_string())
184+
let page_index_eval_time = builder
185+
.clone()
179186
.subset_time("page_index_eval_time", partition);
180187

181-
let page_index_rows_pruned = MetricBuilder::new(metrics)
182-
.with_new_label("filename", filename.to_string())
188+
let page_index_rows_pruned = builder
189+
.clone()
183190
.pruning_metrics("page_index_rows_pruned", partition);
184191

185-
let predicate_cache_inner_records = MetricBuilder::new(metrics)
186-
.with_new_label("filename", filename.to_string())
192+
let predicate_cache_inner_records = builder
193+
.clone()
187194
.with_category(MetricCategory::Rows)
188195
.gauge("predicate_cache_inner_records", partition);
189196

190-
let predicate_cache_records = MetricBuilder::new(metrics)
191-
.with_new_label("filename", filename.to_string())
197+
let predicate_cache_records = builder
192198
.with_category(MetricCategory::Rows)
193199
.gauge("predicate_cache_records", partition);
194200

datafusion/physical-expr-common/src/metrics/builder.rs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@ use crate::metrics::{
2525
};
2626

2727
use super::{
28-
Count, ExecutionPlanMetricsSet, Gauge, Label, Metric, MetricValue, Time, Timestamp,
28+
Count, ExecutionPlanMetricsSet, Gauge, Label, LabelValue, Metric, MetricValue, Time,
29+
Timestamp,
2930
};
3031

3132
/// Structure for constructing metrics, counters, timers, etc.
3233
///
3334
/// Note the use of `Cow<..>` is to avoid allocations in the common
34-
/// case of constant strings
35+
/// case of constant strings. Dynamically created label strings are shared when
36+
/// [`Label`] values are cloned.
3537
///
3638
/// ```rust
3739
/// use datafusion_physical_expr_common::metrics::*;
@@ -47,6 +49,7 @@ use super::{
4749
/// .with_new_label("filename", "my_awesome_file.parquet")
4850
/// .counter("num_bytes", partition);
4951
/// ```
52+
#[derive(Clone)]
5053
pub struct MetricBuilder<'a> {
5154
/// Location that the metric created by this builder will be added do
5255
metrics: &'a ExecutionPlanMetricsSet,
@@ -108,7 +111,10 @@ impl<'a> MetricBuilder<'a> {
108111
name: impl Into<Cow<'static, str>>,
109112
value: impl Into<Cow<'static, str>>,
110113
) -> Self {
111-
self.with_label(Label::new(name.into(), value.into()))
114+
self.with_label(Label::new(
115+
LabelValue::from(name.into()),
116+
LabelValue::from(value.into()),
117+
))
112118
}
113119

114120
/// Set the partition of the metric being constructed

datafusion/physical-expr-common/src/metrics/mod.rs

Lines changed: 105 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ use parking_lot::Mutex;
3030
use std::{
3131
borrow::Cow,
3232
fmt::{self, Debug, Display},
33+
hash::{Hash, Hasher},
3334
sync::Arc,
3435
vec::IntoIter,
3536
};
@@ -519,33 +520,32 @@ impl From<MetricsSet> for ExecutionPlanMetricsSet {
519520
/// telemetry]<https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md>,
520521
/// etc.
521522
///
522-
/// As the name and value are expected to mostly be constant strings,
523-
/// use a [`Cow`] to avoid copying / allocations in this common case.
523+
/// As the name and value are expected to often be constant strings, borrowed
524+
/// static strings avoid allocations in that common case. Dynamic strings are
525+
/// stored behind [`Arc<str>`] so cloning labels does not copy the underlying
526+
/// string data.
524527
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
525528
pub struct Label {
526-
name: Cow<'static, str>,
527-
value: Cow<'static, str>,
529+
name: LabelValue,
530+
value: LabelValue,
528531
}
529532

530533
impl Label {
531534
/// Create a new [`Label`]
532-
pub fn new(
533-
name: impl Into<Cow<'static, str>>,
534-
value: impl Into<Cow<'static, str>>,
535-
) -> Self {
535+
pub fn new(name: impl Into<LabelValue>, value: impl Into<LabelValue>) -> Self {
536536
let name = name.into();
537537
let value = value.into();
538538
Self { name, value }
539539
}
540540

541541
/// Returns the name of this label
542542
pub fn name(&self) -> &str {
543-
self.name.as_ref()
543+
self.name.as_str()
544544
}
545545

546546
/// Returns the value of this label
547547
pub fn value(&self) -> &str {
548-
self.value.as_ref()
548+
self.value.as_str()
549549
}
550550
}
551551

@@ -555,6 +555,89 @@ impl Display for Label {
555555
}
556556
}
557557

558+
/// A label name or value.
559+
///
560+
/// String literals preserve the existing allocation-free path. Dynamic strings
561+
/// can be stored behind [`Arc<str>`], so cloning a [`Label`] only increments an
562+
/// atomic reference count and does not allocate or copy the underlying string
563+
/// data.
564+
#[derive(Clone)]
565+
pub struct LabelValue(LabelValueInner);
566+
567+
/// Internal representation for label names and values.
568+
///
569+
/// `LabelValue` is public because `Label::new` accepts it, but these storage
570+
/// variants are implementation details. Keeping them private prevents external
571+
/// code from constructing or matching on `Static` and `Shared` directly.
572+
#[derive(Clone)]
573+
enum LabelValueInner {
574+
Static(&'static str),
575+
Shared(Arc<str>),
576+
}
577+
578+
impl LabelValue {
579+
/// Return this label value as a string slice.
580+
pub fn as_str(&self) -> &str {
581+
match &self.0 {
582+
LabelValueInner::Static(value) => value,
583+
LabelValueInner::Shared(value) => value.as_ref(),
584+
}
585+
}
586+
}
587+
588+
impl From<&'static str> for LabelValue {
589+
fn from(value: &'static str) -> Self {
590+
Self(LabelValueInner::Static(value))
591+
}
592+
}
593+
594+
impl From<String> for LabelValue {
595+
fn from(value: String) -> Self {
596+
Self(LabelValueInner::Shared(Arc::from(value)))
597+
}
598+
}
599+
600+
impl From<Arc<str>> for LabelValue {
601+
fn from(value: Arc<str>) -> Self {
602+
Self(LabelValueInner::Shared(value))
603+
}
604+
}
605+
606+
impl From<Cow<'static, str>> for LabelValue {
607+
fn from(value: Cow<'static, str>) -> Self {
608+
match value {
609+
Cow::Borrowed(value) => value.into(),
610+
Cow::Owned(value) => value.into(),
611+
}
612+
}
613+
}
614+
615+
impl PartialEq for LabelValue {
616+
fn eq(&self, other: &Self) -> bool {
617+
self.as_str() == other.as_str()
618+
}
619+
}
620+
621+
impl Eq for LabelValue {}
622+
623+
impl Hash for LabelValue {
624+
fn hash<H: Hasher>(&self, state: &mut H) {
625+
self.as_str().hash(state);
626+
}
627+
}
628+
629+
impl Debug for LabelValue {
630+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
631+
Debug::fmt(self.as_str(), f)
632+
}
633+
}
634+
635+
impl Display for LabelValue {
636+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
637+
Display::fmt(self.as_str(), f)
638+
}
639+
}
640+
558641
#[cfg(test)]
559642
mod tests {
560643
use std::time::Duration;
@@ -609,6 +692,18 @@ mod tests {
609692
assert_eq!("output_rows{partition=2, foo=bar}=66", metric.to_string())
610693
}
611694

695+
#[test]
696+
fn test_label_owned_and_borrowed_values_are_equal() {
697+
let borrowed = Label::new("foo", "bar");
698+
let owned = Label::new("foo".to_string(), "bar".to_string());
699+
let shared = Label::new("foo", Arc::<str>::from("bar"));
700+
701+
assert_eq!(borrowed, owned);
702+
assert_eq!(borrowed, shared);
703+
assert_eq!(borrowed.to_string(), owned.to_string());
704+
assert_eq!(borrowed.to_string(), shared.to_string());
705+
}
706+
612707
#[test]
613708
fn test_output_rows() {
614709
let metrics = ExecutionPlanMetricsSet::new();

0 commit comments

Comments
 (0)