Skip to content

Commit 7a104c4

Browse files
martincostellomartincostello
authored
Extend OpenMetrics exporter to include allocations (#3138)
* Extend OpenMetrics exporter to include allocations Extend the OpenMetrics exporter to include a gauge for allocated bytes if the memory diagnoser is enabled. * Address feedback - Emit metrics with `NaN` when the diagnoser is not enabled. - Add test for when disabled. * Update snapshots Update snapshots with allocation gauges. * Only emit GC metrics if memory diagnoser enabled Make GC metrics conditional on the memory diagnoser being enabled.
1 parent 643fd81 commit 7a104c4

9 files changed

Lines changed: 250 additions & 85 deletions

‎src/BenchmarkDotNet/Exporters/OpenMetrics/OpenMetricsExporter.cs‎

Lines changed: 65 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using BenchmarkDotNet.Diagnosers;
12
using BenchmarkDotNet.Engines;
23
using BenchmarkDotNet.Extensions;
34
using BenchmarkDotNet.Helpers;
@@ -29,20 +30,28 @@ public override async ValueTask ExportAsync(Summary summary, CancelableStreamWri
2930
var gcStats = report.GcStats;
3031
var descriptor = benchmark.Descriptor;
3132
var parameters = benchmark.Parameters;
33+
bool hasMemoryDiagnoser = benchmark.Config.HasMemoryDiagnoser();
34+
double? allocatedBytesPerOperation = hasMemoryDiagnoser
35+
? gcStats.GetBytesAllocatedPerOperation(benchmark)
36+
: null;
37+
if (hasMemoryDiagnoser && !allocatedBytesPerOperation.HasValue)
38+
{
39+
allocatedBytesPerOperation = double.NaN;
40+
}
3241

3342
var stats = report.ResultStatistics;
3443
var metrics = report.Metrics;
3544
if (stats == null)
3645
continue;
3746

38-
AddCommonMetrics(metricsSet, descriptor, parameters, stats, gcStats);
39-
AddAdditionalMetrics(metricsSet, metrics, descriptor, parameters);
47+
AddCommonMetrics(metricsSet, descriptor, parameters, stats, gcStats, allocatedBytesPerOperation);
48+
AddAdditionalMetrics(metricsSet, metrics, descriptor, parameters, skipAllocatedMemoryMetric: hasMemoryDiagnoser, reserveMemoryMetricNames: hasMemoryDiagnoser);
4049
}
4150

4251
await WriteMetricsAsync(writer, metricsSet, cancellationToken).ConfigureAwait(false);
4352
}
4453

45-
private static void AddCommonMetrics(HashSet<OpenMetric> metricsSet, Descriptor descriptor, ParameterInstances parameters, Statistics stats, GcStats gcStats)
54+
private static void AddCommonMetrics(HashSet<OpenMetric> metricsSet, Descriptor descriptor, ParameterInstances parameters, Statistics stats, GcStats gcStats, double? allocatedBytesPerOperation)
4655
{
4756
metricsSet.AddRange([
4857
// Mean
@@ -72,80 +81,104 @@ private static void AddCommonMetrics(HashSet<OpenMetric> metricsSet, Descriptor
7281
descriptor,
7382
parameters,
7483
stats.StandardDeviation),
75-
// GC Stats Gen0 - these are counters, not gauges
84+
// P90 - in nanoseconds
85+
OpenMetric.FromStatistics(
86+
$"{MetricPrefix}p90_nanoseconds",
87+
"90th percentile execution time in nanoseconds.",
88+
"gauge",
89+
"nanoseconds",
90+
descriptor,
91+
parameters,
92+
stats.Percentiles.P90),
93+
// P95 - in nanoseconds
7694
OpenMetric.FromStatistics(
95+
$"{MetricPrefix}p95_nanoseconds",
96+
"95th percentile execution time in nanoseconds.",
97+
"gauge",
98+
"nanoseconds",
99+
descriptor,
100+
parameters,
101+
stats.Percentiles.P95)
102+
]);
103+
104+
if (allocatedBytesPerOperation.HasValue)
105+
{
106+
// GC Stats Gen0 - these are counters, not gauges
107+
metricsSet.Add(OpenMetric.FromStatistics(
77108
$"{MetricPrefix}gc_gen0_collections_total",
78109
"Total number of Gen 0 garbage collections during the benchmark execution.",
79110
"counter",
80111
"",
81112
descriptor,
82113
parameters,
83-
gcStats.Gen0Collections),
114+
gcStats.Gen0Collections));
115+
84116
// GC Stats Gen1
85-
OpenMetric.FromStatistics(
117+
metricsSet.Add(OpenMetric.FromStatistics(
86118
$"{MetricPrefix}gc_gen1_collections_total",
87119
"Total number of Gen 1 garbage collections during the benchmark execution.",
88120
"counter",
89121
"",
90122
descriptor,
91123
parameters,
92-
gcStats.Gen1Collections),
124+
gcStats.Gen1Collections));
125+
93126
// GC Stats Gen2
94-
OpenMetric.FromStatistics(
127+
metricsSet.Add(OpenMetric.FromStatistics(
95128
$"{MetricPrefix}gc_gen2_collections_total",
96129
"Total number of Gen 2 garbage collections during the benchmark execution.",
97130
"counter",
98131
"",
99132
descriptor,
100133
parameters,
101-
gcStats.Gen2Collections),
134+
gcStats.Gen2Collections));
135+
102136
// Total GC Operations
103-
OpenMetric.FromStatistics(
137+
metricsSet.Add(OpenMetric.FromStatistics(
104138
$"{MetricPrefix}gc_total_operations_total",
105139
"Total number of garbage collection operations during the benchmark execution.",
106140
"counter",
107141
"",
108142
descriptor,
109143
parameters,
110-
gcStats.TotalOperations),
111-
// P90 - in nanoseconds
112-
OpenMetric.FromStatistics(
113-
$"{MetricPrefix}p90_nanoseconds",
114-
"90th percentile execution time in nanoseconds.",
115-
"gauge",
116-
"nanoseconds",
117-
descriptor,
118-
parameters,
119-
stats.Percentiles.P90),
120-
// P95 - in nanoseconds
121-
OpenMetric.FromStatistics(
122-
$"{MetricPrefix}p95_nanoseconds",
123-
"95th percentile execution time in nanoseconds.",
144+
gcStats.TotalOperations));
145+
146+
metricsSet.Add(OpenMetric.FromStatistics(
147+
$"{MetricPrefix}allocated_bytes",
148+
"Allocated managed memory per single benchmark operation.",
124149
"gauge",
125-
"nanoseconds",
150+
"bytes",
126151
descriptor,
127152
parameters,
128-
stats.Percentiles.P95)
129-
]);
153+
allocatedBytesPerOperation.Value));
154+
}
130155
}
131156

132-
private static void AddAdditionalMetrics(HashSet<OpenMetric> metricsSet, IReadOnlyDictionary<string, Metric> metrics, Descriptor descriptor, ParameterInstances parameters)
157+
private static void AddAdditionalMetrics(HashSet<OpenMetric> metricsSet, IReadOnlyDictionary<string, Metric> metrics, Descriptor descriptor, ParameterInstances parameters, bool skipAllocatedMemoryMetric, bool reserveMemoryMetricNames)
133158
{
134159
var reservedMetricNames = new HashSet<string>
135160
{
136161
$"{MetricPrefix}execution_time_nanoseconds",
137162
$"{MetricPrefix}error_nanoseconds",
138163
$"{MetricPrefix}stddev_nanoseconds",
139-
$"{MetricPrefix}gc_gen0_collections_total",
140-
$"{MetricPrefix}gc_gen1_collections_total",
141-
$"{MetricPrefix}gc_gen2_collections_total",
142-
$"{MetricPrefix}gc_total_operations_total",
143164
$"{MetricPrefix}p90_nanoseconds",
144165
$"{MetricPrefix}p95_nanoseconds"
145166
};
146167

168+
if (reserveMemoryMetricNames)
169+
{
170+
reservedMetricNames.Add($"{MetricPrefix}gc_gen0_collections_total");
171+
reservedMetricNames.Add($"{MetricPrefix}gc_gen1_collections_total");
172+
reservedMetricNames.Add($"{MetricPrefix}gc_gen2_collections_total");
173+
reservedMetricNames.Add($"{MetricPrefix}gc_total_operations_total");
174+
reservedMetricNames.Add($"{MetricPrefix}allocated_bytes");
175+
}
176+
147177
foreach (var metric in metrics)
148178
{
179+
if (skipAllocatedMemoryMetric && metric.Value.Descriptor is AllocatedMemoryMetricDescriptor)
180+
continue;
181+
149182
string metricName = SanitizeMetricName(metric.Key);
150183
string fullMetricName = $"{MetricPrefix}{metricName}";
151184

@@ -246,7 +279,6 @@ public static OpenMetric FromMetric(string fullMetricName, KeyValuePair<string,
246279
return new OpenMetric(fullMetricName, help, type, "", labels, metric.Value.Value);
247280
}
248281

249-
private static readonly Dictionary<string, string> NormalizedLabelKeyCache = [];
250282
private static string NormalizeLabelKey(string key)
251283
{
252284
string normalized = new(key

‎tests/BenchmarkDotNet.Tests/Exporters/OpenMetricsExporterTests.cs‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using BenchmarkDotNet.Configs;
2+
using BenchmarkDotNet.Diagnosers;
23
using BenchmarkDotNet.Engines;
34
using BenchmarkDotNet.Environments;
45
using BenchmarkDotNet.Exporters;
@@ -279,5 +280,134 @@ public async Task DecimalSeparator_UsesInvariantCulture()
279280

280281
return;
281282
}
283+
284+
[Fact]
285+
public async Task MemoryDiagnoser_ExportsAllocatedBytesPerOperation()
286+
{
287+
var config = new ManualConfig().AddDiagnoser(MemoryDiagnoser.Default);
288+
var summary = new Summary(
289+
"MemoryDiagnoserSummary",
290+
[
291+
new BenchmarkReport(
292+
success: true,
293+
benchmarkCase: new BenchmarkCase(
294+
new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo),
295+
Job.Dry,
296+
new ParameterInstances([]),
297+
ImmutableConfigBuilder.Create(config)),
298+
null!,
299+
null!,
300+
[
301+
new ExecuteResult(
302+
[
303+
new Measurement(0, IterationMode.Workload, IterationStage.Result, 4, 4, 40)
304+
],
305+
GcStats.Parse("// GC: 1 2 3 65536 4"))
306+
],
307+
[
308+
new Metric(AllocatedMemoryMetricDescriptor.Instance, 16384),
309+
new(new FakeMetricDescriptor("label", "label"), 42.0)
310+
])
311+
],
312+
HostEnvironmentInfo.GetCurrent(),
313+
"",
314+
"",
315+
TimeSpan.Zero,
316+
CultureInfo.InvariantCulture,
317+
[],
318+
[]);
319+
320+
var logger = new AccumulationLogger();
321+
322+
await ((ExporterBase)OpenMetricsExporter.Default).ExportToLogAsync(summary, logger, CancellationToken.None);
323+
324+
var settings = VerifyHelper.Create();
325+
await Verifier.Verify(logger.GetLog(), settings);
326+
}
327+
328+
[Fact]
329+
public async Task MemoryDiagnoser_WithoutAllocationData_ExportsAllocatedBytesAsNaN()
330+
{
331+
var config = new ManualConfig().AddDiagnoser(MemoryDiagnoser.Default);
332+
var summary = new Summary(
333+
"MemoryDiagnoserNoAllocationDataSummary",
334+
[
335+
new BenchmarkReport(
336+
success: true,
337+
benchmarkCase: new BenchmarkCase(
338+
new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo),
339+
Job.Dry,
340+
new ParameterInstances([]),
341+
ImmutableConfigBuilder.Create(config)),
342+
null!,
343+
null!,
344+
[
345+
new ExecuteResult(
346+
[
347+
new Measurement(0, IterationMode.Workload, IterationStage.Result, 1, 1, 10)
348+
],
349+
GcStats.Empty)
350+
],
351+
[
352+
new Metric(AllocatedMemoryMetricDescriptor.Instance, double.NaN)
353+
])
354+
],
355+
HostEnvironmentInfo.GetCurrent(),
356+
"",
357+
"",
358+
TimeSpan.Zero,
359+
CultureInfo.InvariantCulture,
360+
[],
361+
[]);
362+
363+
var logger = new AccumulationLogger();
364+
365+
await ((ExporterBase)OpenMetricsExporter.Default).ExportToLogAsync(summary, logger, CancellationToken.None);
366+
367+
var log = logger.GetLog();
368+
Assert.Contains("# HELP benchmark_allocated_bytes Allocated managed memory per single benchmark operation.", log);
369+
Assert.Contains("benchmark_allocated_bytes{method=\"Foo\", type=\"MockBenchmarkClass\"} NaN", log);
370+
}
371+
372+
[Fact]
373+
public async Task WithoutMemoryDiagnoser_CustomAllocatedBytesMetricIsNotSuppressed()
374+
{
375+
var summary = new Summary(
376+
"CustomAllocatedBytesMetricSummary",
377+
[
378+
new BenchmarkReport(
379+
success: true,
380+
benchmarkCase: new BenchmarkCase(
381+
new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo),
382+
Job.Dry,
383+
new ParameterInstances([]),
384+
ImmutableConfigBuilder.Create(new ManualConfig())),
385+
null!,
386+
null!,
387+
[
388+
new ExecuteResult([
389+
new Measurement(0, IterationMode.Workload, IterationStage.Result, 1, 1, 10)
390+
])
391+
],
392+
[
393+
new(new FakeMetricDescriptor("allocated_bytes", "allocated bytes"), 42.0)
394+
])
395+
],
396+
HostEnvironmentInfo.GetCurrent(),
397+
"",
398+
"",
399+
TimeSpan.Zero,
400+
CultureInfo.InvariantCulture,
401+
[],
402+
[]);
403+
404+
var logger = new AccumulationLogger();
405+
406+
await ((ExporterBase)OpenMetricsExporter.Default).ExportToLogAsync(summary, logger, CancellationToken.None);
407+
408+
var log = logger.GetLog();
409+
Assert.Contains("# HELP benchmark_allocated_bytes Additional metric allocated_bytes", log);
410+
Assert.Contains("benchmark_allocated_bytes{method=\"Foo\", type=\"MockBenchmarkClass\"} 42", log);
411+
}
282412
}
283413
}

‎tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_Invariant.verified.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,11 @@ MarkdownExporter-stackoverflow
596596
############################################
597597
OpenMetricsExporter
598598
############################################
599+
# HELP benchmark_allocated_bytes Allocated managed memory per single benchmark operation.
600+
# TYPE benchmark_allocated_bytes gauge
601+
# UNIT benchmark_allocated_bytes bytes
602+
benchmark_allocated_bytes{method="Foo", type="MockBenchmarkClass"} NaN
603+
benchmark_allocated_bytes{method="Bar", type="MockBenchmarkClass"} NaN
599604
# HELP benchmark_cachemisses Additional metric CacheMisses
600605
# TYPE benchmark_cachemisses gauge
601606
benchmark_cachemisses{method="Foo", type="MockBenchmarkClass"} 7

‎tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_en-US.verified.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,11 @@ MarkdownExporter-stackoverflow
596596
############################################
597597
OpenMetricsExporter
598598
############################################
599+
# HELP benchmark_allocated_bytes Allocated managed memory per single benchmark operation.
600+
# TYPE benchmark_allocated_bytes gauge
601+
# UNIT benchmark_allocated_bytes bytes
602+
benchmark_allocated_bytes{method="Foo", type="MockBenchmarkClass"} NaN
603+
benchmark_allocated_bytes{method="Bar", type="MockBenchmarkClass"} NaN
599604
# HELP benchmark_cachemisses Additional metric CacheMisses
600605
# TYPE benchmark_cachemisses gauge
601606
benchmark_cachemisses{method="Foo", type="MockBenchmarkClass"} 7

‎tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/CommonExporterVerifyTests.Exporters_ru-RU.verified.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,11 @@ MarkdownExporter-stackoverflow
596596
############################################
597597
OpenMetricsExporter
598598
############################################
599+
# HELP benchmark_allocated_bytes Allocated managed memory per single benchmark operation.
600+
# TYPE benchmark_allocated_bytes gauge
601+
# UNIT benchmark_allocated_bytes bytes
602+
benchmark_allocated_bytes{method="Foo", type="MockBenchmarkClass"} NaN
603+
benchmark_allocated_bytes{method="Bar", type="MockBenchmarkClass"} NaN
599604
# HELP benchmark_cachemisses Additional metric CacheMisses
600605
# TYPE benchmark_cachemisses gauge
601606
benchmark_cachemisses{method="Foo", type="MockBenchmarkClass"} 7

‎tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/OpenMetricsExporterTests.LabelsAreEscapedCorrectly.verified.txt‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,6 @@ benchmark_error_nanoseconds{method="Foo", type="MockBenchmarkClass"} 0
66
# TYPE benchmark_execution_time_nanoseconds gauge
77
# UNIT benchmark_execution_time_nanoseconds nanoseconds
88
benchmark_execution_time_nanoseconds{method="Foo", type="MockBenchmarkClass"} 0.1
9-
# HELP benchmark_gc_gen0_collections_total Total number of Gen 0 garbage collections during the benchmark execution.
10-
# TYPE benchmark_gc_gen0_collections_total counter
11-
benchmark_gc_gen0_collections_total{method="Foo", type="MockBenchmarkClass"} 0
12-
# HELP benchmark_gc_gen1_collections_total Total number of Gen 1 garbage collections during the benchmark execution.
13-
# TYPE benchmark_gc_gen1_collections_total counter
14-
benchmark_gc_gen1_collections_total{method="Foo", type="MockBenchmarkClass"} 0
15-
# HELP benchmark_gc_gen2_collections_total Total number of Gen 2 garbage collections during the benchmark execution.
16-
# TYPE benchmark_gc_gen2_collections_total counter
17-
benchmark_gc_gen2_collections_total{method="Foo", type="MockBenchmarkClass"} 0
18-
# HELP benchmark_gc_total_operations_total Total number of garbage collection operations during the benchmark execution.
19-
# TYPE benchmark_gc_total_operations_total counter
20-
benchmark_gc_total_operations_total{method="Foo", type="MockBenchmarkClass"} 0
219
# HELP benchmark_label_with_dash Additional metric label_with-dash
2210
# TYPE benchmark_label_with_dash gauge
2311
benchmark_label_with_dash{method="Foo", type="MockBenchmarkClass"} 84

0 commit comments

Comments
 (0)