Skip to content

Commit bba18eb

Browse files
charlesmyucharlesmyu
committed
Send summed SQL plan metric values
1 parent 3128df8 commit bba18eb

3 files changed

Lines changed: 177 additions & 1 deletion

File tree

‎dd-java-agent/instrumentation/spark/spark-common/src/main/java/datadog/trace/instrumentation/spark/SparkAggregatedTaskMetrics.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,7 @@ public void externalAccumToJson(JsonGenerator generator, SQLMetricInfo info) thr
321321
if (name != null && hist != null) {
322322
generator.writeStartObject();
323323
generator.writeStringField(name, histogramToBase64(hist));
324+
generator.writeNumberField("sum", hist.getSum());
324325
generator.writeStringField("type", info.metricType());
325326
generator.writeEndObject();
326327
}

‎dd-java-agent/instrumentation/spark/spark-common/src/testFixtures/groovy/datadog/trace/instrumentation/spark/AbstractSpark24SqlTest.groovy‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
157157

158158
assert expectedMetric.size() == actualMetric.size(): prefix + "metric size of $expectedMetric does not match $actualMetric"
159159

160-
// Each metric is a dict { "metric_name": "metric_value", "type": "metric_type" }
160+
// Each metric is a dict { "metric_name": "metric_value", "type": "metric_type", "sum": "metric_sum" }
161161
expectedMetric.each { key, expectedValue ->
162162
assert actualMetric.containsKey(key): prefix + "metric key \"$key\" not found in $actualMetric. \n\tactual: $actual.metrics, \n\texpected: $expected.metrics"
163163

@@ -259,6 +259,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
259259
"metrics": [
260260
{
261261
"data size total (min, med, max)": "any",
262+
"sum": "any",
262263
"type": "size"
263264
}
264265
],
@@ -270,6 +271,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
270271
"metrics": [
271272
{
272273
"duration total (min, med, max)": "any",
274+
"sum": "any",
273275
"type": "timing"
274276
}
275277
],
@@ -290,22 +292,27 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
290292
"metrics": [
291293
{
292294
"aggregate time total (min, med, max)": "any",
295+
"sum": "any",
293296
"type": "timing"
294297
},
295298
{
296299
"number of output rows": "any",
300+
"sum": "any",
297301
"type": "sum"
298302
},
299303
{
300304
"avg hash probe (min, med, max)": "any",
305+
"sum": "any",
301306
"type": "average"
302307
},
303308
{
304309
"peak memory total (min, med, max)": "any",
310+
"sum": "any",
305311
"type": "size"
306312
},
307313
{
308314
"spill size total (min, med, max)": "any",
315+
"sum": "any",
309316
"type": "size"
310317
}
311318
],
@@ -326,6 +333,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
326333
"metrics": [
327334
{
328335
"number of output rows": "CgkJCCGEEEII8T8SZBJgAAAAAAAA8D8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/GAAaAA==",
336+
"sum": 3,
329337
"type": "sum"
330338
}
331339
]
@@ -348,6 +356,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
348356
"metrics": [
349357
{
350358
"duration total (min, med, max)": "any",
359+
"sum": "any",
351360
"type": "timing"
352361
}
353362
],
@@ -368,22 +377,27 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
368377
"metrics": [
369378
{
370379
"aggregate time total (min, med, max)": "any",
380+
"sum": "any",
371381
"type": "timing"
372382
},
373383
{
374384
"avg hash probe (min, med, max)": "any",
385+
"sum": "any",
375386
"type": "average"
376387
},
377388
{
378389
"number of output rows": "CgkJCCGEEEII8T8SDBIIAAAAAAAA8D8YFhoAIQAAAAAAAPA/",
390+
"sum": 2,
379391
"type": "sum"
380392
},
381393
{
382394
"peak memory total (min, med, max)": "any",
395+
"sum": "any",
383396
"type": "size"
384397
},
385398
{
386399
"spill size total (min, med, max)": "any",
400+
"sum": "any",
387401
"type": "size"
388402
}
389403
],
@@ -473,6 +487,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
473487
"metrics": [
474488
{
475489
"data size total (min, med, max)": "any",
490+
"sum": "any",
476491
"type": "size"
477492
}
478493
],
@@ -488,6 +503,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
488503
"metrics": [
489504
{
490505
"number of output rows": "any",
506+
"sum": "any",
491507
"type": "sum"
492508
}
493509
]
@@ -513,6 +529,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
513529
"metrics": [
514530
{
515531
"data size total (min, med, max)": "any",
532+
"sum": "any",
516533
"type": "size"
517534
}
518535
],
@@ -528,6 +545,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
528545
"metrics": [
529546
{
530547
"number of output rows": "any",
548+
"sum": "any",
531549
"type": "sum"
532550
}
533551
]
@@ -548,6 +566,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
548566
"metrics": [
549567
{
550568
"data size total (min, med, max)": "any",
569+
"sum": "any",
551570
"type": "size"
552571
}
553572
],
@@ -559,6 +578,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
559578
"metrics": [
560579
{
561580
"duration total (min, med, max)": "any",
581+
"sum": "any",
562582
"type": "timing"
563583
}
564584
],
@@ -579,22 +599,27 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
579599
"metrics": [
580600
{
581601
"aggregate time total (min, med, max)": "any",
602+
"sum": "any",
582603
"type": "timing"
583604
},
584605
{
585606
"number of output rows": "any",
607+
"sum": "any",
586608
"type": "sum"
587609
},
588610
{
589611
"spill size total (min, med, max)": "any",
612+
"sum": "any",
590613
"type": "size"
591614
},
592615
{
593616
"avg hash probe (min, med, max)": "any",
617+
"sum": "any",
594618
"type": "average"
595619
},
596620
{
597621
"peak memory total (min, med, max)": "any",
622+
"sum": "any",
598623
"type": "size"
599624
}
600625
],
@@ -621,6 +646,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
621646
"metrics": [
622647
{
623648
"number of output rows": "any",
649+
"sum": "any",
624650
"type": "sum"
625651
}
626652
],
@@ -637,6 +663,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
637663
"metrics": [
638664
{
639665
"duration total (min, med, max)": "any",
666+
"sum": "any",
640667
"type": "timing"
641668
}
642669
],
@@ -654,14 +681,17 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
654681
"metrics": [
655682
{
656683
"spill size total (min, med, max)": "any",
684+
"sum": "any",
657685
"type": "size"
658686
},
659687
{
660688
"peak memory total (min, med, max)": "any",
689+
"sum": "any",
661690
"type": "size"
662691
},
663692
{
664693
"sort time total (min, med, max)": "any",
694+
"sum": "any",
665695
"type": "timing"
666696
}
667697
],
@@ -689,6 +719,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
689719
"metrics": [
690720
{
691721
"duration total (min, med, max)": "any",
722+
"sum": "any",
692723
"type": "timing"
693724
}
694725
],
@@ -706,14 +737,17 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
706737
"metrics": [
707738
{
708739
"spill size total (min, med, max)": "any",
740+
"sum": "any",
709741
"type": "size"
710742
},
711743
{
712744
"peak memory total (min, med, max)": "any",
745+
"sum": "any",
713746
"type": "size"
714747
},
715748
{
716749
"sort time total (min, med, max)": "any",
750+
"sum": "any",
717751
"type": "timing"
718752
}
719753
],
@@ -748,6 +782,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
748782
"metrics": [
749783
{
750784
"duration total (min, med, max)": "any",
785+
"sum": "any",
751786
"type": "timing"
752787
}
753788
],
@@ -768,6 +803,7 @@ abstract class AbstractSpark24SqlTest extends InstrumentationSpecification {
768803
"metrics": [
769804
{
770805
"number of output rows": "CgkJCCGEEEII8T8SDBIIAAAAAAAA8D8YABoA",
806+
"sum": 1,
771807
"type": "sum"
772808
}
773809
],

0 commit comments

Comments
 (0)