Skip to content

Commit 828c9cc

Browse files
aboitreaudaboitreaud
authored
Fix sql plan nodes order causing test flakiness (#10911)
Use LinkedHashSet for deterministic ordering Reorder nodes in expected plan Co-authored-by: adrien.boitreaud <adrien.boitreaud@datadoghq.com>
1 parent 51fe2ea commit 828c9cc

File tree

1 file changed

+40
-40
lines changed

1 file changed

+40
-40
lines changed

‎dd-java-agent/instrumentation/spark/spark-common/src/testFixtures/groovy/datadog/trace/instrumentation/spark/AbstractSpark32SqlTest.groovy‎

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
3434
def firstStagePlan = """
3535
{
3636
"node": "Exchange",
37-
"nodeId": "nodeId_4",
37+
"nodeId": "nodeId_1",
3838
"nodeDetailString": "hashpartitioning(string_col#0, 2), ENSURE_REQUIREMENTS, [plan_id\\u003d38]",
3939
"meta": {
4040
"_dd.unparsed" : "any",
@@ -71,7 +71,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
7171
"children": [
7272
{
7373
"node": "WholeStageCodegen (1)",
74-
"nodeId": "nodeId_1",
74+
"nodeId": "nodeId_2",
7575
"meta": {"_dd.unparsed": "any"},
7676
"metrics": [
7777
{
@@ -131,7 +131,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
131131
"children": [
132132
{
133133
"node": "LocalTableScan",
134-
"nodeId": "nodeId_2",
134+
"nodeId": "nodeId_4",
135135
"nodeDetailString": "[string_col#0, double_col#1]",
136136
"meta": {
137137
"output" : [ "string_col#0", "double_col#1" ],
@@ -155,7 +155,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
155155
def secondStagePlan = """
156156
{
157157
"node": "WholeStageCodegen (2)",
158-
"nodeId": "nodeId_8",
158+
"nodeId": "nodeId_5",
159159
"meta": {"_dd.unparsed": "any"},
160160
"metrics": [
161161
{
@@ -167,7 +167,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
167167
"children": [
168168
{
169169
"node": "HashAggregate",
170-
"nodeId": "nodeId_9",
170+
"nodeId": "nodeId_6",
171171
"nodeDetailString": "(keys\\u003d[string_col#0], functions\\u003d[avg(double_col#1)])",
172172
"meta": {
173173
"_dd.unparsed" : "any",
@@ -220,7 +220,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
220220
"children": [
221221
{
222222
"node": "AQEShuffleRead",
223-
"nodeId": "nodeId_5",
223+
"nodeId": "nodeId_8",
224224
"nodeDetailString": "coalesced",
225225
"meta": {
226226
"_dd.unparsed" : "any",
@@ -230,7 +230,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
230230
"children": [
231231
{
232232
"node": "ShuffleQueryStage",
233-
"nodeId": "nodeId_6",
233+
"nodeId": "nodeId_9",
234234
"nodeDetailString": "0",
235235
"meta": {
236236
"_dd.unparsed": "any",
@@ -239,7 +239,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
239239
"children": [
240240
{
241241
"node": "Exchange",
242-
"nodeId": "nodeId_4",
242+
"nodeId": "nodeId_1",
243243
"nodeDetailString": "hashpartitioning(string_col#0, 2), ENSURE_REQUIREMENTS, [plan_id\\u003d38]",
244244
"meta": {
245245
"_dd.unparsed" : "any",
@@ -365,7 +365,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
365365
"children": [
366366
{
367367
"node": "WholeStageCodegen (2)",
368-
"nodeId": "nodeId_8",
368+
"nodeId": "nodeId_5",
369369
"meta": {"_dd.unparsed": "any"},
370370
"metrics": [
371371
{
@@ -377,7 +377,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
377377
"children": [
378378
{
379379
"node": "HashAggregate",
380-
"nodeId": "nodeId_9",
380+
"nodeId": "nodeId_6",
381381
"nodeDetailString": "(keys\\u003d[string_col#0], functions\\u003d[avg(double_col#1)])",
382382
"meta": {
383383
"_dd.unparsed" : "any",
@@ -430,7 +430,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
430430
"children": [
431431
{
432432
"node": "AQEShuffleRead",
433-
"nodeId": "nodeId_5",
433+
"nodeId": "nodeId_8",
434434
"nodeDetailString": "coalesced",
435435
"meta": {
436436
"_dd.unparsed" : "any",
@@ -440,7 +440,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
440440
"children": [
441441
{
442442
"node": "ShuffleQueryStage",
443-
"nodeId": "nodeId_6",
443+
"nodeId": "nodeId_9",
444444
"nodeDetailString": "0",
445445
"meta": {
446446
"_dd.unparsed": "any",
@@ -449,7 +449,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
449449
"children": [
450450
{
451451
"node": "Exchange",
452-
"nodeId": "nodeId_4",
452+
"nodeId": "nodeId_1",
453453
"nodeDetailString": "hashpartitioning(string_col#0, 2), ENSURE_REQUIREMENTS, [plan_id\\u003d38]",
454454
"meta": {
455455
"_dd.unparsed" : "any",
@@ -540,7 +540,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
540540
def fourthStagePlan = """
541541
{
542542
"node": "WholeStageCodegen (3)",
543-
"nodeId": "nodeId_12",
543+
"nodeId": "nodeId_11",
544544
"meta": {"_dd.unparsed": "any"},
545545
"metrics": [
546546
{
@@ -552,7 +552,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
552552
"children": [
553553
{
554554
"node": "Project",
555-
"nodeId": "nodeId_11",
555+
"nodeId": "nodeId_12",
556556
"nodeDetailString": "[string_col#0, cast(avg(double_col)#5 as string) AS avg(double_col)#12]",
557557
"meta": {
558558
"_dd.unparsed" : "any",
@@ -594,7 +594,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
594594
"children": [
595595
{
596596
"node": "AQEShuffleRead",
597-
"nodeId": "nodeId_5",
597+
"nodeId": "nodeId_8",
598598
"nodeDetailString": "coalesced",
599599
"meta": {
600600
"_dd.unparsed" : "any",
@@ -913,7 +913,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
913913
def thirdStagePlan = """
914914
{
915915
"node": "Exchange",
916-
"nodeId": "nodeId_7",
916+
"nodeId": "nodeId_5",
917917
"nodeDetailString": "SinglePartition, ENSURE_REQUIREMENTS, [plan_id\\u003d230]",
918918
"meta": {
919919
"_dd.unparsed" : "any",
@@ -945,7 +945,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
945945
"children": [
946946
{
947947
"node": "WholeStageCodegen (3)",
948-
"nodeId": "nodeId_9",
948+
"nodeId": "nodeId_6",
949949
"meta": {"_dd.unparsed": "any"},
950950
"metrics": [
951951
{
@@ -957,7 +957,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
957957
"children": [
958958
{
959959
"node": "HashAggregate",
960-
"nodeId": "nodeId_16",
960+
"nodeId": "nodeId_7",
961961
"nodeDetailString": "(keys\\u003d[], functions\\u003d[partial_count(1)])",
962962
"meta": {
963963
"_dd.unparsed" : "any",
@@ -1005,15 +1005,15 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
10051005
"children": [
10061006
{
10071007
"node": "Project",
1008-
"nodeId": "nodeId_13",
1008+
"nodeId": "nodeId_8",
10091009
"meta": {
10101010
"_dd.unparsed" : "any",
10111011
"projectList" : [ ]
10121012
},
10131013
"children": [
10141014
{
10151015
"node": "SortMergeJoin",
1016-
"nodeId": "nodeId_15",
1016+
"nodeId": "nodeId_9",
10171017
"nodeDetailString": "[string_col#28], [string_col#32], Inner",
10181018
"meta": {
10191019
"_dd.unparsed" : "any",
@@ -1033,12 +1033,12 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
10331033
"children": [
10341034
{
10351035
"node": "InputAdapter",
1036-
"nodeId": "nodeId_6",
1036+
"nodeId": "nodeId_10",
10371037
"meta": {"_dd.unparsed": "any"},
10381038
"children": [
10391039
{
10401040
"node": "WholeStageCodegen (1)",
1041-
"nodeId": "nodeId_8",
1041+
"nodeId": "nodeId_11",
10421042
"meta": {"_dd.unparsed": "any"},
10431043
"metrics": [
10441044
{
@@ -1050,7 +1050,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
10501050
"children": [
10511051
{
10521052
"node": "Sort",
1053-
"nodeId": "nodeId_11",
1053+
"nodeId": "nodeId_12",
10541054
"nodeDetailString": "[string_col#28 ASC NULLS FIRST], false, 0",
10551055
"meta": {
10561056
"_dd.unparsed" : "any",
@@ -1078,12 +1078,12 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
10781078
"children": [
10791079
{
10801080
"node": "InputAdapter",
1081-
"nodeId": "nodeId_6",
1081+
"nodeId": "nodeId_10",
10821082
"meta": {"_dd.unparsed": "any"},
10831083
"children": [
10841084
{
10851085
"node": "AQEShuffleRead",
1086-
"nodeId": "nodeId_5",
1086+
"nodeId": "nodeId_13",
10871087
"nodeDetailString": "coalesced",
10881088
"meta": {
10891089
"_dd.unparsed" : "any",
@@ -1102,7 +1102,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
11021102
"children": [
11031103
{
11041104
"node": "Exchange",
1105-
"nodeId": "nodeId_2",
1105+
"nodeId": "nodeId_1",
11061106
"nodeDetailString": "hashpartitioning(string_col#28, 2), ENSURE_REQUIREMENTS, [plan_id\\u003d119]",
11071107
"meta": {
11081108
"_dd.unparsed" : "any",
@@ -1191,12 +1191,12 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
11911191
},
11921192
{
11931193
"node": "InputAdapter",
1194-
"nodeId": "nodeId_6",
1194+
"nodeId": "nodeId_10",
11951195
"meta": {"_dd.unparsed": "any"},
11961196
"children": [
11971197
{
11981198
"node": "WholeStageCodegen (2)",
1199-
"nodeId": "nodeId_12",
1199+
"nodeId": "nodeId_15",
12001200
"meta": {"_dd.unparsed": "any"},
12011201
"metrics": [
12021202
{
@@ -1208,7 +1208,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
12081208
"children": [
12091209
{
12101210
"node": "Sort",
1211-
"nodeId": "nodeId_17",
1211+
"nodeId": "nodeId_16",
12121212
"nodeDetailString": "[string_col#32 ASC NULLS FIRST], false, 0",
12131213
"meta": {
12141214
"_dd.unparsed" : "any",
@@ -1236,12 +1236,12 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
12361236
"children": [
12371237
{
12381238
"node": "InputAdapter",
1239-
"nodeId": "nodeId_6",
1239+
"nodeId": "nodeId_10",
12401240
"meta": {"_dd.unparsed": "any"},
12411241
"children": [
12421242
{
12431243
"node": "AQEShuffleRead",
1244-
"nodeId": "nodeId_5",
1244+
"nodeId": "nodeId_13",
12451245
"nodeDetailString": "coalesced",
12461246
"meta": {
12471247
"_dd.unparsed" : "any",
@@ -1251,7 +1251,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
12511251
"children": [
12521252
{
12531253
"node": "ShuffleQueryStage",
1254-
"nodeId": "nodeId_10",
1254+
"nodeId": "nodeId_17",
12551255
"nodeDetailString": "1",
12561256
"meta": {
12571257
"_dd.unparsed": "any",
@@ -1260,7 +1260,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
12601260
"children": [
12611261
{
12621262
"node": "Exchange",
1263-
"nodeId": "nodeId_4",
1263+
"nodeId": "nodeId_3",
12641264
"nodeDetailString": "hashpartitioning(string_col#32, 2), ENSURE_REQUIREMENTS, [plan_id\\u003d120]",
12651265
"meta": {
12661266
"_dd.unparsed" : "any",
@@ -1361,7 +1361,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
13611361
def fourthStagePlan = """
13621362
{
13631363
"node": "WholeStageCodegen (4)",
1364-
"nodeId": "nodeId_20",
1364+
"nodeId": "nodeId_18",
13651365
"meta": {"_dd.unparsed": "any"},
13661366
"metrics": [
13671367
{
@@ -1373,7 +1373,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
13731373
"children": [
13741374
{
13751375
"node": "HashAggregate",
1376-
"nodeId": "nodeId_18",
1376+
"nodeId": "nodeId_19",
13771377
"nodeDetailString": "(keys\\u003d[], functions\\u003d[count(1)])",
13781378
"meta": {
13791379
"_dd.unparsed" : "any",
@@ -1401,12 +1401,12 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
14011401
"children": [
14021402
{
14031403
"node": "InputAdapter",
1404-
"nodeId": "nodeId_6",
1404+
"nodeId": "nodeId_10",
14051405
"meta": {"_dd.unparsed": "any"},
14061406
"children": [
14071407
{
14081408
"node": "ShuffleQueryStage",
1409-
"nodeId": "nodeId_19",
1409+
"nodeId": "nodeId_20",
14101410
"nodeDetailString": "2",
14111411
"meta": {
14121412
"_dd.unparsed": "any",
@@ -1415,7 +1415,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
14151415
"children": [
14161416
{
14171417
"node": "Exchange",
1418-
"nodeId": "nodeId_7",
1418+
"nodeId": "nodeId_5",
14191419
"nodeDetailString": "SinglePartition, ENSURE_REQUIREMENTS, [plan_id\\u003d230]",
14201420
"meta": {
14211421
"_dd.unparsed" : "any",
@@ -1601,7 +1601,7 @@ abstract class AbstractSpark32SqlTest extends InstrumentationSpecification {
16011601
}
16021602

16031603
private static Set<Object> extractNodeIds(Object plan) {
1604-
Set<Object> nodeIds = new HashSet<>()
1604+
Set<Object> nodeIds = new LinkedHashSet<>()
16051605
if (plan instanceof Map) {
16061606
if (plan.containsKey("nodeId")) {
16071607
nodeIds.add(plan.nodeId)

0 commit comments

Comments
 (0)