Skip to content

Commit 58670de

Browse files
tonyredondotonyredondo
committed
Harden coverage backfill publish gating
1 parent 1497c38 commit 58670de

7 files changed

Lines changed: 137 additions & 95 deletions

‎tracer/src/Datadog.Trace/Ci/Coverage/Backfill/CoverageBackfillApplicator.cs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,11 +68,13 @@ public static CoverageBackfillResult ApplyToGlobalCoverage(GlobalCoverageInfo? g
6868

6969
var backendFileCount = CountBackendFilesWithCoverage(backfillData);
7070
var matchedFiles = 0;
71+
var publishableBackendKeys = new HashSet<string>(StringComparer.Ordinal);
7172
foreach (var backendKey in matchedBackendKeys)
7273
{
7374
if (!unsafeBackendKeys.Contains(backendKey))
7475
{
7576
matchedFiles++;
77+
publishableBackendKeys.Add(backendKey);
7678
}
7779
}
7880

@@ -98,7 +100,21 @@ public static CoverageBackfillResult ApplyToGlobalCoverage(GlobalCoverageInfo? g
98100
matchedFiles,
99101
updatedFiles,
100102
hasBackendCoverage: backendFileCount > 0,
101-
canPublishCoverage: true);
103+
canPublishCoverage: CanPublishBackfilledCoverage(backfillData, publishableBackendKeys));
104+
}
105+
106+
private static bool CanPublishBackfilledCoverage(CoverageBackfillData backfillData, HashSet<string> publishableBackendKeys)
107+
{
108+
foreach (var item in backfillData.ExecutedLinesByRelativePath)
109+
{
110+
if (HasActiveBits(item.Value) &&
111+
!publishableBackendKeys.Contains(item.Key))
112+
{
113+
return false;
114+
}
115+
}
116+
117+
return true;
102118
}
103119

104120
private static bool TryGetMergedExecutedBitmap(FileCoverageInfo file, byte[] backendExecutedBitmap, out byte[] mergedBitmap)

‎tracer/src/Datadog.Trace/Ci/Coverage/Backfill/ExternalCoverageXmlBackfill.cs‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2250,15 +2250,24 @@ private static bool CanPublishBackfilledCoverage(CoverageBackfillData? backfillD
22502250
return matchedBackendPaths.Count == 0;
22512251
}
22522252

2253-
foreach (var backendPath in matchedBackendPaths)
2253+
var backendFileCount = 0;
2254+
foreach (var item in backfillData.ExecutedLinesByRelativePath)
22542255
{
2255-
if (!backfillData.ExecutedLinesByRelativePath.ContainsKey(backendPath))
2256+
if (!HasActiveBits(item.Value))
2257+
{
2258+
continue;
2259+
}
2260+
2261+
backendFileCount++;
2262+
if (!matchedBackendPaths.Contains(item.Key) ||
2263+
!representedBackendLines.TryGetValue(item.Key, out var representedLines) ||
2264+
representedLines.Count == 0)
22562265
{
22572266
return false;
22582267
}
22592268
}
22602269

2261-
return true;
2270+
return backendFileCount == matchedBackendPaths.Count;
22622271
}
22632272

22642273
private static bool HasValidatedBackfillCoverage(bool canBackfill, CoverageBackfillData? backfillData, HashSet<string> matchedBackendPaths, Dictionary<string, HashSet<int>> representedBackendLines)

‎tracer/src/Datadog.Trace/Ci/TestOptimizationSkippableFeature.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ internal sealed class TestOptimizationSkippableFeature : ITestOptimizationSkippa
3434
private readonly Dictionary<string, ActualSkippedDictionaryState> _actualSkippedDictionaries = new(StringComparer.Ordinal);
3535
private readonly ConcurrentDictionary<ulong, byte> _itrSkippedSessions = new();
3636
private readonly bool _coverageBackfillRequiresScopedRequests;
37+
private readonly string _coverageBackfillUnsupportedReason;
3738
// Keeps scoped tasks in insertion order so shutdown can wait for them without allocating array snapshots.
3839
private List<Task<SkippableTestsDictionary>>? _scopedSkippableTestsTaskList;
3940

@@ -48,6 +49,10 @@ private TestOptimizationSkippableFeature(TestOptimizationSettings settings, Test
4849
}
4950

5051
_coverageBackfillRequiresScopedRequests = CoverageBackfillCapability.IsCoverageBackfillRequired(settings);
52+
_coverageBackfillUnsupportedReason = _coverageBackfillRequiresScopedRequests &&
53+
!CoverageBackfillCapability.IsActiveCoverageModeBackfillable(settings, out var unsupportedReason) ?
54+
unsupportedReason :
55+
string.Empty;
5156
if (settings.TestsSkippingEnabled == true)
5257
{
5358
Log.Information("TestOptimizationSkippableFeature: Test skipping is enabled.");
@@ -259,6 +264,12 @@ public bool CanSkipWithCoverageBackfill(SkippableTest skippableTest, string? mod
259264
return true;
260265
}
261266

267+
if (!StringUtil.IsNullOrEmpty(_coverageBackfillUnsupportedReason))
268+
{
269+
reason = _coverageBackfillUnsupportedReason;
270+
return false;
271+
}
272+
262273
if (skippableTest.MissingLineCodeCoverage == true)
263274
{
264275
reason = "backend marked the test as missing line coverage";

‎tracer/test/Datadog.Trace.Tests/Ci/CoverageBackfillDataTests.cs‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -447,7 +447,7 @@ public void GlobalCoverageBackfillMatchesUniqueCaseInsensitiveSuffixOnlyOnWindow
447447
else
448448
{
449449
result.MatchedFiles.Should().Be(0);
450-
result.CanPublishCoverage.Should().BeTrue();
450+
result.CanPublishCoverage.Should().BeFalse();
451451
result.UpdatedFiles.Should().Be(0);
452452
globalCoverage.Components[0].Files[0].ExecutedBitmap.Should().Equal([0b_0000_0000]);
453453
}
@@ -487,7 +487,7 @@ public void GlobalCoverageBackfillDoesNotSuffixMatchAbsolutePathOutsideSourceRoo
487487

488488
result.Applied.Should().BeTrue();
489489
result.HasBackendCoverage.Should().BeTrue();
490-
result.CanPublishCoverage.Should().BeTrue();
490+
result.CanPublishCoverage.Should().BeFalse();
491491
result.Backfilled.Should().BeFalse();
492492
result.MatchedFiles.Should().Be(0);
493493
result.UpdatedFiles.Should().Be(0);
@@ -526,7 +526,7 @@ public void GlobalCoverageBackfillDoesNotSuffixMatchAbsolutePathWhenSourceRootIs
526526

527527
result.Applied.Should().BeTrue();
528528
result.HasBackendCoverage.Should().BeTrue();
529-
result.CanPublishCoverage.Should().BeTrue();
529+
result.CanPublishCoverage.Should().BeFalse();
530530
result.Backfilled.Should().BeFalse();
531531
result.MatchedFiles.Should().Be(0);
532532
result.UpdatedFiles.Should().Be(0);
@@ -567,7 +567,7 @@ public void GlobalCoverageBackfillDoesNotSuffixMatchRelativePathWithParentDirect
567567

568568
result.Applied.Should().BeTrue();
569569
result.HasBackendCoverage.Should().BeTrue();
570-
result.CanPublishCoverage.Should().BeTrue();
570+
result.CanPublishCoverage.Should().BeFalse();
571571
result.Backfilled.Should().BeFalse();
572572
result.MatchedFiles.Should().Be(0);
573573
result.UpdatedFiles.Should().Be(0);
@@ -609,7 +609,7 @@ public void GlobalCoverageBackfillIgnoresAmbiguousNonExactBackendKeyMatches()
609609

610610
result.Applied.Should().BeTrue();
611611
result.HasBackendCoverage.Should().BeTrue();
612-
result.CanPublishCoverage.Should().BeTrue();
612+
result.CanPublishCoverage.Should().BeFalse();
613613
result.Backfilled.Should().BeFalse();
614614
result.MatchedFiles.Should().Be(0);
615615
result.UpdatedFiles.Should().Be(0);
@@ -652,7 +652,7 @@ public void GlobalCoverageBackfillIgnoresBackendKeyWhenExactAndDistinctSuffixLoc
652652

653653
result.Applied.Should().BeTrue();
654654
result.HasBackendCoverage.Should().BeTrue();
655-
result.CanPublishCoverage.Should().BeTrue();
655+
result.CanPublishCoverage.Should().BeFalse();
656656
result.Backfilled.Should().BeFalse();
657657
result.MatchedFiles.Should().Be(0);
658658
result.UpdatedFiles.Should().Be(0);
@@ -817,7 +817,7 @@ public void GlobalCoverageBackfillRejectsAbsoluteUriPathBeforeSuffixMatch()
817817

818818
result.Applied.Should().BeTrue();
819819
result.HasBackendCoverage.Should().BeTrue();
820-
result.CanPublishCoverage.Should().BeTrue();
820+
result.CanPublishCoverage.Should().BeFalse();
821821
result.Backfilled.Should().BeFalse();
822822
result.MatchedFiles.Should().Be(0);
823823
result.UpdatedFiles.Should().Be(0);
@@ -907,7 +907,7 @@ public void PathMatcherRejectsAmbiguousCaseInsensitiveExactPaths()
907907
}
908908

909909
[Fact]
910-
public void GlobalCoverageBackfillIgnoresBackendOnlyFiles()
910+
public void GlobalCoverageBackfillBlocksPublishWhenBackendOnlyFilesAreActive()
911911
{
912912
var globalCoverage = new GlobalCoverageInfo
913913
{
@@ -936,14 +936,15 @@ public void GlobalCoverageBackfillIgnoresBackendOnlyFiles()
936936

937937
result.Applied.Should().BeTrue();
938938
result.HasBackendCoverage.Should().BeTrue();
939-
result.CanPublishCoverage.Should().BeTrue();
939+
result.CanPublishCoverage.Should().BeFalse();
940+
result.Backfilled.Should().BeFalse();
940941
result.MatchedFiles.Should().Be(0);
941942
result.UpdatedFiles.Should().Be(0);
942943
globalCoverage.GetTotalPercentage().Should().Be(100);
943944
}
944945

945946
[Fact]
946-
public void GlobalCoverageBackfillAppliesMatchedBackendCoverageAndIgnoresBackendOnlyFiles()
947+
public void GlobalCoverageBackfillBlocksPublishWhenAnyBackendActiveFileIsMissing()
947948
{
948949
var globalCoverage = new GlobalCoverageInfo
949950
{
@@ -973,15 +974,15 @@ public void GlobalCoverageBackfillAppliesMatchedBackendCoverageAndIgnoresBackend
973974

974975
result.Applied.Should().BeTrue();
975976
result.HasBackendCoverage.Should().BeTrue();
976-
result.CanPublishCoverage.Should().BeTrue();
977-
result.Backfilled.Should().BeTrue();
977+
result.CanPublishCoverage.Should().BeFalse();
978+
result.Backfilled.Should().BeFalse();
978979
result.MatchedFiles.Should().Be(1);
979980
result.UpdatedFiles.Should().Be(1);
980981
globalCoverage.Components[0].Files[0].ExecutedBitmap.Should().Equal([0b_1000_0000]);
981982
}
982983

983984
[Fact]
984-
public void GlobalCoverageBackfillCreatesExecutedBitmapForMatchedCoverageAndIgnoresBackendOnlyFiles()
985+
public void GlobalCoverageBackfillCreatesExecutedBitmapForMatchedCoverage()
985986
{
986987
var globalCoverage = new GlobalCoverageInfo
987988
{
@@ -1003,8 +1004,7 @@ public void GlobalCoverageBackfillCreatesExecutedBitmapForMatchedCoverageAndIgno
10031004
var backfill = CoverageBackfillData.FromBackendCoverage(
10041005
new Dictionary<string, string>
10051006
{
1006-
["src/Calculator.cs"] = Convert.ToBase64String([0b_1000_0000]),
1007-
["src/Other.cs"] = Convert.ToBase64String([0b_1000_0000])
1007+
["src/Calculator.cs"] = Convert.ToBase64String([0b_1000_0000])
10081008
});
10091009

10101010
var result = CoverageBackfillApplicator.ApplyToGlobalCoverage(globalCoverage, backfill, CIEnvironmentValues.Instance);

0 commit comments

Comments
 (0)