Skip to content

Commit bfd8734

Browse files
tonyredondotonyredondo
committed
[CI Visibility] Strengthen global coverage regression tests
1 parent 5863a08 commit bfd8734

4 files changed

Lines changed: 144 additions & 6 deletions

File tree

‎tracer/test/Datadog.Trace.Tests/Ci/CoverageEventHandlerTests.cs‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -141,13 +141,19 @@ public void DisposeReleasesEveryModuleAndIsIdempotent()
141141
}
142142

143143
[Fact]
144-
public unsafe void TenThousandClosedContextsKeepOnlyOneCompactBitmap()
144+
public unsafe void TenThousandClosedContextsKeepTheExactUnionInOneCompactBitmap()
145145
{
146146
const int contextCount = 10_000;
147147
const int rawByteLength = 128 * 1024;
148+
const int executableLineCount = 1_024;
148149

149150
var handler = new DefaultWithGlobalCoverageEventHandler();
150-
var metadata = CreateMetadata(totalLines: rawByteLength, coverageMode: 0, lastExecutableLine: 1);
151+
var expectedExecutableBitmap = Enumerable.Repeat((byte)0xff, executableLineCount / 8).ToArray();
152+
var expectedExecutedBitmap = Enumerable.Repeat((byte)0xaa, executableLineCount / 8).ToArray();
153+
var metadata = new TestMetadata(
154+
rawByteLength,
155+
0,
156+
[new FileCoverageMetadata("/src/stress.cs", 0, executableLineCount, expectedExecutableBitmap)]);
151157

152158
for (var i = 0; i < contextCount; i++)
153159
{
@@ -159,7 +165,8 @@ public unsafe void TenThousandClosedContextsKeepOnlyOneCompactBitmap()
159165
out var module)
160166
.Should()
161167
.BeTrue();
162-
*(byte*)module!.FilesLines = 1;
168+
var counters = (byte*)module!.FilesLines;
169+
counters[(i % (executableLineCount / 2)) * 2] = 1;
163170
handler.EndSession(handle);
164171
}
165172

@@ -169,11 +176,18 @@ public unsafe void TenThousandClosedContextsKeepOnlyOneCompactBitmap()
169176
contextDiagnostics.Disposed.Should().Be(contextCount);
170177

171178
var accumulator = handler.AccumulatorDiagnostics;
172-
accumulator.RetainedBitmapBytes.Should().Be(1);
179+
accumulator.RetainedBitmapBytes.Should().Be(expectedExecutedBitmap.Length);
173180
accumulator.ModuleCount.Should().Be(1);
174181
accumulator.FileSlotCount.Should().Be(1);
175182
accumulator.AcceptedContextCount.Should().Be(contextCount);
176183
accumulator.IsValid.Should().BeTrue();
184+
185+
using var snapshot = handler.AcquireGlobalCoverageSnapshot().Snapshot!;
186+
snapshot.MergedContextCount.Should().Be(contextCount);
187+
var file = snapshot.Model.Components.Should().ContainSingle().Subject.Files.Should().ContainSingle().Subject;
188+
file.ExecutableBitmap.Should().Equal(expectedExecutableBitmap);
189+
file.ExecutedBitmap.Should().Equal(expectedExecutedBitmap);
190+
file.Data.Should().Equal(50, executableLineCount, executableLineCount / 2);
177191
}
178192

179193
private static CoverageSessionHandle StartAndAllocate(DefaultWithGlobalCoverageEventHandler handler, ModuleCoverageMetadata metadata)

‎tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageOutputProtocolTests.cs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
using System.Linq;
1111
using System.Text;
1212
using Datadog.Trace.Ci.Coverage;
13+
using Datadog.Trace.Ci.Coverage.Metadata;
1314
using Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.DotnetTest;
1415
using FluentAssertions;
1516
using Newtonsoft.Json.Linq;
@@ -185,6 +186,59 @@ public void NewDirectoryAfterFreezeFailsClosedWithoutAReadyMarker()
185186
}
186187
}
187188

189+
[Fact]
190+
public unsafe void ReconciliationUnionsEveryPublishedGenerationExactly()
191+
{
192+
var directory = CreateDirectory();
193+
try
194+
{
195+
var handler = CreateHandler(directory);
196+
var metadata = new ProtocolTestMetadata(
197+
8,
198+
0,
199+
[new FileCoverageMetadata("/src/generations.cs", 0, 8, [0xff])]);
200+
201+
using (var command = DotnetTestRunState.TryCreate(DotnetTestCommandKind.DotnetTestCommand, null, directory, "run-id"))
202+
{
203+
PublishGeneration(executedOffset: 0);
204+
PublishGeneration(executedOffset: 7);
205+
handler.RequestSeal().Should().BeTrue();
206+
command.ReleaseActivity();
207+
}
208+
209+
var outputPath = Path.Combine(directory, "session-coverage-result.json");
210+
global::CoverageUtils.TryCombineAndGetTotalCoverage(directory, outputPath, out var combined).Should().BeTrue();
211+
212+
var file = combined!.Components.Should().ContainSingle().Subject.Files.Should().ContainSingle().Subject;
213+
file.ExecutableBitmap.Should().Equal(0xff);
214+
file.ExecutedBitmap.Should().Equal(0x81);
215+
file.Data.Should().Equal(25, 8, 2);
216+
Directory.GetFiles(Path.Combine(directory, ".dd-coverage-completed"), "coverage-*.json", SearchOption.AllDirectories).Should().HaveCount(2);
217+
218+
void PublishGeneration(int executedOffset)
219+
{
220+
var handle = handler.StartSession("xunit");
221+
handler.Container!.TryGetOrAddModuleValue(
222+
metadata,
223+
typeof(GlobalCoverageOutputProtocolTests).Module,
224+
CoverageMetadataValidator.ValidateAndGetRawByteLength(metadata),
225+
out var module)
226+
.Should()
227+
.BeTrue();
228+
var counters = (byte*)module!.FilesLines;
229+
counters[executedOffset] = 1;
230+
handler.EndSession(handle);
231+
232+
using var snapshot = handler.AcquireGlobalCoverageSnapshot().Snapshot!;
233+
handler.TryPublishRequiredFiles(snapshot).Should().BeTrue();
234+
}
235+
}
236+
finally
237+
{
238+
Directory.Delete(directory, true);
239+
}
240+
}
241+
188242
[Fact]
189243
public void AuthorizedCombineArchivesRawArtifactsAndRemovesReadyBeforePendingSet()
190244
{
@@ -573,4 +627,12 @@ private static void ProduceCompleteRun(string coordinatorDirectory, string secon
573627
handler.RequestSeal().Should().BeTrue();
574628
command.ReleaseActivity();
575629
}
630+
631+
private sealed class ProtocolTestMetadata : ModuleCoverageMetadata
632+
{
633+
public ProtocolTestMetadata(int totalLines, int coverageMode, FileCoverageMetadata[] files)
634+
: base(totalLines, coverageMode, files)
635+
{
636+
}
637+
}
576638
}

‎tracer/test/Datadog.Trace.Tools.Runner.IntegrationTests/CiRunGlobalCoverageMemoryTests.cs‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ public sealed class CiRunGlobalCoverageMemoryTests
3030
private const long MaximumStressPrivateBytesGrowth = 384L * 1024 * 1024;
3131
private const string SampleName = "NUnitGlobalCoverageMemory";
3232
private const string SampleProjectRelativePath = "tracer/test/test-applications/integrations/Samples.NUnitGlobalCoverageMemory/Samples.NUnitGlobalCoverageMemory.csproj";
33+
private const string SampleSourceFileName = "GlobalCoverageMemoryTests.cs";
34+
private const int CommonCoverageLine = 131_072;
35+
private const int FirstCoverageSentinelLine = 131_073;
36+
private const int MiddleCoverageSentinelLine = 131_074;
37+
private const int LastCoverageSentinelLine = 131_075;
3338
private readonly ITestOutputHelper _output;
3439

3540
public CiRunGlobalCoverageMemoryTests(ITestOutputHelper output)
@@ -145,7 +150,7 @@ private void RunStress(
145150

146151
AssertLaunch(launchedArguments, launchedEnvironment, useDotnetTest, useTestingPlatformCoverage, coverageDirectory);
147152
var testhostProcessId = AssertProgress(progressPath, expectedCaseCount);
148-
var publishedCoverage = AssertPublishedCoverage(coverageDirectory);
153+
var publishedCoverage = AssertPublishedCoverage(coverageDirectory, expectedCaseCount);
149154
File.Move(publishedCoverage, Path.Combine(root.RootPath, $"session-coverage-{runIndex}.json"));
150155
AssertCoverageDiagnostics(logDirectory, testhostProcessId, expectedCaseCount);
151156
if (useDotnetTest)
@@ -338,13 +343,25 @@ private int AssertProgress(string progressPath, int expectedCaseCount)
338343
return records[0]!.Pid;
339344
}
340345

341-
private string AssertPublishedCoverage(string coverageDirectory)
346+
private string AssertPublishedCoverage(string coverageDirectory, int expectedCaseCount)
342347
{
343348
var sessionCoverage = Directory.GetFiles(coverageDirectory, "session-coverage-*.json", SearchOption.TopDirectoryOnly);
344349
sessionCoverage.Should().ContainSingle();
345350
new GlobalCoverageInputReader().TryRead(sessionCoverage[0], out var coverage).Should().BeTrue();
346351
coverage.Should().NotBeNull();
347352
coverage!.GetTotalPercentage().Should().BeGreaterThan(0);
353+
var sampleFile = coverage.Components.SelectMany(static component => component.Files)
354+
.Should()
355+
.ContainSingle(file => string.Equals(Path.GetFileName(file.Path), SampleSourceFileName, StringComparison.OrdinalIgnoreCase))
356+
.Subject;
357+
AssertLine(sampleFile.ExecutableBitmap, CommonCoverageLine, expected: true);
358+
AssertLine(sampleFile.ExecutedBitmap, CommonCoverageLine, expected: true);
359+
AssertLine(sampleFile.ExecutableBitmap, FirstCoverageSentinelLine, expected: true);
360+
AssertLine(sampleFile.ExecutedBitmap, FirstCoverageSentinelLine, expected: true);
361+
AssertLine(sampleFile.ExecutableBitmap, MiddleCoverageSentinelLine, expected: true);
362+
AssertLine(sampleFile.ExecutedBitmap, MiddleCoverageSentinelLine, expected: expectedCaseCount > 1);
363+
AssertLine(sampleFile.ExecutableBitmap, LastCoverageSentinelLine, expected: true);
364+
AssertLine(sampleFile.ExecutedBitmap, LastCoverageSentinelLine, expected: expectedCaseCount > 1);
348365

349366
Directory.GetFiles(coverageDirectory, "coverage-*.json", SearchOption.TopDirectoryOnly).Should().BeEmpty();
350367
Directory.GetFiles(coverageDirectory, ".dd-coverage-process-incomplete-*", SearchOption.TopDirectoryOnly).Should().BeEmpty();
@@ -357,6 +374,16 @@ private string AssertPublishedCoverage(string coverageDirectory)
357374
return sessionCoverage[0];
358375
}
359376

377+
private void AssertLine(byte[]? bitmap, int line, bool expected)
378+
{
379+
bitmap.Should().NotBeNull();
380+
var zeroBasedLine = line - 1;
381+
var byteIndex = zeroBasedLine >> 3;
382+
bitmap!.Length.Should().BeGreaterThan(byteIndex);
383+
var mask = (byte)(128 >> (zeroBasedLine & 7));
384+
((bitmap[byteIndex] & mask) != 0).Should().Be(expected, $"line {line} should have the expected coverage state");
385+
}
386+
360387
private void AssertOuterCommandReconciliation(string logDirectory)
361388
{
362389
var logFiles = Directory.GetFiles(logDirectory, "*.log", SearchOption.AllDirectories);

‎tracer/test/test-applications/integrations/Samples.NUnitGlobalCoverageMemory/GlobalCoverageMemoryTests.cs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,20 @@ public static IEnumerable Cases()
3636
public void ExecutesSparseInstrumentedLine(int value)
3737
{
3838
Assert.That(CoveredMethod(value), Is.EqualTo(value + 1));
39+
if (value == 0)
40+
{
41+
Assert.That(FirstCoverageSentinel(), Is.EqualTo(101));
42+
}
43+
44+
if (value == DefaultCaseCount / 2)
45+
{
46+
Assert.That(MiddleCoverageSentinel(), Is.EqualTo(102));
47+
}
48+
49+
if (value == DefaultCaseCount - 1)
50+
{
51+
Assert.That(LastCoverageSentinel(), Is.EqualTo(103));
52+
}
3953

4054
var completed = Interlocked.Increment(ref _completed);
4155
if (completed % 100 == 0 || completed == 1)
@@ -51,6 +65,27 @@ private static int CoveredMethod(int value)
5165
#line default
5266
}
5367

68+
private static int FirstCoverageSentinel()
69+
{
70+
#line 131073
71+
return 101;
72+
#line default
73+
}
74+
75+
private static int MiddleCoverageSentinel()
76+
{
77+
#line 131074
78+
return 102;
79+
#line default
80+
}
81+
82+
private static int LastCoverageSentinel()
83+
{
84+
#line 131075
85+
return 103;
86+
#line default
87+
}
88+
5489
private static int ParseSmokeCaseCount(string value)
5590
{
5691
if (value == "1")

0 commit comments

Comments
 (0)