Skip to content

Commit c62a892

Browse files
tonyredondotonyredondo
committed
[CI Visibility] Simplify global coverage processing
1 parent 234fe38 commit c62a892

13 files changed

Lines changed: 351 additions & 384 deletions

‎tracer/src/Datadog.Trace.Tools.Runner/CoverageUtils.cs‎

Lines changed: 12 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55

66
using System;
77
using System.IO;
8-
using System.Linq;
9-
using Datadog.Trace;
108
using Datadog.Trace.Ci.Coverage;
119
using Datadog.Trace.Ci.Coverage.Models.Global;
1210
using Spectre.Console;
@@ -15,8 +13,6 @@ namespace Datadog.Trace.Tools.Runner;
1513

1614
internal static class CoverageUtils
1715
{
18-
private const int MaximumInputFiles = 65_536;
19-
2016
public static bool TryCombineAndGetTotalCoverage(string inputFolder, string outputFile, bool useStdOut)
2117
{
2218
return TryCombineAndGetTotalCoverage(inputFolder, outputFile, out _, useStdOut);
@@ -63,7 +59,7 @@ private static bool TryLoadAndCombine(
6359
string outputFile,
6460
out GlobalCoverageInfo globalCoverageInfo,
6561
out GlobalCoverageReconciliationLease reconciliationLease,
66-
bool useStdOut = true)
62+
bool useStdOut)
6763
{
6864
globalCoverageInfo = default;
6965
reconciliationLease = null;
@@ -91,13 +87,11 @@ private static bool TryLoadAndCombine(
9187
var jsonFiles = Array.Empty<string>();
9288
try
9389
{
94-
if (!GlobalCoverageReconciliation.TryAcquire(inputFolder, authority: null, out reconciliationLease, out _) ||
95-
(reconciliationLease is null && HasProtocolMarkers(inputFolder)))
90+
if (!GlobalCoverageFileCombiner.TryAcquireInputFiles(inputFolder, authority: null, out jsonFiles, out reconciliationLease))
9691
{
9792
return false;
9893
}
9994

100-
jsonFiles = reconciliationLease?.SelectedInputs.Select(static input => input.Path).ToArray() ?? GetInputFilesBounded(inputFolder);
10195
if (jsonFiles.Length == 0)
10296
{
10397
reconciliationLease?.Complete();
@@ -115,67 +109,23 @@ private static bool TryLoadAndCombine(
115109
AnsiConsole.WriteException(ex);
116110
}
117111

118-
var inputReader = new GlobalCoverageInputReader();
119-
var accumulator = new GlobalCoverageCombinerAccumulator();
120-
var processedFiles = 0;
121-
var outputFullPath = string.IsNullOrWhiteSpace(outputFile) ? null : Path.GetFullPath(outputFile);
122-
foreach (var file in jsonFiles)
112+
Action<string> onFileProcessed = useStdOut ? file => Utils.WriteSuccess($"Processing: {file}") : null;
113+
if (!GlobalCoverageFileCombiner.TryCombine(
114+
jsonFiles,
115+
outputFile,
116+
reconciliationLease,
117+
onFileProcessed,
118+
out globalCoverageInfo,
119+
out var rejectedInput))
123120
{
124-
if (Path.GetFileName(file).StartsWith("session-coverage-", StringComparison.OrdinalIgnoreCase) ||
125-
(outputFullPath is not null && PathsEqual(Path.GetFullPath(file), outputFullPath)))
126-
{
127-
continue;
128-
}
129-
130-
if (!inputReader.TryRead(file, reconciliationLease?.GetCertifiedInput(file), out var globalCoverage) || globalCoverage is null)
131-
{
132-
if (useStdOut)
133-
{
134-
Utils.WriteError($"Error processing {file}");
135-
}
136-
137-
return false;
138-
}
139-
140-
if (useStdOut)
121+
if (useStdOut && rejectedInput is not null)
141122
{
142-
Utils.WriteSuccess($"Processing: {file}");
123+
Utils.WriteError($"Error processing {rejectedInput}");
143124
}
144125

145-
accumulator.Add(globalCoverage);
146-
processedFiles++;
147-
}
148-
149-
if (processedFiles == 0)
150-
{
151126
return false;
152127
}
153128

154-
globalCoverageInfo = accumulator.Materialize();
155129
return true;
156130
}
157-
158-
private static bool HasProtocolMarkers(string inputFolder)
159-
=> Directory.EnumerateFiles(inputFolder, ".dd-coverage-process-incomplete-*", SearchOption.TopDirectoryOnly).Any() ||
160-
Directory.EnumerateFiles(inputFolder, ".dd-coverage-process-ready-*", SearchOption.TopDirectoryOnly).Any() ||
161-
Directory.EnumerateFiles(inputFolder, ".dd-coverage-command-owner-*.claim", SearchOption.TopDirectoryOnly).Any();
162-
163-
private static string[] GetInputFilesBounded(string inputFolder)
164-
{
165-
var files = Directory.EnumerateFiles(inputFolder, "*.json", SearchOption.TopDirectoryOnly)
166-
.Take(MaximumInputFiles + 1)
167-
.ToArray();
168-
if (files.Length > MaximumInputFiles)
169-
{
170-
throw new InvalidDataException("The global coverage input-file limit was exceeded.");
171-
}
172-
173-
return files;
174-
}
175-
176-
private static bool PathsEqual(string first, string second)
177-
=> string.Equals(
178-
first,
179-
second,
180-
FrameworkDescription.Instance.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
181131
}

‎tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter`1.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,17 +55,14 @@ static CoverageReporter()
5555
ModuleValue? module = null;
5656
try
5757
{
58-
if (handler.Container is { } container &&
59-
container.TryGetOrAddModuleValue(
58+
if (handler.Container is not { } container ||
59+
!container.TryGetOrAddModuleValue(
6060
Metadata,
6161
Module,
6262
ModuleMemorySize,
6363
handler.ModuleValueStrategy,
6464
CoverageModuleValueOrigin.TestContext,
6565
out module))
66-
{
67-
}
68-
else
6966
{
7067
module = GetOrCreateGlobalModuleValue(handler);
7168
}

‎tracer/src/Datadog.Trace/Ci/Coverage/DefaultWithGlobalCoverageEventHandler.cs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -283,11 +283,6 @@ internal bool RequestSeal()
283283
}
284284
}
285285

286-
protected override void OnSessionStart(CoverageContextContainer context)
287-
{
288-
base.OnSessionStart(context);
289-
}
290-
291286
protected virtual void InitializeSnapshotOutput(GlobalCoverageSnapshot snapshot)
292287
=> snapshot.InitializeOutput(_outputManager.FrozenMask, OnSnapshotDisposed);
293288

‎tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageArtifactWriter.cs‎

Lines changed: 4 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,10 @@ private GlobalCoverageStagedArtifact Stage(string destinationPath, GlobalCoverag
7575
try
7676
{
7777
using (var fileStream = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 16 * 1024, FileOptions.SequentialScan))
78-
using (var boundedStream = new BoundedWriteStream(fileStream, _limits.MaximumSerializedBytes))
78+
using (var boundedStream = new GlobalCoverageBoundedWriteStream(
79+
fileStream,
80+
_limits.MaximumSerializedBytes,
81+
"The global coverage serialized-byte limit was exceeded."))
7982
using (var streamWriter = new StreamWriter(boundedStream, Utf8WithoutBom, 16 * 1024, true))
8083
using (var jsonWriter = new JsonTextWriter(streamWriter) { ArrayPool = JsonArrayPool.Shared })
8184
{
@@ -94,51 +97,4 @@ private GlobalCoverageStagedArtifact Stage(string destinationPath, GlobalCoverag
9497
throw;
9598
}
9699
}
97-
98-
private sealed class BoundedWriteStream : Stream
99-
{
100-
private readonly Stream _inner;
101-
private readonly long _maximumBytes;
102-
private long _writtenBytes;
103-
104-
internal BoundedWriteStream(Stream inner, long maximumBytes)
105-
{
106-
_inner = inner;
107-
_maximumBytes = maximumBytes;
108-
}
109-
110-
public override bool CanRead => false;
111-
112-
public override bool CanSeek => false;
113-
114-
public override bool CanWrite => true;
115-
116-
public override long Length => _writtenBytes;
117-
118-
public override long Position
119-
{
120-
get => _writtenBytes;
121-
set => throw new NotSupportedException();
122-
}
123-
124-
public override void Flush() => _inner.Flush();
125-
126-
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
127-
128-
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
129-
130-
public override void SetLength(long value) => throw new NotSupportedException();
131-
132-
public override void Write(byte[] buffer, int offset, int count)
133-
{
134-
var nextLength = checked(_writtenBytes + count);
135-
if (nextLength > _maximumBytes)
136-
{
137-
throw new InvalidDataException("The global coverage serialized-byte limit was exceeded.");
138-
}
139-
140-
_inner.Write(buffer, offset, count);
141-
_writtenBytes = nextLength;
142-
}
143-
}
144100
}

‎tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageBoundedWriteStream.cs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// <copyright file="GlobalCoverageBoundedWriteStream.cs" company="Datadog">
2+
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
4+
// </copyright>
5+
6+
#nullable enable
7+
8+
using System;
9+
using System.IO;
10+
11+
namespace Datadog.Trace.Ci.Coverage;
12+
13+
// This wrapper intentionally does not own the underlying stream. Callers perform the durable file
14+
// flush after the JSON writers have flushed through this byte-counting layer.
15+
internal sealed class GlobalCoverageBoundedWriteStream : Stream
16+
{
17+
private readonly Stream _inner;
18+
private readonly long _maximumBytes;
19+
private readonly string _limitExceededMessage;
20+
private long _writtenBytes;
21+
22+
internal GlobalCoverageBoundedWriteStream(Stream inner, long maximumBytes, string limitExceededMessage)
23+
{
24+
_inner = inner;
25+
_maximumBytes = maximumBytes;
26+
_limitExceededMessage = limitExceededMessage;
27+
}
28+
29+
public override bool CanRead => false;
30+
31+
public override bool CanSeek => false;
32+
33+
public override bool CanWrite => true;
34+
35+
public override long Length => _writtenBytes;
36+
37+
public override long Position
38+
{
39+
get => _writtenBytes;
40+
set => throw new NotSupportedException();
41+
}
42+
43+
public override void Flush() => _inner.Flush();
44+
45+
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
46+
47+
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
48+
49+
public override void SetLength(long value) => throw new NotSupportedException();
50+
51+
public override void Write(byte[] buffer, int offset, int count)
52+
{
53+
var nextLength = checked(_writtenBytes + count);
54+
if (nextLength > _maximumBytes)
55+
{
56+
throw new InvalidDataException(_limitExceededMessage);
57+
}
58+
59+
_inner.Write(buffer, offset, count);
60+
_writtenBytes = nextLength;
61+
}
62+
}

‎tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageFileCombiner.cs‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// <copyright file="GlobalCoverageFileCombiner.cs" company="Datadog">
2+
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
4+
// </copyright>
5+
6+
#nullable enable
7+
8+
using System;
9+
using System.Collections.Generic;
10+
using System.IO;
11+
using System.Linq;
12+
using Datadog.Trace.Ci.Coverage.Models.Global;
13+
using Datadog.Trace.Util;
14+
15+
namespace Datadog.Trace.Ci.Coverage;
16+
17+
internal static class GlobalCoverageFileCombiner
18+
{
19+
private const int MaximumInputFiles = 65_536;
20+
21+
internal static bool TryAcquireInputFiles(
22+
string inputFolder,
23+
GlobalCoverageReconciliationAuthority? authority,
24+
out string[] inputFiles,
25+
out GlobalCoverageReconciliationLease? reconciliationLease)
26+
{
27+
inputFiles = [];
28+
reconciliationLease = null;
29+
30+
if (!GlobalCoverageReconciliation.TryAcquire(inputFolder, authority, out reconciliationLease, out _) ||
31+
(reconciliationLease is null && HasProtocolMarkers(inputFolder)))
32+
{
33+
return false;
34+
}
35+
36+
// The lease certifies a closed protocol generation. Without a protocol, retain the legacy
37+
// behavior of combining every bounded JSON input in the directory.
38+
inputFiles = reconciliationLease?.SelectedInputs.Select(static input => input.Path).ToArray() ?? GetInputFilesBounded(inputFolder);
39+
return true;
40+
}
41+
42+
internal static bool TryCombine(
43+
IReadOnlyList<string> inputFiles,
44+
string? outputFile,
45+
GlobalCoverageReconciliationLease? reconciliationLease,
46+
Action<string>? onFileProcessed,
47+
out GlobalCoverageInfo? globalCoverageInfo,
48+
out string? rejectedInput)
49+
{
50+
globalCoverageInfo = null;
51+
rejectedInput = null;
52+
53+
var inputReader = new GlobalCoverageInputReader();
54+
var accumulator = new GlobalCoverageCombinerAccumulator();
55+
var processedFiles = 0;
56+
var outputFullPath = StringUtil.IsNullOrWhiteSpace(outputFile) ? null : Path.GetFullPath(outputFile);
57+
foreach (var file in inputFiles)
58+
{
59+
if (Path.GetFileName(file).StartsWith("session-coverage-", StringComparison.OrdinalIgnoreCase) ||
60+
(outputFullPath is not null && PathsEqual(Path.GetFullPath(file), outputFullPath)))
61+
{
62+
continue;
63+
}
64+
65+
if (!inputReader.TryRead(file, reconciliationLease?.GetCertifiedInput(file), out var globalCoverage) || globalCoverage is null)
66+
{
67+
rejectedInput = file;
68+
return false;
69+
}
70+
71+
// Reporting remains caller-owned so the tracer can stay silent while the CLI preserves
72+
// its existing per-file progress messages.
73+
onFileProcessed?.Invoke(file);
74+
accumulator.Add(globalCoverage);
75+
processedFiles++;
76+
}
77+
78+
if (processedFiles == 0)
79+
{
80+
return false;
81+
}
82+
83+
globalCoverageInfo = accumulator.Materialize();
84+
return true;
85+
}
86+
87+
private static bool HasProtocolMarkers(string inputFolder)
88+
=> Directory.EnumerateFiles(inputFolder, GlobalCoverageProtocol.PendingMarkerPattern, SearchOption.TopDirectoryOnly).Any() ||
89+
Directory.EnumerateFiles(inputFolder, GlobalCoverageProtocol.ReadyMarkerPattern, SearchOption.TopDirectoryOnly).Any() ||
90+
Directory.EnumerateFiles(inputFolder, GlobalCoverageProtocol.CommandOwnerClaimPattern, SearchOption.TopDirectoryOnly).Any();
91+
92+
private static string[] GetInputFilesBounded(string inputFolder)
93+
{
94+
var files = Directory.EnumerateFiles(inputFolder, "*.json", SearchOption.TopDirectoryOnly)
95+
.Take(MaximumInputFiles + 1)
96+
.ToArray();
97+
if (files.Length > MaximumInputFiles)
98+
{
99+
throw new InvalidDataException("The global coverage input-file limit was exceeded.");
100+
}
101+
102+
return files;
103+
}
104+
105+
private static bool PathsEqual(string first, string second)
106+
=> string.Equals(
107+
first,
108+
second,
109+
FrameworkDescription.Instance.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
110+
}

0 commit comments

Comments
 (0)