diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index a3c572bf0259..33b20539cc83 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -1686,6 +1686,16 @@ stages: parameters: framework: $(framework) includeMultiVersions: false + # The global coverage regression tests exercise two Coverlet versions. Keep the + # Windows download surgical instead of pulling every multi-version sample. + - template: steps/download-artifact.yml + parameters: + artifact: samples-multi-version-net8.0 + path: $(outputDir)/publish + patterns: | + Samples.NUnitGlobalCoverageMemory/release_net8.0_3.2.0/** + Samples.NUnitGlobalCoverageMemory/release_net8.0_6.0.0/** + condition: and(succeeded(), eq(variables['framework'], 'net8.0'), ne(variables['area'], 'ASM')) # Cosmos is _way_ to flaky at the moment. Try enabling again at a later time # - powershell: | diff --git a/Datadog.Trace.Samples.g.sln b/Datadog.Trace.Samples.g.sln index 486ff523abfd..afc5ab076c14 100644 --- a/Datadog.Trace.Samples.g.sln +++ b/Datadog.Trace.Samples.g.sln @@ -227,6 +227,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.XUnitTests", "trace EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.NUnitTests", "tracer\test\test-applications\integrations\Samples.NUnitTests\Samples.NUnitTests.csproj", "{BC998ACD-353B-4A56-8A56-DF6200E141B6}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.NUnitGlobalCoverageMemory", "tracer\test\test-applications\integrations\Samples.NUnitGlobalCoverageMemory\Samples.NUnitGlobalCoverageMemory.csproj", "{EC40C8E7-DC12-4645-8481-E1536726403C}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.MSTestTests", "tracer\test\test-applications\integrations\Samples.MSTestTests\Samples.MSTestTests.csproj", "{8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.Msmq", "tracer\test\test-applications\integrations\Samples.Msmq\Samples.Msmq.csproj", "{662B587F-97B5-4CEF-ABF9-6C76A6DBD29E}" @@ -657,6 +659,10 @@ Global {BC998ACD-353B-4A56-8A56-DF6200E141B6}.Debug|Any CPU.Build.0 = Debug|Any CPU {BC998ACD-353B-4A56-8A56-DF6200E141B6}.Release|Any CPU.ActiveCfg = Release|Any CPU {BC998ACD-353B-4A56-8A56-DF6200E141B6}.Release|Any CPU.Build.0 = Release|Any CPU + {EC40C8E7-DC12-4645-8481-E1536726403C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC40C8E7-DC12-4645-8481-E1536726403C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC40C8E7-DC12-4645-8481-E1536726403C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC40C8E7-DC12-4645-8481-E1536726403C}.Release|Any CPU.Build.0 = Release|Any CPU {8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49}.Debug|Any CPU.Build.0 = Debug|Any CPU {8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -1174,6 +1180,7 @@ Global {3538EF5E-377E-430A-AFB8-F2DB5FAEDE95} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {4AD438D9-D4E3-4EB5-8851-89DB4D1CFB9C} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {BC998ACD-353B-4A56-8A56-DF6200E141B6} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} + {EC40C8E7-DC12-4645-8481-E1536726403C} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {662B587F-97B5-4CEF-ABF9-6C76A6DBD29E} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {94B50277-FB50-4B42-BA79-770ADB24CB80} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} diff --git a/Datadog.Trace.sln b/Datadog.Trace.sln index 1b8ff4771f97..86d274a89c89 100644 --- a/Datadog.Trace.sln +++ b/Datadog.Trace.sln @@ -297,6 +297,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.XUnitTests", "trace EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.NUnitTests", "tracer\test\test-applications\integrations\Samples.NUnitTests\Samples.NUnitTests.csproj", "{BC998ACD-353B-4A56-8A56-DF6200E141B6}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.NUnitGlobalCoverageMemory", "tracer\test\test-applications\integrations\Samples.NUnitGlobalCoverageMemory\Samples.NUnitGlobalCoverageMemory.csproj", "{EC40C8E7-DC12-4645-8481-E1536726403C}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.MSTestTests", "tracer\test\test-applications\integrations\Samples.MSTestTests\Samples.MSTestTests.csproj", "{8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.Msmq", "tracer\test\test-applications\integrations\Samples.Msmq\Samples.Msmq.csproj", "{662B587F-97B5-4CEF-ABF9-6C76A6DBD29E}" @@ -945,6 +947,10 @@ Global {BC998ACD-353B-4A56-8A56-DF6200E141B6}.Debug|Any CPU.Build.0 = Debug|Any CPU {BC998ACD-353B-4A56-8A56-DF6200E141B6}.Release|Any CPU.ActiveCfg = Release|Any CPU {BC998ACD-353B-4A56-8A56-DF6200E141B6}.Release|Any CPU.Build.0 = Release|Any CPU + {EC40C8E7-DC12-4645-8481-E1536726403C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC40C8E7-DC12-4645-8481-E1536726403C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC40C8E7-DC12-4645-8481-E1536726403C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC40C8E7-DC12-4645-8481-E1536726403C}.Release|Any CPU.Build.0 = Release|Any CPU {8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49}.Debug|Any CPU.Build.0 = Debug|Any CPU {8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -1667,6 +1673,7 @@ Global {3538EF5E-377E-430A-AFB8-F2DB5FAEDE95} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {4AD438D9-D4E3-4EB5-8851-89DB4D1CFB9C} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {BC998ACD-353B-4A56-8A56-DF6200E141B6} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} + {EC40C8E7-DC12-4645-8481-E1536726403C} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {8EAABFB9-8A47-4B11-AD7F-AC8B373CDE49} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {662B587F-97B5-4CEF-ABF9-6C76A6DBD29E} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} {94B50277-FB50-4B42-BA79-770ADB24CB80} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A} diff --git a/tracer/build/PackageVersionsGeneratorDefinitions.json b/tracer/build/PackageVersionsGeneratorDefinitions.json index 5b308537a731..41bece896fed 100644 --- a/tracer/build/PackageVersionsGeneratorDefinitions.json +++ b/tracer/build/PackageVersionsGeneratorDefinitions.json @@ -614,6 +614,24 @@ } ] }, + { + "IntegrationName": "NUnitGlobalCoverageMemoryCoverlet", + "SampleProjectName": "Samples.NUnitGlobalCoverageMemory", + "NugetPackageSearchName": "coverlet.collector", + "MinVersion": "3.2.0", + "MaxVersionExclusive": "6.0.1", + "SpecificVersions": [ + "3.2.0", + "6.0.0" + ], + "VersionConditions": [ + { + "IncludeOnlyTargetFrameworks": [ + "net8.0" + ] + } + ] + }, { "IntegrationName": "NUnitRetries", "SampleProjectName": "Samples.NUnitTestsRetries", diff --git a/tracer/build/PackageVersionsLatestMajors.g.props b/tracer/build/PackageVersionsLatestMajors.g.props index 6854289d9971..ba31838eb4d8 100644 --- a/tracer/build/PackageVersionsLatestMajors.g.props +++ b/tracer/build/PackageVersionsLatestMajors.g.props @@ -4602,6 +4602,18 @@ NOTE: This code was generated by the GeneratePackageVersions tool. To safely None Samples.NUnitTests + + ApiVersion=3.2.0;RestoreRecursive=false;BuildProjectReferences=false + net8.0 + None + Samples.NUnitGlobalCoverageMemory + + + ApiVersion=6.0.0;RestoreRecursive=false;BuildProjectReferences=false + net8.0 + None + Samples.NUnitGlobalCoverageMemory + ApiVersion=3.14.0;RestoreRecursive=false;BuildProjectReferences=false net48 diff --git a/tracer/build/PackageVersionsLatestMinors.g.props b/tracer/build/PackageVersionsLatestMinors.g.props index be48b7d76d47..737c0b678bed 100644 --- a/tracer/build/PackageVersionsLatestMinors.g.props +++ b/tracer/build/PackageVersionsLatestMinors.g.props @@ -23382,6 +23382,18 @@ NOTE: This code was generated by the GeneratePackageVersions tool. To safely None Samples.NUnitTests + + ApiVersion=3.2.0;RestoreRecursive=false;BuildProjectReferences=false + net8.0 + None + Samples.NUnitGlobalCoverageMemory + + + ApiVersion=6.0.0;RestoreRecursive=false;BuildProjectReferences=false + net8.0 + None + Samples.NUnitGlobalCoverageMemory + ApiVersion=3.6.1;RestoreRecursive=false;BuildProjectReferences=false net48 diff --git a/tracer/build/PackageVersionsLatestSpecific.g.props b/tracer/build/PackageVersionsLatestSpecific.g.props index f8cf0da6a4e6..ea08b90854e1 100644 --- a/tracer/build/PackageVersionsLatestSpecific.g.props +++ b/tracer/build/PackageVersionsLatestSpecific.g.props @@ -5844,6 +5844,18 @@ NOTE: This code was generated by the GeneratePackageVersions tool. To safely None Samples.NUnitTests + + ApiVersion=3.2.0;RestoreRecursive=false;BuildProjectReferences=false + net8.0 + None + Samples.NUnitGlobalCoverageMemory + + + ApiVersion=6.0.0;RestoreRecursive=false;BuildProjectReferences=false + net8.0 + None + Samples.NUnitGlobalCoverageMemory + ApiVersion=3.6.1;RestoreRecursive=false;BuildProjectReferences=false net48 diff --git a/tracer/build/_build/Honeypot/IntegrationGroups.cs b/tracer/build/_build/Honeypot/IntegrationGroups.cs index fc3253fad06f..b3f9a86619f4 100644 --- a/tracer/build/_build/Honeypot/IntegrationGroups.cs +++ b/tracer/build/_build/Honeypot/IntegrationGroups.cs @@ -118,7 +118,7 @@ static IntegrationMap() NugetPackages.Add("vstest.console.arm64", Array.Empty()); NugetPackages.Add("WebDriver", new[] { "Selenium.WebDriver" }); NugetPackages.Add("Microsoft.AspNetCore.StaticFiles", new [] { "Microsoft.AspNetCore.StaticFiles" }); - NugetPackages.Add("coverlet.core", Array.Empty()); + NugetPackages.Add("coverlet.core", new[] { "coverlet.collector" }); NugetPackages.Add("Microsoft.AspNetCore.Session", new [] { "Microsoft.AspNetCore.Session" }); NugetPackages.Add("Microsoft.TestPlatform.PlatformAbstractions", Array.Empty()); NugetPackages.Add("Microsoft.VisualStudio.TraceDataCollector", Array.Empty()); diff --git a/tracer/build/supported_versions.json b/tracer/build/supported_versions.json index e341208329bb..b404a2d556be 100644 --- a/tracer/build/supported_versions.json +++ b/tracer/build/supported_versions.json @@ -550,7 +550,17 @@ "assemblyName": "coverlet.core", "minAssemblyVersionInclusive": "3.0.0", "maxAssemblyVersionInclusive": "6.65535.65535", - "packages": [] + "packages": [ + { + "name": "coverlet.collector", + "minVersionAvailableInclusive": "1.0.0", + "minVersionSupportedInclusive": "3.0.0", + "minVersionTestedInclusive": "3.2.0", + "maxVersionSupportedInclusive": "6.0.4", + "maxVersionAvailableInclusive": "10.0.1", + "maxVersionTestedInclusive": "6.0.0" + } + ] }, { "integrationName": "DotnetTest", diff --git a/tracer/src/Datadog.Trace.Coverage.collector/InProcCoverageCollector.cs b/tracer/src/Datadog.Trace.Coverage.collector/InProcCoverageCollector.cs index ec11ff0ded91..35805b081448 100644 --- a/tracer/src/Datadog.Trace.Coverage.collector/InProcCoverageCollector.cs +++ b/tracer/src/Datadog.Trace.Coverage.collector/InProcCoverageCollector.cs @@ -5,8 +5,8 @@ using System; using System.IO; +using Datadog.Trace.Ci; using Datadog.Trace.Ci.Coverage; -using Datadog.Trace.Vendors.Newtonsoft.Json; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollector.InProcDataCollector; using Microsoft.VisualStudio.TestPlatform.ObjectModel.InProcDataCollector; @@ -58,6 +58,12 @@ public void TestSessionStart(TestSessionStartArgs testSessionStartArgs) { _outputPathValue = outputPath; } + + if (CoverageReporter.Handler is DefaultWithGlobalCoverageEventHandler coverageHandler) + { + var outputDirectory = _outputPathValue ?? Environment.CurrentDirectory; + coverageHandler.RegisterCollectorOutputDirectory(outputDirectory); + } } /// @@ -82,20 +88,6 @@ public void TestCaseEnd(TestCaseEndArgs testCaseEndArgs) /// Test session end arguments public void TestSessionEnd(TestSessionEndArgs testSessionEndArgs) { - if (CoverageReporter.Handler is DefaultWithGlobalCoverageEventHandler coverageHandler) - { - var globalCoverage = coverageHandler.GetCodeCoveragePercentage(); - var outputPath = $"coverage-{DateTime.Now:yyyy-MM-dd_HH_mm_ss}-{Guid.NewGuid():n}.json"; - if (!string.IsNullOrEmpty(_outputPathValue)) - { - outputPath = Path.Combine(_outputPathValue, outputPath); - } - - using var fileStream = File.OpenWrite(outputPath); - using var streamWriter = new StreamWriter(fileStream); - using var jsonWriter = new JsonTextWriter(streamWriter) { CloseOutput = true }; - var jsonSerializer = new JsonSerializer(); - jsonSerializer.Serialize(jsonWriter, globalCoverage); - } + CoverageReporter.FinalizeGlobalCoverage(); } } diff --git a/tracer/src/Datadog.Trace.Tools.Runner/CoverageUtils.cs b/tracer/src/Datadog.Trace.Tools.Runner/CoverageUtils.cs index b4ea8516bd83..c300dddd3c1c 100644 --- a/tracer/src/Datadog.Trace.Tools.Runner/CoverageUtils.cs +++ b/tracer/src/Datadog.Trace.Tools.Runner/CoverageUtils.cs @@ -4,11 +4,9 @@ // using System; -using System.Collections.Generic; using System.IO; -using System.Text; +using Datadog.Trace.Ci.Coverage; using Datadog.Trace.Ci.Coverage.Models.Global; -using Datadog.Trace.Vendors.Newtonsoft.Json; using Spectre.Console; namespace Datadog.Trace.Tools.Runner; @@ -33,23 +31,27 @@ public static bool TryCombineAndGetTotalCoverage(string inputFolder, string outp return false; } - if (!TryCombineAndGetTotalCoverage(inputFolder, out globalCoverageInfo, useStdOut)) + if (!TryLoadAndCombine(inputFolder, outputFile, out globalCoverageInfo, useStdOut)) { return false; } - using var fStream = File.OpenWrite(outputFile); - using var sWriter = new StreamWriter(fStream, Encoding.UTF8, 4096, false); if (useStdOut) { Utils.WriteSuccess($"Writing {outputFile}"); } - new JsonSerializer().Serialize(sWriter, globalCoverageInfo); + var writer = new GlobalCoverageArtifactWriter(); + using var stagedOutput = writer.StageReplace(outputFile, globalCoverageInfo); + stagedOutput.Commit(); return true; } - private static bool TryCombineAndGetTotalCoverage(string inputFolder, out GlobalCoverageInfo globalCoverageInfo, bool useStdOut = true) + private static bool TryLoadAndCombine( + string inputFolder, + string outputFile, + out GlobalCoverageInfo globalCoverageInfo, + bool useStdOut) { globalCoverageInfo = default; @@ -76,7 +78,11 @@ private static bool TryCombineAndGetTotalCoverage(string inputFolder, out Global var jsonFiles = Array.Empty(); try { - jsonFiles = Directory.GetFiles(inputFolder, "*.json", SearchOption.TopDirectoryOnly); + if (!GlobalCoverageFileCombiner.TryAcquireInputFiles(inputFolder, expectedRunToken: null, out jsonFiles)) + { + return false; + } + if (jsonFiles.Length == 0) { if (useStdOut) @@ -93,34 +99,23 @@ private static bool TryCombineAndGetTotalCoverage(string inputFolder, out Global AnsiConsole.WriteException(ex); } - List globalCoverages = new(); - foreach (var file in jsonFiles) + Action onFileProcessed = useStdOut ? file => Utils.WriteSuccess($"Processing: {file}") : null; + if (!GlobalCoverageFileCombiner.TryCombine( + jsonFiles, + outputFile, + requireAllInputs: false, + onFileProcessed: onFileProcessed, + out globalCoverageInfo, + out var rejectedInput)) { - var fileContent = File.ReadAllText(file); - try + if (useStdOut && rejectedInput is not null) { - if (JsonConvert.DeserializeObject(fileContent) is { } gCoverageInfo) - { - if (useStdOut) - { - Utils.WriteSuccess($"Processing: {file}"); - } - - globalCoverages.Add(gCoverageInfo); - } - else if (useStdOut) - { - Utils.WriteSuccess($"Ignored: {file}"); - } - } - catch (Exception ex) - { - Utils.WriteError($"Error processing {file}"); - AnsiConsole.WriteException(ex); + Utils.WriteError($"Error processing {rejectedInput}"); } + + return false; } - globalCoverageInfo = GlobalCoverageInfo.Combine(globalCoverages.ToArray()); return true; } } diff --git a/tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml b/tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml index 4ae7ff954b05..8f4118fe1cc4 100644 --- a/tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml +++ b/tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml @@ -770,6 +770,7 @@ + @@ -945,6 +946,7 @@ + diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageContextAdmission.cs b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageContextAdmission.cs new file mode 100644 index 000000000000..e71c47d40594 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageContextAdmission.cs @@ -0,0 +1,34 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.Ci.Coverage; + +internal abstract class CoverageContextAdmission +{ + public static readonly CoverageContextAdmission Noop = new NoopCoverageContextAdmission(); + + public abstract void CommitInstalled(); + + public abstract void FailStart(GlobalCoverageFailureReason reason); + + public abstract void Release(); + + private sealed class NoopCoverageContextAdmission : CoverageContextAdmission + { + public override void CommitInstalled() + { + } + + public override void FailStart(GlobalCoverageFailureReason reason) + { + } + + public override void Release() + { + } + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageContextContainer.cs b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageContextContainer.cs index 83a02375a185..6894a3f337c1 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageContextContainer.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageContextContainer.cs @@ -2,114 +2,216 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // + #nullable enable +using System; using System.Collections.Generic; using System.Reflection; -using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading; +using Datadog.Trace.Ci.Coverage.Metadata; +using Datadog.Trace.Util; namespace Datadog.Trace.Ci.Coverage; -/// -/// Coverage context container instance -/// -internal sealed class CoverageContextContainer +internal sealed class CoverageContextContainer : IDisposable { - private readonly List _container = new(); + private readonly object _gate = new(); + private readonly List _modules = new(); + private readonly ModuleValue.BufferKind _bufferKind; private ModuleValue? _currentModuleValue; + private Action? _onDisposed; + private int _activeExecutionContexts; + private int _closed; + private int _disposeRequested; + private int _disposed; - /// - /// Initializes a new instance of the class. - /// - /// State instance - public CoverageContextContainer(object? state = null) + public CoverageContextContainer(object? state = null, ModuleValue.BufferKind bufferKind = ModuleValue.BufferKind.Context) { State = state; + _bufferKind = bufferKind; } - /// - /// Gets or sets the context container state - /// public object? State { get; set; } - /// - /// Gets the current module value - /// - /// Module instance - /// Current module instance - internal ModuleValue? GetModuleValue(Module module) + public bool IsClosed => Volatile.Read(ref _closed) != 0; + + public void OnExecutionContextEntered() => Interlocked.Increment(ref _activeExecutionContexts); + + public void OnExecutionContextExited() + { + if (Interlocked.Decrement(ref _activeExecutionContexts) == 0 && Volatile.Read(ref _disposeRequested) != 0) + { + Dispose(); + } + } + + public ModuleValue? GetModuleValue(Module module) { - if (_currentModuleValue is { } moduleValue && moduleValue.Module == module) + // This is the normal probe path. Session closure is coordinated by the test lifecycle, + // so preserve the historical lock-free lookup and reserve the lock for cache misses. + if (_closed != 0) { - return moduleValue; + return null; } - return GetModuleValueSlow(module); + if (_currentModuleValue is { } current && current.Module == module) + { + return current; + } + + lock (_gate) + { + if (_closed != 0) + { + return null; + } + + return FindModuleValue(module); + } } - private ModuleValue? GetModuleValueSlow(Module module) + public bool TryGetOrAddModuleValue( + ModuleCoverageMetadata metadata, + Module module, + int rawByteLength, + out ModuleValue? moduleValue) { - var container = _container; - lock (container) + moduleValue = GetModuleValue(module); + if (moduleValue is not null) + { + return true; + } + + lock (_gate) { - for (var i = 0; i < container.Count; i++) + if (_closed != 0) { - if (container[i] is { } moduleValueItem && moduleValueItem.Module == module) - { - _currentModuleValue = moduleValueItem; - return moduleValueItem; - } + moduleValue = null; + return false; + } + + moduleValue = FindModuleValue(module); + if (moduleValue is not null) + { + return true; + } + + var provisional = new ModuleValue(metadata, module, rawByteLength, _bufferKind); + try + { + _modules.Add(provisional); + } + catch + { + // List growth can fail after the native buffer was allocated, so the unpublished value must release it here. + provisional.Dispose(); + throw; + } + + _currentModuleValue = provisional; + moduleValue = provisional; + return true; + } + } + + public bool TryCloseAndGetModules(out IReadOnlyList modules) + { + lock (_gate) + { + if (_closed != 0) + { + modules = Array.Empty(); + return false; } + + _closed = 1; + _currentModuleValue = null; + modules = _modules; + return true; } + } - return null; + public ModuleValue[] SnapshotModules(int maximumModules = int.MaxValue) + { + lock (_gate) + { + if (_modules.Count > maximumModules) + { + ThrowHelper.ThrowInvalidOperationException("The global coverage fallback contains too many modules."); + } + + return _modules.Count == 0 ? Array.Empty() : _modules.ToArray(); + } } - /// - /// Stores module data into the context - /// - /// Module instance - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void Add(ModuleValue module) + public void Clear() => Dispose(); + + public void DisposeWhenExecutionContextsAreInactive(Action onDisposed) { - var container = _container; - lock (container) + // A flowed ExecutionContext may still be executing instrumented code with a raw pointer cached + // in an IL local. Inactive contexts cannot hold a live probe frame, and future probes observe + // the closed flag and use the global fallback, so the last active exit is the safe reclamation point. + _onDisposed = onDisposed; + Volatile.Write(ref _disposeRequested, 1); + if (Volatile.Read(ref _activeExecutionContexts) == 0) { - container.Add(module); - _currentModuleValue = module; + Dispose(); } } - /// - /// Clear context data - /// - internal void Clear() + public void Dispose() { - var container = _container; - lock (container) + ExceptionDispatchInfo? firstException = null; + Action? onDisposed = null; + lock (_gate) { - foreach (var moduleValue in container) + if (_disposed != 0) { - moduleValue.Dispose(); + return; } - container.Clear(); + _disposed = 1; + _closed = 1; _currentModuleValue = null; + try + { + foreach (var moduleValue in _modules) + { + try + { + moduleValue.Dispose(); + } + catch (Exception ex) + { + firstException ??= ExceptionDispatchInfo.Capture(ex); + } + } + } + finally + { + _modules.Clear(); + onDisposed = _onDisposed; + _onDisposed = null; + } } + + onDisposed?.Invoke(); + firstException?.Throw(); } - /// - /// Gets modules data from the context - /// - /// Instruction array from the context - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ModuleValue[] CloseContext() + private ModuleValue? FindModuleValue(Module module) { - var container = _container; - lock (container) + for (var i = 0; i < _modules.Count; i++) { - _currentModuleValue = null; - return container.Count == 0 ? [] : container.ToArray(); + if (_modules[i] is { } item && item.Module == module) + { + _currentModuleValue = item; + return item; + } } + + return null; } } diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageEventHandler.cs b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageEventHandler.cs index 7edcfa88f8b4..520c12bfb90b 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageEventHandler.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageEventHandler.cs @@ -2,92 +2,233 @@ // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // + #nullable enable +using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading; using Datadog.Trace.Ci.Telemetry; using Datadog.Trace.Telemetry; using Datadog.Trace.Telemetry.Metrics; +using Datadog.Trace.Util; namespace Datadog.Trace.Ci.Coverage; -/// -/// Coverage event handler -/// internal abstract class CoverageEventHandler { - private readonly AsyncLocal _asyncContext; - private readonly CoverageContextContainer _globalContainer; + private readonly AsyncLocal _asyncContext = new(OnAsyncContextChanged); + private readonly CoverageContextContainer _globalContainer = new(bufferKind: ModuleValue.BufferKind.GlobalFallback); + private readonly ContextDiagnostics _contextDiagnostics = new(); + private readonly Action _recordContextDisposed; protected CoverageEventHandler() { - _asyncContext = new(); - _globalContainer = new CoverageContextContainer(); + // Cache the instance delegate once; closed contexts may retain it briefly while an inherited + // ExecutionContext finishes, so allocating a new delegate for every test would be unnecessary churn. + _recordContextDisposed = _contextDiagnostics.RecordDisposed; } - /// - /// Gets the coverage local container - /// - internal CoverageContextContainer? Container => _asyncContext.Value; + public CoverageContextContainer? Container => _asyncContext.Value; - /// - /// Gets the coverage global container - /// - internal CoverageContextContainer GlobalContainer => _globalContainer; + public CoverageContextContainer GlobalContainer => _globalContainer; - /// - /// Start session - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void StartSession(string? testingFramework = null) + public CoverageSessionHandle StartSession(string? testingFramework = null) { - var telemetryTestingFramework = TelemetryHelper.GetTelemetryTestingFrameworkEnum(testingFramework); - TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageStarted(telemetryTestingFramework, MetricTags.CIVisibilityCoverageLibrary.Custom); - var context = new CoverageContextContainer(telemetryTestingFramework); - OnSessionStart(context); - _asyncContext.Value = context; + CoverageContextAdmission? admission = null; + CoverageContextContainer? context = null; + var transferred = false; + try + { + if (!TryBeginSessionStartAdmission(out admission)) + { + return CoverageSessionHandle.Invalid; + } + + var telemetryTestingFramework = TelemetryHelper.GetTelemetryTestingFrameworkEnum(testingFramework); + TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageStarted(telemetryTestingFramework, MetricTags.CIVisibilityCoverageLibrary.Custom); + context = CreateContext(telemetryTestingFramework); + OnSessionStart(context); + InstallContext(context); + admission.CommitInstalled(); + + var handle = new CoverageSessionHandle(this, context, admission); + _contextDiagnostics.RecordStarted(); + transferred = true; + return handle; + } + finally + { + if (!transferred) + { + if (context is not null) + { + if (ReferenceEquals(_asyncContext.Value, context)) + { + _asyncContext.Value = null; + } + + context.Dispose(); + } + + admission?.FailStart(GlobalCoverageFailureReason.StartFailed); + } + } } - /// - /// End async session - /// - /// Object instance with the final coverage report [MethodImpl(MethodImplOptions.AggressiveInlining)] - public object? EndSession() + public object? EndSession(CoverageSessionHandle? handle) { - if (_asyncContext.Value is { } context) + if (handle is null || !handle.IsValid) + { + return null; + } + + if (!ReferenceEquals(handle.Owner, this)) + { + ThrowHelper.ThrowInvalidOperationException("The coverage session handle belongs to another handler."); + } + + var context = handle.Context!; + if (ReferenceEquals(_asyncContext.Value, context)) { _asyncContext.Value = null; - var sessionEndData = OnSessionFinished(context); - if (context.State is MetricTags.CIVisibilityTestFramework { } telemetryTestingFramework) + } + + if (!context.TryCloseAndGetModules(out var modules)) + { + return null; + } + + _contextDiagnostics.RecordClosed(); + try + { + var sessionEndData = OnSessionFinished(context, modules); + if (context.State is MetricTags.CIVisibilityTestFramework telemetryTestingFramework) { TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageFinished(telemetryTestingFramework, MetricTags.CIVisibilityCoverageLibrary.Custom); } - OnClearContext(context); return sessionEndData; } + finally + { + try + { + context.DisposeWhenExecutionContextsAreInactive(_recordContextDisposed); + } + finally + { + handle.Admission.Release(); + } + } + } + + public void AbortSession(CoverageSessionHandle handle, GlobalCoverageFailureReason reason) + { + try + { + if (!ReferenceEquals(handle.Owner, this) || handle.Context is not { } context) + { + return; + } - return null; + if (ReferenceEquals(_asyncContext.Value, context)) + { + _asyncContext.Value = null; + } + + if (!context.TryCloseAndGetModules(out _)) + { + return; + } + + _contextDiagnostics.RecordClosed(); + MarkGlobalCoverageIncomplete(reason); + try + { + context.DisposeWhenExecutionContextsAreInactive(_recordContextDisposed); + } + finally + { + handle.Admission.Release(); + } + } + catch + { + // Abort is a structural no-throw cleanup path. The original functional exception wins. + } + } + + public void MarkProbeDataIncomplete(GlobalCoverageFailureReason reason) + { + try + { + MarkGlobalCoverageIncomplete(reason); + } + catch + { + // Never replace the probe exception with completeness bookkeeping. + } + } + + protected virtual bool TryBeginSessionStartAdmission(out CoverageContextAdmission admission) + { + admission = CoverageContextAdmission.Noop; + return true; + } + + protected virtual CoverageContextContainer CreateContext(object? state) => new(state); + + protected virtual void InstallContext(CoverageContextContainer context) => _asyncContext.Value = context; + + protected virtual void MarkGlobalCoverageIncomplete(GlobalCoverageFailureReason reason) + { } - /// - /// Method called when a session is started - /// - /// Coverage context container + protected void LogContextDiagnostics(long merged) + => _contextDiagnostics.Log(merged); + protected abstract void OnSessionStart(CoverageContextContainer context); - /// - /// Method called when a session is finished to process all coverage raw data. - /// - /// Coverage context container - /// Instance of the final coverage report - protected abstract object? OnSessionFinished(CoverageContextContainer context); - - /// - /// Method called when the context is cleared - /// - /// Context to be cleared - protected abstract void OnClearContext(CoverageContextContainer context); + protected abstract object? OnSessionFinished(CoverageContextContainer context, IReadOnlyList modules); + + private static void OnAsyncContextChanged(AsyncLocalValueChangedArgs args) + { + if (ReferenceEquals(args.PreviousValue, args.CurrentValue)) + { + return; + } + + // Instrumented methods cache raw counter pointers for the duration of the method. Track active + // ExecutionContexts here so session cleanup never adds synchronization to the probe hot path. + args.PreviousValue?.OnExecutionContextExited(); + args.CurrentValue?.OnExecutionContextEntered(); + } + + private sealed class ContextDiagnostics + { + private long _started; + private long _closed; + private long _disposed; + + public void RecordStarted() => Interlocked.Increment(ref _started); + + public void RecordClosed() => Interlocked.Increment(ref _closed); + + public void RecordDisposed() => Interlocked.Increment(ref _disposed); + + public void Log(long merged) + { + TestOptimization.Instance.Log.Debug( + "Global coverage context diagnostics: pid={ProcessId}, started={Started}, closed={Closed}, disposed={Disposed}, merged={Merged}.", + DomainMetadata.Instance.ProcessId, + Interlocked.Read(ref _started), + Interlocked.Read(ref _closed), + Interlocked.Read(ref _disposed), + merged); + } + } } diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter.cs b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter.cs index b2a8a870b73f..1f42627ca554 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter.cs @@ -7,6 +7,7 @@ using System; using System.ComponentModel; using System.Runtime.CompilerServices; +using System.Threading; using Datadog.Trace.Ci.Configuration; using Datadog.Trace.Util; @@ -19,7 +20,7 @@ namespace Datadog.Trace.Ci.Coverage; [EditorBrowsable(EditorBrowsableState.Never)] public static class CoverageReporter { - private static CoverageEventHandler _handler = CreateDefaultHandler(TestOptimization.Instance.Settings); + private static CoverageEventHandler? _handler; /// /// Gets or sets coverage handler @@ -28,14 +29,38 @@ public static class CoverageReporter internal static CoverageEventHandler Handler { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => _handler; + get => LazyInitializer.EnsureInitialized(ref _handler, static () => CreateDefaultHandler(TestOptimization.Instance.Settings))!; [MethodImpl(MethodImplOptions.AggressiveInlining)] - set => _handler = value ?? throw new ArgumentNullException(nameof(value)); + set + { + if (value is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(value)); + } + + Volatile.Write(ref _handler, value); + } } - internal static CoverageContextContainer? Container => _handler.Container; + internal static CoverageContextContainer? Container => Handler.Container; - internal static CoverageContextContainer GlobalContainer => _handler.GlobalContainer; + internal static CoverageContextContainer GlobalContainer => Handler.GlobalContainer; + + /// + /// Publishes the final global coverage snapshot and seals the process output, if coverage was used by this process. + /// + /// True when global coverage is not active or the process output was sealed completely. + internal static bool FinalizeGlobalCoverage(Action? onCompleted = null) + { + var handler = Volatile.Read(ref _handler); + if (handler is DefaultWithGlobalCoverageEventHandler globalHandler) + { + return globalHandler.FinalizeAndSeal(onCompleted); + } + + onCompleted?.Invoke(true); + return true; + } /// /// Creates the default coverage event handler for the current CI Visibility coverage mode. @@ -44,11 +69,11 @@ internal static CoverageEventHandler CreateDefaultHandler(TestOptimizationSettin { if (settings is null) { - throw new ArgumentNullException(nameof(settings)); + ThrowHelper.ThrowArgumentNullException(nameof(settings)); } return settings.TestsSkippingEnabled == true && StringUtil.IsNullOrWhiteSpace(settings.CodeCoveragePath) ? new DefaultCoverageEventHandler() - : new DefaultWithGlobalCoverageEventHandler(); + : new DefaultWithGlobalCoverageEventHandler(configuredOutputDirectory: settings.CodeCoveragePath); } } diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter`1.cs b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter`1.cs index c8d5f9de58f5..89d1c58a996a 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter`1.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageReporter`1.cs @@ -7,6 +7,7 @@ using System; using System.ComponentModel; using System.Reflection; +using System.Threading; using Datadog.Trace.Ci.Coverage.Metadata; using Datadog.Trace.Util; @@ -26,24 +27,21 @@ public static class CoverageReporter private static readonly TMeta Metadata; private static readonly Module Module; private static readonly int ModuleMemorySize; - private static ModuleValue _globalModuleValue; + private static ModuleValue? _globalModuleValue; static CoverageReporter() { - Metadata = new TMeta(); - Module = typeof(TMeta).Module; - ModuleMemorySize = Metadata.CoverageMode == 0 ? Metadata.TotalLines * sizeof(byte) : Metadata.TotalLines * sizeof(int); - - // Caching the module from the global shared container in case an async container is null - var globalCoverageContextContainer = CoverageReporter.GlobalContainer; - var globalModuleValue = globalCoverageContextContainer.GetModuleValue(Module); - if (globalModuleValue is null) + try { - globalModuleValue = new ModuleValue(Metadata, Module, ModuleMemorySize); - globalCoverageContextContainer.Add(globalModuleValue); + Metadata = new TMeta(); + Module = typeof(TMeta).Module; + ModuleMemorySize = CoverageMetadataValidator.ValidateAndGetRawByteLength(Metadata); + } + catch + { + CoverageReporter.Handler.MarkProbeDataIncomplete(GlobalCoverageFailureReason.ProbeDataIncomplete); + throw; } - - _globalModuleValue = globalModuleValue; } /// @@ -53,37 +51,71 @@ static CoverageReporter() /// Counters for the file public static unsafe void* GetFileCounter(int fileIndex) { - ModuleValue? module; - - // Try to get the async context container - if (CoverageReporter.Container is { } container) + var handler = CoverageReporter.Handler; + try { - // Get the module form the container - module = container.GetModuleValue(Module); + var module = GetModuleValue(handler); if (module is null) { - // If the module is not found, we create a new one for this container - module = new ModuleValue(Metadata, Module, ModuleMemorySize); - container.Add(module); + ThrowHelper.ThrowInvalidOperationException("The global coverage buffer is unexpectedly closed."); } + + return GetPointer(module, fileIndex); } - else + catch { - // If there's no async context container then we use the module from the global shared container. - module = _globalModuleValue; + handler.MarkProbeDataIncomplete(GlobalCoverageFailureReason.ProbeDataIncomplete); + throw; } + } - if (module.FilesLines == IntPtr.Zero) + private static ModuleValue? GetModuleValue(CoverageEventHandler handler) + { + if (handler.Container is { } container && + container.TryGetOrAddModuleValue( + Metadata, + Module, + ModuleMemorySize, + out var module) && + module is not null) { - ThrowHelper.ThrowNullReferenceException("Counter memory was disposed."); + return module; } - // Gets the file counter by using the file offset over the global module memory segment - if (Metadata.CoverageMode == 0) + return TryGetOrCreateGlobalModuleValue(handler); + } + + private static ModuleValue? TryGetOrCreateGlobalModuleValue(CoverageEventHandler handler) + { + if (Volatile.Read(ref _globalModuleValue) is { } cached) { - return ((byte*)module.FilesLines) + Metadata.GetOffset(fileIndex); + return cached; + } + + if (!handler.GlobalContainer.TryGetOrAddModuleValue( + Metadata, + Module, + ModuleMemorySize, + out var module) || module is null) + { + return null; + } + + Interlocked.CompareExchange(ref _globalModuleValue, module, null); + return Volatile.Read(ref _globalModuleValue)!; + } + + private static int GetByteOffset(int fileIndex) + => Metadata.CoverageMode == 0 ? Metadata.GetOffset(fileIndex) : Metadata.GetOffset(fileIndex) * sizeof(int); + + private static unsafe void* GetPointer(ModuleValue module, int fileIndex) + { + var filesLines = module.FilesLines; + if (filesLines == IntPtr.Zero) + { + ThrowHelper.ThrowNullReferenceException("Counter memory was disposed."); } - return ((int*)module.FilesLines) + Metadata.GetOffset(fileIndex); + return (byte*)filesLines + GetByteOffset(fileIndex); } } diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/CoverageSessionHandle.cs b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageSessionHandle.cs new file mode 100644 index 000000000000..9b17fe78d967 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/CoverageSessionHandle.cs @@ -0,0 +1,40 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class CoverageSessionHandle +{ + public static readonly CoverageSessionHandle Invalid = new(); + + private CoverageSessionHandle() + { + Owner = null; + Context = null; + Admission = CoverageContextAdmission.Noop; + } + + public CoverageSessionHandle(CoverageEventHandler owner, CoverageContextContainer context, CoverageContextAdmission admission) + { + Owner = owner; + Context = context; + Admission = admission; + } + + public CoverageEventHandler? Owner { get; } + + public CoverageContextContainer? Context { get; } + + public CoverageContextAdmission Admission { get; } + + public bool IsValid => Owner is not null && Context is not null; + + public void AbortIncomplete(GlobalCoverageFailureReason reason) + { + Owner?.AbortSession(this, reason); + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/DefaultCoverageEventHandler.cs b/tracer/src/Datadog.Trace/Ci/Coverage/DefaultCoverageEventHandler.cs index b818aba8682e..710947af159e 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/DefaultCoverageEventHandler.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/DefaultCoverageEventHandler.cs @@ -26,79 +26,62 @@ protected override void OnSessionStart(CoverageContextContainer context) { } - protected override unsafe object? OnSessionFinished(CoverageContextContainer context) + protected override object? OnSessionFinished(CoverageContextContainer context, IReadOnlyList modules) + => ProcessSessionFinished(modules, out _); + + protected object? ProcessSessionFinished(IReadOnlyList modules, out ModuleCoverageData[] moduleCoverage) { try { - var modules = context.CloseContext(); - + moduleCoverage = new ModuleCoverageData[modules.Count]; Dictionary? fileDictionary = null; - var fileBitmapBuffer = stackalloc byte[512]; - foreach (var moduleValue in modules) + for (var moduleIndex = 0; moduleIndex < modules.Count; moduleIndex++) { - foreach (var moduleFile in moduleValue.Metadata.Files) + var capturedModule = ModuleCoverageData.Capture(modules[moduleIndex]); + moduleCoverage[moduleIndex] = capturedModule; + for (var fileIndex = 0; fileIndex < capturedModule.Metadata.Files.Length; fileIndex++) { - var fileBitmapLastExecutableLine = moduleFile.LastExecutableLine; - var fileBitmapSize = FileBitmap.GetSize(fileBitmapLastExecutableLine); - using var fileBitmap = fileBitmapSize <= 512 ? new FileBitmap(fileBitmapBuffer, fileBitmapSize) : new FileBitmap(new byte[fileBitmapSize]); - if (moduleValue.Metadata.CoverageMode == 0) + var executedBitmap = capturedModule.ExecutedBitmaps[fileIndex]; + if (executedBitmap is null) { - var filesLines = (byte*)moduleValue.FilesLines + moduleFile.Offset; - for (var i = 0; i < fileBitmapLastExecutableLine; i++) - { - if (filesLines[i] > 0) - { - fileBitmap.Set(i + 1); - } - } - } - else if (moduleValue.Metadata.CoverageMode == 1) - { - var filesLines = (int*)moduleValue.FilesLines + moduleFile.Offset; - for (var i = 0; i < fileBitmapLastExecutableLine; i++) - { - if (filesLines[i] > 0) - { - fileBitmap.Set(i + 1); - } - } + continue; } - if (fileBitmap.HasActiveBits()) + var moduleFile = capturedModule.Metadata.Files[fileIndex]; + FileCoverage? fileCoverage; + if (fileDictionary is null) { - FileCoverage? fileCoverage; - if (fileDictionary is null) + fileCoverage = new FileCoverage { - fileCoverage = new FileCoverage - { - FileName = CIEnvironmentValues.Instance.MakeRelativePathFromSourceRoot(moduleFile.Path, false) - }; + FileName = CIEnvironmentValues.Instance.MakeRelativePathFromSourceRoot(moduleFile.Path, false) + }; - fileDictionary = new Dictionary - { - [moduleFile.Path] = fileCoverage - }; - } - else if (!fileDictionary.TryGetValue(moduleFile.Path, out fileCoverage)) + fileDictionary = new Dictionary + { + [moduleFile.Path] = fileCoverage + }; + } + else if (!fileDictionary.TryGetValue(moduleFile.Path, out fileCoverage)) + { + fileCoverage = new FileCoverage { - fileCoverage = new FileCoverage - { - FileName = CIEnvironmentValues.Instance.MakeRelativePathFromSourceRoot(moduleFile.Path, false) - }; + FileName = CIEnvironmentValues.Instance.MakeRelativePathFromSourceRoot(moduleFile.Path, false) + }; - fileDictionary[moduleFile.Path] = fileCoverage; - } + fileDictionary[moduleFile.Path] = fileCoverage; + } - if (fileCoverage.Bitmap is { } bitmap) - { - using var currentBitmap = new FileBitmap(bitmap); - var mergedBitmap = fileBitmap | currentBitmap; - fileCoverage.Bitmap = mergedBitmap.GetInternalArrayOrToArrayAndDispose(); - } - else - { - fileCoverage.Bitmap = fileBitmap.ToArray(); - } + if (fileCoverage.Bitmap is { } bitmap) + { + using var capturedBitmap = new FileBitmap(executedBitmap); + using var currentBitmap = new FileBitmap(bitmap); + var mergedBitmap = capturedBitmap | currentBitmap; + fileCoverage.Bitmap = mergedBitmap.GetInternalArrayOrToArrayAndDispose(); + } + else + { + // The accumulator only reads the bitmap, so the per-test payload can reuse it. + fileCoverage.Bitmap = executedBitmap; } } } @@ -130,9 +113,4 @@ protected override void OnSessionStart(CoverageContextContainer context) throw; } } - - protected override void OnClearContext(CoverageContextContainer context) - { - context.Clear(); - } } diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/DefaultWithGlobalCoverageEventHandler.cs b/tracer/src/Datadog.Trace/Ci/Coverage/DefaultWithGlobalCoverageEventHandler.cs index 363b4abe6c1a..93b3a7de3407 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/DefaultWithGlobalCoverageEventHandler.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/DefaultWithGlobalCoverageEventHandler.cs @@ -1,173 +1,401 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // + #nullable enable using System; using System.Collections.Generic; -using System.Diagnostics; -using System.Reflection; -using Datadog.Trace.Ci.CiEnvironment; -using Datadog.Trace.Ci.Coverage.Metadata; -using Datadog.Trace.Ci.Coverage.Models.Global; -using Datadog.Trace.Ci.Coverage.Util; +using System.Threading; using Datadog.Trace.Telemetry; using Datadog.Trace.Util; -using Datadog.Trace.Util.Json; -using Datadog.Trace.Vendors.Newtonsoft.Json; -using Datadog.Trace.Vendors.Serilog.Events; namespace Datadog.Trace.Ci.Coverage; internal sealed class DefaultWithGlobalCoverageEventHandler : DefaultCoverageEventHandler { - private readonly List _coverages = new(); + private readonly object _lifecycleGate = new(); + private readonly GlobalCoverageAccumulator _accumulator; + private readonly GlobalCoverageOutputManager _outputManager; + private int _inFlightStarts; + private int _activeContexts; + private int _inFlightSnapshots; + private LifecycleState _state; + private Action? _sealCompleted; + private bool _sealStarted; + private bool _sealRequested; + private bool _sealedComplete; + + public DefaultWithGlobalCoverageEventHandler( + GlobalCoverageAccumulatorLimits? limits = null, + string? configuredOutputDirectory = null, + Func? runIdProvider = null) + { + _accumulator = new GlobalCoverageAccumulator(limits); + _outputManager = new GlobalCoverageOutputManager( + configuredOutputDirectory, + Environment.CurrentDirectory, + runIdProvider ?? (() => TestOptimization.Instance.RunId)); + } - protected override void OnSessionStart(CoverageContextContainer context) + private enum AdmissionState { - if (context is not null) + Starting, + Active, + Released, + } + + private enum LifecycleState + { + Running, + Completing, + Sealed, + } + + public GlobalCoverageSnapshotResult AcquireGlobalCoverageSnapshot() + { + var admission = new SnapshotAdmission(this); + lock (_lifecycleGate) { - lock (_coverages) + if (_state != LifecycleState.Running) { - _coverages.Add(context); + return GlobalCoverageSnapshotResult.Suppressed(_accumulator.FailureReason); } - base.OnSessionStart(context); + _inFlightSnapshots++; + } + + try + { + var result = _accumulator.AcquireSnapshot(GlobalContainer, admission.Release); + if (result.Status != GlobalCoverageSnapshotStatus.Success) + { + admission.Release(); + } + + return result; + } + catch + { + admission.Release(); + throw; + } + } + + public bool TryCommit(GlobalCoverageSnapshot snapshot, Action action) + => _accumulator.TryCommit(snapshot, action); + + public bool RegisterCollectorOutputDirectory(string directory) + { + var registered = _outputManager.RegisterCollectorAndFreeze(directory); + if (!registered) + { + _accumulator.Suppress(GlobalCoverageFailureReason.OutputCommitFailed); } + + return registered; } - public void Clear() + public bool FinalizeAndSeal(Action? onCompleted = null) { - lock (_coverages) + var completeNow = false; + bool? completed = null; + lock (_lifecycleGate) { - foreach (var coverage in _coverages) + if (_state == LifecycleState.Sealed) { - coverage.Clear(); + completed = _sealedComplete; } + else + { + if (onCompleted is not null) + { + _sealCompleted += onCompleted; + } - _coverages.Clear(); + _sealRequested = true; + _state = LifecycleState.Completing; + completeNow = HasNoAdmissionsUnderLock(); + } + } + + if (completed is { } completedValue) + { + InvokeSealCompleted(onCompleted, completedValue); + return completedValue; + } + + if (completeNow) + { + CompleteSeal(); + } + + lock (_lifecycleGate) + { + return _state == LifecycleState.Sealed && _sealedComplete; } } - public unsafe GlobalCoverageInfo GetCodeCoveragePercentage() + protected override object? OnSessionFinished(CoverageContextContainer context, IReadOnlyList modules) { + var merged = false; try { - lock (_coverages) + var testCoverage = ProcessSessionFinished(modules, out var moduleCoverage); + merged = _accumulator.TryMerge(moduleCoverage) != GlobalCoverageMergeResult.BecameSuppressedIncomplete; + return testCoverage; + } + catch + { + if (!merged) { - var sw = RefStopwatch.Create(); - var globalCoverage = new GlobalCoverageInfo(); + _accumulator.Suppress(GlobalCoverageFailureReason.PerTestProcessingFailed); + } - IEnumerable GetModuleValues() - { - var globalContainer = GlobalContainer.CloseContext(); - foreach (var moduleValue in globalContainer) - { - yield return moduleValue; - } - - foreach (var coverageContextContainer in _coverages) - { - var container = coverageContextContainer.CloseContext(); - foreach (var moduleValue in container) - { - yield return moduleValue; - } - } - } + throw; + } + } - var componentCoverageInfos = new Dictionary(); - var fileCoverageInfos = new Dictionary(); + protected override bool TryBeginSessionStartAdmission(out CoverageContextAdmission admission) + { + var rejected = false; + lock (_lifecycleGate) + { + if (_state == LifecycleState.Sealed) + { + ThrowHelper.ThrowInvalidOperationException("A coverage session cannot start after the test session has ended."); + } - var fileBitmapBuffer = stackalloc byte[512]; - foreach (var moduleValue in GetModuleValues()) - { - var module = moduleValue.Module; - if (!componentCoverageInfos.TryGetValue(module, out var componentCoverageInfo)) - { - componentCoverageInfo = new ComponentCoverageInfo(module.Name); - globalCoverage.Components.Add(componentCoverageInfo); - componentCoverageInfos[module] = componentCoverageInfo; - } - - foreach (var moduleFile in moduleValue.Metadata.Files) - { - if (!fileCoverageInfos.TryGetValue(moduleFile, out var fileCoverageInfo)) - { - fileCoverageInfo = new FileCoverageInfo(moduleFile.Path) - { - ExecutableBitmap = moduleFile.Bitmap - }; - - componentCoverageInfo.Files.Add(fileCoverageInfo); - fileCoverageInfos[moduleFile] = fileCoverageInfo; - } - - var fileBitmapLastExecutableLine = moduleFile.LastExecutableLine; - var fileBitmapSize = FileBitmap.GetSize(fileBitmapLastExecutableLine); - using var fileBitmap = fileBitmapSize <= 512 ? new FileBitmap(fileBitmapBuffer, fileBitmapSize) : new FileBitmap(new byte[fileBitmapSize]); - if (moduleValue.Metadata.CoverageMode == 0) - { - var filesLines = (byte*)moduleValue.FilesLines + moduleFile.Offset; - for (var i = 0; i < fileBitmapLastExecutableLine; i++) - { - if (filesLines[i] > 0) - { - fileBitmap.Set(i + 1); - } - } - } - else if (moduleValue.Metadata.CoverageMode == 1) - { - var filesLines = (int*)moduleValue.FilesLines + moduleFile.Offset; - for (var i = 0; i < fileBitmapLastExecutableLine; i++) - { - if (filesLines[i] > 0) - { - fileBitmap.Set(i + 1); - } - } - } - - if (fileBitmap.HasActiveBits()) - { - if (fileCoverageInfo.ExecutedBitmap is null) - { - fileCoverageInfo.ExecutedBitmap = fileBitmap.ToArray(); - } - else - { - using var currentExecutedBitmap = new FileBitmap(fileCoverageInfo.ExecutedBitmap); - var mergedBitmap = fileBitmap | currentExecutedBitmap; - fileCoverageInfo.ExecutedBitmap = mergedBitmap.GetInternalArrayOrToArrayAndDispose(); - } - } - } - } + if (_state == LifecycleState.Completing) + { + rejected = true; + } + else + { + _inFlightStarts++; + } + } - if (Log.IsEnabled(LogEventLevel.Debug)) + if (rejected) + { + _accumulator.Suppress(GlobalCoverageFailureReason.StartFailed); + admission = CoverageContextAdmission.Noop; + return false; + } + + if (!_outputManager.EnsureConfiguredAndFreeze()) + { + _accumulator.Suppress(GlobalCoverageFailureReason.OutputCommitFailed); + } + + admission = new GlobalCoverageAdmission(this); + return true; + } + + protected override void MarkGlobalCoverageIncomplete(GlobalCoverageFailureReason reason) + => _accumulator.Suppress(reason); + + private void CommitAdmission(GlobalCoverageAdmission admission) + { + lock (_lifecycleGate) + { + if (admission.TryTransition(AdmissionState.Starting, AdmissionState.Active)) + { + _inFlightStarts--; + _activeContexts++; + } + } + } + + private void FailAdmission(GlobalCoverageAdmission admission, GlobalCoverageFailureReason reason) + { + var completeNow = false; + lock (_lifecycleGate) + { + var previous = admission.ReleaseState(); + if (previous == AdmissionState.Starting) + { + _inFlightStarts--; + } + else if (previous == AdmissionState.Active) + { + _activeContexts--; + } + else + { + return; + } + + completeNow = _sealRequested && HasNoAdmissionsUnderLock(); + } + + _accumulator.Suppress(reason); + if (completeNow) + { + CompleteSeal(); + } + } + + private void ReleaseAdmission(GlobalCoverageAdmission admission) + { + var completeNow = false; + lock (_lifecycleGate) + { + if (admission.ReleaseState() == AdmissionState.Active) + { + _activeContexts--; + completeNow = _sealRequested && HasNoAdmissionsUnderLock(); + } + } + + if (completeNow) + { + CompleteSeal(); + } + } + + private void ReleaseSnapshotAdmission() + { + var completeNow = false; + lock (_lifecycleGate) + { + if (_inFlightSnapshots > 0) + { + _inFlightSnapshots--; + completeNow = _sealRequested && HasNoAdmissionsUnderLock(); + } + } + + if (completeNow) + { + CompleteSeal(); + } + } + + private bool HasNoAdmissionsUnderLock() + => _inFlightStarts == 0 && _activeContexts == 0 && _inFlightSnapshots == 0; + + private void CompleteSeal() + { + lock (_lifecycleGate) + { + if (_state != LifecycleState.Completing || _sealStarted || !HasNoAdmissionsUnderLock()) + { + return; + } + + _sealStarted = true; + } + + LogContextDiagnostics(_accumulator.AcceptedContextCount); + var complete = TryPublishFinalSnapshot(out var failureException); + if (!complete) + { + // Sealing is single-shot, so report the terminal failure here instead of logging each + // lower-level attempt and producing duplicate diagnostics for the same coverage run. + TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageErrors(); + failureException ??= _outputManager.FailureException; + var failureReason = _accumulator.FailureReason; + if (failureException is not null) + { + Log.Error(failureException, "Global code coverage could not be finalized. Reason: {FailureReason}.", failureReason); + } + else + { + Log.Error("Global code coverage could not be finalized. Reason: {FailureReason}.", failureReason); + } + } + + Action? callback; + lock (_lifecycleGate) + { + _sealedComplete = complete; + _state = LifecycleState.Sealed; + callback = _sealCompleted; + _sealCompleted = null; + } + + InvokeSealCompleted(callback, complete); + + ModuleValue.LogNativeMemoryDiagnostics(DomainMetadata.Instance.ProcessId); + } + + private bool TryPublishFinalSnapshot(out Exception? failureException) + { + failureException = null; + try + { + var result = _accumulator.AcquireSnapshot(GlobalContainer); + if (result.Status != GlobalCoverageSnapshotStatus.Success || result.Snapshot is not { } snapshot) + { + return false; + } + + using (snapshot) + { + if (!_accumulator.TryFinalizeSnapshot(snapshot, () => _outputManager.TryPublish(snapshot.Model))) { - Log.Debug("Global Coverage payload: {Payload}", JsonHelper.SerializeObject(globalCoverage)); + _accumulator.Suppress(GlobalCoverageFailureReason.OutputCommitFailed); + return false; } - // Clean coverages - Clear(); - - Log.Information("Total time to calculate global coverage: {TotalMilliseconds}ms", sw.ElapsedMilliseconds); - return globalCoverage; + return true; } } catch (Exception ex) { - TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageErrors(); - Log.Error(ex, "Error processing the global coverage data."); - throw; + failureException = ex; + _accumulator.Suppress(GlobalCoverageFailureReason.SnapshotFailed); + return false; } } - protected override void OnClearContext(CoverageContextContainer context) + private void InvokeSealCompleted(Action? callback, bool complete) { - // None we need to keep all context to calculate the global coverage later + try + { + callback?.Invoke(complete); + } + catch + { + // Publication callbacks must not replace failures from the test lifecycle. + } + } + + private sealed class GlobalCoverageAdmission : CoverageContextAdmission + { + private readonly DefaultWithGlobalCoverageEventHandler _owner; + private int _state; + + public GlobalCoverageAdmission(DefaultWithGlobalCoverageEventHandler owner) => _owner = owner; + + public override void CommitInstalled() => _owner.CommitAdmission(this); + + public override void FailStart(GlobalCoverageFailureReason reason) => _owner.FailAdmission(this, reason); + + public override void Release() => _owner.ReleaseAdmission(this); + + public bool TryTransition(AdmissionState expected, AdmissionState next) + => Interlocked.CompareExchange(ref _state, (int)next, (int)expected) == (int)expected; + + public AdmissionState ReleaseState() + => (AdmissionState)Interlocked.Exchange(ref _state, (int)AdmissionState.Released); + } + + private sealed class SnapshotAdmission + { + private readonly DefaultWithGlobalCoverageEventHandler _owner; + private int _released; + + public SnapshotAdmission(DefaultWithGlobalCoverageEventHandler owner) => _owner = owner; + + public void Release() + { + if (Interlocked.CompareExchange(ref _released, 1, 0) == 0) + { + _owner.ReleaseSnapshotAdmission(); + } + } } } diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageAccumulator.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageAccumulator.cs new file mode 100644 index 000000000000..b6202a83e6e2 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageAccumulator.cs @@ -0,0 +1,386 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.ExceptionServices; +using System.Threading; +using Datadog.Trace.Ci.Coverage.Metadata; +using Datadog.Trace.Ci.Coverage.Models.Global; +using Datadog.Trace.Ci.Coverage.Util; + +namespace Datadog.Trace.Ci.Coverage; + +internal enum GlobalCoverageMergeResult +{ + Merged, + AlreadySuppressed, + BecameSuppressedIncomplete, +} + +internal enum GlobalCoverageSnapshotStatus +{ + Success, + SuppressedIncomplete, +} + +internal sealed class GlobalCoverageAccumulator +{ + private readonly object _mergeGate = new(); + private readonly object _completenessGate = new(); + private readonly SemaphoreSlim _snapshotGate = new(1, 1); + private readonly GlobalCoverageAccumulatorLimits _limits; + private CoverageState? _coverage; + private int _suppressed; + private int _failureReason; + private long _completenessEpoch; + private long _acceptedContextCount; + private bool _completenessFinalized; + + public GlobalCoverageAccumulator(GlobalCoverageAccumulatorLimits? limits = null) + { + _limits = limits ?? GlobalCoverageAccumulatorLimits.Default; + _coverage = new CoverageState(); + } + + public bool IsSuppressed => Volatile.Read(ref _suppressed) != 0; + + public GlobalCoverageFailureReason FailureReason => (GlobalCoverageFailureReason)Volatile.Read(ref _failureReason); + + public long AcceptedContextCount => Volatile.Read(ref _acceptedContextCount); + + private static bool IsRecoverable(Exception exception) + => exception is OutOfMemoryException or OverflowException or GlobalCoverageLimitException or GlobalCoverageMetadataException; + + private static GlobalCoverageInfo Materialize(CoverageState coverage) + { + var globalCoverage = new GlobalCoverageInfo(); + foreach (var pair in coverage.Modules) + { + var component = new ComponentCoverageInfo(pair.Key.Name); + var metadata = pair.Value.Metadata; + for (var i = 0; i < metadata.Files.Length; i++) + { + var fileMetadata = metadata.Files[i]; + component.Files.Add( + new FileCoverageInfo(fileMetadata.Path) + { + ExecutableBitmap = fileMetadata.Bitmap, + // A snapshot outlives the merge lock, so it must not expose mutable accumulator storage. + ExecutedBitmap = pair.Value.ExecutedBitmaps[i] is { } bitmap ? (byte[])bitmap.Clone() : null + }); + } + + globalCoverage.Components.Add(component); + } + + return globalCoverage; + } + + public GlobalCoverageMergeResult TryMerge(IReadOnlyList modules) + { + if (IsSuppressed) + { + return GlobalCoverageMergeResult.AlreadySuppressed; + } + + try + { + lock (_mergeGate) + { + if (IsSuppressed || _coverage is null) + { + return GlobalCoverageMergeResult.AlreadySuppressed; + } + + MergeIntoCoverage(_coverage, modules); + _coverage.AcceptedContextCount++; + _acceptedContextCount++; + + return GlobalCoverageMergeResult.Merged; + } + } + catch (Exception ex) when (IsRecoverable(ex)) + { + Suppress(GlobalCoverageFailureReason.MergeFailed); + return GlobalCoverageMergeResult.BecameSuppressedIncomplete; + } + } + + public void Suppress(GlobalCoverageFailureReason reason) + { + lock (_completenessGate) + { + SuppressUnderCompletenessGate(reason); + } + + ClearCoverage(); + } + + public bool TryCommit(GlobalCoverageSnapshot snapshot, Action action) + { + ExceptionDispatchInfo? exception = null; + var committed = false; + lock (_completenessGate) + { + if (!IsSuppressed && !snapshot.IsDisposed && snapshot.CompletenessEpoch == _completenessEpoch) + { + try + { + action(); + committed = true; + } + catch (Exception ex) + { + SuppressUnderCompletenessGate(GlobalCoverageFailureReason.OutputCommitFailed); + exception = ExceptionDispatchInfo.Capture(ex); + } + } + } + + if (exception is not null) + { + ClearCoverage(); + exception.Throw(); + } + + return committed; + } + + // Keep artifact publication and the final completeness transition under one lock. Otherwise a + // concurrent failure could invalidate coverage after its pending marker has already been removed. + public bool TryFinalizeSnapshot(GlobalCoverageSnapshot snapshot, Func commit) + { + var failed = false; + ExceptionDispatchInfo? exception = null; + lock (_completenessGate) + { + if (IsSuppressed || snapshot.IsDisposed || snapshot.CompletenessEpoch != _completenessEpoch) + { + return false; + } + + try + { + if (!commit()) + { + SuppressUnderCompletenessGate(GlobalCoverageFailureReason.OutputCommitFailed); + failed = true; + } + else + { + _completenessFinalized = true; + return true; + } + } + catch (Exception ex) + { + SuppressUnderCompletenessGate(GlobalCoverageFailureReason.OutputCommitFailed); + failed = true; + exception = ExceptionDispatchInfo.Capture(ex); + } + } + + if (failed) + { + ClearCoverage(); + } + + exception?.Throw(); + + return false; + } + + private void ClearCoverage() + { + lock (_mergeGate) + { + _coverage = null; + } + } + + private void SuppressUnderCompletenessGate(GlobalCoverageFailureReason reason) + { + if (_completenessFinalized) + { + return; + } + + if (Interlocked.CompareExchange(ref _suppressed, 1, 0) == 0) + { + Volatile.Write(ref _failureReason, (int)reason); + _completenessEpoch = checked(_completenessEpoch + 1); + } + } + + public GlobalCoverageSnapshotResult AcquireSnapshot(CoverageContextContainer? globalContainer, Action? releaseAdmission = null) + { + _snapshotGate.Wait(); + var releaseSnapshotGate = true; + try + { + if (IsSuppressed) + { + return GlobalCoverageSnapshotResult.Suppressed(FailureReason); + } + + GlobalCoverageInfo model; + long acceptedContextCount; + lock (_mergeGate) + { + if (IsSuppressed || _coverage is null) + { + return GlobalCoverageSnapshotResult.Suppressed(FailureReason); + } + + if (globalContainer is not null) + { + var globalModules = globalContainer.SnapshotModules(_limits.MaximumModules); + var globalCoverage = new ModuleCoverageData[globalModules.Length]; + for (var i = 0; i < globalModules.Length; i++) + { + globalCoverage[i] = ModuleCoverageData.Capture(globalModules[i]); + } + + MergeIntoCoverage(_coverage, globalCoverage); + } + + model = Materialize(_coverage); + _ = model.GetTotalPercentage(); + acceptedContextCount = _coverage.AcceptedContextCount; + } + + lock (_completenessGate) + { + if (IsSuppressed) + { + return GlobalCoverageSnapshotResult.Suppressed(FailureReason); + } + + var snapshot = new GlobalCoverageSnapshot(model, acceptedContextCount, _completenessEpoch, _snapshotGate, releaseAdmission); + releaseSnapshotGate = false; + return GlobalCoverageSnapshotResult.Success(snapshot); + } + } + catch (Exception ex) when (IsRecoverable(ex)) + { + Suppress(GlobalCoverageFailureReason.SnapshotFailed); + return GlobalCoverageSnapshotResult.Suppressed(FailureReason); + } + catch + { + Suppress(GlobalCoverageFailureReason.SnapshotFailed); + throw; + } + finally + { + if (releaseSnapshotGate) + { + _snapshotGate.Release(); + } + } + } + + private void MergeIntoCoverage(CoverageState coverage, IReadOnlyList modules) + { + foreach (var moduleCoverage in modules) + { + var metadata = moduleCoverage.Metadata; + + if (!coverage.Modules.TryGetValue(moduleCoverage.Module, out var moduleEntry)) + { + if (coverage.Modules.Count >= _limits.MaximumModules) + { + throw new GlobalCoverageLimitException("The global coverage module limit was exceeded."); + } + + var newFileSlotCount = checked(coverage.FileSlotCount + metadata.Files.Length); + if (newFileSlotCount > _limits.MaximumFileSlots) + { + throw new GlobalCoverageLimitException("The global coverage file-slot limit was exceeded."); + } + + moduleEntry = new ModuleEntry(metadata); + coverage.Modules.Add(moduleCoverage.Module, moduleEntry); + coverage.FileSlotCount = newFileSlotCount; + } + else if (!ReferenceEquals(moduleEntry.Metadata, metadata)) + { + throw new GlobalCoverageMetadataException("The same module was observed with different coverage metadata."); + } + + for (var fileIndex = 0; fileIndex < metadata.Files.Length; fileIndex++) + { + var file = metadata.Files[fileIndex]; + var sourceBitmap = moduleCoverage.ExecutedBitmaps[fileIndex]; + if (sourceBitmap is null) + { + continue; + } + + var expectedBitmapLength = FileBitmap.GetSize(file.LastExecutableLine); + if (sourceBitmap.Length != expectedBitmapLength) + { + throw new GlobalCoverageMetadataException("A captured coverage bitmap length does not match its metadata."); + } + + var executedBitmap = moduleEntry.ExecutedBitmaps[fileIndex]; + executedBitmap ??= AllocateBitmap(coverage, file.LastExecutableLine); + for (var byteIndex = 0; byteIndex < sourceBitmap.Length; byteIndex++) + { + executedBitmap[byteIndex] |= sourceBitmap[byteIndex]; + } + + moduleEntry.ExecutedBitmaps[fileIndex] = executedBitmap; + } + } + } + + private byte[] AllocateBitmap(CoverageState coverage, int lineCount) + { + var byteLength = FileBitmap.GetSize(lineCount); + if (byteLength > _limits.MaximumSingleBitmapBytes) + { + throw new GlobalCoverageLimitException("A global coverage bitmap exceeds the per-file limit."); + } + + var retainedBytes = checked(coverage.RetainedBitmapBytes + byteLength); + if (retainedBytes > _limits.MaximumRetainedBitmapBytes) + { + throw new GlobalCoverageLimitException("The global coverage bitmap budget was exceeded."); + } + + var bitmap = new byte[byteLength]; + coverage.RetainedBitmapBytes = retainedBytes; + return bitmap; + } + + private sealed class CoverageState + { + public Dictionary Modules { get; } = new(); + + public int RetainedBitmapBytes { get; set; } + + public int FileSlotCount { get; set; } + + public long AcceptedContextCount { get; set; } + } + + private sealed class ModuleEntry + { + public ModuleEntry(ModuleCoverageMetadata metadata) + { + Metadata = metadata; + ExecutedBitmaps = new byte[metadata.Files.Length][]; + } + + public ModuleCoverageMetadata Metadata { get; } + + public byte[]?[] ExecutedBitmaps { get; } + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageAccumulatorLimits.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageAccumulatorLimits.cs new file mode 100644 index 000000000000..395b77735411 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageAccumulatorLimits.cs @@ -0,0 +1,29 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageAccumulatorLimits +{ + public static readonly GlobalCoverageAccumulatorLimits Default = new(8 * 1024 * 1024, 64 * 1024 * 1024, 10_000, 100_000); + + public GlobalCoverageAccumulatorLimits(int maximumSingleBitmapBytes, int maximumRetainedBitmapBytes, int maximumModules, int maximumFileSlots) + { + MaximumSingleBitmapBytes = maximumSingleBitmapBytes; + MaximumRetainedBitmapBytes = maximumRetainedBitmapBytes; + MaximumModules = maximumModules; + MaximumFileSlots = maximumFileSlots; + } + + public int MaximumSingleBitmapBytes { get; } + + public int MaximumRetainedBitmapBytes { get; } + + public int MaximumModules { get; } + + public int MaximumFileSlots { get; } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageArtifactLimits.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageArtifactLimits.cs new file mode 100644 index 000000000000..cce5f849094e --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageArtifactLimits.cs @@ -0,0 +1,67 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageArtifactLimits +{ + public static readonly GlobalCoverageArtifactLimits Default = new( + maximumSerializedBytes: 256L * 1024 * 1024, + maximumBitmapBytes: 8 * 1024 * 1024, + maximumModelBitmapBytes: 128L * 1024 * 1024, + maximumComponents: 10_000, + maximumEntries: 100_000, + maximumIdentityCharacters: 32_000_000, + maximumPropertyCharacters: 1_024, + maximumScalarCharacters: 16_777_216, + maximumDepth: 64, + scannerBufferCharacters: 16 * 1024); + + public GlobalCoverageArtifactLimits( + long maximumSerializedBytes, + int maximumBitmapBytes, + long maximumModelBitmapBytes, + int maximumComponents, + int maximumEntries, + int maximumIdentityCharacters, + int maximumPropertyCharacters, + int maximumScalarCharacters, + int maximumDepth, + int scannerBufferCharacters) + { + MaximumSerializedBytes = maximumSerializedBytes; + MaximumBitmapBytes = maximumBitmapBytes; + MaximumModelBitmapBytes = maximumModelBitmapBytes; + MaximumComponents = maximumComponents; + MaximumEntries = maximumEntries; + MaximumIdentityCharacters = maximumIdentityCharacters; + MaximumPropertyCharacters = maximumPropertyCharacters; + MaximumScalarCharacters = maximumScalarCharacters; + MaximumDepth = maximumDepth; + ScannerBufferCharacters = scannerBufferCharacters; + } + + public long MaximumSerializedBytes { get; } + + public int MaximumBitmapBytes { get; } + + public long MaximumModelBitmapBytes { get; } + + public int MaximumComponents { get; } + + public int MaximumEntries { get; } + + public int MaximumIdentityCharacters { get; } + + public int MaximumPropertyCharacters { get; } + + public int MaximumScalarCharacters { get; } + + public int MaximumDepth { get; } + + public int ScannerBufferCharacters { get; } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageArtifactWriter.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageArtifactWriter.cs new file mode 100644 index 000000000000..4410a10f905b --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageArtifactWriter.cs @@ -0,0 +1,104 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.IO; +using System.Text; +using Datadog.Trace.Ci.Coverage.Models.Global; +using Datadog.Trace.Util; +using Datadog.Trace.Util.Json; +using Datadog.Trace.Vendors.Newtonsoft.Json; + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageArtifactWriter +{ + private static readonly Encoding Utf8WithoutBom = new UTF8Encoding(false, true); + + private readonly GlobalCoverageArtifactLimits _limits; + private readonly GlobalCoverageInputReader _validator; + + public GlobalCoverageArtifactWriter(GlobalCoverageArtifactLimits? limits = null) + { + _limits = limits ?? GlobalCoverageArtifactLimits.Default; + _validator = new GlobalCoverageInputReader(_limits); + } + + public void WriteAtomicNoReplace(string destinationPath, GlobalCoverageInfo model) + { + using var staged = Stage(destinationPath, model, replaceExisting: false); + staged.Commit(); + } + + public void WriteAtomicReplace(string destinationPath, GlobalCoverageInfo model) + { + using var staged = Stage(destinationPath, model, replaceExisting: true); + staged.Commit(); + } + + public GlobalCoverageStagedArtifact StageNoReplace(string destinationPath, GlobalCoverageInfo model) + => Stage(destinationPath, model, replaceExisting: false); + + public GlobalCoverageStagedArtifact StageReplace(string destinationPath, GlobalCoverageInfo model) + => Stage(destinationPath, model, replaceExisting: true); + + private void TryDelete(string path) + { + try + { + File.Delete(path); + } + catch + { + // A failed write must preserve the primary exception. The non-json temporary is never a valid input. + } + } + + private GlobalCoverageStagedArtifact Stage(string destinationPath, GlobalCoverageInfo model, bool replaceExisting) + { + if (StringUtil.IsNullOrWhiteSpace(destinationPath)) + { + ThrowHelper.ThrowArgumentException("A global coverage destination path is required.", nameof(destinationPath)); + } + + _validator.ValidateModel(model); + + var fullDestinationPath = Path.GetFullPath(destinationPath); + var directory = Path.GetDirectoryName(fullDestinationPath); + if (StringUtil.IsNullOrEmpty(directory)) + { + ThrowHelper.ThrowInvalidOperationException("The global coverage destination has no parent directory."); + } + + Directory.CreateDirectory(directory); + var temporaryPath = Path.Combine(directory, $".{Path.GetFileName(fullDestinationPath)}.{Guid.NewGuid():N}.tmp"); + try + { + using (var fileStream = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 16 * 1024, FileOptions.SequentialScan)) + using (var boundedStream = new GlobalCoverageBoundedWriteStream( + fileStream, + _limits.MaximumSerializedBytes, + "The global coverage serialized-byte limit was exceeded.")) + using (var streamWriter = new StreamWriter(boundedStream, Utf8WithoutBom, 16 * 1024, true)) + using (var jsonWriter = new JsonTextWriter(streamWriter) { ArrayPool = JsonArrayPool.Shared }) + { + JsonSerializer.Create().Serialize(jsonWriter, model); + jsonWriter.Flush(); + streamWriter.Flush(); + boundedStream.Flush(); + fileStream.Flush(true); + } + + return new GlobalCoverageStagedArtifact(temporaryPath, fullDestinationPath, replaceExisting); + } + catch + { + TryDelete(temporaryPath); + throw; + } + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageBoundedWriteStream.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageBoundedWriteStream.cs new file mode 100644 index 000000000000..67fecc1ab014 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageBoundedWriteStream.cs @@ -0,0 +1,62 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.IO; + +namespace Datadog.Trace.Ci.Coverage; + +// This wrapper intentionally does not own the underlying stream. Callers perform the durable file +// flush after the JSON writers have flushed through this byte-counting layer. +internal sealed class GlobalCoverageBoundedWriteStream : Stream +{ + private readonly Stream _inner; + private readonly long _maximumBytes; + private readonly string _limitExceededMessage; + private long _writtenBytes; + + public GlobalCoverageBoundedWriteStream(Stream inner, long maximumBytes, string limitExceededMessage) + { + _inner = inner; + _maximumBytes = maximumBytes; + _limitExceededMessage = limitExceededMessage; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _writtenBytes; + + public override long Position + { + get => _writtenBytes; + set => throw new NotSupportedException(); + } + + public override void Flush() => _inner.Flush(); + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) + { + var nextLength = checked(_writtenBytes + count); + if (nextLength > _maximumBytes) + { + throw new InvalidDataException(_limitExceededMessage); + } + + _inner.Write(buffer, offset, count); + _writtenBytes = nextLength; + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageCombinerAccumulator.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageCombinerAccumulator.cs new file mode 100644 index 000000000000..483c3fff931c --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageCombinerAccumulator.cs @@ -0,0 +1,216 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using Datadog.Trace.Ci.Coverage.Models.Global; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageCombinerAccumulator +{ + private readonly GlobalCoverageArtifactLimits _limits; + private readonly Dictionary _components = new(); + private readonly List _componentOrder = new(); + private int _entryCount; + private long _identityCharacters; + private long _bitmapBytes; + private bool _materialized; + + public GlobalCoverageCombinerAccumulator(GlobalCoverageArtifactLimits? limits = null) + { + _limits = limits ?? GlobalCoverageArtifactLimits.Default; + } + + public void Add(GlobalCoverageInfo model) + { + if (_materialized) + { + ThrowHelper.ThrowInvalidOperationException("The global coverage accumulator has already been materialized."); + } + + foreach (var component in model.Components) + { + var componentKey = new NullableStringKey(component.Name); + if (!_components.TryGetValue(componentKey, out var destinationComponent)) + { + if (_components.Count >= _limits.MaximumComponents) + { + throw new InvalidDataException("The combined global coverage component limit was exceeded."); + } + + AddIdentity(component.Name); + destinationComponent = new ComponentAccumulator(component.Name); + _components.Add(componentKey, destinationComponent); + _componentOrder.Add(destinationComponent); + } + + foreach (var file in component.Files) + { + var fileKey = new NullableStringKey(file.Path); + if (!destinationComponent.Files.TryGetValue(fileKey, out var destinationFile)) + { + _entryCount = checked(_entryCount + 1); + if (_entryCount > _limits.MaximumEntries) + { + throw new InvalidDataException("The combined global coverage entry limit was exceeded."); + } + + AddIdentity(file.Path); + ValidateBitmap(file.ExecutableBitmap); + ValidateBitmap(file.ExecutedBitmap); + AddBitmapBytes(file.ExecutableBitmap?.Length ?? 0); + AddBitmapBytes(file.ExecutedBitmap?.Length ?? 0); + destinationFile = new FileAccumulator(file.Path, file.ExecutableBitmap, file.ExecutedBitmap); + destinationComponent.Files.Add(fileKey, destinationFile); + destinationComponent.FileOrder.Add(destinationFile); + } + else + { + destinationFile.ExecutableBitmap = MergeBitmap(destinationFile.ExecutableBitmap, file.ExecutableBitmap); + destinationFile.ExecutedBitmap = MergeBitmap(destinationFile.ExecutedBitmap, file.ExecutedBitmap); + } + } + } + } + + public GlobalCoverageInfo Materialize() + { + if (_materialized) + { + ThrowHelper.ThrowInvalidOperationException("The global coverage accumulator has already been materialized."); + } + + _materialized = true; + var model = new GlobalCoverageInfo(); + foreach (var componentAccumulator in _componentOrder) + { + var component = new ComponentCoverageInfo(componentAccumulator.Name); + foreach (var fileAccumulator in componentAccumulator.FileOrder) + { + component.Files.Add( + new FileCoverageInfo(fileAccumulator.Path) + { + ExecutableBitmap = fileAccumulator.ExecutableBitmap, + ExecutedBitmap = fileAccumulator.ExecutedBitmap, + }); + } + + model.Components.Add(component); + } + + _ = model.GetTotalPercentage(); + return model; + } + + private byte[]? MergeBitmap(byte[]? current, byte[]? incoming) + { + if (incoming is null) + { + return current; + } + + ValidateBitmap(incoming); + if (current is null) + { + AddBitmapBytes(incoming.Length); + return incoming; + } + + if (incoming.Length > current.Length) + { + AddBitmapBytes(incoming.Length - current.Length); + for (var i = 0; i < current.Length; i++) + { + incoming[i] |= current[i]; + } + + return incoming; + } + + for (var i = 0; i < incoming.Length; i++) + { + current[i] |= incoming[i]; + } + + return current; + } + + private void AddIdentity(string? identity) + { + _identityCharacters = checked(_identityCharacters + (identity?.Length ?? 0)); + if (_identityCharacters > _limits.MaximumIdentityCharacters) + { + throw new InvalidDataException("The combined global coverage path/name character limit was exceeded."); + } + } + + private void ValidateBitmap(byte[]? bitmap) + { + if (bitmap is not null && bitmap.Length > _limits.MaximumBitmapBytes) + { + throw new InvalidDataException("A combined global coverage bitmap exceeds the per-bitmap limit."); + } + } + + private void AddBitmapBytes(int count) + { + _bitmapBytes = checked(_bitmapBytes + count); + if (_bitmapBytes > _limits.MaximumModelBitmapBytes) + { + throw new InvalidDataException("The combined global coverage bitmap limit was exceeded."); + } + } + + private readonly struct NullableStringKey : IEquatable + { + public NullableStringKey(string? value) + { + Value = value; + } + + private string? Value { get; } + + public bool Equals(NullableStringKey other) => string.Equals(Value, other.Value, StringComparison.Ordinal); + + public override bool Equals(object? obj) => obj is NullableStringKey other && Equals(other); + + public override int GetHashCode() => Value is null ? 0 : StringComparer.Ordinal.GetHashCode(Value); + } + + private sealed class ComponentAccumulator + { + public ComponentAccumulator(string? name) + { + Name = name; + } + + public string? Name { get; } + + public Dictionary Files { get; } = new(); + + public List FileOrder { get; } = new(); + } + + private sealed class FileAccumulator + { + public FileAccumulator(string? path, byte[]? executableBitmap, byte[]? executedBitmap) + { + Path = path; + ExecutableBitmap = executableBitmap; + ExecutedBitmap = executedBitmap; + } + + public string? Path { get; } + + public byte[]? ExecutableBitmap { get; set; } + + public byte[]? ExecutedBitmap { get; set; } + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageFailureReason.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageFailureReason.cs new file mode 100644 index 000000000000..b6059cf16c9b --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageFailureReason.cs @@ -0,0 +1,21 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.Ci.Coverage; + +internal enum GlobalCoverageFailureReason +{ + None, + StartFailed, + TestConstructionFailed, + TestCloseBeforeCoverage, + ProbeDataIncomplete, + PerTestProcessingFailed, + MergeFailed, + SnapshotFailed, + OutputCommitFailed, +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageFileCombiner.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageFileCombiner.cs new file mode 100644 index 000000000000..c8b95a298e62 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageFileCombiner.cs @@ -0,0 +1,109 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Datadog.Trace.Ci.Coverage.Models.Global; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Ci.Coverage; + +internal static class GlobalCoverageFileCombiner +{ + private const int MaximumInputFiles = 65_536; + + public static bool TryAcquireInputFiles(string inputFolder, string? expectedRunToken, out string[] inputFiles) + { + inputFiles = []; + if (expectedRunToken is not null && + Directory.EnumerateFiles( + inputFolder, + GlobalCoverageProtocol.PendingMarkerPrefix + expectedRunToken + "-*", + SearchOption.TopDirectoryOnly) + .Any()) + { + return false; + } + + var inputPattern = expectedRunToken is null + ? "*.json" + : GlobalCoverageProtocol.CoverageFilePrefix + expectedRunToken + "-*" + GlobalCoverageProtocol.JsonExtension; + inputFiles = GetInputFilesBounded(inputFolder, inputPattern); + return true; + } + + public static bool TryCombine( + IReadOnlyList inputFiles, + string? outputFile, + bool requireAllInputs, + Action? onFileProcessed, + out GlobalCoverageInfo? globalCoverageInfo, + out string? rejectedInput) + { + globalCoverageInfo = null; + rejectedInput = null; + + var inputReader = new GlobalCoverageInputReader(); + var accumulator = new GlobalCoverageCombinerAccumulator(); + var processedFiles = 0; + var outputFullPath = StringUtil.IsNullOrWhiteSpace(outputFile) ? null : Path.GetFullPath(outputFile); + foreach (var file in inputFiles) + { + if (Path.GetFileName(file).StartsWith("session-coverage-", StringComparison.OrdinalIgnoreCase) || + (outputFullPath is not null && PathsEqual(Path.GetFullPath(file), outputFullPath))) + { + continue; + } + + if (!inputReader.TryRead(file, out var globalCoverage) || globalCoverage is null) + { + // Legacy directories may contain unrelated JSON. Run-scoped callers require every + // selected process artifact to be valid so incomplete coverage cannot be published. + if (requireAllInputs) + { + rejectedInput = file; + return false; + } + + continue; + } + + onFileProcessed?.Invoke(file); + accumulator.Add(globalCoverage); + processedFiles++; + } + + if (processedFiles == 0) + { + return false; + } + + globalCoverageInfo = accumulator.Materialize(); + return true; + } + + private static string[] GetInputFilesBounded(string inputFolder, string pattern) + { + var files = Directory.EnumerateFiles(inputFolder, pattern, SearchOption.TopDirectoryOnly) + .Take(MaximumInputFiles + 1) + .ToArray(); + if (files.Length > MaximumInputFiles) + { + throw new InvalidDataException("The global coverage input-file limit was exceeded."); + } + + return files; + } + + private static bool PathsEqual(string first, string second) + => string.Equals( + first, + second, + FrameworkDescription.Instance.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal); +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageInputReader.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageInputReader.cs new file mode 100644 index 000000000000..f71954a14dbc --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageInputReader.cs @@ -0,0 +1,247 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using Datadog.Trace.Ci.Coverage.Models.Global; +using Datadog.Trace.Vendors.Newtonsoft.Json; + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageInputReader +{ + private readonly GlobalCoverageArtifactLimits _limits; + + public GlobalCoverageInputReader(GlobalCoverageArtifactLimits? limits = null) + { + _limits = limits ?? GlobalCoverageArtifactLimits.Default; + } + + private static bool HashesMatch(byte[] first, byte[] second) + { + if (first.Length != second.Length) + { + return false; + } + + var difference = 0; + for (var i = 0; i < first.Length; i++) + { + difference |= first[i] ^ second[i]; + } + + return difference == 0; + } + + public bool TryRead(string path, out GlobalCoverageInfo? model) + { + model = null; + try + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + var expectedLength = stream.Length; + if (expectedLength < 0 || expectedLength > _limits.MaximumSerializedBytes) + { + return false; + } + + var preflightHash = RunPreflight(stream); + if (stream.Length != expectedLength) + { + return false; + } + + stream.Seek(0, SeekOrigin.Begin); + var deserialized = Deserialize(stream, out var deserializeHash); + if (stream.Length != expectedLength || !HashesMatch(preflightHash, deserializeHash)) + { + return false; + } + + ValidateModel(deserialized); + model = deserialized; + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or JsonException or OverflowException or OutOfMemoryException or CryptographicException) + { + return false; + } + } + + public void ValidateModel(GlobalCoverageInfo model) + { + if (model.Components.Count > _limits.MaximumComponents) + { + throw new InvalidDataException("The global coverage component limit was exceeded."); + } + + var entryCount = 0; + long identityCharacters = 0; + long bitmapBytes = 0; + foreach (var component in model.Components) + { + if (component is null) + { + throw new InvalidDataException("The global coverage input contains a null component."); + } + + identityCharacters = checked(identityCharacters + (component.Name?.Length ?? 0)); + foreach (var file in component.Files) + { + if (file is null) + { + throw new InvalidDataException("The global coverage input contains a null file."); + } + + entryCount = checked(entryCount + 1); + identityCharacters = checked(identityCharacters + (file.Path?.Length ?? 0)); + AddBitmap(file.ExecutableBitmap); + AddBitmap(file.ExecutedBitmap); + ValidateExecutedBitmap(file.ExecutableBitmap, file.ExecutedBitmap); + } + } + + if (entryCount > _limits.MaximumEntries) + { + throw new InvalidDataException("The global coverage entry limit was exceeded."); + } + + if (identityCharacters > _limits.MaximumIdentityCharacters) + { + throw new InvalidDataException("The global coverage path/name character limit was exceeded."); + } + + void AddBitmap(byte[]? bitmap) + { + if (bitmap is null) + { + return; + } + + if (bitmap.Length > _limits.MaximumBitmapBytes) + { + throw new InvalidDataException("A global coverage bitmap exceeds the per-bitmap limit."); + } + + bitmapBytes = checked(bitmapBytes + bitmap.Length); + if (bitmapBytes > _limits.MaximumModelBitmapBytes) + { + throw new InvalidDataException("The global coverage model bitmap limit was exceeded."); + } + } + + static void ValidateExecutedBitmap(byte[]? executable, byte[]? executed) + { + if (executed is null) + { + return; + } + + if (executable is null || executed.Length > executable.Length) + { + throw new InvalidDataException("A global coverage executed bitmap has no compatible executable bitmap."); + } + + for (var i = 0; i < executed.Length; i++) + { + if ((executed[i] & ~executable[i]) != 0) + { + throw new InvalidDataException("A global coverage executed bitmap contains a non-executable line."); + } + } + } + } + + private byte[] RunPreflight(FileStream stream) + { + using var sha256 = SHA256.Create(); + using var hashingStream = new HashingReadStream(stream, sha256); + var scanner = new GlobalCoverageJsonPreflightScanner(_limits); + scanner.Scan(hashingStream); + + return sha256.Hash ?? throw new CryptographicException("Unable to hash the global coverage input."); + } + + private GlobalCoverageInfo Deserialize(FileStream stream, out byte[] hash) + { + using var sha256 = SHA256.Create(); + GlobalCoverageInfo? model; + using (var hashingStream = new HashingReadStream(stream, sha256)) + using (var streamReader = new StreamReader(hashingStream, new UTF8Encoding(false, true), true, _limits.ScannerBufferCharacters, true)) + using (var jsonReader = new JsonTextReader(streamReader) { MaxDepth = _limits.MaximumDepth, DateParseHandling = DateParseHandling.None }) + { + model = JsonSerializer.Create().Deserialize(jsonReader); + if (model is null) + { + throw new InvalidDataException("The global coverage input does not contain a model."); + } + + if (jsonReader.Read()) + { + throw new InvalidDataException("The global coverage input contains trailing JSON tokens."); + } + } + + hash = sha256.Hash ?? throw new CryptographicException("Unable to hash the global coverage input."); + return model; + } + + private sealed class HashingReadStream : Stream + { + private readonly Stream _inner; + private readonly HashAlgorithm _hash; + private bool _completed; + + public HashingReadStream(Stream inner, HashAlgorithm hash) + { + _inner = inner; + _hash = hash; + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) + { + var read = _inner.Read(buffer, offset, count); + if (read > 0) + { + _hash.TransformBlock(buffer, offset, read, buffer, offset); + } + else if (!_completed) + { + _completed = true; + _hash.TransformFinalBlock(Array.Empty(), 0, 0); + } + + return read; + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageJsonPreflightScanner.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageJsonPreflightScanner.cs new file mode 100644 index 000000000000..e5c126e1dfa4 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageJsonPreflightScanner.cs @@ -0,0 +1,477 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.IO; +using System.Text; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageJsonPreflightScanner +{ + private readonly GlobalCoverageArtifactLimits _limits; + private readonly char[] _readBuffer; + private readonly char[] _propertyBuffer; + private readonly Frame[] _frames; + private int _readOffset; + private int _readCount; + private int _depth; + private int _componentCount; + private int _entryCount; + private long _identityCharacters; + private long _bitmapBytes; + private TextReader? _reader; + + public GlobalCoverageJsonPreflightScanner(GlobalCoverageArtifactLimits limits) + { + _limits = limits; + _readBuffer = new char[limits.ScannerBufferCharacters]; + _propertyBuffer = new char[limits.MaximumPropertyCharacters]; + _frames = new Frame[limits.MaximumDepth]; + } + + private enum ContainerKind + { + Object, + Array, + } + + private enum ArrayKind + { + Unknown, + Components, + Files, + } + + private enum PropertyKind + { + Unknown, + Components, + Files, + Name, + Path, + ExecutableBitmap, + ExecutedBitmap, + } + + private static bool IsBase64Character(char character, ref bool paddingStarted, ref int padding) + { + if (character == '=') + { + paddingStarted = true; + padding++; + return true; + } + + if (paddingStarted) + { + return false; + } + + return (character >= 'A' && character <= 'Z') || + (character >= 'a' && character <= 'z') || + (character >= '0' && character <= '9') || + character is '+' or '/'; + } + + public void Scan(Stream stream) + { + _reader = new StreamReader(stream, new UTF8Encoding(false, true), true, _limits.ScannerBufferCharacters, true); + _readOffset = 0; + _readCount = 0; + _depth = 0; + _componentCount = 0; + _entryCount = 0; + _identityCharacters = 0; + _bitmapBytes = 0; + + var sawToken = false; + int value; + while ((value = Read()) >= 0) + { + var character = (char)value; + if (char.IsWhiteSpace(character)) + { + continue; + } + + sawToken = true; + switch (character) + { + case '{': + BeginContainer(ContainerKind.Object); + break; + case '[': + BeginContainer(ContainerKind.Array); + break; + case '}': + EndContainer(ContainerKind.Object); + break; + case ']': + EndContainer(ContainerKind.Array); + break; + case ',': + OnComma(); + break; + case ':': + break; + case '"': + ReadStringToken(); + break; + default: + ReadScalarToken(character); + CompleteScalarValue(); + break; + } + } + + if (!sawToken || _depth != 0) + { + throw new InvalidDataException("The global coverage JSON is incomplete."); + } + } + + private void BeginContainer(ContainerKind kind) + { + var arrayKind = ArrayKind.Unknown; + if (_depth > 0) + { + ref var parent = ref _frames[_depth - 1]; + if (parent.Kind == ContainerKind.Array) + { + if (kind == ContainerKind.Object && parent.ArrayKind == ArrayKind.Components) + { + _componentCount = checked(_componentCount + 1); + if (_componentCount > _limits.MaximumComponents) + { + throw new InvalidDataException("The global coverage component limit was exceeded."); + } + } + else if (kind == ContainerKind.Object && parent.ArrayKind == ArrayKind.Files) + { + _entryCount = checked(_entryCount + 1); + if (_entryCount > _limits.MaximumEntries) + { + throw new InvalidDataException("The global coverage entry limit was exceeded."); + } + } + } + else if (kind == ContainerKind.Array) + { + arrayKind = parent.PendingProperty switch + { + PropertyKind.Components => ArrayKind.Components, + PropertyKind.Files => ArrayKind.Files, + _ => ArrayKind.Unknown, + }; + } + + parent.PendingProperty = PropertyKind.Unknown; + } + + if (_depth >= _limits.MaximumDepth) + { + throw new InvalidDataException("The global coverage JSON nesting limit was exceeded."); + } + + _frames[_depth++] = new Frame(kind, arrayKind, kind == ContainerKind.Object); + } + + private void EndContainer(ContainerKind expectedKind) + { + if (_depth == 0 || _frames[_depth - 1].Kind != expectedKind) + { + throw new InvalidDataException("The global coverage JSON contains mismatched containers."); + } + + _depth--; + } + + private void OnComma() + { + if (_depth > 0 && _frames[_depth - 1].Kind == ContainerKind.Object) + { + ref var frame = ref _frames[_depth - 1]; + frame.ExpectingProperty = true; + frame.PendingProperty = PropertyKind.Unknown; + } + } + + private void ReadStringToken() + { + var isProperty = _depth > 0 && + _frames[_depth - 1].Kind == ContainerKind.Object && + _frames[_depth - 1].ExpectingProperty; + var valueProperty = !isProperty && _depth > 0 && _frames[_depth - 1].Kind == ContainerKind.Object + ? _frames[_depth - 1].PendingProperty + : PropertyKind.Unknown; + var decodedLength = 0; + var base64Characters = 0; + var base64Padding = 0; + var base64PaddingStarted = false; + + while (true) + { + var value = Read(); + if (value < 0) + { + throw new InvalidDataException("The global coverage JSON contains an unterminated string."); + } + + var character = (char)value; + if (character == '"') + { + break; + } + + if (character < 0x20) + { + throw new InvalidDataException("The global coverage JSON contains an invalid control character."); + } + + if (character == '\\') + { + character = ReadEscapedCharacter(); + } + + decodedLength = checked(decodedLength + 1); + if (decodedLength > _limits.MaximumScalarCharacters) + { + throw new InvalidDataException("The global coverage scalar-token limit was exceeded."); + } + + if (isProperty) + { + if (decodedLength > _limits.MaximumPropertyCharacters) + { + throw new InvalidDataException("The global coverage property-name limit was exceeded."); + } + + _propertyBuffer[decodedLength - 1] = character; + } + else if (valueProperty is PropertyKind.ExecutableBitmap or PropertyKind.ExecutedBitmap) + { + if (!IsBase64Character(character, ref base64PaddingStarted, ref base64Padding)) + { + throw new InvalidDataException("The global coverage JSON contains an invalid base64 bitmap."); + } + + base64Characters = checked(base64Characters + 1); + } + } + + if (isProperty) + { + ref var frame = ref _frames[_depth - 1]; + frame.PendingProperty = GetPropertyKind(decodedLength); + frame.ExpectingProperty = false; + return; + } + + if (valueProperty is PropertyKind.Name or PropertyKind.Path) + { + _identityCharacters = checked(_identityCharacters + decodedLength); + if (_identityCharacters > _limits.MaximumIdentityCharacters) + { + throw new InvalidDataException("The global coverage path/name character limit was exceeded."); + } + } + else if (valueProperty is PropertyKind.ExecutableBitmap or PropertyKind.ExecutedBitmap) + { + if ((base64Characters & 3) != 0 || base64Padding > 2) + { + throw new InvalidDataException("The global coverage JSON contains an invalid base64 bitmap length."); + } + + var decodedBytes = checked(((long)base64Characters / 4 * 3) - base64Padding); + if (decodedBytes > _limits.MaximumBitmapBytes) + { + throw new InvalidDataException("A global coverage bitmap exceeds the per-bitmap limit."); + } + + _bitmapBytes = checked(_bitmapBytes + decodedBytes); + if (_bitmapBytes > _limits.MaximumModelBitmapBytes) + { + throw new InvalidDataException("The global coverage model bitmap limit was exceeded."); + } + } + + CompleteScalarValue(); + } + + private char ReadEscapedCharacter() + { + var value = Read(); + if (value < 0) + { + throw new InvalidDataException("The global coverage JSON contains an incomplete escape."); + } + + return (char)value switch + { + '"' => '"', + '\\' => '\\', + '/' => '/', + 'b' => '\b', + 'f' => '\f', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'u' => ReadUnicodeEscape(), + _ => throw new InvalidDataException("The global coverage JSON contains an invalid escape."), + }; + } + + private char ReadUnicodeEscape() + { + var result = 0; + for (var i = 0; i < 4; i++) + { + var value = Read(); + if (value < 0) + { + throw new InvalidDataException("The global coverage JSON contains an incomplete unicode escape."); + } + + var digit = (char)value; + result = checked((result << 4) + (digit switch + { + >= '0' and <= '9' => digit - '0', + >= 'a' and <= 'f' => digit - 'a' + 10, + >= 'A' and <= 'F' => digit - 'A' + 10, + _ => throw new InvalidDataException("The global coverage JSON contains an invalid unicode escape."), + })); + } + + return (char)result; + } + + private PropertyKind GetPropertyKind(int length) + { + if (Matches("components", length)) + { + return PropertyKind.Components; + } + + if (Matches("files", length)) + { + return PropertyKind.Files; + } + + if (Matches("name", length)) + { + return PropertyKind.Name; + } + + if (Matches("path", length)) + { + return PropertyKind.Path; + } + + if (Matches("executableBitmap", length)) + { + return PropertyKind.ExecutableBitmap; + } + + return Matches("executedBitmap", length) ? PropertyKind.ExecutedBitmap : PropertyKind.Unknown; + } + + private bool Matches(string expected, int actualLength) + { + if (expected.Length != actualLength) + { + return false; + } + + for (var i = 0; i < actualLength; i++) + { + if (_propertyBuffer[i] != expected[i]) + { + return false; + } + } + + return true; + } + + private void ReadScalarToken(char firstCharacter) + { + var length = 1; + var character = firstCharacter; + while (true) + { + var value = Read(); + if (value < 0) + { + return; + } + + character = (char)value; + if (char.IsWhiteSpace(character) || character is ',' or ']' or '}') + { + Unread(); + return; + } + + length = checked(length + 1); + if (length > _limits.MaximumScalarCharacters) + { + throw new InvalidDataException("The global coverage scalar-token limit was exceeded."); + } + } + } + + private void CompleteScalarValue() + { + if (_depth > 0 && _frames[_depth - 1].Kind == ContainerKind.Object) + { + _frames[_depth - 1].PendingProperty = PropertyKind.Unknown; + } + } + + private int Read() + { + if (_readOffset >= _readCount) + { + _readCount = _reader!.Read(_readBuffer, 0, _readBuffer.Length); + _readOffset = 0; + if (_readCount == 0) + { + return -1; + } + } + + return _readBuffer[_readOffset++]; + } + + private void Unread() + { + if (_readOffset == 0) + { + ThrowHelper.ThrowInvalidOperationException("The global coverage scanner cannot unread across a buffer boundary."); + } + + _readOffset--; + } + + private struct Frame + { + public ContainerKind Kind; + public ArrayKind ArrayKind; + public bool ExpectingProperty; + public PropertyKind PendingProperty; + + public Frame(ContainerKind kind, ArrayKind arrayKind, bool expectingProperty) + { + Kind = kind; + ArrayKind = arrayKind; + ExpectingProperty = expectingProperty; + PendingProperty = PropertyKind.Unknown; + } + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageLimitException.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageLimitException.cs new file mode 100644 index 000000000000..473dc211530a --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageLimitException.cs @@ -0,0 +1,18 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageLimitException : Exception +{ + public GlobalCoverageLimitException(string message) + : base(message) + { + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageMetadataException.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageMetadataException.cs new file mode 100644 index 000000000000..092a726f87b9 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageMetadataException.cs @@ -0,0 +1,18 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageMetadataException : Exception +{ + public GlobalCoverageMetadataException(string message) + : base(message) + { + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageOutputManager.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageOutputManager.cs new file mode 100644 index 000000000000..8cedb54ac732 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageOutputManager.cs @@ -0,0 +1,146 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.IO; +using Datadog.Trace.Ci.Coverage.Models.Global; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Ci.Coverage; + +/// +/// Publishes one atomic, process-wide coverage artifact. The pending marker is intentionally the +/// only coordination primitive: it makes an interrupted producer visible without requiring owner +/// election, cross-process leases, or a multi-file commit protocol. +/// +internal sealed class GlobalCoverageOutputManager +{ + private readonly object _gate = new(); + private readonly string? _configuredDirectory; + private readonly string _baseDirectory; + private readonly Func _runIdProvider; + private string? _directory; + private string? _coveragePath; + private string? _pendingPath; + private bool _frozen; + private bool _failed; + private bool _published; + private Exception? _failureException; + + public GlobalCoverageOutputManager(string? configuredDirectory, string baseDirectory, Func runIdProvider) + { + _configuredDirectory = configuredDirectory; + _baseDirectory = baseDirectory; + _runIdProvider = runIdProvider; + } + + public Exception? FailureException + { + get + { + lock (_gate) + { + return _failureException; + } + } + } + + public bool EnsureConfiguredAndFreeze() + { + lock (_gate) + { + if (!_frozen) + { + _frozen = true; + if (!StringUtil.IsNullOrWhiteSpace(_configuredDirectory)) + { + ConfigureUnderLock(_configuredDirectory!); + } + } + + return !_failed; + } + } + + public bool RegisterCollectorAndFreeze(string directory) + { + lock (_gate) + { + if (!_frozen || (!_failed && _directory is null)) + { + _frozen = true; + ConfigureUnderLock(StringUtil.IsNullOrWhiteSpace(_configuredDirectory) ? directory : _configuredDirectory!); + } + + return !_failed; + } + } + + public bool TryPublish(GlobalCoverageInfo model) + { + lock (_gate) + { + if (_failed || _published) + { + return !_failed; + } + + try + { + if (_coveragePath is null) + { + // Coverage can be enabled for in-memory module percentages without configuring + // an artifact directory. In that case there is simply nothing to publish. + _published = true; + return true; + } + + var writer = new GlobalCoverageArtifactWriter(); + using var staged = writer.StageNoReplace(_coveragePath, model); + staged.Commit(); + if (_pendingPath is not null) + { + File.Delete(_pendingPath); + } + + _published = true; + return true; + } + catch (Exception ex) + { + _failureException ??= ex; + _failed = true; + return false; + } + } + } + + private void ConfigureUnderLock(string directory) + { + try + { + var candidate = Path.IsPathRooted(directory) ? directory : Path.Combine(_baseDirectory, directory); + _directory = Path.GetFullPath(candidate); + Directory.CreateDirectory(_directory); + + var runToken = GlobalCoverageProtocol.GetRunToken(_runIdProvider()); + var processIdentity = GlobalCoverageProtocol.GetProcessIdentity(runToken, DomainMetadata.Instance.ProcessId, Guid.NewGuid().ToString("N")); + _coveragePath = Path.Combine(_directory, GlobalCoverageProtocol.GetCoverageFileName(processIdentity)); + _pendingPath = Path.Combine(_directory, GlobalCoverageProtocol.GetPendingMarkerFileName(processIdentity)); + + // FileMode.CreateNew keeps identities collision-free and leaves a durable blocker if + // the process exits before the atomic coverage artifact is committed. + using var pending = new FileStream(_pendingPath, FileMode.CreateNew, FileAccess.Write, FileShare.Read); + pending.Flush(true); + } + catch (Exception ex) + { + _failureException ??= ex; + _failed = true; + } + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageProtocol.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageProtocol.cs new file mode 100644 index 000000000000..55567bb0cd0d --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageProtocol.cs @@ -0,0 +1,54 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace Datadog.Trace.Ci.Coverage; + +// Keep the small on-disk vocabulary and run identity hashing shared by producers and consumers. +internal static class GlobalCoverageProtocol +{ + public const string PendingMarkerPrefix = ".dd-coverage-process-incomplete-"; + public const string CoverageFilePrefix = "coverage-"; + public const string JsonExtension = ".json"; + public const string PendingMarkerPattern = PendingMarkerPrefix + "*"; + public const string CoverageFilePattern = CoverageFilePrefix + "*" + JsonExtension; + + private static readonly Encoding Utf8WithoutBom = new UTF8Encoding(false, true); + + public static string GetRunToken(string runId) + { + var bytes = Utf8WithoutBom.GetBytes(runId); +#if NET6_0_OR_GREATER + var hash = SHA256.HashData(bytes); +#else + byte[] hash; + using (var sha256 = SHA256.Create()) + { + hash = sha256.ComputeHash(bytes); + } +#endif + var builder = new StringBuilder(hash.Length * 2); + foreach (var value in hash) + { + builder.Append(value.ToString("x2", CultureInfo.InvariantCulture)); + } + + return builder.ToString(); + } + + public static string GetProcessIdentity(string runToken, int processId, string nonce) + => $"{runToken}-{processId.ToString(CultureInfo.InvariantCulture)}-{nonce}"; + + public static string GetPendingMarkerFileName(string processIdentity) + => PendingMarkerPrefix + processIdentity; + + public static string GetCoverageFileName(string processIdentity) + => $"{CoverageFilePrefix}{processIdentity}{JsonExtension}"; +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageSnapshot.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageSnapshot.cs new file mode 100644 index 000000000000..233ea9d082f0 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageSnapshot.cs @@ -0,0 +1,55 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Threading; +using Datadog.Trace.Ci.Coverage.Models.Global; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Ci.Coverage; + +internal sealed class GlobalCoverageSnapshot : IDisposable +{ + private SemaphoreSlim? _snapshotGate; + private Action? _releaseAdmission; + private int _disposed; + + public GlobalCoverageSnapshot( + GlobalCoverageInfo model, + long mergedContextCount, + long completenessEpoch, + SemaphoreSlim snapshotGate, + Action? releaseAdmission) + { + Model = model; + MergedContextCount = mergedContextCount; + CompletenessEpoch = completenessEpoch; + _snapshotGate = snapshotGate; + _releaseAdmission = releaseAdmission; + } + + public GlobalCoverageInfo Model { get; } + + public long MergedContextCount { get; } + + public long CompletenessEpoch { get; } + + public bool IsDisposed => Volatile.Read(ref _disposed) != 0; + + public void Dispose() + { + Volatile.Write(ref _disposed, 1); + try + { + Interlocked.Exchange(ref _snapshotGate, null)?.Release(); + } + finally + { + Interlocked.Exchange(ref _releaseAdmission, null)?.Invoke(); + } + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageSnapshotResult.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageSnapshotResult.cs new file mode 100644 index 000000000000..d24b54e19b2f --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageSnapshotResult.cs @@ -0,0 +1,30 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.Ci.Coverage; + +internal readonly struct GlobalCoverageSnapshotResult +{ + private GlobalCoverageSnapshotResult(GlobalCoverageSnapshotStatus status, GlobalCoverageSnapshot? snapshot, GlobalCoverageFailureReason failureReason) + { + Status = status; + Snapshot = snapshot; + FailureReason = failureReason; + } + + public GlobalCoverageSnapshotStatus Status { get; } + + public GlobalCoverageSnapshot? Snapshot { get; } + + public GlobalCoverageFailureReason FailureReason { get; } + + public static GlobalCoverageSnapshotResult Success(GlobalCoverageSnapshot snapshot) + => new(GlobalCoverageSnapshotStatus.Success, snapshot, GlobalCoverageFailureReason.None); + + public static GlobalCoverageSnapshotResult Suppressed(GlobalCoverageFailureReason reason) + => new(GlobalCoverageSnapshotStatus.SuppressedIncomplete, null, reason); +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageStagedArtifact.cs b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageStagedArtifact.cs new file mode 100644 index 000000000000..ce8c97cfb84b --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/GlobalCoverageStagedArtifact.cs @@ -0,0 +1,66 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.IO; +using Datadog.Trace.Util; + +namespace Datadog.Trace.Ci.Coverage; + +// Coverage payloads remain hidden at a temporary path until the caller commits them. +internal sealed class GlobalCoverageStagedArtifact : IDisposable +{ + private readonly string _destinationPath; + private readonly bool _replaceExisting; + private string? _temporaryPath; + + public GlobalCoverageStagedArtifact(string temporaryPath, string destinationPath, bool replaceExisting) + { + _temporaryPath = temporaryPath; + _destinationPath = destinationPath; + _replaceExisting = replaceExisting; + } + + public void Commit() + { + var temporaryPath = _temporaryPath; + if (temporaryPath is null) + { + ThrowHelper.ThrowInvalidOperationException("The staged global coverage artifact is no longer available."); + } + + if (_replaceExisting && File.Exists(_destinationPath)) + { + File.Replace(temporaryPath, _destinationPath, null); + } + else + { + File.Move(temporaryPath, _destinationPath); + } + + _temporaryPath = null; + } + + public void Dispose() + { + var temporaryPath = _temporaryPath; + _temporaryPath = null; + if (temporaryPath is null) + { + return; + } + + try + { + File.Delete(temporaryPath); + } + catch + { + // A staged artifact is never a valid input and cleanup must not hide the primary failure. + } + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/Metadata/CoverageMetadataValidator.cs b/tracer/src/Datadog.Trace/Ci/Coverage/Metadata/CoverageMetadataValidator.cs new file mode 100644 index 000000000000..126c76d601e5 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/Metadata/CoverageMetadataValidator.cs @@ -0,0 +1,61 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using Datadog.Trace.Util; + +namespace Datadog.Trace.Ci.Coverage.Metadata; + +internal static class CoverageMetadataValidator +{ + public static int ValidateAndGetRawByteLength(ModuleCoverageMetadata metadata) + { + if (metadata.CoverageMode is not 0 and not 1) + { + ThrowHelper.ThrowInvalidOperationException($"Unsupported coverage mode '{metadata.CoverageMode}'."); + } + + if (metadata.TotalLines < 0) + { + ThrowHelper.ThrowInvalidOperationException("Coverage metadata contains a negative total line count."); + } + + var bytesPerLine = metadata.CoverageMode == 0 ? sizeof(byte) : sizeof(int); + if (metadata.TotalLines > int.MaxValue / bytesPerLine) + { + ThrowHelper.ThrowInvalidOperationException("Coverage metadata requires a raw counter buffer larger than the supported size."); + } + + var rawByteLength = metadata.TotalLines * bytesPerLine; + + foreach (var file in metadata.Files) + { + if (file.Offset < 0 || file.LastExecutableLine < 0) + { + ThrowHelper.ThrowInvalidOperationException("Coverage metadata contains a negative file offset or line count."); + } + + var end = (long)file.Offset + file.LastExecutableLine; + if (end > int.MaxValue) + { + ThrowHelper.ThrowInvalidOperationException("Coverage metadata contains an overflowing file range."); + } + + if (end > metadata.TotalLines) + { + ThrowHelper.ThrowInvalidOperationException("Coverage metadata contains a file range outside the module counter buffer."); + } + + var expectedBitmapLength = ((long)file.LastExecutableLine + 7) / 8; + if (file.Bitmap is null || file.Bitmap.Length != expectedBitmapLength) + { + ThrowHelper.ThrowInvalidOperationException("Coverage metadata contains an executable bitmap with an invalid length."); + } + } + + return rawByteLength; + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/ModuleCoverageData.cs b/tracer/src/Datadog.Trace/Ci/Coverage/ModuleCoverageData.cs new file mode 100644 index 000000000000..dd0cd3249d29 --- /dev/null +++ b/tracer/src/Datadog.Trace/Ci/Coverage/ModuleCoverageData.cs @@ -0,0 +1,94 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Reflection; +using Datadog.Trace.Ci.Coverage.Metadata; +using Datadog.Trace.Ci.Coverage.Util; + +namespace Datadog.Trace.Ci.Coverage; + +/// +/// Contains the compact executed-line bitmaps captured from one module buffer. +/// +internal readonly struct ModuleCoverageData +{ + public ModuleCoverageData(Module module, ModuleCoverageMetadata metadata, byte[]?[] executedBitmaps) + { + Module = module; + Metadata = metadata; + ExecutedBitmaps = executedBitmaps; + } + + public Module Module { get; } + + public ModuleCoverageMetadata Metadata { get; } + + public byte[]?[] ExecutedBitmaps { get; } + + /// + /// Scans the native counters once, outside the accumulator lock, and retains only executed bits. + /// Counter values are intentionally read without synchronization because coverage only depends on + /// whether a counter is non-zero. + /// + public static unsafe ModuleCoverageData Capture(ModuleValue moduleValue) + { + var metadata = moduleValue.Metadata; + var expectedRawByteLength = CoverageMetadataValidator.ValidateAndGetRawByteLength(metadata); + if (moduleValue.AllocatedByteLength != expectedRawByteLength) + { + throw new GlobalCoverageMetadataException("A coverage buffer length does not match its metadata."); + } + + var rawPointer = moduleValue.FilesLines; + if (rawPointer == IntPtr.Zero) + { + throw new GlobalCoverageMetadataException("A coverage buffer was disposed before aggregation."); + } + + var executedBitmaps = new byte[]?[metadata.Files.Length]; + for (var fileIndex = 0; fileIndex < metadata.Files.Length; fileIndex++) + { + var file = metadata.Files[fileIndex]; + byte[]? executedBitmap = null; + if (metadata.CoverageMode == 0) + { + var counters = (byte*)rawPointer + file.Offset; + for (var lineIndex = 0; lineIndex < file.LastExecutableLine; lineIndex++) + { + if (counters[lineIndex] != 0) + { + executedBitmap ??= new byte[FileBitmap.GetSize(file.LastExecutableLine)]; + SetBit(executedBitmap, lineIndex); + } + } + } + else + { + var counters = (int*)rawPointer + file.Offset; + for (var lineIndex = 0; lineIndex < file.LastExecutableLine; lineIndex++) + { + if (counters[lineIndex] != 0) + { + executedBitmap ??= new byte[FileBitmap.GetSize(file.LastExecutableLine)]; + SetBit(executedBitmap, lineIndex); + } + } + } + + executedBitmaps[fileIndex] = executedBitmap; + } + + return new ModuleCoverageData(moduleValue.Module, metadata, executedBitmaps); + } + + private static void SetBit(byte[] bitmap, int zeroBasedLine) + { + var byteIndex = zeroBasedLine >> 3; + bitmap[byteIndex] |= (byte)(128 >> (zeroBasedLine & 7)); + } +} diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/ModuleValue.cs b/tracer/src/Datadog.Trace/Ci/Coverage/ModuleValue.cs index 18b671040a01..78f4fdeef120 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/ModuleValue.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/ModuleValue.cs @@ -9,19 +9,55 @@ using System.Reflection; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Threading; using Datadog.Trace.Ci.Coverage.Metadata; +using Datadog.Trace.Logging; +using Datadog.Trace.Vendors.Serilog.Events; namespace Datadog.Trace.Ci.Coverage; internal sealed class ModuleValue : IDisposable { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public unsafe ModuleValue(ModuleCoverageMetadata metadata, Module module, int fileLinesMemorySize) + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + private static readonly NativeMemoryDebugMetrics ContextBufferMetrics = new(); + private static readonly NativeMemoryDebugMetrics GlobalFallbackBufferMetrics = new(); + private readonly BufferKind _bufferKind; + private readonly bool _recordNativeMemoryDiagnostics; + private IntPtr _filesLines; + private int _allocatedByteLength; + + public ModuleValue(ModuleCoverageMetadata metadata, Module module, int fileLinesMemorySize, BufferKind bufferKind) { Metadata = metadata; Module = module; - FilesLines = Marshal.AllocHGlobal(fileLinesMemorySize); - Unsafe.InitBlockUnaligned((byte*)FilesLines, 0, (uint)fileLinesMemorySize); + _bufferKind = bufferKind; + // Keep allocation and free accounting paired even if debug logging changes while this value is alive. + _recordNativeMemoryDiagnostics = Log.IsEnabled(LogEventLevel.Debug); + + var pointer = IntPtr.Zero; + try + { + pointer = Marshal.AllocHGlobal(fileLinesMemorySize); + unsafe + { + Unsafe.InitBlockUnaligned((byte*)pointer, 0, (uint)fileLinesMemorySize); + } + + _allocatedByteLength = fileLinesMemorySize; + _filesLines = pointer; + pointer = IntPtr.Zero; + if (_recordNativeMemoryDiagnostics) + { + GetMetrics(bufferKind).OnAllocated(fileLinesMemorySize); + } + } + finally + { + if (pointer != IntPtr.Zero) + { + Marshal.FreeHGlobal(pointer); + } + } } ~ModuleValue() @@ -29,21 +65,177 @@ public unsafe ModuleValue(ModuleCoverageMetadata metadata, Module module, int fi Dispose(); } + public enum BufferKind + { + Context, + GlobalFallback, + } + public ModuleCoverageMetadata Metadata { get; } public Module Module { get; } - public IntPtr FilesLines { get; private set; } + // Instrumented methods read this on every entry. The test lifecycle owns the buffer lifetime, + // so keep the read identical to the original lock-free counter path. + public IntPtr FilesLines => _filesLines; + + public int AllocatedByteLength => _allocatedByteLength; + + public static void LogNativeMemoryDiagnostics(int processId) + { + if (!Log.IsEnabled(LogEventLevel.Debug)) + { + return; + } + + LogNativeMemoryDiagnostics(processId, BufferKind.Context, ContextBufferMetrics.GetSnapshot()); + LogNativeMemoryDiagnostics(processId, BufferKind.GlobalFallback, GlobalFallbackBufferMetrics.GetSnapshot()); + } + + private static NativeMemoryDebugMetrics GetMetrics(BufferKind bufferKind) + => bufferKind == BufferKind.Context ? ContextBufferMetrics : GlobalFallbackBufferMetrics; + + private static void LogNativeMemoryDiagnostics(int processId, BufferKind bufferKind, NativeMemoryDebugSnapshot diagnostics) + { + if (bufferKind == BufferKind.Context) + { + Log.Debug( + "Global coverage native context-buffer diagnostics: pid={ProcessId}, currentBytes={CurrentBytes}, peakBytes={PeakBytes}, activeBuffers={ActiveBuffers}, peakBuffers={PeakBuffers}.", + processId, + diagnostics.CurrentBytes, + diagnostics.PeakBytes, + diagnostics.ActiveBuffers, + diagnostics.PeakBuffers); + Log.Debug( + "Global coverage native context-buffer allocation diagnostics: pid={ProcessId}, allocations={Allocations}, frees={Frees}, maximumBufferBytes={MaximumBufferBytes}.", + processId, + diagnostics.AllocationCount, + diagnostics.FreeCount, + diagnostics.MaximumBufferBytes); + return; + } + + Log.Debug( + "Global coverage native fallback-buffer diagnostics: pid={ProcessId}, currentBytes={CurrentBytes}, peakBytes={PeakBytes}, activeBuffers={ActiveBuffers}, peakBuffers={PeakBuffers}.", + processId, + diagnostics.CurrentBytes, + diagnostics.PeakBytes, + diagnostics.ActiveBuffers, + diagnostics.PeakBuffers); + Log.Debug( + "Global coverage native fallback-buffer allocation diagnostics: pid={ProcessId}, allocations={Allocations}, frees={Frees}, maximumBufferBytes={MaximumBufferBytes}.", + processId, + diagnostics.AllocationCount, + diagnostics.FreeCount, + diagnostics.MaximumBufferBytes); + } public void Dispose() { - var filesLines = FilesLines; + FreeBuffer(); + GC.SuppressFinalize(this); + } + + private void FreeBuffer() + { + var filesLines = Interlocked.Exchange(ref _filesLines, IntPtr.Zero); if (filesLines != IntPtr.Zero) { - FilesLines = IntPtr.Zero; + var byteLength = Volatile.Read(ref _allocatedByteLength); Marshal.FreeHGlobal(filesLines); + Volatile.Write(ref _allocatedByteLength, 0); + if (_recordNativeMemoryDiagnostics) + { + GetMetrics(_bufferKind).OnFreed(byteLength); + } } + } - GC.SuppressFinalize(this); + private readonly struct NativeMemoryDebugSnapshot + { + public NativeMemoryDebugSnapshot( + long currentBytes, + long peakBytes, + long activeBuffers, + long peakBuffers, + long allocationCount, + long freeCount, + long maximumBufferBytes) + { + CurrentBytes = currentBytes; + PeakBytes = peakBytes; + ActiveBuffers = activeBuffers; + PeakBuffers = peakBuffers; + AllocationCount = allocationCount; + FreeCount = freeCount; + MaximumBufferBytes = maximumBufferBytes; + } + + public long CurrentBytes { get; } + + public long PeakBytes { get; } + + public long ActiveBuffers { get; } + + public long PeakBuffers { get; } + + public long AllocationCount { get; } + + public long FreeCount { get; } + + public long MaximumBufferBytes { get; } + } + + private sealed class NativeMemoryDebugMetrics + { + private long _currentBytes; + private long _peakBytes; + private long _activeBuffers; + private long _peakBuffers; + private long _allocationCount; + private long _freeCount; + private long _maximumBufferBytes; + + public void OnAllocated(int byteLength) + { + var currentBytes = Interlocked.Add(ref _currentBytes, byteLength); + var activeBuffers = Interlocked.Increment(ref _activeBuffers); + Interlocked.Increment(ref _allocationCount); + SetMaximum(ref _peakBytes, currentBytes); + SetMaximum(ref _peakBuffers, activeBuffers); + SetMaximum(ref _maximumBufferBytes, byteLength); + } + + public void OnFreed(int byteLength) + { + Interlocked.Add(ref _currentBytes, -byteLength); + Interlocked.Decrement(ref _activeBuffers); + Interlocked.Increment(ref _freeCount); + } + + public NativeMemoryDebugSnapshot GetSnapshot() + => new( + Interlocked.Read(ref _currentBytes), + Interlocked.Read(ref _peakBytes), + Interlocked.Read(ref _activeBuffers), + Interlocked.Read(ref _peakBuffers), + Interlocked.Read(ref _allocationCount), + Interlocked.Read(ref _freeCount), + Interlocked.Read(ref _maximumBufferBytes)); + + private static void SetMaximum(ref long target, long value) + { + var current = Interlocked.Read(ref target); + while (value > current) + { + var observed = Interlocked.CompareExchange(ref target, value, current); + if (observed == current) + { + return; + } + + current = observed; + } + } } } diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/Util/CoverageUtils.cs b/tracer/src/Datadog.Trace/Ci/Coverage/Util/CoverageUtils.cs index 37e158c0c4ad..f833b1c88c1e 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/Util/CoverageUtils.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/Util/CoverageUtils.cs @@ -1,4 +1,4 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // @@ -6,14 +6,12 @@ #nullable enable using System; -using System.Collections.Generic; using System.IO; -using System.Text; using Datadog.Trace.Ci; +using Datadog.Trace.Ci.Coverage; using Datadog.Trace.Ci.Coverage.Models.Global; using Datadog.Trace.Logging; -using Datadog.Trace.Util.Json; -using Datadog.Trace.Vendors.Newtonsoft.Json; +using Datadog.Trace.Util; internal static class CoverageUtils { @@ -26,23 +24,24 @@ public static bool TryCombineAndGetTotalCoverage(string inputFolder, string outp public static bool TryCombineAndGetTotalCoverage(string? inputFolder, string? outputFile, out GlobalCoverageInfo? globalCoverageInfo) { - if (string.IsNullOrEmpty(outputFile)) + globalCoverageInfo = null; + if (StringUtil.IsNullOrEmpty(outputFile)) { globalCoverageInfo = null; return false; } - if (!TryCombineAndGetTotalCoverage(inputFolder, out globalCoverageInfo)) - { - return false; - } - try { - using var fStream = File.OpenWrite(outputFile); - using var sWriter = new StreamWriter(fStream, Encoding.UTF8, 4096, false); - using var jsonWriter = new JsonTextWriter(sWriter) { ArrayPool = JsonArrayPool.Shared }; - new JsonSerializer().Serialize(jsonWriter, globalCoverageInfo); + if (!TryReadAndCombine(inputFolder, outputFile, expectedRunToken: null, out globalCoverageInfo)) + { + return false; + } + + var writer = new GlobalCoverageArtifactWriter(); + using var stagedOutput = writer.StageReplace(outputFile!, globalCoverageInfo!); + stagedOutput.Commit(); + return true; } catch (Exception ex) @@ -53,13 +52,17 @@ public static bool TryCombineAndGetTotalCoverage(string? inputFolder, string? ou return false; } - private static bool TryCombineAndGetTotalCoverage(string? inputFolder, out GlobalCoverageInfo? globalCoverageInfo) + public static bool TryReadAndCombine( + string? inputFolder, + string? outputFile, + string? expectedRunToken, + out GlobalCoverageInfo? globalCoverageInfo) { globalCoverageInfo = default; try { - if (string.IsNullOrEmpty(inputFolder)) + if (StringUtil.IsNullOrEmpty(inputFolder)) { return false; } @@ -70,31 +73,33 @@ private static bool TryCombineAndGetTotalCoverage(string? inputFolder, out Globa return false; } - var jsonFiles = Directory.GetFiles(inputFolder, "*.json", SearchOption.TopDirectoryOnly); + if (!GlobalCoverageFileCombiner.TryAcquireInputFiles(inputFolder!, expectedRunToken, out var jsonFiles)) + { + return false; + } + if (jsonFiles.Length == 0) { Log.ErrorSkipTelemetry("'{InputFolder}' doesn't contain any json file.", inputFolder); return false; } - List globalCoverages = new(); - foreach (var file in jsonFiles) + if (!GlobalCoverageFileCombiner.TryCombine( + jsonFiles, + outputFile, + requireAllInputs: expectedRunToken is not null, + onFileProcessed: null, + out globalCoverageInfo, + out var rejectedInput)) { - var fileContent = File.ReadAllText(file); - try + if (rejectedInput is not null) { - if (JsonHelper.DeserializeObject(fileContent) is { } gCoverageInfo) - { - globalCoverages.Add(gCoverageInfo); - } - } - catch (Exception ex) - { - Log.Error(ex, "Error processing {File}", file); + Log.Error("Error processing global coverage input: {File}", rejectedInput); } + + return false; } - globalCoverageInfo = GlobalCoverageInfo.Combine(globalCoverages.ToArray()); return true; } catch (Exception globalEx) diff --git a/tracer/src/Datadog.Trace/Ci/Coverage/Util/FileBitmap.cs b/tracer/src/Datadog.Trace/Ci/Coverage/Util/FileBitmap.cs index 4a32c1646e67..20ba15af4a31 100644 --- a/tracer/src/Datadog.Trace/Ci/Coverage/Util/FileBitmap.cs +++ b/tracer/src/Datadog.Trace/Ci/Coverage/Util/FileBitmap.cs @@ -12,6 +12,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; +using Datadog.Trace.Util; #if NETCOREAPP3_1_OR_GREATER using System.Runtime.Intrinsics; @@ -144,7 +145,7 @@ public static FileBitmap FromActiveRange(int fromLine, int toLine) if (fromLine <= 0 || toLine < fromLine) { - throw new ArgumentException("Invalid range"); + ThrowHelper.ThrowArgumentException("Invalid range"); } for (var i = fromLine; i <= toLine; i++) @@ -581,7 +582,15 @@ public static FileBitmap Not(FileBitmap fileBitmap, bool reuseBufferFromBitmap) /// The number of lines. /// The required storage size in bytes. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetSize(int numOfLines) => (numOfLines + 7) / 8; + public static int GetSize(int numOfLines) + { + if (numOfLines < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException(nameof(numOfLines)); + } + + return checked((int)(((long)numOfLines + 7) / 8)); + } #if !NETCOREAPP3_1_OR_GREATER /// diff --git a/tracer/src/Datadog.Trace/Ci/Test.cs b/tracer/src/Datadog.Trace/Ci/Test.cs index f238c09ff0ea..3f2e8f350123 100644 --- a/tracer/src/Datadog.Trace/Ci/Test.cs +++ b/tracer/src/Datadog.Trace/Ci/Test.cs @@ -29,9 +29,10 @@ public sealed class Test { private static readonly AsyncLocal CurrentTest = new(); private static readonly HashSet OpenedTests = new(); - private readonly ITestOptimization _testOptimization; private readonly Scope _scope; + private readonly Test? _priorTest; + private Coverage.CoverageSessionHandle? _coverageSessionHandle; private int _finished; private List>? _onCloseActions; @@ -44,59 +45,96 @@ internal Test(TestSuite suite, string name, DateTimeOffset? startDate, TraceId t { Suite = suite; var module = suite.Module; - - var tags = new TestSpanTags(Suite.Tags, name); - var tracer = Tracer.Instance; - var span = tracer.StartSpan( - string.IsNullOrEmpty(module.Framework) ? "test" : $"{module.Framework!.ToLowerInvariant()}.test", - tags: tags, - startTime: startDate, - traceId: traceId, - spanId: spanId); - var scope = tracer.TracerManager.ScopeManager.Activate(span, true); - - scope.Span.Type = SpanTypes.Test; - scope.Span.ResourceName = $"{suite.Name}.{name}"; - scope.Span.Context.TraceContext.SetSamplingPriority(SamplingPriorityValues.AutoKeep, SamplingMechanism.Manual); - scope.Span.Context.TraceContext.Origin = TestTags.CIAppTestOriginName; - TelemetryFactory.Metrics.RecordCountSpanCreated(MetricTags.IntegrationName.CiAppManual); - - _scope = scope; + _priorTest = CurrentTest.Value; _testOptimization = TestOptimization.Instance; - if (_testOptimization.Settings.CodeCoverageEnabled == true) + Scope? activatedScope = null; + Coverage.CoverageSessionHandle? coverageSessionHandle = null; + try { - Coverage.CoverageReporter.Handler.StartSession(module.Framework); - } + var tags = new TestSpanTags(Suite.Tags, name); + var tracer = Tracer.Instance; + var span = tracer.StartSpan( + StringUtil.IsNullOrEmpty(module.Framework) ? "test" : $"{module.Framework!.ToLowerInvariant()}.test", + tags: tags, + startTime: startDate, + traceId: traceId, + spanId: spanId); + activatedScope = tracer.TracerManager.ScopeManager.Activate(span, true); + _scope = activatedScope; + + activatedScope.Span.Type = SpanTypes.Test; + activatedScope.Span.ResourceName = $"{suite.Name}.{name}"; + activatedScope.Span.Context.TraceContext.SetSamplingPriority(SamplingPriorityValues.AutoKeep, SamplingMechanism.Manual); + activatedScope.Span.Context.TraceContext.Origin = TestTags.CIAppTestOriginName; + TelemetryFactory.Metrics.RecordCountSpanCreated(MetricTags.IntegrationName.CiAppManual); + + if (_testOptimization.Settings.CodeCoverageEnabled == true) + { + coverageSessionHandle = Coverage.CoverageReporter.Handler.StartSession(module.Framework); + } - // Capabilities tags (yes they are strings, this is because previously the values were "true" or "false" and we changed the format in attempt_to_fix-v2) - tags.CapabilitiesTestImpactAnalysis = "1"; - tags.CapabilitiesEarlyFlakeDetection = "1"; - tags.CapabilitiesAutoTestRetries = "1"; - tags.CapabilitiesTestManagementQuarantine = "1"; - tags.CapabilitiesTestManagementDisable = "1"; - tags.CapabilitiesTestManagementAttemptToFix = "4"; + // Capabilities tags (yes they are strings, this is because previously the values were "true" or "false" and we changed the format in attempt_to_fix-v2) + tags.CapabilitiesTestImpactAnalysis = "1"; + tags.CapabilitiesEarlyFlakeDetection = "1"; + tags.CapabilitiesAutoTestRetries = "1"; + tags.CapabilitiesTestManagementQuarantine = "1"; + tags.CapabilitiesTestManagementDisable = "1"; + tags.CapabilitiesTestManagementAttemptToFix = "4"; - CurrentTest.Value = this; - lock (OpenedTests) - { - OpenedTests.Add(this); - } + CurrentTest.Value = this; + lock (OpenedTests) + { + OpenedTests.Add(this); + } - _testOptimization.Log.Debug("######### New Test Created: {Name} ({Suite} | {Module})", Name, Suite.Name, Suite.Module.Name); + _testOptimization.Log.Debug("######### New Test Created: {Name} ({Suite} | {Module})", Name, Suite.Name, Suite.Module.Name); - if (startDate is null) - { - // If a test doesn't have a fixed start time we reset it before running the test code - scope.Span.ResetStartTime(); - } + if (startDate is null) + { + // If a test doesn't have a fixed start time we reset it before running the test code + activatedScope.Span.ResetStartTime(); + } - // Record EventCreate telemetry metric - if (TelemetryHelper.GetEventTypeWithCodeOwnerAndSupportedCiAndBenchmark( - MetricTags.CIVisibilityTestingEventType.Test, - module.Framework == CommonTags.TestingFrameworkNameBenchmarkDotNet) is { } eventTypeWithMetadata) + // Record EventCreate telemetry metric + if (TelemetryHelper.GetEventTypeWithCodeOwnerAndSupportedCiAndBenchmark( + MetricTags.CIVisibilityTestingEventType.Test, + module.Framework == CommonTags.TestingFrameworkNameBenchmarkDotNet) is { } eventTypeWithMetadata) + { + TelemetryFactory.Metrics.RecordCountCIVisibilityEventCreated(TelemetryHelper.GetTelemetryTestingFrameworkEnum(module.Framework), eventTypeWithMetadata); + } + + _coverageSessionHandle = coverageSessionHandle; + } + catch { - TelemetryFactory.Metrics.RecordCountCIVisibilityEventCreated(TelemetryHelper.GetTelemetryTestingFrameworkEnum(module.Framework), eventTypeWithMetadata); + try + { + (coverageSessionHandle ?? _coverageSessionHandle)?.AbortIncomplete(Coverage.GlobalCoverageFailureReason.TestConstructionFailed); + } + catch + { + } + + try + { + activatedScope?.Dispose(); + } + catch + { + } + + if (ReferenceEquals(CurrentTest.Value, this)) + { + CurrentTest.Value = _priorTest; + } + + lock (OpenedTests) + { + OpenedTests.Remove(this); + } + + throw; } } @@ -472,24 +510,38 @@ public void Close(TestStatus status, TimeSpan? duration, string? skipReason) var tags = (TestSpanTags)scope.Span.Tags; // Calculate duration beforehand - duration ??= _scope.Span.Context.TraceContext.Clock.ElapsedSince(scope.Span.StartTime); + duration ??= scope.Span.Context.TraceContext.Clock.ElapsedSince(scope.Span.StartTime); - // Set coverage - if (_testOptimization.Settings.CodeCoverageEnabled == true) + var coverageSessionHandle = Interlocked.Exchange(ref _coverageSessionHandle, null); + var coverageEnded = coverageSessionHandle is null || !coverageSessionHandle.IsValid; + try { - if (Coverage.CoverageReporter.Handler.EndSession() is Coverage.Models.Tests.TestCoverage testCoverage) + // Set coverage through the exact handler/context captured when the test was constructed. + if (coverageSessionHandle is { IsValid: true }) { - testCoverage.SessionId = tags.SessionId; - testCoverage.SuiteId = tags.SuiteId; - testCoverage.SpanId = _scope.Span.SpanId; + var coverageResult = coverageSessionHandle.Owner!.EndSession(coverageSessionHandle); + coverageEnded = true; + if (coverageResult is Coverage.Models.Tests.TestCoverage testCoverage) + { + testCoverage.SessionId = tags.SessionId; + testCoverage.SuiteId = tags.SuiteId; + testCoverage.SpanId = scope.Span.SpanId; - _testOptimization.Log.Debug("Coverage data for SessionId={SessionId}, SuiteId={SuiteId} and SpanId={SpanId} processed.", testCoverage.SessionId, testCoverage.SuiteId, testCoverage.SpanId); - _testOptimization.TracerManagement?.Manager?.WriteEvent(testCoverage); + _testOptimization.Log.Debug("Coverage data for SessionId={SessionId}, SuiteId={SuiteId} and SpanId={SpanId} processed.", testCoverage.SessionId, testCoverage.SuiteId, testCoverage.SpanId); + _testOptimization.TracerManagement?.Manager?.WriteEvent(testCoverage); + } + else if (status != TestStatus.Skip) + { + var testName = scope.Span.ResourceName; + _testOptimization.Log.Warning("Coverage data for test: {TestName} with Status: {Status} is empty. File: {File}", testName, status, tags.SourceFile); + } } - else if (status != TestStatus.Skip) + } + finally + { + if (!coverageEnded) { - var testName = scope.Span.ResourceName; - _testOptimization.Log.Warning("Coverage data for test: {TestName} with Status: {Status} is empty. File: {File}", testName, status, tags.SourceFile); + coverageSessionHandle?.AbortIncomplete(Coverage.GlobalCoverageFailureReason.TestCloseBeforeCoverage); } } @@ -532,7 +584,6 @@ public void Close(TestStatus status, TimeSpan? duration, string? skipReason) TelemetryFactory.Metrics.RecordCountCIVisibilityITRForcedRun(MetricTags.CIVisibilityTestingEventType.Test); } - // Call close actions if (_onCloseActions is not null) { foreach (var action in _onCloseActions) @@ -543,17 +594,15 @@ public void Close(TestStatus status, TimeSpan? duration, string? skipReason) _onCloseActions.Clear(); } - // Finish scope.Span.Finish(duration.Value); scope.Dispose(); - // Record EventFinished telemetry metric if (TelemetryHelper.GetEventTypeWithCodeOwnerAndSupportedCiAndBenchmarkAndEarlyFlakeDetection( MetricTags.CIVisibilityTestingEventType.Test, tags.Type == TestTags.TypeBenchmark, tags.TestIsNew == "true", tags.EarlyFlakeDetectionTestAbortReason == "slow", - !string.IsNullOrEmpty(tags.BrowserDriver), + !StringUtil.IsNullOrEmpty(tags.BrowserDriver), tags.IsRumActive == "true") is { } eventTypeWithMetadata) { var retryReasonTag = tags.TestRetryReason switch diff --git a/tracer/src/Datadog.Trace/Ci/TestModule.cs b/tracer/src/Datadog.Trace/Ci/TestModule.cs index cadda159cf63..06c0e306a38b 100644 --- a/tracer/src/Datadog.Trace/Ci/TestModule.cs +++ b/tracer/src/Datadog.Trace/Ci/TestModule.cs @@ -407,33 +407,35 @@ private bool InternalClose(TimeSpan? duration) Tags.Status ??= TestTags.StatusPass; if (_testOptimization.Settings.CodeCoverageEnabled == true && - CoverageReporter.Handler is DefaultWithGlobalCoverageEventHandler coverageHandler && - coverageHandler.GetCodeCoveragePercentage() is { } globalCoverage) + CoverageReporter.Handler is DefaultWithGlobalCoverageEventHandler coverageHandler) { - // We only report global code coverage if ITR is disabled and we are in a fake session (like the internal testlogger scenario) - // For a normal customer session we never report the percentage of total lines on modules - if (!_testOptimization.Settings.IntelligentTestRunnerEnabled && _fakeSession is not null) + var snapshotResult = coverageHandler.AcquireGlobalCoverageSnapshot(); + if (snapshotResult.Status == GlobalCoverageSnapshotStatus.Success && snapshotResult.Snapshot is { } snapshot) { - // Adds the global code coverage percentage to the module - var codeCoveragePercentage = globalCoverage.GetTotalPercentage(); - SetTag(CodeCoverageTags.PercentageOfTotalLines, codeCoveragePercentage); - _fakeSession.SetTag(CodeCoverageTags.PercentageOfTotalLines, codeCoveragePercentage); - } - - // If the code coverage path environment variable is set, we store the json file - if (!string.IsNullOrWhiteSpace(_testOptimization.Settings.CodeCoveragePath)) - { - var codeCoveragePath = Path.Combine(_testOptimization.Settings.CodeCoveragePath, $"coverage-{DateTime.Now:yyyy-MM-dd_HH_mm_ss}-{Guid.NewGuid():n}.json"); - try - { - using var fStream = File.OpenWrite(codeCoveragePath); - using var sWriter = new StreamWriter(fStream, Encoding.UTF8, 4096, false); - using var jsonWriter = new JsonTextWriter(sWriter) { ArrayPool = JsonArrayPool.Shared }; - JsonSerializer.Create().Serialize(jsonWriter, globalCoverage); - } - catch (Exception ex) + using (snapshot) { - _testOptimization.Log.Error(ex, "Error writing global code coverage."); + var globalCoverage = snapshot.Model; + try + { + coverageHandler.TryCommit( + snapshot, + () => + { + // We only report global code coverage if ITR is disabled and we are in a fake session (like the internal testlogger scenario) + // For a normal customer session we never report the percentage of total lines on modules + if (!_testOptimization.Settings.IntelligentTestRunnerEnabled && _fakeSession is not null) + { + // Adds the global code coverage percentage to the module + var codeCoveragePercentage = globalCoverage.GetTotalPercentage(); + SetTag(CodeCoverageTags.PercentageOfTotalLines, codeCoveragePercentage); + _fakeSession.SetTag(CodeCoverageTags.PercentageOfTotalLines, codeCoveragePercentage); + } + }); + } + catch (Exception ex) + { + _testOptimization.Log.Error(ex, "Error writing global code coverage."); + } } } } diff --git a/tracer/src/Datadog.Trace/Ci/TestOptimization.cs b/tracer/src/Datadog.Trace/Ci/TestOptimization.cs index 1dab4e5e6bef..d145c43a12dc 100644 --- a/tracer/src/Datadog.Trace/Ci/TestOptimization.cs +++ b/tracer/src/Datadog.Trace/Ci/TestOptimization.cs @@ -11,6 +11,7 @@ using Datadog.Trace.Agent.DiscoveryService; using Datadog.Trace.Ci.CiEnvironment; using Datadog.Trace.Ci.Configuration; +using Datadog.Trace.Ci.Coverage; using Datadog.Trace.Ci.Net; using Datadog.Trace.Configuration; using Datadog.Trace.Logging; @@ -549,6 +550,15 @@ private async Task ShutdownAsync(Exception? exception) await testModule.CloseAsync().ConfigureAwait(false); } + try + { + CoverageReporter.FinalizeGlobalCoverage(); + } + catch (Exception ex) + { + Log.Error(ex, "TestOptimization: Error finalizing global code coverage during shutdown."); + } + foreach (var testSession in TestSession.ActiveTestSessions) { if (exception is not null) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs index 5037de9b084c..e9eea378c657 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/DotnetTest/DotnetCommon.cs @@ -288,45 +288,7 @@ internal static void FinalizeSession(TestSession? session, int exitCode, Excepti // Note: we also write the total global code coverage to the `session-coverage-{date}.json` file if (!StringUtil.IsNullOrEmpty(codeCoveragePath)) { - try - { - var outputPath = Path.Combine(codeCoveragePath, $"session-coverage-{DateTime.Now:yyyy-MM-dd_HH_mm_ss}.json"); - if (CoverageUtils.TryCombineAndGetTotalCoverage(codeCoveragePath, outputPath, out var globalCoverage) && - globalCoverage is not null) - { - var backfillResult = TryApplyItrCoverageBackfill(session, globalCoverage); - if (!backfillResult.CanPublishCoverage) - { - Log.Warning("RunCiCommand: ITR coverage backfill could not match backend coverage to Datadog internal coverage. The coverage result will not be published."); - TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageErrors(); - TryDeleteFile(outputPath); - } - else - { - if (backfillResult.Backfilled) - { - File.WriteAllText(outputPath, JsonHelper.SerializeObject(globalCoverage)); - } - - // We only report the code coverage percentage if the customer manually sets the 'DD_CIVISIBILITY_CODE_COVERAGE_ENABLED' environment variable according to the new spec. - if (EnvironmentHelpers.GetEnvironmentVariable(Configuration.ConfigurationKeys.CIVisibility.CodeCoverage)?.ToBoolean() == true) - { - var data = globalCoverage.Data; - session.RecordCodeCoverage( - CodeCoverageReportSource.DatadogInternal, - globalCoverage.GetTotalPercentage(), - backfillResult.Backfilled, - executableLines: data[1], - coveredLines: data[2]); - } - } - } - } - catch (Exception ex) - { - Log.Warning(ex, "RunCiCommand: Error while reading or backfilling Datadog internal code coverage."); - TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageErrors(); - } + TryFinalizeDatadogInternalCoverage(session, codeCoveragePath!); } try @@ -369,6 +331,59 @@ internal static void FinalizeSession(TestSession? session, int exitCode, Excepti session.Close(exitCode == 0 ? TestStatus.Pass : TestStatus.Fail); } + private static void TryFinalizeDatadogInternalCoverage(TestSession? session, string codeCoveragePath) + { + try + { + var outputPath = Path.Combine(codeCoveragePath, $"session-coverage-{DateTime.UtcNow:yyyy-MM-dd_HH_mm_ss_fffffff}-{Guid.NewGuid():N}.json"); + var runId = EnvironmentHelpers.GetEnvironmentVariable(ConfigurationKeys.CIVisibility.TestOptimizationRunId) ?? TestOptimization.Instance.RunId; + var runToken = GlobalCoverageProtocol.GetRunToken(runId); + if (!CoverageUtils.TryReadAndCombine(codeCoveragePath, outputPath, runToken, out var globalCoverage) || + globalCoverage is null) + { + return; + } + + var canPublishCoverage = true; + var backfilled = false; + if (session is not null) + { + var backfillResult = TryApplyItrCoverageBackfill(session, globalCoverage); + canPublishCoverage = backfillResult.CanPublishCoverage; + backfilled = backfillResult.Backfilled; + } + + if (!canPublishCoverage) + { + Log.Warning("RunCiCommand: ITR coverage backfill could not match backend coverage to Datadog internal coverage. The coverage result will not be published."); + TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageErrors(); + return; + } + + var writer = new GlobalCoverageArtifactWriter(); + using var stagedOutput = writer.StageReplace(outputPath, globalCoverage); + stagedOutput.Commit(); + + // We only report the code coverage percentage if the customer manually sets the 'DD_CIVISIBILITY_CODE_COVERAGE_ENABLED' environment variable according to the new spec. + if (session is not null && + EnvironmentHelpers.GetEnvironmentVariable(Configuration.ConfigurationKeys.CIVisibility.CodeCoverage)?.ToBoolean() == true) + { + var data = globalCoverage.Data; + session.RecordCodeCoverage( + CodeCoverageReportSource.DatadogInternal, + globalCoverage.GetTotalPercentage(), + backfilled, + executableLines: data[1], + coveredLines: data[2]); + } + } + catch (Exception ex) + { + Log.Warning(ex, "RunCiCommand: Error while reading or backfilling Datadog internal code coverage."); + TelemetryFactory.Metrics.RecordCountCIVisibilityCodeCoverageErrors(); + } + } + internal static void FinalizeCoverageResultsBeforeSessionClose(TestSession session) => FinalizeCoverageResultsBeforeSessionClose(session, static session => TryProcessCoverletCollectorXmlReports(session, recordCoverageResult: true)); @@ -2323,7 +2338,7 @@ internal static void InjectCodeCoverageCollectorToDotnetTest(ref IEnumerable ?? [..msbuildArgs]; + var msbuildArgsList = msbuildArgs as List ?? [.. msbuildArgs]; for (var i = 0; i < msbuildArgsList.Count; i++) { var arg = msbuildArgsList[i]; diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersions.g.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersions.g.cs index 43271741e358..389ed07dcc6b 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersions.g.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersions.g.cs @@ -94,6 +94,8 @@ public class PackageVersions public static IEnumerable NUnit => IsAllMinorPackageVersions ? PackageVersionsLatestMinors.NUnit : PackageVersionsLatestSpecific.NUnit; + public static IEnumerable NUnitGlobalCoverageMemoryCoverlet => IsAllMinorPackageVersions ? PackageVersionsLatestMinors.NUnitGlobalCoverageMemoryCoverlet : PackageVersionsLatestSpecific.NUnitGlobalCoverageMemoryCoverlet; + public static IEnumerable NUnitRetries => IsAllMinorPackageVersions ? PackageVersionsLatestMinors.NUnitRetries : PackageVersionsLatestSpecific.NUnitRetries; public static IEnumerable MSTest => IsAllMinorPackageVersions ? PackageVersionsLatestMinors.MSTest : PackageVersionsLatestSpecific.MSTest; diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestMajors.g.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestMajors.g.cs index b30c0bb5260a..f79cbc8fe52a 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestMajors.g.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestMajors.g.cs @@ -1743,6 +1743,20 @@ public class PackageVersionsLatestMajors new object[] { "3.14.0" }, new object[] { "4.6.1" }, #endif +#endif + }; + + public static IEnumerable NUnitGlobalCoverageMemoryCoverlet => + + new List + { +#if DEFAULT_SAMPLES + new object[] { string.Empty }, +#else +#if NET8_0 + new object[] { "3.2.0" }, + new object[] { "6.0.0" }, +#endif #endif }; diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestMinors.g.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestMinors.g.cs index 80db08c2c5c5..ab450b9ca76d 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestMinors.g.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestMinors.g.cs @@ -4873,6 +4873,20 @@ public class PackageVersionsLatestMinors new object[] { "4.5.1" }, new object[] { "4.6.1" }, #endif +#endif + }; + + public static IEnumerable NUnitGlobalCoverageMemoryCoverlet => + + new List + { +#if DEFAULT_SAMPLES + new object[] { string.Empty }, +#else +#if NET8_0 + new object[] { "3.2.0" }, + new object[] { "6.0.0" }, +#endif #endif }; diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestSpecific.g.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestSpecific.g.cs index 93cb4d00ce0e..63057e382f68 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestSpecific.g.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/PackageVersionsLatestSpecific.g.cs @@ -1950,6 +1950,20 @@ public class PackageVersionsLatestSpecific new object[] { "3.14.0" }, new object[] { "4.6.1" }, #endif +#endif + }; + + public static IEnumerable NUnitGlobalCoverageMemoryCoverlet => + + new List + { +#if DEFAULT_SAMPLES + new object[] { string.Empty }, +#else +#if NET8_0 + new object[] { "3.2.0" }, + new object[] { "6.0.0" }, +#endif #endif }; diff --git a/tracer/test/Datadog.Trace.Tests/Ci/CoverageEventHandlerTests.cs b/tracer/test/Datadog.Trace.Tests/Ci/CoverageEventHandlerTests.cs new file mode 100644 index 000000000000..6760eb4cf9c2 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/CoverageEventHandlerTests.cs @@ -0,0 +1,318 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Ci.Coverage; +using Datadog.Trace.Ci.Coverage.Metadata; +using Datadog.Trace.Ci.Coverage.Models.Tests; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Ci; + +[Collection(nameof(CoverageGlobalStateTestCollection))] +public class CoverageEventHandlerTests +{ + [Fact] + public async Task OwnerBoundHandlesCloseExactContextsFromAnUnrelatedExecutionContext() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var metadata = CreateMetadata(totalLines: 128, coverageMode: 0, lastExecutableLine: 128); + + var firstHandle = await Task.Run(() => StartAndAllocate(handler, metadata)); + var secondHandle = await Task.Run(() => StartAndAllocate(handler, metadata)); + + Task closeTask; + using (ExecutionContext.SuppressFlow()) + { + closeTask = Task.Run( + () => + { + handler.EndSession(secondHandle); + handler.EndSession(firstHandle); + }); + } + + await closeTask; + + using var snapshot = handler.AcquireGlobalCoverageSnapshot().Snapshot!; + snapshot.MergedContextCount.Should().Be(2); + firstHandle.Context!.SnapshotModules().Should().BeEmpty(); + secondHandle.Context!.SnapshotModules().Should().BeEmpty(); + } + + [Fact] + public async Task ClosedFlowUsesLazyGlobalFallbackInsteadOfReopeningLocalContext() + { + var previousHandler = CoverageReporter.Handler; + var handler = new DefaultWithGlobalCoverageEventHandler(); + CoverageReporter.Handler = handler; + + try + { + var handle = handler.StartSession("xunit"); + WriteStaleFlowCounter(); + + var probe = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var child = Task.Run( + async () => + { + await probe.Task; + WriteStaleFlowCounter(); + }); + + handler.EndSession(handle); + probe.SetResult(true); + await child; + handle.Context!.SnapshotModules().Should().BeEmpty(); + handler.GlobalContainer.SnapshotModules().Should().ContainSingle(); + } + finally + { + CoverageReporter.Handler = previousHandler; + } + } + + [Fact] + public async Task EndSessionDefersBufferReleaseWhileAnInheritedExecutionContextIsActive() + { + var previousHandler = CoverageReporter.Handler; + var handler = new DefaultWithGlobalCoverageEventHandler(); + CoverageReporter.Handler = handler; + using var pointerAcquired = new ManualResetEventSlim(); + using var releaseProbe = new ManualResetEventSlim(); + var writeAfterClose = 0; + + try + { + var handle = handler.StartSession("xunit"); + var child = Task.Run( + () => + { + var pointer = GetStaleFlowCounter(); + WriteCounter(pointer); + pointerAcquired.Set(); + releaseProbe.Wait(); + if (Volatile.Read(ref writeAfterClose) != 0) + { + WriteCounter(pointer); + } + }); + + pointerAcquired.Wait(TimeSpan.FromSeconds(5)).Should().BeTrue(); + var module = handle.Context!.SnapshotModules().Should().ContainSingle().Subject; + try + { + handler.EndSession(handle); + + module.FilesLines.Should().NotBe( + IntPtr.Zero, + "the inherited execution context can still hold a raw counter pointer"); + Volatile.Write(ref writeAfterClose, 1); + } + finally + { + releaseProbe.Set(); + await child; + } + + SpinWait.SpinUntil(() => module.FilesLines == IntPtr.Zero, TimeSpan.FromSeconds(5)).Should().BeTrue(); + module.AllocatedByteLength.Should().Be(0); + using var snapshot = handler.AcquireGlobalCoverageSnapshot().Snapshot!; + snapshot.Model.Data.Should().Equal(100, 1, 1); + } + finally + { + CoverageReporter.Handler = previousHandler; + } + } + + [Fact] + public async Task ConcurrentFirstProbePublishesExactlyOneNativeBuffer() + { + var context = new CoverageContextContainer(); + var metadata = CreateMetadata(totalLines: 1024, coverageMode: 0, lastExecutableLine: 1); + var rawByteLength = CoverageMetadataValidator.ValidateAndGetRawByteLength(metadata); + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var tasks = Enumerable.Range(0, 32) + .Select( + _ => Task.Run( + async () => + { + await start.Task; + context.TryGetOrAddModuleValue( + metadata, + typeof(CoverageEventHandlerTests).Module, + rawByteLength, + out var module) + .Should() + .BeTrue(); + return module; + })) + .ToArray(); + + start.SetResult(true); + var modules = await Task.WhenAll(tasks); + + modules.Distinct().Should().ContainSingle(); + modules.Should().OnlyContain(value => value != null); + var module = modules[0]!; + context.SnapshotModules().Should().ContainSingle().Which.Should().BeSameAs(module); + module.FilesLines.Should().NotBe(IntPtr.Zero); + module.AllocatedByteLength.Should().Be(rawByteLength); + + context.Dispose(); + module.FilesLines.Should().Be(IntPtr.Zero); + module.AllocatedByteLength.Should().Be(0); + context.SnapshotModules().Should().BeEmpty(); + } + + [Fact] + public void DisposeReleasesEveryModuleAndIsIdempotent() + { + var context = new CoverageContextContainer(); + var metadata = CreateMetadata(totalLines: 8, coverageMode: 0, lastExecutableLine: 1); + + context.TryGetOrAddModuleValue(metadata, typeof(CoverageEventHandlerTests).Module, 8, out _).Should().BeTrue(); + context.TryGetOrAddModuleValue(metadata, typeof(string).Module, 8, out _).Should().BeTrue(); + var modules = context.SnapshotModules(); + + context.Dispose(); + + modules.Should().OnlyContain(module => module.FilesLines == IntPtr.Zero && module.AllocatedByteLength == 0); + context.SnapshotModules().Should().BeEmpty(); + context.Dispose(); + context.SnapshotModules().Should().BeEmpty(); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + public unsafe void PerTestCoveragePreservesTheExactUnionAcrossModules(int coverageMode) + { + var handler = new DefaultCoverageEventHandler(); + var metadata = new TestModuleCoverageMetadata( + 8, + coverageMode, + [new FileCoverageMetadata("/src/shared.cs", 0, 8, [0xff])]); + var rawByteLength = CoverageMetadataValidator.ValidateAndGetRawByteLength(metadata); + var handle = handler.StartSession("xunit"); + handler.Container!.TryGetOrAddModuleValue( + metadata, + typeof(CoverageEventHandlerTests).Module, + rawByteLength, + out var firstModule) + .Should() + .BeTrue(); + handler.Container.TryGetOrAddModuleValue( + metadata, + typeof(string).Module, + rawByteLength, + out var secondModule) + .Should() + .BeTrue(); + + if (coverageMode == 0) + { + ((byte*)firstModule!.FilesLines)[0] = 1; + ((byte*)secondModule!.FilesLines)[7] = 1; + } + else + { + // Call counters are deliberately unsynchronized; only zero versus non-zero is meaningful. + ((int*)firstModule!.FilesLines)[0] = -1; + ((int*)secondModule!.FilesLines)[7] = int.MinValue; + } + + var coverage = handler.EndSession(handle).Should().BeOfType().Subject; + + coverage.Files.Should().ContainSingle().Subject.Bitmap.Should().Equal(0x81); + firstModule.FilesLines.Should().Be(IntPtr.Zero); + secondModule.FilesLines.Should().Be(IntPtr.Zero); + } + + [Fact] + public unsafe void TenThousandClosedContextsKeepTheExactUnionInOneCompactBitmap() + { + const int contextCount = 10_000; + const int rawByteLength = 128 * 1024; + const int executableLineCount = contextCount * 2; + + var handler = new DefaultWithGlobalCoverageEventHandler(); + var expectedExecutableBitmap = Enumerable.Repeat((byte)0xff, executableLineCount / 8).ToArray(); + var expectedExecutedBitmap = Enumerable.Repeat((byte)0xaa, executableLineCount / 8).ToArray(); + var metadata = new TestModuleCoverageMetadata( + rawByteLength, + 0, + [new FileCoverageMetadata("/src/stress.cs", 0, executableLineCount, expectedExecutableBitmap)]); + + for (var i = 0; i < contextCount; i++) + { + var handle = handler.StartSession("xunit"); + handler.Container!.TryGetOrAddModuleValue( + metadata, + typeof(CoverageEventHandlerTests).Module, + rawByteLength, + out var module) + .Should() + .BeTrue(); + var counters = (byte*)module!.FilesLines; + counters[i * 2] = 1; + handler.EndSession(handle); + module.FilesLines.Should().Be(IntPtr.Zero, "closed contexts must release native buffers immediately"); + module.AllocatedByteLength.Should().Be(0); + } + + using var snapshot = handler.AcquireGlobalCoverageSnapshot().Snapshot!; + snapshot.MergedContextCount.Should().Be(contextCount); + var file = snapshot.Model.Components.Should().ContainSingle().Subject.Files.Should().ContainSingle().Subject; + file.ExecutableBitmap.Should().Equal(expectedExecutableBitmap); + file.ExecutedBitmap.Should().Equal(expectedExecutedBitmap); + file.Data.Should().Equal(50, executableLineCount, executableLineCount / 2); + } + + private static CoverageSessionHandle StartAndAllocate(DefaultWithGlobalCoverageEventHandler handler, ModuleCoverageMetadata metadata) + { + var handle = handler.StartSession("xunit"); + handler.Container!.TryGetOrAddModuleValue( + metadata, + typeof(CoverageEventHandlerTests).Module, + CoverageMetadataValidator.ValidateAndGetRawByteLength(metadata), + out _) + .Should() + .BeTrue(); + return handle; + } + + private static unsafe void WriteStaleFlowCounter() + { + var pointer = (byte*)CoverageReporter.GetFileCounter(0); + *pointer = 1; + } + + private static unsafe IntPtr GetStaleFlowCounter() + => (IntPtr)CoverageReporter.GetFileCounter(0); + + private static unsafe void WriteCounter(IntPtr pointer) + => *(byte*)pointer = 1; + + private static TestModuleCoverageMetadata CreateMetadata(int totalLines, int coverageMode, int lastExecutableLine) + => new( + totalLines, + coverageMode, + [new FileCoverageMetadata("/src/example.cs", 0, lastExecutableLine, new byte[(lastExecutableLine + 7) / 8])]); + + private sealed class StaleFlowMetadata : TestModuleCoverageMetadata + { + public StaleFlowMetadata() + : base(1, 0, [new FileCoverageMetadata("/src/stale.cs", 0, 1, [0x80])]) + { + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Ci/CoverageGlobalStateTestCollection.cs b/tracer/test/Datadog.Trace.Tests/Ci/CoverageGlobalStateTestCollection.cs new file mode 100644 index 000000000000..34d908b06a4b --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/CoverageGlobalStateTestCollection.cs @@ -0,0 +1,13 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Xunit; + +namespace Datadog.Trace.Tests.Ci; + +[CollectionDefinition(nameof(CoverageGlobalStateTestCollection), DisableParallelization = true)] +public sealed class CoverageGlobalStateTestCollection +{ +} diff --git a/tracer/test/Datadog.Trace.Tests/Ci/CoverageMetadataValidatorTests.cs b/tracer/test/Datadog.Trace.Tests/Ci/CoverageMetadataValidatorTests.cs new file mode 100644 index 000000000000..32402b1baa26 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/CoverageMetadataValidatorTests.cs @@ -0,0 +1,83 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using Datadog.Trace.Ci.Coverage; +using Datadog.Trace.Ci.Coverage.Metadata; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Ci; + +[Collection(nameof(CoverageGlobalStateTestCollection))] +public class CoverageMetadataValidatorTests +{ + [Fact] + public void RejectsUnsupportedCoverageMode() + => Validate(new TestModuleCoverageMetadata(1, 2, [new FileCoverageMetadata("file", 0, 1, [0x80])])) + .Should().Throw(); + + [Fact] + public void RejectsNegativeTotalLines() + => Validate(new TestModuleCoverageMetadata(-1, 0, [])).Should().Throw(); + + [Fact] + public void RejectsRawCounterSizeOverflow() + => Validate(new TestModuleCoverageMetadata(int.MaxValue, 1, [])).Should().Throw(); + + [Fact] + public void RejectsNegativeFileRange() + => Validate(new TestModuleCoverageMetadata(1, 0, [new FileCoverageMetadata("file", -1, 1, [0x80])])) + .Should().Throw(); + + [Fact] + public void RejectsOverflowingFileRange() + => Validate(new TestModuleCoverageMetadata(int.MaxValue, 0, [new FileCoverageMetadata("file", int.MaxValue, 1, [0x80])])) + .Should().Throw(); + + [Fact] + public void RejectsFileRangeOutsideRawBuffer() + => Validate(new TestModuleCoverageMetadata(1, 0, [new FileCoverageMetadata("file", 1, 1, [0x80])])) + .Should().Throw(); + + [Fact] + public void RejectsExecutableBitmapWithWrongSize() + => Validate(new TestModuleCoverageMetadata(9, 0, [new FileCoverageMetadata("file", 0, 9, [0xff])])) + .Should().Throw(); + + [Fact] + public unsafe void InvalidReporterMetadataSuppressesGlobalOutput() + { + var previousHandler = CoverageReporter.Handler; + var handler = new DefaultWithGlobalCoverageEventHandler(); + CoverageReporter.Handler = handler; + try + { + var action = () => Probe(); + + action.Should().Throw(); + handler.AcquireGlobalCoverageSnapshot().Status.Should().Be(GlobalCoverageSnapshotStatus.SuppressedIncomplete); + } + finally + { + CoverageReporter.Handler = previousHandler; + } + } + + private static Action Validate(ModuleCoverageMetadata metadata) + => () => CoverageMetadataValidator.ValidateAndGetRawByteLength(metadata); + + private static unsafe void Probe() + where TMetadata : ModuleCoverageMetadata, new() + => _ = CoverageReporter.GetFileCounter(0); + + private sealed class InvalidReporterMetadata : TestModuleCoverageMetadata + { + public InvalidReporterMetadata() + : base(int.MaxValue, 1, []) + { + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageAccumulatorTests.cs b/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageAccumulatorTests.cs new file mode 100644 index 000000000000..6b647c999003 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageAccumulatorTests.cs @@ -0,0 +1,289 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Ci.Coverage; +using Datadog.Trace.Ci.Coverage.Metadata; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Ci; + +[Collection(nameof(CoverageGlobalStateTestCollection))] +public class GlobalCoverageAccumulatorTests +{ + [Theory] + [InlineData(0)] + [InlineData(1)] + public unsafe void AggregatesLineExecutionAndLineCallCountModes(int coverageMode) + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var metadata = new TestModuleCoverageMetadata( + 16, + coverageMode, + [new FileCoverageMetadata("/src/mode.cs", 0, 16, [0xff, 0xff])]); + var handle = handler.StartSession("xunit"); + var rawByteLength = CoverageMetadataValidator.ValidateAndGetRawByteLength(metadata); + handler.Container!.TryGetOrAddModuleValue( + metadata, + typeof(GlobalCoverageAccumulatorTests).Module, + rawByteLength, + out var module) + .Should() + .BeTrue(); + + if (coverageMode == 0) + { + var counters = (byte*)module!.FilesLines; + counters[0] = 1; + counters[8] = 1; + } + else + { + var counters = (int*)module!.FilesLines; + counters[0] = 3; + counters[8] = 7; + } + + handler.EndSession(handle); + + var result = handler.AcquireGlobalCoverageSnapshot(); + result.Status.Should().Be(GlobalCoverageSnapshotStatus.Success); + using var snapshot = result.Snapshot!; + var file = snapshot.Model.Components.Should().ContainSingle().Subject.Files.Should().ContainSingle().Subject; + file.ExecutedBitmap.Should().Equal(0x80, 0x80); + file.Data.Should().Equal(12.5, 16, 2); + } + + [Fact] + public unsafe void ExceedingBitmapBudgetSuppressesGlobalCoverageButFreesNativeContext() + { + var handler = new DefaultWithGlobalCoverageEventHandler( + new GlobalCoverageAccumulatorLimits(maximumSingleBitmapBytes: 0, maximumRetainedBitmapBytes: 0, maximumModules: 1, maximumFileSlots: 1)); + var metadata = new TestModuleCoverageMetadata(8, 0, [new FileCoverageMetadata("/src/limit.cs", 0, 8, [0xff])]); + var handle = handler.StartSession("xunit"); + handler.Container!.TryGetOrAddModuleValue( + metadata, + typeof(GlobalCoverageAccumulatorTests).Module, + 8, + out var module) + .Should() + .BeTrue(); + *(byte*)module!.FilesLines = 1; + + handler.EndSession(handle).Should().NotBeNull(); + + module.FilesLines.Should().Be(IntPtr.Zero); + module.AllocatedByteLength.Should().Be(0); + var snapshotResult = handler.AcquireGlobalCoverageSnapshot(); + snapshotResult.Status.Should().Be(GlobalCoverageSnapshotStatus.SuppressedIncomplete); + snapshotResult.FailureReason.Should().Be(GlobalCoverageFailureReason.MergeFailed); + } + + [Fact] + public void ConcurrentSuppressionInvalidatesAnAlreadyAcquiredSnapshotBeforeCommit() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var result = handler.AcquireGlobalCoverageSnapshot(); + using var snapshot = result.Snapshot!; + var committed = false; + + handler.MarkProbeDataIncomplete(GlobalCoverageFailureReason.ProbeDataIncomplete); + + handler.TryCommit(snapshot, () => committed = true).Should().BeFalse(); + committed.Should().BeFalse(); + snapshot.Dispose(); + handler.AcquireGlobalCoverageSnapshot().Status.Should().Be(GlobalCoverageSnapshotStatus.SuppressedIncomplete); + } + + [Fact] + public void CommitFailureIsStickyAndPreservesTheOriginalException() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var result = handler.AcquireGlobalCoverageSnapshot(); + using var snapshot = result.Snapshot!; + var expected = new InvalidOperationException("Injected output failure."); + + var action = () => handler.TryCommit(snapshot, () => throw expected); + + action.Should().Throw().Which.Should().BeSameAs(expected); + snapshot.Dispose(); + var suppressed = handler.AcquireGlobalCoverageSnapshot(); + suppressed.Status.Should().Be(GlobalCoverageSnapshotStatus.SuppressedIncomplete); + suppressed.FailureReason.Should().Be(GlobalCoverageFailureReason.OutputCommitFailed); + } + + [Fact] + public async Task FinalPublicationAndCompletenessAreCommittedAtomically() + { + var accumulator = new GlobalCoverageAccumulator(); + using var snapshot = accumulator.AcquireSnapshot(globalContainer: null).Snapshot!; + using var suppressionStarted = new ManualResetEventSlim(); + Task suppressionTask = null; + + var finalized = accumulator.TryFinalizeSnapshot( + snapshot, + () => + { + suppressionTask = Task.Run( + () => + { + suppressionStarted.Set(); + accumulator.Suppress(GlobalCoverageFailureReason.ProbeDataIncomplete); + }); + suppressionStarted.Wait(TimeSpan.FromSeconds(5)).Should().BeTrue(); + return true; + }); + + finalized.Should().BeTrue(); + suppressionTask.Should().NotBeNull(); + await suppressionTask; + accumulator.IsSuppressed.Should().BeFalse("a failure observed after the atomic final commit cannot invalidate published coverage"); + } + + [Fact] + public void FailedFinalPublicationSuppressesCoverage() + { + var accumulator = new GlobalCoverageAccumulator(); + using var snapshot = accumulator.AcquireSnapshot(globalContainer: null).Snapshot!; + + accumulator.TryFinalizeSnapshot(snapshot, static () => false).Should().BeFalse(); + + accumulator.IsSuppressed.Should().BeTrue(); + accumulator.FailureReason.Should().Be(GlobalCoverageFailureReason.OutputCommitFailed); + } + + [Fact] + public void DisposedSnapshotCannotCommitSideEffects() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var result = handler.AcquireGlobalCoverageSnapshot(); + var snapshot = result.Snapshot!; + snapshot.Dispose(); + var committed = false; + + handler.TryCommit(snapshot, () => committed = true).Should().BeFalse(); + committed.Should().BeFalse(); + } + + [Fact] + public void SealWaitsForSnapshotLeaseAndCompletesAfterItsDisposal() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var result = handler.AcquireGlobalCoverageSnapshot(); + var snapshot = result.Snapshot!; + var completed = false; + + handler.FinalizeAndSeal(value => completed = value).Should().BeFalse(); + completed.Should().BeFalse(); + + snapshot.Dispose(); + + completed.Should().BeTrue(); + handler.AcquireGlobalCoverageSnapshot().Status.Should().Be(GlobalCoverageSnapshotStatus.SuppressedIncomplete); + } + + [Fact] + public void StartDuringCompletingIsRejectedAndMakesSealIncomplete() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var handle = handler.StartSession("xunit"); + + var completed = false; + handler.FinalizeAndSeal(value => completed = value).Should().BeFalse(); + handler.StartSession("xunit").IsValid.Should().BeFalse(); + handler.EndSession(handle); + + completed.Should().BeFalse(); + handler.AcquireGlobalCoverageSnapshot().Status.Should().Be(GlobalCoverageSnapshotStatus.SuppressedIncomplete); + } + + [Fact] + public void StartAfterSealThrowsBeforeCreatingAContext() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + handler.FinalizeAndSeal().Should().BeTrue(); + + var action = () => handler.StartSession("xunit"); + + action.Should().Throw(); + handler.Container.Should().BeNull(); + } + + [Fact] + public unsafe void UnionsOverlappingContextsAcrossModulesAndFilesWithoutInflatingDenominator() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var firstMetadata = new TestModuleCoverageMetadata( + 16, + 0, + [ + new FileCoverageMetadata("/src/a.cs", 0, 8, [0xff]), + new FileCoverageMetadata("/src/b.cs", 8, 8, [0xff]) + ]); + var secondMetadata = new TestModuleCoverageMetadata(8, 0, [new FileCoverageMetadata("/src/c.cs", 0, 8, [0xff])]); + + MergeContext(handler, firstMetadata, typeof(GlobalCoverageAccumulatorTests).Module, [0, 8]); + MergeContext(handler, firstMetadata, typeof(GlobalCoverageAccumulatorTests).Module, [0, 1, 8, 15]); + MergeContext(handler, secondMetadata, typeof(string).Module, [3]); + + using var snapshot = handler.AcquireGlobalCoverageSnapshot().Snapshot!; + snapshot.Model.Components.Should().HaveCount(2); + snapshot.Model.Components.SelectMany(component => component.Files).Should().HaveCount(3); + snapshot.Model.Data[0].Should().Be(20.83); + snapshot.Model.Data[1].Should().Be(24); + snapshot.Model.Data[2].Should().Be(5); + snapshot.MergedContextCount.Should().Be(3); + } + + [Fact] + public unsafe void ContextClosedAfterSnapshotAppearsInTheNextSnapshot() + { + var handler = new DefaultWithGlobalCoverageEventHandler(); + var metadata = new TestModuleCoverageMetadata(8, 0, [new FileCoverageMetadata("/src/late.cs", 0, 8, [0xff])]); + var handle = handler.StartSession("xunit"); + handler.Container!.TryGetOrAddModuleValue( + metadata, + typeof(GlobalCoverageAccumulatorTests).Module, + 8, + out var module) + .Should() + .BeTrue(); + *(byte*)module!.FilesLines = 1; + + using (var first = handler.AcquireGlobalCoverageSnapshot().Snapshot!) + { + first.Model.Components.Should().BeEmpty(); + } + + handler.EndSession(handle); + using var second = handler.AcquireGlobalCoverageSnapshot().Snapshot!; + second.Model.Data.Should().Equal(12.5, 8, 1); + second.MergedContextCount.Should().Be(1); + } + + private static unsafe void MergeContext(DefaultWithGlobalCoverageEventHandler handler, ModuleCoverageMetadata metadata, Module module, int[] executedOffsets) + { + var handle = handler.StartSession("xunit"); + handler.Container!.TryGetOrAddModuleValue( + metadata, + module, + CoverageMetadataValidator.ValidateAndGetRawByteLength(metadata), + out var moduleValue) + .Should() + .BeTrue(); + var counters = (byte*)moduleValue!.FilesLines; + foreach (var offset in executedOffsets) + { + counters[offset] = 1; + } + + handler.EndSession(handle); + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageConsumerTests.cs b/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageConsumerTests.cs new file mode 100644 index 000000000000..2bab63610088 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageConsumerTests.cs @@ -0,0 +1,305 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.IO; +using System.Text; +using Datadog.Trace.Ci.Coverage; +using Datadog.Trace.Ci.Coverage.Models.Global; +using Datadog.Trace.Vendors.Newtonsoft.Json; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Ci; + +[Collection(nameof(CoverageGlobalStateTestCollection))] +public class GlobalCoverageConsumerTests +{ + [Fact] + public void InputReaderAcceptsBoundedCoverageWithUtf8Bom() + { + var path = Path.GetTempFileName(); + try + { + var model = CreateModel("component", "/src/example.cs", [0xff], [0x80]); + using (var stream = File.Create(path)) + using (var writer = new StreamWriter(stream, Encoding.UTF8)) + { + JsonSerializer.Create().Serialize(writer, model); + } + + var reader = new GlobalCoverageInputReader(); + reader.TryRead(path, out var result).Should().BeTrue(); + var file = result!.Components.Should().ContainSingle().Subject.Files.Should().ContainSingle().Subject; + file.Path.Should().Be("/src/example.cs"); + file.ExecutedBitmap.Should().Equal(0x80); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void PreflightRejectsBitmapBeforeGeneralDeserializationCanMaterializeIt() + { + var limits = CreateSmallLimits(maximumBitmapBytes: 2, maximumIdentityCharacters: 128); + var path = Path.GetTempFileName(); + try + { + File.WriteAllText( + path, + "{\"components\":[{\"name\":\"c\",\"files\":[{\"path\":\"p\",\"executableBitmap\":\"AQID\",\"executedBitmap\":\"AQ==\"}]}]}", + new UTF8Encoding(false)); + + var reader = new GlobalCoverageInputReader(limits); + reader.TryRead(path, out _).Should().BeFalse(); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void PreflightRejectsIdentityCharacterBudget() + { + var limits = CreateSmallLimits(maximumBitmapBytes: 8, maximumIdentityCharacters: 3); + var path = Path.GetTempFileName(); + try + { + File.WriteAllText( + path, + "{\"components\":[{\"name\":\"name\",\"files\":[]}]}", + new UTF8Encoding(false)); + + var reader = new GlobalCoverageInputReader(limits); + reader.TryRead(path, out _).Should().BeFalse(); + } + finally + { + File.Delete(path); + } + } + + [Theory] + [InlineData("{\"components\":[null]}")] + [InlineData("{\"components\":[{\"name\":\"component\",\"files\":[null]}]}")] + public void InputReaderRejectsNullCoverageEntries(string json) + { + var path = Path.GetTempFileName(); + try + { + File.WriteAllText(path, json, new UTF8Encoding(false)); + + var reader = new GlobalCoverageInputReader(); + reader.TryRead(path, out _).Should().BeFalse(); + } + finally + { + File.Delete(path); + } + } + + [Theory] + [InlineData("{\"components\":[{\"name\":\"c\",\"files\":[{\"path\":\"p\",\"executableBitmap\":\"gA==\",\"executedBitmap\":\"/w==\"}]}]}")] + [InlineData("{\"components\":[{\"name\":\"c\",\"files\":[{\"path\":\"p\",\"executedBitmap\":\"gA==\"}]}]}")] + [InlineData("{\"components\":[{\"name\":\"c\",\"files\":[{\"path\":\"p\",\"executableBitmap\":\"gA==\",\"executedBitmap\":\"gAA=\"}]}]}")] + public void InputReaderRejectsExecutedBitmapThatIsNotACompatibleExecutableSubset(string json) + { + var path = Path.GetTempFileName(); + try + { + File.WriteAllText(path, json, new UTF8Encoding(false)); + + var reader = new GlobalCoverageInputReader(); + reader.TryRead(path, out _).Should().BeFalse(); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void InputReaderAcceptsShorterExecutedBitmapAsImplicitTrailingZeros() + { + var path = Path.GetTempFileName(); + try + { + File.WriteAllText( + path, + "{\"components\":[{\"name\":\"c\",\"files\":[{\"path\":\"p\",\"executableBitmap\":\"//8=\",\"executedBitmap\":\"gA==\"}]}]}", + new UTF8Encoding(false)); + + var reader = new GlobalCoverageInputReader(); + reader.TryRead(path, out var result).Should().BeTrue(); + result!.Components.Should().ContainSingle().Subject.Files.Should().ContainSingle().Subject.Data.Should().Equal(6.25, 16, 1); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void CombinerUnionsInputsIncrementallyWithoutRetainingInputModels() + { + var accumulator = new GlobalCoverageCombinerAccumulator(CreateSmallLimits(maximumBitmapBytes: 8, maximumIdentityCharacters: 128)); + accumulator.Add(CreateModel(null, null, [0xf0], [0x80])); + accumulator.Add(CreateModel(null, null, [0x0f, 0xff], [0x08, 0x80])); + + var model = accumulator.Materialize(); + var file = model.Components.Should().ContainSingle().Subject.Files.Should().ContainSingle().Subject; + file.ExecutableBitmap.Should().Equal(0xff, 0xff); + file.ExecutedBitmap.Should().Equal(0x88, 0x80); + file.Data.Should().Equal(18.75, 16, 3); + } + + [Fact] + public void RunScopedCombinerRejectsMalformedProcessArtifact() + { + var path = Path.GetTempFileName(); + try + { + File.WriteAllText(path, "{not-json", new UTF8Encoding(false)); + + GlobalCoverageFileCombiner.TryCombine( + [path], + outputFile: null, + requireAllInputs: true, + onFileProcessed: null, + out var coverage, + out var rejectedInput).Should().BeFalse(); + + coverage.Should().BeNull(); + rejectedInput.Should().Be(path); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void ArtifactWriterPublishesUtf8WithoutBomAndReaderAcceptsIt() + { + var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"))).FullName; + var path = Path.Combine(directory, "coverage.json"); + try + { + var writer = new GlobalCoverageArtifactWriter(); + writer.WriteAtomicNoReplace(path, CreateModel("component", "/src/example.cs", [0xff], [0x80])); + + var bytes = File.ReadAllBytes(path); + (bytes.Length >= 3 && bytes[0] == 0xef && bytes[1] == 0xbb && bytes[2] == 0xbf).Should().BeFalse(); + var reader = new GlobalCoverageInputReader(); + reader.TryRead(path, out _).Should().BeTrue(); + Directory.GetFiles(directory, "*.tmp").Should().BeEmpty(); + } + finally + { + Directory.Delete(directory, true); + } + } + + [Fact] + public void ArtifactWriterLimitFailurePreservesDestinationAndCleansTemporary() + { + var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"))).FullName; + var path = Path.Combine(directory, "coverage.json"); + var original = new byte[] { 1, 2, 3, 4 }; + File.WriteAllBytes(path, original); + try + { + var limits = CreateSmallLimits(maximumBitmapBytes: 8, maximumIdentityCharacters: 128, maximumSerializedBytes: 1); + var writer = new GlobalCoverageArtifactWriter(limits); + var action = () => writer.WriteAtomicReplace(path, CreateModel("component", "/src/example.cs", [0xff], [0x80])); + + action.Should().Throw(); + File.ReadAllBytes(path).Should().Equal(original); + Directory.GetFiles(directory).Should().ContainSingle().Which.Should().Be(path); + } + finally + { + Directory.Delete(directory, true); + } + } + + [Fact] + public void ArtifactWriterNoReplaceNeverOverwritesExistingArtifact() + { + var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"))).FullName; + var path = Path.Combine(directory, "coverage.json"); + var original = new byte[] { 5, 6, 7, 8 }; + File.WriteAllBytes(path, original); + try + { + var writer = new GlobalCoverageArtifactWriter(); + var action = () => writer.WriteAtomicNoReplace(path, CreateModel("component", "/src/example.cs", [0xff], [0x80])); + + action.Should().Throw(); + File.ReadAllBytes(path).Should().Equal(original); + Directory.GetFiles(directory).Should().ContainSingle().Which.Should().Be(path); + } + finally + { + Directory.Delete(directory, true); + } + } + + [Fact] + public void StagedArtifactDoesNotPublishUntilCommitAndDisposeCleansUncommittedTemporary() + { + var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"))).FullName; + var path = Path.Combine(directory, "coverage.json"); + try + { + var writer = new GlobalCoverageArtifactWriter(); + using (var staged = writer.StageNoReplace(path, CreateModel("component", "/src/example.cs", [0xff], [0x80]))) + { + File.Exists(path).Should().BeFalse(); + Directory.GetFiles(directory).Should().ContainSingle().Which.Should().EndWith(".tmp"); + } + + Directory.GetFiles(directory).Should().BeEmpty(); + using (var staged = writer.StageNoReplace(path, CreateModel("component", "/src/example.cs", [0xff], [0x80]))) + { + staged.Commit(); + } + + File.Exists(path).Should().BeTrue(); + Directory.GetFiles(directory).Should().ContainSingle().Which.Should().Be(path); + } + finally + { + Directory.Delete(directory, true); + } + } + + private static GlobalCoverageInfo CreateModel(string? componentName, string? path, byte[] executable, byte[] executed) + { + var model = new GlobalCoverageInfo(); + var component = new ComponentCoverageInfo(componentName); + component.Files.Add(new FileCoverageInfo(path) { ExecutableBitmap = executable, ExecutedBitmap = executed }); + model.Components.Add(component); + return model; + } + + private static GlobalCoverageArtifactLimits CreateSmallLimits(int maximumBitmapBytes, int maximumIdentityCharacters, long maximumSerializedBytes = 4 * 1024) + => new( + maximumSerializedBytes: maximumSerializedBytes, + maximumBitmapBytes: maximumBitmapBytes, + maximumModelBitmapBytes: 64, + maximumComponents: 4, + maximumEntries: 8, + maximumIdentityCharacters: maximumIdentityCharacters, + maximumPropertyCharacters: 64, + maximumScalarCharacters: 256, + maximumDepth: 16, + scannerBufferCharacters: 128); +} diff --git a/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageOutputTests.cs b/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageOutputTests.cs new file mode 100644 index 000000000000..065fe9e04547 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/GlobalCoverageOutputTests.cs @@ -0,0 +1,167 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Datadog.Trace.Ci.Coverage; +using Datadog.Trace.Ci.Coverage.Metadata; +using Datadog.Trace.Telemetry; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Ci; + +[Collection(nameof(CoverageGlobalStateTestCollection))] +public class GlobalCoverageOutputTests +{ + [Fact] + public unsafe void SealPublishesOneProcessArtifactWithTheExactAccumulatedUnion() + { + var directory = CreateTemporaryDirectory(); + try + { + const string runId = "one-process-artifact"; + var handler = new DefaultWithGlobalCoverageEventHandler(configuredOutputDirectory: directory, runIdProvider: () => runId); + var metadata = new TestModuleCoverageMetadata(8, 0, [new FileCoverageMetadata("/src/file.cs", 0, 8, [0xff])]); + + MergeLine(handler, metadata, 0); + using (var intermediate = handler.AcquireGlobalCoverageSnapshot().Snapshot!) + { + intermediate.Model.Data.Should().Equal(12.5, 8, 1); + } + + MergeLine(handler, metadata, 7); + handler.FinalizeAndSeal().Should().BeTrue(); + handler.FinalizeAndSeal().Should().BeTrue("sealing is idempotent"); + + Directory.GetFiles(directory, GlobalCoverageProtocol.PendingMarkerPattern).Should().BeEmpty(); + var artifact = Directory.GetFiles(directory, GlobalCoverageProtocol.CoverageFilePattern).Should().ContainSingle().Subject; + var reader = new GlobalCoverageInputReader(); + reader.TryRead(artifact, out var coverage).Should().BeTrue(); + coverage!.Data.Should().Equal(25, 8, 2); + coverage.Components.Should().ContainSingle().Subject.Files.Should().ContainSingle().Subject.ExecutedBitmap.Should().Equal(0x81); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void PendingProducerPreventsRunScopedConsumption() + { + var directory = CreateTemporaryDirectory(); + try + { + const string runId = "pending-producer"; + var output = new GlobalCoverageOutputManager(directory, directory, () => runId); + output.EnsureConfiguredAndFreeze().Should().BeTrue(); + + GlobalCoverageFileCombiner.TryAcquireInputFiles(directory, GlobalCoverageProtocol.GetRunToken(runId), out var files).Should().BeFalse(); + files.Should().BeEmpty(); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void UnrelatedPendingProducerDoesNotPreventLegacyConsumption() + { + var directory = CreateTemporaryDirectory(); + try + { + var input = Path.Combine(directory, "legacy.json"); + File.WriteAllText(input, "{}"); + File.WriteAllText( + Path.Combine(directory, GlobalCoverageProtocol.GetPendingMarkerFileName("unrelated-process")), + string.Empty); + + GlobalCoverageFileCombiner.TryAcquireInputFiles(directory, expectedRunToken: null, out var files).Should().BeTrue(); + files.Should().Equal(input); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void CollectorCanProvideOutputDirectoryAfterInMemoryCoverageStarts() + { + var directory = CreateTemporaryDirectory(); + try + { + var output = new GlobalCoverageOutputManager( + configuredDirectory: null, + baseDirectory: directory, + runIdProvider: () => "late-collector"); + + output.EnsureConfiguredAndFreeze().Should().BeTrue(); + output.RegisterCollectorAndFreeze(directory).Should().BeTrue(); + + Directory.GetFiles(directory, GlobalCoverageProtocol.PendingMarkerPattern).Should().ContainSingle(); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public async Task SealFailurePreservesTheOriginalExceptionAndRecordsCoverageTelemetry() + { + var directory = CreateTemporaryDirectory(); + var invalidDirectory = Path.Combine(directory, "not-a-directory"); + File.WriteAllText(invalidDirectory, string.Empty); + + var collector = new CiVisibilityMetricsTelemetryCollector(Timeout.InfiniteTimeSpan); + var previousMetrics = TelemetryFactory.SetMetricsForTesting(collector); + try + { + var output = new GlobalCoverageOutputManager(invalidDirectory, directory, () => "failed-output"); + output.EnsureConfiguredAndFreeze().Should().BeFalse(); + output.FailureException.Should().BeAssignableTo(); + + var handler = new DefaultWithGlobalCoverageEventHandler(configuredOutputDirectory: invalidDirectory); + handler.RegisterCollectorOutputDirectory(invalidDirectory).Should().BeFalse(); + handler.FinalizeAndSeal().Should().BeFalse(); + handler.FinalizeAndSeal().Should().BeFalse("failed sealing is idempotent"); + collector.AggregateMetrics(); + + var metric = collector.GetMetrics().Metrics.Should() + .ContainSingle(metric => metric.Metric == "code_coverage.errors" && metric.Namespace == "civisibility") + .Subject; + metric.Points.Should().ContainSingle().Subject.Value.Should().Be(1); + } + finally + { + TelemetryFactory.SetMetricsForTesting(previousMetrics); + await collector.DisposeAsync(); + Directory.Delete(directory, recursive: true); + } + } + + private static unsafe void MergeLine(DefaultWithGlobalCoverageEventHandler handler, ModuleCoverageMetadata metadata, int line) + { + var handle = handler.StartSession("xunit"); + handler.Container!.TryGetOrAddModuleValue(metadata, typeof(GlobalCoverageOutputTests).Module, 8, out var module).Should().BeTrue(); + ((byte*)module!.FilesLines)[line] = 1; + handler.EndSession(handle); + module.FilesLines.Should().Be(IntPtr.Zero); + } + + private static string CreateTemporaryDirectory() + { + var directory = Path.Combine(Path.GetTempPath(), $"dd-coverage-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + return directory; + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Ci/TestCoverageLifecycleTests.cs b/tracer/test/Datadog.Trace.Tests/Ci/TestCoverageLifecycleTests.cs new file mode 100644 index 000000000000..c7ad7356e7a7 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/TestCoverageLifecycleTests.cs @@ -0,0 +1,146 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using Datadog.Trace.Ci; +using Datadog.Trace.Ci.CiEnvironment; +using Datadog.Trace.Ci.Configuration; +using Datadog.Trace.Ci.Coverage; +using Datadog.Trace.Configuration; +using Datadog.Trace.Configuration.Telemetry; +using Datadog.Trace.Logging; +using Datadog.Trace.TestHelpers; +using FluentAssertions; +using Moq; +using Xunit; + +namespace Datadog.Trace.Tests.Ci; + +[Collection(nameof(CoverageGlobalStateTestCollection))] +public class TestCoverageLifecycleTests : SettingsTestsBase +{ + [Fact] + public void ConstructorFailureAfterCoverageStartAbortsCoverageContext() + { + using var harness = new TestHarness(); + var logger = new Mock(); + logger.Setup( + x => x.Debug( + "######### New Test Created: {Name} ({Suite} | {Module})", + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new InvalidOperationException("Injected construction failure.")); + harness.TestOptimizationMock.Setup(x => x.Log).Returns(logger.Object); + + var action = () => harness.Suite.CreateTest("constructor-failure"); + + action.Should().Throw().WithMessage("Injected construction failure."); + AssertBalancedSuppressedCoverage(harness.GlobalHandler, GlobalCoverageFailureReason.TestConstructionFailed); + } + + [Fact] + public void CoverageEndFailureStillClosesAndDisposesTheContext() + { + var handler = new ThrowingCoverageEventHandler(); + using var harness = new TestHarness(handler); + var test = harness.Suite.CreateTest("close-failure"); + + var action = () => test.Close(TestStatus.Pass); + + action.Should().Throw().WithMessage("Injected coverage-end failure."); + test.IsClosed.Should().BeTrue(); + handler.Container.Should().BeNull(); + } + + private static void AssertBalancedSuppressedCoverage(DefaultWithGlobalCoverageEventHandler handler, GlobalCoverageFailureReason reason) + { + handler.Container.Should().BeNull(); + var snapshotResult = handler.AcquireGlobalCoverageSnapshot(); + snapshotResult.Status.Should().Be(GlobalCoverageSnapshotStatus.SuppressedIncomplete); + snapshotResult.FailureReason.Should().Be(reason); + } + + private sealed class TestHarness : IDisposable + { + private readonly ITestOptimization _previousTestOptimization; + private readonly CoverageEventHandler _previousCoverageHandler; + + public TestHarness(CoverageEventHandler? handler = null) + { + _previousTestOptimization = TestOptimization.Instance; + _previousCoverageHandler = CoverageReporter.Handler; + + var settings = new TestOptimizationSettings( + CreateConfigurationSource((ConfigurationKeys.CIVisibility.CodeCoverage, "1")), + NullConfigurationTelemetry.Instance); + TestOptimizationMock = new Mock(); + var hostInfo = new Mock(); + hostInfo.Setup(x => x.GetOperatingSystemVersion()).Returns("test-os-version"); + TestOptimizationMock.Setup(x => x.Settings).Returns(settings); + TestOptimizationMock.Setup(x => x.Log).Returns(DatadogLogging.GetLoggerFor(typeof(TestCoverageLifecycleTests))); + TestOptimizationMock.Setup(x => x.CIValues).Returns(new TestCIEnvironmentValues(Directory.GetCurrentDirectory())); + TestOptimizationMock.Setup(x => x.HostInfo).Returns(hostInfo.Object); + + Handler = handler ?? new DefaultWithGlobalCoverageEventHandler(); + TestOptimization.Instance = TestOptimizationMock.Object; + CoverageReporter.Handler = Handler; + Session = TestSession.GetOrCreate("dotnet test", workingDirectory: null, framework: "xunit", startDate: null); + Module = Session.CreateModule("coverage-lifecycle"); + Suite = Module.GetOrCreateSuite("coverage-lifecycle-suite"); + } + + public Mock TestOptimizationMock { get; } + + public CoverageEventHandler Handler { get; } + + public DefaultWithGlobalCoverageEventHandler GlobalHandler => (DefaultWithGlobalCoverageEventHandler)Handler; + + public TestSession Session { get; } + + public TestModule Module { get; } + + public TestSuite Suite { get; } + + public void Dispose() + { + Suite.Close(); + Module.Close(); + Session.Close(TestStatus.Pass); + CoverageReporter.Handler = _previousCoverageHandler; + TestOptimization.Instance = _previousTestOptimization; + } + } + + private sealed class ThrowingCoverageEventHandler : CoverageEventHandler + { + protected override void OnSessionStart(CoverageContextContainer context) + { + } + + protected override object? OnSessionFinished(CoverageContextContainer context, IReadOnlyList modules) + => throw new InvalidOperationException("Injected coverage-end failure."); + } + + private sealed class TestCIEnvironmentValues : CIEnvironmentValues + { + public TestCIEnvironmentValues(string workspacePath) + { + WorkspacePath = workspacePath; + Repository = "https://github.com/DataDog/dd-trace-dotnet"; + Commit = "abcdef123456"; + } + + protected override void Setup(IGitInfo gitInfo) + { + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Ci/TestModuleCoverageMetadata.cs b/tracer/test/Datadog.Trace.Tests/Ci/TestModuleCoverageMetadata.cs new file mode 100644 index 000000000000..c7ed7cb5c44d --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/TestModuleCoverageMetadata.cs @@ -0,0 +1,32 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Reflection; +using Datadog.Trace.Ci.Coverage.Metadata; + +namespace Datadog.Trace.Tests.Ci; + +internal class TestModuleCoverageMetadata : ModuleCoverageMetadata +{ + private static readonly FieldInfo TotalLinesField = typeof(ModuleCoverageMetadata).GetField(nameof(TotalLines))!; + private static readonly FieldInfo CoverageModeField = typeof(ModuleCoverageMetadata).GetField(nameof(CoverageMode))!; + private static readonly FieldInfo FilesField = typeof(ModuleCoverageMetadata).GetField("Files", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)!; + + public TestModuleCoverageMetadata() + { + } + + public TestModuleCoverageMetadata(int totalLines, int coverageMode, FileCoverageMetadata[] files) + => Initialize(totalLines, coverageMode, files); + + protected void Initialize(int totalLines, int coverageMode, FileCoverageMetadata[] files) + { + // Production metadata is populated by the coverage rewriter. Tests use reflection to model + // that generated state without adding a test-only construction path to the shipped assembly. + TotalLinesField.SetValue(this, totalLines); + CoverageModeField.SetValue(this, coverageMode); + FilesField.SetValue(this, files); + } +} diff --git a/tracer/test/Datadog.Trace.Tools.Runner.IntegrationTests/CiRunCommandTests.cs b/tracer/test/Datadog.Trace.Tools.Runner.IntegrationTests/CiRunCommandTests.cs index 60a1f56ea684..d711af4404f1 100644 --- a/tracer/test/Datadog.Trace.Tools.Runner.IntegrationTests/CiRunCommandTests.cs +++ b/tracer/test/Datadog.Trace.Tools.Runner.IntegrationTests/CiRunCommandTests.cs @@ -2472,8 +2472,11 @@ public void DatadogInternalCoverageFailsClosedWhenBackendPathDoesNotMatch() CoverageBackfillCapability.ResetCommandLineCacheForTests(); TestOptimization.Instance.InitializeFromRunner(TestOptimization.Instance.Settings, NullDiscoveryService.Instance, eventPlatformProxyEnabled: true); + // Run-scoped consumers deliberately ignore unrelated JSON files, so publish the fixture using the production artifact naming protocol. + var runToken = GlobalCoverageProtocol.GetRunToken(TestOptimization.Instance.RunId); + var processIdentity = GlobalCoverageProtocol.GetProcessIdentity(runToken, processId: 1, nonce: "fixture"); File.WriteAllText( - Path.Combine(coverageDirectory.RootPath, "coverage-input.json"), + Path.Combine(coverageDirectory.RootPath, GlobalCoverageProtocol.GetCoverageFileName(processIdentity)), JsonHelper.SerializeObject(CreateGlobalCoverage("src/Calculator.cs"))); CoverageBackfillDataStore.Persist(TestOptimization.Instance, CreateCoverageBackfillData("src/Other.cs", SimplePassTestCoveredLine)); CoverageBackfillDataStore.RecordActualItrSkip(); diff --git a/tracer/test/Datadog.Trace.Tools.Runner.IntegrationTests/CiRunGlobalCoverageMemoryTests.cs b/tracer/test/Datadog.Trace.Tools.Runner.IntegrationTests/CiRunGlobalCoverageMemoryTests.cs new file mode 100644 index 000000000000..280d6d3310f3 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tools.Runner.IntegrationTests/CiRunGlobalCoverageMemoryTests.cs @@ -0,0 +1,364 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if NET8_0 + +#nullable enable + +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Datadog.Trace.Ci.Coverage; +using Datadog.Trace.Configuration; +using Datadog.Trace.TestHelpers; +using Datadog.Trace.Vendors.Newtonsoft.Json; +using FluentAssertions; +using Xunit; +using Xunit.Abstractions; + +namespace Datadog.Trace.Tools.Runner.IntegrationTests; + +[Collection(nameof(ConsoleTestsCollection))] +public sealed class CiRunGlobalCoverageMemoryTests +{ + private const long MaximumStressPrivateBytesGrowth = 384L * 1024 * 1024; + private const string SampleName = "NUnitGlobalCoverageMemory"; + private const string SampleSourceFileName = "GlobalCoverageMemoryTests.cs"; + private const int CommonCoverageLine = 131_072; + private const int FirstCoverageSentinelLine = 131_073; + private const int MiddleCoverageSentinelLine = 131_074; + private const int LastCoverageSentinelLine = 131_075; + private readonly ITestOutputHelper _output; + + public CiRunGlobalCoverageMemoryTests(ITestOutputHelper output) + { + _output = output; + } + + [SkippableTheory] + [InlineData("3.2.0")] + [InlineData("6.0.0")] + [Trait("RunOnWindows", "True")] + [Trait("Category", "EndToEnd")] + [Trait("Category", "TestIntegrations")] + [Trait("Category", "LoadTest")] + public void SixThousandNUnitContextsDoNotRetainNativeCoverageBuffers(string coverletVersion) + { + Skip.IfNot(FrameworkDescription.Instance.IsWindows()); + RunStress( + packageVersion: coverletVersion, + expectedCaseCount: 6_000, + includeCoverlet: true); + } + + private void RunStress( + string packageVersion, + int expectedCaseCount, + bool includeCoverlet) + { + var environmentHelper = new EnvironmentHelper(SampleName, typeof(CiRunGlobalCoverageMemoryTests), _output); + var sampleAssembly = environmentHelper.GetTestCommandForSampleApplicationPath(packageVersion, "net8.0"); + File.Exists(sampleAssembly).Should().BeTrue($"the required sample output must be present at {sampleAssembly}"); + + var runnerDirectory = GetRunnerDirectory(); + var runnerAssembly = Path.Combine(runnerDirectory, "Datadog.Trace.Tools.Runner.dll"); + File.Exists(runnerAssembly).Should().BeTrue("the runner tool output must contain the runner assembly"); + File.Exists(Path.Combine(runnerDirectory, "Datadog.Trace.Coverage.collector.dll")) + .Should() + .BeTrue("the runner tool output must contain the Datadog VSTest collector"); + File.Exists(Path.Combine(runnerDirectory, "Datadog.collector.dll")) + .Should() + .BeFalse("the production runner layout must not contain the test suite's legacy collector with the same VSTest friendly name"); + + using var root = new TemporaryDirectory("dd-global-coverage-memory-"); + var coverageDirectory = Directory.CreateDirectory(Path.Combine(root.RootPath, "coverage")).FullName; + + using var agent = MockTracerAgent.Create(null, TcpPortProvider.GetOpenPort()); + var logDirectory = Directory.CreateDirectory(Path.Combine(root.RootPath, "logs")).FullName; + var progressPath = Path.Combine(root.RootPath, "progress.jsonl"); + var targetCommand = CreateVstestCommand(environmentHelper.GetDotnetExe(), sampleAssembly, includeCoverlet); + var arguments = CreateCiRunArguments( + environmentHelper.MonitoringHome, + agent.Port, + coverageDirectory, + logDirectory, + progressPath, + expectedCaseCount, + targetCommand); + + var result = RunRunner(environmentHelper.GetDotnetExe(), runnerAssembly, arguments, logDirectory); + result.ExitCode.Should().Be(0, result.Error); + + AssertLaunch(result.Output, runnerDirectory); + var testhostProcessId = AssertProgress(progressPath, expectedCaseCount); + AssertPublishedCoverage(coverageDirectory, expectedCaseCount); + AssertCoverageDiagnostics(logDirectory, testhostProcessId, expectedCaseCount); + + static string GetRunnerDirectory() + { + // The integration-test output also contains Datadog.collector.dll from DatadogTestCollector. + // Use the tool output so VSTest resolves the same collector and directory layout shipped to customers. + var pivot = $"{EnvironmentTools.GetBuildConfiguration().ToLowerInvariant()}_net8.0"; + return Path.Combine( + EnvironmentTools.GetSolutionDirectory(), + "artifacts", + "bin", + "Datadog.Trace.Tools.Runner.Tool", + pivot); + } + } + + private ProcessResult RunRunner(string dotnetExecutable, string runnerAssembly, string[] arguments, string logDirectory) + { + using var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = dotnetExecutable, + WorkingDirectory = EnvironmentTools.GetSolutionDirectory(), + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + } + }; + process.StartInfo.ArgumentList.Add(runnerAssembly); + foreach (var argument in arguments) + { + process.StartInfo.ArgumentList.Add(argument); + } + + // Run the CLI in its own process so this load test exercises only production paths. + // Debug output also gives the test an observable record of the normalized child command. + process.StartInfo.Environment[ConfigurationKeys.DebugEnabled] = "1"; + process.StartInfo.Environment[ConfigurationKeys.LogDirectory] = logDirectory; + process.Start().Should().BeTrue(); + var outputTask = process.StandardOutput.ReadToEndAsync(); + var errorTask = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit((int)TimeSpan.FromMinutes(20).TotalMilliseconds)) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(); + throw new TimeoutException("The dd-trace CI run memory test exceeded 20 minutes."); + } + + var output = outputTask.GetAwaiter().GetResult(); + var error = errorTask.GetAwaiter().GetResult(); + _output.WriteLine(output); + if (!string.IsNullOrWhiteSpace(error)) + { + _output.WriteLine(error); + } + + return new ProcessResult(process.ExitCode, output, error); + } + + private string[] CreateVstestCommand( + string dotnetExecutable, + string sampleAssembly, + bool includeCoverlet) + { + var command = new System.Collections.Generic.List + { + dotnetExecutable, + "vstest", + sampleAssembly, + }; + + if (includeCoverlet) + { + command.Add("/Collect:XPlat Code Coverage;IncludeTestAssembly=true"); + } + + return command.ToArray(); + } + + private string[] CreateCiRunArguments( + string monitoringHome, + int agentPort, + string coverageDirectory, + string logDirectory, + string progressPath, + int expectedCaseCount, + string[] targetCommand) + { + var arguments = new System.Collections.Generic.List + { + "ci", + "run", + "--tracer-home", + monitoringHome, + "--agent-url", + $"http://127.0.0.1:{agentPort}", + "--set-env", + $"{ConfigurationKeys.CIVisibility.CodeCoverage}=1", + "--set-env", + $"{ConfigurationKeys.CIVisibility.CodeCoveragePath}={coverageDirectory}", + "--set-env", + $"{ConfigurationKeys.LogDirectory}={logDirectory}", + "--set-env", + $"NUNIT_GLOBAL_COVERAGE_PROGRESS_PATH={progressPath}", + "--set-env", + $"NUNIT_GLOBAL_COVERAGE_CASE_COUNT={(expectedCaseCount == 1 ? "1" : string.Empty)}", + "--set-env", + $"{ConfigurationKeys.DebugEnabled}=1", + "--" + }; + + arguments.AddRange(targetCommand); + return arguments.ToArray(); + } + + private void AssertLaunch(string output, string runnerDirectory) + { + var launchLine = output.Split(["\r\n", "\n"], StringSplitOptions.RemoveEmptyEntries) + .Should() + .ContainSingle(static line => line.StartsWith("Running:", StringComparison.Ordinal)) + .Subject; + var datadogCollectorCount = Regex.Matches(launchLine, @"(? !string.IsNullOrWhiteSpace(line)) + .Select(static line => JsonConvert.DeserializeObject(line)) + .ToArray(); + + records.Should().NotBeEmpty(); + records.Should().OnlyContain(record => record != null && record.Pid > 0 && record.PrivateBytes > 0 && record.ManagedBytes > 0); + records.Select(static record => record!.Pid).Distinct().Should().ContainSingle("all test cases must run in one testhost process"); + records[^1]!.Completed.Should().Be(expectedCaseCount); + + if (expectedCaseCount > 1) + { + var initialPrivateBytes = records[0]!.PrivateBytes; + var maximumPrivateBytes = records.Max(static record => record!.PrivateBytes); + (maximumPrivateBytes - initialPrivateBytes).Should() + .BeLessThan( + MaximumStressPrivateBytesGrowth, + "completed test contexts must not retain their 128 KiB native coverage buffers"); + } + + return records[0]!.Pid; + } + + private void AssertPublishedCoverage(string coverageDirectory, int expectedCaseCount) + { + var sessionCoverage = Directory.GetFiles(coverageDirectory, "session-coverage-*.json", SearchOption.TopDirectoryOnly); + sessionCoverage.Should().ContainSingle(); + var reader = new GlobalCoverageInputReader(); + reader.TryRead(sessionCoverage[0], out var coverage).Should().BeTrue(); + coverage.Should().NotBeNull(); + coverage!.GetTotalPercentage().Should().BeGreaterThan(0); + var sampleFile = coverage.Components.SelectMany(static component => component.Files) + .Should() + .ContainSingle(file => string.Equals(Path.GetFileName(file.Path), SampleSourceFileName, StringComparison.OrdinalIgnoreCase)) + .Subject; + AssertLine(sampleFile.ExecutableBitmap, CommonCoverageLine, expected: true); + AssertLine(sampleFile.ExecutedBitmap, CommonCoverageLine, expected: true); + AssertLine(sampleFile.ExecutableBitmap, FirstCoverageSentinelLine, expected: true); + AssertLine(sampleFile.ExecutedBitmap, FirstCoverageSentinelLine, expected: true); + AssertLine(sampleFile.ExecutableBitmap, MiddleCoverageSentinelLine, expected: true); + AssertLine(sampleFile.ExecutedBitmap, MiddleCoverageSentinelLine, expected: expectedCaseCount > 1); + AssertLine(sampleFile.ExecutableBitmap, LastCoverageSentinelLine, expected: true); + AssertLine(sampleFile.ExecutedBitmap, LastCoverageSentinelLine, expected: expectedCaseCount > 1); + + Directory.GetFiles(coverageDirectory, GlobalCoverageProtocol.CoverageFilePattern, SearchOption.TopDirectoryOnly).Should().NotBeEmpty(); + Directory.GetFiles(coverageDirectory, ".dd-coverage-process-incomplete-*", SearchOption.TopDirectoryOnly).Should().BeEmpty(); + } + + private void AssertLine(byte[]? bitmap, int line, bool expected) + { + bitmap.Should().NotBeNull(); + var zeroBasedLine = line - 1; + var byteIndex = zeroBasedLine >> 3; + bitmap!.Length.Should().BeGreaterThan(byteIndex); + var mask = (byte)(128 >> (zeroBasedLine & 7)); + ((bitmap[byteIndex] & mask) != 0).Should().Be(expected, $"line {line} should have the expected coverage state"); + } + + private void AssertCoverageDiagnostics(string logDirectory, int testhostProcessId, int expectedCaseCount) + { + var logLines = Directory.GetFiles(logDirectory, "*.log", SearchOption.AllDirectories) + .SelectMany(File.ReadLines) + .ToArray(); + var contextMatch = FindLastMatch( + logLines, + $@"Global coverage context diagnostics: pid={testhostProcessId}, started=(\d+), closed=(\d+), disposed=(\d+), merged=(\d+)\."); + var nativeMatch = FindLastMatch( + logLines, + $@"Global coverage native context-buffer diagnostics: pid={testhostProcessId}, currentBytes=(\d+), peakBytes=(\d+), activeBuffers=(\d+), peakBuffers=(\d+)\."); + var nativeAllocationMatch = FindLastMatch( + logLines, + $@"Global coverage native context-buffer allocation diagnostics: pid={testhostProcessId}, allocations=(\d+), frees=(\d+), maximumBufferBytes=(\d+)\."); + + Parse(contextMatch, 1).Should().Be(expectedCaseCount); + Parse(contextMatch, 2).Should().Be(expectedCaseCount); + Parse(contextMatch, 3).Should().Be(expectedCaseCount); + Parse(contextMatch, 4).Should().Be(expectedCaseCount); + + Parse(nativeMatch, 1).Should().Be(0); + Parse(nativeMatch, 2).Should().BeGreaterThanOrEqualTo(128 * 1024); + Parse(nativeMatch, 3).Should().Be(0); + Parse(nativeMatch, 4).Should().BeGreaterThanOrEqualTo(1); + var allocations = Parse(nativeAllocationMatch, 1); + allocations.Should().BeGreaterThanOrEqualTo(expectedCaseCount); + Parse(nativeAllocationMatch, 2).Should().Be(allocations); + Parse(nativeAllocationMatch, 3).Should().BeGreaterThanOrEqualTo(128 * 1024); + + static Match FindLastMatch(string[] lines, string pattern) + { + var match = lines.Select(line => Regex.Match(line, pattern, RegexOptions.CultureInvariant)) + .LastOrDefault(static candidate => candidate.Success); + match.Should().NotBeNull("the sealed testhost must emit global coverage lifecycle diagnostics"); + return match!; + } + + static long Parse(Match match, int group) + => long.Parse(match.Groups[group].Value, NumberStyles.None, CultureInfo.InvariantCulture); + } + + private readonly record struct ProcessResult(int ExitCode, string Output, string Error); + + private sealed class ProgressRecord + { + public int Pid { get; set; } + + public int Completed { get; set; } + + public long PrivateBytes { get; set; } + + public long ManagedBytes { get; set; } + } + + private sealed class TemporaryDirectory : IDisposable + { + public TemporaryDirectory(string prefix) + { + RootPath = Path.Combine(Path.GetTempPath(), prefix + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(RootPath); + } + + public string RootPath { get; } + + public void Dispose() + { + if (Directory.Exists(RootPath)) + { + Directory.Delete(RootPath, recursive: true); + } + } + } +} + +#endif diff --git a/tracer/test/Datadog.Trace.Tools.Runner.Tests/CoverageRewriteTests.cs b/tracer/test/Datadog.Trace.Tools.Runner.Tests/CoverageRewriteTests.cs index c92d59b07cf9..9f56456f31fb 100644 --- a/tracer/test/Datadog.Trace.Tools.Runner.Tests/CoverageRewriteTests.cs +++ b/tracer/test/Datadog.Trace.Tools.Runner.Tests/CoverageRewriteTests.cs @@ -7,9 +7,11 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Reflection; using System.Threading.Tasks; using System.Xml; using Datadog.Trace.Ci.Configuration; +using Datadog.Trace.Ci.Coverage; using Datadog.Trace.Ci.Coverage.Attributes; using Datadog.Trace.Configuration; using Datadog.Trace.Configuration.Telemetry; @@ -19,6 +21,7 @@ using ICSharpCode.Decompiler; using ICSharpCode.Decompiler.CSharp; using Mono.Cecil; +using Mono.Cecil.Cil; using VerifyTests; using VerifyXunit; using Xunit; @@ -204,6 +207,115 @@ public async Task WithFilters(string targetSnapshot, string configurationSetting await Verifier.Verify(transCode, transVerifySettings); } + [Fact] + public void RewrittenExceptionPathReleasesTheContextBufferAtSessionEnd() + { + var tempFileName = GetTempFile(); + var previousHandler = CoverageReporter.Handler; + try + { + using (var assembly = AssemblyDefinition.ReadAssembly(tempFileName, new ReaderParameters { ReadSymbols = true, InMemory = true })) + { + var main = assembly.MainModule.Types.Single(type => type.Name == "Class1").Methods.Single(method => method.Name == "Main"); + var firstInstruction = main.Body.Instructions[0]; + var constructor = assembly.MainModule.ImportReference(typeof(InvalidOperationException).GetConstructor(Type.EmptyTypes)!); + var processor = main.Body.GetILProcessor(); + processor.InsertBefore(firstInstruction, processor.Create(OpCodes.Newobj, constructor)); + processor.InsertBefore(firstInstruction, processor.Create(OpCodes.Throw)); + AssemblyProcessor.WriteTargetAssembly(assembly, tempFileName, strongNameKeyBlob: null); + } + + var asmProcessor = new AssemblyProcessor(tempFileName, new CoverageSettings(null, string.Empty)); + asmProcessor.Process(); + var rewrittenAssembly = Assembly.Load(File.ReadAllBytes(tempFileName)); + var instance = Activator.CreateInstance(rewrittenAssembly.GetType("CoverageRewriterAssembly.Class1")!); + var mainMethod = instance!.GetType().GetMethod("Main")!; + var handler = new DefaultWithGlobalCoverageEventHandler(); + CoverageReporter.Handler = handler; + var handle = handler.StartSession("xunit"); + + var invoke = () => mainMethod.Invoke(instance, null); + invoke.Should().Throw().WithInnerException(); + var module = handler.Container!.SnapshotModules().Should().ContainSingle().Subject; + + handler.EndSession(handle); + + module.FilesLines.Should().Be(IntPtr.Zero); + module.AllocatedByteLength.Should().Be(0); + } + finally + { + CoverageReporter.Handler = previousHandler; + File.Delete(tempFileName); + File.Delete(Path.ChangeExtension(tempFileName, ".pdb")); + } + } + + [Fact] + public void TailCallMethodKeepsTheExistingCounterInstrumentationShape() + { + var tempFileName = GetTempFile(); + try + { + using (var assembly = AssemblyDefinition.ReadAssembly(tempFileName, new ReaderParameters { ReadSymbols = true, InMemory = true })) + { + var type = assembly.MainModule.Types.Single(candidate => candidate.Name == "Class1"); + var templateSequencePoint = type.Methods + .SelectMany(method => method.DebugInformation.SequencePoints) + .First(sequencePoint => !sequencePoint.IsHidden && sequencePoint.Document is not null); + var method = new MethodDefinition( + "TailCountdown", + Mono.Cecil.MethodAttributes.Public | Mono.Cecil.MethodAttributes.Static, + assembly.MainModule.TypeSystem.Int32); + method.Parameters.Add(new ParameterDefinition("value", Mono.Cecil.ParameterAttributes.None, assembly.MainModule.TypeSystem.Int32)); + type.Methods.Add(method); + + var processor = method.Body.GetILProcessor(); + var firstInstruction = processor.Create(OpCodes.Ldarg_0); + var recurse = processor.Create(OpCodes.Ldarg_0); + processor.Append(firstInstruction); + processor.Append(processor.Create(OpCodes.Brtrue, recurse)); + processor.Append(processor.Create(OpCodes.Ldc_I4_0)); + processor.Append(processor.Create(OpCodes.Ret)); + processor.Append(recurse); + processor.Append(processor.Create(OpCodes.Ldc_I4_1)); + processor.Append(processor.Create(OpCodes.Sub)); + processor.Append(processor.Create(OpCodes.Tail)); + processor.Append(processor.Create(OpCodes.Call, method)); + processor.Append(processor.Create(OpCodes.Ret)); + method.DebugInformation.SequencePoints.Add( + new SequencePoint(firstInstruction, templateSequencePoint.Document) + { + StartLine = templateSequencePoint.StartLine, + StartColumn = templateSequencePoint.StartColumn, + EndLine = templateSequencePoint.EndLine, + EndColumn = templateSequencePoint.EndColumn, + }); + + AssemblyProcessor.WriteTargetAssembly(assembly, tempFileName, strongNameKeyBlob: null); + } + + var assemblyProcessor = new AssemblyProcessor(tempFileName, new CoverageSettings(null, string.Empty)); + assemblyProcessor.Process(); + + using var rewritten = AssemblyDefinition.ReadAssembly(tempFileName, new ReaderParameters { ReadSymbols = true, InMemory = true }); + var tailMethod = rewritten.MainModule.Types.Single(type => type.Name == "Class1").Methods.Single(method => method.Name == "TailCountdown"); + var tailInstruction = tailMethod.Body.Instructions.Should().ContainSingle(instruction => instruction.OpCode == OpCodes.Tail).Subject; + tailMethod.Body.Instructions[tailMethod.Body.Instructions.IndexOf(tailInstruction) + 1].OpCode.Should().Be(OpCodes.Call); + var calledMethods = tailMethod.Body.Instructions + .Where(instruction => instruction.Operand is MethodReference) + .Select(instruction => ((MethodReference)instruction.Operand).Name); + calledMethods.Should().Contain("GetFileCounter"); + calledMethods.Should().NotContain("AcquireFileCounter"); + tailMethod.Body.ExceptionHandlers.Should().BeEmpty("coverage instrumentation must not rewrite the method control flow"); + } + finally + { + File.Delete(tempFileName); + File.Delete(Path.ChangeExtension(tempFileName, ".pdb")); + } + } + [Theory] [MemberData(nameof(CoverageToolingAssemblyData))] public void SkipsCoverageToolingAssemblies(string assemblyFileName) diff --git a/tracer/test/Datadog.Trace.Tools.Runner.Tests/CoverageUtilsTests.cs b/tracer/test/Datadog.Trace.Tools.Runner.Tests/CoverageUtilsTests.cs index e35e65c3ca12..dfee9cc1be78 100644 --- a/tracer/test/Datadog.Trace.Tools.Runner.Tests/CoverageUtilsTests.cs +++ b/tracer/test/Datadog.Trace.Tools.Runner.Tests/CoverageUtilsTests.cs @@ -138,10 +138,48 @@ internal void CoverageCombineTest(GlobalCoverageInfo a, GlobalCoverageInfo b, Gl File.WriteAllText(bPath, JsonConvert.SerializeObject(b)); var outputFile = Path.GetTempFileName(); - global::CoverageUtils.TryCombineAndGetTotalCoverage(tmpFolder, outputFile, out var actualGlobalCoverageInfo).Should().BeTrue(); + CoverageUtils.TryCombineAndGetTotalCoverage(tmpFolder, outputFile, out var actualGlobalCoverageInfo, useStdOut: false).Should().BeTrue(); actualGlobalCoverageInfo.Should().BeEquivalentTo(expected); var outputContent = File.ReadAllText(outputFile); JsonConvert.DeserializeObject(outputContent).Should().BeEquivalentTo(expected); } + + [Fact] + internal void LegacyDirectorySkipsMalformedJsonAndCombinesValidCoverage() + { + var directory = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"))).FullName; + try + { + var expected = new GlobalCoverageInfo + { + Components = + { + new ComponentCoverageInfo("component") + { + Files = + { + new FileCoverageInfo("file.cs") + { + ExecutableBitmap = [0xff], + ExecutedBitmap = [0x81] + } + } + } + } + }; + File.WriteAllText(Path.Combine(directory, "unrelated.json"), "{not-json"); + File.WriteAllText(Path.Combine(directory, "coverage.json"), JsonConvert.SerializeObject(expected)); + var outputPath = Path.Combine(directory, "combined.json"); + + CoverageUtils.TryCombineAndGetTotalCoverage(directory, outputPath, out var actual, useStdOut: false).Should().BeTrue(); + + actual.Should().BeEquivalentTo(expected); + JsonConvert.DeserializeObject(File.ReadAllText(outputPath)).Should().BeEquivalentTo(expected); + } + finally + { + Directory.Delete(directory, true); + } + } } diff --git a/tracer/test/test-applications/integrations/Samples.NUnitGlobalCoverageMemory/GlobalCoverageMemoryTests.cs b/tracer/test/test-applications/integrations/Samples.NUnitGlobalCoverageMemory/GlobalCoverageMemoryTests.cs new file mode 100644 index 000000000000..20644928f21b --- /dev/null +++ b/tracer/test/test-applications/integrations/Samples.NUnitGlobalCoverageMemory/GlobalCoverageMemoryTests.cs @@ -0,0 +1,120 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Threading; +using NUnit.Framework; + +namespace Samples.NUnitGlobalCoverageMemory; + +[TestFixture] +[NonParallelizable] +public class GlobalCoverageMemoryTests +{ + private const int DefaultCaseCount = 6_000; + private const string CaseCountEnvironmentVariable = "NUNIT_GLOBAL_COVERAGE_CASE_COUNT"; + private const string ProgressPathEnvironmentVariable = "NUNIT_GLOBAL_COVERAGE_PROGRESS_PATH"; + private static int _completed; + + public static IEnumerable Cases() + { + var configuredCount = Environment.GetEnvironmentVariable(CaseCountEnvironmentVariable); + var count = string.IsNullOrEmpty(configuredCount) ? DefaultCaseCount : ParseSmokeCaseCount(configuredCount); + for (var i = 0; i < count; i++) + { + yield return new TestCaseData(i).SetName($"GlobalCoverageMemory_{i:D4}"); + } + } + + [TestCaseSource(nameof(Cases))] + public void ExecutesSparseInstrumentedLine(int value) + { + Assert.That(CoveredMethod(value), Is.EqualTo(value + 1)); + if (value == 0) + { + Assert.That(FirstCoverageSentinel(), Is.EqualTo(101)); + } + + if (value == DefaultCaseCount / 2) + { + Assert.That(MiddleCoverageSentinel(), Is.EqualTo(102)); + } + + if (value == DefaultCaseCount - 1) + { + Assert.That(LastCoverageSentinel(), Is.EqualTo(103)); + } + + var completed = Interlocked.Increment(ref _completed); + if (completed % 100 == 0 || completed == 1) + { + WriteProgress(completed); + } + } + + private static int CoveredMethod(int value) + { +#line 131072 + return value + 1; +#line default + } + + private static int FirstCoverageSentinel() + { +#line 131073 + return 101; +#line default + } + + private static int MiddleCoverageSentinel() + { +#line 131074 + return 102; +#line default + } + + private static int LastCoverageSentinel() + { +#line 131075 + return 103; +#line default + } + + private static int ParseSmokeCaseCount(string value) + { + if (value == "1") + { + return 1; + } + + throw new InvalidOperationException($"{CaseCountEnvironmentVariable} only accepts the test-only value '1'."); + } + + private static void WriteProgress(int completed) + { + var path = Environment.GetEnvironmentVariable(ProgressPathEnvironmentVariable); + if (string.IsNullOrEmpty(path)) + { + return; + } + + using var process = Process.GetCurrentProcess(); + var record = string.Format( + CultureInfo.InvariantCulture, + "{{\"pid\":{0},\"completed\":{1},\"privateBytes\":{2},\"managedBytes\":{3},\"timestamp\":\"{4:O}\"}}{5}", + process.Id, + completed, + process.PrivateMemorySize64, + GC.GetTotalMemory(forceFullCollection: false), + DateTimeOffset.UtcNow, + Environment.NewLine); + + File.AppendAllText(path, record); + } +} diff --git a/tracer/test/test-applications/integrations/Samples.NUnitGlobalCoverageMemory/Samples.NUnitGlobalCoverageMemory.csproj b/tracer/test/test-applications/integrations/Samples.NUnitGlobalCoverageMemory/Samples.NUnitGlobalCoverageMemory.csproj new file mode 100644 index 000000000000..90e9d3948cd3 --- /dev/null +++ b/tracer/test/test-applications/integrations/Samples.NUnitGlobalCoverageMemory/Samples.NUnitGlobalCoverageMemory.csproj @@ -0,0 +1,23 @@ + + + + net8.0 + Library + false + 6.0.0 + false + false + true + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + +