diff --git a/tracer/src/Datadog.Trace/Ci/Tagging/TestSpanTags.cs b/tracer/src/Datadog.Trace/Ci/Tagging/TestSpanTags.cs index ebcdb501158e..103924400e15 100644 --- a/tracer/src/Datadog.Trace/Ci/Tagging/TestSpanTags.cs +++ b/tracer/src/Datadog.Trace/Ci/Tagging/TestSpanTags.cs @@ -127,6 +127,9 @@ public TestSpanTags(TestSuiteSpanTags suiteTags, string testName) [Tag(TestTags.TestAttemptToFixPassed)] public string? AttemptToFixPassed { get; set; } + [Tag(TestTags.TestFinalStatus)] + public string? FinalStatus { get; set; } + [Tag(CapabilitiesTags.LibraryCapabilitiesTestImpactAnalysis)] public string? CapabilitiesTestImpactAnalysis { get; set; } diff --git a/tracer/src/Datadog.Trace/Ci/Tags/TestTags.cs b/tracer/src/Datadog.Trace/Ci/Tags/TestTags.cs index 1075b2ebb518..73217bf7de65 100644 --- a/tracer/src/Datadog.Trace/Ci/Tags/TestTags.cs +++ b/tracer/src/Datadog.Trace/Ci/Tags/TestTags.cs @@ -156,6 +156,21 @@ internal static class TestTags /// public const string TestRetryReason = "test.retry_reason"; + /// + /// Retry reason value for Early Flake Detection + /// + public const string TestRetryReasonEfd = "efd"; + + /// + /// Retry reason value for Auto Test Retries + /// + public const string TestRetryReasonAtr = "atr"; + + /// + /// Retry reason value for Attempt to Fix (Test Management) + /// + public const string TestRetryReasonAttemptToFix = "attempt_to_fix"; + /// /// Test is quarantined flag /// @@ -185,4 +200,9 @@ internal static class TestTags /// Test management enabled flag /// public const string TestManagementEnabled = "test.test_management.enabled"; + + /// + /// Test final status - the adjusted test outcome for CI pipelines + /// + public const string TestFinalStatus = "test.final_status"; } diff --git a/tracer/src/Datadog.Trace/Ci/Test.cs b/tracer/src/Datadog.Trace/Ci/Test.cs index 63ac18ec312e..b4d80f11cd51 100644 --- a/tracer/src/Datadog.Trace/Ci/Test.cs +++ b/tracer/src/Datadog.Trace/Ci/Test.cs @@ -555,8 +555,8 @@ public void Close(TestStatus status, TimeSpan? duration, string? skipReason) { var retryReasonTag = tags.TestRetryReason switch { - "efd" => MetricTags.CIVisibilityTestingEventTypeRetryReason.EarlyFlakeDetection, - "atr" => MetricTags.CIVisibilityTestingEventTypeRetryReason.AutomaticTestRetry, + TestTags.TestRetryReasonEfd => MetricTags.CIVisibilityTestingEventTypeRetryReason.EarlyFlakeDetection, + TestTags.TestRetryReasonAtr => MetricTags.CIVisibilityTestingEventTypeRetryReason.AutomaticTestRetry, _ => MetricTags.CIVisibilityTestingEventTypeRetryReason.None }; diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/Common.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/Common.cs index 7a245f887954..ab65e3fe1e7a 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/Common.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/Common.cs @@ -10,6 +10,7 @@ using System.Threading; using Datadog.Trace.Ci; using Datadog.Trace.Ci.Net; +using Datadog.Trace.Ci.Tagging; using Datadog.Trace.Ci.Tags; using Datadog.Trace.Logging; using Datadog.Trace.Util; @@ -188,7 +189,7 @@ internal static bool SetEarlyFlakeDetectionTestTagsAndAbortReason(Test test, boo if (isRetry) { testTags.TestIsRetry = "true"; - testTags.TestRetryReason = "efd"; + testTags.TestRetryReason = TestTags.TestRetryReasonEfd; } else { @@ -211,7 +212,7 @@ internal static bool SetFlakyRetryTags(Test test, bool isRetry) { var testTags = test.GetTags(); testTags.TestIsRetry = "true"; - testTags.TestRetryReason = "atr"; + testTags.TestRetryReason = TestTags.TestRetryReasonAtr; } return flakyRetryFeature; @@ -244,7 +245,7 @@ internal static TestOptimizationClient.TestManagementResponseTestPropertiesAttri if (isRetry) { testTags.TestIsRetry = "true"; - testTags.TestRetryReason = "attempt_to_fix"; + testTags.TestRetryReason = TestTags.TestRetryReasonAttemptToFix; } } @@ -273,4 +274,49 @@ internal static void CheckFaultyThreshold(Test test, long nTestCases, long tTest } } } + + /// + /// Calculates the final status for a test based on execution results and test management tags. + /// Priority order (first match wins): + /// 1. Quarantined/disabled -> skip (always mask to skip) + /// 2. For ATF tests: any execution failed -> fail (flaky test = fix didn't work) + /// 3. Any execution passed -> pass + /// 4. Skip/inconclusive AND no pass -> skip + /// 5. All executions failed -> fail + /// + /// True if any execution (initial or retry) passed. + /// True if any execution (initial or retry) failed. + /// True if the current/last execution was skip or inconclusive. + /// The test tags to check for quarantine/disabled/ATF status. + /// The final status string: "pass", "fail", or "skip". + internal static string CalculateFinalStatus(bool anyExecutionPassed, bool anyExecutionFailed, bool isSkippedOrInconclusive, TestSpanTags? testTags) + { + // Priority 1: Quarantined/disabled tests always mask to skip + if (testTags?.IsQuarantined == "true" || testTags?.IsDisabled == "true") + { + return TestTags.StatusSkip; + } + + // Priority 2: For ATF tests, any failure means fix didn't work (test is still flaky) + // This must be checked BEFORE anyPassed for ATF tests + if (testTags?.IsAttemptToFix == "true" && anyExecutionFailed) + { + return TestTags.StatusFail; + } + + // Priority 3: Any execution passed -> pass (pass takes precedence over skip) + if (anyExecutionPassed) + { + return TestTags.StatusPass; + } + + // Priority 4: Skip/inconclusive AND no pass -> skip + if (isSkippedOrInconclusive) + { + return TestTags.StatusSkip; + } + + // Priority 5: All executions failed -> fail + return TestTags.StatusFail; + } } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/SkipTestMethodExecutor.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/SkipTestMethodExecutor.cs index 696caeb8324f..3c8f3d5aa4cf 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/SkipTestMethodExecutor.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/SkipTestMethodExecutor.cs @@ -8,6 +8,7 @@ using System.Reflection; using System.Threading.Tasks; using Datadog.Trace.Ci; +using Datadog.Trace.Ci.Tags; using Datadog.Trace.DuckTyping; namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.MsTestV2; @@ -37,8 +38,13 @@ protected void ProcessTestMethod(object testMethod) if (testMethod.TryDuckCast(out var testMethodInfo)) { // Create the skip span - MsTestIntegration.OnMethodBegin(testMethodInfo, testMethod.GetType(), isRetry: false)? - .Close(TestStatus.Skip, TimeSpan.Zero, _skipReason); + var test = MsTestIntegration.OnMethodBegin(testMethodInfo, testMethod.GetType(), isRetry: false); + if (test is not null) + { + // Set final_status = skip for pre-execution skipped tests (ITR/attribute-based skips) + test.GetTags().FinalStatus = TestTags.StatusSkip; + test.Close(TestStatus.Skip, TimeSpan.Zero, _skipReason); + } } } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/TestMethodAttributeExecuteIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/TestMethodAttributeExecuteIntegration.cs index 2283520d730c..2c45047960d2 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/TestMethodAttributeExecuteIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/TestMethodAttributeExecuteIntegration.cs @@ -7,14 +7,18 @@ using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Datadog.Trace.Ci; +using Datadog.Trace.Ci.Tags; using Datadog.Trace.ClrProfiler.CallTarget; using Datadog.Trace.ClrProfiler.CallTarget.Handlers; using Datadog.Trace.DuckTyping; +using Datadog.Trace.Util; namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.MsTestV2; @@ -102,6 +106,15 @@ internal static CallTargetState OnMethodBegin(TTarget inst public sealed class TestMethodAttributeExecuteAsyncIntegration #pragma warning restore SA1402 { + // Per-row cache for parameterized test execution results, keyed by test identifier (DisplayName) + // This survives across initial and retry executions for the same test method + // Use ConcurrentDictionary for thread safety - MSTest can run parameterized rows in parallel + // ConditionalWeakTable allows garbage collection of testMethod without manual cleanup + private static readonly ConditionalWeakTable> InitialExecutionPassedCache = new(); + private static readonly ConditionalWeakTable> InitialExecutionFailedCache = new(); + private static readonly ConditionalWeakTable> AnyRetryPassedCache = new(); + private static readonly ConditionalWeakTable> AllAttemptsPassedCache = new(); + private static int _totalRetries = -1; internal static CallTargetState OnMethodBegin(TTarget instance, TTestMethod testMethod) @@ -153,6 +166,8 @@ internal static CallTargetState OnMethodBegin(TTarget inst } MsTestIntegration.AddTotalTestCases(returnValueList.Count - 1); + var initialExecutionPassed = false; + var initialExecutionFailed = false; for (var i = 0; i < returnValueList.Count; i++) { var test = i == 0 ? testMethodState.Test : MsTestIntegration.OnMethodBegin(testMethodState.TestMethod, testMethodState.TestMethod.Type, isRetry: false, testMethodState.Test.StartTime); @@ -174,9 +189,33 @@ internal static CallTargetState OnMethodBegin(TTarget inst if (returnValueList[i].TryDuckCast(out var testResult)) { - var retryState = new RetryState(); + // Check if this test will have EFD/ATF retries (based on tags) + var testIsEfd = testTags.TestIsNew == "true"; + var testIsAtf = testTags.IsAttemptToFix == "true"; + + var retryState = new RetryState + { + IsEfdOrAtfTest = testIsEfd || testIsAtf + }; resultStatus = HandleTestResult(test, testMethod, testResult, exception, retryState); allowRetries = allowRetries || resultStatus != TestStatus.Skip; + + // Track if initial execution passed/failed (for final_status) - both aggregate and per-row + var displayName = testResult.DisplayName ?? test.Name; + if (resultStatus == TestStatus.Pass) + { + initialExecutionPassed = true; + var cacheKey = GetCacheKey(displayName); + // Cache per-row initial execution result for parameterized tests + SetInitialExecutionPassed(testMethodState.TestMethod, cacheKey, true); + } + else if (resultStatus == TestStatus.Fail) + { + initialExecutionFailed = true; + var cacheKey = GetCacheKey(displayName); + // Cache per-row initial execution failure for ATF tracking + SetInitialExecutionFailed(testMethodState.TestMethod, cacheKey, true); + } } else { @@ -209,6 +248,10 @@ internal static CallTargetState OnMethodBegin(TTarget inst { IsARetry = true, IsAttemptToFix = isAttemptToFix, + IsEfdOrAtfTest = true, + TotalExecutions = 1 + remainingRetries, + InitialExecutionPassed = initialExecutionPassed, + InitialExecutionFailed = initialExecutionFailed, }; // Handle retries @@ -237,19 +280,26 @@ internal static CallTargetState OnMethodBegin(TTarget inst } // Flaky retry is enabled and the test failed - var retryState = new RetryState - { - IsARetry = true, - IsAttemptToFix = false, - }; Interlocked.CompareExchange(ref _totalRetries, testOptimization.FlakyRetryFeature?.TotalFlakyRetryCount ?? TestOptimizationFlakyRetryFeature.TotalFlakyRetryCountDefault, -1); var remainingRetries = testOptimization.FlakyRetryFeature?.FlakyRetryCount ?? TestOptimizationFlakyRetryFeature.FlakyRetryCountDefault; if (remainingRetries > 0) { + var retryState = new RetryState + { + IsARetry = true, + IsAttemptToFix = false, + IsEfdOrAtfTest = false, + TotalExecutions = 1 + remainingRetries, + InitialExecutionPassed = initialExecutionPassed, + InitialExecutionFailed = initialExecutionFailed, + }; + // Handle retries var results = new List { returnValueList }; for (var i = 0; i < remainingRetries; i++) { + retryState.IsLastRetry = i == remainingRetries - 1; + if (Interlocked.Decrement(ref _totalRetries) <= 0) { Common.Log.Debug("TestMethodAttributeExecuteIntegration: FlakyRetry: Exceeded number of total retries. [{Number}]", testOptimization.FlakyRetryFeature?.TotalFlakyRetryCount); @@ -367,37 +417,105 @@ private static TestStatus HandleTestResult(Test test, MsTestIntegration.UpdateTestParameters(test, testMethodProxy, testResult.DisplayName); } + // Get display name for per-row caching - must be before exception branch + var displayName = testResult.DisplayName ?? test.Name; + var cacheKey = GetCacheKey(displayName); + + var shouldMaskOutcome = false; try { if (exception is not null) { + // Track failure for ATF - both shared state and per-row cache + if (retryState.IsAttemptToFix) + { + retryState.AllAttemptsPassed = false; + + // Cache per-row ATF failure for parameterized tests (retry path) + if (retryState.IsARetry && testMethod is not null) + { + SetAllAttemptsPassed(testMethod, cacheKey, false); + } + } + + // Track initial execution failure - both shared state and per-row cache + if (!retryState.IsARetry) + { + retryState.InitialExecutionFailed = true; + if (testMethod is not null) + { + SetInitialExecutionFailed(testMethod, cacheKey, true); + } + } + + // Set final_status before closing + SetFinalStatusIfApplicable(test, testMethod, cacheKey, TestStatus.Fail, retryState); + test.Close(TestStatus.Fail); return TestStatus.Fail; } var testStatus = GetStatusFromOutcome(testResult.Outcome); + + // Track pass status for final_status calculation if (retryState.IsARetry) { + // Retry execution if (testStatus != TestStatus.Fail) { retryState.AllRetriesFailed = false; } - else if (retryState.IsAttemptToFix) + + if (testStatus == TestStatus.Pass && testMethod is not null) + { + retryState.AnyRetryPassed = true; + + // Cache per-row retry pass result for parameterized tests + SetAnyRetryPassed(testMethod, cacheKey, true); + } + else if (retryState.IsAttemptToFix && testStatus == TestStatus.Fail && testMethod is not null) { retryState.AllAttemptsPassed = false; + + // Cache per-row ATF failure for parameterized tests + SetAllAttemptsPassed(testMethod, cacheKey, false); } + } + else + { + // Initial execution + if (testStatus == TestStatus.Pass && testMethod is not null) + { + retryState.InitialExecutionPassed = true; - if (retryState.IsLastRetry && test.GetTags() is { } testTags) + // Cache per-row initial pass result for parameterized tests + SetInitialExecutionPassed(testMethod, cacheKey, true); + } + else if (testStatus == TestStatus.Fail && testMethod is not null) { - if (retryState.IsAttemptToFix) - { - testTags.AttemptToFixPassed = retryState.AllAttemptsPassed ? "true" : "false"; - } + retryState.InitialExecutionFailed = true; - if (retryState.AllRetriesFailed) - { - testTags.HasFailedAllRetries = "true"; - } + // Cache per-row initial execution failure for ATF tracking + SetInitialExecutionFailed(testMethod, cacheKey, true); + } + } + + // Set final_status before closing the test + SetFinalStatusIfApplicable(test, testMethod, cacheKey, testStatus, retryState); + + // Determine if we should mask outcome (quarantined/ATF) - only on final execution + var testTags = test.GetTags(); + if (TestOptimization.Instance.TestManagementFeature?.Enabled == true && testTags is not null) + { + var isQuarantined = testTags.IsQuarantined == "true"; + var isAttemptToFix = testTags.IsAttemptToFix == "true"; + var isDisabled = testTags.IsDisabled == "true"; + + // Only mask outcome on final execution for ATF + // Quarantined and disabled tests always mask outcome + if (isQuarantined || isDisabled || (isAttemptToFix && (retryState.IsLastRetry || (!retryState.IsARetry && !retryState.IsEfdOrAtfTest)))) + { + shouldMaskOutcome = true; } } @@ -419,17 +537,11 @@ private static TestStatus HandleTestResult(Test test, } finally { - if (TestOptimization.Instance.TestManagementFeature?.Enabled == true) + if (shouldMaskOutcome) { - var testTags = test.GetTags(); - var isQuarantined = testTags.IsQuarantined == "true"; - var isAttemptToFix = testTags.IsAttemptToFix == "true"; - if (isQuarantined || isAttemptToFix) - { - Common.Log.Debug("TestMethodAttributeExecuteIntegration: Test is quarantined or is an attempt to fix. Skipping test."); - testResult.Outcome = UnitTestOutcome.Ignored; - testResult.TestFailureException = null; - } + Common.Log.Debug("TestMethodAttributeExecuteIntegration: Test is quarantined, disabled, or is an attempt to fix (final). Masking outcome."); + testResult.Outcome = UnitTestOutcome.Ignored; + testResult.TestFailureException = null; } } } @@ -451,6 +563,104 @@ static TestStatus Unknown(UnitTestOutcome outcome) } } + private static void SetFinalStatusIfApplicable(Test test, TTestMethod testMethod, string cacheKey, TestStatus testStatus, RetryState retryState) + { + var testTags = test.GetTags(); + if (testTags == null) + { + return; + } + + // Per-span guard to prevent duplicate setting + if (testTags.FinalStatus is not null) + { + return; + } + + // Determine if this is a "final execution" for final_status calculation + bool isFinalExecution; + + if (retryState.IsARetry) + { + // For retries, check various conditions + var isAtrRetry = retryState is { IsAttemptToFix: false, IsEfdOrAtfTest: false }; + + // ATR early exit: test actually passed (Skip doesn't trigger early exit) + var isAtrEarlyExit = isAtrRetry && testStatus == TestStatus.Pass; + + // ATR budget exhaustion: test failed and budget is about to run out + // This check runs before retry scheduling decrements the shared ATR budget. + // <= 1 means this failure is effectively final because the next decrement exhausts budget. + var isAtrBudgetExhausted = isAtrRetry && testStatus == TestStatus.Fail && GetRemainingAtrBudget() <= 1; + + isFinalExecution = retryState.IsLastRetry || isAtrEarlyExit || isAtrBudgetExhausted; + } + else + { + // Initial execution - it's final if no retries will happen + // For EFD/ATF, retries will always happen + // For ATR, retries happen only if test fails and ATR is enabled + var atrEnabled = TestOptimization.Instance.FlakyRetryFeature?.Enabled == true; + var willHaveAtrRetries = atrEnabled && testStatus == TestStatus.Fail; + isFinalExecution = !retryState.IsEfdOrAtfTest && !willHaveAtrRetries; + } + + if (!isFinalExecution) + { + return; + } + + // Only set retry-specific tags for tests with actual retries + if (retryState.TotalExecutions > 1) + { + if (retryState.AllRetriesFailed) + { + testTags.HasFailedAllRetries = "true"; + } + } + + // Calculate final_status using PER-ROW CACHE for parameterized tests + // This ensures each row gets the correct final_status based on its own execution results, + // not the shared RetryState which aggregates across all rows + bool anyExecutionPassed; + bool anyExecutionFailed; + if (retryState.TotalExecutions == 1) + { + // Single execution: current status determines pass/fail + anyExecutionPassed = testStatus == TestStatus.Pass; + anyExecutionFailed = testStatus == TestStatus.Fail; + } + else if (testMethod is not null) + { + // Retry: use per-row cache for correct parameterized test handling + var initialPassed = GetInitialExecutionPassed(testMethod, cacheKey); + var initialFailed = GetInitialExecutionFailed(testMethod, cacheKey); + var retryPassed = GetAnyRetryPassed(testMethod, cacheKey); + var allAttemptsPassed = GetAllAttemptsPassed(testMethod, cacheKey); + anyExecutionPassed = initialPassed || retryPassed; + // For ATF: any actual failure (initial or retry) means the fix didn't work (test is still flaky) + // Note: skip does NOT count as failure per ATF semantics + anyExecutionFailed = initialFailed || !allAttemptsPassed; + } + else + { + // Fallback: use shared RetryState (shouldn't happen in normal flow) + anyExecutionPassed = retryState.InitialExecutionPassed || retryState.AnyRetryPassed; + // Use explicit InitialExecutionFailed - skip does NOT count as failure per ATF semantics + anyExecutionFailed = retryState.InitialExecutionFailed || !retryState.AllAttemptsPassed; + } + + var isSkippedOrInconclusive = testStatus == TestStatus.Skip; + testTags.FinalStatus = Common.CalculateFinalStatus(anyExecutionPassed, anyExecutionFailed, isSkippedOrInconclusive, testTags); + + // ATF: AttemptToFixPassed should be consistent with final_status + // If any execution failed, the fix didn't work + if (retryState.TotalExecutions > 1 && retryState.IsAttemptToFix) + { + testTags.AttemptToFixPassed = anyExecutionFailed ? "false" : "true"; + } + } + private static IList GetFinalResults(List executionStatuses) { var lstExceptions = new List(); @@ -517,6 +727,119 @@ object GetResultFromRetries(IList results) } } + /// + /// Read-only snapshot of remaining ATR budget for pre-close checks. + /// Value meanings: -1 = uninitialized, 0 = exhausted, positive = nominally available. + /// This value is read before retry scheduling decrements budget, so values of 1 or 0 mean no + /// further retry can run after the current failed execution. + /// + internal static int GetRemainingAtrBudget() + => Interlocked.CompareExchange(ref _totalRetries, 0, 0); + + /// + /// Gets the cache key for per-row tracking. Uses displayName (includes parameter values for parameterized tests). + /// Falls back to a default key if displayName is null/empty to avoid collision. + /// + private static string GetCacheKey(string? displayName) + { + // For parameterized tests: displayName includes parameter values (e.g., "TestMethod (1, 2)") + // For non-parameterized tests: displayName may be empty, use a default fallback + if (StringUtil.IsNullOrEmpty(displayName)) + { + return "__default__"; + } + + return displayName; + } + + /// + /// Gets whether the initial execution passed for a specific parameterized row. + /// + private static bool GetInitialExecutionPassed(object testMethodKey, string cacheKey) + { + if (InitialExecutionPassedCache.TryGetValue(testMethodKey, out var cache) && cache.TryGetValue(cacheKey, out var passed)) + { + return passed; + } + + return false; // Default: assume initial failed if not cached + } + + /// + /// Sets whether the initial execution passed for a specific parameterized row. + /// + private static void SetInitialExecutionPassed(object testMethodKey, string cacheKey, bool passed) + { + var cache = InitialExecutionPassedCache.GetOrCreateValue(testMethodKey); + cache[cacheKey] = passed; + } + + /// + /// Gets whether any retry execution passed for a specific parameterized row. + /// + private static bool GetAnyRetryPassed(object testMethodKey, string cacheKey) + { + if (AnyRetryPassedCache.TryGetValue(testMethodKey, out var cache) && cache.TryGetValue(cacheKey, out var passed)) + { + return passed; + } + + return false; // Default: no retry has passed yet + } + + /// + /// Sets whether any retry execution passed for a specific parameterized row. + /// + private static void SetAnyRetryPassed(object testMethodKey, string cacheKey, bool passed) + { + var cache = AnyRetryPassedCache.GetOrCreateValue(testMethodKey); + cache.AddOrUpdate(cacheKey, passed, (k, v) => passed || v); + } + + /// + /// Gets whether the initial execution failed for a specific parameterized row. + /// + private static bool GetInitialExecutionFailed(object testMethodKey, string cacheKey) + { + if (InitialExecutionFailedCache.TryGetValue(testMethodKey, out var cache) && cache.TryGetValue(cacheKey, out var failed)) + { + return failed; + } + + return false; // Default: assume initial didn't fail if not cached + } + + /// + /// Sets whether the initial execution failed for a specific parameterized row. + /// + private static void SetInitialExecutionFailed(object testMethodKey, string cacheKey, bool failed) + { + var cache = InitialExecutionFailedCache.GetOrCreateValue(testMethodKey); + cache[cacheKey] = failed; + } + + /// + /// Gets whether all attempts passed for a specific parameterized row (ATF tracking). + /// + private static bool GetAllAttemptsPassed(object testMethodKey, string cacheKey) + { + if (AllAttemptsPassedCache.TryGetValue(testMethodKey, out var cache) && cache.TryGetValue(cacheKey, out var allPassed)) + { + return allPassed; + } + + return true; // Default: assume all passed until a failure is recorded + } + + /// + /// Sets whether all attempts passed for a specific parameterized row (ATF tracking). + /// + private static void SetAllAttemptsPassed(object testMethodKey, string cacheKey, bool allPassed) + { + var cache = AllAttemptsPassedCache.GetOrCreateValue(testMethodKey); + cache[cacheKey] = allPassed; + } + private readonly struct TestRunnerState { private readonly TraceClock _clock; @@ -546,5 +869,32 @@ private sealed class RetryState public bool AllRetriesFailed { get; set; } = true; public bool IsAttemptToFix { get; set; } = false; + + /// + /// Gets or sets a value indicating whether this test is an EFD or ATF test (retries will always happen). + /// + public bool IsEfdOrAtfTest { get; set; } = false; + + /// + /// Gets or sets a value indicating whether the initial execution passed. Only PASS counts as passed, not SKIP. + /// + public bool InitialExecutionPassed { get; set; } = false; + + /// + /// Gets or sets a value indicating whether the initial execution failed. Only FAIL counts as failed, not SKIP. + /// Used for ATF final_status calculation. + /// + public bool InitialExecutionFailed { get; set; } = false; + + /// + /// Gets or sets a value indicating whether any retry execution passed. Only PASS counts as passed, not SKIP. + /// Used for final_status calculation. + /// + public bool AnyRetryPassed { get; set; } = false; + + /// + /// Gets or sets the total number of executions for this test (1 = single execution, >1 = has retries). + /// + public int TotalExecutions { get; set; } = 1; } } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestAsyncIntegration3_8.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestAsyncIntegration3_8.cs index 2f66d15dade7..6cea7d6a59d6 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestAsyncIntegration3_8.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestAsyncIntegration3_8.cs @@ -9,6 +9,7 @@ using System.ComponentModel; using System.Runtime.CompilerServices; using Datadog.Trace.Ci; +using Datadog.Trace.Ci.Tags; using Datadog.Trace.ClrProfiler.CallTarget; using Datadog.Trace.DuckTyping; @@ -92,7 +93,17 @@ internal static CallTargetState OnMethodBegin(out var assemblyInfoExceptionsStruct) == true) { if (assemblyInfoExceptionsStruct.AssemblyInitializationException is { } assemblyInitializationException && - MsTestIntegration.OnMethodBegin(testMethodInfo, instance.GetType(), isRetry: false) is { } test) + MsTestIntegration.OnMethodBegin(testMethodInfo, instance.GetType(), isRetry: false) is { } asmTest) { - test.SetErrorInfo(assemblyInitializationException); - test.Close(TestStatus.Fail); + asmTest.SetErrorInfo(assemblyInitializationException); + + // Set final_status = fail for assembly initialization failures + if (asmTest.GetTags() is { } asmTestTags) + { + asmTestTags.FinalStatus = TestTags.StatusFail; + } + + asmTest.Close(TestStatus.Fail); } } else diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestIntegration.cs index 1439bd9b269e..6698088c3c51 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestIntegration.cs @@ -8,6 +8,7 @@ using System.Collections; using System.ComponentModel; using Datadog.Trace.Ci; +using Datadog.Trace.Ci.Tags; using Datadog.Trace.ClrProfiler.CallTarget; using Datadog.Trace.DuckTyping; @@ -57,7 +58,17 @@ public static class UnitTestRunnerRunSingleTestIntegration if (!skipHandled) { // This instrumentation catches all tests being ignored - MsTestIntegration.OnMethodBegin(testMethod, instance.GetType(), isRetry: false)?.Close(TestStatus.Skip, TimeSpan.Zero, unitTestResult.ErrorMessage); + var test = MsTestIntegration.OnMethodBegin(testMethod, instance.GetType(), isRetry: false); + if (test is not null) + { + // Set final_status = skip for ignored/inconclusive tests + if (test.GetTags() is { } testTags) + { + testTags.FinalStatus = TestTags.StatusSkip; + } + + test.Close(TestStatus.Skip, TimeSpan.Zero, unitTestResult.ErrorMessage); + } } } else if (unitTestResult.Outcome is UnitTestResultOutcome.Error or UnitTestResultOutcome.Failed) @@ -71,6 +82,13 @@ public static class UnitTestRunnerRunSingleTestIntegration MsTestIntegration.OnMethodBegin(testMethodInfo, instance.GetType(), isRetry: false) is { } test) { test.SetErrorInfo(classInitializationException); + + // Set final_status = fail for class initialization failures + if (test.GetTags() is { } testTags) + { + testTags.FinalStatus = TestTags.StatusFail; + } + test.Close(TestStatus.Fail); } } @@ -97,10 +115,17 @@ public static class UnitTestRunnerRunSingleTestIntegration if (testMethodInfo.Parent?.Parent?.Instance.TryDuckCast(out var assemblyInfoExceptionsStruct) == true) { if (assemblyInfoExceptionsStruct.AssemblyInitializationException is { } assemblyInitializationException && - MsTestIntegration.OnMethodBegin(testMethodInfo, instance.GetType(), isRetry: false) is { } test) + MsTestIntegration.OnMethodBegin(testMethodInfo, instance.GetType(), isRetry: false) is { } asmTest) { - test.SetErrorInfo(assemblyInitializationException); - test.Close(TestStatus.Fail); + asmTest.SetErrorInfo(assemblyInitializationException); + + // Set final_status = fail for assembly initialization failures + if (asmTest.GetTags() is { } asmTestTags) + { + asmTestTags.FinalStatus = TestTags.StatusFail; + } + + asmTest.Close(TestStatus.Fail); } } else diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestIntegration3_8.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestIntegration3_8.cs index 03fedb8cafa0..24918bb68104 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestIntegration3_8.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/MsTestV2/UnitTestRunnerRunSingleTestIntegration3_8.cs @@ -7,6 +7,7 @@ using System.Collections; using System.ComponentModel; using Datadog.Trace.Ci; +using Datadog.Trace.Ci.Tags; using Datadog.Trace.ClrProfiler.CallTarget; using Datadog.Trace.DuckTyping; @@ -65,7 +66,17 @@ public static class UnitTestRunnerRunSingleTestIntegration3_8 if (!skipHandled) { // This instrumentation catches all tests being ignored - MsTestIntegration.OnMethodBegin(testMethod, instance.GetType(), isRetry: false)?.Close(TestStatus.Skip, TimeSpan.Zero, unitTestResult.IgnoreReason); + var test = MsTestIntegration.OnMethodBegin(testMethod, instance.GetType(), isRetry: false); + if (test is not null) + { + // Set final_status = skip for ignored/inconclusive tests + if (test.GetTags() is { } testTags) + { + testTags.FinalStatus = TestTags.StatusSkip; + } + + test.Close(TestStatus.Skip, TimeSpan.Zero, unitTestResult.IgnoreReason); + } } } else if (unitTestResult.Outcome is UnitTestOutcome.Error or UnitTestOutcome.Failed) @@ -79,6 +90,13 @@ public static class UnitTestRunnerRunSingleTestIntegration3_8 MsTestIntegration.OnMethodBegin(testMethodInfo, instance.GetType(), isRetry: false) is { } test) { test.SetErrorInfo(classInitializationException); + + // Set final_status = fail for class initialization failures + if (test.GetTags() is { } testTags) + { + testTags.FinalStatus = TestTags.StatusFail; + } + test.Close(TestStatus.Fail); } } @@ -105,10 +123,17 @@ public static class UnitTestRunnerRunSingleTestIntegration3_8 if (testMethodInfo.Parent?.Parent?.Instance.TryDuckCast(out var assemblyInfoExceptionsStruct) == true) { if (assemblyInfoExceptionsStruct.AssemblyInitializationException is { } assemblyInitializationException && - MsTestIntegration.OnMethodBegin(testMethodInfo, instance.GetType(), isRetry: false) is { } test) + MsTestIntegration.OnMethodBegin(testMethodInfo, instance.GetType(), isRetry: false) is { } asmTest) { - test.SetErrorInfo(assemblyInitializationException); - test.Close(TestStatus.Fail); + asmTest.SetErrorInfo(assemblyInitializationException); + + // Set final_status = fail for assembly initialization failures + if (asmTest.GetTags() is { } asmTestTags) + { + asmTestTags.FinalStatus = TestTags.StatusFail; + } + + asmTest.Close(TestStatus.Fail); } } else diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/FlakyRetryBehavior.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/FlakyRetryBehavior.cs index 34b9fcd18892..4ac02c03eed5 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/FlakyRetryBehavior.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/FlakyRetryBehavior.cs @@ -27,4 +27,13 @@ public bool ShouldRetry(ITestResult result) => result.ResultState.Status == TestStatus.Failed && Interlocked.Decrement(ref _totalRetries) > 0; public ITestResult ResultChanges(ITestResult result) => result; + + /// + /// Read-only snapshot of remaining ATR budget for pre-close checks. + /// Value meanings: -1 = uninitialized, 0 = exhausted, positive = nominally available. + /// This value is observed before ShouldRetry() decrements budget, so values of 1 or 0 mean no + /// further retry can run after the current failed execution. + /// + internal static int GetRemainingBudget() + => Interlocked.CompareExchange(ref _totalRetries, 0, 0); } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/NUnitIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/NUnitIntegration.cs index 623946b4fdc5..03d769487800 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/NUnitIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/NUnitIntegration.cs @@ -316,6 +316,12 @@ internal static void GetExceptionAndMessage(ITestResult result, out string excep // Skip tests if (skipReason is not null) { + // Set final_status = skip for pre-execution skipped tests (ITR/attribute-based skips) + if (test.GetTags() is { } skipTestTags) + { + skipTestTags.FinalStatus = TestTags.StatusSkip; + } + test.Close(Ci.TestStatus.Skip, skipReason: skipReason, duration: TimeSpan.Zero); return test; } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/TestOptimizationTestCommand.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/TestOptimizationTestCommand.cs index 8b3826d6e15f..ad3a17093529 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/TestOptimizationTestCommand.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/NUnit/TestOptimizationTestCommand.cs @@ -1,4 +1,4 @@ -// +// // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. // @@ -8,6 +8,7 @@ using Datadog.Trace.Ci; using Datadog.Trace.Ci.Net; using Datadog.Trace.Ci.Tagging; +using Datadog.Trace.Ci.Tags; using Datadog.Trace.DuckTyping; namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.NUnit; @@ -47,6 +48,12 @@ public TestOptimizationTestCommand(ITestCommand innerCommand) SetSkippedResult(result, "Flaky test is disabled by Datadog."); if (NUnitIntegration.GetOrCreateTest(context.CurrentTest, 0) is { } test) { + // Set final_status = skip for disabled tests (no execution, so direct skip) + if (test.GetTags() is { } disabledTestTags) + { + disabledTestTags.FinalStatus = TestTags.StatusSkip; + } + NUnitIntegration.FinishTest(test, result); } @@ -66,19 +73,45 @@ public TestOptimizationTestCommand(ITestCommand innerCommand) SetSkippedResult(result, "Flaky test is quarantined by Datadog."); } + // Determine if retries will happen (used for final_status decision) + var isSkippedOrInconclusive = resultStatus is TestStatus.Skipped or TestStatus.Inconclusive; + var efdWillRetry = testOptimization.EarlyFlakeDetectionFeature?.Enabled == true && testTags?.TestIsNew == "true"; + var atrRemainingBudget = testOptimization.FlakyRetryFeature?.Enabled == true ? FlakyRetryBehavior.GetRemainingBudget() : 0; + // -1 = uninitialized → assume retries possible; 0 = exhausted → no retries + var atrHasBudget = atrRemainingBudget != 0; + var atrWillRetry = resultStatus == TestStatus.Failed && + testOptimization.FlakyRetryFeature?.Enabled == true && + atrHasBudget; + var atfWillRetry = testManagementProperties is { AttemptToFix: true }; + var willRetry = !isSkippedOrInconclusive && (efdWillRetry || atrWillRetry || atfWillRetry); + // We bailout if the test was skipped or inconclusive - if (resultStatus is TestStatus.Skipped or TestStatus.Inconclusive) + // But first set final_status for the bailout path + if (isSkippedOrInconclusive || !willRetry) + { + // Set final_status for tests that won't retry (single-execution or skip/inconclusive bailout) + if (testTags is not null && testTags.FinalStatus is null) + { + // Single execution: passed if status == Pass (not skip, not fail) + var anyExecutionPassed = resultStatus == TestStatus.Passed; + var anyExecutionFailed = resultStatus == TestStatus.Failed; + testTags.FinalStatus = Common.CalculateFinalStatus(anyExecutionPassed, anyExecutionFailed, isSkippedOrInconclusive, testTags); + } + } + + // Early bailout for skip/inconclusive + if (isSkippedOrInconclusive) { context.CurrentResult = result; return result.Instance; } // Apply retries - if (testOptimization.EarlyFlakeDetectionFeature?.Enabled == true && testTags?.TestIsNew == "true") + if (efdWillRetry) { result = DoRetries(new EarlyFlakeDetectionRetryBehavior(duration), context, result); } - else if (resultStatus == TestStatus.Failed && testOptimization.FlakyRetryFeature?.Enabled == true) + else if (atrWillRetry) { // check if is the first execution and the dynamic instrumentation feature is enabled if (testOptimization.DynamicInstrumentationFeature?.Enabled == true) @@ -91,9 +124,10 @@ public TestOptimizationTestCommand(ITestCommand innerCommand) result = DoRetries(new FlakyRetryBehavior(testOptimization), context, result); } - else if (testManagementProperties is { AttemptToFix: true }) + else if (atfWillRetry) { - result = DoRetries(new AttemptToFixRetryBehavior(testOptimization, testManagementProperties), context, result); + // testManagementProperties is validated non-null by the atfWillRetry pattern match above + result = DoRetries(new AttemptToFixRetryBehavior(testOptimization, testManagementProperties!), context, result); } context.CurrentResult = result; @@ -176,13 +210,26 @@ private ITestResult ExecuteTest(ITestExecutionContext context, int executionNumb } var attemptToFixRetryBehavior = retryState.BehaviorType == typeof(AttemptToFixRetryBehavior); + var resultStatus = testResult.ResultState.Status; + if (retryState.IsARetry) { - if (testResult.ResultState.Status != TestStatus.Failed) + // Track if any retry passed (for final_status calculation) + // CRITICAL: Only PASS counts as passed, not SKIP! + if (resultStatus == TestStatus.Passed) + { + retryState.AnyRetryPassed = true; + } + + // PRESERVED: AllRetriesFailed clears on pass OR skip (existing behavior for has_failed_all_retries) + if (resultStatus != TestStatus.Failed) { retryState.AllRetriesFailed = false; } - else if (attemptToFixRetryBehavior) + + // ATF: AllAttemptsPassed clears only on actual failure (not skip) + // Per ATF semantics: "failed" means actual failure, not skip/inconclusive + if (attemptToFixRetryBehavior && resultStatus == TestStatus.Failed) { retryState.AllAttemptsPassed = false; } @@ -197,17 +244,45 @@ private ITestResult ExecuteTest(ITestExecutionContext context, int executionNumb testTags.EarlyFlakeDetectionTestAbortReason = "slow"; } - if (retryState.IsLastRetry) + // Determine if this is the final execution + var isAtrBehavior = retryState.BehaviorType == typeof(FlakyRetryBehavior); + var isAtrEarlyExit = isAtrBehavior && + resultStatus == TestStatus.Passed && + !retryState.IsLastRetry; + + // Pre-compute ATR budget exhaustion + var isAtrBudgetExhausted = false; + if (isAtrBehavior && resultStatus == TestStatus.Failed && !retryState.IsLastRetry) { - if (attemptToFixRetryBehavior) - { - testTags.AttemptToFixPassed = retryState.AllAttemptsPassed ? "true" : "false"; - } + var remainingBudget = FlakyRetryBehavior.GetRemainingBudget(); + // This pre-close check runs before ShouldRetry() decrements the ATR budget. + // If budget is 1 now, the next decrement reaches 0, so no further retry will run. + isAtrBudgetExhausted = remainingBudget <= 1; + } + + var isFinalExecution = retryState.IsLastRetry || isAtrEarlyExit || isAtrBudgetExhausted; + if (isFinalExecution) + { if (retryState.AllRetriesFailed) { testTags.HasFailedAllRetries = "true"; } + + // Set final_status on the final retry + var anyExecutionPassed = retryState.InitialExecutionPassed || retryState.AnyRetryPassed; + // For ATF: any actual failure (initial or retry) means the fix didn't work (test is still flaky) + // Note: skip/inconclusive does NOT count as failure per ATF semantics + var anyExecutionFailed = retryState.InitialExecutionFailed || !retryState.AllAttemptsPassed; + var isSkippedOrInconclusive = resultStatus is TestStatus.Skipped or TestStatus.Inconclusive; + testTags.FinalStatus = Common.CalculateFinalStatus(anyExecutionPassed, anyExecutionFailed, isSkippedOrInconclusive, testTags); + + // ATF: AttemptToFixPassed should be consistent with final_status + // If any execution failed, the fix didn't work + if (attemptToFixRetryBehavior) + { + testTags.AttemptToFixPassed = anyExecutionFailed ? "false" : "true"; + } } NUnitIntegration.FinishTest(test, testResult); @@ -223,10 +298,17 @@ private ITestResult DoRetries(in TBehavior behavior, ITestExecutionCo var remainingRetries = behavior.RemainingRetries; var retryNumber = 0; var totalRetries = remainingRetries; + // Initialize InitialExecutionPassed/Failed from the initial execution result + // CRITICAL: Only PASS counts as passed, only FAIL counts as failed (not skip) + var initialStatus = result.ResultState.Status; + var initialPassed = initialStatus == TestStatus.Passed; + var initialFailed = initialStatus == TestStatus.Failed; var retryState = new RetryState { IsARetry = true, - BehaviorType = typeof(AttemptToFixRetryBehavior) + BehaviorType = typeof(TBehavior), + InitialExecutionPassed = initialPassed, + InitialExecutionFailed = initialFailed }; while (remainingRetries-- > 0) { @@ -259,6 +341,9 @@ private ref struct RetryState public bool IsLastRetry; public bool AllAttemptsPassed; public bool AllRetriesFailed; + public bool InitialExecutionPassed; + public bool InitialExecutionFailed; + public bool AnyRetryPassed; public Type? BehaviorType; public RetryState() @@ -267,6 +352,9 @@ public RetryState() IsLastRetry = false; AllAttemptsPassed = true; AllRetriesFailed = true; + InitialExecutionPassed = false; + InitialExecutionFailed = false; + AnyRetryPassed = false; BehaviorType = null; } } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/TestCaseMetadata.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/TestCaseMetadata.cs index 1e1c5f416663..83437559a4d8 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/TestCaseMetadata.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/TestCaseMetadata.cs @@ -47,5 +47,22 @@ public TestCaseMetadata(string uniqueID, int totalExecution, int countDownExecut public bool HasAnException { get; set; } = false; + /// + /// Gets or sets a value indicating whether the initial execution passed. Only PASS counts as passed, not SKIP. + /// + public bool InitialExecutionPassed { get; set; } = false; + + /// + /// Gets or sets a value indicating whether the initial execution failed. Only actual FAIL counts, not SKIP. + /// Used for ATF final_status calculation. + /// + public bool InitialExecutionFailed { get; set; } = false; + + /// + /// Gets or sets a value indicating whether any retry execution passed. Only PASS counts as passed, not SKIP. + /// Used for final_status calculation (distinct from AllRetriesFailed which clears on pass OR skip). + /// + public bool AnyRetryPassed { get; set; } = false; + public string UniqueID { get; } } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/V3/XUnitTestMethodRunnerBaseRunTestCaseV3Integration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/V3/XUnitTestMethodRunnerBaseRunTestCaseV3Integration.cs index e67c7158bb20..d79f316cf7f7 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/V3/XUnitTestMethodRunnerBaseRunTestCaseV3Integration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/V3/XUnitTestMethodRunnerBaseRunTestCaseV3Integration.cs @@ -325,6 +325,15 @@ internal static async Task OnAsyncMethodEnd(TTarget i return returnValue; } + /// + /// Read-only snapshot of remaining ATR budget for pre-close checks (XUnit v3). + /// Value meanings: -1 = uninitialized, 0 = exhausted, positive = nominally available. + /// This value is observed before retry scheduling decrements budget, so values of 1 or 0 mean no + /// further retry can run after the current failed execution. + /// + internal static int GetRemainingAtrBudget() + => Interlocked.CompareExchange(ref _totalRetries, 0, 0); + private readonly struct TestRunnerState { public readonly RetryMessageBus MessageBus; diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/XUnitIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/XUnitIntegration.cs index 8a565568fab4..dfd9f48d67ea 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/XUnitIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/XUnitIntegration.cs @@ -131,7 +131,7 @@ internal static class XUnitIntegration if (testIsNew && testCaseMetadata.ExecutionIndex > 0) { testTags.TestIsRetry = "true"; - testTags.TestRetryReason = "efd"; + testTags.TestRetryReason = TestTags.TestRetryReasonEfd; } Common.CheckFaultyThreshold(test, Interlocked.Read(ref _newTestCases), Interlocked.Read(ref _totalTestCases)); @@ -162,6 +162,8 @@ internal static class XUnitIntegration // Skip tests if (runnerInstance.SkipReason is { } skipReason) { + // Set final_status = skip for pre-execution skipped tests (ITR/attribute-based skips) + testTags.FinalStatus = TestTags.StatusSkip; test.Close(TestStatus.Skip, skipReason: skipReason, duration: TimeSpan.Zero); return null; } @@ -204,7 +206,20 @@ internal static void FinishTest(Test test, IExceptionAggregator? exceptionAggreg testCaseMetadata.AllRetriesFailed = false; } - WriteFinalTagsFromMetadata(test, testCaseMetadata); + // Set Skipped flag for dynamic skips + if (testCaseMetadata is not null) + { + testCaseMetadata.Skipped = true; + } + + WriteFinalTagsFromMetadata(test, testCaseMetadata, isSkip: true); + + // Handle null metadata - set final_status for tests without retry features + if (testCaseMetadata is null && test.GetTags() is { } nullMetaSkipTags) + { + nullMetaSkipTags.FinalStatus = TestTags.StatusSkip; + } + var skipReason = exception.Message.Replace("$XunitDynamicSkip$", string.Empty); test.Close(TestStatus.Skip, TimeSpan.Zero, skipReason); } @@ -213,13 +228,27 @@ internal static void FinishTest(Test test, IExceptionAggregator? exceptionAggreg if (testCaseMetadata != null) { testCaseMetadata.HasAnException = true; + // ATF: AllAttemptsPassed clears only on actual failure (not skip) if (testCaseMetadata.IsAttemptToFix) { testCaseMetadata.AllAttemptsPassed = false; } + + // Track initial execution failure for ATF final_status + if (testCaseMetadata.ExecutionIndex == 0) + { + testCaseMetadata.InitialExecutionFailed = true; + } + } + + WriteFinalTagsFromMetadata(test, testCaseMetadata, isSkip: false); + + // Handle null metadata - set final_status for tests without retry features + if (testCaseMetadata is null && test.GetTags() is { } nullMetaFailTags) + { + nullMetaFailTags.FinalStatus = TestTags.StatusFail; } - WriteFinalTagsFromMetadata(test, testCaseMetadata); if (Common.Log.IsEnabled(LogEventLevel.Debug)) { var span = Tracer.Instance.ActiveScope?.Span; @@ -233,12 +262,35 @@ internal static void FinishTest(Test test, IExceptionAggregator? exceptionAggreg } else { - if (testCaseMetadata?.TotalExecutions > 1) + // Test passed + if (testCaseMetadata is not null) + { + // Track pass status for final_status calculation + if (testCaseMetadata.ExecutionIndex == 0) + { + // Initial execution passed + testCaseMetadata.InitialExecutionPassed = true; + } + else + { + // Retry execution passed + testCaseMetadata.AnyRetryPassed = true; + } + + if (testCaseMetadata.TotalExecutions > 1) + { + testCaseMetadata.AllRetriesFailed = false; + } + } + + WriteFinalTagsFromMetadata(test, testCaseMetadata, isSkip: false); + + // Handle null metadata - set final_status for tests without retry features + if (testCaseMetadata is null && test.GetTags() is { } nullMetaPassTags) { - testCaseMetadata.AllRetriesFailed = false; + nullMetaPassTags.FinalStatus = TestTags.StatusPass; } - WriteFinalTagsFromMetadata(test, testCaseMetadata); test.Close(TestStatus.Pass, duration); } } @@ -256,7 +308,7 @@ internal static void FinishTest(Test test, IExceptionAggregator? exceptionAggreg } } - private static void WriteFinalTagsFromMetadata(Test test, TestCaseMetadata? testCaseMetadata) + private static void WriteFinalTagsFromMetadata(Test test, TestCaseMetadata? testCaseMetadata, bool isSkip) { if (testCaseMetadata == null) { @@ -264,19 +316,79 @@ private static void WriteFinalTagsFromMetadata(Test test, TestCaseMetadata? test } var tags = test.GetTags(); - if (!testCaseMetadata.IsLastRetry) + + // Per-span guard to prevent duplicate setting + if (tags.FinalStatus is not null) + { + return; + } + + // Determine if this is a "final execution" for final_status calculation + // XUnit has a timing issue: TotalExecutions is stale during initial EFD execution + // (it's updated in OnAsyncMethodEnd AFTER FinishTest). Use guards to handle this. + var isInitialEfdOrAtfExecution = testCaseMetadata.ExecutionIndex == 0 && + (testCaseMetadata.EarlyFlakeDetectionEnabled || testCaseMetadata.IsAttemptToFix); + + // Check if EFD/ATF will retry (even if TotalExecutions is not yet set) + // If EFD is enabled for a new test, retries will happen regardless of initial result + // If ATF is enabled, retries will happen regardless of initial result + var willHaveRetries = isInitialEfdOrAtfExecution && testCaseMetadata.TotalExecutions <= 1; + + // ATR early exit detection + var isAtrRetry = testCaseMetadata.IsRetry && + tags.TestRetryReason == TestTags.TestRetryReasonAtr; + var isAtrEarlyExit = isAtrRetry && + testCaseMetadata is { HasAnException: false, Skipped: false, IsLastRetry: false }; + + // ATR budget exhaustion detection (Edge Case 23) + var isAtrBudgetExhausted = false; + if (isAtrRetry && testCaseMetadata is { HasAnException: true, IsLastRetry: false }) + { + var remainingBudget = GetRemainingAtrBudget(); + // This pre-close check runs before the retry scheduler decrements budget. + // If budget is 1 now, the next decrement reaches 0, so no further retry will run. + isAtrBudgetExhausted = remainingBudget <= 1; + } + + // Single-execution test: TotalExecutions == 1 means no retries were scheduled + var isSingleExecution = testCaseMetadata.TotalExecutions == 1 && !willHaveRetries; + + var isFinalExecution = testCaseMetadata.IsLastRetry || isSingleExecution || isAtrEarlyExit || isAtrBudgetExhausted; + + if (!isFinalExecution) { return; } - if (testCaseMetadata.IsAttemptToFix) + // Only set retry-specific tags for tests with actual retries + if (testCaseMetadata.TotalExecutions > 1) { - tags.AttemptToFixPassed = testCaseMetadata.AllAttemptsPassed ? "true" : "false"; + if (testCaseMetadata.AllRetriesFailed) + { + tags.HasFailedAllRetries = "true"; + } } - if (testCaseMetadata.AllRetriesFailed) + // Calculate final_status + // For single-execution tests, use HasAnException to determine pass/fail + var anyExecutionPassed = testCaseMetadata.TotalExecutions == 1 + ? testCaseMetadata is { HasAnException: false, Skipped: false } // Single: no exception and not skipped = passed + : testCaseMetadata.InitialExecutionPassed || testCaseMetadata.AnyRetryPassed; // Retry: tracked values + + // For ATF: any actual failure (initial or retry) means the fix didn't work (test is still flaky) + // Note: skip does NOT count as failure per ATF semantics + var anyExecutionFailed = testCaseMetadata.TotalExecutions == 1 + ? testCaseMetadata is { HasAnException: true, Skipped: false } // Single: exception and not skip = failed + : testCaseMetadata.InitialExecutionFailed || !testCaseMetadata.AllAttemptsPassed; // Retry: initial failed OR any retry failed + + var isSkippedOrInconclusive = isSkip || testCaseMetadata.Skipped; + tags.FinalStatus = Common.CalculateFinalStatus(anyExecutionPassed, anyExecutionFailed, isSkippedOrInconclusive, tags); + + // ATF: AttemptToFixPassed should be consistent with final_status + // If any execution failed, the fix didn't work + if (testCaseMetadata is { TotalExecutions: > 1, IsAttemptToFix: true }) { - tags.HasFailedAllRetries = "true"; + tags.AttemptToFixPassed = anyExecutionFailed ? "false" : "true"; } } @@ -317,4 +429,18 @@ internal static void IncrementTotalTestCases() { Interlocked.Increment(ref _totalTestCases); } + + /// + /// Unified read-only snapshot of remaining ATR budget for pre-close checks. + /// Uses Math.Max to handle both v2 and v3 scenarios (they have separate counters). + /// Value meanings: -1 = uninitialized, 0 = exhausted, positive = nominally available. + /// This value is observed before retry scheduling decrements the budget, so values of 1 or 0 mean + /// the current failed execution is the last one before exhaustion. + /// + internal static int GetRemainingAtrBudget() + { + var v2Budget = XUnitTestRunnerRunAsyncIntegration.GetRemainingAtrBudget(); + var v3Budget = V3.XUnitTestMethodRunnerBaseRunTestCaseV3Integration.GetRemainingAtrBudget(); + return Math.Max(v2Budget, v3Budget); + } } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/XUnitTestRunnerRunAsyncIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/XUnitTestRunnerRunAsyncIntegration.cs index 087ca2b93f33..b2e3a9156e3f 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/XUnitTestRunnerRunAsyncIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Testing/XUnit/XUnitTestRunnerRunAsyncIntegration.cs @@ -338,6 +338,15 @@ internal static async Task OnAsyncMethodEnd(TTarget i return returnValue; } + /// + /// Read-only snapshot of remaining ATR budget for pre-close checks (XUnit v2). + /// Value meanings: -1 = uninitialized, 0 = exhausted, positive = nominally available. + /// This value is observed before retry scheduling decrements budget, so values of 1 or 0 mean no + /// further retry can run after the current failed execution. + /// + internal static int GetRemainingAtrBudget() + => Interlocked.CompareExchange(ref _totalRetries, 0, 0); + private readonly struct TestRunnerState { public readonly DateTimeOffset StartTime; diff --git a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs index 8e238fb9ec83..2a7c42ee1cbc 100644 --- a/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs @@ -151,6 +151,12 @@ partial class TestSpanTags private static ReadOnlySpan AttemptToFixPassedBytes => new byte[] { 217, 42, 116, 101, 115, 116, 46, 116, 101, 115, 116, 95, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 116, 116, 101, 109, 112, 116, 95, 116, 111, 95, 102, 105, 120, 95, 112, 97, 115, 115, 101, 100 }; #else private static readonly byte[] AttemptToFixPassedBytes = new byte[] { 217, 42, 116, 101, 115, 116, 46, 116, 101, 115, 116, 95, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 116, 116, 101, 109, 112, 116, 95, 116, 111, 95, 102, 105, 120, 95, 112, 97, 115, 115, 101, 100 }; +#endif + // FinalStatusBytes = MessagePack.Serialize("test.final_status"); +#if NETCOREAPP + private static ReadOnlySpan FinalStatusBytes => new byte[] { 177, 116, 101, 115, 116, 46, 102, 105, 110, 97, 108, 95, 115, 116, 97, 116, 117, 115 }; +#else + private static readonly byte[] FinalStatusBytes = new byte[] { 177, 116, 101, 115, 116, 46, 102, 105, 110, 97, 108, 95, 115, 116, 97, 116, 117, 115 }; #endif // CapabilitiesTestImpactAnalysisBytes = MessagePack.Serialize("_dd.library_capabilities.test_impact_analysis"); #if NETCOREAPP @@ -214,6 +220,7 @@ partial class TestSpanTags "test.test_management.is_attempt_to_fix" => IsAttemptToFix, "test.has_failed_all_retries" => HasFailedAllRetries, "test.test_management.attempt_to_fix_passed" => AttemptToFixPassed, + "test.final_status" => FinalStatus, "_dd.library_capabilities.test_impact_analysis" => CapabilitiesTestImpactAnalysis, "_dd.library_capabilities.early_flake_detection" => CapabilitiesEarlyFlakeDetection, "_dd.library_capabilities.auto_test_retries" => CapabilitiesAutoTestRetries, @@ -291,6 +298,9 @@ public override void SetTag(string key, string? value) case "test.test_management.attempt_to_fix_passed": AttemptToFixPassed = value; break; + case "test.final_status": + FinalStatus = value; + break; case "_dd.library_capabilities.test_impact_analysis": CapabilitiesTestImpactAnalysis = value; break; @@ -422,6 +432,11 @@ public override void EnumerateTags(ref TProcessor processor) processor.Process(new TagItem("test.test_management.attempt_to_fix_passed", AttemptToFixPassed, AttemptToFixPassedBytes)); } + if (FinalStatus is not null) + { + processor.Process(new TagItem("test.final_status", FinalStatus, FinalStatusBytes)); + } + if (CapabilitiesTestImpactAnalysis is not null) { processor.Process(new TagItem("_dd.library_capabilities.test_impact_analysis", CapabilitiesTestImpactAnalysis, CapabilitiesTestImpactAnalysisBytes)); @@ -604,6 +619,13 @@ protected override void WriteAdditionalTags(System.Text.StringBuilder sb) .Append(','); } + if (FinalStatus is not null) + { + sb.Append("test.final_status (tag):") + .Append(FinalStatus) + .Append(','); + } + if (CapabilitiesTestImpactAnalysis is not null) { sb.Append("_dd.library_capabilities.test_impact_analysis (tag):") diff --git a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs index 8e238fb9ec83..2a7c42ee1cbc 100644 --- a/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/net6.0/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs @@ -151,6 +151,12 @@ partial class TestSpanTags private static ReadOnlySpan AttemptToFixPassedBytes => new byte[] { 217, 42, 116, 101, 115, 116, 46, 116, 101, 115, 116, 95, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 116, 116, 101, 109, 112, 116, 95, 116, 111, 95, 102, 105, 120, 95, 112, 97, 115, 115, 101, 100 }; #else private static readonly byte[] AttemptToFixPassedBytes = new byte[] { 217, 42, 116, 101, 115, 116, 46, 116, 101, 115, 116, 95, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 116, 116, 101, 109, 112, 116, 95, 116, 111, 95, 102, 105, 120, 95, 112, 97, 115, 115, 101, 100 }; +#endif + // FinalStatusBytes = MessagePack.Serialize("test.final_status"); +#if NETCOREAPP + private static ReadOnlySpan FinalStatusBytes => new byte[] { 177, 116, 101, 115, 116, 46, 102, 105, 110, 97, 108, 95, 115, 116, 97, 116, 117, 115 }; +#else + private static readonly byte[] FinalStatusBytes = new byte[] { 177, 116, 101, 115, 116, 46, 102, 105, 110, 97, 108, 95, 115, 116, 97, 116, 117, 115 }; #endif // CapabilitiesTestImpactAnalysisBytes = MessagePack.Serialize("_dd.library_capabilities.test_impact_analysis"); #if NETCOREAPP @@ -214,6 +220,7 @@ partial class TestSpanTags "test.test_management.is_attempt_to_fix" => IsAttemptToFix, "test.has_failed_all_retries" => HasFailedAllRetries, "test.test_management.attempt_to_fix_passed" => AttemptToFixPassed, + "test.final_status" => FinalStatus, "_dd.library_capabilities.test_impact_analysis" => CapabilitiesTestImpactAnalysis, "_dd.library_capabilities.early_flake_detection" => CapabilitiesEarlyFlakeDetection, "_dd.library_capabilities.auto_test_retries" => CapabilitiesAutoTestRetries, @@ -291,6 +298,9 @@ public override void SetTag(string key, string? value) case "test.test_management.attempt_to_fix_passed": AttemptToFixPassed = value; break; + case "test.final_status": + FinalStatus = value; + break; case "_dd.library_capabilities.test_impact_analysis": CapabilitiesTestImpactAnalysis = value; break; @@ -422,6 +432,11 @@ public override void EnumerateTags(ref TProcessor processor) processor.Process(new TagItem("test.test_management.attempt_to_fix_passed", AttemptToFixPassed, AttemptToFixPassedBytes)); } + if (FinalStatus is not null) + { + processor.Process(new TagItem("test.final_status", FinalStatus, FinalStatusBytes)); + } + if (CapabilitiesTestImpactAnalysis is not null) { processor.Process(new TagItem("_dd.library_capabilities.test_impact_analysis", CapabilitiesTestImpactAnalysis, CapabilitiesTestImpactAnalysisBytes)); @@ -604,6 +619,13 @@ protected override void WriteAdditionalTags(System.Text.StringBuilder sb) .Append(','); } + if (FinalStatus is not null) + { + sb.Append("test.final_status (tag):") + .Append(FinalStatus) + .Append(','); + } + if (CapabilitiesTestImpactAnalysis is not null) { sb.Append("_dd.library_capabilities.test_impact_analysis (tag):") diff --git a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs index 8e238fb9ec83..2a7c42ee1cbc 100644 --- a/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netcoreapp3.1/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs @@ -151,6 +151,12 @@ partial class TestSpanTags private static ReadOnlySpan AttemptToFixPassedBytes => new byte[] { 217, 42, 116, 101, 115, 116, 46, 116, 101, 115, 116, 95, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 116, 116, 101, 109, 112, 116, 95, 116, 111, 95, 102, 105, 120, 95, 112, 97, 115, 115, 101, 100 }; #else private static readonly byte[] AttemptToFixPassedBytes = new byte[] { 217, 42, 116, 101, 115, 116, 46, 116, 101, 115, 116, 95, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 116, 116, 101, 109, 112, 116, 95, 116, 111, 95, 102, 105, 120, 95, 112, 97, 115, 115, 101, 100 }; +#endif + // FinalStatusBytes = MessagePack.Serialize("test.final_status"); +#if NETCOREAPP + private static ReadOnlySpan FinalStatusBytes => new byte[] { 177, 116, 101, 115, 116, 46, 102, 105, 110, 97, 108, 95, 115, 116, 97, 116, 117, 115 }; +#else + private static readonly byte[] FinalStatusBytes = new byte[] { 177, 116, 101, 115, 116, 46, 102, 105, 110, 97, 108, 95, 115, 116, 97, 116, 117, 115 }; #endif // CapabilitiesTestImpactAnalysisBytes = MessagePack.Serialize("_dd.library_capabilities.test_impact_analysis"); #if NETCOREAPP @@ -214,6 +220,7 @@ partial class TestSpanTags "test.test_management.is_attempt_to_fix" => IsAttemptToFix, "test.has_failed_all_retries" => HasFailedAllRetries, "test.test_management.attempt_to_fix_passed" => AttemptToFixPassed, + "test.final_status" => FinalStatus, "_dd.library_capabilities.test_impact_analysis" => CapabilitiesTestImpactAnalysis, "_dd.library_capabilities.early_flake_detection" => CapabilitiesEarlyFlakeDetection, "_dd.library_capabilities.auto_test_retries" => CapabilitiesAutoTestRetries, @@ -291,6 +298,9 @@ public override void SetTag(string key, string? value) case "test.test_management.attempt_to_fix_passed": AttemptToFixPassed = value; break; + case "test.final_status": + FinalStatus = value; + break; case "_dd.library_capabilities.test_impact_analysis": CapabilitiesTestImpactAnalysis = value; break; @@ -422,6 +432,11 @@ public override void EnumerateTags(ref TProcessor processor) processor.Process(new TagItem("test.test_management.attempt_to_fix_passed", AttemptToFixPassed, AttemptToFixPassedBytes)); } + if (FinalStatus is not null) + { + processor.Process(new TagItem("test.final_status", FinalStatus, FinalStatusBytes)); + } + if (CapabilitiesTestImpactAnalysis is not null) { processor.Process(new TagItem("_dd.library_capabilities.test_impact_analysis", CapabilitiesTestImpactAnalysis, CapabilitiesTestImpactAnalysisBytes)); @@ -604,6 +619,13 @@ protected override void WriteAdditionalTags(System.Text.StringBuilder sb) .Append(','); } + if (FinalStatus is not null) + { + sb.Append("test.final_status (tag):") + .Append(FinalStatus) + .Append(','); + } + if (CapabilitiesTestImpactAnalysis is not null) { sb.Append("_dd.library_capabilities.test_impact_analysis (tag):") diff --git a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs index 8e238fb9ec83..2a7c42ee1cbc 100644 --- a/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs +++ b/tracer/src/Datadog.Trace/Generated/netstandard2.0/Datadog.Trace.SourceGenerators/TagListGenerator/TestSpanTags.g.cs @@ -151,6 +151,12 @@ partial class TestSpanTags private static ReadOnlySpan AttemptToFixPassedBytes => new byte[] { 217, 42, 116, 101, 115, 116, 46, 116, 101, 115, 116, 95, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 116, 116, 101, 109, 112, 116, 95, 116, 111, 95, 102, 105, 120, 95, 112, 97, 115, 115, 101, 100 }; #else private static readonly byte[] AttemptToFixPassedBytes = new byte[] { 217, 42, 116, 101, 115, 116, 46, 116, 101, 115, 116, 95, 109, 97, 110, 97, 103, 101, 109, 101, 110, 116, 46, 97, 116, 116, 101, 109, 112, 116, 95, 116, 111, 95, 102, 105, 120, 95, 112, 97, 115, 115, 101, 100 }; +#endif + // FinalStatusBytes = MessagePack.Serialize("test.final_status"); +#if NETCOREAPP + private static ReadOnlySpan FinalStatusBytes => new byte[] { 177, 116, 101, 115, 116, 46, 102, 105, 110, 97, 108, 95, 115, 116, 97, 116, 117, 115 }; +#else + private static readonly byte[] FinalStatusBytes = new byte[] { 177, 116, 101, 115, 116, 46, 102, 105, 110, 97, 108, 95, 115, 116, 97, 116, 117, 115 }; #endif // CapabilitiesTestImpactAnalysisBytes = MessagePack.Serialize("_dd.library_capabilities.test_impact_analysis"); #if NETCOREAPP @@ -214,6 +220,7 @@ partial class TestSpanTags "test.test_management.is_attempt_to_fix" => IsAttemptToFix, "test.has_failed_all_retries" => HasFailedAllRetries, "test.test_management.attempt_to_fix_passed" => AttemptToFixPassed, + "test.final_status" => FinalStatus, "_dd.library_capabilities.test_impact_analysis" => CapabilitiesTestImpactAnalysis, "_dd.library_capabilities.early_flake_detection" => CapabilitiesEarlyFlakeDetection, "_dd.library_capabilities.auto_test_retries" => CapabilitiesAutoTestRetries, @@ -291,6 +298,9 @@ public override void SetTag(string key, string? value) case "test.test_management.attempt_to_fix_passed": AttemptToFixPassed = value; break; + case "test.final_status": + FinalStatus = value; + break; case "_dd.library_capabilities.test_impact_analysis": CapabilitiesTestImpactAnalysis = value; break; @@ -422,6 +432,11 @@ public override void EnumerateTags(ref TProcessor processor) processor.Process(new TagItem("test.test_management.attempt_to_fix_passed", AttemptToFixPassed, AttemptToFixPassedBytes)); } + if (FinalStatus is not null) + { + processor.Process(new TagItem("test.final_status", FinalStatus, FinalStatusBytes)); + } + if (CapabilitiesTestImpactAnalysis is not null) { processor.Process(new TagItem("_dd.library_capabilities.test_impact_analysis", CapabilitiesTestImpactAnalysis, CapabilitiesTestImpactAnalysisBytes)); @@ -604,6 +619,13 @@ protected override void WriteAdditionalTags(System.Text.StringBuilder sb) .Append(','); } + if (FinalStatus is not null) + { + sb.Append("test.final_status (tag):") + .Append(FinalStatus) + .Append(','); + } + if (CapabilitiesTestImpactAnalysis is not null) { sb.Append("_dd.library_capabilities.test_impact_analysis (tag):") diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/MsTestV2EvpTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/MsTestV2EvpTests.cs index 1818ee5d98c3..d9f33aec3f60 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/MsTestV2EvpTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/MsTestV2EvpTests.cs @@ -378,6 +378,9 @@ await Verifier.Verify( targetTest.Meta.Remove(TestTags.TestIsNew); targetTest.Meta.Remove(TestTags.TestIsRetry); + // Remove test final status + targetTest.Meta.Remove(TestTags.TestFinalStatus); + // Remove capabilities targetTest.Meta.Remove(CapabilitiesTags.LibraryCapabilitiesAutoTestRetries); targetTest.Meta.Remove(CapabilitiesTags.LibraryCapabilitiesTestManagementQuarantine); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/MsTestV2Tests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/MsTestV2Tests.cs index 41ea731b5887..2a73d67eb252 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/MsTestV2Tests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/MsTestV2Tests.cs @@ -104,6 +104,9 @@ public async Task SubmitTraces(string packageVersion) targetSpan.Tags.Remove(TestTags.TestIsNew); targetSpan.Tags.Remove(TestTags.TestIsRetry); + // Remove test final status + targetSpan.Tags.Remove(TestTags.TestFinalStatus); + // Remove capabilities targetSpan.Tags.Remove(CapabilitiesTags.LibraryCapabilitiesAutoTestRetries); targetSpan.Tags.Remove(CapabilitiesTags.LibraryCapabilitiesTestManagementQuarantine); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/NUnitEvpTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/NUnitEvpTests.cs index b4e0e8980d50..9ddce83cacc4 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/NUnitEvpTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/NUnitEvpTests.cs @@ -406,6 +406,9 @@ public async Task SubmitTraces(string packageVersion, string evpVersionToRemove, targetTest.Meta.Remove(TestTags.TestIsNew); targetTest.Meta.Remove(TestTags.TestIsRetry); + // Remove test final status + targetTest.Meta.Remove(TestTags.TestFinalStatus); + // Remove capabilities targetTest.Meta.Remove(CapabilitiesTags.LibraryCapabilitiesAutoTestRetries); targetTest.Meta.Remove(CapabilitiesTags.LibraryCapabilitiesTestManagementQuarantine); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/NUnitTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/NUnitTests.cs index 4e8f2d5f0174..c495d702a6c4 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/NUnitTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/NUnitTests.cs @@ -98,6 +98,9 @@ public async Task SubmitTraces(string packageVersion) targetSpan.Tags.Remove(TestTags.TestIsNew); targetSpan.Tags.Remove(TestTags.TestIsRetry); + // Remove test final status + targetSpan.Tags.Remove(TestTags.TestFinalStatus); + // Remove capabilities targetSpan.Tags.Remove(CapabilitiesTags.LibraryCapabilitiesAutoTestRetries); targetSpan.Tags.Remove(CapabilitiesTags.LibraryCapabilitiesTestManagementQuarantine); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkEvpTest.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkEvpTest.cs index ea9378a36d5c..b1b47c65d4cf 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkEvpTest.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/TestingFrameworkEvpTest.cs @@ -534,6 +534,7 @@ await Verifier.Verify( .ThenBy(s => GetValueOrDefault(s.Meta, TestTags.TestIsRetry)) .ThenBy(s => GetValueOrDefault(s.Meta, TestTags.TestAttemptToFixPassed)) .ThenBy(s => GetValueOrDefault(s.Meta, TestTags.TestHasFailedAllRetries)) + .ThenBy(s => GetValueOrDefault(s.Meta, TestTags.TestFinalStatus)) .ThenBy(s => GetValueOrDefault(s.Meta, EarlyFlakeDetectionTags.AbortReason)), settings); } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTests.cs index b594ba2bfef7..2c1ff1366c87 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTests.cs @@ -381,6 +381,9 @@ public virtual async Task SubmitTraces(string packageVersion, string evpVersionT targetTest.Meta.Remove(TestTags.TestIsNew); targetTest.Meta.Remove(TestTags.TestIsRetry); + // Remove test final status + targetTest.Meta.Remove(TestTags.TestFinalStatus); + // Remove capabilities targetTest.Meta.Remove(CapabilitiesTags.LibraryCapabilitiesAutoTestRetries); targetTest.Meta.Remove(CapabilitiesTags.LibraryCapabilitiesTestManagementQuarantine); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTestsV3.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTestsV3.cs index fbb06f422cce..54ef9b3098af 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTestsV3.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitEvpTestsV3.cs @@ -387,6 +387,9 @@ public async Task SubmitTraces(string packageVersion, string evpVersionToRemove, targetTest.Meta.Remove(TestTags.TestIsNew); targetTest.Meta.Remove(TestTags.TestIsRetry); + // Remove test final status + targetTest.Meta.Remove(TestTags.TestFinalStatus); + // Remove capabilities targetTest.Meta.Remove(CapabilitiesTags.LibraryCapabilitiesAutoTestRetries); targetTest.Meta.Remove(CapabilitiesTags.LibraryCapabilitiesTestManagementQuarantine); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitTests.cs index b90ed8384df6..b132000c0800 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/CI/XUnitTests.cs @@ -72,6 +72,9 @@ public virtual async Task SubmitTraces(string packageVersion) targetSpan.Tags.Remove(TestTags.TestIsNew); targetSpan.Tags.Remove(TestTags.TestIsRetry); + // Remove test final status + targetSpan.Tags.Remove(TestTags.TestFinalStatus); + // Remove capabilities targetSpan.Tags.Remove(CapabilitiesTags.LibraryCapabilitiesAutoTestRetries); targetSpan.Tags.Remove(CapabilitiesTags.LibraryCapabilitiesTestManagementQuarantine); diff --git a/tracer/test/Datadog.Trace.Tests/Ci/TestFinalStatusTests.cs b/tracer/test/Datadog.Trace.Tests/Ci/TestFinalStatusTests.cs new file mode 100644 index 000000000000..412cf688cf1c --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Ci/TestFinalStatusTests.cs @@ -0,0 +1,1041 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using Datadog.Trace.Ci.Tagging; +using Datadog.Trace.Ci.Tags; +using Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Ci; + +/// +/// Unit tests for the test.final_status tag implementation. +/// Tests the CalculateFinalStatus helper and related functionality. +/// +public class TestFinalStatusTests +{ + // CalculateFinalStatus Helper Tests + + [Fact] + public void CalculateFinalStatus_NullTags_AnyPassed_ReturnsPass() + { + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: null); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_NullTags_NotPassed_NotSkip_ReturnsFail() + { + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: null); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_NullTags_NotPassed_Skip_ReturnsSkip() + { + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: null); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_NullTags_Passed_Skip_ReturnsPass() + { + // Pass takes precedence over skip + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: null); + result.Should().Be(TestTags.StatusPass); + } + + // Priority 1: Quarantined/Disabled Tests + + [Fact] + public void CalculateFinalStatus_Quarantined_AnyPassed_ReturnsSkip() + { + var tags = new TestSpanTags { IsQuarantined = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "quarantined tests always return skip regardless of actual result"); + } + + [Fact] + public void CalculateFinalStatus_Quarantined_AllFailed_ReturnsSkip() + { + var tags = new TestSpanTags { IsQuarantined = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "quarantined tests always return skip regardless of actual result"); + } + + [Fact] + public void CalculateFinalStatus_Disabled_AnyPassed_ReturnsSkip() + { + var tags = new TestSpanTags { IsDisabled = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "disabled tests always return skip regardless of actual result"); + } + + [Fact] + public void CalculateFinalStatus_Disabled_AllFailed_ReturnsSkip() + { + var tags = new TestSpanTags { IsDisabled = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "disabled tests always return skip regardless of actual result"); + } + + [Fact] + public void CalculateFinalStatus_QuarantinedWithATF_Passed_ReturnsSkip() + { + // Quarantine always masks to skip, even with ATF enabled + var tags = new TestSpanTags { IsQuarantined = "true", IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "quarantined tests with ATF still return skip"); + } + + [Fact] + public void CalculateFinalStatus_QuarantinedWithATF_AllFailed_ReturnsSkip() + { + var tags = new TestSpanTags { IsQuarantined = "true", IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "quarantined tests with ATF still return skip"); + } + + [Fact] + public void CalculateFinalStatus_DisabledWithATF_Passed_ReturnsSkip() + { + var tags = new TestSpanTags { IsDisabled = "true", IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "disabled tests with ATF still return skip"); + } + + // Priority 2: For ATF - Any Execution Failed + + [Fact] + public void CalculateFinalStatus_ATF_AnyFailed_ReturnsFail() + { + // ATF: if any execution failed, the fix didn't work (test is still flaky) + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail, "ATF with any failure means fix didn't work"); + } + + [Fact] + public void CalculateFinalStatus_ATF_AllPassed_ReturnsPass() + { + // ATF: if all executions passed, the fix worked + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "ATF with all passes means fix worked"); + } + + // ATF Skip/Inconclusive Semantics - Skip does NOT count as failure + + [Fact] + public void CalculateFinalStatus_ATF_SkipDoesNotCountAsFailure_ReturnsPass() + { + // ATF: if initial passes and retry is skipped (not failed), the test is considered passing + // Skip does NOT count as failure per ATF semantics + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + // anyExecutionPassed=true (initial passed), anyExecutionFailed=false (skip doesn't count as fail) + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusPass, "ATF with skip (not fail) should pass - skip != fail"); + } + + [Fact] + public void CalculateFinalStatus_ATF_AllSkipped_ReturnsSkip() + { + // ATF: if all executions were skipped (none passed, none failed), final_status is skip + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + // anyExecutionPassed=false (none passed), anyExecutionFailed=false (skip doesn't count as fail) + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "ATF with all skips should return skip"); + } + + [Fact] + public void CalculateFinalStatus_ATF_InitialFailsRetrySkipped_ReturnsFail() + { + // ATF: if initial fails but retry is skipped, the test still failed (initial failure counts) + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + // anyExecutionPassed=false (none passed), anyExecutionFailed=true (initial failed) + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusFail, "ATF with initial failure should fail even if retry skipped"); + } + + [Fact] + public void CalculateFinalStatus_ATF_InitialSkippedRetryPasses_ReturnsPass() + { + // ATF: if initial is skipped (not failed) but retry passes, the fix worked + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + // anyExecutionPassed=true (retry passed), anyExecutionFailed=false (initial skip doesn't count as fail) + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "ATF with skip then pass should pass"); + } + + [Fact] + public void CalculateFinalStatus_ATF_InitialSkippedRetryFails_ReturnsFail() + { + // ATF: if initial is skipped but retry fails, the test failed (retry failure counts) + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + // anyExecutionPassed=false (none passed), anyExecutionFailed=true (retry failed) + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail, "ATF with skip then fail should fail"); + } + + // Priority 3: Any Execution Passed (non-ATF) + + [Fact] + public void CalculateFinalStatus_AnyPassed_NotSkip_ReturnsPass() + { + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_AnyPassed_CurrentSkip_ReturnsPass() + { + // "Pass then skip" scenario - pass takes precedence + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusPass, "pass takes precedence over skip"); + } + + // Priority 4: Skip/Inconclusive + + [Fact] + public void CalculateFinalStatus_NotPassed_Skip_ReturnsSkip() + { + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_NotPassed_Inconclusive_ReturnsSkip() + { + // Inconclusive is treated as skip + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + // Priority 5: All Failed + + [Fact] + public void CalculateFinalStatus_AllFailed_ReturnsFail() + { + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + // Single Execution Scenarios + + [Theory] + [InlineData(true, false, false, TestTags.StatusPass)] // Single pass + [InlineData(false, true, false, TestTags.StatusFail)] // Single fail + [InlineData(false, false, true, TestTags.StatusSkip)] // Single skip + public void CalculateFinalStatus_SingleExecution(bool passed, bool failed, bool skipped, string expected) + { + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: passed, anyExecutionFailed: failed, isSkippedOrInconclusive: skipped, testTags: tags); + result.Should().Be(expected); + } + + // EFD Scenarios + + [Fact] + public void CalculateFinalStatus_EFD_AllRetriesPass_ReturnsPass() + { + var tags = new TestSpanTags { TestIsNew = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_EFD_FirstFailsLaterPasses_ReturnsPass() + { + var tags = new TestSpanTags { TestIsNew = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "any pass means final status is pass (EFD is not ATF)"); + } + + [Fact] + public void CalculateFinalStatus_EFD_AllRetriesFail_ReturnsFail() + { + var tags = new TestSpanTags { TestIsNew = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_EFD_FirstPassesLaterFails_ReturnsPass() + { + // Edge Case 17: Initial passes, all retries fail -> pass (because initial passed) + var tags = new TestSpanTags { TestIsNew = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "initial execution passed, so final status is pass"); + } + + // ATR Scenarios + + [Fact] + public void CalculateFinalStatus_ATR_FirstFailsRetryPasses_ReturnsPass() + { + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_ATR_AllRetriesFail_ReturnsFail() + { + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_ATR_PassesOnLastRetry_ReturnsPass() + { + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + // ATF (Attempt to Fix) Scenarios + + [Fact] + public void CalculateFinalStatus_ATF_NonQuarantined_FirstFailsLaterPasses_ReturnsFail() + { + // ATF: if ANY execution failed, the test is still flaky, so fix didn't work + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail, "ATF with any failure means fix didn't work, test is still flaky"); + } + + [Fact] + public void CalculateFinalStatus_ATF_NonQuarantined_AllFail_ReturnsFail() + { + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_ATF_InitialPassesAllRetriesFail_ReturnsFail() + { + // ATF: even if initial passed, if any retry failed, the test is still flaky + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail, "ATF with any failure means fix didn't work"); + } + + [Fact] + public void CalculateFinalStatus_ATF_AllExecutionsPass_ReturnsPass() + { + // ATF: only if ALL executions pass, the fix worked + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "ATF with no failures means fix worked"); + } + + // ITR and Pre-execution Skip Scenarios + + [Fact] + public void CalculateFinalStatus_ITRSkipped_ReturnsSkip() + { + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_AttributeSkipped_ReturnsSkip() + { + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + // Edge Cases + + [Fact] + public void CalculateFinalStatus_IsQuarantinedFalse_NotTreatedAsQuarantined() + { + var tags = new TestSpanTags { IsQuarantined = "false" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "IsQuarantined='false' should not be treated as quarantined"); + } + + [Fact] + public void CalculateFinalStatus_IsDisabledFalse_NotTreatedAsDisabled() + { + var tags = new TestSpanTags { IsDisabled = "false" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "IsDisabled='false' should not be treated as disabled"); + } + + [Fact] + public void CalculateFinalStatus_EmptyTags_BehavesAsNormalTest() + { + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Theory] + [InlineData(true, false, false, false, false, TestTags.StatusSkip)] // Quarantined + passed + [InlineData(false, true, false, false, false, TestTags.StatusSkip)] // Disabled + passed + [InlineData(false, false, true, false, false, TestTags.StatusPass)] // Normal + passed + [InlineData(false, false, false, true, false, TestTags.StatusFail)] // Normal + failed + [InlineData(false, false, false, false, true, TestTags.StatusSkip)] // Normal + skipped + public void CalculateFinalStatus_PriorityOrder(bool quarantined, bool disabled, bool passed, bool failed, bool skipped, string expected) + { + var tags = new TestSpanTags + { + IsQuarantined = quarantined ? "true" : null, + IsDisabled = disabled ? "true" : null + }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: passed, anyExecutionFailed: failed, isSkippedOrInconclusive: skipped, testTags: tags); + result.Should().Be(expected); + } + + // Pass Then Skip Scenarios (CI Behavior Match) + + [Fact] + public void CalculateFinalStatus_PassThenSkip_ReturnsPass() + { + // CI behavior: if test passed once, pipeline passes + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusPass, "pass takes precedence over skip to match CI behavior"); + } + + [Fact] + public void CalculateFinalStatus_FailThenSkip_ReturnsSkip() + { + // No pass + skip = skip + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "no pass + skip = skip"); + } + + // Mixed Retry Reason Scenarios + + [Theory] + [InlineData(TestTags.TestRetryReasonEfd, true, false, TestTags.StatusPass)] + [InlineData(TestTags.TestRetryReasonEfd, false, true, TestTags.StatusFail)] + [InlineData(TestTags.TestRetryReasonAtr, true, false, TestTags.StatusPass)] + [InlineData(TestTags.TestRetryReasonAtr, false, true, TestTags.StatusFail)] + [InlineData(TestTags.TestRetryReasonAttemptToFix, true, false, TestTags.StatusPass)] + [InlineData(TestTags.TestRetryReasonAttemptToFix, false, true, TestTags.StatusFail)] + public void CalculateFinalStatus_RetryReasons_CorrectFinalStatus(string retryReason, bool anyPassed, bool anyFailed, string expected) + { + // Note: This test doesn't set IsAttemptToFix="true", so ATF-specific behavior won't trigger + var tags = new TestSpanTags { TestRetryReason = retryReason }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: anyPassed, anyExecutionFailed: anyFailed, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(expected); + } + + // Test Tag Existence Verification + + [Fact] + public void TestTags_TestFinalStatus_ConstantExists() + { + TestTags.TestFinalStatus.Should().Be("test.final_status"); + } + + [Fact] + public void TestTags_StatusConstants_Exist() + { + TestTags.StatusPass.Should().Be("pass"); + TestTags.StatusFail.Should().Be("fail"); + TestTags.StatusSkip.Should().Be("skip"); + } + + // Has Failed All Retries Interaction + + [Fact] + public void CalculateFinalStatus_HasFailedAllRetries_StillCalculatesCorrectly() + { + // HasFailedAllRetries is a separate tag, doesn't affect final_status calculation + var tags = new TestSpanTags { HasFailedAllRetries = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_HasFailedAllRetriesButPassed_ReturnsPass() + { + // Edge case: HasFailedAllRetries might be set incorrectly, but if anyPassed is true, should be pass + var tags = new TestSpanTags { HasFailedAllRetries = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + // Attempt to Fix Passed Interaction + + [Fact] + public void CalculateFinalStatus_AttemptToFixPassed_StillCalculatesCorrectly() + { + // AttemptToFixPassed is a separate tag, doesn't affect final_status calculation + var tags = new TestSpanTags { AttemptToFixPassed = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_AttemptToFixPassedFalse_AllFailed_ReturnsFail() + { + var tags = new TestSpanTags { AttemptToFixPassed = "false" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + // Comprehensive Scenario Matrix + + /// + /// Test all combinations of the three input parameters plus quarantined/disabled status. + /// + [Theory] + // Normal tests (not quarantined, not disabled) + [InlineData(false, false, true, false, false, TestTags.StatusPass)] // passed, not skip + [InlineData(false, false, true, false, true, TestTags.StatusPass)] // passed, skip (pass takes precedence) + [InlineData(false, false, false, true, false, TestTags.StatusFail)] // not passed, not skip, failed + [InlineData(false, false, false, false, true, TestTags.StatusSkip)] // not passed, skip + // Quarantined tests (always skip) + [InlineData(true, false, true, false, false, TestTags.StatusSkip)] // quarantined, passed + [InlineData(true, false, true, false, true, TestTags.StatusSkip)] // quarantined, passed, skip + [InlineData(true, false, false, true, false, TestTags.StatusSkip)] // quarantined, not passed + [InlineData(true, false, false, false, true, TestTags.StatusSkip)] // quarantined, not passed, skip + // Disabled tests (always skip) + [InlineData(false, true, true, false, false, TestTags.StatusSkip)] // disabled, passed + [InlineData(false, true, true, false, true, TestTags.StatusSkip)] // disabled, passed, skip + [InlineData(false, true, false, true, false, TestTags.StatusSkip)] // disabled, not passed + [InlineData(false, true, false, false, true, TestTags.StatusSkip)] // disabled, not passed, skip + public void CalculateFinalStatus_ComprehensiveMatrix( + bool isQuarantined, + bool isDisabled, + bool anyPassed, + bool anyFailed, + bool isSkip, + string expected) + { + var tags = new TestSpanTags + { + IsQuarantined = isQuarantined ? "true" : null, + IsDisabled = isDisabled ? "true" : null + }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: anyPassed, anyExecutionFailed: anyFailed, isSkippedOrInconclusive: isSkip, testTags: tags); + result.Should().Be(expected); + } + + // XUnit Dynamic Skip (SkipException) Scenarios - Tests 50-51 + + [Fact] + public void CalculateFinalStatus_DynamicSkip_SingleExecution_ReturnsSkip() + { + // Test 50: XUnit single-execution dynamic skip (SkipException) + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_DynamicSkip_LastRetry_ReturnsSkip() + { + // Test 51: XUnit retry test dynamic skip on last retry + var tags = new TestSpanTags { TestIsRetry = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + // ATR Initial Pass Scenarios - Tests 52-53 + + [Fact] + public void CalculateFinalStatus_ATR_InitialPass_ReturnsPass() + { + // Test 52: XUnit ATR enabled, initial pass -> final_status = "pass" + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_ATR_InitialFailRetryPasses_ReturnsPass() + { + // Test 53: XUnit ATR enabled, initial fail, retry passes + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + // EFD Single Execution Scenarios - Tests 54-56 + + [Theory] + [InlineData(true, false, false, TestTags.StatusPass)] // Test 54/56: EFD slow abort pass + [InlineData(false, true, false, TestTags.StatusFail)] // Test 54: EFD slow abort fail + [InlineData(false, false, true, TestTags.StatusSkip)] // EFD slow abort skip + public void CalculateFinalStatus_EFD_SingleExecution_SlowAbort(bool passed, bool failed, bool skipped, string expected) + { + var tags = new TestSpanTags { TestIsNew = "true", EarlyFlakeDetectionTestAbortReason = "slow" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: passed, anyExecutionFailed: failed, isSkippedOrInconclusive: skipped, testTags: tags); + result.Should().Be(expected); + } + + [Fact] + public void CalculateFinalStatus_ATF_SingleRetryConfigured_ReturnsCorrectStatus() + { + // Test 55: XUnit ATF with only 1 retry configured (TotalExecutions=1) + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + // ATF Framework Outcome Consistency - Tests 57-62 + + [Theory] + [InlineData(true, false, TestTags.StatusPass)] // Test 57/61/62: ATF on normal test, all pass + [InlineData(false, true, TestTags.StatusFail)] // Test 58: ATF on normal test fails all + public void CalculateFinalStatus_ATF_NormalTest_FrameworkOutcome(bool passed, bool failed, string expected) + { + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: passed, anyExecutionFailed: failed, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(expected); + } + + [Fact] + public void CalculateFinalStatus_ATF_Quarantined_Passes_ReturnsSkip() + { + // Test 59: MsTest ATF on quarantined test (passes) -> skip + var tags = new TestSpanTags { IsAttemptToFix = "true", IsQuarantined = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "quarantined always takes precedence"); + } + + [Fact] + public void CalculateFinalStatus_ATF_Disabled_Fails_ReturnsSkip() + { + // Test 60: MsTest ATF on disabled test (fails) -> skip + var tags = new TestSpanTags { IsAttemptToFix = "true", IsDisabled = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "disabled always takes precedence"); + } + + // ATR with FlakyRetryCount=0 Edge Cases - Tests 63-65 + + [Theory] + [InlineData(false, true, TestTags.StatusFail)] // Test 63: ATR FlakyRetryCount=0, initial fail + [InlineData(true, false, TestTags.StatusPass)] // Test 64: ATR FlakyRetryCount=0, initial pass + public void CalculateFinalStatus_ATR_ZeroRetries_SingleExecution(bool passed, bool failed, string expected) + { + // Treated as single-execution when FlakyRetryCount=0 + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: passed, anyExecutionFailed: failed, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(expected); + } + + [Fact] + public void CalculateFinalStatus_ATF_SingleRetry_InitialExecution_ReturnsCorrectStatus() + { + // Test 65: XUnit ATF with TestManagementAttemptToFixRetryCount=1 + var tags = new TestSpanTags { IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + // MsTest Exception Branch - Tests 66 + + [Fact] + public void CalculateFinalStatus_MsTest_ExceptionBranch_ReturnsFail() + { + // Test 66: MsTest early exception branch + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + // Parameterized Test Per-Row Scenarios - Tests 67-68 + + [Theory] + [InlineData(true, false, TestTags.StatusPass)] // Row passes + [InlineData(false, true, TestTags.StatusFail)] // Row fails + public void CalculateFinalStatus_ParameterizedTest_PerRowStatus(bool rowPassed, bool rowFailed, string expected) + { + // Tests 67-68: Each parameterized row gets its own final_status + var tags = new TestSpanTags { TestIsNew = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: rowPassed, anyExecutionFailed: rowFailed, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(expected); + } + + // Pre-execution Skip Paths - Tests 70-75 + + [Fact] + public void CalculateFinalStatus_NUnit_ITRSkipped_ReturnsSkip() + { + // Test 70: NUnit ITR-skipped test + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_NUnit_AttributeSkipped_ReturnsSkip() + { + // Test 71: NUnit attribute-skipped test + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_MsTest_Inconclusive_ReturnsSkip() + { + // Test 72: MsTest Inconclusive + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_MsTest_NotRunnable_ReturnsSkip() + { + // Test 73: MsTest NotRunnable + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_MsTest_ClassInitError_ReturnsFail() + { + // Test 74: MsTest class initialization error + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_MsTest_AssemblyInitError_ReturnsFail() + { + // Test 75: MsTest assembly initialization error + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + // EFD Single-Execution and Initial Pass Gaps - Tests 76-80 + + [Theory] + [InlineData(true, false, false, TestTags.StatusPass)] // Test 76: EFD fast test pass + [InlineData(false, true, false, TestTags.StatusFail)] // Test 76: EFD fast test fail + [InlineData(false, false, true, TestTags.StatusSkip)] // Test 77: EFD fast test skip + public void CalculateFinalStatus_EFD_FastTest_DurationBasedCount1(bool passed, bool failed, bool skipped, string expected) + { + var tags = new TestSpanTags { TestIsNew = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: passed, anyExecutionFailed: failed, isSkippedOrInconclusive: skipped, testTags: tags); + result.Should().Be(expected); + } + + [Theory] + [InlineData("MsTest")] // Test 78 + [InlineData("NUnit")] // Test 79 + [InlineData("XUnit")] // Test 80 + public void CalculateFinalStatus_EFD_InitialPassesAllRetriesFail_ReturnsPass(string framework) + { + // Tests 78-80: Initial PASSES, all 3 retries fail -> final_status = "pass" + // HasFailedAllRetries = "true" is set separately (preserves retries-only semantics) + var tags = new TestSpanTags { TestIsNew = "true", HasFailedAllRetries = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, $"{framework}: initial execution passed, so final status is pass"); + } + + // Review Round 7 Fixes - Tests 81-86 + + [Fact] + public void CalculateFinalStatus_MsTest_RetryExceptionBranch_LastRetry_ReturnsFail() + { + // Test 81: MsTest retry execution with exception (early branch) + var tags = new TestSpanTags { TestIsRetry = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_MsTest_ATREarlyExit_RetryPasses_ReturnsPass() + { + // Test 82: MsTest retry execution, retry 2 of 5 passes (ATR early exit) + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_XUnit_EFD_TotalExecutions1_ReturnsCorrectStatus() + { + // Test 83: XUnit EFD TotalExecutions=1 uses metadata + var tags = new TestSpanTags { TestIsNew = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_XUnit_NullMetadata_Pass_ReturnsPass() + { + // Test 84: XUnit null metadata pass (TestManagement disabled) + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: null); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_XUnit_NullMetadata_Fail_ReturnsFail() + { + // Test 85: XUnit null metadata fail + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: null); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_XUnit_NullMetadata_Skip_ReturnsSkip() + { + // Test 86: XUnit null metadata skip + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: false, isSkippedOrInconclusive: true, testTags: null); + result.Should().Be(TestTags.StatusSkip); + } + + // ATR Budget Exhaustion Scenarios - Tests 87-90, 95-96, 104-107 + + [Theory] + [InlineData("XUnit")] // Test 87 + [InlineData("NUnit")] // Test 88 + [InlineData("MsTest")] // Test 89 + public void CalculateFinalStatus_ATR_BudgetExhausted_MidRetry_Fail_ReturnsFail(string framework) + { + // Tests 87-89: ATR budget exhausted mid-retry (fail) + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail, $"{framework}: budget exhausted with fail -> final_status = fail"); + } + + [Fact] + public void CalculateFinalStatus_ATR_BudgetExhausted_Pass_ReturnsPass() + { + // Test 90: ATR budget exhausted, but test passed (covered by early exit) + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_NUnit_ATR_BudgetLessThanOrEqual1_IsFinal_ReturnsFail() + { + // Test 95: NUnit ATR budget exhaustion: GetRemainingBudget() <= 1 -> isFinalExecution = true + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Theory] + [InlineData("XUnit")] // Test 104 + [InlineData("MsTest")] // Test 105 + [InlineData("NUnit")] // Test 106 + public void CalculateFinalStatus_ATR_BudgetExhausted_BeforeFirstRetry_Fail_ReturnsFail(string framework) + { + // Tests 104-106: ATR budget exhausted BEFORE first retry, test fails -> final_status = "fail" on initial + var tags = new TestSpanTags(); + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail, $"{framework}: budget exhausted before retry, test fails -> final_status = fail"); + } + + [Fact] + public void CalculateFinalStatus_XUnit_ATR_BudgetDecrementedTo0_DuringRetry_ReturnsFail() + { + // Test 107: XUnit ATR budget = 1 at initial, test fails, retry 1 -> budget decremented to 0 + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + // MsTest Parameterized Per-Row Cache Scenarios - Tests 91-94 + + [Fact] + public void CalculateFinalStatus_MsTest_Parameterized_Row1Passes_Row2Fails_CorrectPerRowStatus() + { + // Test 91: MsTest parameterized (row1 passes, row2 fails) with EFD retries + // Row 1 + var row1Tags = new TestSpanTags { TestIsNew = "true" }; + var row1Result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: row1Tags); + row1Result.Should().Be(TestTags.StatusPass, "row1 passes -> final_status = pass"); + + // Row 2 + var row2Tags = new TestSpanTags { TestIsNew = "true" }; + var row2Result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: row2Tags); + row2Result.Should().Be(TestTags.StatusFail, "row2 fails -> final_status = fail"); + } + + [Fact] + public void CalculateFinalStatus_MsTest_Parameterized_Row1Fails_Row2Passes_CorrectPerRowStatus() + { + // Test 92: MsTest parameterized (row1 fails, row2 passes) with ATF retries + // Row 1 + var row1Tags = new TestSpanTags { IsAttemptToFix = "true" }; + var row1Result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: row1Tags); + row1Result.Should().Be(TestTags.StatusFail, "row1 fails -> final_status = fail"); + + // Row 2 + var row2Tags = new TestSpanTags { IsAttemptToFix = "true" }; + var row2Result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: row2Tags); + row2Result.Should().Be(TestTags.StatusPass, "row2 passes -> final_status = pass"); + } + + [Fact] + public void CalculateFinalStatus_MsTest_Parameterized_AggregateRetries_AllRowsGetFinalStatus() + { + // Tests 93-94: MsTest parameterized with aggregate retries -> all rows get final_status on last retry + var tags = new TestSpanTags { TestIsNew = "true", TestIsRetry = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + // Mixed-Feature Test Cases (EFD + ATR/ATF interactions) - Tests 98-103 + + [Fact] + public void CalculateFinalStatus_EFD_Plus_ATR_NewTestFails_ReturnsCorrectStatus() + { + // Test 98: EFD + ATR: new test fails -> EFD retries all, final_status on last EFD retry + var tags = new TestSpanTags { TestIsNew = "true", TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonEfd }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_EFD_Plus_ATF_NewTestWithATF_ReturnsCorrectStatus() + { + // Test 99: EFD + ATF: new test marked for ATF -> EFD retries take precedence + var tags = new TestSpanTags { TestIsNew = "true", IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_ATR_Plus_ATF_ATFTestFails_ReturnsCorrectStatus() + { + // Test 100: ATR + ATF: ATF test fails -> ATF retries all + var tags = new TestSpanTags { IsAttemptToFix = "true", TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAttemptToFix }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + [Fact] + public void CalculateFinalStatus_XUnit_EFD_Plus_ATR_NewTestPassesImmediately_LastRetryGetsStatus() + { + // Test 101: XUnit EFD + ATR: new test passes immediately -> EFD retries happen, final_status = "pass" on last retry + var tags = new TestSpanTags { TestIsNew = "true", TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonEfd }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_MsTest_EFD_Plus_ATF_NewTestAlsoATF_EFDBehavior() + { + // Test 102: MsTest EFD + ATF: new test also ATF -> EFD behavior (TestIsNew checked before ATF) + var tags = new TestSpanTags { TestIsNew = "true", IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass); + } + + [Fact] + public void CalculateFinalStatus_NUnit_EFD_Plus_ATR_KnownTestFails_ATRRetries() + { + // Test 103: NUnit EFD + ATR: known test fails -> ATR retries (not EFD) + var tags = new TestSpanTags { TestIsRetry = "true", TestRetryReason = TestTags.TestRetryReasonAtr }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: false, anyExecutionFailed: true, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusFail); + } + + // Quarantined + Disabled Combined Scenarios + + [Fact] + public void CalculateFinalStatus_BothQuarantinedAndDisabled_ReturnsSkip() + { + // Edge case: both flags set (shouldn't happen but test the priority) + var tags = new TestSpanTags { IsQuarantined = "true", IsDisabled = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip); + } + + [Fact] + public void CalculateFinalStatus_QuarantinedDisabledAndATF_AllSet_ReturnsSkip() + { + // Edge case: all flags set + var tags = new TestSpanTags { IsQuarantined = "true", IsDisabled = "true", IsAttemptToFix = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusSkip, "quarantined/disabled always takes precedence over ATF"); + } + + // EFD + New Test Combinations + + [Theory] + [InlineData(true, false, false, TestTags.StatusPass)] + [InlineData(false, true, false, TestTags.StatusFail)] + [InlineData(false, false, true, TestTags.StatusSkip)] + [InlineData(true, false, true, TestTags.StatusPass)] // Pass takes precedence over skip + public void CalculateFinalStatus_NewTest_AllCombinations(bool passed, bool failed, bool skipped, string expected) + { + var tags = new TestSpanTags { TestIsNew = "true" }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: passed, anyExecutionFailed: failed, isSkippedOrInconclusive: skipped, testTags: tags); + result.Should().Be(expected); + } + + // Retry with Various Tags + + [Theory] + [InlineData(true, false, false, false, TestTags.StatusPass)] // Retry passed + [InlineData(false, true, false, false, TestTags.StatusFail)] // Retry failed + [InlineData(false, false, true, false, TestTags.StatusSkip)] // Retry skipped + [InlineData(true, false, false, true, TestTags.StatusSkip)] // Retry passed but quarantined + [InlineData(false, true, false, true, TestTags.StatusSkip)] // Retry failed but quarantined + public void CalculateFinalStatus_RetryWithQuarantine(bool passed, bool failed, bool skipped, bool quarantined, string expected) + { + var tags = new TestSpanTags + { + TestIsRetry = "true", + IsQuarantined = quarantined ? "true" : null + }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: passed, anyExecutionFailed: failed, isSkippedOrInconclusive: skipped, testTags: tags); + result.Should().Be(expected); + } + + // Edge case: Empty string vs null for tag values + + [Fact] + public void CalculateFinalStatus_IsQuarantinedEmptyString_NotTreatedAsQuarantined() + { + var tags = new TestSpanTags { IsQuarantined = string.Empty }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "empty string should not be treated as quarantined"); + } + + [Fact] + public void CalculateFinalStatus_IsDisabledEmptyString_NotTreatedAsDisabled() + { + var tags = new TestSpanTags { IsDisabled = string.Empty }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + result.Should().Be(TestTags.StatusPass, "empty string should not be treated as disabled"); + } + + // Case sensitivity check + + [Theory] + [InlineData("TRUE")] + [InlineData("True")] + [InlineData("TRUE ")] + [InlineData(" true")] + public void CalculateFinalStatus_IsQuarantined_CaseSensitive_OnlyLowercaseTrue(string value) + { + var tags = new TestSpanTags { IsQuarantined = value }; + var result = Common.CalculateFinalStatus(anyExecutionPassed: true, anyExecutionFailed: false, isSkippedOrInconclusive: false, testTags: tags); + // Only exact "true" should be treated as quarantined + result.Should().Be(TestTags.StatusPass, $"'{value}' should not be treated as quarantined (only exact 'true' matches)"); + } +} diff --git a/tracer/test/snapshots/MsTestV2EvpTests.AttemptToFixTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.AttemptToFixTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt index 43bc20500ef6..f6f41813f0b0 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.AttemptToFixTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.AttemptToFixTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -111,6 +112,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -180,6 +182,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -249,6 +252,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -318,6 +322,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -391,6 +396,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -465,6 +471,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -539,6 +546,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1304,6 +1312,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1379,6 +1388,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1449,6 +1459,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1519,6 +1530,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1589,6 +1601,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1658,6 +1671,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1729,6 +1743,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1801,6 +1816,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1873,6 +1889,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1945,6 +1962,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2720,6 +2738,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2796,6 +2815,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2866,6 +2886,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2938,6 +2959,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.AttemptToFixTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.AttemptToFixTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt index 59b349e67b73..c90d5124ceed 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.AttemptToFixTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.AttemptToFixTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -111,6 +112,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -180,6 +182,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -249,6 +252,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -318,6 +322,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -391,6 +396,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -465,6 +471,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -539,6 +546,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1304,6 +1312,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1379,6 +1388,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1449,6 +1459,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1519,6 +1530,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1589,6 +1601,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1658,6 +1671,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1729,6 +1743,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1801,6 +1816,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2576,6 +2592,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2652,6 +2669,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2722,6 +2740,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2794,6 +2813,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.DisabledTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.DisabledTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt index 23a7fa9beaf0..da3792a4e590 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.DisabledTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.DisabledTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -111,6 +112,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -180,6 +182,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -249,6 +252,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -318,6 +322,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -391,6 +396,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -465,6 +471,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -539,6 +546,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -609,6 +617,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -681,6 +690,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -751,6 +761,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -821,6 +832,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -891,6 +903,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -960,6 +973,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1031,6 +1045,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1103,6 +1118,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1175,6 +1191,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1247,6 +1264,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1318,6 +1336,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1391,6 +1410,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1461,6 +1481,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1533,6 +1554,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.DisabledTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.DisabledTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt index 923dceb7fc6f..cb673c023b8f 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.DisabledTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.DisabledTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -111,6 +112,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -180,6 +182,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -249,6 +252,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -318,6 +322,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -391,6 +396,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -465,6 +471,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -539,6 +546,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -613,6 +621,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -683,6 +692,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -753,6 +763,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -823,6 +834,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -893,6 +905,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -962,6 +975,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1033,6 +1047,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1105,6 +1120,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1180,6 +1196,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1251,6 +1268,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1321,6 +1339,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1393,6 +1412,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=post_2_2_4_all_efd.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=post_2_2_4_all_efd.verified.txt index 1caa39d21d9f..33b9c57711ac 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=post_2_2_4_all_efd.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=post_2_2_4_all_efd.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -698,6 +699,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -1497,6 +1499,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -2225,6 +2228,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -3073,6 +3077,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3852,6 +3857,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -4631,6 +4637,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -5401,6 +5408,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -6139,6 +6147,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6877,6 +6886,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7615,6 +7625,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8344,6 +8355,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8417,6 +8429,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8490,6 +8503,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8564,6 +8578,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8638,6 +8653,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8712,6 +8728,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -9489,6 +9506,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -10228,6 +10246,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -10302,6 +10321,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -11031,6 +11051,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=post_2_2_4_efd_with_test_bypass.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=post_2_2_4_efd_with_test_bypass.verified.txt index e0269e31b6fd..e2a22d889273 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=post_2_2_4_efd_with_test_bypass.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=post_2_2_4_efd_with_test_bypass.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -698,6 +699,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -1497,6 +1499,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -2225,6 +2228,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -3073,6 +3077,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3852,6 +3857,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -4631,6 +4637,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -5401,6 +5408,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -6139,6 +6147,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6877,6 +6886,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7615,6 +7625,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8344,6 +8355,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8417,6 +8429,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8490,6 +8503,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8564,6 +8578,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8638,6 +8653,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8712,6 +8728,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -9489,6 +9506,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -9564,6 +9582,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -9635,6 +9654,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -10364,6 +10384,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=pre_2_2_4_all_efd.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=pre_2_2_4_all_efd.verified.txt index cd785b0d7e7d..17d02a451c28 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=pre_2_2_4_all_efd.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=pre_2_2_4_all_efd.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -698,6 +699,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -1497,6 +1499,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -2225,6 +2228,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -3073,6 +3077,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3852,6 +3857,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -4631,6 +4637,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -5401,6 +5408,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -6139,6 +6147,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6877,6 +6886,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7615,6 +7625,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8344,6 +8355,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8417,6 +8429,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8490,6 +8503,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8564,6 +8578,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -9341,6 +9356,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -10080,6 +10096,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -10154,6 +10171,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -10883,6 +10901,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=pre_2_2_4_efd_with_test_bypass.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=pre_2_2_4_efd_with_test_bypass.verified.txt index d651707a1e70..b7e8d676e2a5 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=pre_2_2_4_efd_with_test_bypass.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.EarlyFlakeDetection_packageVersion=pre_2_2_4_efd_with_test_bypass.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -698,6 +699,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -1497,6 +1499,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -2225,6 +2228,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -3073,6 +3077,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3852,6 +3857,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -4631,6 +4637,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -5401,6 +5408,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -6139,6 +6147,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6877,6 +6886,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7615,6 +7625,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8344,6 +8355,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8417,6 +8429,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8490,6 +8503,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8564,6 +8578,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -9341,6 +9356,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -9416,6 +9432,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -9487,6 +9504,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, @@ -10216,6 +10234,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.is_new: true, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.QuarantinedTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.QuarantinedTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt index e451fe57d676..1a6b26977caa 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.QuarantinedTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.QuarantinedTests_packageVersion=post_2_2_4_quarantined_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -111,6 +112,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -180,6 +182,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -249,6 +252,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -318,6 +322,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -391,6 +396,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -466,6 +472,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -541,6 +548,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -616,6 +624,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -686,6 +695,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -756,6 +766,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -826,6 +837,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -896,6 +908,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -965,6 +978,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1036,6 +1050,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1108,6 +1123,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1180,6 +1196,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1252,6 +1269,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1327,6 +1345,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1398,6 +1417,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1468,6 +1488,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1540,6 +1561,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.QuarantinedTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.QuarantinedTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt index 3a9aa6bf26ac..3c18e3a57795 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.QuarantinedTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.QuarantinedTests_packageVersion=pre_2_2_4_quarantined_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -111,6 +112,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -180,6 +182,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -249,6 +252,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -318,6 +322,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -391,6 +396,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -466,6 +472,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -541,6 +548,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -616,6 +624,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -686,6 +695,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -756,6 +766,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -826,6 +837,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -896,6 +908,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -965,6 +978,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1036,6 +1050,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1108,6 +1123,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1183,6 +1199,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1254,6 +1271,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1324,6 +1342,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1396,6 +1415,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.SubmitTraces_packageVersion=post_2_2_4.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.SubmitTraces_packageVersion=post_2_2_4.verified.txt index e9584f388ceb..ffccb602691f 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.SubmitTraces_packageVersion=post_2_2_4.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.SubmitTraces_packageVersion=post_2_2_4.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -113,6 +114,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -183,6 +185,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -253,6 +256,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -323,6 +327,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -397,6 +402,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -472,6 +478,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -547,6 +554,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -622,6 +630,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -692,6 +701,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -763,6 +773,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -834,6 +845,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -905,6 +917,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -975,6 +988,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1047,6 +1061,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1120,6 +1135,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1193,6 +1209,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1266,6 +1283,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1342,6 +1360,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1413,6 +1432,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1484,6 +1504,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1557,6 +1578,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2EvpTests.SubmitTraces_packageVersion=pre_2_2_4.verified.txt b/tracer/test/snapshots/MsTestV2EvpTests.SubmitTraces_packageVersion=pre_2_2_4.verified.txt index 22001828e599..b3878d304d21 100644 --- a/tracer/test/snapshots/MsTestV2EvpTests.SubmitTraces_packageVersion=pre_2_2_4.verified.txt +++ b/tracer/test/snapshots/MsTestV2EvpTests.SubmitTraces_packageVersion=pre_2_2_4.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -113,6 +114,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -183,6 +185,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -253,6 +256,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -323,6 +327,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -397,6 +402,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -472,6 +478,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -547,6 +554,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -622,6 +630,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -692,6 +701,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -763,6 +773,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -834,6 +845,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -905,6 +917,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -975,6 +988,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1047,6 +1061,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1120,6 +1135,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1196,6 +1212,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1267,6 +1284,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1338,6 +1356,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1411,6 +1430,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2Tests.SubmitTraces_packageVersion=post_2_2_4.verified.txt b/tracer/test/snapshots/MsTestV2Tests.SubmitTraces_packageVersion=post_2_2_4.verified.txt index b1089aede2c8..7c0b691ddb11 100644 --- a/tracer/test/snapshots/MsTestV2Tests.SubmitTraces_packageVersion=post_2_2_4.verified.txt +++ b/tracer/test/snapshots/MsTestV2Tests.SubmitTraces_packageVersion=post_2_2_4.verified.txt @@ -40,6 +40,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -109,6 +110,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -178,6 +180,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -247,6 +250,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -316,6 +320,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -389,6 +394,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -463,6 +469,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -537,6 +544,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -611,6 +619,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -680,6 +689,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -750,6 +760,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -820,6 +831,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -890,6 +902,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -959,6 +972,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1030,6 +1044,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1102,6 +1117,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1174,6 +1190,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1246,6 +1263,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1321,6 +1339,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1391,6 +1410,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1461,6 +1481,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1533,6 +1554,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2Tests.SubmitTraces_packageVersion=pre_2_2_4.verified.txt b/tracer/test/snapshots/MsTestV2Tests.SubmitTraces_packageVersion=pre_2_2_4.verified.txt index 6746805d9bfd..59bc099afba2 100644 --- a/tracer/test/snapshots/MsTestV2Tests.SubmitTraces_packageVersion=pre_2_2_4.verified.txt +++ b/tracer/test/snapshots/MsTestV2Tests.SubmitTraces_packageVersion=pre_2_2_4.verified.txt @@ -40,6 +40,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -109,6 +110,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -178,6 +180,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -247,6 +250,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -316,6 +320,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -389,6 +394,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -463,6 +469,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -537,6 +544,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -611,6 +619,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -680,6 +689,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -750,6 +760,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -820,6 +831,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -890,6 +902,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -959,6 +972,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1030,6 +1044,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1102,6 +1117,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1177,6 +1193,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1247,6 +1264,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1317,6 +1335,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1389,6 +1408,7 @@ test.bundle: Samples.MSTestTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2Tests2.SubmitTraces_packageVersion=post_2_2_4.verified.txt b/tracer/test/snapshots/MsTestV2Tests2.SubmitTraces_packageVersion=post_2_2_4.verified.txt index 6a7f27f4e402..ca80b2594787 100644 --- a/tracer/test/snapshots/MsTestV2Tests2.SubmitTraces_packageVersion=post_2_2_4.verified.txt +++ b/tracer/test/snapshots/MsTestV2Tests2.SubmitTraces_packageVersion=post_2_2_4.verified.txt @@ -36,6 +36,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -105,6 +106,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -174,6 +176,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -243,6 +246,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -316,6 +320,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -390,6 +395,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -464,6 +470,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -538,6 +545,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -607,6 +615,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -677,6 +686,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -747,6 +757,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -817,6 +828,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -886,6 +898,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -957,6 +970,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1029,6 +1043,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1101,6 +1116,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1173,6 +1189,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1248,6 +1265,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1318,6 +1336,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1388,6 +1407,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1460,6 +1480,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/MsTestV2Tests2.SubmitTraces_packageVersion=pre_2_2_4.verified.txt b/tracer/test/snapshots/MsTestV2Tests2.SubmitTraces_packageVersion=pre_2_2_4.verified.txt index f9e867b95fb5..938f49e708fc 100644 --- a/tracer/test/snapshots/MsTestV2Tests2.SubmitTraces_packageVersion=pre_2_2_4.verified.txt +++ b/tracer/test/snapshots/MsTestV2Tests2.SubmitTraces_packageVersion=pre_2_2_4.verified.txt @@ -36,6 +36,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -105,6 +106,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -174,6 +176,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -243,6 +246,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -316,6 +320,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -390,6 +395,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -464,6 +470,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -538,6 +545,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -607,6 +615,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -677,6 +686,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -747,6 +757,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -817,6 +828,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -886,6 +898,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -957,6 +970,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1029,6 +1043,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1104,6 +1119,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1174,6 +1190,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1244,6 +1261,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1316,6 +1334,7 @@ test.bundle: Samples.MSTestTests2, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: MSTestV2, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/NUnitEvpTests.AttemptToFixTests_quarantined_and_disabled.verified.txt b/tracer/test/snapshots/NUnitEvpTests.AttemptToFixTests_quarantined_and_disabled.verified.txt index f2190d7251d6..c6931326fe05 100644 --- a/tracer/test/snapshots/NUnitEvpTests.AttemptToFixTests_quarantined_and_disabled.verified.txt +++ b/tracer/test/snapshots/NUnitEvpTests.AttemptToFixTests_quarantined_and_disabled.verified.txt @@ -184,6 +184,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -254,6 +255,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -328,6 +330,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -401,6 +404,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -474,6 +478,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -547,6 +552,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -620,6 +626,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -693,6 +700,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -762,6 +770,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -831,6 +840,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -902,6 +912,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -976,6 +987,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1050,6 +1062,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1124,6 +1137,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1889,6 +1903,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1964,6 +1979,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2035,6 +2051,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2106,6 +2123,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2177,6 +2195,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2246,6 +2265,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2533,6 +2553,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -3308,6 +3329,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3384,6 +3406,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -3454,6 +3477,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -3526,6 +3550,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -3595,6 +3620,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -3668,6 +3694,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -3737,6 +3764,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/NUnitEvpTests.DisabledTests_disabled_tests.verified.txt b/tracer/test/snapshots/NUnitEvpTests.DisabledTests_disabled_tests.verified.txt index ac55a7b5e1ad..71b4edf2db62 100644 --- a/tracer/test/snapshots/NUnitEvpTests.DisabledTests_disabled_tests.verified.txt +++ b/tracer/test/snapshots/NUnitEvpTests.DisabledTests_disabled_tests.verified.txt @@ -184,6 +184,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -254,6 +255,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -328,6 +330,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -401,6 +404,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -474,6 +478,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -547,6 +552,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -620,6 +626,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -693,6 +700,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -762,6 +770,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -831,6 +840,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -902,6 +912,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -976,6 +987,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1050,6 +1062,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1124,6 +1137,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1194,6 +1208,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1266,6 +1281,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1337,6 +1353,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1408,6 +1425,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1479,6 +1497,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1548,6 +1567,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1835,6 +1855,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1906,6 +1927,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1979,6 +2001,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2049,6 +2072,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2121,6 +2145,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2190,6 +2215,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2263,6 +2289,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2332,6 +2359,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/NUnitEvpTests.EarlyFlakeDetection_all_efd.verified.txt b/tracer/test/snapshots/NUnitEvpTests.EarlyFlakeDetection_all_efd.verified.txt index f2850aa0b60f..8044d1770d89 100644 --- a/tracer/test/snapshots/NUnitEvpTests.EarlyFlakeDetection_all_efd.verified.txt +++ b/tracer/test/snapshots/NUnitEvpTests.EarlyFlakeDetection_all_efd.verified.txt @@ -853,6 +853,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -866,7 +867,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestFixtureTest("Test01"), - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -1592,6 +1592,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -1605,7 +1606,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestFixtureTest("Test02"), - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -2362,6 +2362,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2375,7 +2376,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -3132,6 +3132,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3145,7 +3146,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -3902,6 +3902,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3915,7 +3916,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -4672,6 +4672,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -4685,7 +4686,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -5442,6 +5442,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -5455,7 +5456,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -6212,6 +6212,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -6225,7 +6226,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -6942,6 +6942,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6954,7 +6955,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestString, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -7016,6 +7016,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7753,6 +7754,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7766,7 +7768,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -8532,6 +8533,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -8546,7 +8548,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -9312,6 +9313,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -9326,7 +9328,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -10092,6 +10093,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -10106,7 +10108,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -10863,6 +10864,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -10876,7 +10878,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -11611,6 +11612,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -11624,7 +11626,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.traits: {"Category":["ParemeterizedTest","FirstCase"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -12360,6 +12361,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -12373,7 +12375,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.traits: {"Category":["ParemeterizedTest","SecondCase"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -13109,6 +13110,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -13122,7 +13124,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.traits: {"Category":["ParemeterizedTest","ThirdCase"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -13840,6 +13841,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -13852,7 +13854,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -13914,6 +13915,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -14209,6 +14211,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -14986,6 +14989,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -14999,7 +15003,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.traits: {"Category":["Category01"],"Compatibility":["Windows","Linux"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -15726,6 +15729,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -15738,7 +15742,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.traits: {"Category":["Category01"],"Compatibility":["Windows","Linux"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -15801,6 +15804,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -16530,6 +16534,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -16542,7 +16547,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -17259,6 +17263,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -17271,7 +17276,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestTearDown2Error, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -18028,6 +18032,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -18041,7 +18046,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestTearDownError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -18758,6 +18762,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -18770,7 +18775,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.UnSkippableSuite, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, diff --git a/tracer/test/snapshots/NUnitEvpTests.EarlyFlakeDetection_efd_with_test_bypass.verified.txt b/tracer/test/snapshots/NUnitEvpTests.EarlyFlakeDetection_efd_with_test_bypass.verified.txt index 410f42ff3872..4b16403c9995 100644 --- a/tracer/test/snapshots/NUnitEvpTests.EarlyFlakeDetection_efd_with_test_bypass.verified.txt +++ b/tracer/test/snapshots/NUnitEvpTests.EarlyFlakeDetection_efd_with_test_bypass.verified.txt @@ -853,6 +853,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -866,7 +867,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestFixtureTest("Test01"), - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -1592,6 +1592,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -1605,7 +1606,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestFixtureTest("Test02"), - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -2362,6 +2362,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2375,7 +2376,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -3132,6 +3132,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3145,7 +3146,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -3902,6 +3902,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3915,7 +3916,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -4672,6 +4672,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -4685,7 +4686,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -5442,6 +5442,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -5455,7 +5456,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -6212,6 +6212,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -6225,7 +6226,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSetupError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -6942,6 +6942,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6954,7 +6955,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestString, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -7016,6 +7016,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7753,6 +7754,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7766,7 +7768,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -8532,6 +8533,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -8546,7 +8548,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -9312,6 +9313,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -9326,7 +9328,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -10092,6 +10093,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -10106,7 +10108,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -10863,6 +10864,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -10876,7 +10878,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -11611,6 +11612,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -11624,7 +11626,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.traits: {"Category":["ParemeterizedTest","FirstCase"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -12360,6 +12361,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -12373,7 +12375,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.traits: {"Category":["ParemeterizedTest","SecondCase"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -13109,6 +13110,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -13122,7 +13124,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.traits: {"Category":["ParemeterizedTest","ThirdCase"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -13840,6 +13841,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -13852,7 +13854,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -13914,6 +13915,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -14209,6 +14211,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -14986,6 +14989,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -14999,7 +15003,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: false, test.traits: {"Category":["Category01"],"Compatibility":["Windows","Linux"]}, test.type: test, test.working_directory: CommandWorkingDirectory, @@ -15062,6 +15065,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -15133,6 +15137,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -15862,6 +15867,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -15874,7 +15880,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestSuite, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -16591,6 +16596,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -16603,7 +16609,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.TestTearDown2Error, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -17360,6 +17365,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -17373,7 +17379,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: fail, test.suite: Samples.NUnitTests.TestTearDownError, - test.test_management.attempt_to_fix_passed: false, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, @@ -18090,6 +18095,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -18102,7 +18108,6 @@ test.source.file: tracer/test/test-applications/integrations/Samples.NUnitTests/TestSuite.cs, test.status: pass, test.suite: Samples.NUnitTests.UnSkippableSuite, - test.test_management.attempt_to_fix_passed: true, test.type: test, test.working_directory: CommandWorkingDirectory, version: 1.0.0, diff --git a/tracer/test/snapshots/NUnitEvpTests.QuarantinedTests_quarantined_tests.verified.txt b/tracer/test/snapshots/NUnitEvpTests.QuarantinedTests_quarantined_tests.verified.txt index 1f3c449ab860..334fa04f9fb1 100644 --- a/tracer/test/snapshots/NUnitEvpTests.QuarantinedTests_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/NUnitEvpTests.QuarantinedTests_quarantined_tests.verified.txt @@ -184,6 +184,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -254,6 +255,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -328,6 +330,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -401,6 +404,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -474,6 +478,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -547,6 +552,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -620,6 +626,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -693,6 +700,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -762,6 +770,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -831,6 +840,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -902,6 +912,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -976,6 +987,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1051,6 +1063,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1126,6 +1139,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1201,6 +1215,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1271,6 +1286,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1342,6 +1358,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1413,6 +1430,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1484,6 +1502,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1553,6 +1572,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1840,6 +1860,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1915,6 +1936,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1986,6 +2008,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2056,6 +2079,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2128,6 +2152,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2197,6 +2222,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2270,6 +2296,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2339,6 +2366,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/NUnitEvpTests.SubmitTraces_packageVersion=all.verified.txt b/tracer/test/snapshots/NUnitEvpTests.SubmitTraces_packageVersion=all.verified.txt index efe8e93c2b79..88a464e4ca9e 100644 --- a/tracer/test/snapshots/NUnitEvpTests.SubmitTraces_packageVersion=all.verified.txt +++ b/tracer/test/snapshots/NUnitEvpTests.SubmitTraces_packageVersion=all.verified.txt @@ -187,6 +187,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -258,6 +259,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -333,6 +335,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -407,6 +410,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -481,6 +485,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -555,6 +560,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -629,6 +635,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -703,6 +710,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -773,6 +781,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -843,6 +852,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -915,6 +925,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -990,6 +1001,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1065,6 +1077,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1140,6 +1153,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1215,6 +1229,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1285,6 +1300,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1357,6 +1373,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1429,6 +1446,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1501,6 +1519,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1571,6 +1590,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1862,6 +1882,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1938,6 +1959,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2009,6 +2031,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2080,6 +2103,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2153,6 +2177,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2223,6 +2248,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2297,6 +2323,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2367,6 +2394,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/NUnitTests.SubmitTraces_packageVersion=all.verified.txt b/tracer/test/snapshots/NUnitTests.SubmitTraces_packageVersion=all.verified.txt index 85fafb98c841..433082779f14 100644 --- a/tracer/test/snapshots/NUnitTests.SubmitTraces_packageVersion=all.verified.txt +++ b/tracer/test/snapshots/NUnitTests.SubmitTraces_packageVersion=all.verified.txt @@ -182,6 +182,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -252,6 +253,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -326,6 +328,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -399,6 +402,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -472,6 +476,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -545,6 +550,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -618,6 +624,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -691,6 +698,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -760,6 +768,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -829,6 +838,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -900,6 +910,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -974,6 +985,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1048,6 +1060,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1122,6 +1135,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1196,6 +1210,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1265,6 +1280,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1336,6 +1352,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1407,6 +1424,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1478,6 +1496,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1547,6 +1566,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1834,6 +1854,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1909,6 +1930,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1979,6 +2001,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2049,6 +2072,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2121,6 +2145,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2190,6 +2215,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2263,6 +2289,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2332,6 +2359,7 @@ test.bundle: Samples.NUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: NUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/SeleniumTests.Injection_all_versions.verified.txt b/tracer/test/snapshots/SeleniumTests.Injection_all_versions.verified.txt index d7086752acae..23249ae1403e 100644 --- a/tracer/test/snapshots/SeleniumTests.Injection_all_versions.verified.txt +++ b/tracer/test/snapshots/SeleniumTests.Injection_all_versions.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.Selenium, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_rum_active: true, diff --git a/tracer/test/snapshots/XUnitEvpTests.AttemptToFixTests_quarantined_and_disabled.verified.txt b/tracer/test/snapshots/XUnitEvpTests.AttemptToFixTests_quarantined_and_disabled.verified.txt index dea75f96422e..e2287cfc4c3d 100644 --- a/tracer/test/snapshots/XUnitEvpTests.AttemptToFixTests_quarantined_and_disabled.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTests.AttemptToFixTests_quarantined_and_disabled.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -116,6 +117,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -190,6 +192,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -955,6 +958,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1030,6 +1034,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1100,6 +1105,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1170,6 +1176,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1240,6 +1247,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1309,6 +1317,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1380,6 +1389,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1451,6 +1461,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2226,6 +2237,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2302,6 +2314,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2372,6 +2385,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2444,6 +2458,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2513,6 +2528,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/XUnitEvpTests.DisabledTests_disabled_tests.verified.txt b/tracer/test/snapshots/XUnitEvpTests.DisabledTests_disabled_tests.verified.txt index d61893b06466..a8909c8cc0e0 100644 --- a/tracer/test/snapshots/XUnitEvpTests.DisabledTests_disabled_tests.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTests.DisabledTests_disabled_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -116,6 +117,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -190,6 +192,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -260,6 +263,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -332,6 +336,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -402,6 +407,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -472,6 +478,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -542,6 +549,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -611,6 +619,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -682,6 +691,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -753,6 +763,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -824,6 +835,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -897,6 +909,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -967,6 +980,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1039,6 +1053,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1108,6 +1123,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/XUnitEvpTests.EarlyFlakeDetection_all_efd.verified.txt b/tracer/test/snapshots/XUnitEvpTests.EarlyFlakeDetection_all_efd.verified.txt index 74e7776d98a8..4c4c5782b8f6 100644 --- a/tracer/test/snapshots/XUnitEvpTests.EarlyFlakeDetection_all_efd.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTests.EarlyFlakeDetection_all_efd.verified.txt @@ -743,6 +743,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1522,6 +1523,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2301,6 +2303,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3071,6 +3074,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3809,6 +3813,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -4547,6 +4552,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -5285,6 +5291,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6014,6 +6021,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6087,6 +6095,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6160,6 +6169,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6233,6 +6243,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7010,6 +7021,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -7749,6 +7761,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7823,6 +7836,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8552,6 +8566,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -9280,6 +9295,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, diff --git a/tracer/test/snapshots/XUnitEvpTests.EarlyFlakeDetection_efd_with_test_bypass.verified.txt b/tracer/test/snapshots/XUnitEvpTests.EarlyFlakeDetection_efd_with_test_bypass.verified.txt index eb8331a3136a..27b8b3afdf58 100644 --- a/tracer/test/snapshots/XUnitEvpTests.EarlyFlakeDetection_efd_with_test_bypass.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTests.EarlyFlakeDetection_efd_with_test_bypass.verified.txt @@ -743,6 +743,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1522,6 +1523,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2301,6 +2303,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3071,6 +3074,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3809,6 +3813,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -4547,6 +4552,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -5285,6 +5291,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6014,6 +6021,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6087,6 +6095,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6160,6 +6169,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6233,6 +6243,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7010,6 +7021,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -7085,6 +7097,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -7156,6 +7169,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7885,6 +7899,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8613,6 +8628,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.is_new: true, diff --git a/tracer/test/snapshots/XUnitEvpTests.QuarantinedTests_quarantined_tests.verified.txt b/tracer/test/snapshots/XUnitEvpTests.QuarantinedTests_quarantined_tests.verified.txt index f38230314db1..abca7648c0e5 100644 --- a/tracer/test/snapshots/XUnitEvpTests.QuarantinedTests_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTests.QuarantinedTests_quarantined_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -117,6 +118,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -192,6 +194,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -267,6 +270,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -337,6 +341,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -407,6 +412,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -477,6 +483,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -547,6 +554,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -616,6 +624,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -687,6 +696,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -758,6 +768,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -833,6 +844,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -904,6 +916,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -974,6 +987,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1046,6 +1060,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1115,6 +1130,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/XUnitEvpTests.SubmitTraces_packageVersion=all.verified.txt b/tracer/test/snapshots/XUnitEvpTests.SubmitTraces_packageVersion=all.verified.txt index f9e5c96e383e..99fe52ca700a 100644 --- a/tracer/test/snapshots/XUnitEvpTests.SubmitTraces_packageVersion=all.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTests.SubmitTraces_packageVersion=all.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -118,6 +119,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -193,6 +195,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -268,6 +271,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -338,6 +342,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -409,6 +414,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -480,6 +486,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -551,6 +558,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -621,6 +629,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -693,6 +702,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -765,6 +775,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -841,6 +852,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -912,6 +924,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -983,6 +996,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1056,6 +1070,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1126,6 +1141,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/XUnitEvpTestsV3.AttemptToFixTests_quarantined_and_disabled.verified.txt b/tracer/test/snapshots/XUnitEvpTestsV3.AttemptToFixTests_quarantined_and_disabled.verified.txt index cc90145be313..9e21ff2a4b32 100644 --- a/tracer/test/snapshots/XUnitEvpTestsV3.AttemptToFixTests_quarantined_and_disabled.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTestsV3.AttemptToFixTests_quarantined_and_disabled.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -116,6 +117,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -190,6 +192,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -955,6 +958,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1030,6 +1034,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1100,6 +1105,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1170,6 +1176,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1240,6 +1247,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1309,6 +1317,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1380,6 +1389,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1451,6 +1461,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2226,6 +2237,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2302,6 +2314,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2372,6 +2385,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2444,6 +2458,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -2513,6 +2528,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/XUnitEvpTestsV3.DisabledTests_disabled_tests.verified.txt b/tracer/test/snapshots/XUnitEvpTestsV3.DisabledTests_disabled_tests.verified.txt index ad490fcb4463..eeafb3143931 100644 --- a/tracer/test/snapshots/XUnitEvpTestsV3.DisabledTests_disabled_tests.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTestsV3.DisabledTests_disabled_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -116,6 +117,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -190,6 +192,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -260,6 +263,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -332,6 +336,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -402,6 +407,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -472,6 +478,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -542,6 +549,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -611,6 +619,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -682,6 +691,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -753,6 +763,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -824,6 +835,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -897,6 +909,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -967,6 +980,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1039,6 +1053,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1108,6 +1123,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/XUnitEvpTestsV3.EarlyFlakeDetection_all_efd.verified.txt b/tracer/test/snapshots/XUnitEvpTestsV3.EarlyFlakeDetection_all_efd.verified.txt index 039c47bd7690..e714dfc283b1 100644 --- a/tracer/test/snapshots/XUnitEvpTestsV3.EarlyFlakeDetection_all_efd.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTestsV3.EarlyFlakeDetection_all_efd.verified.txt @@ -743,6 +743,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1522,6 +1523,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2301,6 +2303,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3071,6 +3074,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3809,6 +3813,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -4547,6 +4552,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -5285,6 +5291,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6014,6 +6021,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6087,6 +6095,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6160,6 +6169,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6233,6 +6243,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7010,6 +7021,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -7749,6 +7761,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7823,6 +7836,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8552,6 +8566,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -9280,6 +9295,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, diff --git a/tracer/test/snapshots/XUnitEvpTestsV3.EarlyFlakeDetection_efd_with_test_bypass.verified.txt b/tracer/test/snapshots/XUnitEvpTestsV3.EarlyFlakeDetection_efd_with_test_bypass.verified.txt index 6b71151fd681..4881e22bee7a 100644 --- a/tracer/test/snapshots/XUnitEvpTestsV3.EarlyFlakeDetection_efd_with_test_bypass.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTestsV3.EarlyFlakeDetection_efd_with_test_bypass.verified.txt @@ -743,6 +743,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -1522,6 +1523,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -2301,6 +2303,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3071,6 +3074,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -3809,6 +3813,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -4547,6 +4552,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -5285,6 +5291,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6014,6 +6021,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6087,6 +6095,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6160,6 +6169,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -6233,6 +6243,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7010,6 +7021,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.has_failed_all_retries: true, @@ -7085,6 +7097,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -7156,6 +7169,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -7885,6 +7899,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, @@ -8613,6 +8628,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.is_new: true, diff --git a/tracer/test/snapshots/XUnitEvpTestsV3.QuarantinedTests_quarantined_tests.verified.txt b/tracer/test/snapshots/XUnitEvpTestsV3.QuarantinedTests_quarantined_tests.verified.txt index 131005ba0e9a..2af14794bd38 100644 --- a/tracer/test/snapshots/XUnitEvpTestsV3.QuarantinedTests_quarantined_tests.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTestsV3.QuarantinedTests_quarantined_tests.verified.txt @@ -42,6 +42,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -117,6 +118,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -192,6 +194,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -267,6 +270,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -337,6 +341,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -407,6 +412,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -477,6 +483,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -547,6 +554,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -616,6 +624,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -687,6 +696,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -758,6 +768,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -833,6 +844,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -904,6 +916,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -974,6 +987,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1046,6 +1060,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1115,6 +1130,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/XUnitEvpTestsV3.SubmitTraces_packageVersion=all.verified.txt b/tracer/test/snapshots/XUnitEvpTestsV3.SubmitTraces_packageVersion=all.verified.txt index 66c33eea4709..41d52d799a6f 100644 --- a/tracer/test/snapshots/XUnitEvpTestsV3.SubmitTraces_packageVersion=all.verified.txt +++ b/tracer/test/snapshots/XUnitEvpTestsV3.SubmitTraces_packageVersion=all.verified.txt @@ -43,6 +43,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -118,6 +119,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -193,6 +195,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -268,6 +271,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -338,6 +342,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -409,6 +414,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -480,6 +486,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -551,6 +558,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -621,6 +629,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -693,6 +702,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -765,6 +775,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -841,6 +852,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -912,6 +924,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -983,6 +996,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1056,6 +1070,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1126,6 +1141,7 @@ test.bundle: Samples.XUnitTestsV3, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit.v3, test.framework_version: FrameworkVersion, test.itr.forced_run: false, diff --git a/tracer/test/snapshots/XUnitTests.SubmitTraces_packageVersion=all.verified.txt b/tracer/test/snapshots/XUnitTests.SubmitTraces_packageVersion=all.verified.txt index 3f6a6c43e584..9a0a42ce6550 100644 --- a/tracer/test/snapshots/XUnitTests.SubmitTraces_packageVersion=all.verified.txt +++ b/tracer/test/snapshots/XUnitTests.SubmitTraces_packageVersion=all.verified.txt @@ -40,6 +40,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -114,6 +115,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -188,6 +190,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -262,6 +265,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -331,6 +335,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -401,6 +406,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -471,6 +477,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -541,6 +548,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -610,6 +618,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -681,6 +690,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -752,6 +762,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -827,6 +838,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: fail, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -897,6 +909,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -967,6 +980,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: skip, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1039,6 +1053,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false, @@ -1108,6 +1123,7 @@ test.bundle: Samples.XUnitTests, test.codeowners: [ "@MyTeam" ], test.command: Command, + test.final_status: pass, test.framework: xUnit, test.framework_version: FrameworkVersion, test.itr.forced_run: false,