Skip to content

Commit 8ab56fe

Browse files
authored
chore: Add setting to skip long running tests on draft PR (#3121)
* chore: add logics to detect github draft pr * chore: add skip tests logics when ci running on draft pr
1 parent af4f924 commit 8ab56fe

14 files changed

Lines changed: 69 additions & 29 deletions

.github/workflows/run-tests.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,13 @@ run-name: Run tests / ${{ github.event.head_commit.message }}
33

44
on:
55
pull_request:
6+
types: [opened, synchronize, reopened, ready_for_review]
67
push:
78
workflow_dispatch:
89

10+
env:
11+
IS_DRAFT_PR: ${{ github.event.pull_request.draft }}
12+
913
concurrency:
1014
group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }}
1115
cancel-in-progress: true
@@ -24,6 +28,7 @@ jobs:
2428
- uses: ./.github/actions/add-windowsdefender-exclusions
2529
# Setup additional tools
2630
- name: Setup additional tools for Wasm/NativeAot tests
31+
if: ${{ github.event.pull_request.draft != true }}
2732
uses: ./.github/actions/setup-additional-tools
2833
# Build and Test
2934
- name: Run task 'build'
@@ -66,6 +71,7 @@ jobs:
6671
- uses: ./.github/actions/add-windowsdefender-exclusions
6772
# Setup additional tools
6873
- name: Setup additional tools for Wasm/NativeAot tests
74+
if: ${{ github.event.pull_request.draft != true }}
6975
uses: ./.github/actions/setup-additional-tools
7076
# Build and Test
7177
- name: Run task 'build'
@@ -105,6 +111,7 @@ jobs:
105111
- uses: actions/checkout@v6
106112
# Setup additional tools
107113
- name: Setup additional tools for Wasm/NativeAot tests
114+
if: ${{ github.event.pull_request.draft != true }}
108115
uses: ./.github/actions/setup-additional-tools
109116
# Build and Test
110117
- name: Run task 'build'
@@ -145,6 +152,7 @@ jobs:
145152
- uses: actions/checkout@v6
146153
# Setup additional tools
147154
- name: Setup additional tools for Wasm/NativeAot tests
155+
if: ${{ github.event.pull_request.draft != true }}
148156
uses: ./.github/actions/setup-additional-tools
149157
# Build and Test
150158
- name: Run task 'build'

tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@ namespace BenchmarkDotNet.IntegrationTests
1111
public class ArgumentsTests : BenchmarkTestExecutor
1212
{
1313
public static IEnumerable<object[]> GetToolchains()
14-
=>
15-
[
16-
[Job.Default.GetToolchain()],
17-
[InProcessEmitToolchain.Default],
18-
];
14+
{
15+
yield return [InProcessEmitToolchain.Default];
16+
17+
if (ContinuousIntegration.IsGitHubDraftPR())
18+
yield break;
19+
20+
yield return [Job.Default.GetToolchain()];
21+
}
1922

2023
public ArgumentsTests(ITestOutputHelper output) : base(output) { }
2124

tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public void BenchmarkWithCancellationTokenProperty_ReceivesToken_InProcessEmit()
5252
CanExecute<BenchmarkWithCancellationToken>(config);
5353
}
5454

55-
[TheoryEnvSpecific("JSVU does not support ARM on Windows or Linux", EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm)]
55+
[TheoryEnvSpecific("JSVU does not support ARM on Windows or Linux", EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm, EnvRequirement.NonGitHubDraftPR)]
5656
[InlineData("v8")]
5757
[InlineData("node")]
5858
public void BenchmarkWithCancellationTokenProperty_ReceivesToken_Wasm(string javaScriptEngine)
@@ -116,7 +116,7 @@ public async Task RunWithCancellationTokenIsCancelled_InProcessEmit()
116116
}
117117

118118
[Theory]
119-
[InlineDataEnvSpecific("v8", "JSVU does not support ARM on Windows or Linux", [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])]
119+
[InlineDataEnvSpecific("v8", "JSVU does not support ARM on Windows or Linux", [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm, EnvRequirement.NonGitHubDraftPR])]
120120
[InlineData("node")]
121121
public async Task RunWithCancellationTokenIsCancelled_Wasm(string javaScriptEngine)
122122
{

tests/BenchmarkDotNet.IntegrationTests/ContinuousIntegration.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
using BenchmarkDotNet.Detectors;
2+
using BenchmarkDotNet.Extensions;
3+
using BenchmarkDotNet.Tests.XUnit;
24

35
namespace BenchmarkDotNet.IntegrationTests
46
{
@@ -10,5 +12,8 @@ internal static bool IsGitHubActionsOnWindows()
1012
=> OsDetector.IsWindows() && IsGitHubActions();
1113

1214
internal static bool IsLocalRun() => !IsGitHubActions();
15+
16+
internal static bool IsGitHubDraftPR()
17+
=> EnvRequirementChecker.GetSkip(EnvRequirement.NonGitHubDraftPR).IsNotBlank();
1318
}
1419
}

tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ public static IEnumerable<object[]> GetAllJits()
2525
{
2626
yield return [JitInfo.GetCurrentJit(), RuntimeInformation.GetCurrentPlatform(), InProcessEmitToolchain.Default]; // InProcess
2727

28+
if (ContinuousIntegration.IsGitHubDraftPR())
29+
yield break;
30+
2831
if (RuntimeInformation.IsFullFramework)
2932
{
3033
yield return [Jit.LegacyJit, Platform.X86, CsProjClassicNetToolchain.Net472]; // 32bit LegacyJit for desktop .NET

tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,14 @@ public class MemoryDiagnoserTests
3232

3333
public static IEnumerable<object[]> GetToolchains()
3434
{
35-
yield return new object[] { Job.Default.GetToolchain() };
3635
// InProcessEmit reports flaky allocations in current .Net 8.
3736
if (!RuntimeInformation.IsNetCore)
38-
{
3937
yield return new object[] { InProcessEmitToolchain.Default };
40-
}
38+
39+
if (ContinuousIntegration.IsGitHubDraftPR())
40+
yield break;
41+
42+
yield return new object[] { Job.Default.GetToolchain() };
4143
}
4244

4345
public class AccurateAllocations
@@ -68,7 +70,7 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain)
6870
});
6971
}
7072

71-
[FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 8.0)", EnvRequirement.DotNetCoreOnly)]
73+
[FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 8.0)", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR])]
7274
public void MemoryDiagnoserSupportsNativeAOT()
7375
{
7476
if (OsDetector.IsMacOS())
@@ -77,13 +79,13 @@ public void MemoryDiagnoserSupportsNativeAOT()
7779
MemoryDiagnoserIsAccurate(NativeAotToolchain.Net80);
7880
}
7981

80-
[FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0), and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm])]
82+
[FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 8.0), and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm, EnvRequirement.NonGitHubDraftPR])]
8183
public void MemoryDiagnoserSupportsModernMono()
8284
{
8385
MemoryDiagnoserIsAccurate(MonoToolchain.Mono80);
8486
}
8587

86-
[TheoryEnvSpecific("JSVU does not support ARM on Windows or Linux", EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm)]
88+
[TheoryEnvSpecific("JSVU does not support ARM on Windows or Linux", [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm, EnvRequirement.NonGitHubDraftPR])]
8789
[InlineData(MonoAotCompilerMode.mini)]
8890
// BUG: https://github.com/dotnet/BenchmarkDotNet/issues/3036
8991
[InlineData(MonoAotCompilerMode.wasm, Skip = "AOT is broken")]

tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ public class MonoTests : BenchmarkTestExecutor
1212
{
1313
public MonoTests(ITestOutputHelper output) : base(output) { }
1414

15-
[FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6, and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm])]
15+
[FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6, and it's not supported on Windows+Arm", [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm, EnvRequirement.NonGitHubDraftPR])]
1616
public void Mono80IsSupported()
1717
{
1818
var logger = new OutputLogger(Output);

tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ private ManualConfig GetConfig()
3333
.WithEnvironmentVariable(NativeAotBenchmark.EnvVarKey, IsAvx2Supported().ToString().ToLower()));
3434
}
3535

36-
[FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly)]
36+
[FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)]
3737
public void LatestNativeAotVersionIsSupported()
3838
{
3939
if (!GetShouldRunTest())
@@ -52,7 +52,7 @@ public void LatestNativeAotVersionIsSupported()
5252
}
5353
}
5454

55-
[FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly)]
55+
[FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)]
5656
public void NativeAotSupportsInProcessDiagnosers()
5757
{
5858
if (!GetShouldRunTest())

tests/BenchmarkDotNet.IntegrationTests/ParamSourceTests.cs

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,14 @@ public class ParamSourceTests : BenchmarkTestExecutor
1414
public ParamSourceTests(ITestOutputHelper output) : base(output) { }
1515

1616
public static IEnumerable<object[]> GetToolchains()
17-
=>
18-
[
19-
[Job.Default.GetToolchain()],
20-
[InProcessEmitToolchain.Default],
21-
];
17+
{
18+
yield return [InProcessEmitToolchain.Default];
19+
20+
if (ContinuousIntegration.IsGitHubDraftPR())
21+
yield break;
22+
23+
yield return [Job.Default.GetToolchain()];
24+
}
2225

2326
[Fact]
2427
public void ParamSourceCanHandleStringWithSurrogates()

tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ private static R2RRuntime GetCurrentR2RRuntime()
5454
#endif
5555
}
5656

57-
[FactEnvSpecific("R2R requires .NET Core runtime", EnvRequirement.DotNetCoreOnly)]
57+
[FactEnvSpecific("R2R requires .NET Core runtime", EnvRequirement.DotNetCoreOnly, EnvRequirement.NonGitHubDraftPR)]
5858
public void R2RToolchainCanExecuteBenchmarks()
5959
{
6060
try

0 commit comments

Comments
 (0)