Skip to content

Commit 259c1ff

Browse files
ChrisJollyAUChrisJollyAU
committed
Split FunctionalTests into Query/Non-Query shards
EFCore.Jet.FunctionalTests are now split into two shards (Query and Non-Query) in both pull_request.yml and push.yml workflows. Each shard runs independently with its own results directory and up to three retries on crash. Test results from both shards are merged for reporting and comparison, ensuring accurate tracking of passing tests across all shards.
1 parent 5f4f57c commit 259c1ff

2 files changed

Lines changed: 142 additions & 40 deletions

File tree

‎.github/workflows/pull_request.yml‎

Lines changed: 71 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -229,30 +229,31 @@ jobs:
229229
run: |
230230
$env:EFCoreJet_DefaultConnection = '${{ env.defaultConnection }}'
231231
& '${{ env.dotnetExecutable }}' test .\test\EFCore.Jet.Tests --configuration '${{ env.buildConfiguration }}' -p:FixedTestOrder=${{ env.deterministicTests }} --logger trx --verbosity detailed --blame-hang-timeout 3m
232-
- name: 'Run Tests: EFCore.Jet.FunctionalTests'
232+
- name: 'Run Tests: EFCore.Jet.FunctionalTests (Shard 1 - Query)'
233233
if: always() && env.skipTests != 'true'
234234
shell: pwsh
235235
run: |
236+
$shardDir = '.\test\EFCore.Jet.FunctionalTests\TestResults\shard1'
237+
236238
for ($i = 0; $i -lt 3; $i++) {
237-
if (Test-Path '.\test\EFCore.Jet.FunctionalTests\TestResults' -PathType Container) {
238-
Get-ChildItem '.\test\EFCore.Jet.FunctionalTests\TestResults' | Remove-Item -Recurse -Force
239+
if (Test-Path $shardDir -PathType Container) {
240+
Get-ChildItem $shardDir | Remove-Item -Recurse -Force
239241
}
240242
241243
$env:EFCoreJet_DefaultConnection = '${{ env.defaultConnection }}'
242-
& '${{ env.dotnetExecutable }}' test .\test\EFCore.Jet.FunctionalTests --configuration '${{ env.buildConfiguration }}' -p:FixedTestOrder=${{ env.deterministicTests }} --logger trx --verbosity detailed --blame-hang-timeout 3m
244+
& '${{ env.dotnetExecutable }}' test .\test\EFCore.Jet.FunctionalTests --configuration '${{ env.buildConfiguration }}' -p:FixedTestOrder=${{ env.deterministicTests }} --logger trx --verbosity detailed --blame-hang-timeout 3m --results-directory $shardDir --filter "FullyQualifiedName~.FunctionalTests.Query."
243245
244246
#
245247
# Check for test runner crashes:
246248
#
247249
248-
$testResultsDir = '.\test\EFCore.Jet.FunctionalTests\TestResults'
249-
$currentTestRunTrx = Get-ChildItem $testResultsDir -Filter '*.trx' | Sort-Object LastWriteTime | Select-Object -Last 1
250+
$currentTestRunTrx = Get-ChildItem $shardDir -Filter '*.trx' | Sort-Object LastWriteTime | Select-Object -Last 1
250251
if ($null -eq $currentTestRunTrx) {
251252
echo 'Test runner log file is missing.'
252253
exit 3
253254
}
254255
255-
$currentTestRunDir = Join-Path $testResultsDir $currentTestRunTrx.BaseName
256+
$currentTestRunDir = Join-Path $shardDir $currentTestRunTrx.BaseName
256257
if (Test-Path $currentTestRunDir) {
257258
if ($null -ne (Get-ChildItem $currentTestRunDir -Filter 'Sequence_*' -Recurse)) {
258259
# Split string because searching the log for that phrase should only show actual crashes and not this line.
@@ -267,7 +268,52 @@ jobs:
267268
268269
$establishedGreenTestsFilePath = ".\test\EFCore.Jet.FunctionalTests\GreenTests\ace_${{ matrix.aceVersion }}_$('${{ matrix.dataAccessProviderType }}'.Replace(' ', '').ToLowerInvariant())_${{ matrix.aceArchitecture }}.txt"
269270
$failIfKeepsCrashing = Test-Path $establishedGreenTestsFilePath
270-
271+
272+
if ($i -ge 3 -and $failIfKeepsCrashing) {
273+
echo 'Test runner keeps crashing.'
274+
exit 2
275+
}
276+
exit 0
277+
- name: 'Run Tests: EFCore.Jet.FunctionalTests (Shard 2 - Non-Query)'
278+
if: always() && env.skipTests != 'true'
279+
shell: pwsh
280+
run: |
281+
$shardDir = '.\test\EFCore.Jet.FunctionalTests\TestResults\shard2'
282+
283+
for ($i = 0; $i -lt 3; $i++) {
284+
if (Test-Path $shardDir -PathType Container) {
285+
Get-ChildItem $shardDir | Remove-Item -Recurse -Force
286+
}
287+
288+
$env:EFCoreJet_DefaultConnection = '${{ env.defaultConnection }}'
289+
& '${{ env.dotnetExecutable }}' test .\test\EFCore.Jet.FunctionalTests --configuration '${{ env.buildConfiguration }}' -p:FixedTestOrder=${{ env.deterministicTests }} --logger trx --verbosity detailed --blame-hang-timeout 3m --results-directory $shardDir --filter "FullyQualifiedName!~.FunctionalTests.Query."
290+
291+
#
292+
# Check for test runner crashes:
293+
#
294+
295+
$currentTestRunTrx = Get-ChildItem $shardDir -Filter '*.trx' | Sort-Object LastWriteTime | Select-Object -Last 1
296+
if ($null -eq $currentTestRunTrx) {
297+
echo 'Test runner log file is missing.'
298+
exit 3
299+
}
300+
301+
$currentTestRunDir = Join-Path $shardDir $currentTestRunTrx.BaseName
302+
if (Test-Path $currentTestRunDir) {
303+
if ($null -ne (Get-ChildItem $currentTestRunDir -Filter 'Sequence_*' -Recurse)) {
304+
# Split string because searching the log for that phrase should only show actual crashes and not this line.
305+
echo ('Test runner cras' + 'hed.')
306+
continue
307+
}
308+
}
309+
310+
echo 'Test runner ran until the end.'
311+
break
312+
}
313+
314+
$establishedGreenTestsFilePath = ".\test\EFCore.Jet.FunctionalTests\GreenTests\ace_${{ matrix.aceVersion }}_$('${{ matrix.dataAccessProviderType }}'.Replace(' ', '').ToLowerInvariant())_${{ matrix.aceArchitecture }}.txt"
315+
$failIfKeepsCrashing = Test-Path $establishedGreenTestsFilePath
316+
271317
if ($i -ge 3 -and $failIfKeepsCrashing) {
272318
echo 'Test runner keeps crashing.'
273319
exit 2
@@ -285,46 +331,51 @@ jobs:
285331
name: test-results_${{ env.matrixId }}
286332
path: |
287333
test\EFCore.Jet.Data.Tests\TestResults\*.trx
288-
test\EFCore.Jet.FunctionalTests\TestResults\*.trx
334+
test\EFCore.Jet.FunctionalTests\TestResults\**\*.trx
289335
test\EFCore.Jet.Tests\TestResults\*.trx
290336
- name: 'Check Tests: EFCore.Jet.FunctionalTests'
291337
if: env.skipTests != 'true'
292338
shell: pwsh
293339
run: |
294-
# Create text file with all tests that passed.
340+
# Collect and merge test results from all shards.
295341
$testResultsDir = '.\test\EFCore.Jet.FunctionalTests\TestResults'
296-
$currentTestRunTrx = Get-ChildItem $testResultsDir -Filter '*.trx' | Sort-Object LastWriteTime | Select-Object -Last 1
342+
$allTrxFiles = @(Get-ChildItem $testResultsDir -Filter '*.trx' -Recurse)
297343
298-
if ($null -eq $currentTestRunTrx) {
299-
echo 'Test runner log file is missing.'
344+
if ($allTrxFiles.Count -eq 0) {
345+
echo 'No test result files found.'
300346
exit 3
301347
}
302348
303-
$allTestsFilePath = Join-Path $currentTestRunTrx.DirectoryName ($currentTestRunTrx.BaseName + '_All.txt')
304-
(Select-Xml -Path $currentTestRunTrx.FullName -XPath "//ns:UnitTestResult" -Namespace @{"ns"="http://microsoft.com/schemas/VisualStudio/TeamTest/2010"}).Node | Sort-Object -Property testName -CaseSensitive | ForEach-Object { "$($_.outcome -eq 'Passed' ? 'P' : $_.outcome -eq 'NotExecuted' ? 'N' : $_.outcome -eq 'Failed' ? 'F' : 'U') $($_.testName)" } | Add-Content $allTestsFilePath
349+
$allNodes = @()
350+
foreach ($trxFile in $allTrxFiles) {
351+
$allNodes += (Select-Xml -Path $trxFile.FullName -XPath "//ns:UnitTestResult" -Namespace @{"ns"="http://microsoft.com/schemas/VisualStudio/TeamTest/2010"}).Node
352+
}
353+
354+
$allTestsFilePath = Join-Path $testResultsDir 'combined_All.txt'
355+
$allNodes | Sort-Object -Property testName -CaseSensitive | ForEach-Object { "$($_.outcome -eq 'Passed' ? 'P' : $_.outcome -eq 'NotExecuted' ? 'N' : $_.outcome -eq 'Failed' ? 'F' : 'U') $($_.testName)" } | Set-Content $allTestsFilePath
305356
306-
$greenTestsFilePath = Join-Path $currentTestRunTrx.DirectoryName ($currentTestRunTrx.BaseName + '_Passed.txt')
307-
Get-Content $allTestsFilePath | Where-Object { $_.StartsWith('P ') } | ForEach-Object { $_.Substring(2) } | Add-Content $greenTestsFilePath
357+
$greenTestsFilePath = Join-Path $testResultsDir 'combined_Passed.txt'
358+
Get-Content $allTestsFilePath | Where-Object { $_.StartsWith('P ') } | ForEach-Object { $_.Substring(2) } | Set-Content $greenTestsFilePath
308359
309360
# Compare test file against previously committed file.
310361
$establishedGreenTestsFilePath = ".\test\EFCore.Jet.FunctionalTests\GreenTests\ace_${{ matrix.aceVersion }}_$('${{ matrix.dataAccessProviderType }}'.Replace(' ', '').ToLowerInvariant())_${{ matrix.aceArchitecture }}.txt"
311362
312363
if (Test-Path $establishedGreenTestsFilePath) {
313364
$diffResult = Compare-Object (Get-Content $establishedGreenTestsFilePath) (Get-Content $greenTestsFilePath)
314-
365+
315366
$notGreenAnymore = $diffResult | Where-Object { $_.SideIndicator -eq '<=' } | Select-Object -ExpandProperty InputObject
316367
if ($null -ne $notGreenAnymore) {
317368
echo "`nThe following $(@($notGreenAnymore).Length) tests passed in previous runs, but didn't pass in this run:`n"
318369
$notGreenAnymore
319370
exit 1
320371
}
321-
372+
322373
echo 'All tests that passed in previous runs still passed in this run.'
323374
324375
$newlyGreenTests = $diffResult | Where-Object { $_.SideIndicator -eq '=>' } | Select-Object -ExpandProperty InputObject
325376
if ($newlyGreenTests.Length -gt 0) {
326377
Copy-Item $greenTestsFilePath $establishedGreenTestsFilePath -Force -Verbose
327-
378+
328379
echo "`nThe following new tests passed that did not pass before:`n"
329380
$newlyGreenTests
330381

‎.github/workflows/push.yml‎

Lines changed: 71 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -231,30 +231,31 @@ jobs:
231231
run: |
232232
$env:EFCoreJet_DefaultConnection = '${{ env.defaultConnection }}'
233233
& '${{ env.dotnetExecutable }}' test .\test\EFCore.Jet.Tests --configuration '${{ env.buildConfiguration }}' -p:FixedTestOrder=${{ env.deterministicTests }} --logger trx --verbosity detailed --blame-hang-timeout 3m
234-
- name: 'Run Tests: EFCore.Jet.FunctionalTests'
234+
- name: 'Run Tests: EFCore.Jet.FunctionalTests (Shard 1 - Query)'
235235
if: always() && env.skipTests != 'true'
236236
shell: pwsh
237237
run: |
238+
$shardDir = '.\test\EFCore.Jet.FunctionalTests\TestResults\shard1'
239+
238240
for ($i = 0; $i -lt 3; $i++) {
239-
if (Test-Path '.\test\EFCore.Jet.FunctionalTests\TestResults' -PathType Container) {
240-
Get-ChildItem '.\test\EFCore.Jet.FunctionalTests\TestResults' | Remove-Item -Recurse -Force
241+
if (Test-Path $shardDir -PathType Container) {
242+
Get-ChildItem $shardDir | Remove-Item -Recurse -Force
241243
}
242244
243245
$env:EFCoreJet_DefaultConnection = '${{ env.defaultConnection }}'
244-
& '${{ env.dotnetExecutable }}' test .\test\EFCore.Jet.FunctionalTests --configuration '${{ env.buildConfiguration }}' -p:FixedTestOrder=${{ env.deterministicTests }} --logger trx --verbosity detailed --blame-hang-timeout 3m
246+
& '${{ env.dotnetExecutable }}' test .\test\EFCore.Jet.FunctionalTests --configuration '${{ env.buildConfiguration }}' -p:FixedTestOrder=${{ env.deterministicTests }} --logger trx --verbosity detailed --blame-hang-timeout 3m --results-directory $shardDir --filter "FullyQualifiedName~.FunctionalTests.Query."
245247
246248
#
247249
# Check for test runner crashes:
248250
#
249251
250-
$testResultsDir = '.\test\EFCore.Jet.FunctionalTests\TestResults'
251-
$currentTestRunTrx = Get-ChildItem $testResultsDir -Filter '*.trx' | Sort-Object LastWriteTime | Select-Object -Last 1
252+
$currentTestRunTrx = Get-ChildItem $shardDir -Filter '*.trx' | Sort-Object LastWriteTime | Select-Object -Last 1
252253
if ($null -eq $currentTestRunTrx) {
253254
echo 'Test runner log file is missing.'
254255
exit 3
255256
}
256257
257-
$currentTestRunDir = Join-Path $testResultsDir $currentTestRunTrx.BaseName
258+
$currentTestRunDir = Join-Path $shardDir $currentTestRunTrx.BaseName
258259
if (Test-Path $currentTestRunDir) {
259260
if ($null -ne (Get-ChildItem $currentTestRunDir -Filter 'Sequence_*' -Recurse)) {
260261
# Split string because searching the log for that phrase should only show actual crashes and not this line.
@@ -269,7 +270,52 @@ jobs:
269270
270271
$establishedGreenTestsFilePath = ".\test\EFCore.Jet.FunctionalTests\GreenTests\ace_${{ matrix.aceVersion }}_$('${{ matrix.dataAccessProviderType }}'.Replace(' ', '').ToLowerInvariant())_${{ matrix.aceArchitecture }}.txt"
271272
$failIfKeepsCrashing = Test-Path $establishedGreenTestsFilePath
272-
273+
274+
if ($i -ge 3 -and $failIfKeepsCrashing) {
275+
echo 'Test runner keeps crashing.'
276+
exit 2
277+
}
278+
exit 0
279+
- name: 'Run Tests: EFCore.Jet.FunctionalTests (Shard 2 - Non-Query)'
280+
if: always() && env.skipTests != 'true'
281+
shell: pwsh
282+
run: |
283+
$shardDir = '.\test\EFCore.Jet.FunctionalTests\TestResults\shard2'
284+
285+
for ($i = 0; $i -lt 3; $i++) {
286+
if (Test-Path $shardDir -PathType Container) {
287+
Get-ChildItem $shardDir | Remove-Item -Recurse -Force
288+
}
289+
290+
$env:EFCoreJet_DefaultConnection = '${{ env.defaultConnection }}'
291+
& '${{ env.dotnetExecutable }}' test .\test\EFCore.Jet.FunctionalTests --configuration '${{ env.buildConfiguration }}' -p:FixedTestOrder=${{ env.deterministicTests }} --logger trx --verbosity detailed --blame-hang-timeout 3m --results-directory $shardDir --filter "FullyQualifiedName!~.FunctionalTests.Query."
292+
293+
#
294+
# Check for test runner crashes:
295+
#
296+
297+
$currentTestRunTrx = Get-ChildItem $shardDir -Filter '*.trx' | Sort-Object LastWriteTime | Select-Object -Last 1
298+
if ($null -eq $currentTestRunTrx) {
299+
echo 'Test runner log file is missing.'
300+
exit 3
301+
}
302+
303+
$currentTestRunDir = Join-Path $shardDir $currentTestRunTrx.BaseName
304+
if (Test-Path $currentTestRunDir) {
305+
if ($null -ne (Get-ChildItem $currentTestRunDir -Filter 'Sequence_*' -Recurse)) {
306+
# Split string because searching the log for that phrase should only show actual crashes and not this line.
307+
echo ('Test runner cras' + 'hed.')
308+
continue
309+
}
310+
}
311+
312+
echo 'Test runner ran until the end.'
313+
break
314+
}
315+
316+
$establishedGreenTestsFilePath = ".\test\EFCore.Jet.FunctionalTests\GreenTests\ace_${{ matrix.aceVersion }}_$('${{ matrix.dataAccessProviderType }}'.Replace(' ', '').ToLowerInvariant())_${{ matrix.aceArchitecture }}.txt"
317+
$failIfKeepsCrashing = Test-Path $establishedGreenTestsFilePath
318+
273319
if ($i -ge 3 -and $failIfKeepsCrashing) {
274320
echo 'Test runner keeps crashing.'
275321
exit 2
@@ -287,26 +333,31 @@ jobs:
287333
name: test-results_${{ env.matrixId }}
288334
path: |
289335
test\EFCore.Jet.Data.Tests\TestResults\*.trx
290-
test\EFCore.Jet.FunctionalTests\TestResults\*.trx
336+
test\EFCore.Jet.FunctionalTests\TestResults\**\*.trx
291337
test\EFCore.Jet.Tests\TestResults\*.trx
292338
- name: 'Check Tests: EFCore.Jet.FunctionalTests'
293339
if: env.skipTests != 'true'
294340
shell: pwsh
295341
run: |
296-
# Create text file with all tests that passed.
342+
# Collect and merge test results from all shards.
297343
$testResultsDir = '.\test\EFCore.Jet.FunctionalTests\TestResults'
298-
$currentTestRunTrx = Get-ChildItem $testResultsDir -Filter '*.trx' | Sort-Object LastWriteTime | Select-Object -Last 1
344+
$allTrxFiles = @(Get-ChildItem $testResultsDir -Filter '*.trx' -Recurse)
299345
300-
if ($null -eq $currentTestRunTrx) {
301-
echo 'Test runner log file is missing.'
346+
if ($allTrxFiles.Count -eq 0) {
347+
echo 'No test result files found.'
302348
exit 3
303349
}
304350
305-
$allTestsFilePath = Join-Path $currentTestRunTrx.DirectoryName ($currentTestRunTrx.BaseName + '_All.txt')
306-
(Select-Xml -Path $currentTestRunTrx.FullName -XPath "//ns:UnitTestResult" -Namespace @{"ns"="http://microsoft.com/schemas/VisualStudio/TeamTest/2010"}).Node | Sort-Object -Property testName -CaseSensitive | ForEach-Object { "$($_.outcome -eq 'Passed' ? 'P' : $_.outcome -eq 'NotExecuted' ? 'N' : $_.outcome -eq 'Failed' ? 'F' : 'U') $($_.testName)" } | Add-Content $allTestsFilePath
351+
$allNodes = @()
352+
foreach ($trxFile in $allTrxFiles) {
353+
$allNodes += (Select-Xml -Path $trxFile.FullName -XPath "//ns:UnitTestResult" -Namespace @{"ns"="http://microsoft.com/schemas/VisualStudio/TeamTest/2010"}).Node
354+
}
355+
356+
$allTestsFilePath = Join-Path $testResultsDir 'combined_All.txt'
357+
$allNodes | Sort-Object -Property testName -CaseSensitive | ForEach-Object { "$($_.outcome -eq 'Passed' ? 'P' : $_.outcome -eq 'NotExecuted' ? 'N' : $_.outcome -eq 'Failed' ? 'F' : 'U') $($_.testName)" } | Set-Content $allTestsFilePath
307358
308-
$greenTestsFilePath = Join-Path $currentTestRunTrx.DirectoryName ($currentTestRunTrx.BaseName + '_Passed.txt')
309-
Get-Content $allTestsFilePath | Where-Object { $_.StartsWith('P ') } | ForEach-Object { $_.Substring(2) } | Add-Content $greenTestsFilePath
359+
$greenTestsFilePath = Join-Path $testResultsDir 'combined_Passed.txt'
360+
Get-Content $allTestsFilePath | Where-Object { $_.StartsWith('P ') } | ForEach-Object { $_.Substring(2) } | Set-Content $greenTestsFilePath
310361
311362
# Compare test file against previously committed file.
312363
$establishedGreenTestsFilePath = ".\test\EFCore.Jet.FunctionalTests\GreenTests\ace_${{ matrix.aceVersion }}_$('${{ matrix.dataAccessProviderType }}'.Replace(' ', '').ToLowerInvariant())_${{ matrix.aceArchitecture }}.txt"
@@ -320,19 +371,19 @@ jobs:
320371
$notGreenAnymore
321372
exit 1
322373
}
323-
374+
324375
echo 'All tests that passed in previous runs still passed in this run.'
325376
326377
$newlyGreenTests = $diffResult | Where-Object { $_.SideIndicator -eq '=>' } | Select-Object -ExpandProperty InputObject
327378
if ($newlyGreenTests.Length -gt 0) {
328379
Copy-Item $greenTestsFilePath $establishedGreenTestsFilePath -Force -Verbose
329-
380+
330381
echo "`nThe following new tests passed that did not pass before:`n"
331382
$newlyGreenTests
332383
333384
$commitGreenTestsFile = $establishedGreenTestsFilePath
334385
echo "commitGreenTestsFile=$commitGreenTestsFile" >> $env:GITHUB_ENV
335-
}
386+
}
336387
}
337388
echo 'Check succeeded.'
338389
- name: 'Upload Green Tests'

0 commit comments

Comments
 (0)