Skip to content

Commit dbe45fb

Browse files
williamjallenwilliamjallen
authored
Run newer tests in transactions (#3375)
We currently apply the `RUN_SERIAL` property to tests which need an empty database, so they can wipe the database without conflicting with other tests. The growing number of tests using `RUN_SERIAL` bottlenecks our ability to parallelize the test suite. This PR adds a new testing database and switches most of Laravel tests to use it, running each test in a transaction so the database is always clean. This resulted in a 35-second reduction in overall running time for the test suite on my development machine.
1 parent 48486ba commit dbe45fb

69 files changed

Lines changed: 296 additions & 190 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CMakeLists.txt‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ find_program(NPX_EXE NAMES npx PATHS c:/usr/bin/npx /usr/bin/npx REQUIRED)
5959
set(PHPUNIT "${CDash_SOURCE_DIR}/vendor/bin/phpunit")
6060

6161
set_property(GLOBAL PROPERTY INSTALL_CDASH_COUNT "0")
62-
function(cdash_install TestName)
62+
function(cdash_install TestName DatabaseName)
6363
add_test(
6464
NAME ${TestName}
6565
COMMAND ${PHP_EXE} ${testing_dir}/singletest.php ${testing_dir}/test_install.php
@@ -69,6 +69,13 @@ function(cdash_install TestName)
6969
FAIL_REGULAR_EXPRESSION ".*Failures: [1-9]+.*;.*Exceptions: [1-9]+.*"
7070
ATTACHED_FILES_ON_FAIL "${CDash_SOURCE_DIR}/storage/logs/cdash.log"
7171
)
72+
73+
if(DatabaseName)
74+
set_tests_properties(
75+
${TestName} PROPERTIES
76+
ENVIRONMENT "DB_DATABASE=${DatabaseName}"
77+
)
78+
endif()
7279
endfunction()
7380

7481
# add the new tests directory

‎app/cdash/tests/CMakeLists.txt‎

Lines changed: 128 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# We define these here so they can be referenced in functions below.
2-
cdash_install(install_1)
3-
cdash_install(install_2)
2+
cdash_install(install_1 "")
3+
cdash_install(install_2 "")
4+
cdash_install(install_clean_database testing_clean)
45

56
# function to add a new PHP based coverage test to CDash
67
function(add_php_test TestName)
@@ -65,6 +66,17 @@ function(add_feature_test TestName)
6566
)
6667
endfunction()
6768

69+
# Tests which run entirely within a transaction, leaving no trace in the database.
70+
function(add_feature_test_in_transaction TestName)
71+
add_laravel_test(${TestName})
72+
set_property(TEST ${TestName} APPEND PROPERTY DEPENDS install_clean_database)
73+
set_tests_properties(
74+
${TestName} PROPERTIES
75+
ATTACHED_FILES_ON_FAIL "${CDash_SOURCE_DIR}/storage/logs/cdash.log"
76+
ENVIRONMENT "DB_DATABASE=testing_clean"
77+
)
78+
endfunction()
79+
6880
function(add_browser_test TestName)
6981
add_feature_test(${TestName})
7082
set_property(TEST ${TestName} APPEND PROPERTY RESOURCE_LOCK
@@ -158,104 +170,162 @@ add_legacy_unit_test(/CDash/XmlHandler/UpdateHandler)
158170
set_tests_properties(/CDash/XmlHandler/UpdateHandler PROPERTIES DEPENDS /CDash/XmlHandler/TestingHandler)
159171

160172
###################################################################################################
161-
# Tests from this point to the next cdash_install() are expected to be completely independent.
162-
# Tests may wipe the database, provided that they use RUN_SERIAL. All tests in this section should
163-
# depend upon the last test prior to this section.
173+
# Tests in this section are expected to run against the testing_clean database in transactions,
174+
# such that the database is empty at any given point.
164175

165-
add_feature_test(/Feature/LoginAndRegistration)
176+
add_feature_test_in_transaction(/Feature/LoginAndRegistration)
166177

167-
add_feature_test(/Feature/ProjectPermissions)
178+
add_feature_test_in_transaction(/Feature/ProjectPermissions)
168179

169-
add_feature_test(/Feature/SlowPageTest)
180+
add_feature_test_in_transaction(/Feature/SlowPageTest)
170181

171-
add_feature_test(/Feature/GitHubWebhook)
182+
add_feature_test_in_transaction(/Feature/GitHubWebhook)
172183

173-
add_feature_test(/Feature/GlobalBannerTest)
184+
add_feature_test_in_transaction(/Feature/GlobalBannerTest)
174185

175-
add_legacy_unit_test(/CDash/Lib/Repository/GitHub)
176-
set_tests_properties(/CDash/Lib/Repository/GitHub PROPERTIES
177-
DEPENDS /CDash/XmlHandler/UpdateHandler
178-
DISABLED "$<STREQUAL:${CDASH_IMAGE},ubi>"
179-
)
186+
add_feature_test_in_transaction(/Feature/GraphQL/BuildTypeTest)
180187

181-
add_legacy_unit_test(/CDash/Messaging/Subscription/CommitAuthorSubscriptionBuilder)
182-
set_tests_properties(/CDash/Messaging/Subscription/CommitAuthorSubscriptionBuilder PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
188+
add_feature_test_in_transaction(/Feature/ProjectInvitationAcceptanceTest)
183189

184-
add_legacy_unit_test(/CDash/Messaging/Subscription/UserSubscriptionBuilder)
185-
set_tests_properties(/CDash/Messaging/Subscription/UserSubscriptionBuilder PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
190+
add_feature_test_in_transaction(/Feature/GraphQL/FilterTest)
186191

187-
add_legacy_unit_test(/CDash/Messaging/Topic/UpdateErrorTopic)
188-
set_tests_properties(/CDash/Messaging/Topic/UpdateErrorTopic PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
192+
add_feature_test_in_transaction(/Feature/GraphQL/QueryTypeTest)
189193

190-
add_legacy_unit_test(/CDash/Service/RepositoryService)
191-
set_tests_properties(/CDash/Service/RepositoryService PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
194+
add_feature_test_in_transaction(/Feature/GraphQL/UserTypeTest)
192195

193-
# Set this to RUN_SERIAL as we are changing the config and don't want to clobber other tests.
194-
# Only requires exclusive .env access.
195-
add_feature_test(/Feature/SubmissionValidation)
196-
set_property(TEST /Feature/SubmissionValidation APPEND PROPERTY RUN_SERIAL TRUE)
196+
add_feature_test_in_transaction(/Feature/GraphQL/ProjectTypeTest)
197197

198-
add_feature_test(/Feature/ProjectInvitationAcceptanceTest)
198+
add_feature_test_in_transaction(/Feature/GraphQL/SiteTypeTest)
199199

200-
add_feature_test(/Feature/GraphQL/FilterTest)
201-
set_property(TEST /Feature/GraphQL/FilterTest APPEND PROPERTY RUN_SERIAL TRUE)
200+
add_feature_test_in_transaction(/Feature/GraphQL/ConfigureTypeTest)
202201

203-
add_feature_test(/Feature/GraphQL/QueryTypeTest)
202+
add_feature_test_in_transaction(/Feature/GraphQL/TestTypeTest)
204203

205-
add_feature_test(/Feature/GraphQL/UserTypeTest)
204+
add_feature_test_in_transaction(/Feature/GraphQL/TestMeasurementTypeTest)
206205

207-
add_feature_test(/Feature/GraphQL/ProjectTypeTest)
208-
set_property(TEST /Feature/GraphQL/ProjectTypeTest APPEND PROPERTY RUN_SERIAL TRUE)
206+
add_feature_test_in_transaction(/Feature/GraphQL/NoteTypeTest)
209207

210-
add_feature_test(/Feature/GraphQL/SiteTypeTest)
211-
set_property(TEST /Feature/GraphQL/SiteTypeTest APPEND PROPERTY RUN_SERIAL TRUE)
208+
add_feature_test_in_transaction(/Feature/GraphQL/CoverageTypeTest)
212209

213-
add_feature_test(/Feature/GraphQL/BuildTypeTest)
210+
add_feature_test_in_transaction(/Feature/GraphQL/DynamicAnalysisTypeTest)
214211

215-
add_feature_test(/Feature/GraphQL/ConfigureTypeTest)
212+
add_feature_test_in_transaction(/Feature/GraphQL/DynamicAnalysisDefectTypeTest)
216213

217-
add_feature_test(/Feature/GraphQL/TestTypeTest)
214+
add_feature_test_in_transaction(/Feature/GraphQL/ProjectInvitationTypeTest)
218215

219-
add_feature_test(/Feature/GraphQL/TestMeasurementTypeTest)
216+
add_feature_test_in_transaction(/Feature/GraphQL/LabelTypeTest)
220217

221-
add_feature_test(/Feature/GraphQL/NoteTypeTest)
218+
add_feature_test_in_transaction(/Feature/GraphQL/TargetTypeTest)
222219

223-
add_feature_test(/Feature/GraphQL/CoverageTypeTest)
220+
add_feature_test_in_transaction(/Feature/GraphQL/BuildCommandTypeTest)
224221

225-
add_feature_test(/Feature/GraphQL/DynamicAnalysisTypeTest)
222+
add_feature_test_in_transaction(/Feature/GraphQL/BuildCommandOutputTypeTest)
226223

227-
add_feature_test(/Feature/GraphQL/DynamicAnalysisDefectTypeTest)
224+
add_feature_test_in_transaction(/Feature/GraphQL/UpdateTypeTest)
228225

229-
add_feature_test(/Feature/GraphQL/ProjectInvitationTypeTest)
226+
add_feature_test_in_transaction(/Feature/GraphQL/UpdateFileTypeTest)
230227

231-
add_feature_test(/Feature/GraphQL/LabelTypeTest)
228+
add_feature_test_in_transaction(/Feature/RouteAccessTest)
232229

233-
add_feature_test(/Feature/GraphQL/TargetTypeTest)
230+
add_feature_test_in_transaction(/Feature/Monitor)
234231

235-
add_feature_test(/Feature/GraphQL/BuildCommandTypeTest)
232+
add_feature_test_in_transaction(/Feature/PasswordRotation)
236233

237-
add_feature_test(/Feature/GraphQL/BuildCommandOutputTypeTest)
234+
add_feature_test_in_transaction(/Feature/PurgeUnusedProjectsCommand)
238235

239-
add_feature_test(/Feature/GraphQL/UpdateTypeTest)
236+
add_feature_test_in_transaction(/Feature/Jobs/PruneJobsTest)
240237

241-
add_feature_test(/Feature/GraphQL/UpdateFileTypeTest)
238+
add_feature_test_in_transaction(/Feature/Jobs/PruneAuthTokensTest)
242239

243-
add_feature_test(/Feature/Submission/Instrumentation/BuildInstrumentationTest)
244-
add_feature_test(/Feature/Submission/Tests/TestXMLTest)
240+
add_feature_test_in_transaction(/Feature/UserCommand)
245241

246-
add_feature_test(/Feature/RouteAccessTest)
242+
add_feature_test_in_transaction(/Feature/RemoteWorkers)
243+
set_property(TEST /Feature/RemoteWorkers APPEND PROPERTY
244+
DISABLED "$<NOT:$<STREQUAL:${CDASH_STORAGE_TYPE},local>>"
245+
)
247246

248-
add_feature_test(/Feature/Monitor)
249-
set_property(TEST /Feature/Monitor APPEND PROPERTY RUN_SERIAL TRUE)
247+
add_feature_test_in_transaction(/Feature/Jobs/NotifyExpiringAuthTokensTest)
250248

251-
add_feature_test(/Feature/PasswordRotation)
249+
add_feature_test_in_transaction(/Feature/Jobs/PruneUploadsTest)
252250

253-
add_browser_test(/Browser/Pages/SitesIdPageTest)
251+
add_feature_test_in_transaction(/Feature/Mail/AuthTokenExpiringTest)
252+
253+
add_feature_test_in_transaction(/Feature/Mail/AuthTokenExpiredTest)
254+
255+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/CreateProjectTest)
256+
257+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/UpdateSiteDescriptionTest)
258+
259+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/InviteToProjectTest)
260+
261+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/RevokeProjectInvitationTest)
262+
263+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/ChangeProjectRoleTest)
264+
265+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/RemoveProjectUserTest)
266+
267+
add_feature_test_in_transaction(/Feature/Traits/UpdatesSiteInformationTest)
268+
269+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/ChangeGlobalRoleTest)
270+
271+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/JoinProjectTest)
272+
273+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/CreateGlobalInvitationTest)
274+
275+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/RevokeGlobalInvitationTest)
276+
277+
add_feature_test_in_transaction(/Feature/GlobalInvitationAcceptanceTest)
278+
279+
add_feature_test_in_transaction(/Feature/GraphQL/GlobalInvitationTypeTest)
280+
281+
add_feature_test_in_transaction(/Feature/GraphQL/Mutations/RemoveUserTest)
282+
283+
add_feature_test_in_transaction(/Feature/Jobs/PruneBuildsTest)
284+
285+
add_feature_test_in_transaction(/Feature/Jobs/PruneEmailsTest)
286+
287+
add_feature_test_in_transaction(/Feature/Services/ProjectServiceTest)
288+
289+
add_feature_test_in_transaction(/Feature/AutoRemoveBuildsCommand)
290+
291+
###################################################################################################
292+
# Tests from this point to the next cdash_install() are expected to be completely independent.
293+
# Tests may wipe the database, provided that they use RUN_SERIAL. All tests in this section should
294+
# depend upon the last test prior to this section.
295+
296+
add_legacy_unit_test(/CDash/Lib/Repository/GitHub)
297+
set_tests_properties(/CDash/Lib/Repository/GitHub PROPERTIES
298+
DEPENDS /CDash/XmlHandler/UpdateHandler
299+
DISABLED "$<STREQUAL:${CDASH_IMAGE},ubi>"
300+
)
301+
302+
add_legacy_unit_test(/CDash/Messaging/Subscription/CommitAuthorSubscriptionBuilder)
303+
set_tests_properties(/CDash/Messaging/Subscription/CommitAuthorSubscriptionBuilder PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
304+
305+
add_legacy_unit_test(/CDash/Messaging/Subscription/UserSubscriptionBuilder)
306+
set_tests_properties(/CDash/Messaging/Subscription/UserSubscriptionBuilder PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
307+
308+
add_legacy_unit_test(/CDash/Messaging/Topic/UpdateErrorTopic)
309+
set_tests_properties(/CDash/Messaging/Topic/UpdateErrorTopic PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
310+
311+
add_legacy_unit_test(/CDash/Service/RepositoryService)
312+
set_tests_properties(/CDash/Service/RepositoryService PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
313+
314+
# Set this to RUN_SERIAL as we are changing the config and don't want to clobber other tests.
315+
# Only requires exclusive .env access.
316+
add_feature_test(/Feature/SubmissionValidation)
317+
set_property(TEST /Feature/SubmissionValidation APPEND PROPERTY RUN_SERIAL TRUE)
254318

255319
# Technically this test doesn't have to run serially. It just needs to have exclusive access to the .env
256320
add_feature_test(/Feature/LdapIntegration)
257321
set_property(TEST /Feature/LdapIntegration APPEND PROPERTY RUN_SERIAL TRUE)
258322

323+
add_feature_test(/Feature/Submission/Instrumentation/BuildInstrumentationTest)
324+
325+
add_feature_test(/Feature/Submission/Tests/TestXMLTest)
326+
327+
add_browser_test(/Browser/Pages/SitesIdPageTest)
328+
259329
add_browser_test(/Browser/Pages/ProjectSitesPageTest)
260330

261331
add_browser_test(/Browser/Pages/ProjectMembersPageTest)
@@ -265,17 +335,6 @@ add_browser_test(/Browser/Pages/ProjectBuildsPageTest)
265335
add_browser_test(/Browser/Pages/ProjectsPageTest)
266336
set_property(TEST /Browser/Pages/ProjectsPageTest APPEND PROPERTY RUN_SERIAL TRUE)
267337

268-
add_feature_test(/Feature/PurgeUnusedProjectsCommand)
269-
set_property(TEST /Feature/PurgeUnusedProjectsCommand APPEND PROPERTY RUN_SERIAL TRUE)
270-
271-
# This could theoretically conflict with another test removing jobs from the submission tables in the future
272-
add_feature_test(/Feature/Jobs/PruneJobsTest)
273-
274-
add_feature_test(/Feature/Jobs/PruneAuthTokensTest)
275-
set_property(TEST /Feature/Jobs/PruneAuthTokensTest APPEND PROPERTY RUN_SERIAL TRUE)
276-
277-
add_feature_test(/Feature/UserCommand)
278-
279338
add_legacy_unit_test(/CDash/Database)
280339
set_tests_properties(/CDash/Database PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
281340

@@ -306,54 +365,6 @@ set_tests_properties(/CDash/Messaging/Topic/TestFailureTopic PROPERTIES DEPENDS
306365
add_legacy_unit_test(/CDash/Messaging/Topic/TopicDecorator)
307366
set_tests_properties(/CDash/Messaging/Topic/TopicDecorator PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
308367

309-
add_feature_test(/Feature/RemoteWorkers)
310-
set_property(TEST /Feature/RemoteWorkers APPEND PROPERTY
311-
DISABLED "$<NOT:$<STREQUAL:${CDASH_STORAGE_TYPE},local>>"
312-
)
313-
314-
add_feature_test(/Feature/Jobs/NotifyExpiringAuthTokensTest)
315-
set_property(TEST /Feature/Jobs/NotifyExpiringAuthTokensTest APPEND PROPERTY RUN_SERIAL TRUE)
316-
317-
add_feature_test(/Feature/Jobs/PruneUploadsTest)
318-
319-
add_feature_test(/Feature/Mail/AuthTokenExpiringTest)
320-
321-
add_feature_test(/Feature/Mail/AuthTokenExpiredTest)
322-
323-
add_feature_test(/Feature/GraphQL/Mutations/CreateProjectTest)
324-
325-
add_feature_test(/Feature/GraphQL/Mutations/UpdateSiteDescriptionTest)
326-
327-
add_feature_test(/Feature/GraphQL/Mutations/InviteToProjectTest)
328-
329-
add_feature_test(/Feature/GraphQL/Mutations/RevokeProjectInvitationTest)
330-
331-
add_feature_test(/Feature/GraphQL/Mutations/ChangeProjectRoleTest)
332-
333-
add_feature_test(/Feature/GraphQL/Mutations/RemoveProjectUserTest)
334-
335-
add_feature_test(/Feature/Traits/UpdatesSiteInformationTest)
336-
337-
add_feature_test(/Feature/GraphQL/Mutations/ChangeGlobalRoleTest)
338-
339-
add_feature_test(/Feature/GraphQL/Mutations/JoinProjectTest)
340-
341-
# Needs RUN_SERIAL to verify that no unexpected invitations are created
342-
add_feature_test(/Feature/GraphQL/Mutations/CreateGlobalInvitationTest)
343-
set_property(TEST /Feature/GraphQL/Mutations/CreateGlobalInvitationTest APPEND PROPERTY RUN_SERIAL TRUE)
344-
345-
add_feature_test(/Feature/GraphQL/Mutations/RevokeGlobalInvitationTest)
346-
347-
# Needs RUN_SERIAL to verify that only the expected users are created
348-
add_feature_test(/Feature/GlobalInvitationAcceptanceTest)
349-
set_property(TEST /Feature/GlobalInvitationAcceptanceTest APPEND PROPERTY RUN_SERIAL TRUE)
350-
351-
# Requires exclusive access to the global_invitations table
352-
add_feature_test(/Feature/GraphQL/GlobalInvitationTypeTest)
353-
set_property(TEST /Feature/GraphQL/GlobalInvitationTypeTest APPEND PROPERTY RUN_SERIAL TRUE)
354-
355-
add_feature_test(/Feature/GraphQL/Mutations/RemoveUserTest)
356-
357368
add_browser_test(/Browser/Pages/UsersPageTest)
358369

359370
add_browser_test(/Browser/Pages/BuildFilesPageTest)
@@ -413,19 +424,6 @@ set_tests_properties(configurewarnings PROPERTIES DEPENDS /CDash/XmlHandler/Upda
413424
add_php_test(lotsofsubprojects)
414425
set_tests_properties(lotsofsubprojects PROPERTIES DEPENDS /CDash/XmlHandler/UpdateHandler)
415426

416-
add_feature_test(/Feature/AutoRemoveBuildsCommand)
417-
418-
# Must run serially since it can delete builds other than the ones it creates.
419-
add_feature_test(/Feature/Jobs/PruneBuildsTest)
420-
set_property(TEST /Feature/Jobs/PruneBuildsTest APPEND PROPERTY RUN_SERIAL TRUE)
421-
422-
# Theoretically it's possible for this test to conflict with other tests because this test can
423-
# delete email history records other than the ones it creates. The odds of such a conflict
424-
# occurring are too low to justify running serially.
425-
add_feature_test(/Feature/Jobs/PruneEmailsTest)
426-
427-
add_feature_test(/Feature/Services/ProjectServiceTest)
428-
429427
###################################################################################################
430428

431429
set_property(TEST install_2 APPEND PROPERTY DEPENDS

0 commit comments

Comments
 (0)