Skip to content

Commit a73bf6e

Browse files
StuartWheaterStuartWheater
authored
Merge pull request #272 from datashield/ci-phase-iii
Initial CI "bug phase" support
2 parents f8d1dc4 + 5f5c197 commit a73bf6e

24 files changed

Lines changed: 139 additions & 86 deletions

‎azure-pipelines.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,59 @@ jobs:
401401
condition: succeededOrFailed()
402402
403403
404+
#####################################################################################
405+
# Essentially run devtools::test() on the checked out code for bug reporting.
406+
# The actual command is vary convoluted as it had to do some things
407+
# which are not default behaviour: output the results to a JUnit xml file, not stop
408+
# when a small number of errors have happened, run through the code coverage tool.
409+
# TODO: Tidy up variable names - use timestamps here.
410+
- bash: |
411+
412+
# junit reporter output is to test_results_bug.xml
413+
sudo R -q -e '
414+
library(testthat);
415+
output_file <- file("test_console_output_bug.txt");
416+
sink(output_file);
417+
sink(output_file, type = "message");
418+
junit_rep <- JunitReporter$new(file = "test_results_bug.xml");
419+
progress_rep <- ProgressReporter$new(max_failures = 999999);
420+
multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep));
421+
testthat::test_package("$(projectName)", filter = "__bug-|datachk_bug-|smk_bug-|arg_bug-|disc_bug-|smk_expt_bug-|expt_bug-|math_bug-", reporter = multi_rep, stop_on_failure = FALSE)'
422+
423+
cat test_console_output_bug.txt
424+
425+
mv test_results_bug.xml $(Pipeline.Workspace)/logs
426+
427+
workingDirectory: $(Pipeline.Workspace)/dsBaseClient
428+
displayName: 'Bug report output'
429+
condition: succeededOrFailed()
430+
431+
432+
#####################################################################################
433+
# Parse the JUnit file to see if there are any errors/warnings. If there are then
434+
# echo them so finding bugs should be easier.
435+
# This should run even if previous steps have failed.
436+
- bash: |
437+
438+
# Strip out when error and failure = 0 and count the number of times it does not.
439+
issue_count=$(sed 's/failures="0" errors="0"//' test_results_bug.xml | sed 's/errors="0" failures="0"//' | grep --count errors=)
440+
echo "Number of testsuites with issues: "$issue_count
441+
echo "Testsuites with issues:"
442+
sed 's/failures="0" errors="0"//' test_results_bug.xml | sed 's/errors="0" failures="0"//' | grep errors= > issues.log
443+
cat issues.log
444+
no_issue_count=$(sed 's/failures="0" errors="0"//' test_results_bug.xml | sed 's/errors="0" failures="0"//' | grep -v --count errors=)
445+
echo
446+
echo "Number of testsuites with no issues: "$no_issue_count
447+
echo "Testsuites with issues:"
448+
sed 's/failures="0" errors="0"//' test_results_bug.xml | sed 's/errors="0" failures="0"//' | grep "<testsuite " | grep -v errors= > no_issues.log
449+
cat no_issues.log
450+
exit 0
451+
452+
workingDirectory: $(Pipeline.Workspace)/logs
453+
displayName: 'Bug summary report output'
454+
condition: succeededOrFailed()
455+
456+
404457
#####################################################################################
405458
# Output some important version numbers to file. This gets added to the testStatus
406459
# commit so it can be parsed and used on the status table.

tests/testthat/atest-arg-ds.dataFrameSort.R renamed to tests/testthat/test-arg_bug-ds.dataFrameSort.R

File renamed without changes.

‎tests/testthat/test-expt-ds.mean.R‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ source("connection_to_datasets/init_testing_datasets.R")
1414
source("definition_tests/def-ds.mean.R")
1515

1616

17-
context("ds.mean()::expt::combine::multiple")
17+
context("ds.mean::expt::combine::multiple")
1818
test_that("combined data set",
1919
{
2020
connect.all.datasets()
@@ -28,7 +28,7 @@ test_that("combined data set",
2828
.test.mean.combined('D$NEGATIVE_NUMERIC',ds.test_env$local.values[,13])
2929
})
3030

31-
context("ds.mean()::expt::split::multiple")
31+
context("ds.mean::expt::split::multiple")
3232
test_that("split data set",
3333
{
3434
connect.all.datasets()
@@ -43,7 +43,7 @@ test_that("split data set",
4343
.test.mean.split('D$NEGATIVE_NUMERIC',ds.test_env$local.values.1[,13],ds.test_env$local.values.2[,13],ds.test_env$local.values.3[,13])
4444
})
4545

46-
context("ds.mean()::expt::single")
46+
context("ds.mean::expt::single")
4747
test_that("combined data set",
4848
{
4949
connect.dataset.1()
@@ -57,7 +57,7 @@ test_that("combined data set",
5757
.test.mean.combined('D$NEGATIVE_NUMERIC',ds.test_env$local.values.1[,13])
5858
})
5959

60-
context("ds.mean()::expt::large_values::single")
60+
context("ds.mean::expt::large_values::single")
6161
test_that("combined data set",
6262
{
6363
connect.dataset.1()
@@ -71,7 +71,7 @@ test_that("combined data set",
7171
.test.mean.large('D$NEGATIVE_NUMERIC',ds.test_env$local.values.1[,13])
7272
})
7373

74-
context("ds.mean()::expt::large_values::multiple")
74+
context("ds.mean::expt::large_values::multiple")
7575
test_that("combined data set",
7676
{
7777
connect.all.datasets()

‎tests/testthat/test-expt-ds.reshape.R‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ source("definition_tests/def-ds.reshape.R")
33
source("definition_tests/def-assign-stats.R")
44

55

6-
context("ds.reShape()::expt::multiple")
6+
context("ds.reShape::expt::multiple")
77
test_that("copy and transform",
88
{
99
init.testing.datasets()

‎tests/testthat/test-expt-ds.skewness.R‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
source("connection_to_datasets/init_testing_datasets.R")
1414
source("definition_tests/def-ds.skewness.R")
1515

16-
context("ds.skewness()::expt::combine::multiple")
16+
context("ds.skewness::expt::combine::multiple")
1717

1818
test_that("combined data set",
1919
{
@@ -28,7 +28,7 @@ test_that("combined data set",
2828
.test.skewness.combined('D$NEGATIVE_NUMERIC',ds.test_env$local.values[,13], method = 1)
2929
})
3030

31-
context("ds.skewness()::expt::split::multiple")
31+
context("ds.skewness::expt::split::multiple")
3232
test_that("split data set",
3333
{
3434
connect.all.datasets()
@@ -42,7 +42,7 @@ test_that("split data set",
4242
.test.skewness.split('D$NEGATIVE_NUMERIC',ds.test_env$local.values.1[,13],ds.test_env$local.values.2[,13],ds.test_env$local.values.3[,13], method = 2)
4343
})
4444

45-
context("ds.skewness()::expt::single")
45+
context("ds.skewness::expt::single")
4646
test_that("combined data set",
4747
{
4848
connect.dataset.1()
@@ -54,4 +54,4 @@ test_that("combined data set",
5454
.test.skewness.combined('D$NON_NEGATIVE_NUMERIC',ds.test_env$local.values.1[,11], method = 3)
5555
.test.skewness.combined('D$POSITIVE_NUMERIC',ds.test_env$local.values.1[,12], method = 3)
5656
.test.skewness.combined('D$NEGATIVE_NUMERIC',ds.test_env$local.values.1[,13], method = 3)
57-
})
57+
})

‎tests/testthat/test-expt-ds.var.R‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ source("connection_to_datasets/init_testing_datasets.R")
22
source("definition_tests/def-ds.var.R")
33

44

5-
context("ds.var()::expt::multiple")
5+
context("ds.var::expt::multiple")
66
test_that("combined data set",
77
{
88
connect.all.datasets()
@@ -16,7 +16,7 @@ test_that("combined data set",
1616
.test.var.combined('D$NEGATIVE_NUMERIC',ds.test_env$local.values[,13])
1717
})
1818

19-
context("ds.var()::expt::single")
19+
context("ds.var::expt::single")
2020
test_that("split data set",
2121
{
2222
connect.all.datasets()
@@ -30,7 +30,7 @@ test_that("split data set",
3030
.test.var.split('D$NEGATIVE_NUMERIC',ds.test_env$local.values.1[,13],ds.test_env$local.values.2[,13],ds.test_env$local.values.3[,13])
3131
})
3232

33-
context("ds.var()::expt::large_values::multiple")
33+
context("ds.var::expt::large_values::multiple")
3434
test_that("large values",
3535
{
3636
connect.all.datasets()
@@ -44,7 +44,7 @@ test_that("large values",
4444
.test.variance.large('D$NEGATIVE_NUMERIC',ds.test_env$local.values[,13])
4545
})
4646

47-
context("ds.var()::expt::large_values::single")
47+
context("ds.var::expt::large_values::single")
4848
test_that("large values",
4949
{
5050
connect.dataset.1()

tests/testthat/atest-expt-ds.asFactor.R renamed to tests/testthat/test-expt_bug-ds.asFactor.R

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
source("definition_tests/def-ds.asFactor.R")
22

33

4-
context("ds.asFactor()::expt::multiple")
4+
context("ds.asFactor::expt::multiple")
55
test_that("combined data set",
66
{
77
connect.all.datasets()
@@ -16,7 +16,7 @@ test_that("combined data set",
1616
.test.find.factor('D$FACTOR_INTEGER', 'FACTOR_INT.f',ds.test_env$local.values,15)
1717
})
1818

19-
context("ds.asFactor()::expt::single")
19+
context("ds.asFactor::expt::single")
2020
test_that("single dataset ",
2121
{
2222
connect.dataset.1()

tests/testthat/atest-expt-ds.dataFrameSort.R renamed to tests/testthat/test-expt_bug-ds.dataFrameSort.R

File renamed without changes.

tests/testthat/atest-expt-ds.make.R renamed to tests/testthat/test-expt_bug-ds.make.R

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ source("definition_tests/def-ds.make.R")
33
source("definition_tests/def-assign-stats.R")
44

55

6-
context("ds.make()::expt::copy_transform:single")
6+
context("ds.make::expt::copy_transform:single")
77
test_that("copy and transform",
88
{
99
connect.dataset.1()
@@ -20,7 +20,7 @@ test_that("copy and transform",
2020

2121

2222

23-
context("ds.make()::expt::copy_data::single")
23+
context("ds.make::expt::copy_data::single")
2424
test_that("copy data without any changes applied",
2525
{
2626
connect.dataset.1()
@@ -35,7 +35,7 @@ test_that("copy data without any changes applied",
3535
.test.copy.data('D$FACTOR_INTEGER','FACTOR_INTEGER_created',ds.test_env$local.values.1[,15])
3636
})
3737

38-
context("ds.make()::expt::copy_data::multiple")
38+
context("ds.make::expt::copy_data::multiple")
3939
test_that("copy data without any changes applied",
4040
{
4141
connect.all.datasets()
@@ -50,7 +50,7 @@ test_that("copy data without any changes applied",
5050
.test.copy.data('D$FACTOR_INTEGER','FACTOR_INTEGER_created',ds.test_env$local.values[,15])
5151
})
5252

53-
context("ds.make()::expt::sum_of_two_vectors::single")
53+
context("ds.make::expt::sum_of_two_vectors::single")
5454
test_that("apply the the sum some vectors",
5555
{
5656
connect.dataset.1()
@@ -82,7 +82,7 @@ test_that("apply the the sum some vectors",
8282
})
8383

8484

85-
context("ds.make()::expt::sum_of_two_vectors::multiple")
85+
context("ds.make::expt::sum_of_two_vectors::multiple")
8686
test_that("add some vectors",
8787
{
8888
connect.all.datasets()
@@ -111,7 +111,7 @@ test_that("add some vectors",
111111
.test.operation.vectors('D$NUMERIC','D$NON_NEGATIVE_NUMERIC','NUMERIC_created',"+",result.local = .add.vectors(ds.test_env$local.values[,10],ds.test_env$local.values[,11]))
112112
})
113113

114-
context("ds.make()::expt::product_of_two_vectors::single")
114+
context("ds.make::expt::product_of_two_vectors::single")
115115
test_that("multiply two vectors",
116116
{
117117
connect.dataset.1()
@@ -144,7 +144,7 @@ test_that("multiply two vectors",
144144

145145

146146

147-
context("ds.make()::expt::product_of_two_vectors::multiple")
147+
context("ds.make::expt::product_of_two_vectors::multiple")
148148
test_that("multiply two vectors",
149149
{
150150
connect.all.datasets()
@@ -173,7 +173,7 @@ test_that("multiply two vectors",
173173
.test.operation.vectors('D$NUMERIC','D$NON_NEGATIVE_NUMERIC','NUMERIC_created',"*",result.local = .mult.vectors(ds.test_env$local.values[,10],ds.test_env$local.values[,11]))
174174
})
175175

176-
context("ds.make()::expt::division_of_two_vectors::multiple")
176+
context("ds.make::expt::division_of_two_vectors::multiple")
177177
test_that("divide two vectors",
178178
{
179179
connect.all.datasets()
@@ -202,7 +202,7 @@ test_that("divide two vectors",
202202
})
203203

204204

205-
context("ds.make()::expt::division_of_two_vectors::single")
205+
context("ds.make::expt::division_of_two_vectors::single")
206206
test_that("divide two vectors",
207207
{
208208
connect.dataset.1()
@@ -231,7 +231,7 @@ test_that("divide two vectors",
231231
})
232232

233233

234-
context("ds.make()::expt::substract_of_two_vectors::multiple")
234+
context("ds.make::expt::substract_of_two_vectors::multiple")
235235
test_that("substract two vectors",
236236
{
237237
connect.all.datasets()
@@ -260,7 +260,7 @@ test_that("substract two vectors",
260260
})
261261

262262

263-
context("ds.make()::expt::substract_of_two_vectors::single")
263+
context("ds.make::expt::substract_of_two_vectors::single")
264264
test_that("substract two vectors",
265265
{
266266
connect.dataset.1()
@@ -289,7 +289,7 @@ test_that("substract two vectors",
289289
})
290290

291291

292-
context("ds.make()::expt::sum_of_constant::single")
292+
context("ds.make::expt::sum_of_constant::single")
293293
test_that("apply the the sum a vector and a constant value",
294294
{
295295
connect.dataset.1()
@@ -306,7 +306,7 @@ test_that("apply the the sum a vector and a constant value",
306306

307307
})
308308

309-
context("ds.make()::expt::substract_of_constant::single")
309+
context("ds.make::expt::substract_of_constant::single")
310310
test_that("substract a constant value to all the values of a vector",
311311
{
312312
connect.dataset.1()
@@ -324,7 +324,7 @@ test_that("substract a constant value to all the values of a vector",
324324
})
325325

326326

327-
context("ds.make()::expt::divide_of_constant::single")
327+
context("ds.make::expt::divide_of_constant::single")
328328
test_that("divide a constant value to all the values of a vector",
329329
{
330330
connect.dataset.1()
@@ -341,7 +341,7 @@ test_that("divide a constant value to all the values of a vector",
341341

342342
})
343343

344-
context("ds.make()::expt::multiply_of_constant::single")
344+
context("ds.make::expt::multiply_of_constant::single")
345345
test_that("multiply a constant value to all the values of a vector",
346346
{
347347
connect.dataset.1()
@@ -359,7 +359,7 @@ test_that("multiply a constant value to all the values of a vector",
359359
})
360360

361361

362-
context("ds.make()::expt::sum_of_constant::multiple")
362+
context("ds.make::expt::sum_of_constant::multiple")
363363
test_that("apply the the sum a vector and a constant value",
364364
{
365365
connect.all.datasets()
@@ -376,7 +376,7 @@ test_that("apply the the sum a vector and a constant value",
376376

377377
})
378378

379-
context("ds.make()::expt::substract_of_constant::multiple")
379+
context("ds.make::expt::substract_of_constant::multiple")
380380
test_that("substract a constant value to all the values of a vector",
381381
{
382382
connect.all.datasets()
@@ -394,7 +394,7 @@ test_that("substract a constant value to all the values of a vector",
394394
})
395395

396396

397-
context("ds.make()::expt::divide_of_constant::multiple")
397+
context("ds.make::expt::divide_of_constant::multiple")
398398
test_that("divide a constant value to all the values of a vector",
399399
{
400400
connect.all.datasets()
@@ -411,7 +411,7 @@ test_that("divide a constant value to all the values of a vector",
411411

412412
})
413413

414-
context("ds.make()::expt::multiply_of_constant::multiple")
414+
context("ds.make::expt::multiply_of_constant::multiple")
415415
test_that("multiply a constant value to all the values of a vector",
416416
{
417417
connect.all.datasets()

0 commit comments

Comments
 (0)