Skip to content

Commit 7f7b790

Browse files
StuartWheaterStuartWheater
authored
Merge pull request datashield#542 from StuartWheater/v6.3.1-RC2-dev
Durable performance tests
2 parents f63f4b0 + c310ce3 commit 7f7b790

19 files changed

Lines changed: 304 additions & 28 deletions

‎opal_azure-pipelines.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ jobs:
272272
# junit reporter output is to test_results.xml
273273
#
274274
# "_-|arg-|smk-|datachk-|disc-|math-|expt-|expt_smk-"
275-
# testthat::test_package("$(projectName)", filter = "_-|datachk-|smk-|arg-|disc-|smk_expt-|expt-|math-", reporter = multi_rep, stop_on_failure = FALSE)
275+
# testthat::test_package("$(projectName)", filter = "_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-", reporter = multi_rep, stop_on_failure = FALSE)
276276
sudo R -q -e '
277277
library(covr);
278278
dsbase.res <- covr::package_coverage(
@@ -288,7 +288,7 @@ jobs:
288288
progress_rep <- ProgressReporter$new(max_failures = 999999);
289289
multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep));
290290
options("default_driver" = "OpalDriver");
291-
testthat::test_package("$(projectName)", filter = "_-|datachk-|smk-|arg-|disc-|smk_expt-|expt-|math-", reporter = multi_rep, stop_on_failure = FALSE)
291+
testthat::test_package("$(projectName)", filter = "_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-", reporter = multi_rep, stop_on_failure = FALSE)
292292
'"'"'
293293
)
294294
);

‎tests/testthat/perf_files/armadillo_hp-laptop_quay.csv‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
"refer_name","rate","lower_tolerance","upper_tolerance"
2+
"conndisconn::perf::simple0","0.05471","0.5","2"
3+
"ds.abs::perf::0","0.5769","0.5","2"
4+
"ds.asInteger::perf:0","0.6371","0.5","2"
5+
"ds.asList::perf:0","1.296","0.5","2"
6+
"ds.asNumeric::perf:0","0.6456","0.5","2"
7+
"ds.assign::perf::0","1.349","0.5","2"
8+
"ds.class::perf::combine:0","1.490","0.5","2"
9+
"ds.colnames::perf:0","0.9683","0.5","2"
10+
"ds.exists::perf::combine:0","2.999","0.5","2"
11+
"ds.length::perf::combine:0","2.963","0.5","2"
12+
"ds.mean::perf::combine:0","2.976","0.5","2"
13+
"ds.mean::perf::split:0","2.758","0.5","2"
14+
"void::perf::void::0","23050","0.5","2"

‎tests/testthat/perf_files/default_perf_profile.csv‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
"refer_name","rate","lower_tolerance","upper_tolerance"
2+
"conndisconn::perf::simple0","0.229","0.5","2"
3+
"ds.abs::perf::0","1.796","0.5","2"
4+
"ds.asInteger::perf:0","1.538","0.5","2"
5+
"ds.asList::perf:0","3.061","0.5","2"
6+
"ds.asNumeric::perf:0","1.548","0.5","2"
7+
"ds.assign::perf::0","3.556","0.5","2"
8+
"ds.class::perf::combine:0","3.149","0.5","2"
9+
"ds.colnames::perf:0","2.584","0.5","2"
10+
"ds.exists::perf::combine:0","7.138","0.5","2"
11+
"ds.length::perf::combine:0","6.385","0.5","2"
12+
"ds.mean::perf::combine:0","5.982","0.5","2"
13+
"ds.mean::perf::split:0","6.923","0.5","2"
14+
"void::perf::void::0","41810.0","0.5","2"

‎tests/testthat/perf_files/opal_azure-pipeline.csv‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
"refer_name","rate","lower_tolerance","upper_tolerance"
2+
"conndisconn::perf::simple0","0.229","0.5","2"
3+
"ds.abs::perf::0","1.796","0.5","2"
4+
"ds.asInteger::perf:0","1.538","0.5","2"
5+
"ds.asList::perf:0","3.061","0.5","2"
6+
"ds.asNumeric::perf:0","1.548","0.5","2"
7+
"ds.assign::perf::0","3.556","0.5","2"
8+
"ds.class::perf::combine:0","3.149","0.5","2"
9+
"ds.colnames::perf:0","2.584","0.5","2"
10+
"ds.exists::perf::combine:0","7.138","0.5","2"
11+
"ds.length::perf::combine:0","6.385","0.5","2"
12+
"ds.mean::perf::combine:0","5.982","0.5","2"
13+
"ds.mean::perf::split:0","6.923","0.5","2"
14+
"void::perf::void::0","41810.0","0.5","2"

‎tests/testthat/perf_files/opal_hp-laptop_quay.csv‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
"refer_name","rate","lower_tolerance","upper_tolerance"
2+
"conndisconn::perf::simple0","0.145258829974366","0.5","2"
3+
"ds.abs::perf::0","0.685354162883962","0.5","2"
4+
"ds.asInteger::perf:0","0.757343916339244","0.5","2"
5+
"ds.asList::perf:0","1.39411478465579","0.5","2"
6+
"ds.asNumeric::perf:0","0.695929483688074","0.5","2"
7+
"ds.assign::perf::0","1.98579096107968","0.5","2"
8+
"ds.class::perf::combine:0","1.83959664387857","0.5","2"
9+
"ds.colnames::perf:0","1.38390626627527","0.5","2"
10+
"ds.exists::perf::combine:0","4.37111331491725","0.5","2"
11+
"ds.length::perf::combine:0","2.99769707214692","0.5","2"
12+
"ds.mean::perf::combine:0","3.14458208946095","0.5","2"
13+
"ds.mean::perf::split:0","4.258064355457","0.5","2"
14+
"void::perf::void::0","20588.7831697084","0.5","2"

‎tests/testthat/perf_tests/perf_rate.R‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#-------------------------------------------------------------------------------
2+
# Copyright (c) 2024 Arjuna Technologies, Newcastle upon Tyne. All rights reserved.
3+
#
4+
# This program and the accompanying materials
5+
# are made available under the terms of the GNU Public License v3.0.
6+
#
7+
# You should have received a copy of the GNU General Public License
8+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
9+
#-------------------------------------------------------------------------------
10+
11+
.perf.reference.filename <- 'perf_files/default_perf_profile.csv'
12+
13+
.perf.reference <- NULL
14+
15+
.load.pref <- function() {
16+
.perf.reference <<- read.csv(.perf.reference.filename, header = TRUE, sep = ",")
17+
}
18+
19+
perf.reference.save <- function(perf.ref.name, rate, tolerance.lower, tolerance.upper) {
20+
if (is.null(.perf.reference))
21+
load.pref()
22+
23+
.perf.reference[nrow(.perf.reference)+1,] <- c(perf.ref.name, rate, tolerance.lower, tolerance.upper)
24+
25+
write.csv(.perf.reference, .perf.reference.filename, row.names = FALSE)
26+
27+
.perf.reference <<- .perf.reference
28+
}
29+
30+
perf.reference.rate <- function(perf.ref.name) {
31+
if (is.null(.perf.reference))
32+
.load.pref()
33+
34+
return(as.numeric(.perf.reference[which(.perf.reference$refer_name == perf.ref.name),]$rate))
35+
}
36+
37+
perf.reference.tolerance.lower <- function(perf.ref.name) {
38+
if (is.null(.perf.reference))
39+
.load.pref()
40+
41+
return(as.numeric(.perf.reference[which(.perf.reference$refer_name == perf.ref.name),]$lower_tolerance))
42+
}
43+
44+
perf.reference.tolerance.upper <- function(perf.ref.name) {
45+
if (is.null(.perf.reference))
46+
.load.pref()
47+
48+
return(as.numeric(.perf.reference[which(.perf.reference$refer_name == perf.ref.name),]$upper_tolerance))
49+
}
50+

‎tests/testthat/setup.R‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ library(DSMolgenisArmadillo)
2626
library(DSLite)
2727

2828
source("dstest_functions/ds_expect_variables.R")
29+
source("perf_tests/perf_rate.R")
2930
source("connection_to_datasets/login_details.R")
3031
source("connection_to_datasets/init_testing_datasets.R")
3132
source("connection_to_datasets/init_studies_datasets.R")

‎tests/testthat/test-perf-conndisconn.R‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,22 @@ test_that("simple connect - disconnect performance", {
3434
.count <- .count + 1
3535
.current.time <- Sys.time()
3636
}
37-
expect_true(TRUE)
3837

39-
print(paste("conndisconn::perf::simple::0:", format(.count / (difftime(.current.time, .start.time, units = "secs")[[1]]), digits = 8)))
38+
.current.rate <- .count / (difftime(.current.time, .start.time, units = "secs")[[1]])
39+
.reference.rate <- perf.reference.rate("conndisconn::perf::simple0")
40+
if (any(length(.reference.rate) == 0) || any(is.null(.reference.rate))) {
41+
print(paste("conndisconn::perf::simple0 ", .current.rate, 0.5, 2.0))
42+
perf.reference.save("conndisconn::perf::simple0", .current.rate, 0.5, 2.0)
43+
} else {
44+
print(paste("conndisconn::perf::simple0 ", format(.current.rate, digits = 8), ", ", format(100.0 * .current.rate / .reference.rate, digits = 4), "%", sep = ''))
45+
}
46+
47+
.reference.rate <- perf.reference.rate("conndisconn::perf::simple0")
48+
.reference.tolerance.lower <- perf.reference.tolerance.lower("conndisconn::perf::simple0")
49+
.reference.tolerance.upper <- perf.reference.tolerance.upper("conndisconn::perf::simple0")
50+
51+
expect_gt(.current.rate, .reference.rate * .reference.tolerance.lower, label = "Observed rate", expected.label = "lower threshold on rate")
52+
expect_lt(.current.rate, .reference.rate * .reference.tolerance.upper, label = "Observed rate", expected.label = "upper threshold on rate")
4053
})
4154

4255
#

‎tests/testthat/test-perf-ds.abs.R‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,22 @@ test_that("combine - performance", {
3232
.count <- .count + 1
3333
.current.time <- Sys.time()
3434
}
35-
expect_true(TRUE)
3635

37-
print(paste("ds.abs::perf::0:", format(.count / (difftime(.current.time, .start.time, units = "secs")[[1]]), digits = 8)))
36+
.current.rate <- .count / (difftime(.current.time, .start.time, units = "secs")[[1]])
37+
.reference.rate <- perf.reference.rate("ds.abs::perf::0")
38+
if (any(length(.reference.rate) == 0) || any(is.null(.reference.rate))) {
39+
print(paste("ds.abs::perf::0 ", .current.rate, 0.5, 2.0))
40+
perf.reference.save("ds.abs::perf::0", .current.rate, 0.5, 2.0)
41+
} else {
42+
print(paste("ds.abs::perf::0 ", format(.current.rate, digits = 8), ", ", format(100.0 * .current.rate / .reference.rate, digits = 4), "%", sep = ''))
43+
}
44+
45+
.reference.rate <- perf.reference.rate("ds.abs::perf::0")
46+
.reference.tolerance.lower <- perf.reference.tolerance.lower("ds.abs::perf::0")
47+
.reference.tolerance.upper <- perf.reference.tolerance.upper("ds.abs::perf::0")
48+
49+
expect_gt(.current.rate, .reference.rate * .reference.tolerance.lower, label = "Observed rate", expected.label = "lower threshold on rate")
50+
expect_lt(.current.rate, .reference.rate * .reference.tolerance.upper, label = "Observed rate", expected.label = "upper threshold on rate")
3851
})
3952

4053
#

‎tests/testthat/test-perf-ds.asInteger.R‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,22 @@ test_that("combine - performance", {
3232
.count <- .count + 1
3333
.current.time <- Sys.time()
3434
}
35-
expect_true(TRUE)
3635

37-
print(paste("ds.asInteger::perf::0:", format(.count / (difftime(.current.time, .start.time, units = "secs")[[1]]), digits = 8)))
36+
.current.rate <- .count / (difftime(.current.time, .start.time, units = "secs")[[1]])
37+
.reference.rate <- perf.reference.rate("ds.asInteger::perf:0")
38+
if (any(length(.reference.rate) == 0) || any(is.null(.reference.rate))) {
39+
print(paste("ds.asInteger::perf:0 ", .current.rate, 0.5, 2.0))
40+
perf.reference.save("ds.asInteger::perf:0", .current.rate, 0.5, 2.0)
41+
} else {
42+
print(paste("ds.asInteger::perf:0 ", format(.current.rate, digits = 8), ", ", format(100.0 * .current.rate / .reference.rate, digits = 4), "%", sep = ''))
43+
}
44+
45+
.reference.rate <- perf.reference.rate("ds.asInteger::perf:0")
46+
.reference.tolerance.lower <- perf.reference.tolerance.lower("ds.asInteger::perf:0")
47+
.reference.tolerance.upper <- perf.reference.tolerance.upper("ds.asInteger::perf:0")
48+
49+
expect_gt(.current.rate, .reference.rate * .reference.tolerance.lower, label = "Observed rate", expected.label = "lower threshold on rate")
50+
expect_lt(.current.rate, .reference.rate * .reference.tolerance.upper, label = "Observed rate", expected.label = "upper threshold on rate")
3851
})
3952

4053
#

0 commit comments

Comments
 (0)