Skip to content

Commit 240d60a

Browse files
Merge branch 'refactor/perf-batch-2' into perf-batch-1
2 parents 1c2bebf + 2da6e87 commit 240d60a

97 files changed

Lines changed: 1005 additions & 603 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.Rbuildignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,4 @@
2323
^\.circleci$
2424
^\.circleci/config\.yml$
2525
^\.github$
26+
^man-roxygen$

DESCRIPTION

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ Description: Base 'DataSHIELD' functions for the client side. 'DataSHIELD' is a
55
you to do non-disclosive federated analysis on sensitive data. 'DataSHIELD' analytic functions have
66
been designed to only share non disclosive summary statistics, with built in automated output
77
checking based on statistical disclosure control. With data sites setting the threshold values for
8-
the automated output checks. For more details, see citation("dsBaseClient").
8+
the automated output checks. For more details, see citation('dsBaseClient').
99
Authors@R: c(person(given = "Paul",
1010
family = "Burton",
1111
role = c("aut"),
@@ -56,7 +56,12 @@ Authors@R: c(person(given = "Paul",
5656
family = "Wheater",
5757
role = c("aut", "cre"),
5858
email = "stuart.wheater@arjuna.com",
59-
comment = c(ORCID = "0009-0003-2419-1964")))
59+
comment = c(ORCID = "0009-0003-2419-1964")),
60+
person(given = "Tim",
61+
family = "Cadman",
62+
role = c("aut"),
63+
comment = c(ORCID = "0000-0002-7682-5645",
64+
affiliation = "Genomics Coordination Centre, UMCG, Netherlands")))
6065
License: GPL-3
6166
Depends:
6267
R (>= 4.0.0),
@@ -82,6 +87,6 @@ Suggests:
8287
DSOpal,
8388
DSMolgenisArmadillo,
8489
DSLite
85-
RoxygenNote: 7.3.3
90+
RoxygenNote: 8.0.0
8691
Encoding: UTF-8
8792
Language: en-GB

R/checkClass.R

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
checkClass <- function(datasources=NULL, obj=NULL){
1414
# check the class of the input object
1515
cally <- call("classDS", obj)
16-
classesBy <- DSI::datashield.aggregate(datasources, cally, async = FALSE)
16+
classesBy <- DSI::datashield.aggregate(datasources, cally)
1717
classes <- unique(unlist(classesBy))
1818
for (n in names(classesBy)) {
1919
if (!all(classes == classesBy[[n]])) {

R/ds.class.R

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
1212
#' @return \code{ds.class} returns the type of the R object.
1313
#' @author DataSHIELD Development Team
14+
#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
1415
#' @seealso \code{\link{ds.exists}} to verify if an object is defined (exists) on the server-side.
1516
#' @examples
1617
#' \dontrun{
@@ -54,23 +55,12 @@
5455
#'
5556
ds.class <- function(x=NULL, datasources=NULL) {
5657

57-
# look for DS connections
58-
if(is.null(datasources)){
59-
datasources <- datashield.connections_find()
60-
}
61-
62-
# ensure datasources is a list of DSConnection-class
63-
if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
64-
stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
65-
}
58+
datasources <- .set_datasources(datasources)
6659

6760
if(is.null(x)){
6861
stop("Please provide the name of the input object!", call.=FALSE)
6962
}
7063

71-
# check if the input object is defined in all the studies
72-
defined <- isDefined(datasources, x)
73-
7464
cally <- call('classDS', x)
7565
output <- DSI::datashield.aggregate(datasources, cally)
7666

R/ds.completeCases.R

Lines changed: 3 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -68,123 +68,22 @@
6868
#' }
6969
#'
7070
#' @author DataSHIELD Development Team
71+
#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
7172
#' @export
7273
#'
7374
ds.completeCases <- function(x1=NULL, newobj=NULL, datasources=NULL){
74-
75-
# if no connection login details are provided look for 'connection' objects in the environment
76-
if(is.null(datasources)){
77-
datasources <- datashield.connections_find()
78-
}
7975

80-
# ensure datasources is a list of DSConnection-class
81-
if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
82-
stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
83-
}
76+
datasources <- .set_datasources(datasources)
8477

85-
# check if a value has been provided for x1
8678
if(is.null(x1)){
8779
return("Error: x1 must be a character string naming a serverside data.frame, matrix or vector")
8880
}
89-
90-
# check if the input object is defined in all the studies
91-
isDefined(datasources, x1)
92-
93-
# rename target object for transfer (not strictly necessary as string will pass parser anyway)
94-
# but maintains consistency with other functions
95-
x1.transmit <- x1
9681

97-
# if no value specified for output object, then specify a default
9882
if(is.null(newobj)){
9983
newobj <- paste0(x1,"_complete.cases")
10084
}
10185

102-
# CALL THE MAIN SERVER SIDE FUNCTION
103-
calltext <- call("completeCasesDS", x1.transmit)
86+
calltext <- call("completeCasesDS", x1)
10487
DSI::datashield.assign(datasources, newobj, calltext)
10588

106-
107-
#############################################################################################################
108-
#DataSHIELD CLIENTSIDE MODULE: CHECK KEY DATA OBJECTS SUCCESSFULLY CREATED #
109-
#
110-
#SET APPROPRIATE PARAMETERS FOR THIS PARTICULAR FUNCTION #
111-
test.obj.name<-newobj #
112-
#
113-
#TRACER #
114-
#return(test.obj.name) #
115-
#} #
116-
#
117-
#
118-
# CALL SEVERSIDE FUNCTION #
119-
calltext <- call("testObjExistsDS", test.obj.name) #
120-
#
121-
object.info<-DSI::datashield.aggregate(datasources, calltext) #
122-
#
123-
# CHECK IN EACH SOURCE WHETHER OBJECT NAME EXISTS #
124-
# AND WHETHER OBJECT PHYSICALLY EXISTS WITH A NON-NULL CLASS #
125-
num.datasources<-length(object.info) #
126-
#
127-
#
128-
obj.name.exists.in.all.sources<-TRUE #
129-
obj.non.null.in.all.sources<-TRUE #
130-
#
131-
for(j in 1:num.datasources){ #
132-
if(!object.info[[j]]$test.obj.exists){ #
133-
obj.name.exists.in.all.sources<-FALSE #
134-
} #
135-
if(is.null(object.info[[j]]$test.obj.class) || ("ABSENT" %in% object.info[[j]]$test.obj.class)){ #
136-
obj.non.null.in.all.sources<-FALSE #
137-
} #
138-
} #
139-
#
140-
if(obj.name.exists.in.all.sources && obj.non.null.in.all.sources){ #
141-
#
142-
return.message<- #
143-
paste0("A data object <", test.obj.name, "> has been created in all specified data sources") #
144-
#
145-
#
146-
}else{ #
147-
#
148-
return.message.1<- #
149-
paste0("Error: A valid data object <", test.obj.name, "> does NOT exist in ALL specified data sources") #
150-
#
151-
return.message.2<- #
152-
paste0("It is either ABSENT and/or has no valid content/class,see return.info above") #
153-
#
154-
return.message.3<- #
155-
paste0("Please use ds.ls() to identify where missing") #
156-
#
157-
#
158-
return.message<-list(return.message.1,return.message.2,return.message.3) #
159-
#
160-
} #
161-
#
162-
calltext <- call("messageDS", test.obj.name) #
163-
studyside.message<-DSI::datashield.aggregate(datasources, calltext) #
164-
#
165-
no.errors<-TRUE #
166-
for(nd in 1:num.datasources){ #
167-
if(studyside.message[[nd]]!="ALL OK: there are no studysideMessage(s) on this datasource"){ #
168-
no.errors<-FALSE #
169-
} #
170-
} #
171-
#
172-
#
173-
if(no.errors){ #
174-
validity.check<-paste0("<",test.obj.name, "> appears valid in all sources") #
175-
return(list(is.object.created=return.message,validity.check=validity.check)) #
176-
} #
177-
#
178-
if(!no.errors){ #
179-
validity.check<-paste0("<",test.obj.name,"> invalid in at least one source. See studyside.messages:") #
180-
return(list(is.object.created=return.message,validity.check=validity.check, #
181-
studyside.messages=studyside.message)) #
182-
} #
183-
#
184-
#END OF CHECK OBJECT CREATED CORECTLY MODULE #
185-
#############################################################################################################
186-
18789
}
188-
#ds.completeCases
189-
190-

R/ds.dataFrameFill.R

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
#' client-side indicating the name of the \code{newobj} that has been created in each data source
2222
#' and if it is in a valid form.
2323
#' @author Demetris Avraam for DataSHIELD Development Team
24-
#'
24+
#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
25+
#'
2526
#' @examples
2627
#' \dontrun{
2728
#'
@@ -134,9 +135,17 @@ ds.dataFrameFill <- function(df.name=NULL, newobj=NULL, datasources=NULL){
134135
defined.vect1 <- lapply(defined.list, function(x){unlist(x)})
135136
defined.vect2 <- lapply(defined.vect1, function(x){which(x == FALSE)})
136137

137-
# get the class of each variable in the dataframes
138-
class.list <- lapply(allNames, function(x){lapply(datasources, function(dts){DSI::datashield.aggregate(dts, call('classDS', paste0(df.name, '$', x)))})})
139-
class.vect1 <- lapply(class.list, function(x){unlist(x)})
138+
# get the class of each variable in the dataframes, skipping servers where the column doesn't exist
139+
class.list <- lapply(seq_along(allNames), function(idx){
140+
sapply(seq_along(datasources), function(ds_idx){
141+
if(ds_idx %in% defined.vect2[[idx]]){
142+
"NULL"
143+
} else {
144+
DSI::datashield.aggregate(datasources[ds_idx], call('classDS', paste0(df.name, '$', allNames[idx])))[[1]]
145+
}
146+
})
147+
})
148+
class.vect1 <- class.list
140149
# the loop below is to avoid autocompletion of variable name
141150
for (i in 1:length(allNames.transmit)){
142151
if(length(defined.vect2[[i]])>0){class.vect1[[i]][defined.vect2[[i]]]<-'NULL'}

R/ds.dim.R

Lines changed: 21 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -7,28 +7,25 @@
77
#' from every single study and the pooled dimension of the object by summing up the individual
88
#' dimensions returned from each study.
99
#'
10-
#' In \code{checks} parameter is suggested that checks should only be undertaken once the
11-
#' function call has failed.
12-
#'
1310
#' Server function called: \code{dimDS}
14-
#'
15-
#' @param x a character string providing the name of the input object.
16-
#' @param type a character string that represents the type of analysis to carry out.
11+
#'
12+
#' @param x a character string providing the name of the input object.
13+
#' @param type a character string that represents the type of analysis to carry out.
1714
#' If \code{type} is set to \code{'combine'}, \code{'combined'}, \code{'combines'} or \code{'c'},
18-
#' the global dimension is returned.
19-
#' If \code{type} is set to \code{'split'}, \code{'splits'} or \code{'s'},
15+
#' the global dimension is returned.
16+
#' If \code{type} is set to \code{'split'}, \code{'splits'} or \code{'s'},
2017
#' the dimension is returned separately for each study.
2118
#' If \code{type} is set to \code{'both'} or \code{'b'}, both sets of outputs are produced.
22-
#' Default \code{'both'}.
23-
#' @param checks logical. If TRUE undertakes all DataSHIELD checks (time-consuming).
24-
#' Default FALSE.
19+
#' Default \code{'both'}.
20+
#' @template classConsistencyCheck
2521
#' @param datasources a list of \code{\link[DSI]{DSConnection-class}}
2622
#' objects obtained after login. If the \code{datasources} argument is not specified
2723
#' the default set of connections will be used: see \code{\link[DSI]{datashield.connections_default}}.
2824
#' @return \code{ds.dim} retrieves to the client-side the dimension of the object
2925
#' in the form of a vector where the first
3026
#' element indicates the number of rows and the second element indicates the number of columns.
3127
#' @author DataSHIELD Development Team
28+
#' @author Tim Cadman, Genomics Coordination Centre, UMCG, Netherlands
3229
#' @seealso \code{\link{ds.dataFrame}} to generate a table of the type data frame.
3330
#' @seealso \code{\link{ds.changeRefGroup}} to change the reference level of a factor.
3431
#' @seealso \code{\link{ds.colnames}} to obtain the column names of a matrix or a data frame
@@ -67,68 +64,44 @@
6764
#' # Calculate the dimension
6865
#' ds.dim(x="D",
6966
#' type="combine", #global dimension
70-
#' checks = FALSE,
71-
#' datasources = connections)#all opal servers are used
67+
#'#' datasources = connections)#all opal servers are used
7268
#' ds.dim(x="D",
7369
#' type = "both",#separate dimension for each study
7470
#' #and the pooled dimension (default)
75-
#' checks = FALSE,
76-
#' datasources = connections)#all opal servers are used
71+
#'#' datasources = connections)#all opal servers are used
7772
#' ds.dim(x="D",
7873
#' type="split", #separate dimension for each study
79-
#' checks = FALSE,
80-
#' datasources = connections[1])#only the first opal server is used ("study1")
74+
#'#' datasources = connections[1])#only the first opal server is used ("study1")
8175
#'
8276
#' # clear the Datashield R sessions and logout
8377
#' datashield.logout(connections)
8478
#'
8579
#' }
8680
#'
87-
ds.dim <- function(x=NULL, type='both', checks=FALSE, datasources=NULL) {
81+
ds.dim <- function(x=NULL, type='both', classConsistencyCheck=TRUE, datasources=NULL) {
8882

89-
# look for DS connections
90-
if(is.null(datasources)){
91-
datasources <- datashield.connections_find()
92-
}
93-
94-
# ensure datasources is a list of DSConnection-class
95-
if(!(is.list(datasources) && all(unlist(lapply(datasources, function(d) {methods::is(d,"DSConnection")}))))){
96-
stop("The 'datasources' were expected to be a list of DSConnection-class objects", call.=FALSE)
97-
}
83+
datasources <- .set_datasources(datasources)
9884

9985
if(is.null(x)){
10086
stop("Please provide the name of a data.frame or matrix!", call.=FALSE)
10187
}
10288

103-
########################################################################################################
104-
# MODULE: GENERIC OPTIONAL CHECKS TO ENSURE CONSISTENT STRUCTURE OF KEY VARIABLES IN DIFFERENT SOURCES #
105-
# beginning of optional checks - the process stops and reports as soon as one check fails #
106-
# #
107-
if(checks){ #
108-
message(" -- Verifying the variables in the model") #
109-
# check if the input object(s) is(are) defined in all the studies #
110-
defined <- isDefined(datasources, x) # #
111-
# call the internal function that checks the input object is suitable in all studies #
112-
typ <- checkClass(datasources, x) #
113-
# throw a message and stop if input is not table structure #
114-
if(!('data.frame' %in% typ) & !('matrix' %in% typ)){ #
115-
stop("The input object must be a table structure!", call.=FALSE) #
116-
} #
117-
} #
118-
########################################################################################################
119-
120-
12189
###################################################################################################
12290
#MODULE: EXTEND "type" argument to include "both" and enable valid aliases #
12391
if(type == 'combine' | type == 'combined' | type == 'combines' | type == 'c') type <- 'combine' #
12492
if(type == 'split' | type == 'splits' | type == 's') type <- 'split' #
12593
if(type == 'both' | type == 'b' ) type <- 'both' #
126-
#
127-
#MODIFY FUNCTION CODE TO DEAL WITH ALL THREE TYPES #
12894
###################################################################################################
12995

13096
cally <- call("dimDS", x)
131-
dimensions <- DSI::datashield.aggregate(datasources, cally)
97+
results <- DSI::datashield.aggregate(datasources, cally)
98+
99+
if(classConsistencyCheck){
100+
.checkClassConsistency(results)
101+
}
102+
103+
# extract dimensions from results
104+
dimensions <- lapply(results, function(r) r$dim)
132105

133106
# names of the studies to be used in the output
134107
stdnames <- names(datasources)

0 commit comments

Comments
 (0)