optimize.Rraw

require(methods)
if (exists("test.data.table", .GlobalEnv, inherits=FALSE)) {
  if ((tt<-compiler::enableJIT(-1))>0)
    cat("This is dev mode and JIT is enabled (level ", tt, ") so there will be a brief pause around the first test.\n", sep="")
} else {
  require(data.table)
  test = data.table:::test
  null.data.table = data.table:::null.data.table
  INT = data.table:::INT
}

sugg = c("bit64")
for (s in sugg) {
  assign(paste0("test_",s), loaded<-suppressWarnings(suppressMessages(
    library(s, character.only=TRUE, logical.return=TRUE, quietly=TRUE, warn.conflicts=FALSE, pos="package:base")  # attach at the end for #5101
  )))
  if (!loaded) cat("\n**** Suggested package",s,"is not installed or has dependencies missing. Tests using it will be skipped.\n\n")
}

# := by group
DT = data.table(a=1:3,b=(1:9)/10)
test(611.1, levels=c(0L, 2L), DT[,v:=sum(b),by=a], data.table(a=1:3,b=(1:9)/10,v=c(1.2,1.5,1.8)))
setkey(DT,a)
test(611.2, levels=c(0L, 2L), DT[,v:=min(b),by=a], data.table(a=1:3,b=(1:9)/10,v=(1:3)/10,key="a"))
# Combining := by group with i
test(611.3, levels=c(0L, 2L), DT[a>1,p:=sum(b)]$p, rep(c(NA,3.3),c(3,6)))
test(611.4, levels=c(0L, 2L), DT[a>1,q:=sum(b),by=a]$q, rep(c(NA,1.5,1.8),each=3))
# 612 was just level repetition of 611
# Assign to subset ok (NA initialized in the other items) ok :
test(613, levels=c(0L, 2L), DT[J(2),w:=8.3]$w, rep(c(NA,8.3,NA),each=3))
test(614, levels=c(0L, 2L), DT[J(3),x:=9L]$x, rep(c(NA_integer_,NA_integer_,9L),each=3))
test(615, levels=c(0L, 2L), DT[J(2),z:=list(list(c(10L,11L)))]$z, rep(list(NULL, 10:11, NULL),each=3))

# Empty i clause, #2034. Thanks to Chris for testing, tests from him. Plus changes from #759
ans = copy(DT)[,r:=NA_real_]
test(618.1, levels=c(0L, 2L), copy(DT)[a>3,r:=sum(b)],   ans)
test(618.2, levels=c(0L, 2L), copy(DT)[J(-1),r:=sum(b)], ans)
test(618.3, levels=c(0L, 2L), copy(DT)[NA,r:=sum(b)],    ans)
test(618.4, levels=c(0L, 2L), copy(DT)[0,r:=sum(b)],     ans)
test(618.5, levels=c(0L, 2L), copy(DT)[NULL,r:=sum(b)],  null.data.table())
# test 619 was level 2 of 618

DT = data.table(x=letters, key="x")
test(621, levels=c(0L, 2L), copy(DT)[J("bb"), x:="foo"], DT)  # when no update, key should be retained
test(622, levels=c(0L, 2L), copy(DT)[J("bb"), x:="foo",nomatch=0], DT, warning="ignoring nomatch")

set.seed(2)
DT = data.table(a=rnorm(5)*10, b=1:5)
test(623, levels=c(0L, 2L), copy(DT)[,s:=sum(b),by=round(a)%%2]$s, c(10L,5L,5L,10L,10L))

# Setup for test 656.x - gforce tests
set.seed(9)
n = 1e3
DT = data.table(grp1=sample.int(150L, n, replace=TRUE),
                grp2=sample.int(150L, n, replace=TRUE),
                x=rnorm(n),
                y=rnorm(n))
opt = 0:2
out = c('GForce FALSE', 'GForce FALSE' ,'GForce TRUE')
test(656.1, levels=opt, DT[ , mean(x), by=grp1, verbose=TRUE],                output=out)
test(656.2, levels=opt, DT[ , list(mean(x)), by=grp1, verbose=TRUE],          output=out)
test(656.3, levels=opt, DT[ , list(mean(x), mean(y)), by=grp1, verbose=TRUE], output=out)

# Test := keyby does setkey, #2065
DT = data.table(x=1:2, y=1:6)
ans = data.table(x=rep(1:2,each=3),y=c(1L,3L,5L,2L,4L,6L),z=rep(c(9L,12L),each=3),key="x")
test(670.1, levels=c(0L, 2L), copy(DT)[,z:=sum(y),keyby=x], ans)
test(670.2, levels=c(0L, 2L), copy(DT)[,z:=sum(y),keyby="x"], ans)
test(670.3, levels=c(0L, 2L), copy(DT)[,z:=sum(y),keyby=x%%2], data.table(x=1:2,y=1:6,z=c(9L,12L)),
          warning="The setkey() normally performed by keyby= has been skipped (as if by= was used) because := is being used together with keyby= but the keyby= contains some expressions. To avoid this warning, use by= instead, or provide existing column names to keyby=")
test(670.4, levels=c(0L, 2L), copy(DT)[,z:=sum(y),by=x%%2], data.table(x=1:2,y=1:6,z=c(9L,12L)))
test(670.5, levels=c(0L, 2L), copy(DT)[x>1,z:=sum(y),keyby=x], error=":= with keyby is only possible when i is not supplied since")
# test 671 was level 2 of 670

# varname holding colnames, by group, linked from #2120.
DT = data.table(a=rep(1:3,1:3),b=1:6)
colname = "newcol"
test(751, levels=c(0L, 2L), DT[,(colname):=sum(b),by=a], data.table(a=rep(1:3,1:3),b=1:6,newcol=INT(1,5,5,15,15,15)))

# Add tests for nested := in j by group, #1987
DT = data.table(a=rep(1:3,2:4),b=1:9)
test(752, levels=c(0L, 2L), DT[,head(.SD,2)[,new:=1:.N],by=a], data.table(a=rep(1:3,each=2),b=c(1:4,6:7),new=1:2))

DT = data.table(a=1:3,b=1:3,v=1:6,w=1:6)
opt = c(0:2, Inf)
out = c('GForce FALSE', 'GForce FALSE', 'GForce TRUE', 'GForce TRUE')
# v1.9.7 treats wrapped {} better, so this is now optimized
test(865, levels=opt, DT[,{list(name1=sum(v),name2=sum(w))},by=c('a', 'b'),verbose=TRUE], output=out)
test(867, levels=opt, names(DT[,list(name1=sum(v),name2=sum(w)),by=c('a', 'b')]), c("a", "b", "name1", "name2"))  # list names extracted here

# tests of gsum and gmean with NA
DT = data.table(x=rep(c("a","b","c","d"),each=3), y=c(1L,3L,6L), v=as.numeric(1:12))
set(DT,c(3L,8L),"y",NA)
set(DT,c(5L,9L),"v",NA)
set(DT,10:12,"y",NA)
set(DT,10:12,"v",NA)
opt = c(1L, 2L)
out = c("(GForce FALSE)", "GForce optimized j to")
test(1184.1, levels=opt, DT[, sum(v), by=x, verbose=TRUE], output=out)
test(1184.2, levels=1L, DT[, mean(v), by=x, verbose=TRUE], output="(GForce FALSE)")
test(1185.2, levels=c(0L,1L,Inf), DT[, list(mean(y), mean(v), mean(y,na.rm=TRUE), mean(v,na.rm=TRUE)), by=x, verbose=TRUE], 
  output=c("All optimizations.*off", "Old mean.*changed j", "GForce optimized j to"))
test(1187.1, levels=c(0L, 1L, 2L), DT[, list(sum(y), sum(v), sum(y,na.rm=TRUE), sum(v,na.rm=TRUE)), by=x],
           data.table(x=c("a","b","c","d"), V1=c(NA,10L,NA,NA), V2=c(6,NA,NA,NA), V3=c(4L,10L,7L,0L), V4=c(6,10,15,0)))
MyVar = TRUE
test(1187.2, levels=opt, DT[, list(sum(y,na.rm=MyVar), mean(y,na.rm=MyVar)), by=x, verbose=TRUE], output=out)
test(1187.3, levels=opt, DT[, mean(y,na.rm=MyVar), by=x, verbose=TRUE], output=out)
MyVar = FALSE
test(1187.4, levels=opt, DT[, list(sum(y,na.rm=MyVar), mean(y,na.rm=MyVar)), by=x, verbose=TRUE], output=out)
test(1187.5, levels=opt, DT[, mean(y,na.rm=MyVar), by=x, verbose=TRUE], output=out)
# GForce should not turn on when the .ok function isn't triggered
test(1187.6, levels=2L, DT[, mean(y, trim=.2), by=x, verbose=TRUE],
             data.table(x = c("a", "b", "c", "d"), V1 = c(NA, 3.33333333333333, NA, NA)),
             output='j unchanged', warning="'trim' is not yet optimized")

# FR #334. Test to just make sure that GForce and dogroups with .N are giving the same results.
set.seed(2L)
dt <- data.table(x=sample(rep(1:5e3, each=3)), y=sample(10))
test(1304.1, levels=0:2, dt[, list(.N, sum(y)), by=x])
dt <- data.table(x=sample(rep(1:5e3, each=3)), y=sample(10), key="x")
test(1304.2, levels=0:2, dt[, list(.N, sum(y)), by=x])

# gmin and gmax extensive testing (because there are tricky cases)
DT <- data.table(x=rep(1:6, each=3), y=INT(4,-1,0, NA,4,10, 4,NA,10, 4,10,NA, -2147483647, -2147483647, -2147483647, 2147483647, 2147483647, 2147483647))
opts = 0:2
# for integers
test(1313.01, levels=opts, DT[, min(y), by=x], DT[, base::min(y), by=x])
test(1313.02, levels=opts, DT[, max(y), by=x], DT[, base::max(y), by=x])
test(1313.03, levels=opts, DT[, min(y, na.rm=TRUE), by=x], DT[, base::min(y, na.rm=TRUE), by=x])
test(1313.04, levels=opts, DT[, max(y, na.rm=TRUE), by=x], DT[, base::max(y, na.rm=TRUE), by=x])
# testing all NA - GForce automatically converts to numeric.. optimize=1L errors due to change from integer/numeric (like median)
DT[x==6, y := INT(NA)]
test(1313.05, levels=opts, DT[, min(y), by=x], DT[, base::min(y), by=x])
test(1313.06, levels=opts, DT[, max(y), by=x], DT[, base::max(y), by=x])
test(1313.07, levels=2L,   DT[, min(y, na.rm=TRUE), by=x], data.table(x=1:6, V1=INT(-1,4,4,4,-2147483647,NA)))
test(1313.08, levels=2L,   DT[, max(y, na.rm=TRUE), by=x], data.table(x=1:6, V1=INT(4,10,10,10,-2147483647,NA)))
# for numeric
DT <- data.table(x=rep(1:6, each=3), y=c(4,-1,0, NA,4,10, 4,NA,10, 4,10,NA, -Inf, NA, NA, Inf, NA, NA))
test(1313.09, levels=opts, DT[, min(y), by=x], DT[, base::min(y), by=x])
test(1313.10, levels=opts, DT[, max(y), by=x], DT[, base::max(y), by=x])
test(1313.11, levels=opts, DT[, min(y, na.rm=TRUE), by=x], DT[, base::min(y, na.rm=TRUE), by=x])
test(1313.12, levels=opts, DT[, max(y, na.rm=TRUE), by=x], DT[, base::max(y, na.rm=TRUE), by=x])
# testing all NA - GForce automatically converts to numeric.. optimize=1L errors due to change from integer/numeric (like median)
DT[x==6, y := NA_real_]
test(1313.13, levels=opts, DT[, min(y), by=x], DT[, base::min(y), by=x])
test(1313.14, levels=opts, DT[, max(y), by=x], DT[, base::max(y), by=x])
test(1313.15, levels=2L,   DT[, min(y, na.rm=TRUE), by=x], data.table(x=1:6, V1=c(-1,4,4,4,-Inf,NA)))
test(1313.16, levels=2L,   DT[, max(y, na.rm=TRUE), by=x], data.table(x=1:6, V1=c(4,10,10,10,-Inf,NA)))
# for date (attribute check.. especially after issues/689 !!!)
DT <- data.table(x = rep(letters[1:2], each=5), y = as.POSIXct('2010-01-01', tz="UTC") + seq(0, 86400*9, 86400))
test(1313.17, levels=opts, DT[, list(y=min(y)), by=x], DT[c(1,6)])
test(1313.18, levels=opts, DT[, list(y=max(y)), by=x], DT[c(5,10)])
DT[c(1,6), y := NA]
test(1313.19, levels=opts, DT[, list(y=min(y)), by=x], DT[c(1,6)])
test(1313.20, levels=opts, DT[, list(y=max(y)), by=x], DT[c(1,6)])
test(1313.21, levels=opts, DT[, list(y=min(y, na.rm=TRUE)), by=x], DT[c(2,7)])
test(1313.22, levels=opts, DT[, list(y=max(y, na.rm=TRUE)), by=x], DT[c(5,10)])
# for character
set.seed(1L)
DT <- data.table(x=rep(1:7, each=3), y=sample(c("", letters[1:3], NA), 21, TRUE))
DT[x==7, y := c("","b","c")]
test(1313.23, levels=opts, DT[, min(y), by=x], DT[, base::min(y), by=x])
test(1313.24, levels=opts, DT[, max(y), by=x], DT[, base::max(y), by=x])
test(1313.25, levels=opts, DT[, min(y, na.rm=TRUE), by=x], DT[, base::min(y, na.rm=TRUE), by=x])
test(1313.26, levels=opts, DT[, max(y, na.rm=TRUE), by=x], DT[, base::max(y, na.rm=TRUE), by=x])
DT[x==6, y := NA_character_]
test(1313.27, levels=opts, DT[, min(y), by=x], DT[, base::min(y), by=x])
test(1313.28, levels=opts, DT[, max(y), by=x], DT[, base::max(y), by=x])
test(1313.29, levels=2L,   DT[, min(y, na.rm=TRUE), by=x], data.table(x=1:7, V1=c("a","a","c","","a",NA,"")))
test(1313.30, levels=2L,   DT[, max(y, na.rm=TRUE), by=x], data.table(x=1:7, V1=c("b","a","c","a","c",NA,"c")))

# Fix for #1251, DT[, .N, by=a] and DT[, .(.N), by=a] uses GForce now
dt = data.table(a=sample(3,20,TRUE), b=1:10)
test(1565.1, levels=c(0,1,Inf), dt[, .N, by=a, verbose=TRUE], 
  output=c("All optimizations are turned off", "lapply optimization is on, j unchanged", "GForce optimized j to"))

# gforce optimisations
dt = data.table(x  = sample(letters, 300, TRUE),
                i1 = sample(-10:10, 300, TRUE),
                i2 = sample(c(-10:10, NA), 300, TRUE),
                d1 = as.numeric(sample(-10:10, 300, TRUE)),
                d2 = as.numeric(sample(c(NA, NaN, -10:10), 300, TRUE)))
if (test_bit64) {
  dt[, `:=`(d3 = as.integer64(sample(-10:10, 300, TRUE)))]
  dt[, `:=`(d4 = as.integer64(sample(c(-10:10,NA), 300, TRUE)))]
}
opt = 0:2
out = c('GForce FALSE', 'GForce FALSE' ,'GForce TRUE')
# make sure gforce is on
# testing gforce::gmedian
test(1579.01, levels=2L,  dt[, lapply(.SD, median), by=x, verbose=TRUE],
                          dt[, lapply(.SD, function(x) median(as.numeric(x))), by=x], output="GForce optimized")
test(1579.02, levels=2L,  dt[, lapply(.SD, median, na.rm=TRUE), by=x],
                          dt[, lapply(.SD, function(x) median(as.numeric(x), na.rm=TRUE)), by=x])
test(1579.03, levels=2L, dt[, lapply(.SD, median), keyby=x],
                          dt[, lapply(.SD, function(x) median(as.numeric(x))), keyby=x])
test(1579.04, levels=2L, dt[, lapply(.SD, median, na.rm=TRUE), keyby=x],
                          dt[, lapply(.SD, function(x) median(as.numeric(x), na.rm=TRUE)), keyby=x])
# testing gforce::ghead and gforce::gtail
# head(.SD, 1) and tail(.SD, 1) optimisation
test(1579.06, levels=opt, dt[, head(.SD,1),  by=x,    verbose=TRUE], output=out)
test(1579.08, levels=opt, dt[, head(.SD,1),  keyby=x, verbose=TRUE], output=out)
test(1579.10, levels=opt, dt[, head(.SD,1L), by=x,    verbose=TRUE], output=out)
test(1579.12, levels=opt, dt[, head(.SD,1L), keyby=x, verbose=TRUE], output=out)
test(1579.14, levels=opt, dt[, tail(.SD,1),  by=x,    verbose=TRUE], output=out)
test(1579.16, levels=opt, dt[, tail(.SD,1),  keyby=x, verbose=TRUE], output=out)
test(1579.18, levels=opt, dt[, tail(.SD,1L), by=x,    verbose=TRUE], output=out)
test(1579.20, levels=opt, dt[, tail(.SD,1L), keyby=x, verbose=TRUE], output=out)
# 1579.22 tested gtail with n>1; now 1579.4+ below
mysub <- function(x, n) x[n]
test(1579.23, levels=2L,  dt[, .SD[2],  by=x, verbose=TRUE], dt[, mysub(.SD,2),  by=x], output="GForce optimized.*g[[]")
test(1579.24, levels=opt, dt[, .SD[2],  keyby=x], dt[, mysub(.SD,2),  keyby=x])
test(1579.25, levels=opt, dt[, .SD[2L], by=x],    dt[, mysub(.SD,2L), by=x])
test(1579.26, levels=opt, dt[, .SD[2L], keyby=x], dt[, mysub(.SD,2L), keyby=x])
test(1579.27, levels=opt, dt[, .SD[15], by=x],    dt[, mysub(.SD,15), by=x])   # tests 15 > grpsize and that NA is correct including for integer64
test(1579.28, levels=opt, dt[, .SD[15], keyby=x], dt[, mysub(.SD,15), keyby=x])
# gforce head/tail for n>1, #5060
set.seed(99)
DT = data.table(x = sample(letters[1:5], 20, TRUE),
                y = rep.int(1:2, 10),  # to test 2 grouping columns get rep'd properly
                i = sample(c(-2L,0L,3L,NA), 20, TRUE),
                d = sample(c(1.2,-3.4,5.6,NA), 20, TRUE),
                s = sample(c("foo","bar",NA), 20, TRUE),
                l = sample(list(1:3, mean, letters[4:5], NULL), 20, replace=TRUE))
if (test_bit64) DT[, i64:=sample(as.integer64(c(-2200000000,+2400000000,NA)), 20, TRUE)]
test(1579.401, levels=0:2, DT[, .N, by=x]$N, INT(4,6,5,2,3))   # the smallest group is 2, so n=5 tests n constrained to grpsize
test(1579.402, levels=2L,  DT[, head(.SD,2),    by=x, verbose=TRUE], DT[, utils::head(.SD,2),    by=x], output="optimized.*ghead")
test(1579.403, levels=2L,  DT[, head(.SD,2), keyby=x, verbose=TRUE], DT[, utils::head(.SD,2), keyby=x], output="optimized.*ghead")
test(1579.404, levels=2L,  DT[, head(.SD,5),    by=x, verbose=TRUE], DT[, utils::head(.SD,5),    by=x], output="optimized.*ghead")
test(1579.405, levels=2L,  DT[, head(.SD,5), keyby=x, verbose=TRUE], DT[, utils::head(.SD,5), keyby=x], output="optimized.*ghead")
test(1579.406, levels=2L,  DT[, tail(.SD,2),    by=x, verbose=TRUE], DT[, utils::tail(.SD,2),    by=x], output="optimized.*gtail")
test(1579.407, levels=2L,  DT[, tail(.SD,2), keyby=x, verbose=TRUE], DT[, utils::tail(.SD,2), keyby=x], output="optimized.*gtail")
test(1579.408, levels=2L,  DT[, tail(.SD,5),    by=x, verbose=TRUE], DT[, utils::tail(.SD,5),    by=x], output="optimized.*gtail")
test(1579.409, levels=2L,  DT[, tail(.SD,5), keyby=x, verbose=TRUE], DT[, utils::tail(.SD,5), keyby=x], output="optimized.*gtail")
test(1579.410, levels=2L,  DT[, tail(.SD,2), by=.(x,y), verbose=TRUE], DT[, utils::tail(.SD,2), by=.(x,y)], output="optimized.*gtail")


# FR #971, partly addressed (only subsets in 'i')
# make sure GForce kicks in and the results are identical
dt = data.table(x  = sample(letters, 300, TRUE),
                d1 = as.numeric(sample(-10:10, 300, TRUE)),
                d2 = as.numeric(sample(c(NA, NaN, -10:10), 300, TRUE)))
opt = 1:2
out = c("GForce FALSE","GForce optimized j")
test(1581.01, levels=opt, ans1 <- dt[x %in% letters[15:20],
                        c(.N, lapply(.SD, sum, na.rm=TRUE),
                              lapply(.SD, min, na.rm=TRUE),
                              lapply(.SD, max, na.rm=TRUE),
                              lapply(.SD, mean, na.rm=TRUE),
                              lapply(.SD, median, na.rm=TRUE)
                          ), by=x, verbose=TRUE],
  output = out)

# subsets in 'i' for head and tail
test(1581.04, levels=opt, dt[x %in% letters[15:20], head(.SD,1), by=x, verbose=TRUE], output=out)
test(1581.07, levels=opt, dt[x %in% letters[15:20], tail(.SD,1), by=x, verbose=TRUE], output=out)
test(1581.10, levels=opt, dt[x %in% letters[15:20], .SD[2], by=x, verbose=TRUE], output=out)
# #3209 g[[
test(1581.13, levels=opt, dt[x %in% letters[15:20], d1[[2]], by=x, verbose=TRUE], output=out)
# also, block for non-atomic input, #4159
dt = data.table(a=1:3)
dt[ , l := .(list(1, 2, 3))]
test(1581.16, dt[ , .(l = l[[1L]]), by=a, verbose=TRUE],
     dt[ , l := unlist(l)], output='(GForce FALSE)')
# make sure not to apply when `[[` is applied to a nested call, #4413
DT = data.table(f1=c("a","b"), f2=c("x","y"))
l = list(a = c(x = "ax", y = "ay"), b = c(x = "bx", y = "by"))
test(1581.17, DT[ , as.list(l[[f1]])[[f2]], by=c("f1","f2")],
     data.table(f1 = c("a", "b"), f2 = c("x", "y"), V1 = c("ax", "by")))
test(1581.18, DT[, v:=l[[f1]][f2], by=c("f1","f2")],
     data.table(f1=c("a","b"), f2=c("x","y"), v=c("ax", "by")))
# When the object being [[ is in parent.frame(), not x,
#   need eval to have enclos=parent.frame(), #4612
DT = data.table(id = c(1, 1, 2), value = c("a", "b", "c"))
DT0 = copy(DT)
fun = function(DT, tag = c("A", "B")) DT[, var := tag[[.GRP]], by = "id"]
fun(DT)
test(1581.19, DT, DT0[ , var := c('A', 'A', 'B')])

# bug fix #1461 related to NaN not being recognized due to ISNA vs ISNAN at C level
# verbatim test from the original report:
DT = data.table(
     C1 = c(rep("A", 4), rep("B",4), rep("C", 4)),
     C2 = c(rep("a", 3), rep("b",3), rep("c",3), rep("d",3)),
     Val = c(1:5, NaN, NaN, 8,9,10,NaN,12))
opt = 0:2
test(1583.1, levels=opt, DT[, .(agg = min(Val, na.rm=TRUE)), by=c('C1', 'C2')],
             data.table(C1=c("A","A","B","B","C","C"),
                        C2=c("a","b","b","c","c","d"),
                        agg=c(1,4,5,8,9,10)))
# extra test with a size-1 group containing one NaN too
DT = data.table(x=INT(1,1,1,2,2,2,3,3,3,4,4,4,5), y=c(NaN,1,2, 2,NaN,1, NA,NaN,2, NaN,NA,NaN, NaN))
test(1583.2, levels=2L, DT[, min(y, na.rm=TRUE), by=x], data.table(x=1:5, V1=c(1,1,2,NA,NA)))
test(1583.3, levels=2L, DT[, max(y, na.rm=TRUE), by=x], data.table(x=1:5, V1=c(2,2,2,NA,NA)))
test(1583.4, levels=opt, DT[, min(y), by=x], data.table(x=1:5, V1=c(NaN,NaN,NA,NaN,NaN)))
test(1583.5, levels=opt, DT[, max(y), by=x], data.table(x=1:5, V1=c(NaN,NaN,NA,NaN,NaN)))

# FR #523, var, sd and prod
DT = data.table(x=sample(5, 100, TRUE),
               y1=sample(6, 100, TRUE),
               y2=sample(c(1:10,NA), 100, TRUE),
               z1=runif(100),
               z2=sample(c(runif(10),NA,NaN), 100, TRUE))
opt = 0:2
out = c("GForce FALSE", "GForce FALSE", "GForce optimized j to")
test(1594.01, levels=opt, DT[, lapply(.SD, var, na.rm=FALSE), by=x])
test(1594.02, levels=opt, DT[, lapply(.SD, var, na.rm=TRUE), by=x])
test(1594.03, levels=opt, DT[, lapply(.SD, var, na.rm=TRUE), by=x, verbose=TRUE], output=out)
# coverage: default group .N=1 case
idx=DT[ , .I[1L], by=x]$V1
ans=data.table(x=DT[(idx), x], V1=NA_real_)
test(1594.05, levels=opt, DT[(idx), var(y1), by=x], ans)
test(1594.06, levels=opt, DT[(idx), var(y1, na.rm=TRUE), by=x], ans)
test(1594.07, levels=opt, DT[(idx), var(z1), by=x], ans)
test(1594.08, levels=opt, DT[(idx), var(z1, na.rm=TRUE), by=x], ans)

test(1594.09, levels=opt,DT[, lapply(.SD, sd, na.rm=FALSE), by=x])
test(1594.10, levels=opt, DT[, lapply(.SD, sd, na.rm=TRUE), by=x], DT[, lapply(.SD, stats::sd, na.rm=TRUE), by=x])
test(1594.11, levels=opt, DT[, lapply(.SD, sd, na.rm=TRUE), by=x, verbose=TRUE], output=out)
 
test(1594.12, levels=opt, DT[, lapply(.SD, prod, na.rm=FALSE), by=x])
test(1594.13, levels=opt, DT[, lapply(.SD, prod, na.rm=TRUE), by=x])
test(1594.14, levels=opt, DT[, lapply(.SD, prod, na.rm=TRUE), by=x, verbose=TRUE], output=out)

# when datatable.optimize<1, no optimisation of j should take place:
dt = data.table(x=1:5, y=6:10, z=c(1,1,1,2,2))
test(1638, options=c(datatable.optimize=0L), dt[, .SD, by=z, verbose=TRUE], output="All optimizations are turned off")

# weighted.mean GForce optimized, #3977
DT = data.table(x=c(3.7,3.3,3.5,2.8), w=c(5,5,4,1), g=1L)
opt = c(1L,2L)
out = c("GForce FALSE", "GForce optimized j to")
test(2231.01, levels=opt, DT[, weighted.mean(x, w), g, verbose=TRUE], data.table(g=1L, V1=3.45333333333333), output=out)
test(2231.02, levels=opt, DT[, weighted.mean(w, x), g, verbose=TRUE], data.table(g=1L, V1=3.89473684210526), output=out)
test(2231.03, levels=opt, DT[, weighted.mean(x), g, verbose=TRUE], data.table(g=1L, V1=3.325), output=out)
# multiple groups
DT = data.table(x=c(1L,2L,2L,3L,4L,5L,5L,6L), w=c(1L,1L,1L,1L,2L,2L,2L,2L), g=rep(1L:2L, each=4L))
test(2231.04, levels=opt, DT[, weighted.mean(x, w), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2,5)), output=out)
test(2231.05, levels=opt, DT[, weighted.mean(x, w), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2,5)), output=out)
test(2231.06, levels=opt, DT[, weighted.mean(x, w), seq(nrow(DT)), verbose=TRUE], data.table(seq=1L:8L, V1=c(1,2,2,3,4,5,5,6)), output=out)
# (only x XOR w) containing NA
DT = data.table(x=c(1L,NA,2L,3L,4L,5L,5L,6L), w=c(1L,1L,1L,1L,2L,NA,2L,2L), g=rep(1L:2L, each=4L))
test(2231.07, levels=opt, DT[, weighted.mean(x, w, na.rm=FALSE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(NA_real_, NA_real_)), output=out)
test(2231.08, levels=opt, DT[, weighted.mean(x, w, na.rm=TRUE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2, NA_real_)), output=out)
test(2231.09, levels=opt, DT[, weighted.mean(x, na.rm=FALSE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(NA, 5)), output=out)
test(2231.10, levels=opt, DT[, weighted.mean(x, na.rm=TRUE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2, 5)), output=out)
# (only x XOR w) containing NaN
DT = data.table(x=c(1L,2L,NaN,3L,4L,5L,5L,6L), w=c(1L,1L,1L,1L,2L,2L,NaN,2L), g=rep(1L:2L, each=4L))
test(2231.11, levels=opt, DT[, weighted.mean(x, w, na.rm=FALSE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(NaN, NA)), output=out)
test(2231.12, levels=opt, DT[, weighted.mean(x, w, na.rm=TRUE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2, NA)), output=out)
test(2231.13, levels=opt, DT[, weighted.mean(x, na.rm=FALSE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(NaN, 5)), output=out)
test(2231.14, levels=opt, DT[, weighted.mean(x, na.rm=TRUE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2, 5)), output=out)
# (only x XOR w) containing NA and NaN
DT = data.table(x=c(1L,NA,NaN,3L,4L,5L,5L,6L), w=c(1L,1L,1L,1L,2L,NA,NaN,2L), g=rep(1L:2L, each=4L))
test(2231.15, levels=opt, DT[, weighted.mean(x, w, na.rm=FALSE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(NA_real_, NA_real_)), output=out)
test(2231.16, levels=opt, DT[, weighted.mean(x, w, na.rm=TRUE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2, NA)), output=out)
test(2231.17, levels=opt, DT[, weighted.mean(x, na.rm=FALSE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(NA, 5)), output=out)
test(2231.18, levels=opt, DT[, weighted.mean(x, na.rm=TRUE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2, 5)), output=out)
# (x and w) containing NA and NaN
DT = data.table(x=c(1L,NA,NaN,3L,4L,5L,5L,6L), w=c(1L,NA,NaN,1L,2L,2L,2L,2L), g=rep(1L:2L, each=4L))
test(2231.19, levels=opt, DT[, weighted.mean(x, w, na.rm=FALSE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(NA, 5)), output=out)
test(2231.20, levels=opt, DT[, weighted.mean(x, w, na.rm=TRUE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2, 5)), output=out)
DT = data.table(x=c(1L,NA,NaN,3L,4L,5L,5L,6L), w=c(1L,NaN,NA,1L,2L,2L,2L,2L), g=rep(1L:2L, each=4L))
test(2231.21, levels=opt, DT[, weighted.mean(x, w, na.rm=FALSE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(NA, 5)), output=out)
test(2231.22, levels=opt, DT[, weighted.mean(x, w, na.rm=TRUE), g, verbose=TRUE], data.table(g=c(1L,2L), V1=c(2, 5)), output=out)
# let wrongly named arguments get lost in ellipsis #5543
DT = data.table(x=c(3.7,3.3,3.5,2.8), w=c(5,5,4,1), g=1L)
test(2231.61, levels=opt, DT[, weighted.mean(x, w),        g, verbose=TRUE], data.table(g=1L, V1=3.45+1/300), output=out)
test(2231.62, levels=opt, DT[, weighted.mean(x, weight=w), g, verbose=TRUE], data.table(g=1L, V1=3.325),      output=out)
test(2231.63, levels=opt, DT[, weighted.mean(x, w, na.rm=FALSE), g],    DT[, stats::weighted.mean(x, w, na.rm=FALSE), g])
test(2231.64, levels=opt, DT[, weighted.mean(x, weight=w, na.rm=TRUE)], DT[, stats::weighted.mean(x, weight=w, na.rm=TRUE)])

# GForce retains attributes in by arguments #5567
dt = data.table(a=letters[1:4], b=structure(1:4, class = c("class_b", "integer"), att=1), c=structure(c(1L,2L,1L,2L), class = c("class_c", "integer")))
opt = c(0,Inf)
out = c("GForce FALSE", "GForce optimized j to")
test(2263.1, levels=opt, options=list(datatable.verbose=TRUE), dt[, .N, b], data.table(b=dt$b, N=1L),               output=out)
test(2263.2, levels=opt, options=list(datatable.verbose=TRUE), dt[, .N, .(b,c)], data.table(b=dt$b, c=dt$c, N=1L),  output=out)
test(2263.3, levels=opt, options=list(datatable.verbose=TRUE), names(attributes(dt[, .N, b]$b)), c("class", "att"), output=out)

# named arguments of c() in j get prepended to lapply(.SD, FUN) #2311
M <- as.data.table(mtcars)
M[, " " := hp]
M[, "." := hp]

sdnames <- setdiff(names(M), "cyl")
sdlist <- vector("list", length(sdnames))
names(sdlist) <- sdnames

opts = 0:2
test(2283 + 0.01, levels=opts,
       names(M[, c(m=lapply(.SD, mean)), by="cyl"]),
       c("cyl", names(c(m=sdlist))))
test(2283 + 0.02, levels=opts,
       names(M[, c(Mpg=list(mpg), lapply(.SD, mean)), by="cyl"]),
       c("cyl", "Mpg", sdnames))
test(2283 + 0.03, levels=opts,
       names(M[, c(Mpg=list(mpg), m=lapply(.SD, mean)), by="cyl"]),
       c("cyl", "Mpg", names(c(m=sdlist))))
test(2283 + 0.04, levels=opts,
       names(M[, c(mpg=list(mpg), mpg=lapply(.SD, mean)), by="cyl"]),
       c("cyl", "mpg", names(c(mpg=sdlist))))
test(2283 + 0.05, levels=opts,
       names(M[, c(list(mpg), lapply(.SD, mean)), by="cyl"]),
       c("cyl", "V1", sdnames))
test(2283 + 0.06, levels=opts,
       names(M[, c(lapply(.SD, mean), list(mpg)), by="cyl"]),
       c("cyl", sdnames, sprintf("V%d", length(sdnames)+1L)))
test(2283 + 0.07, levels=opts,
       names(M[, c(lapply(.SD, mean), lapply(.SD, sum)), by="cyl"]),
       c("cyl", sdnames, sdnames))
test(2283 + 0.08, levels=opts,
       names(M[, c(mean=lapply(.SD, mean), sum=lapply(.SD,  sum)), by="cyl"]),
       c("cyl", names(c(mean=sdlist, sum=sdlist))))
test(2283 + 0.09, levels=opts,
       names(M[, c(lapply(.SD, mean), sum=lapply(.SD,  sum)), by="cyl"]),
       c("cyl", sdnames, names(c(sum=sdlist))) )
test(2283 + 0.10, levels=opts,
       names(M[, c(" "=lapply(.SD, mean), "."=lapply(.SD,  sum)), by="cyl"]),
       c("cyl", names(c(" "=sdlist, "."=sdlist))))
test(2283 + 0.11, levels=opts,
       names(M[, c(A=list(a=mpg, b=hp), lapply(.SD, mean)), by="cyl"]),
       c("cyl", names(c(A=list(a=0, b=0))), sdnames))
test(2283 + 0.12, levels=opts,
       names(M[, c(A=list(mpg, hp), lapply(.SD, mean)), by="cyl"]),
       c("cyl", names(c(A=list(0, 0))), sdnames))
test(2283 + 0.13, levels=opts,
       names(M[, c(A=list(mpg, b=hp, wt), lapply(.SD, mean)), by="cyl"]),
       c("cyl", names(c(A=list(0, b=0, 0))), sdnames))
test(2283 + 0.14, levels=opts,
       names(M[, c(A=list(mpg), lapply(.SD, mean)), by="cyl"]),
       c("cyl", names(c(A=list(0))), sdnames))
test(2283 + 0.15, levels=opts,
       names(M[, c(" "=list(" "=hp, "."=disp, mpg), lapply(.SD, mean)), by="cyl"]),
       c("cyl", names(c(" "=list(" "=0, "."=0, 0))), sdnames))
test(2283 + 0.16, levels=opts,
       names(M[, c("."=list(" "=hp, "."=disp, mpg), lapply(.SD, mean)), by="cyl"]),
       c("cyl", names(c("."=list(" "=0, "."=0, 0))), sdnames))
test(2283 + 0.17, levels=opts,
       names(M[, c(list(mpg, b=hp), lapply(.SD, mean)), by="cyl", .SDcols=c("vs", "am")]),
       c("cyl", "V1", "b", "vs", "am"))
test(2283 + 0.18, levels=opts,
       names(M[, c(list(mpg, b=hp), c(lapply(.SD, mean))), by="cyl", .SDcols=c("vs", "am")]),
       c("cyl", "V1", "b", "vs", "am"))
test(2283 + 0.19, levels=opts,
       names(M[, c(mpg[1], list(mpg, b=hp), c(lapply(.SD, mean))), by="cyl", .SDcols=c("vs", "am")]),
       c("cyl", "V1", "V2", "b", "vs", "am"))