Merge branch 'master' into fread-file-spaces

Author: aitap

.Rbuildignore

@@ -17,6 +17,7 @@
 ^\.devcontainer$
 ^\.graphics$
 ^\.github$
+^\.jj$
 ^\.vscode$
 ^\.zed$
 ^\.lintr$

.github/workflows/test-coverage.yaml

@@ -51,11 +51,11 @@ jobs:
           covr::to_cobertura(cov)
         shell: Rscript {0}
 
-      - uses: codecov/codecov-action@v4
+      - uses: codecov/codecov-action@v5
         with:
-          fail_ci_if_error: ${{ github.event_name != 'pull_request' && true || false }}
-          file: ./cobertura.xml
-          plugin: noop
+          fail_ci_if_error: ${{ github.event_name != 'pull_request' || secrets.CODECOV_TOKEN }}
+          files: ./cobertura.xml
+          plugins: noop
           disable_search: true
           token: ${{ secrets.CODECOV_TOKEN }}
 

DESCRIPTION

@@ -3,7 +3,7 @@ Version: 1.18.99
 Title: Extension of `data.frame`
 Depends: R (>= 3.5.0)
 Imports: methods
-Suggests: bit64 (>= 4.0.0), bit (>= 4.0.4), R.utils, xts, zoo (>= 1.8-1), yaml, litedown
+Suggests: bit64 (>= 4.0.0), R.utils, xts, zoo (>= 1.8-1), yaml, litedown
 Enhances: knitr, xfun
 Description: Fast aggregation of large data (e.g. 100GB in RAM), fast ordered joins, fast add/modify/delete of columns by group using no copies at all, list columns, friendly and fast character-separated-value read/write. Offers a natural and flexible syntax, for faster development.
 License: MPL-2.0 | file LICENSE

NEWS.md

@@ -28,7 +28,9 @@
 
 1. `fread()` with `skip=0` and `(header=TRUE|FALSE)` no longer skips the first row when it has fewer fields than subsequent rows, [#7463](https://github.com/Rdatatable/data.table/issues/7463). Thanks @emayerhofer for the report and @ben-schwen for the fix.
 
-2. `fread("file://...")` works for file URIs with spaces, [#7550](https://github.com/Rdatatable/data.table/issues/7550). Thanks @aitap for the report and @MichaelChirico for the PR.
+2. `set()` now automatically pre-allocates new column slots if needed, similar to what `:=` already does, [#1831](https://github.com/Rdatatable/data.table/issues/1831) [#4100](https://github.com/Rdatatable/data.table/issues/4100). Thanks to @zachokeeffe and @tyner for the report and @ben-schwen for the fix.
+
+3. `fread("file://...")` works for file URIs with spaces, [#7550](https://github.com/Rdatatable/data.table/issues/7550). Thanks @aitap for the report and @MichaelChirico for the PR.
 
 ## data.table [v1.18.0](https://github.com/Rdatatable/data.table/milestone/37?closed=1)  23 December 2025
 

R/data.table.R

@@ -2854,10 +2854,20 @@ setcolorder = function(x, neworder=key(x), before=NULL, after=NULL, skip_absent=
   invisible(x)
 }
 
+.set_needs_alloccol = function(x, value) {
+  # automatically allocate more space when tl <= ncol (either full or loaded from disk)
+  if (truelength(x) <= length(x)) return(TRUE)
+  if (selfrefok(x, verbose=FALSE) >= 1L) return(FALSE)
+  # value can be NULL or list with NULLs inside
+  if (is.null(value)) return(TRUE)
+  if (!is.list(value)) return(FALSE)
+  any(vapply_1b(value, is.null))
+}
+
 set = function(x,i=NULL,j,value)  # low overhead, loopable
 {
   # If removing columns from a table that's not selfrefok, need to call setalloccol first, #7488
-  if ((is.null(value) || (is.list(value) && any(vapply_1b(value, is.null)))) && selfrefok(x, verbose=FALSE) < 1L) {
+  if (.set_needs_alloccol(x, value)) {
     name = substitute(x)
     setalloccol(x, verbose=FALSE)
     if (is.name(name)) {

inst/tests/froll.Rraw

@@ -11,7 +11,7 @@ if (exists("test.data.table", .GlobalEnv, inherits=FALSE)) {
 
 exact_NaN = isTRUE(capabilities()["long.double"]) && identical(as.integer(.Machine$longdouble.digits), 64L)
 if (!exact_NaN) {
-  cat("\n**** Skipping 7 NaN/NA algo='exact' tests because .Machine$longdouble.digits==", .Machine$longdouble.digits, " (!=64); e.g. under valgrind\n\n", sep="")
+  cat("\n**** Skipping 8 NaN/NA algo='exact' tests because .Machine$longdouble.digits==", .Machine$longdouble.digits, " (!=64); e.g. under valgrind\n\n", sep="")
   # for Matt when he runs valgrind it is 53, but 64 when running regular R
   # froll.c uses long double and appears to require full long double accuracy in the algo='exact'
 }
@@ -1448,9 +1448,12 @@ test(6001.727, frollvar(adaptive=TRUE, c(1:2,NA), c(2,0,2), algo="exact"), c(NA_
 test(6001.728, frollvar(adaptive=TRUE, c(1:2,NA), c(2,0,2), algo="exact", na.rm=TRUE), c(NA_real_,NA_real_,NA_real_))
 test(6001.729, frollvar(adaptive=TRUE, c(1:2,NA), c(2,0,2), algo="exact", na.rm=TRUE, partial=TRUE), c(NA_real_,NA_real_,NA_real_))
 test(6001.730, frollvar(adaptive=TRUE, c(1:2,NA), c(2,0,2), fill=99, algo="exact", na.rm=TRUE), c(99,NA,NA))
-y = c(1e8+2.980232e-8, 1e8, 1e8, 1e8) # CLAMP0 test
-test(6001.731, frollvar(y, 3)[4L], 0)
-test(6001.732, frollsd(y, 3)[4L], 0)
+# numerical stability: we need to guarantee frollvar(x, n) >= 0 for all x, n
+#   the exact epsilon here is a bit implementation-dependent (as in #7546), but what's
+#   crucial is the output is never negative (or NaN after sqrt() for frollsd).
+y = c(1e8+2.980232e-8, 1e8, 1e8, 1e8)
+test(6001.731, between(frollvar(y, 3)[4L], 0, 1e-7))
+test(6001.732, between(frollsd(y, 3)[4L], 0, 1e-7))
 test(6001.733, frollvar(y, c(3,3,3,3), adaptive=TRUE)[4L], 0)
 test(6001.734, frollsd(y, c(3,3,3,3), adaptive=TRUE)[4L], 0)
 test(6001.740, frollvar(c(1.5,2.5,2,NA), c(3,3)), list(c(NA,NA,0.25,NA), c(NA,NA,0.25,NA)), output="running sequentially, because outer parallelism has been used", options=c(datatable.verbose=TRUE)) # ensure no nested parallelism in rolling functions #7352
@@ -2084,7 +2087,8 @@ if (use.fork) {
   test(6010.772, .selfref.ok(ans[[2L]]))
   ans = frollapply(1:2, 2, function(x) list(data.table(x)), fill=list(data.table(NA)), simplify=FALSE)
   test(6010.773, !.selfref.ok(ans[[2L]][[1L]]))
-  test(6010.7731, set(ans[[2L]][[1L]],, "newcol", 1L), error="data.table has either been loaded from disk")
+  # deactivated by #5443
+  # test(6010.7731, set(ans[[2L]][[1L]],, "newcol", 1L), error="data.table has either been loaded from disk")
   ans = lapply(ans, lapply, setDT)
   test(6010.774, .selfref.ok(ans[[2L]][[1L]])) ## fix after
   ans = frollapply(1:2, 2, function(x) list(data.table(x)), fill=list(data.table(NA)), simplify=function(x) lapply(x, lapply, setDT))

inst/tests/tests.Rraw

@@ -14797,7 +14797,7 @@ test(2016.1, name, "DT")
 test(2016.2, DT, data.table(a=1:3))
 test(2016.3, DT[2,a:=4L], data.table(a=INT(1,4,3)))        # no error for := when existing column
 test(2016.4, set(DT,3L,1L,5L), data.table(a=INT(1,4,5)))   # no error for set() when existing column
-test(2016.5, set(DT,2L,"newCol",5L), error="either been loaded from disk.*or constructed manually.*Please run setDT.*setalloccol.*on it first")   # just set()
+test(2016.5, set(DT,2L,"newCol",5L), data.table(a=INT(1,4,5), newCol=INT(NA,5L,NA)))   # works since set overallocates #4100
 test(2016.6, DT[2,newCol:=6L], data.table(a=INT(1,4,5), newCol=INT(NA,6L,NA)))  # := ok (it changes DT in caller)
 unlink(tt)
 
@@ -19478,7 +19478,7 @@ test(2290.4, DT[, `:=`(a = 2, c := 3)], error="It looks like you re-used `:=` in
 df = data.frame(a=1:3)
 setDT(df)
 attr(df, "att") = 1
-test(2291.1, set(df, NULL, "new", "new"), error="either been loaded from disk.*or constructed manually.*Please run setDT.*setalloccol.*on it first")
+test(2291.1, set(df, NULL, "new", "new"), setattr(data.table(a=1:3, new="new"), "att", 1)) # fixed when calling setalloccol before set #4100
 
 # ns-qualified bysub error, #6493
 DT = data.table(a = 1)
@@ -21960,12 +21960,21 @@ test(2355.2, fread(txt, skip=0, header=TRUE), data.table(V1 = c("b1", "c1"), a1
 test(2355.3, fread(txt, skip=0, header=FALSE), data.table(V1=character(), V2=character(), V3=character()), warning="Consider fill=TRUE")
 test(2355.4, fread(txt, skip=0, fill=TRUE), data.table(V1 = c("a1", "b1", "c1"), V2 = c("a2", "b2", "c2"), V3 = c("", "b3", "c3")))
 
+# re-overallocate in set if quota is reached #496 #1831 #4100
+DT = data.table()
+test(2356.1, options=c(datatable.alloccol=1L), {for (i in seq(10L)) set(DT, j = paste0("V",i), value = i); ncol(DT)}, 10L)
+DT = structure(list(a = 1, b = 2), class = c("data.table", "data.frame"))
+test(2356.2, options=c(datatable.alloccol=1L), set(DT, j="c", value=3), data.table(a=1, b=2, c=3))
+# ensure := and set are consistent if they need to overallocate
+DT = data.table(); DT2 = data.table()
+test(2356.3, options=c(datatable.alloccol=1L), {for (i in seq(10L)) set(DT, j = sprintf("V%d",i), value = i); DT}, {for (i in seq(10)) DT2[, sprintf("V%d",i) := i]; DT2})
+
 # fread works on file:// URIs with spaces, #7550
 local({
   f = tempfile("with spaces"); on.exit(unlink(f))
   DT = data.table(a = 1L, b = 2L)
   fwrite(DT, f)
 
-  test(2356.1, fread(f), DT)
-  test(2356.2, fread(paste0("file://", f)), DT)
+  test(2357.1, fread(f), DT)
+  test(2357.2, fread(paste0("file://", f)), DT)
 })

src/cj.c

@@ -6,21 +6,22 @@
    - The memory copying operations (blockwise replication of data using memcpy)
    - The creation of all combinations of the input vectors over the cross-product space
 */
-SEXP cj(SEXP base_list) {
+SEXP cj(SEXP base_list)
+{
   int ncol = LENGTH(base_list);
   SEXP out = PROTECT(allocVector(VECSXP, ncol));
   int nrow = 1;
   // already confirmed to be less than .Machine$integer.max at R level
-  for (int j=0; j<ncol; ++j) nrow *= length(VECTOR_ELT(base_list, j));
+  for (int j = 0; j < ncol; j++) nrow *= length(VECTOR_ELT(base_list, j));
   int eachrep = 1;
-  for (int j=ncol-1; j>=0; --j) {
+  for (int j = ncol - 1; j >= 0; j--) {
     SEXP source = VECTOR_ELT(base_list, j), target;
-    SET_VECTOR_ELT(out, j, target=allocVector(TYPEOF(source), nrow));
+    SET_VECTOR_ELT(out, j, target = allocVector(TYPEOF(source), nrow));
     copyMostAttrib(source, target);  // includes levels of factors, integer64, custom classes, etc
-    if (nrow==0) continue;  // one or more columns are empty so the result will be empty, #2511
+    if (nrow == 0) continue;  // one or more columns are empty so the result will be empty, #2511
     int thislen = LENGTH(source);
-    int blocklen = thislen*eachrep;
-    int ncopy = nrow/blocklen;
+    int blocklen = thislen * eachrep;
+    int ncopy = nrow / blocklen;
     switch(TYPEOF(source)) {
     case LGLSXP:
     case INTSXP: {
@@ -29,64 +30,64 @@ SEXP cj(SEXP base_list) {
       #pragma omp parallel for num_threads(getDTthreads(thislen*eachrep, true))
       // default static schedule so two threads won't write to same cache line in last column
       // if they did write to same cache line (and will when last column's thislen is small) there's no correctness issue
-      for (int i=0; i<thislen; ++i) {
+      for (int i = 0; i < thislen; i++) {
         const int item = sourceP[i];
-        const int end = (i+1)*eachrep;
-        for (int j=i*eachrep; j<end; ++j) targetP[j] = item;  // no div, mod or read ops inside loop; just rep a const contiguous write
+        const int end = (i + 1) * eachrep;
+        for (int j = i * eachrep; j < end; j++) targetP[j] = item;  // no div, mod or read ops inside loop; just rep a const contiguous write
       }
       #pragma omp parallel for num_threads(getDTthreads(ncopy*blocklen, true))
-      for (int i=1; i<ncopy; ++i) {
-        memcpy(targetP + i*blocklen, targetP, blocklen*sizeof(*targetP));
+      for (int i = 1; i < ncopy; i++) {
+        memcpy(targetP + i * blocklen, targetP, blocklen * sizeof(*targetP));
       }
     } break;
     case REALSXP: {
       const double *restrict sourceP = REAL(source);
       double *restrict targetP = REAL(target);
       #pragma omp parallel for num_threads(getDTthreads(thislen*eachrep, true))
-      for (int i=0; i<thislen; ++i) {
+      for (int i = 0; i < thislen; i++) {
         const double item = sourceP[i];
-        const int end=(i+1)*eachrep;
-        for (int j=i*eachrep; j<end; ++j) targetP[j] = item;
+        const int end = (i + 1) * eachrep;
+        for (int j = i * eachrep; j < end; j++) targetP[j] = item;
       }
       #pragma omp parallel for num_threads(getDTthreads(ncopy*blocklen, true))
-      for (int i=1; i<ncopy; ++i) {
-        memcpy(targetP + i*blocklen, targetP, blocklen*sizeof(double));
+      for (int i = 1; i < ncopy; i++) {
+        memcpy(targetP + i * blocklen, targetP, blocklen * sizeof(double));
       }
     } break;
     case CPLXSXP: {
       const Rcomplex *restrict sourceP = COMPLEX(source);
       Rcomplex *restrict targetP = COMPLEX(target);
       #pragma omp parallel for num_threads(getDTthreads(thislen*eachrep, true))
-      for (int i=0; i<thislen; ++i) {
+      for (int i = 0; i < thislen; i++) {
         const Rcomplex item = sourceP[i];
-        const int end=(i+1)*eachrep;
-        for (int j=i*eachrep; j<end; ++j) targetP[j] = item;
+        const int end = (i + 1) * eachrep;
+        for (int j = i * eachrep; j < end; j++) targetP[j] = item;
       }
       #pragma omp parallel for num_threads(getDTthreads(ncopy*blocklen, true))
-      for (int i=1; i<ncopy; ++i) {
-        memcpy(targetP + i*blocklen, targetP, blocklen*sizeof(Rcomplex));
+      for (int i = 1; i < ncopy; i++) {
+        memcpy(targetP + i * blocklen, targetP, blocklen * sizeof(Rcomplex));
       }
     } break;
     case STRSXP: {
       const SEXP *sourceP = STRING_PTR_RO(source);
       int start = 0;
-      for (int i=0; i<ncopy; ++i) {
-        for (int j=0; j<thislen; ++j) {
+      for (int i = 0; i < ncopy; i++) {
+        for (int j = 0; j < thislen; j++) {
           const SEXP item = sourceP[j];
-          const int end = start+eachrep;
-          for (int k=start; k<end; ++k) SET_STRING_ELT(target, k, item);  // no div, mod, or read-API call to STRING_ELT
+          const int end = start + eachrep;
+          for (int k = start; k < end; k++) SET_STRING_ELT(target, k, item);  // no div, mod, or read-API call to STRING_ELT
           start = end;
         }
       }
     } break;
     case VECSXP: {
       const SEXP *sourceP = SEXPPTR_RO(source);
       int start = 0;
-      for (int i=0; i<ncopy; ++i) {
-        for (int j=0; j<thislen; ++j) {
+      for (int i = 0; i < ncopy; i++) {
+        for (int j = 0; j < thislen; j++) {
           const SEXP item = sourceP[j];
-          const int end = start+eachrep;
-          for (int k=start; k<end; ++k) SET_VECTOR_ELT(target, k, item);
+          const int end = start + eachrep;
+          for (int k = start; k < end; k++) SET_VECTOR_ELT(target, k, item);
           start = end;
         }
       }