Teach forder to re-use existing key and index attributes instead of sorting from scratch

R/bmerge.R

@@ -116,9 +116,6 @@ bmerge = function(i, x, icols, xcols, roll, rollends, nomatch, mult, ops, verbos
     }
   }
 
-  ## after all modifications of i, check if i has a proper key on all icols
-  io = identical(icols, head(chmatch(key(i), names(i)), length(icols)))
-
   ## after all modifications of x, check if x has a proper key on all xcols.
   ## If not, calculate the order. Also for non-equi joins, the order must be calculated.
   non_equi = which.first(ops != 1L) # 1 is "==" operator
@@ -178,7 +175,7 @@ bmerge = function(i, x, icols, xcols, roll, rollends, nomatch, mult, ops, verbos
   }
 
   if (verbose) {last.started.at=proc.time();cat("Starting bmerge ...\n");flush.console()}
-  ans = .Call(Cbmerge, i, x, as.integer(icols), as.integer(xcols), io, xo, roll, rollends, nomatch, mult, ops, nqgrp, nqmaxgrp)
+  ans = .Call(Cbmerge, i, x, as.integer(icols), as.integer(xcols), xo, roll, rollends, nomatch, mult, ops, nqgrp, nqmaxgrp)
   if (verbose) {cat("bmerge done in",timetaken(last.started.at),"\n"); flush.console()}
   # TO DO: xo could be moved inside Cbmerge
 

R/setkey.R

@@ -172,8 +172,7 @@ is.sorted = function(x, by=seq_along(x)) {
 }
 
 ORDERING_TYPES = c('logical', 'integer', 'double', 'complex', 'character')
-forderv = function(x, by=seq_along(x), retGrp=FALSE, sort=TRUE, order=1L, na.last=FALSE)
-{
+forderv = function(x, by=seq_along(x), retGrp=FALSE, sort=TRUE, order=1L, na.last=FALSE, lazy=NA) {
   if (is.atomic(x)) {  # including forderv(NULL) which returns error consistent with base::order(NULL),
     if (!missing(by) && !is.null(by)) stop("x is a single vector, non-NULL 'by' doesn't make sense")
     by = NULL
@@ -183,7 +182,7 @@ forderv = function(x, by=seq_along(x), retGrp=FALSE, sort=TRUE, order=1L, na.las
     if (length(order) == 1L) order = rep(order, length(by))
   }
   order = as.integer(order) # length and contents of order being +1/-1 is checked at C level
-  .Call(Cforder, x, by, retGrp, sort, order, na.last)  # returns integer() if already sorted, regardless of sort=TRUE|FALSE
+  .Call(Cforder, x, by, retGrp, sort, order, na.last, lazy)  # returns integer() if already sorted, regardless of sort=TRUE|FALSE
 }
 
 forder = function(..., na.last=TRUE, decreasing=FALSE)

inst/tests/tests.Rraw

@@ -13364,7 +13364,7 @@ test(1962.0461, forderv(DT, order = c(1L, -1L)),       error="Either order= is n
 test(1962.0462, forderv(DT, order = 2L),               error='Item 1 of order (ascending/descending) is 2. Must be +1 or -1')
 test(1962.0471, forderv(mean),                         error="'x' argument must be data.table compatible")
 test(1962.0472, forderv(DT, by=mean),                  error="argument specifying columns must be character or numeric")
-test(1962.0473, forderv(NULL),                         error="DT is an empty list() of 0 columns")
+test(1962.0473, forderv(NULL),                         error="DT is NULL")
 
 setDF(DT)
 test(1962.0481, forder(DT), 3:1)
@@ -16846,3 +16846,123 @@ A = data.table(A=c(complex(real = 1:3, imaginary=c(0, -1, 1)), NaN))
 test(2138.3, rbind(A,B), data.table(A=c(as.character(A$A), B$A)))
 A = data.table(A=as.complex(rep(NA, 5)))
 test(2138.4, rbind(A,B), data.table(A=c(as.character(A$A), B$A)))
+
+# lazy forder
+dd = data.table(a=1:2, b=2:1)
+d = copy(dd)
+op = options(datatable.verbose=TRUE)
+test(2139.01, forderv(d, "b"), 2:1, output="forder.*opt=-1.*took")
+test(2139.02, forderv(d, "b", lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+setkeyv(d, "b")
+options(datatable.verbose=TRUE)
+test(2139.03, forderv(d, "b"), integer(), output="forder.*opt=1.*took")
+test(2139.04, forderv(d, "b", lazy=FALSE), integer(), output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+d = copy(dd)
+setindexv(d, "b")
+options(datatable.verbose=TRUE)
+test(2139.05, forderv(d, "b"), 2:1, output="forder.*opt=2.*took")
+test(2139.06, forderv(d, "b", lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+d = copy(dd)
+options(datatable.verbose=TRUE)
+test(2139.11, forderv(d, c("a","b")), integer(), output="forder.*opt=-1.*took")
+test(2139.12, forderv(d, c("a","b"), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.13, forderv(d, c("b","a")), 2:1, output="forder.*opt=-1.*took")
+test(2139.14, forderv(d, c("b","a"), lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+setkeyv(d, c("a","b"))
+options(datatable.verbose=TRUE)
+test(2139.21, forderv(d, c("a","b")), integer(), output="forder.*opt=1.*took")
+test(2139.22, forderv(d, c("a","b"), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.23, forderv(d, c("b","a")), 2:1, output="forder.*opt=-1.*took")
+test(2139.24, forderv(d, c("b","a"), lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+setkeyv(d, c("b","a"))
+options(datatable.verbose=TRUE)
+test(2139.25, forderv(d, c("a","b")), 2:1, output="forder.*opt=-1.*took")
+test(2139.26, forderv(d, c("a","b"), lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+test(2139.27, forderv(d, c("b","a")), integer(), output="forder.*opt=1.*took")
+test(2139.28, forderv(d, c("b","a"), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+d = copy(dd)
+setindexv(d, c("a","b"))
+options(datatable.verbose=TRUE)
+test(2139.31, forderv(d, c("a","b")), integer(), output="forder.*opt=2.*took")
+test(2139.32, forderv(d, c("a","b"), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.33, forderv(d, c("b","a")), 2:1, output="forder.*opt=-1.*took")
+test(2139.34, forderv(d, c("b","a"), lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+setindexv(d, NULL)
+setindexv(d, c("b","a"))
+options(datatable.verbose=TRUE)
+test(2139.35, forderv(d, c("a","b")), integer(), output="forder.*opt=-1.*took")
+test(2139.36, forderv(d, c("a","b"), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.37, forderv(d, c("b","a")), 2:1, output="forder.*opt=2.*took")
+test(2139.38, forderv(d, c("b","a"), lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+setindexv(d, NULL)
+setindexv(d, list(c("a","b"), c("b","a")))
+options(datatable.verbose=TRUE)
+test(2139.41, forderv(d, c("a","b")), integer(), output="forder.*opt=2.*took")
+test(2139.42, forderv(d, c("a","b"), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.43, forderv(d, c("b","a")), 2:1, output="forder.*opt=2.*took")
+test(2139.44, forderv(d, c("b","a"), lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+d = copy(dd)
+setkeyv(d, c("a","b"))
+setindexv(d, list(c("a","b"), c("b","a")))
+options(datatable.verbose=TRUE)
+test(2139.51, forderv(d, c("a","b")), integer(), output="forder.*opt=1.*took", notOutput="forder.*opt=2.*took") # idxOpt is not reached
+test(2139.52, forderv(d, c("a","b"), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.53, forderv(d, c("b","a")), 2:1, output="forder.*opt=2.*took")
+test(2139.54, forderv(d, c("b","a"), lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+setkeyv(d, NULL)
+setindexv(d, list(c("a","b"), c("b","a")))
+options(datatable.verbose=TRUE)
+test(2139.55, forderv(d, c("a","b")), integer(), output="forder.*opt=2.*took", notOutput="forder.*opt=1.*took")
+test(2139.56, forderv(d, c("a","b"), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+d = copy(dd)
+setkeyv(d, c("a","b"))
+setindexv(d, list(c("a","a"), c("b","a")))
+options(datatable.verbose=TRUE)
+ab = structure(integer(), starts=1:2, maxgrpn=1L)
+ba = structure(2:1, starts=1:2, maxgrpn=1L)
+test(2139.61, forderv(d, c("a","b"), retGrp=TRUE), ab, output="forder.*opt=0.*took")
+test(2139.62, forderv(d, c("a","b"), retGrp=TRUE, lazy=FALSE), ab, output="forder.*opt=0.*took")
+test(2139.63, forderv(d, c("b","a"), retGrp=TRUE), ba, output="forder.*opt=0.*took")
+test(2139.64, forderv(d, c("b","a"), retGrp=TRUE, lazy=FALSE), ba, output="forder.*opt=0.*took")
+test(2139.65, forderv(d, c("a","b"), na.last=TRUE), integer(), output="forder.*opt=0.*took")
+test(2139.66, forderv(d, c("a","b"), na.last=TRUE, lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.67, forderv(d, c("b","a"), na.last=TRUE), 2:1, output="forder.*opt=0.*took")
+test(2139.68, forderv(d, c("b","a"), na.last=TRUE, lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+test(2139.69, forderv(d, c("a","b"), sort=FALSE, retGrp=TRUE), ab, output="forder.*opt=0.*took")
+test(2139.70, forderv(d, c("a","b"), sort=FALSE, retGrp=TRUE, lazy=FALSE), ab, output="forder.*opt=0.*took")
+test(2139.71, forderv(d, c("b","a"), sort=FALSE, retGrp=TRUE), ab, output="forder.*opt=0.*took")
+test(2139.72, forderv(d, c("b","a"), sort=FALSE, retGrp=TRUE, lazy=FALSE), ab, output="forder.*opt=0.*took")
+test(2139.73, forderv(d, c("a","b"), order=-1L), 2:1, output="forder.*opt=0.*took")
+test(2139.74, forderv(d, c("a","b"), order=-1L, lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+test(2139.75, forderv(d, c("b","a"), order=-1L), integer(), output="forder.*opt=0.*took")
+test(2139.76, forderv(d, c("b","a"), order=-1L, lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.77, forderv(d, c("a","b"), order=c(1L,-1L)), integer(), output="forder.*opt=0.*took")
+test(2139.78, forderv(d, c("a","b"), order=c(1L,-1L), lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.79, forderv(d, c("b","a"), order=c(1L,-1L)), 2:1, output="forder.*opt=0.*took")
+test(2139.80, forderv(d, c("b","a"), order=c(1L,-1L), lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+test(2139.81, forderv(1:2), integer(), output="forder.*opt=0.*took")
+test(2139.82, forderv(1:2, lazy=FALSE), integer(), output="forder.*opt=0.*took")
+test(2139.83, forderv(2:1), 2:1, output="forder.*opt=0.*took")
+test(2139.84, forderv(2:1, lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(datatable.verbose=FALSE)
+d = copy(dd)
+setindexv(d, "b")
+options(datatable.verbose=TRUE)
+op2 = options(datatable.use.index=FALSE)
+test(2139.91, forderv(d, "b"), 2:1, output="forder.*opt=-1.*took")
+test(2139.92, forderv(d, "b", lazy=FALSE), 2:1, output="forder.*opt=0.*took")
+options(op2)
+options(op)
+test(2139.99, forderv(data.table(a=1), lazy=c(TRUE, TRUE)), error="lazy must be")
+

src/bmerge.c

@@ -37,7 +37,11 @@ static Rboolean rollToNearest=FALSE;
 
 void bmerge_r(int xlowIn, int xuppIn, int ilowIn, int iuppIn, int col, int thisgrp, int lowmax, int uppmax);
 
-SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg, SEXP isorted, SEXP xoArg, SEXP rollarg, SEXP rollendsArg, SEXP nomatchArg, SEXP multArg, SEXP opArg, SEXP nqgrpArg, SEXP nqmaxgrpArg) {
+SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg, SEXP xoArg, SEXP rollarg, SEXP rollendsArg, SEXP nomatchArg, SEXP multArg, SEXP opArg, SEXP nqgrpArg, SEXP nqmaxgrpArg) {
+  const bool verbose = GetVerbose();
+  double tic=0.0, tic0=0.0;
+  if (verbose)
+    tic = omp_get_wtime();
   int xN, iN, protecti=0;
   ctr=0; // needed for non-equi join case
   SEXP retFirstArg, retLengthArg, retIndexArg, allLen1Arg, allGrp1Arg;
@@ -138,17 +142,15 @@ SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg, SEXP isorted, SE
   allGrp1[0] = TRUE;
   protecti += 2;
 
-  // isorted arg
-  o = NULL;
-  if (!LOGICAL(isorted)[0]) {
-    SEXP order = PROTECT(allocVector(INTSXP, length(icolsArg)));
-    protecti++;
-    for (int j=0; j<LENGTH(order); j++) INTEGER(order)[j]=1;   // rep(1L, length(icolsArg))
-    SEXP oSxp = PROTECT(forder(i, icolsArg, ScalarLogical(FALSE), ScalarLogical(TRUE), order, ScalarLogical(FALSE)));
-    protecti++;
-    // TODO - split head of forder into C-level callable
-    if (!LENGTH(oSxp)) o = NULL; else o = INTEGER(oSxp);
-  }
+  SEXP order = PROTECT(allocVector(INTSXP, length(icolsArg)));
+  protecti++;
+  for (int j=0; j<LENGTH(order); j++)
+    INTEGER(order)[j]=1;   // rep(1L, length(icolsArg))
+  SEXP oSxp = PROTECT(forder(i, icolsArg, ScalarLogical(FALSE), ScalarLogical(TRUE), order, ScalarLogical(FALSE), ScalarLogical(TRUE))); protecti++;
+  if (!LENGTH(oSxp))
+    o = NULL;
+  else
+    o = INTEGER(oSxp);
 
   // xo arg
   xo = NULL;
@@ -159,10 +161,14 @@ SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg, SEXP isorted, SE
 
   // start bmerge
   if (iN) {
+    if (verbose)
+      tic0 = omp_get_wtime();
     // embarassingly parallel if we've storage space for nqmaxgrp*iN
     for (int kk=0; kk<nqmaxgrp; kk++) {
       bmerge_r(-1,xN,-1,iN,scols,kk+1,1,1);
     }
+    if (verbose)
+      Rprintf("bmerge: looping bmerge_r took %.3fs\n", omp_get_wtime()-tic0);
   }
   ctr += iN;
   if (nqmaxgrp > 1 && mult == ALL) {
@@ -193,6 +199,8 @@ SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg, SEXP isorted, SE
     Free(retLength);
     Free(retIndex);
   }
+  if (verbose)
+    Rprintf("bmerge: took %.3fs\n", omp_get_wtime()-tic);
   UNPROTECT(protecti);
   return (ans);
 }

src/data.table.h

@@ -123,7 +123,8 @@ int checkOverAlloc(SEXP x);
 // forder.c
 int StrCmp(SEXP x, SEXP y);
 uint64_t dtwiddle(void *p, int i);
-SEXP forder(SEXP DT, SEXP by, SEXP retGrp, SEXP sortStrArg, SEXP orderArg, SEXP naArg);
+SEXP forderDo(SEXP DT, SEXP by, SEXP retGrp, SEXP sortStrArg, SEXP orderArg, SEXP naArg);
+SEXP forder(SEXP DT, SEXP by, SEXP retGrp, SEXP sortStrArg, SEXP orderArg, SEXP naArg, SEXP lazyArg); // lazy wrapper to forderDo
 int getNumericRounding_C();
 
 // reorder.c
@@ -170,7 +171,7 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols,
                 SEXP on, SEXP verbose);
 
 // bmerge.c
-SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg, SEXP isorted,
+SEXP bmerge(SEXP iArg, SEXP xArg, SEXP icolsArg, SEXP xcolsArg,
                 SEXP xoArg, SEXP rollarg, SEXP rollendsArg, SEXP nomatchArg,
                 SEXP multArg, SEXP opArg, SEXP nqgrpArg, SEXP nqmaxgrpArg);