Merge branch 'master' into rchk-gha

.Rbuildignore

@@ -46,3 +46,6 @@
 ^lib$
 ^library$
 ^devwd$
+
+# only the inst/po compressed files are needed, not raw .pot/.po
+^po$

.ci/atime/tests.R

@@ -121,6 +121,24 @@ test.list <- atime::atime_test_list(
     Slow = "c4a2085e35689a108d67dacb2f8261e4964d7e12", # Parent of the first commit in the PR that fixes the issue (https://github.com/Rdatatable/data.table/commit/7cc4da4c1c8e568f655ab5167922dcdb75953801)
     Fast = "1872f473b20fdcddc5c1b35d79fe9229cd9a1d15"), # Last commit in the PR that fixes the issue (https://github.com/Rdatatable/data.table/pull/5427/commits)
 
+  # Issue reported in: https://github.com/Rdatatable/data.table/issues/4200
+  # To be fixed in: https://github.com/Rdatatable/data.table/pull/4558
+  "DT[by] fixed in #4558" = atime::atime_test(
+    N = 10^seq(1, 20),
+    setup = {
+      d <- data.table(
+        id3 = sample(c(seq.int(N*0.9), sample( N*0.9, N*0.1, TRUE))),
+        v1 = sample(5L, N, TRUE),
+        v2 = sample(5L, N, TRUE)
+      )
+    },
+    expr = {
+      expr=data.table:::`[.data.table`(d, , max(v1) - min(v2), by = id3)
+    },
+    Before = "7a9eaf62ede487625200981018d8692be8c6f134", # Parent of the first commit (https://github.com/Rdatatable/data.table/commit/515de90a6068911a148e54343a3503043b8bb87c) in the PR (https://github.com/Rdatatable/data.table/pull/4164/commits) that introduced the regression
+    Regression = "c152ced0e5799acee1589910c69c1a2c6586b95d", # Parent of the first commit (https://github.com/Rdatatable/data.table/commit/15f0598b9828d3af2eb8ddc9b38e0356f42afe4f) in the PR (https://github.com/Rdatatable/data.table/pull/4558/commits) that fixes the regression
+    Fixed = "f750448a2efcd258b3aba57136ee6a95ce56b302"), # Second commit of the PR (https://github.com/Rdatatable/data.table/pull/4558/commits) that fixes the regression
+
   # Issue with sorting again when already sorted: https://github.com/Rdatatable/data.table/issues/4498
   # Fixed in: https://github.com/Rdatatable/data.table/pull/4501
   "DT[,.SD] improved in #4501" = atime::atime_test(
@@ -136,8 +154,7 @@ test.list <- atime::atime_test_list(
     },
     Fast = "353dc7a6b66563b61e44b2fa0d7b73a0f97ca461", # Close-to-last merge commit in the PR (https://github.com/Rdatatable/data.table/pull/4501/commits) that fixes the issue 
     Slow = "3ca83738d70d5597d9e168077f3768e32569c790", # Circa 2024 master parent of close-to-last merge commit (https://github.com/Rdatatable/data.table/commit/353dc7a6b66563b61e44b2fa0d7b73a0f97ca461) in the PR (https://github.com/Rdatatable/data.table/pull/4501/commits) that fixes the issue 
-    Slower = "cacdc92df71b777369a217b6c902c687cf35a70d" # Circa 2020 parent of the first commit (https://github.com/Rdatatable/data.table/commit/74636333d7da965a11dad04c322c752a409db098) in the PR (https://github.com/Rdatatable/data.table/pull/4501/commits) that fixes the issue 
-  ),
+    Slower = "cacdc92df71b777369a217b6c902c687cf35a70d"), # Circa 2020 parent of the first commit (https://github.com/Rdatatable/data.table/commit/74636333d7da965a11dad04c322c752a409db098) in the PR (https://github.com/Rdatatable/data.table/pull/4501/commits) that fixes the issue 
 
   NULL)
 # nolint end: undesirable_operator_linter.

.ci/linters/c/omp_set_num_threads_linter.R

@@ -0,0 +1,13 @@
+
+# Ensure no calls to omp_set_num_threads() [to avoid affecting other packages and base R]
+# Only comments referring to it should be in openmp-utils.c
+omp_set_num_threads_linter = function(c_file) {
+  # strip comments, we only care if the function appears in actual code.
+  processed_lines = system2("gcc", c("-fpreprocessed", "-E", c_file), stdout=TRUE, stderr=FALSE)
+  idx = grep("omp_set_num_threads", processed_lines, fixed = TRUE)
+  if (!length(idx)) return()
+  stop(sprintf(
+    "In %s, found omp_set_num_threads() usage, which could affect other packages and base R:\n%s",
+    c_file, paste0("  ", format(idx), ":", processed_lines[idx], collapse = "\n")
+  ))
+}

.ci/publish.R

@@ -163,7 +163,7 @@ r.ver <- function(x) {
   v = tmp[3L]
   if (identical(v, "rel")) "r-release"
   else if (identical(v, "dev")) "r-devel"
-  else if (identical(v, "old")) "r-oldrel"
+  else if (identical(v, "old") || identical(v, "ancient")) "r-oldrel"
   else {
     if (grepl("\\D", v)) stop("third word in test job name must be rel/dev/old or numbers of R version")
     paste0("r-", paste(strsplit(v, "")[[1L]], collapse="."))

.dev/CRAN_Release.cmd

@@ -53,9 +53,6 @@ grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -P
 # Unicode is now ok. This unicode in tests.Rraw is passing on CRAN.
 grep -RI --exclude-dir=".git" --exclude="*.md" --exclude="*~" --color='auto' -n "[\]u[0-9]" ./
 
-# Ensure no calls to omp_set_num_threads() [to avoid affecting other packages and base R]
-# Only comments referring to it should be in openmp-utils.c
-grep omp_set_num_threads ./src/*
 
 # Ensure no calls to omp_set_nested() as i) it's hard to fully honor OMP_THREAD_LIMIT as required by CRAN, and
 #                                        ii) a simpler non-nested approach is always preferable if possible, as has been the case so far

.github/CONTRIBUTING.md

@@ -114,7 +114,7 @@ and then the `commit` and `push` shown above would push to the branch in the mai
 
 #### Translations
 
-`data.table` offers some translations so that our users can get feedback (errors, warnings, verbose messages) in their native language. Currently we only support Mandarin Chinese.
+`data.table` offers some translations so that our users can get feedback (errors, warnings, verbose messages) in their native language. There is a [translation team](https://github.com/orgs/Rdatatable/teams/translators/teams) for each currently supported translation.
 
 The data for these translations lives in the `po` folder. You do not need to make any changes here for your PR -- translations are updated in a batch before each CRAN release.
 

.github/workflows/code-quality.yaml

@@ -40,9 +40,10 @@ jobs:
       - uses: r-lib/actions/setup-r@v2
       - name: Lint
         run: |
-          for (f in list.files('.ci/linters/c', full.names=TRUE)) source(f)
-          for (f in list.files('src', pattern='[.]c$', full.names=TRUE)) {
-            alloc_linter(f)
+          linter_env = new.env()
+          for (f in list.files('.ci/linters/c', full.names=TRUE)) sys.source(f, linter_env)
+          for (f in list.files('src', pattern='[.][ch]$', full.names=TRUE)) {
+            for (linter in ls(linter_env)) linter_env[[linter]](f)
             # TODO(#6272): Incorporate more checks from CRAN_Release
           }
         shell: Rscript {0}

.gitlab-ci.yml

@@ -110,6 +110,7 @@ test-lin-rel:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-data.table
   variables:
+    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
     _R_CHECK_CRAN_INCOMING_: "FALSE"
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
     _R_CHECK_FORCE_SUGGESTS_: "TRUE"
@@ -131,6 +132,8 @@ test-lin-rel:
 test-lin-rel-vanilla:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-base-gcc
+  variables:
+    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
   script:
     - echo 'CFLAGS=-g -O0 -fno-openmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
     - echo 'CXXFLAGS=-g -O0 -fno-openmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
@@ -143,6 +146,7 @@ test-lin-rel-cran:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-base
   variables:
+    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
     _R_CHECK_CRAN_INCOMING_: "TRUE"           ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes
@@ -163,6 +167,7 @@ test-lin-dev-gcc-strict-cran:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-devel-gcc-strict
   variables:
+    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
     _R_CHECK_CRAN_INCOMING_: "TRUE"
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
     _R_S3_METHOD_LOOKUP_BASEENV_AFTER_GLOBALENV_: "FALSE" ## detects S3 method lookup found on search path #4777
@@ -184,6 +189,7 @@ test-lin-dev-clang-cran:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-devel-clang
   variables:
+    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
     _R_CHECK_CRAN_INCOMING_: "TRUE"
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
     _R_S3_METHOD_LOOKUP_BASEENV_AFTER_GLOBALENV_: "FALSE"
@@ -197,10 +203,9 @@ test-lin-dev-clang-cran:
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, installed package size, non-API calls) but ", shQuote(l)) else q("no")'
 
-## R 3.1.0
 # stated dependency on R
-test-lin-310-cran:
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0
+test-lin-ancient-cran:
+  image: registry.gitlab.com/jangorecki/dockerfiles/r-3.3.0
   <<: *test-lin
   script:
     - *install-deps
@@ -295,7 +300,7 @@ integration:
     - saas-linux-medium-amd64
   only:
     - master
-  needs: ["mirror-packages","build","test-lin-rel","test-lin-rel-cran","test-lin-dev-gcc-strict-cran","test-lin-dev-clang-cran","test-lin-rel-vanilla","test-lin-310-cran","test-win-rel","test-win-dev" ,"test-win-old"]
+  needs: ["mirror-packages","build","test-lin-rel","test-lin-rel-cran","test-lin-dev-gcc-strict-cran","test-lin-dev-clang-cran","test-lin-rel-vanilla","test-lin-ancient-cran","test-win-rel","test-win-dev" ,"test-win-old"]
   script:
     - R --version
     - *install-deps ## markdown pkg not present in r-pkgdown image

DESCRIPTION

@@ -86,5 +86,7 @@ Authors@R: c(
   person("Dmitry", "Shemetov",     role="ctb"),
   person("Nitish", "Jha",          role="ctb"),
   person("Joshua", "Wu",           role="ctb"),
-  person("Iago", "Giné-Vázquez",   role="ctb")
+  person("Iago", "Giné-Vázquez",   role="ctb"),
+  person("Anirban", "Chetia",      role="ctb"),
+  person("Doris", "Amoakohene",    role="ctb")
   )

NAMESPACE

@@ -201,5 +201,5 @@ S3method(format_col, expression)
 export(format_list_item)
 S3method(format_list_item, default)
 
-export(fdroplevels)
+export(fdroplevels, setdroplevels)
 S3method(droplevels, data.table)