From 3e4739363c73e3695f7ae9270e201f259d82b5a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Wed, 29 Jan 2025 14:26:44 -0300 Subject: [PATCH 1/8] include AlertaDengueAnalise gitmodule --- .github/workflows/main.yaml | 4 ++-- .gitmodules | 3 +++ AlertaDengueAnalise | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 160000 AlertaDengueAnalise diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 769312c..353307e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4 - name: Build - run: docker build -t ADAnalise . + run: docker build -t adanalise . - name: Test - run: docker run ADAnalise arg1 arg2 arg3 + run: docker run adanalise arg1 arg2 arg3 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..dc554da --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "AlertaDengueAnalise"] + path = AlertaDengueAnalise + url = https://github.com/AlertaDengue/AlertaDengueAnalise diff --git a/AlertaDengueAnalise b/AlertaDengueAnalise new file mode 160000 index 0000000..a0c211d --- /dev/null +++ b/AlertaDengueAnalise @@ -0,0 +1 @@ +Subproject commit a0c211d60216e3a561dee0d29c554573feba359a From 581cf93aea16becd3064be07384fd9f75a2ed9e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Fri, 31 Jan 2025 14:09:03 -0300 Subject: [PATCH 2/8] main.R --- .github/workflows/main.yaml | 17 ++++ Dockerfile | 18 ++--- README.md | 11 ++- main.R | 151 ++++++++++++++++++++++++++++++++++++ setup.R | 3 +- your_script.R | 11 --- 6 files changed, 187 insertions(+), 24 deletions(-) create mode 100644 main.R delete mode 100644 your_script.R diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 353307e..ce6f318 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -22,6 +22,23 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: "latest" + mamba-version: "*" + environment-file: conda/dev.yaml + channels: conda-forge,nodefaults + activate-environment: adcaptura + use-mamba: true + miniforge-variant: Mambaforge + + - name: Update all submodules + run: | + git submodule update --init --recursive - name: Build run: docker build -t adanalise . diff --git a/Dockerfile b/Dockerfile index 522b7a7..4dc93ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,16 @@ FROM rocker/tidyverse:latest -RUN echo 'options(repos = c(CRAN = "https://cloud.r-project.org"))' >>"${R_HOME}/etc/Rprofile.site" -COPY . /app -WORKDIR /app +RUN echo 'options(repos = c(CRAN = "https://cloud.r-project.org"))' >> "${R_HOME}/etc/Rprofile.site" -# Copy the dependencies file and the R script into the container -COPY dependencies.txt setup.R ./ -COPY your_script.R . +WORKDIR /app +COPY AlertaDengueAnalise/ ./ -# Install the R package dependencies +COPY dependencies.txt ./ RUN R -e "install.packages(readLines('dependencies.txt'), repos='http://cran.rstudio.com/')" + +COPY setup.R ./ RUN Rscript setup.R -# Command to run the R script -ENTRYPOINT ["Rscript", "your_script.R"] +COPY main.R ./ + +ENTRYPOINT ["Rscript", "main.R"] diff --git a/README.md b/README.md index 00ed382..fec7ebe 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,22 @@ # r-script-container docker template to run R scripts +## Install git submodules +```sh +git submodule update --init --recursive +``` + ## Building the container ```bash -docker build -t r-script-container . +docker build -t adanalise . ``` ## Running the container in an interactive terminal ```bash -docker run -v $(pwd):/app -it r-script-container Rscript /app/your_script.R arg1 arg2 arg3 +docker run -v $(pwd):/app -it adanalise Rscript /app/your_script.R arg1 arg2 arg3 ``` ## Running the container without a terminal ```bash -docker run r-script-container arg1 arg2 arg3 +docker run adanalise arg1 arg2 arg3 ``` diff --git a/main.R b/main.R new file mode 100644 index 0000000..fce1a9c --- /dev/null +++ b/main.R @@ -0,0 +1,151 @@ +library(argparse) +library(RPostgres) +source("AlertaDengueAnalise/config/config_global_2020.R") + +ufs <- data.frame( + estado = c( + "Acre", "Amazonas", "Amapá", "Pará", "Rondônia", "Roraima", "Tocantins", + "Alagoas", "Bahia", "Ceará", "Maranhão", "Piauí", "Pernambuco", "Paraíba", + "Rio Grande do Norte", "Sergipe", "Goiás", "Mato Grosso", + "Mato Grosso do Sul", "Distrito Federal", "Espírito Santo", "Minas Gerais", + "Rio de Janeiro", "São Paulo", "Paraná", "Rio Grande do Sul", + "Santa Catarina" + ), + sigla = c( + "AC", "AM", "AP", "PA", "RO", "RR", "TO", + "AL", "BA", "CE", "MA", "PI", "PE", "PB", "RN", "SE", + "GO", "MT", "MS", "DF", + "ES", "MG", "RJ", "SP", + "PR", "RS", "SC" + ), +) + +parser <- ArgumentParser() +parser$add_argument( + "--uf", + choices = ufs$sigla, + help = "Choose the UFs. Default to all UFs. Example: `--uf SP RJ MG`", + nargs = "+" +) +parser$add_argument( + "--disease", + choices = c("dengue", "chik", "zika"), + help = "Specify the disease: dengue, chik, or zika", + default = "dengue" +) +parser$add_argument( + "--epiweek", + type = "character", + required = TRUE, + help = "Epidemiological week. Format: YYYYWW" +) +parser$add_argument( + "--uri", + type = "character", + required = TRUE, + help = "Database URI" +) + +args <- parser$parse_args() + +if (!is.null(args$uf)) { + ufs <- ufs[ufs$sigla %in% args$uf, ] +} + +disease <- args$disease +epiweek <- args$epiweek +finalday <- seqSE(epiweek, epiweek)$Termino +output_dir <- "./" + +print(ufs) +cat("Disease:", disease, "\n") +cat("Epidemiological Week:", epiweek, "\n") +cat("Final Day:", finalday, "\n") +cat("Output Directory:", output_dir, "\n") + +# if (!dir.exists(paste0(output_dir, epiweek))) { +# dir.create(paste0(output_dir, epiweek), recursive = TRUE) +# } +# +# con <- dbConnect(Rpostgres::Postgres(), url = args$uri) +# +# t1 <- Sys.time() +# for (i in seq_len(nrow(ufs))) { +# print(i) +# estado <- ufs$estado[i] +# +# cid10 <- list(dengue = "A90", chik = "A92", zika = "A92.8") +# filename <- paste0("ale-", ufs$sigla[i], "-", epiweek, ".RData") +# cities <- getCidades(uf = estado)[, "municipio_geocodigo"] +# +# res <- list() +# +# res[[paste0("ale.", disease)]] <- pipe_infodengue( +# cities, +# cid10 = cid10[[disease]], +# nowcasting = ifelse(disease == "dengue", "none", "bayesian"), +# finalday = finalday, +# narule = "arima", +# iniSE = 201001, +# dataini = "sinpri", +# completetail = 0 +# ) +# +# res[[paste0("restab.", disease)]] <- tabela_historico( +# res[[paste0("ale.", disease)]], +# iniSE = epiweek - 100 +# ) +# +# save(res, file = paste0(output_dir, epiweek, "/", filename)) +# } +# t2 <- Sys.time() +# message(paste("total time was", t2 - t1)) +# +# dbDisconnect(con) +# +# file_paths <- fs::dir_ls(paste0(output_dir, epiweek, "/")) +# +# j <- 1 +# for (i in seq_along(file_paths)) { +# load.Rdata(file_paths[i], "res") +# assign(paste0("res", j), res) +# j <- j + 1 +# load(file_paths[i]) +# } +# rm(res) +# +# output_path <- paste0("AlertaDengueAnalise/main/sql/output_", disease, ".sql") +# +# lapply(seq_along(file_paths), function(j) { +# restab <- eval(parse( +# text = paste0("res", j, "[['restab.", disease, "']] %>% bind_rows()") +# )) +# data <- do.call(rbind, restab) +# data$casos_est_max[data$casos_est_max > 10000] <- NA +# summary(data) +# write_alerta(data, writetofile = TRUE, arq = output_path) +# }) +# +# ale_data <- list() +# +# for (i in seq_along(file_paths)) { +# data <- eval(parse( +# text = paste0( +# "transpose(res", i, "[['ale.", disease, "']])[[1]] %>% bind_rows()" +# ) +# )) +# indices <- eval(parse( +# text = paste0( +# "transpose(res", i, "[['ale.", disease, "']])[[2]] %>% bind_rows()" +# ) +# )) +# ale_data[[i]] <- cbind(data, indices) +# } +# +# res <- do.call(rbind, ale_data) +# +# if (!dir.exists(paste0(output_dir, "BR"))) { +# dir.create(paste0(output_dir, "BR")) +# } +# +# save(res, file = paste0(output_dir, "BR/ale-BR-", epiweek, ".RData")) diff --git a/setup.R b/setup.R index 6b3c5a4..182f1fd 100644 --- a/setup.R +++ b/setup.R @@ -15,7 +15,8 @@ cran <- c( "cgwtools", "fs", "miceadds", - "brpop" + "brpop", + "argparse" ) install <- function(pkg) { diff --git a/your_script.R b/your_script.R deleted file mode 100644 index 70718c5..0000000 --- a/your_script.R +++ /dev/null @@ -1,11 +0,0 @@ -library(ggplot2) - -#commandArgs picks up the variables you pass from the command line -args <- commandArgs(trailingOnly = TRUE) - -# Use the input argument -if (length(args) > 0) { - print(args) -} else { - cat("No argument was passed.\n") -} From 58e0ed9aec6120ea114855f3ffa05647995f755d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Tue, 4 Feb 2025 15:06:26 -0300 Subject: [PATCH 3/8] Solve requirements for ADanalise --- Dockerfile | 44 ++++++++++++++++++++++++++++++++++++++++---- main.R | 2 +- setup.R | 13 +++++++++---- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4dc93ed..96e5048 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,52 @@ FROM rocker/tidyverse:latest +RUN apt-get update && apt-get install -y \ + libcurl4-openssl-dev \ + libssl-dev \ + libxml2-dev \ + libgmp-dev \ + libmpfr-dev \ + libxt-dev \ + libopenblas-dev \ + libudunits2-dev \ + libgdal-dev \ + libgeos-dev \ + libproj-dev \ + gdal-bin \ + proj-bin \ + liblzma-dev \ + libxt-dev \ + libpng-dev \ + libtiff-dev \ + zlib1g-dev \ + libjpeg-dev \ + libicu-dev \ + gfortran \ + liblapack-dev \ + libatlas-base-dev \ + libnlopt-dev \ + make + RUN echo 'options(repos = c(CRAN = "https://cloud.r-project.org"))' >> "${R_HOME}/etc/Rprofile.site" WORKDIR /app -COPY AlertaDengueAnalise/ ./ - -COPY dependencies.txt ./ -RUN R -e "install.packages(readLines('dependencies.txt'), repos='http://cran.rstudio.com/')" COPY setup.R ./ RUN Rscript setup.R +COPY dependencies.txt ./ +RUN R -e "options(error = traceback); install.packages(readLines('dependencies.txt'), repos='http://cran.rstudio.com/')" + +RUN R -e "options(error = traceback); install.packages('units', repos='https://cloud.r-project.org')" +RUN R -e "options(error = traceback); install.packages('sf', repos='https://cloud.r-project.org')" + +RUN R -e 'options(error = traceback); install.packages("INLA",repos=c(getOption("repos"),INLA="https://inla.r-inla-download.org/R/stable"), dep=TRUE)' + +RUN R -e 'options(error = traceback); utils::install.packages("miceadds")' +RUN R -e "options(error = traceback); remotes::install_github('inlabru-org/fmesher', ref = 'stable')" + +COPY AlertaDengueAnalise/ ./AlertaDengueAnalise/ + COPY main.R ./ ENTRYPOINT ["Rscript", "main.R"] diff --git a/main.R b/main.R index fce1a9c..4f69330 100644 --- a/main.R +++ b/main.R @@ -17,7 +17,7 @@ ufs <- data.frame( "GO", "MT", "MS", "DF", "ES", "MG", "RJ", "SP", "PR", "RS", "SC" - ), + ) ) parser <- ArgumentParser() diff --git a/setup.R b/setup.R index 182f1fd..4f9c454 100644 --- a/setup.R +++ b/setup.R @@ -10,17 +10,22 @@ cran <- c( "devtools", "lubridate", "grid", - "INLA", "parallel", "cgwtools", "fs", - "miceadds", "brpop", - "argparse" + "argparse", + "futile.logger", + "units", + "sf", + "fmesher", + "miceadds" ) +options(error = traceback) + install <- function(pkg) { - if (!require(pkg, character.only = TRUE)) { + if (!nzchar(system.file(package = pkg))) { install.packages(pkg, dependencies = TRUE) } } From 9cfbe3855ecfa2f58b347572447f97fafc1cd89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Wed, 5 Feb 2025 10:13:59 -0300 Subject: [PATCH 4/8] Use .env to pass URI connection --- .env.tpl | 1 + .gitignore | 1 + main.R | 38 ++++++++++++++++---------------------- setup.R | 3 ++- 4 files changed, 20 insertions(+), 23 deletions(-) create mode 100644 .env.tpl create mode 100644 .gitignore diff --git a/.env.tpl b/.env.tpl new file mode 100644 index 0000000..2964920 --- /dev/null +++ b/.env.tpl @@ -0,0 +1 @@ +DB_URI=$DB_URI diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env diff --git a/main.R b/main.R index 4f69330..880f762 100644 --- a/main.R +++ b/main.R @@ -39,12 +39,12 @@ parser$add_argument( required = TRUE, help = "Epidemiological week. Format: YYYYWW" ) -parser$add_argument( - "--uri", - type = "character", - required = TRUE, - help = "Database URI" -) + +uri <- Sys.getenv("DB_URI") + +if (uri == "") { + stop("Error: DB_URI environment variable is not set.") +} args <- parser$parse_args() @@ -57,19 +57,13 @@ epiweek <- args$epiweek finalday <- seqSE(epiweek, epiweek)$Termino output_dir <- "./" -print(ufs) -cat("Disease:", disease, "\n") -cat("Epidemiological Week:", epiweek, "\n") -cat("Final Day:", finalday, "\n") -cat("Output Directory:", output_dir, "\n") +if (!dir.exists(paste0(output_dir, epiweek))) { + dir.create(paste0(output_dir, epiweek), recursive = TRUE) +} -# if (!dir.exists(paste0(output_dir, epiweek))) { -# dir.create(paste0(output_dir, epiweek), recursive = TRUE) -# } -# -# con <- dbConnect(Rpostgres::Postgres(), url = args$uri) -# -# t1 <- Sys.time() +con <- dbConnect(Rpostgres::Postgres(), url = args$uri) + +t1 <- Sys.time() # for (i in seq_len(nrow(ufs))) { # print(i) # estado <- ufs$estado[i] @@ -98,10 +92,10 @@ cat("Output Directory:", output_dir, "\n") # # save(res, file = paste0(output_dir, epiweek, "/", filename)) # } -# t2 <- Sys.time() -# message(paste("total time was", t2 - t1)) -# -# dbDisconnect(con) +t2 <- Sys.time() +message(paste("total time was", t2 - t1)) + +dbDisconnect(con) # # file_paths <- fs::dir_ls(paste0(output_dir, epiweek, "/")) # diff --git a/setup.R b/setup.R index 4f9c454..733f136 100644 --- a/setup.R +++ b/setup.R @@ -19,7 +19,8 @@ cran <- c( "units", "sf", "fmesher", - "miceadds" + "miceadds", + "RPostgres" ) options(error = traceback) From 2b8a76ff8be6bdb9722859b3fcf24b6288018b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Wed, 5 Feb 2025 11:37:23 -0300 Subject: [PATCH 5/8] Fix sql connection & prepare to attach volumes to the sql output files --- .env.tpl | 2 +- Dockerfile | 16 ++++++++++++++++ README.md | 15 +++++++-------- main.R | 23 +++++++++++++++-------- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/.env.tpl b/.env.tpl index 2964920..17a6b89 100644 --- a/.env.tpl +++ b/.env.tpl @@ -1 +1 @@ -DB_URI=$DB_URI +DB_URI=$DB_URI # NOTE: Don't use double quotes in this variable diff --git a/Dockerfile b/Dockerfile index 96e5048..3b8d432 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,20 @@ FROM rocker/tidyverse:latest +ARG UID=1000 +ARG GID=1000 + +RUN if getent group $GID > /dev/null; then \ + groupmod -n adanalise $(getent group $GID | cut -d: -f1); \ + else \ + groupadd -g $GID adanalise; \ + fi && \ + if getent passwd $UID > /dev/null; then \ + usermod -l adanalise $(getent passwd $UID | cut -d: -f1) && \ + usermod -g adanalise adanalise; \ + else \ + useradd -m -u $UID -g adanalise adanalise; \ + fi + RUN apt-get update && apt-get install -y \ libcurl4-openssl-dev \ libssl-dev \ @@ -30,6 +45,7 @@ RUN apt-get update && apt-get install -y \ RUN echo 'options(repos = c(CRAN = "https://cloud.r-project.org"))' >> "${R_HOME}/etc/Rprofile.site" WORKDIR /app +USER adanalise COPY setup.R ./ RUN Rscript setup.R diff --git a/README.md b/README.md index fec7ebe..aa8b942 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,16 @@ docker template to run R scripts git submodule update --init --recursive ``` -## Building the container -```bash -docker build -t adanalise . -``` +## Create & populate .env file +See .env.tpl to check the required env variables in .env file -## Running the container in an interactive terminal + +## Building the container ```bash -docker run -v $(pwd):/app -it adanalise Rscript /app/your_script.R arg1 arg2 arg3 +docker build --build-arg UID=$(id -u) --build-arg GID=$(id -g) -t adanalise . ``` -## Running the container without a terminal +## Running the container (all UFs) ```bash -docker run adanalise arg1 arg2 arg3 +docker run --env-file .env --network infodengue-dev_infodengue -i adanalise --epiweek 202022 ``` diff --git a/main.R b/main.R index 880f762..11699e2 100644 --- a/main.R +++ b/main.R @@ -1,6 +1,8 @@ +source("AlertaDengueAnalise/config/config_global_2020.R") + library(argparse) library(RPostgres) -source("AlertaDengueAnalise/config/config_global_2020.R") +library(httr) ufs <- data.frame( estado = c( @@ -25,7 +27,8 @@ parser$add_argument( "--uf", choices = ufs$sigla, help = "Choose the UFs. Default to all UFs. Example: `--uf SP RJ MG`", - nargs = "+" + nargs = "+", + required = FALSE ) parser$add_argument( "--disease", @@ -40,11 +43,8 @@ parser$add_argument( help = "Epidemiological week. Format: YYYYWW" ) -uri <- Sys.getenv("DB_URI") - -if (uri == "") { - stop("Error: DB_URI environment variable is not set.") -} +uri <- parse_url(Sys.getenv("DB_URI")) +print(uri) args <- parser$parse_args() @@ -61,7 +61,14 @@ if (!dir.exists(paste0(output_dir, epiweek))) { dir.create(paste0(output_dir, epiweek), recursive = TRUE) } -con <- dbConnect(Rpostgres::Postgres(), url = args$uri) +con <- dbConnect( + RPostgres::Postgres(), + dbname = sub("^/", "", uri$path), + host = uri$hostname, + port = uri$port, + user = uri$username, + password = uri$password +) t1 <- Sys.time() # for (i in seq_len(nrow(ufs))) { From a63e30fe47c713cb4cdd3b7d26340edf4b371bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Wed, 5 Feb 2025 17:26:18 -0300 Subject: [PATCH 6/8] rollback non-root config due to missing R packages --- Dockerfile | 16 ---------------- README.md | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b8d432..96e5048 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,5 @@ FROM rocker/tidyverse:latest -ARG UID=1000 -ARG GID=1000 - -RUN if getent group $GID > /dev/null; then \ - groupmod -n adanalise $(getent group $GID | cut -d: -f1); \ - else \ - groupadd -g $GID adanalise; \ - fi && \ - if getent passwd $UID > /dev/null; then \ - usermod -l adanalise $(getent passwd $UID | cut -d: -f1) && \ - usermod -g adanalise adanalise; \ - else \ - useradd -m -u $UID -g adanalise adanalise; \ - fi - RUN apt-get update && apt-get install -y \ libcurl4-openssl-dev \ libssl-dev \ @@ -45,7 +30,6 @@ RUN apt-get update && apt-get install -y \ RUN echo 'options(repos = c(CRAN = "https://cloud.r-project.org"))' >> "${R_HOME}/etc/Rprofile.site" WORKDIR /app -USER adanalise COPY setup.R ./ RUN Rscript setup.R diff --git a/README.md b/README.md index aa8b942..c5f894d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ See .env.tpl to check the required env variables in .env file ## Building the container ```bash -docker build --build-arg UID=$(id -u) --build-arg GID=$(id -g) -t adanalise . +docker build -t adanalise . ``` ## Running the container (all UFs) From 689c6021d222dca7c2116aad83d0a7b007361cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Wed, 5 Feb 2025 18:54:50 -0300 Subject: [PATCH 7/8] remove the need for AlertaDengueAnalise repo --- .gitignore | 1 + AlertaDengueAnalise | 1 - Dockerfile | 2 - README.md | 6 -- main.R | 156 +++++++++++++++++++++++++------------------- 5 files changed, 90 insertions(+), 76 deletions(-) delete mode 160000 AlertaDengueAnalise diff --git a/.gitignore b/.gitignore index 4c49bd7..eb2f977 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .env +output/ diff --git a/AlertaDengueAnalise b/AlertaDengueAnalise deleted file mode 160000 index a0c211d..0000000 --- a/AlertaDengueAnalise +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a0c211d60216e3a561dee0d29c554573feba359a diff --git a/Dockerfile b/Dockerfile index 96e5048..59e3c99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,8 +45,6 @@ RUN R -e 'options(error = traceback); install.packages("INLA",repos=c(getOption( RUN R -e 'options(error = traceback); utils::install.packages("miceadds")' RUN R -e "options(error = traceback); remotes::install_github('inlabru-org/fmesher', ref = 'stable')" -COPY AlertaDengueAnalise/ ./AlertaDengueAnalise/ - COPY main.R ./ ENTRYPOINT ["Rscript", "main.R"] diff --git a/README.md b/README.md index c5f894d..b5dc9b0 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,9 @@ # r-script-container docker template to run R scripts -## Install git submodules -```sh -git submodule update --init --recursive -``` - ## Create & populate .env file See .env.tpl to check the required env variables in .env file - ## Building the container ```bash docker build -t adanalise . diff --git a/main.R b/main.R index 11699e2..e38943d 100644 --- a/main.R +++ b/main.R @@ -1,8 +1,26 @@ -source("AlertaDengueAnalise/config/config_global_2020.R") +pkgs <- c( + "foreign", + "tidyverse", + "forecast", + "RPostgreSQL", + "xtable", + "zoo", + "assertthat", + "DBI", + "futile.logger", + "lubridate", + "grid", + "INLA", + "cgwtools", + "fs", + "miceadds", + "AlertTools", + "parallel", + "argparse", + "httr" +) -library(argparse) -library(RPostgres) -library(httr) +lapply(pkgs, library, character.only = TRUE, quietly = T) ufs <- data.frame( estado = c( @@ -44,7 +62,6 @@ parser$add_argument( ) uri <- parse_url(Sys.getenv("DB_URI")) -print(uri) args <- parser$parse_args() @@ -53,24 +70,28 @@ if (!is.null(args$uf)) { } disease <- args$disease -epiweek <- args$epiweek +epiweek <- as.numeric(args$epiweek) finalday <- seqSE(epiweek, epiweek)$Termino -output_dir <- "./" +output_dir <- "./output/" + +if (!dir.exists(paste0(output_dir, "sql/"))) { + dir.create(paste0(output_dir, "sql/"), recursive = TRUE) +} if (!dir.exists(paste0(output_dir, epiweek))) { dir.create(paste0(output_dir, epiweek), recursive = TRUE) } -con <- dbConnect( - RPostgres::Postgres(), - dbname = sub("^/", "", uri$path), - host = uri$hostname, - port = uri$port, - user = uri$username, - password = uri$password -) - -t1 <- Sys.time() +# con <- dbConnect( +# RPostgreSQL::PostgreSQL(), +# dbname = sub("^/", "", uri$path), +# host = uri$hostname, +# port = uri$port, +# user = uri$username, +# password = uri$password +# ) +# +# t1 <- Sys.time() # for (i in seq_len(nrow(ufs))) { # print(i) # estado <- ufs$estado[i] @@ -99,54 +120,55 @@ t1 <- Sys.time() # # save(res, file = paste0(output_dir, epiweek, "/", filename)) # } -t2 <- Sys.time() -message(paste("total time was", t2 - t1)) - -dbDisconnect(con) -# -# file_paths <- fs::dir_ls(paste0(output_dir, epiweek, "/")) -# -# j <- 1 -# for (i in seq_along(file_paths)) { -# load.Rdata(file_paths[i], "res") -# assign(paste0("res", j), res) -# j <- j + 1 -# load(file_paths[i]) -# } -# rm(res) -# -# output_path <- paste0("AlertaDengueAnalise/main/sql/output_", disease, ".sql") -# -# lapply(seq_along(file_paths), function(j) { -# restab <- eval(parse( -# text = paste0("res", j, "[['restab.", disease, "']] %>% bind_rows()") -# )) -# data <- do.call(rbind, restab) -# data$casos_est_max[data$casos_est_max > 10000] <- NA -# summary(data) -# write_alerta(data, writetofile = TRUE, arq = output_path) -# }) -# -# ale_data <- list() -# -# for (i in seq_along(file_paths)) { -# data <- eval(parse( -# text = paste0( -# "transpose(res", i, "[['ale.", disease, "']])[[1]] %>% bind_rows()" -# ) -# )) -# indices <- eval(parse( -# text = paste0( -# "transpose(res", i, "[['ale.", disease, "']])[[2]] %>% bind_rows()" -# ) -# )) -# ale_data[[i]] <- cbind(data, indices) -# } +# t2 <- Sys.time() +# message(paste("total time was", t2 - t1)) # -# res <- do.call(rbind, ale_data) -# -# if (!dir.exists(paste0(output_dir, "BR"))) { -# dir.create(paste0(output_dir, "BR")) -# } -# -# save(res, file = paste0(output_dir, "BR/ale-BR-", epiweek, ".RData")) +# dbDisconnect(con) + +file_paths <- fs::dir_ls(paste0(output_dir, epiweek, "/")) + +j <- 1 +for (i in seq_along(file_paths)) { + load.Rdata(file_paths[i], "res") + assign(paste0("res", j), res) + j <- j + 1 + load(file_paths[i]) +} +rm(res) + +output_path <- paste0(output_dir, "sql/output_", disease, ".sql") + +lapply(seq_along(file_paths), function(j) { + restab <- eval(parse( + text = paste0("res", j, "[['restab.", disease, "']] %>% bind_rows()") + )) + data <- do.call(rbind, restab) + print(data) + data$casos_est_max[data$casos_est_max > 10000] <- NA + summary(data) + write_alerta(data, writetofile = TRUE, arq = output_path) +}) + +ale_data <- list() + +for (i in seq_along(file_paths)) { + data <- eval(parse( + text = paste0( + "transpose(res", i, "[['ale.", disease, "']])[[1]] %>% bind_rows()" + ) + )) + indices <- eval(parse( + text = paste0( + "transpose(res", i, "[['ale.", disease, "']])[[2]] %>% bind_rows()" + ) + )) + ale_data[[i]] <- cbind(data, indices) +} + +res <- do.call(rbind, ale_data) + +if (!dir.exists(paste0(output_dir, "BR"))) { + dir.create(paste0(output_dir, "BR")) +} + +save(res, file = paste0(output_dir, "BR/ale-BR-", epiweek, ".RData")) From c405e4413c258819da0730c83921c9fba66cc548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=A3=20Bida=20Vacaro?= Date: Thu, 6 Feb 2025 21:50:37 -0300 Subject: [PATCH 8/8] transpose the dataframe from pipe_infodengue (bug) --- .github/workflows/main.yaml | 21 +------- README.md | 3 +- main.R | 95 ++++++++++++++++++++----------------- 3 files changed, 54 insertions(+), 65 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index ce6f318..1f55d00 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -22,26 +22,9 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@v3 - with: - miniconda-version: "latest" - mamba-version: "*" - environment-file: conda/dev.yaml - channels: conda-forge,nodefaults - activate-environment: adcaptura - use-mamba: true - miniforge-variant: Mambaforge - - - name: Update all submodules - run: | - git submodule update --init --recursive - name: Build run: docker build -t adanalise . - - name: Test - run: docker run adanalise arg1 arg2 arg3 + # - name: Test + # run: docker run adanalise arg1 arg2 arg3 diff --git a/README.md b/README.md index b5dc9b0..814daab 100644 --- a/README.md +++ b/README.md @@ -11,5 +11,4 @@ docker build -t adanalise . ## Running the container (all UFs) ```bash -docker run --env-file .env --network infodengue-dev_infodengue -i adanalise --epiweek 202022 -``` +docker run --env-file .env --volume ./output:/app/output -i adanalise --disease dengue --epiweek 202501 # --uf RS SC PR diff --git a/main.R b/main.R index e38943d..cfaf76f 100644 --- a/main.R +++ b/main.R @@ -82,48 +82,47 @@ if (!dir.exists(paste0(output_dir, epiweek))) { dir.create(paste0(output_dir, epiweek), recursive = TRUE) } -# con <- dbConnect( -# RPostgreSQL::PostgreSQL(), -# dbname = sub("^/", "", uri$path), -# host = uri$hostname, -# port = uri$port, -# user = uri$username, -# password = uri$password -# ) -# -# t1 <- Sys.time() -# for (i in seq_len(nrow(ufs))) { -# print(i) -# estado <- ufs$estado[i] -# -# cid10 <- list(dengue = "A90", chik = "A92", zika = "A92.8") -# filename <- paste0("ale-", ufs$sigla[i], "-", epiweek, ".RData") -# cities <- getCidades(uf = estado)[, "municipio_geocodigo"] -# -# res <- list() -# -# res[[paste0("ale.", disease)]] <- pipe_infodengue( -# cities, -# cid10 = cid10[[disease]], -# nowcasting = ifelse(disease == "dengue", "none", "bayesian"), -# finalday = finalday, -# narule = "arima", -# iniSE = 201001, -# dataini = "sinpri", -# completetail = 0 -# ) -# -# res[[paste0("restab.", disease)]] <- tabela_historico( -# res[[paste0("ale.", disease)]], -# iniSE = epiweek - 100 -# ) -# -# save(res, file = paste0(output_dir, epiweek, "/", filename)) -# } -# t2 <- Sys.time() -# message(paste("total time was", t2 - t1)) -# -# dbDisconnect(con) +con <- dbConnect( + RPostgreSQL::PostgreSQL(), + dbname = sub("^/", "", uri$path), + host = uri$hostname, + port = uri$port, + user = uri$username, + password = uri$password +) + +t1 <- Sys.time() +for (i in seq_len(nrow(ufs))) { + print(i) + estado <- ufs$estado[i] + cid10 <- list(dengue = "A90", chik = "A92", zika = "A92.8") + filename <- paste0("ale-", ufs$sigla[i], "-", epiweek, ".RData") + cities <- getCidades(uf = estado)[, "municipio_geocodigo"] + + res <- list() + + res[[paste0("ale.", disease)]] <- pipe_infodengue( + cities, + cid10 = cid10[[disease]], + nowcasting = ifelse(disease == "dengue", "none", "bayesian"), + finalday = finalday, + narule = "arima", + iniSE = 201001, + dataini = "sinpri", + completetail = 0 + ) + + res[[paste0("restab.", disease)]] <- tabela_historico( + res[[paste0("ale.", disease)]], + iniSE = epiweek - 100 + ) + + save(res, file = paste0(output_dir, epiweek, "/", filename)) +} +t2 <- Sys.time() +message(paste("total time was", t2 - t1)) + +dbDisconnect(con) file_paths <- fs::dir_ls(paste0(output_dir, epiweek, "/")) @@ -143,8 +142,16 @@ lapply(seq_along(file_paths), function(j) { text = paste0("res", j, "[['restab.", disease, "']] %>% bind_rows()") )) data <- do.call(rbind, restab) - print(data) - data$casos_est_max[data$casos_est_max > 10000] <- NA + data <- as.data.frame(t(data)) # FIX: t() is an workaround + + if (!"casos_est_max" %in% names(data)) { + data$casos_est_max <- NA + } + + if (is.numeric(data$casos_est_max)) { + data$casos_est_max[data$casos_est_max > 10000] <- NA + } + summary(data) write_alerta(data, writetofile = TRUE, arq = output_path) })