adjust process resources usage

grunwaldlab · Nov 15, 2024 · 89700ff · 89700ff
1 parent 16a4a19
commit 89700ff
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 19 deletions.
diff --git a/conf/modules.config b/conf/modules.config
@@ -281,9 +281,10 @@ process {
 
     withName: SOURMASH_COMPARE {
         ext.args = "-k 31 --ani"
-        cpus      = { 4     * task.attempt }
-        memory    = { 16.GB * task.attempt }
-        time      = { 12.h  * task.attempt }
+        maxRetries = 2
+        cpus   = { 4     * task.attempt }
+        memory = { 16.GB * Math.pow(3, task.attempt - 1) }
+        time   = { 12.h  * task.attempt }
     }
 
     withName: 'MAIN_REPORT.*' {
@@ -302,6 +303,12 @@ process {
         time   = { 12.h  * task.attempt }
     }
 
+    withName: BUSCO {
+        cpus   = { 6     * task.attempt }
+        memory = { 16.GB * task.attempt }
+        time   = { 120.h * task.attempt }
+    }
+
     withName: PIRATE {
         cpus   = { 8     * task.attempt }
         memory = { 24.GB * task.attempt }

diff --git a/nextflow.config b/nextflow.config
@@ -22,12 +22,12 @@ params {
     allow_non_refseq             = false
     allow_partial_refs           = false
     allow_unannotated            = false
-    n_ref_strains                = 10
-    n_ref_species                = 20
-    n_ref_genera                 = 20
-    n_ref_closest                = 3
-    n_ref_closest_named          = 2
-    n_ref_context                = 7
+    n_ref_strains                = 5
+    n_ref_species                = 10
+    n_ref_genera                 = 10
+    n_ref_closest                = 2
+    n_ref_closest_named          = 1
+    n_ref_context                = 5
     ref_min_ani                  = 0.95
     phylo_min_genes              = 10
     phylo_max_genes              = 300
@@ -43,7 +43,7 @@ params {
 
     // Boilerplate options
     outdir                       = null
-    trace_dir                    = null
+    trace_dir                    = "${params.outdir}/pipeline_info"
     publish_dir_mode             = 'copy'
     copymode                     = 'medium'
     email                        = null

diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -73,7 +73,8 @@
                 "download_bakta_db": {
                     "type": "boolean",
                     "description": "Download the database required for running Bakta. This or --bakta_db must be included. Note that this will download gigabytes of information, so if you are planning to do repeated runs without --resume it would be better to download the database manually according to the bakta documentaion and specify it with --bakta_db.",
-                    "fa_icon": "fas fa-download"
+                    "fa_icon": "fas fa-download",
+                    "default":"true"
                 },
                 "bakta_db_type": {
                     "type": "string",
@@ -145,19 +146,19 @@
                 },
                 "n_ref_closest": {
                     "type": "number",
-                    "default": 3,
+                    "default": 2,
                     "description": "The number of references most similar to each sample based on estimated ANI to include in phyogenetic anlyses.",
                     "fa_icon": "fas fa-download"
                 },
                 "n_ref_closest_named": {
                     "type": "number",
-                    "default": 2,
+                    "default": 1,
                     "description": "Same as the 'n_ref_closest' option except that it only applies to referneces with what apppear to be standard latin binomaial names (i.e. two words with no numbers or symbols). This is intended to ensure that a refernece with an informative name is present even if it is not the most similar.",
                     "fa_icon": "fas fa-download"
                 },
                 "n_ref_context": {
                     "type": "number",
-                    "default": 7,
+                    "default": 5,
                     "description": "The number of references representing the entire range of ANI relative to each sample. These are meant to provide context for more similar references. For a group of samples, the fewest total references will be selected that satisify this count for each sample.",
                     "fa_icon": "fas fa-download"
                 },
@@ -169,7 +170,7 @@
                 },
                 "phylo_max_genes": {
                     "type": "number",
-                    "default": 200,
+                    "default": 300,
                     "description": "The maximum number of genes used to conduct a core gene phylogeny.",
                     "fa_icon": "fas fa-align-center"
                 },
@@ -399,7 +400,6 @@
                 },
                 "hpc_queue": {
                     "type": "string",
-		            "default": "",
                     "description": "Name of queue in HPC environment to run jobs.",
                     "hidden": true,
                     "fa_icon": "fas fa-cogs"

diff --git a/subworkflows/local/variant_analysis.nf b/subworkflows/local/variant_analysis.nf
@@ -78,12 +78,12 @@ workflow VARIANT_ANALYSIS {
         }
         .groupTuple(by: [0,1])
     filtered_sample_data_with_refs = grouped_sample_data_with_refs
-        .filter{ it[2].size() > 2 }
+        .filter{ it[2].size() >= 3 }
         .transpose(by: 2)
         .map{ it[2] }
     messages = messages.mix(
         grouped_sample_data_with_refs
-            .filter{ it[2].size() <= 2 }
+            .filter{ it[2].size() < 3 }
             .map{ report_meta, ref_meta, data ->
                 [data[0][0], report_meta, ref_meta, "VARIANT_ANALYSIS", "WARNING", "Sample is excluded from variant calling analysis because there are too few samples aligned to this reference to make a tree."]
             }
@@ -105,7 +105,7 @@ workflow VARIANT_ANALYSIS {
         .map { sample_meta, chopped_reads, report_meta, ref_meta, ref_path, usage, read_paths, sequence_type ->
             [sample_meta, report_meta, ref_meta, ref_path, usage, chopped_reads, sequence_type]
         }
-    filtered_input = sample_data_with_refs.filtered
+    filtered_input = filtered_sample_data_with_refs
         .filter { sample_meta, report_meta, ref_meta, ref_path, usage, read_paths, sequence_type ->
             sequence_type == "illumina" || sequence_type == "bgiseq"
         }