Updates for QC, tranches and optional EXIT-RIF gvcf dataset

.gitattributes

@@ -0,0 +1,2 @@
+resources/exit_rif/EXIT-RIF.g.vcf.gz filter=lfs diff=lfs merge=lfs -text
+resources/exit_rif/EXIT-RIF.g.vcf.gz.tbi filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -17,3 +17,7 @@ results*
 *.nextflow*
 
 conda_envs/xbs-nf-env*
+containers/**/*yml
+
+samplesheet.csv
+xbs-nf.sh

README.md

@@ -1,64 +1,31 @@
 # XBS-nf
 
-# Benefits of the Nextflow wrapper
+XBS-nf (compleX Bacterial Samples) is a pipeline for comprehensive genomic analyses of Mycobacterium tuberculosis with a focus on clinical decision making as well as research. 
+
+# Salient features of the implementation
 
 - Fine-grained control over resource allocation (CPU/Memory/Storage)
-- Reliance of bioconda and biocontainers for installing packages for reproducibility
-- Ease of use on a range of infrastructure (cloud/on-prem clusters/local machine)
+- Reliance of bioconda for installing packages for reproducibility
+- Ease of use on a range of infrastructure (cloud/on-prem HPC clusters/ servers (or local machines))
 - Resumability for failed processes
 - Centralized locations for specifying analysis parameters and hardware requirements
-    - XBS-nf parameters (`conf/global_parameters.config`)
+    - XBS-nf parameters (`default_parameters.config`)
     - Hardware requirements (`conf/standard.config`)
-    - Software requirements (`conf/docker.config` or `conf/conda.config`)
-
-# Quickstart for a server/laptop
-
-**NOTE**: The instructions for a cluster system like SLURM/PBS are slightly different!
-
-The simplest use case is to analyze a few genomes on a single machine environment. Almost all aspects are customizable but for the sake of brevity, a bare bones guide for any beginner user is as shown below
-
-- [ ] Clone the project 
-
-```shell
-git clone https://github.com/abhi18av/xbs-nf
-cd xbs-nf
-```
-
-- [ ] Move your genomes (`fastq.gz files`) to a specific folder. For example `xbs-nf/data/full_data` folder
-
-- [ ] Prepare a samplesheet using `xbs-nf/resources/reference_set/xbs-nf.test.csv` as a reference for the format.
+    - Execution (software) requirements (`conf/docker.config` or `conf/conda.config`)
+- A GVCF reference dataset for ~600 samples
 
-You can optionally put your sample samplsheet in `xbs-nf/resources/reference_set/` folder.
+# Usage and Tutorial
 
-- [ ] Update the `xbs-nf/conf/server.config` file to point to the reference sheet
+TODO: For the usage and tutorials please refer the XBS-nf website
 
-- [ ] To run the pipeline, make sure you have `conda` installed. Moreover, if you don't already have `nextflow` installed, you can use the following commands to install it 
-
-```shell
-conda create -n xbs-nf-env -c bioconda -c conda-forge nextflow mamba openjdk=11
-```
-
-
-You can confirm the setup by activating that environment and using the `nextflow info`  command
-
-```
-conda activate -n xbs-nf-env
-
-nextflow info 
-```
-
-- [ ] Then simply issue the following command on the command line 
-
-```
-nextflow run main.nf -profile conda,server
-```
+# Citation
 
+TODO: Update this section and add a citation.cff file 
 
 # Contributions
 
 Contributions are warmly accepted!
 
-
 # License
 
 TODO

conda_envs/setup_conda_envs.sh

@@ -2,10 +2,10 @@
 
 set -xue
 
-# NOTE: These environments must be in /path/to/xbs-nf/conda_envs folder
+# NOTE: Please replace `conda` with `mamba` if it is installed for faster installs.
 
-# NOTE: If there are problems in `mamba`, replace it with `conda`
+# NOTE: The conda environments are expected by the `conda_local` profile to be created within `xbs-nf/conda_envs` directory
 
-mamba create -p ./xbs-nf-env-1 bioconda::gatk4=4.2.4.1 conda-forge::R=4.1  conda-forge::r-ggplot2=3.3.5 bioconda::datamash=1.1.0 bioconda::delly=0.8.7 bioconda::lofreq=2.1.5 bioconda::delly=0.8.7 bioconda::lofreq=2.1.5 bioconda::tb-profiler=4.1.1 bioconda::multiqc=1.11 bioconda::fastqc=0.11.8
+conda env create -p xbs-nf-env-1 --file xbs-nf-env-1.yml
 
-mamba create -p ./xbs-nf-env-2 jemunro::quanttb=1.01 bioconda::bwa=0.7.17 bioconda::samtools=1.9 bioconda::iqtree=2.1.2 bioconda::snp-dists=0.8.2 bioconda::snp-sites=2.4.0 bioconda::bcftools=1.9 bioconda::snpeff=4.3.1t bioconda::clusterpicker=1.2.3
+conda env create -p xbs-nf-env-2 --file xbs-nf-env-2.yml

conda_envs/xbs-nf-env-1.yml

@@ -0,0 +1,16 @@
+name: xbs-nf-env-1
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  - bioconda::gatk4=4.2.4.1 
+  - conda-forge::R=4.1 
+  - conda-forge::r-ggplot2=3.3.5 
+  - bioconda::datamash=1.1.0 
+  - bioconda::delly=0.8.7 
+  - bioconda::lofreq=2.1.5 
+  - bioconda::delly=0.8.7 
+  - bioconda::lofreq=2.1.5 
+  - bioconda::tb-profiler=4.1.1 
+  - bioconda::multiqc=1.11 
+  - bioconda::fastqc=0.11.8

conda_envs/xbs-nf-env-2.yml

@@ -0,0 +1,15 @@
+name: xbs-nf-env-2
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  - jemunro::quanttb=1.01 
+  - bioconda::bwa=0.7.17 
+  - bioconda::samtools=1.9 
+  - bioconda::iqtree=2.1.2 
+  - bioconda::snp-dists=0.8.2 
+  - bioconda::snp-sites=2.4.0 
+  - bioconda::bcftools=1.9 
+  - bioconda::snpeff=4.3.1t 
+  - bioconda::clusterpicker=1.2.3
+

conf/docker.config

@@ -6,12 +6,12 @@ process {
 
     withName:
     'GATK.*|LOFREQ.*|DELLY.*|TBPROFILER.*|MULTIQC.*|FASTQC.*|UTILS.*' {
-        container = "rg.nl-ams.scw.cloud/xbs-nf-containers/xbs-nf-container-1:0.3.0"
+        container = "rg.nl-ams.scw.cloud/xbs-nf-containers/xbs-nf-container-1:0.5.0"
     }
 
     withName:
     'QUANTTB.*|BWA.*|IQTREE.*|SNPDISTS.*|SNPSITES.*|BCFTOOLS.*|BGZIP.*|SAMTOOLS.*|SNPEFF.*|CLUSTERPICKER.*' {
-        container = "rg.nl-ams.scw.cloud/xbs-nf-containers/xbs-nf-container-2:0.3.0"
+        container = "rg.nl-ams.scw.cloud/xbs-nf-containers/xbs-nf-container-2:0.5.0"
     }
 
 /*

conf/optimized_processes.config

@@ -14,11 +14,6 @@ process {
         memory = 2.GB
     }
 
-    withName: 'TBPROFILER_LOAD_LIBRARY' {
-        cpus = 2
-        memory = 2.GB
-    }
-
     withName: 'UTILS_QUANTTB_.*' {
         cpus = 1
         memory = 2.GB
@@ -98,4 +93,8 @@ process {
         cpus = 1
         memory = 2.GB
     }
+
+    withName: 'TBPROFILER_VCF_PROFILE__LOFREQ.*' {
+        maxForks = 1
+     }
 }

conf/test.config

@@ -0,0 +1,57 @@
+manifest {
+
+	name = "SLURM testing with 5 samples with optimization without EXIT-RIF GVCF"
+}
+
+params {
+    outdir = "${projectDir}/results"
+    optimize_variant_recalibration = false
+    compute_minor_variants = true
+    dataset_is_not_contaminated = true
+
+    use_ref_exit_rif_gvcf = false
+
+    //The path to resistance database to use for resistance calling
+    resistance_db = "NONE"
+
+
+    save_mode = 'symlink'
+
+
+    //NOTE: This is customized version for dev time testing (remove gaussian param)
+    GATK_VARIANT_RECALIBRATOR__SNP {
+        results_dir = "${params.outdir}/gatk/variant_recalibrator__snp"
+
+        arguments = " --use-allele-specific-annotations \
+                        -AS \
+                        --target-titv 1.7 \
+                        --truth-sensitivity-tranche 100.0 \
+                        --truth-sensitivity-tranche 99.9 \
+                        --truth-sensitivity-tranche 99.8 \
+                        --truth-sensitivity-tranche 99.7 \
+                        --truth-sensitivity-tranche 99.6 \
+                        --truth-sensitivity-tranche 99.5 \
+                        --truth-sensitivity-tranche 99.4 \
+                        --truth-sensitivity-tranche 99.3 \
+                        --truth-sensitivity-tranche 99.2 \
+                        --truth-sensitivity-tranche 99.1 \
+                        --truth-sensitivity-tranche 99.0 \
+                        --max-gaussians 1 \
+                        -mq-cap 60"
+    }
+
+
+}
+
+executor {
+//    queueSize = 1
+    pollInterval = '5sec'
+}
+
+process {
+
+    executor = "slurm"
+    errorStrategy = { task.attempt < 3 ? 'retry' : 'ignore' }
+
+    time = '1h'
+}

containers/build.sh

@@ -5,10 +5,13 @@ set -uex
 
 DOCKER_NAMESPACE="rg.nl-ams.scw.cloud/xbs-nf-containers"
 
+cp ../conda_envs/xbs-nf-env-1.yml ./xbs-nf-container-1
+cp ../conda_envs/xbs-nf-env-2.yml ./xbs-nf-container-2
+
 for container_dir in $(find * -type d); do
   echo "Building $container_dir ..."
   cd $container_dir
-  CONTAINER_TAG=0.3.0
+  CONTAINER_TAG=0.5.0
   CONTAINER_NAME=$DOCKER_NAMESPACE/$container_dir:$CONTAINER_TAG
   echo "Container Name : $CONTAINER_NAME "
   docker build -t $CONTAINER_NAME .

containers/xbs-nf-container-1/Dockerfile

@@ -1,22 +1,9 @@
 FROM mambaorg/micromamba
 MAINTAINER abhi18av@outlook.com
 
+COPY --chown=$MAMBA_USER:$MAMBA_USER xbs-nf-env-1.yml /tmp/xbs-nf-env-1.yml
 
-RUN \
-   micromamba install -y -n base  -c conda-forge -c bioconda \
-	bioconda::gatk4=4.2.4.1 \
-	conda-forge::R=4.1 \
-	conda-forge::r-ggplot2=3.3.5 \
-	bioconda::datamash=1.1.0 \
-	bioconda::delly=0.8.7 \
-	bioconda::lofreq=2.1.5 \
-	bioconda::delly=0.8.7 \
-	bioconda::lofreq=2.1.5 \
-	bioconda::tb-profiler=4.1.1 \
-	bioconda::multiqc=1.11 \
-	bioconda::fastqc=0.11.8 \
-   && micromamba clean -a -y
-
+RUN micromamba install -y -f /tmp/xbs-nf-env-1.yml -n base
 
-RUN micromamba install -y -n base  conda-forge::procps-ng  \
+RUN micromamba install -y -n base conda-forge::procps-ng  \
    && micromamba clean -a -y

containers/xbs-nf-container-2/Dockerfile

@@ -1,19 +1,9 @@
 FROM mambaorg/micromamba
 MAINTAINER abhi18av@outlook.com
 
+COPY --chown=$MAMBA_USER:$MAMBA_USER xbs-nf-env-2.yml /tmp/xbs-nf-env-2.yml
 
-RUN \
-   micromamba install -y -n base  -c conda-forge -c bioconda \
-     jemunro::quanttb=1.01 \
-     bioconda::bwa=0.7.17 \
-     bioconda::samtools=1.9 \
-     bioconda::iqtree=2.1.2 \
-     bioconda::snp-dists=0.8.2 \
-     bioconda::snp-sites=2.4.0 \
-     bioconda::bcftools=1.9 \
-     bioconda::snpeff=4.3.1t \
-     bioconda::clusterpicker=1.2.3 \
-   && micromamba clean -a -y
+RUN micromamba install -y -f /tmp/xbs-nf-env-2.yml -n base
 
-RUN micromamba install -y -n base conda-forge::procps-ng=3.3.16 conda-forge::bc=v1.07.1  \
+RUN micromamba install -y -n base conda-forge::procps-ng  \
    && micromamba clean -a -y