From 5d138c8fa970e2db72d093be2b06fab777e182cf Mon Sep 17 00:00:00 2001 From: Gianna Pohl Date: Tue, 6 Feb 2024 08:58:09 +0100 Subject: [PATCH 01/10] changed automated naming of software version files to avoid problems when process is called within a submodule --- modules/01_fastp.nf | 12 ++++++------ modules/02_bwa_aln.nf | 28 ++++++++++++++-------------- modules/02_bwa_mem.nf | 16 ++++++++-------- modules/02_bwa_mem_2.nf | 16 ++++++++-------- modules/02_star.nf | 12 ++++++------ modules/03_index.nf | 6 +++--- 6 files changed, 45 insertions(+), 45 deletions(-) diff --git a/modules/01_fastp.nf b/modules/01_fastp.nf index 7638f8c..a972813 100644 --- a/modules/01_fastp.nf +++ b/modules/01_fastp.nf @@ -16,7 +16,7 @@ process FASTP_PAIRED { file("${fastq2.baseName}.trimmed.fq.gz"), emit: trimmed_fastqs file("${name}.fastp_stats.json") file("${name}.fastp_stats.html") - file("software_versions.${task.process}.txt") + file("software_versions.FASTP_PAIRED.txt") """ # --input_files needs to be forced, otherwise it is inherited from profile in tests @@ -29,8 +29,8 @@ process FASTP_PAIRED { --html ${name}.fastp_stats.html \ --thread ${params.cpus} - echo ${params.manifest} >> software_versions.${task.process}.txt - fastp --version 2>> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.FASTP_PAIRED.txt + fastp --version 2>> software_versions.FASTP_PAIRED.txt """ } @@ -50,7 +50,7 @@ process FASTP_SINGLE { tuple val(name), file("${fastq1.baseName}.trimmed.fq.gz"), emit: trimmed_fastqs file("${name}.fastp_stats.json") file("${name}.fastp_stats.html") - file("software_versions.${task.process}.txt") + file("software_versions.FASTP_SINGLE.txt") """ # --input_files needs to be forced, otherwise it is inherited from profile in tests @@ -61,7 +61,7 @@ process FASTP_SINGLE { --html ${name}.fastp_stats.html \ --thread ${params.cpus} - echo ${params.manifest} >> software_versions.${task.process}.txt - fastp --version 2>> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.FASTP_SINGLE.txt + fastp --version 2>> software_versions.FASTP_SINGLE.txt """ } \ No newline at end of file diff --git a/modules/02_bwa_aln.nf b/modules/02_bwa_aln.nf index 7f9b4ad..c960ab2 100644 --- a/modules/02_bwa_aln.nf +++ b/modules/02_bwa_aln.nf @@ -12,13 +12,13 @@ process BWA_ALN { output: tuple val("${name}"), file("${fastq}"), file("${fastq.baseName}.sai"), emit: alignment_output - file("software_versions.${task.process}.txt") + file("software_versions.BWA_ALN.txt") """ bwa aln -t ${task.cpus} ${params.reference} ${fastq} > ${fastq.baseName}.sai - echo ${params.manifest} >> software_versions.${task.process}.txt - echo "bwa=0.7.17" >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.BWA_ALN.txt + echo "bwa=0.7.17" >> software_versions.BWA_ALN.txt """ } @@ -37,14 +37,14 @@ process BWA_SAMPE { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.BWA_SAMPE.txt") """ bwa sampe ${params.reference} ${sai1} ${sai2} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - echo "bwa=0.7.17" >> software_versions.${task.process}.txt - samtools --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.BWA_SAMPE.txt + echo "bwa=0.7.17" >> software_versions.BWA_SAMPE.txt + samtools --version >> software_versions.BWA_SAMPE.txt """ } @@ -63,13 +63,13 @@ process BWA_SAMSE { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.BWA_SAMSE.txt") """ bwa samse ${params.reference} ${sai} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - echo "bwa=0.7.17" >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.BWA_SAMSE.txt + echo "bwa=0.7.17" >> software_versions.BWA_SAMSE.txt """ } @@ -89,15 +89,15 @@ process BWA_ALN_INCEPTION { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.BWA_ALN_INCEPTION.txt") """ bwa sampe ${params.reference} <( bwa aln -t ${params.cpus} ${params.reference} ${fastq1} ) \ <( bwa aln -t ${params.cpus} ${params.reference} ${fastq2} ) ${fastq1} ${fastq2} \ | samtools view -uS - | samtools sort - > ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - echo "bwa=0.7.17" >> software_versions.${task.process}.txt - samtools --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.BWA_ALN_INCEPTION.txt + echo "bwa=0.7.17" >> software_versions.BWA_ALN_INCEPTION.txt + samtools --version >> software_versions.BWA_ALN_INCEPTION.txt """ } diff --git a/modules/02_bwa_mem.nf b/modules/02_bwa_mem.nf index 626a170..ac1b6b0 100644 --- a/modules/02_bwa_mem.nf +++ b/modules/02_bwa_mem.nf @@ -13,14 +13,14 @@ process BWA_MEM { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.BWA_MEM.txt") """ bwa mem ${params.additional_args} -t ${task.cpus} ${params.reference} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - echo "bwa=0.7.17" >> software_versions.${task.process}.txt - samtools --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.BWA_MEM.txt + echo "bwa=0.7.17" >> software_versions.BWA_MEM.txt + samtools --version >> software_versions.BWA_MEM.txt """ } @@ -39,13 +39,13 @@ process BWA_MEM_SE { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.BWA_MEM_SE.txt") """ bwa mem ${params.additional_args} -t ${task.cpus} ${params.reference} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - echo "bwa=0.7.17" >> software_versions.${task.process}.txt - samtools --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.BWA_MEM_SE.txt + echo "bwa=0.7.17" >> software_versions.BWA_MEM_SE.txt + samtools --version >> software_versions.BWA_MEM_SE.txt """ } diff --git a/modules/02_bwa_mem_2.nf b/modules/02_bwa_mem_2.nf index 58ad725..e61f606 100755 --- a/modules/02_bwa_mem_2.nf +++ b/modules/02_bwa_mem_2.nf @@ -13,14 +13,14 @@ process BWA_MEM_2 { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.BWA_MEM_2.txt") """ bwa-mem2 mem ${params.additional_args} -t ${task.cpus} ${params.reference} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - bwa-mem2 version >> software_versions.${task.process}.txt - samtools --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.BWA_MEM_2.txt + bwa-mem2 version >> software_versions.BWA_MEM_2.txt + samtools --version >> software_versions.BWA_MEM_2.txt """ } @@ -39,13 +39,13 @@ process BWA_MEM_2_SE { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.BWA_MEM_2_SE.txt") """ bwa-mem2 mem ${params.additional_args} -t ${task.cpus} ${params.reference} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - bwa-mem2 version >> software_versions.${task.process}.txt - samtools --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.BWA_MEM_2_SE.txt + bwa-mem2 version >> software_versions.BWA_MEM_2_SE.txt + samtools --version >> software_versions.BWA_MEM_2_SE.txt """ } diff --git a/modules/02_star.nf b/modules/02_star.nf index 66c1b95..948bb06 100644 --- a/modules/02_star.nf +++ b/modules/02_star.nf @@ -13,7 +13,7 @@ process STAR { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.STAR.txt") script: two_pass_mode_param = params.star_two_pass_mode ? "--twopassMode Basic" : "" @@ -32,8 +32,8 @@ process STAR { mv ${name}.Aligned.sortedByCoord.out.bam ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - STAR --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.STAR.txt + STAR --version >> software_versions.STAR.txt """ } @@ -51,7 +51,7 @@ process STAR_SE { output: tuple val("${name}"), file("${name}.bam"), emit: bams - file("software_versions.${task.process}.txt") + file("software_versions.STAR_SE.txt") script: two_pass_mode_param = params.star_two_pass_mode ? "--twopassMode Basic" : "" @@ -70,7 +70,7 @@ process STAR_SE { mv ${name}.Aligned.sortedByCoord.out.bam ${name}.bam - echo ${params.manifest} >> software_versions.${task.process}.txt - STAR --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.STAR_SE.txt + STAR --version >> software_versions.STAR_SE.txt """ } diff --git a/modules/03_index.nf b/modules/03_index.nf index a6fbcdf..7162aef 100644 --- a/modules/03_index.nf +++ b/modules/03_index.nf @@ -13,12 +13,12 @@ process INDEX_BAM { output: file("${name}.bam.bai") - file("software_versions.${task.process}.txt") + file("software_versions.INDEX_BAM.txt") """ samtools index -@ ${task.cpus} ${bam} - echo ${params.manifest} >> software_versions.${task.process}.txt - samtools --version >> software_versions.${task.process}.txt + echo ${params.manifest} >> software_versions.INDEX_BAM.txt + samtools --version >> software_versions.INDEX_BAM.txt """ } From 00ff6abc34787fba64f69355b0cc81461a9b9cbf Mon Sep 17 00:00:00 2001 From: Gianna Pohl Date: Tue, 6 Feb 2024 09:10:11 +0100 Subject: [PATCH 02/10] changed reference passing for easier submodule usage --- main.nf | 22 +++++++++++----------- modules/02_bwa_aln.nf | 14 +++++++++----- modules/02_bwa_mem.nf | 6 ++++-- modules/02_bwa_mem_2.nf | 6 ++++-- modules/02_star.nf | 5 +++-- 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/main.nf b/main.nf index 41aa6bb..d9cac7e 100755 --- a/main.nf +++ b/main.nf @@ -78,25 +78,25 @@ workflow { // alignment if (params.algorithm == "aln" && !params.inception) { - BWA_ALN(trimmed_fastqs.map {name, fq1, fq2 -> tuple(name, fq1)}) - BWA_ALN_2(trimmed_fastqs.map {name, fq1, fq2 -> tuple(name, fq2)}) - BWA_SAMPE(BWA_ALN.out.alignment_output.join(BWA_ALN_2.out.alignment_output)) + BWA_ALN(trimmed_fastqs.map {name, fq1, fq2 -> tuple(name, fq1)},params.reference) + BWA_ALN_2(trimmed_fastqs.map {name, fq1, fq2 -> tuple(name, fq2)},params.reference) + BWA_SAMPE(BWA_ALN.out.alignment_output.join(BWA_ALN_2.out.alignment_output),params.reference) output_bams = BWA_SAMPE.out.bams } else if (params.algorithm == "aln" && params.inception) { - BWA_ALN_INCEPTION(trimmed_fastqs) + BWA_ALN_INCEPTION(trimmed_fastqs,params.reference) output_bams = BWA_ALN_INCEPTION.out.bams } else if (params.algorithm == "mem") { - BWA_MEM(trimmed_fastqs) + BWA_MEM(trimmed_fastqs,params.reference) output_bams = BWA_MEM.out.bams } else if (params.algorithm == "mem2") { - BWA_MEM_2(trimmed_fastqs) + BWA_MEM_2(trimmed_fastqs,params.reference) output_bams = BWA_MEM_2.out.bams } else if (params.algorithm == "star") { - STAR(trimmed_fastqs) + STAR(trimmed_fastqs,params.reference) output_bams = STAR.out.bams } else { @@ -112,20 +112,20 @@ workflow { trimmed_fastqs = FASTP_SINGLE.out.trimmed_fastqs } if (params.algorithm == "aln" && !params.inception) { - BWA_ALN(trimmed_fastqs) + BWA_ALN(trimmed_fastqs,params.reference) BWA_SAMSE(BWA_ALN.out.alignment_output) output_bams = BWA_SAMSE.out.bams } else if (params.algorithm == "mem") { - BWA_MEM_SE(trimmed_fastqs) + BWA_MEM_SE(trimmed_fastqs,params.reference) output_bams = BWA_MEM_SE.out.bams } else if (params.algorithm == "mem2") { - BWA_MEM_2_SE(trimmed_fastqs) + BWA_MEM_2_SE(trimmed_fastqs,params.reference) output_bams = BWA_MEM_2_SE.out.bams } else if (params.algorithm == "star") { - STAR_SE(trimmed_fastqs) + STAR_SE(trimmed_fastqs,params.reference) output_bams = STAR_SE.out.bams } else { diff --git a/modules/02_bwa_aln.nf b/modules/02_bwa_aln.nf index c960ab2..c7381d8 100644 --- a/modules/02_bwa_aln.nf +++ b/modules/02_bwa_aln.nf @@ -9,13 +9,14 @@ process BWA_ALN { input: tuple val(name), file(fastq) + val(reference) output: tuple val("${name}"), file("${fastq}"), file("${fastq.baseName}.sai"), emit: alignment_output file("software_versions.BWA_ALN.txt") """ - bwa aln -t ${task.cpus} ${params.reference} ${fastq} > ${fastq.baseName}.sai + bwa aln -t ${task.cpus} ${reference} ${fastq} > ${fastq.baseName}.sai echo ${params.manifest} >> software_versions.BWA_ALN.txt echo "bwa=0.7.17" >> software_versions.BWA_ALN.txt @@ -34,13 +35,14 @@ process BWA_SAMPE { input: // joins both channels by key using the first element in the tuple, the name tuple val(name), file(fastq1), file(sai1), file(fastq2), file(sai2) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams file("software_versions.BWA_SAMPE.txt") """ - bwa sampe ${params.reference} ${sai1} ${sai2} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam + bwa sampe ${reference} ${sai1} ${sai2} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam echo ${params.manifest} >> software_versions.BWA_SAMPE.txt echo "bwa=0.7.17" >> software_versions.BWA_SAMPE.txt @@ -60,13 +62,14 @@ process BWA_SAMSE { input: // joins both channels by key using the first element in the tuple, the name tuple val(name), file(fastq), file(sai) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams file("software_versions.BWA_SAMSE.txt") """ - bwa samse ${params.reference} ${sai} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam + bwa samse ${reference} ${sai} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam echo ${params.manifest} >> software_versions.BWA_SAMSE.txt echo "bwa=0.7.17" >> software_versions.BWA_SAMSE.txt @@ -86,14 +89,15 @@ process BWA_ALN_INCEPTION { input: // joins both channels by key using the first element in the tuple, the name tuple val(name), file(fastq1), file(fastq2) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams file("software_versions.BWA_ALN_INCEPTION.txt") """ - bwa sampe ${params.reference} <( bwa aln -t ${params.cpus} ${params.reference} ${fastq1} ) \ - <( bwa aln -t ${params.cpus} ${params.reference} ${fastq2} ) ${fastq1} ${fastq2} \ + bwa sampe ${reference} <( bwa aln -t ${params.cpus} ${reference} ${fastq1} ) \ + <( bwa aln -t ${params.cpus} ${reference} ${fastq2} ) ${fastq1} ${fastq2} \ | samtools view -uS - | samtools sort - > ${name}.bam echo ${params.manifest} >> software_versions.BWA_ALN_INCEPTION.txt diff --git a/modules/02_bwa_mem.nf b/modules/02_bwa_mem.nf index ac1b6b0..93bf7ce 100644 --- a/modules/02_bwa_mem.nf +++ b/modules/02_bwa_mem.nf @@ -10,13 +10,14 @@ process BWA_MEM { input: tuple val(name), file(fastq1), file(fastq2) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams file("software_versions.BWA_MEM.txt") """ - bwa mem ${params.additional_args} -t ${task.cpus} ${params.reference} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam + bwa mem ${params.additional_args} -t ${task.cpus} ${reference} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam echo ${params.manifest} >> software_versions.BWA_MEM.txt echo "bwa=0.7.17" >> software_versions.BWA_MEM.txt @@ -36,13 +37,14 @@ process BWA_MEM_SE { input: // joins both channels by key using the first element in the tuple, the name tuple val(name), file(fastq) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams file("software_versions.BWA_MEM_SE.txt") """ - bwa mem ${params.additional_args} -t ${task.cpus} ${params.reference} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam + bwa mem ${params.additional_args} -t ${task.cpus} ${reference} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam echo ${params.manifest} >> software_versions.BWA_MEM_SE.txt echo "bwa=0.7.17" >> software_versions.BWA_MEM_SE.txt diff --git a/modules/02_bwa_mem_2.nf b/modules/02_bwa_mem_2.nf index e61f606..03a5034 100755 --- a/modules/02_bwa_mem_2.nf +++ b/modules/02_bwa_mem_2.nf @@ -10,13 +10,14 @@ process BWA_MEM_2 { input: tuple val(name), file(fastq1), file(fastq2) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams file("software_versions.BWA_MEM_2.txt") """ - bwa-mem2 mem ${params.additional_args} -t ${task.cpus} ${params.reference} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam + bwa-mem2 mem ${params.additional_args} -t ${task.cpus} ${reference} ${fastq1} ${fastq2} | samtools view -uS - | samtools sort - > ${name}.bam echo ${params.manifest} >> software_versions.BWA_MEM_2.txt bwa-mem2 version >> software_versions.BWA_MEM_2.txt @@ -36,13 +37,14 @@ process BWA_MEM_2_SE { input: // joins both channels by key using the first element in the tuple, the name tuple val(name), file(fastq) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams file("software_versions.BWA_MEM_2_SE.txt") """ - bwa-mem2 mem ${params.additional_args} -t ${task.cpus} ${params.reference} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam + bwa-mem2 mem ${params.additional_args} -t ${task.cpus} ${reference} ${fastq} | samtools view -uS - | samtools sort - > ${name}.bam echo ${params.manifest} >> software_versions.BWA_MEM_2_SE.txt bwa-mem2 version >> software_versions.BWA_MEM_2_SE.txt diff --git a/modules/02_star.nf b/modules/02_star.nf index 948bb06..bd2d3e8 100644 --- a/modules/02_star.nf +++ b/modules/02_star.nf @@ -18,7 +18,7 @@ process STAR { script: two_pass_mode_param = params.star_two_pass_mode ? "--twopassMode Basic" : "" """ - STAR --genomeDir ${params.reference} ${two_pass_mode_param} ${params.additional_args} \ + STAR --genomeDir ${reference} ${two_pass_mode_param} ${params.additional_args} \ --readFilesCommand "gzip -d -c -f" \ --readFilesIn ${fastq1} ${fastq2} \ --outSAMmode Full \ @@ -48,6 +48,7 @@ process STAR_SE { input: tuple val(name), file(fastq) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams @@ -56,7 +57,7 @@ process STAR_SE { script: two_pass_mode_param = params.star_two_pass_mode ? "--twopassMode Basic" : "" """ - STAR --genomeDir ${params.reference} ${two_pass_mode_param} ${params.additional_args} \ + STAR --genomeDir ${reference} ${two_pass_mode_param} ${params.additional_args} \ --readFilesCommand "gzip -d -c -f" \ --readFilesIn ${fastq} \ --outSAMmode Full \ From 6daa929286446f75ac1ebdf128deb641cea88f90 Mon Sep 17 00:00:00 2001 From: Gianna Pohl Date: Tue, 6 Feb 2024 09:30:25 +0100 Subject: [PATCH 03/10] Bugfix: added missing refrence parameter --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index d9cac7e..6f753b6 100755 --- a/main.nf +++ b/main.nf @@ -113,7 +113,7 @@ workflow { } if (params.algorithm == "aln" && !params.inception) { BWA_ALN(trimmed_fastqs,params.reference) - BWA_SAMSE(BWA_ALN.out.alignment_output) + BWA_SAMSE(BWA_ALN.out.alignment_output,params.reference) output_bams = BWA_SAMSE.out.bams } else if (params.algorithm == "mem") { From 848eda9061a0f81dcddd70a74159683f1cdfd56c Mon Sep 17 00:00:00 2001 From: Gianna Pohl Date: Tue, 6 Feb 2024 15:23:49 +0100 Subject: [PATCH 04/10] added mising input parameter for reference --- modules/02_star.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/02_star.nf b/modules/02_star.nf index bd2d3e8..f06d9b1 100644 --- a/modules/02_star.nf +++ b/modules/02_star.nf @@ -10,6 +10,7 @@ process STAR { input: tuple val(name), file(fastq1), file(fastq2) + val(reference) output: tuple val("${name}"), file("${name}.bam"), emit: bams From 173ff843534080870064d1847f156b51ad762a2e Mon Sep 17 00:00:00 2001 From: Pablo Riesgo-Ferreiro Date: Thu, 23 May 2024 13:30:10 +0000 Subject: [PATCH 05/10] add new conda.enable and useMamba --- nextflow.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nextflow.config b/nextflow.config index 0223a82..820a198 100644 --- a/nextflow.config +++ b/nextflow.config @@ -22,6 +22,8 @@ params.additional_args = "" profiles { conda { params.enable_conda = true + conda.enabled = true + conda.useMamba = true } debug { process.beforeScript = 'echo $HOSTNAME' } test { From 5538ef9fba3e90d4523dfa0f28df5032e22fbea6 Mon Sep 17 00:00:00 2001 From: Pablo Riesgo-Ferreiro Date: Thu, 23 May 2024 13:34:10 +0000 Subject: [PATCH 06/10] add CI profile --- nextflow.config | 8 +++++--- tests/run_test_1.sh | 2 +- tests/run_test_10.sh | 2 +- tests/run_test_11.sh | 2 +- tests/run_test_12.sh | 2 +- tests/run_test_13.sh | 2 +- tests/run_test_2.sh | 2 +- tests/run_test_3.sh | 2 +- tests/run_test_4.sh | 2 +- tests/run_test_5.sh | 2 +- tests/run_test_6.sh | 2 +- tests/run_test_7.sh | 2 +- tests/run_test_8.sh | 2 +- tests/run_test_9.sh | 2 +- 14 files changed, 18 insertions(+), 16 deletions(-) diff --git a/nextflow.config b/nextflow.config index 820a198..2a5e029 100644 --- a/nextflow.config +++ b/nextflow.config @@ -26,9 +26,7 @@ profiles { conda.useMamba = true } debug { process.beforeScript = 'echo $HOSTNAME' } - test { - params.input_files = "test_data/test_input_paired.txt" - params.reference = "$baseDir/test_data/ucsc.hg19.minimal.fasta" + ci { params.cpus = 1 params.memory = "3g" timeline.enabled = false @@ -36,6 +34,10 @@ profiles { trace.enabled = false dag.enabled = false } + test { + params.input_files = "test_data/test_input_paired.txt" + params.reference = "$baseDir/test_data/ucsc.hg19.minimal.fasta" + } } // Export this variable to prevent local Python libraries from conflicting with those in the container diff --git a/tests/run_test_1.sh b/tests/run_test_1.sh index 8105cef..eb110e0 100644 --- a/tests/run_test_1.sh +++ b/tests/run_test_1.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test1 -nextflow main.nf -profile test,conda --output $output_folder +nextflow main.nf -profile test,conda,ci --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 1 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 1 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 1 output file!"; exit 1; } diff --git a/tests/run_test_10.sh b/tests/run_test_10.sh index fcda739..9b343a4 100644 --- a/tests/run_test_10.sh +++ b/tests/run_test_10.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test10 -nextflow main.nf -profile test,conda --library single --algorithm mem2 --output $output_folder +nextflow main.nf -profile test,conda,ci --library single --algorithm mem2 --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 3 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 3 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 3 output file!"; exit 1; } diff --git a/tests/run_test_11.sh b/tests/run_test_11.sh index e176cb8..8b68dde 100644 --- a/tests/run_test_11.sh +++ b/tests/run_test_11.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test11 -nextflow main.nf -profile test,conda --algorithm star --reference `pwd`/test_data --output $output_folder +nextflow main.nf -profile test,conda,ci --algorithm star --reference `pwd`/test_data --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 4 output file!"; exit 1; } diff --git a/tests/run_test_12.sh b/tests/run_test_12.sh index cca6ace..b7144ec 100644 --- a/tests/run_test_12.sh +++ b/tests/run_test_12.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test12 -nextflow main.nf -profile test,conda --algorithm star --library single --reference `pwd`/test_data --output $output_folder +nextflow main.nf -profile test,conda,ci --algorithm star --library single --reference `pwd`/test_data --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 4 output file!"; exit 1; } diff --git a/tests/run_test_13.sh b/tests/run_test_13.sh index a59d46d..8e3a3c8 100644 --- a/tests/run_test_13.sh +++ b/tests/run_test_13.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test13 -nextflow main.nf -profile test,conda --algorithm star --star_two_pass_mode --reference `pwd`/test_data --output $output_folder +nextflow main.nf -profile test,conda,ci --algorithm star --star_two_pass_mode --reference `pwd`/test_data --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 4 output file!"; exit 1; } diff --git a/tests/run_test_2.sh b/tests/run_test_2.sh index c525376..01086b0 100644 --- a/tests/run_test_2.sh +++ b/tests/run_test_2.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test2 -nextflow main.nf -profile test,conda --inception --output $output_folder +nextflow main.nf -profile test,conda,ci --inception --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 2 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 2 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 2 output file!"; exit 1; } diff --git a/tests/run_test_3.sh b/tests/run_test_3.sh index 89bcc88..56be9c6 100644 --- a/tests/run_test_3.sh +++ b/tests/run_test_3.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test3 -nextflow main.nf -profile test,conda --library single --output $output_folder +nextflow main.nf -profile test,conda,ci --library single --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 3 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 3 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 3 output file!"; exit 1; } diff --git a/tests/run_test_4.sh b/tests/run_test_4.sh index 67313e7..eafa40d 100644 --- a/tests/run_test_4.sh +++ b/tests/run_test_4.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test4 -nextflow main.nf -profile test,conda --algorithm mem --output $output_folder +nextflow main.nf -profile test,conda,ci --algorithm mem --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 4 output file!"; exit 1; } diff --git a/tests/run_test_5.sh b/tests/run_test_5.sh index bceb660..2eb3680 100644 --- a/tests/run_test_5.sh +++ b/tests/run_test_5.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test5 -nextflow main.nf -profile test,conda --algorithm mem --library single --output $output_folder +nextflow main.nf -profile test,conda,ci --algorithm mem --library single --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 5 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 5 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 5 output file!"; exit 1; } diff --git a/tests/run_test_6.sh b/tests/run_test_6.sh index f278fcb..245a1a6 100644 --- a/tests/run_test_6.sh +++ b/tests/run_test_6.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test6 -nextflow main.nf -profile test,conda --output $output_folder --input_files false \ +nextflow main.nf -profile test,conda,ci --output $output_folder --input_files false \ --input_fastq1 test_data/TESTX_S1_L001_R1_001.fastq.gz \ --input_fastq2 test_data/TESTX_S1_L001_R2_001.fastq.gz --input_name test test -s $output_folder/test.bam || { echo "Missing test 6 output file!"; exit 1; } diff --git a/tests/run_test_7.sh b/tests/run_test_7.sh index ebcdeeb..0619a60 100644 --- a/tests/run_test_7.sh +++ b/tests/run_test_7.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test7 -nextflow main.nf -profile test,conda --output $output_folder --input_files false \ +nextflow main.nf -profile test,conda,ci --output $output_folder --input_files false \ --input_fastq1 test_data/TESTX_S1_L001_R1_001.fastq.gz \ --library single --input_name test test -s $output_folder/test.bam || { echo "Missing test 7 output file!"; exit 1; } diff --git a/tests/run_test_8.sh b/tests/run_test_8.sh index 8005d8f..f40d476 100644 --- a/tests/run_test_8.sh +++ b/tests/run_test_8.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test8 -nextflow main.nf -profile test,conda --algorithm mem --skip_trimming --output $output_folder +nextflow main.nf -profile test,conda,ci --algorithm mem --skip_trimming --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 8 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 8 output file!"; exit 1; } if [ -f $output_folder/TESTX_S1_L001.fastp_stats.html ]; then diff --git a/tests/run_test_9.sh b/tests/run_test_9.sh index 638f5f5..71a3cf0 100644 --- a/tests/run_test_9.sh +++ b/tests/run_test_9.sh @@ -1,7 +1,7 @@ #!/bin/bash output_folder=output/test9 -nextflow main.nf -profile test,conda --algorithm mem2 --output $output_folder +nextflow main.nf -profile test,conda,ci --algorithm mem2 --output $output_folder test -s $output_folder/TESTX_S1_L001.bam || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.bam.bai || { echo "Missing test 4 output file!"; exit 1; } test -s $output_folder/TESTX_S1_L001.fastp_stats.html || { echo "Missing test 4 output file!"; exit 1; } From e7599e9e31838acae6cd628ff0a61df2ecd5caeb Mon Sep 17 00:00:00 2001 From: Pablo Riesgo-Ferreiro Date: Thu, 23 May 2024 13:38:47 +0000 Subject: [PATCH 07/10] introduce mamba in CI --- .github/workflows/automated_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automated_tests.yml b/.github/workflows/automated_tests.yml index c253dfb..84de0d0 100644 --- a/.github/workflows/automated_tests.yml +++ b/.github/workflows/automated_tests.yml @@ -12,7 +12,9 @@ jobs: with: distribution: 'zulu' # See 'Supported distributions' for available options java-version: '11' - - uses: conda-incubator/setup-miniconda@v2 + - uses: mamba-org/provision-with-micromamba@v16 + with: + cache-env: true - name: Install dependencies run: | apt-get update && apt-get --assume-yes install wget make procps software-properties-common From 1bac1216cf4b9786f9df7d9226199b1a09f65602 Mon Sep 17 00:00:00 2001 From: Pablo Riesgo-Ferreiro Date: Thu, 23 May 2024 13:40:35 +0000 Subject: [PATCH 08/10] mamba on ci again --- .github/workflows/automated_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/automated_tests.yml b/.github/workflows/automated_tests.yml index 84de0d0..0b73c91 100644 --- a/.github/workflows/automated_tests.yml +++ b/.github/workflows/automated_tests.yml @@ -12,9 +12,9 @@ jobs: with: distribution: 'zulu' # See 'Supported distributions' for available options java-version: '11' - - uses: mamba-org/provision-with-micromamba@v16 + - uses: mamba-org/setup-micromamba@v1 with: - cache-env: true + cache-environment: true - name: Install dependencies run: | apt-get update && apt-get --assume-yes install wget make procps software-properties-common From b8868fcf0eabeeffc1d936dc9396987083a1ad59 Mon Sep 17 00:00:00 2001 From: Pablo Riesgo-Ferreiro Date: Thu, 23 May 2024 13:41:24 +0000 Subject: [PATCH 09/10] one more --- .github/workflows/automated_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/automated_tests.yml b/.github/workflows/automated_tests.yml index 0b73c91..e6e7150 100644 --- a/.github/workflows/automated_tests.yml +++ b/.github/workflows/automated_tests.yml @@ -13,8 +13,6 @@ jobs: distribution: 'zulu' # See 'Supported distributions' for available options java-version: '11' - uses: mamba-org/setup-micromamba@v1 - with: - cache-environment: true - name: Install dependencies run: | apt-get update && apt-get --assume-yes install wget make procps software-properties-common From a9e8dd67bbe6eb31f00c2380eb54518a227d429d Mon Sep 17 00:00:00 2001 From: Pablo Riesgo-Ferreiro Date: Thu, 23 May 2024 13:46:39 +0000 Subject: [PATCH 10/10] remove explicit support for mamba --- nextflow.config | 1 - 1 file changed, 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 2a5e029..d909fe8 100644 --- a/nextflow.config +++ b/nextflow.config @@ -23,7 +23,6 @@ profiles { conda { params.enable_conda = true conda.enabled = true - conda.useMamba = true } debug { process.beforeScript = 'echo $HOSTNAME' } ci {