From cb0e10f2a05767391eba84ab142b8e1ad139c463 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Mon, 18 Dec 2023 14:21:01 +0100 Subject: [PATCH 01/20] Add `.nf-test*` to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1eb2788..0699257 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ log reports .nf-test/ nf-test +.nf-test* test-datasets From a9d02149c398f0462c85b2f66beb697ed874575a Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Mon, 18 Dec 2023 14:21:23 +0100 Subject: [PATCH 02/20] Use compressed tarball for downstream test data Use a compressed tarball of the downstream (post-Space Ranger) test data as a workaround of not being able to stage a remote directory. --- conf/test.config | 6 ++-- subworkflows/local/input_check.nf | 53 +++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/conf/test.config b/conf/test.config index 0a91e5c..c2eec5c 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,9 +20,9 @@ params { max_time = '2.h' // Input and output - input = './test-datasets/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_spaceranger.csv' - spaceranger_probeset = "./test-datasets/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_probe_set.csv" - spaceranger_reference = "./test-datasets/testdata/homo_sapiens_chr22_reference.tar.gz" + input = 'https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_downstream.csv' + spaceranger_probeset = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/outs/probe_set.csv" + spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" st_preprocess_min_counts = 5 st_preprocess_min_genes = 3 outdir = 'results' diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf index c714514..6aa1545 100644 --- a/subworkflows/local/input_check.nf +++ b/subworkflows/local/input_check.nf @@ -2,27 +2,62 @@ // Check input samplesheet and get read channels // +include { UNTAR as UNTAR_COMPRESSED_INPUT } from "../../modules/nf-core/untar" + workflow INPUT_CHECK { take: samplesheet // file: /path/to/samplesheet.csv main: - ch_st = Channel.from(samplesheet).splitCsv( - header: true, - sep: ',' - ).branch { - spaceranger: !it.containsKey("spaceranger_dir") - downstream: it.containsKey("spaceranger_dir") + ch_st = Channel.from(samplesheet) + .splitCsv ( header: true, sep: ',') + .branch { + spaceranger: !it.containsKey("spaceranger_dir") + downstream: it.containsKey("spaceranger_dir") + } + + // Pre-Space Ranger analysis: create meta map and check input existance + ch_spaceranger_input = ch_st.spaceranger.map { create_channel_spaceranger(it) } + + // Post-Space Ranger analysis: + // Check if running the `test` profile, which uses a tarball of the testdata + if ( workflow.profile.contains('test') ) { + + // Untar Space Ranger output stored as tarball + ch_downstream_tar = ch_st.downstream.map { create_channel_downstream_tar(it) } + UNTAR_COMPRESSED_INPUT ( + ch_downstream_tar + ) + + // Create meta map corresponding to non-tarballed input + ch_downstream = UNTAR_COMPRESSED_INPUT.out.untar + .map { meta, dir -> [sample: meta.id, spaceranger_dir: dir] } + + } else { + + // Non-tarballed input data + ch_downstream = ch_st.downstream + } - ch_spaceranger_input = ch_st.spaceranger.map{create_channel_spaceranger(it)} - ch_downstream_input = ch_st.downstream.map{create_channel_downstream(it)} + + // Create meta map and check input existance + ch_downstream_input = ch_downstream.map { create_channel_downstream(it) } emit: ch_spaceranger_input // channel: [ val(meta), [ st data ] ] ch_downstream_input // channel: [ val(meta), [ st data ] ] } +// Function to get list of [ meta, [ spaceranger_dir ]] +def create_channel_downstream_tar(LinkedHashMap meta) { + meta['id'] = meta.remove('sample') + spaceranger_dir = meta.remove('spaceranger_dir') + return [meta, spaceranger_dir] +} + +// Function to get list of [ meta, [ raw_feature_bc_matrix, tissue_positions, +// scalefactors, hires_image, lowres_image ]] def create_channel_downstream(LinkedHashMap meta) { meta["id"] = meta.remove("sample") spaceranger_dir = file("${meta.remove('spaceranger_dir')}/**") @@ -34,7 +69,7 @@ def create_channel_downstream(LinkedHashMap meta) { return [meta, spaceranger_dir] } -// Function to get list of [ meta, [ fastq_dir, tissue_hires_image, slide, area ] +// Function to get list of [ meta, [ fastq_dir, tissue_hires_image, slide, area ]] def create_channel_spaceranger(LinkedHashMap meta) { meta["id"] = meta.remove("sample") From f567498b5d23e620b28535c3593180e57ac1b141 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Mon, 18 Dec 2023 15:19:32 +0100 Subject: [PATCH 03/20] Remove old `run_spaceranger` parameter mention --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index f1f86ea..dddcd0c 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -134,7 +134,7 @@ The typical command for running the pipeline is as follows: ```bash # Run the pipeline with raw data yet to be processed by Space Ranger -nextflow run nf-core/spatialtranscriptomics --input samplesheet.csv --outdir -profile docker --run_spaceranger +nextflow run nf-core/spatialtranscriptomics --input samplesheet.csv --outdir -profile docker # Run pipeline with data already processed by Space Ranger nextflow run nf-core/spatialtranscriptomics --input samplesheet.csv --outdir -profile docker From 3d08dd6e16384fce3e0a4c0d3bdad91a7f77c01e Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Mon, 18 Dec 2023 15:20:40 +0100 Subject: [PATCH 04/20] Clarify test profile description --- conf/test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/test.config b/conf/test.config index c2eec5c..29f95c6 100644 --- a/conf/test.config +++ b/conf/test.config @@ -12,7 +12,7 @@ params { config_profile_name = 'Test profile' - config_profile_description = 'Test pipeline incl. spaceranger with cytassist ffpe sample' + config_profile_description = 'Test pipeline for post-Space Ranger functionality' // Limit resources so that this can run on GitHub Actions max_cpus = 2 From 00185fbd9d2a06e033c99ecc1ed0098df77ad711 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Mon, 18 Dec 2023 16:02:48 +0100 Subject: [PATCH 05/20] Use remote files for `nf-test` downstream test --- tests/pipeline/test_downstream.nf.test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pipeline/test_downstream.nf.test b/tests/pipeline/test_downstream.nf.test index 8ed783a..9307b88 100644 --- a/tests/pipeline/test_downstream.nf.test +++ b/tests/pipeline/test_downstream.nf.test @@ -6,8 +6,9 @@ nextflow_pipeline { test("CytAssist_11mm_FFPE_Human_Glioblastoma_2") { when { params { - input = 'test-datasets/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_downstream.csv' - spaceranger_probeset = null + input = "https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_downstream.csv" + spaceranger_probeset = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/outs/probe_set.csv" + spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" outdir = "$outputDir" } } From 0e51ef30b503aa3c9ffc5c377c2c391711c09cf2 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Mon, 18 Dec 2023 17:09:20 +0100 Subject: [PATCH 06/20] Update nf-test snapshot --- tests/pipeline/test_downstream.nf.test.snap | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pipeline/test_downstream.nf.test.snap b/tests/pipeline/test_downstream.nf.test.snap index a2bee29..a74e549 100644 --- a/tests/pipeline/test_downstream.nf.test.snap +++ b/tests/pipeline/test_downstream.nf.test.snap @@ -8,8 +8,8 @@ }, "software_versions": { "content": [ - "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QC_AND_NORMALISATION={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" + "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.12.0, yaml=6.0.1}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QC_AND_NORMALISATION={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" ], - "timestamp": "2023-06-25T06:54:28+0000" + "timestamp": "2023-12-18T16:46:26.313038" } -} +} \ No newline at end of file From 152f0fd278d39a5ed6f5f1ea2a221839f004d6e3 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Mon, 18 Dec 2023 17:52:51 +0100 Subject: [PATCH 07/20] Use channel branching to check input type Use channel branching to check the input type (directory or tarball) of post-Space Ranger input instead of checking for the `test` profile. This is more generalisable and a better overall solution to the problem of different input types. --- subworkflows/local/input_check.nf | 35 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf index 6aa1545..006367f 100644 --- a/subworkflows/local/input_check.nf +++ b/subworkflows/local/input_check.nf @@ -21,28 +21,27 @@ workflow INPUT_CHECK { ch_spaceranger_input = ch_st.spaceranger.map { create_channel_spaceranger(it) } // Post-Space Ranger analysis: - // Check if running the `test` profile, which uses a tarball of the testdata - if ( workflow.profile.contains('test') ) { - // Untar Space Ranger output stored as tarball - ch_downstream_tar = ch_st.downstream.map { create_channel_downstream_tar(it) } - UNTAR_COMPRESSED_INPUT ( - ch_downstream_tar - ) - - // Create meta map corresponding to non-tarballed input - ch_downstream = UNTAR_COMPRESSED_INPUT.out.untar - .map { meta, dir -> [sample: meta.id, spaceranger_dir: dir] } - - } else { + // Split channel into tarballed and directory inputs + ch_downstream = ch_st.downstream + .map { create_channel_downstream_tar(it) } + .branch { + tar: it[1].contains(".tar.gz") + dir: !it[1].contains(".tar.gz") + } - // Non-tarballed input data - ch_downstream = ch_st.downstream + // Extract tarballed inputs + UNTAR_COMPRESSED_INPUT ( + ch_downstream.tar + ) - } + // Combine extracted and directory inputs into one channel + ch_downstream_combined = UNTAR_COMPRESSED_INPUT.out.untar + .mix ( ch_downstream.dir ) + .map { meta, dir -> [sample: meta.id, spaceranger_dir: dir] } - // Create meta map and check input existance - ch_downstream_input = ch_downstream.map { create_channel_downstream(it) } + // Create final meta map and check input file existance + ch_downstream_input = ch_downstream_combined.map { create_channel_downstream(it) } emit: ch_spaceranger_input // channel: [ val(meta), [ st data ] ] From f248416782d9fcf90138fc9c17ba447fd45e8ddf Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Tue, 19 Dec 2023 16:07:38 +0100 Subject: [PATCH 08/20] Add info on tarball input in usage docs --- docs/usage.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/usage.md b/docs/usage.md index dddcd0c..2837793 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -85,6 +85,14 @@ SAMPLE_2,results/SAMPLE_2/outs | `sample` | Unique sample identifier. | | `spaceranger_dir` | Output directory generated by spaceranger. This is typically called `outs` and contains both gene expression matrices and spatial information | +You may alternatively supply a compressed tarball containing the Space Ranger output: + +```no-highlight +sample,spaceranger_dir +SAMPLE_1,outs.tar.gz +SAMPLE_2,outs.tar.gz +``` + ## Space Ranger The pipeline exposes several of Space Ranger's parameters when executing with From 2b357640afc6582a058de01d590700f52c7dc75a Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Tue, 19 Dec 2023 17:48:37 +0100 Subject: [PATCH 09/20] Use compressed tarball of Space Ranger test data --- subworkflows/local/input_check.nf | 33 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf index 006367f..8d9630d 100644 --- a/subworkflows/local/input_check.nf +++ b/subworkflows/local/input_check.nf @@ -2,7 +2,8 @@ // Check input samplesheet and get read channels // -include { UNTAR as UNTAR_COMPRESSED_INPUT } from "../../modules/nf-core/untar" +include { UNTAR as UNTAR_SPACERANGER_INPUT } from "../../modules/nf-core/untar" +include { UNTAR as UNTAR_DOWNSTREAM_INPUT } from "../../modules/nf-core/untar" workflow INPUT_CHECK { @@ -17,10 +18,28 @@ workflow INPUT_CHECK { downstream: it.containsKey("spaceranger_dir") } - // Pre-Space Ranger analysis: create meta map and check input existance - ch_spaceranger_input = ch_st.spaceranger.map { create_channel_spaceranger(it) } + // Space Ranger analysis: -------------------------------------------------- - // Post-Space Ranger analysis: + // Split channel into tarballed and directory inputs + ch_spaceranger = ch_st.spaceranger + .map { it -> [it, it.fastq_dir]} + .branch { + tar: it[1].contains(".tar.gz") + dir: !it[1].contains(".tar.gz") + } + + // Extract tarballed inputs + UNTAR_SPACERANGER_INPUT ( ch_spaceranger.tar ) + + // Combine extracted and directory inputs into one channel + ch_spaceranger_combined = UNTAR_SPACERANGER_INPUT.out.untar + .mix ( ch_spaceranger.dir ) + .map { meta, dir -> meta + [fastq_dir: dir] } + + // Create final meta map and check input existance + ch_spaceranger_input = ch_spaceranger_combined.map { create_channel_spaceranger(it) } + + // Downstream analysis: ---------------------------------------------------- // Split channel into tarballed and directory inputs ch_downstream = ch_st.downstream @@ -31,12 +50,10 @@ workflow INPUT_CHECK { } // Extract tarballed inputs - UNTAR_COMPRESSED_INPUT ( - ch_downstream.tar - ) + UNTAR_DOWNSTREAM_INPUT ( ch_downstream.tar ) // Combine extracted and directory inputs into one channel - ch_downstream_combined = UNTAR_COMPRESSED_INPUT.out.untar + ch_downstream_combined = UNTAR_DOWNSTREAM_INPUT.out.untar .mix ( ch_downstream.dir ) .map { meta, dir -> [sample: meta.id, spaceranger_dir: dir] } From 5ae23b027462ce119c80854f6180f39b99681ec8 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Tue, 19 Dec 2023 18:04:55 +0100 Subject: [PATCH 10/20] Add info on Space Ranger tarball input in docs --- docs/usage.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 2837793..8010490 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -29,6 +29,15 @@ SAMPLE_1,fastqs_1/,hires_1.png,V11J26,B1 SAMPLE_2,fastqs_2/,hires_2.png,V11J26,B1 ``` +You may also supply a compressed tarball containing the FASTQ files in lieu of a +directory path: + +```no-highlight +sample,fastq_dir,image,slide,area +SAMPLE_1,fastqs_1.tar.gz,hires_1.png,V11J26,B1 +SAMPLE_2,fastqs_2.tar.gz,hires_2.png,V11J26,B1 +``` + For Cytassist samples, the `image` column gets replaced with the `cytaimage` column: ```no-highlight @@ -45,7 +54,7 @@ Please refer to the following table for an overview of all supported columns: | Column | Description | | ------------------ | ------------------------------------------------------------------------------------------------------------------- | | `sample` | Unique sample identifier. MUST match the prefix of the fastq files | -| `fastq_dir` | Path to directory where the sample FASTQ files are stored. | +| `fastq_dir` | Path to directory where the sample FASTQ files are stored. May be a `.tar.gz` file instead of a directory. | | `image` | Brightfield microscopy image | | `cytaimage` | Brightfield tissue image captured with Cytassist device | | `colorizedimage` | A color composite of one or more fluorescence image channels saved as a single-page, single-file color TIFF or JPEG | @@ -80,11 +89,6 @@ SAMPLE_1,results/SAMPLE_1/outs SAMPLE_2,results/SAMPLE_2/outs ``` -| Column | Description | -| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| `sample` | Unique sample identifier. | -| `spaceranger_dir` | Output directory generated by spaceranger. This is typically called `outs` and contains both gene expression matrices and spatial information | - You may alternatively supply a compressed tarball containing the Space Ranger output: ```no-highlight @@ -93,6 +97,14 @@ SAMPLE_1,outs.tar.gz SAMPLE_2,outs.tar.gz ``` +| Column | Description | +| ----------------- | ----------------------------------------------------------------------------------------- | +| `sample` | Unique sample identifier. | +| `spaceranger_dir` | Output directory generated by spaceranger. May be a `.tar.gz` file instead of a directory | + +The Space Ranger output directory is typically called `outs` and contains both +gene expression matrices as well as spatial information. + ## Space Ranger The pipeline exposes several of Space Ranger's parameters when executing with From 1c9ad186ac84cf82735a1e9eacaa0bd67c113b09 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Tue, 19 Dec 2023 18:29:43 +0100 Subject: [PATCH 11/20] Update downstream `nf-test` tests --- tests/pipeline/test_downstream.nf.test | 20 +++++++++++--------- tests/pipeline/test_downstream.nf.test.snap | 7 ------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/tests/pipeline/test_downstream.nf.test b/tests/pipeline/test_downstream.nf.test index 9307b88..07dec42 100644 --- a/tests/pipeline/test_downstream.nf.test +++ b/tests/pipeline/test_downstream.nf.test @@ -6,29 +6,31 @@ nextflow_pipeline { test("CytAssist_11mm_FFPE_Human_Glioblastoma_2") { when { params { - input = "https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_downstream.csv" - spaceranger_probeset = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/outs/probe_set.csv" - spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" outdir = "$outputDir" } } then { assertAll( + // Workflow { assert workflow.success }, { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") }, - // data + + // Data { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_processed.h5ad").exists() }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_norm.h5ad").exists() }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_plain.h5ad").exists() }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_raw.h5ad").exists() }, - // reports - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_clustering.html").text.contains("Saving anndata file for future use") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_qc_and_normalisation.html").text.contains("Distribution after filtering") }, + + // Reports + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_qc_and_normalisation.html").text.contains("Distributions after filtering") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_clustering.html").text.contains("AnnData object is saved in the") }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_spatial_de.html").text.contains("Spatially variable genes") }, - // degs + + // DEGs { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/degs/st_spatial_de.csv").exists() }, - // multiqc + + // MultiQC { assert file("$outputDir/multiqc/multiqc_report.html").exists() } ) } diff --git a/tests/pipeline/test_downstream.nf.test.snap b/tests/pipeline/test_downstream.nf.test.snap index a74e549..58aa1a3 100644 --- a/tests/pipeline/test_downstream.nf.test.snap +++ b/tests/pipeline/test_downstream.nf.test.snap @@ -1,11 +1,4 @@ { - "reports": { - "content": [ - "st_clustering.html:md5,b52e0fd3843c595634f1f2c6dbf76a3a", - "st_qc_and_normalisation.html:md5,82746189bdc1bef051724cd843fe8b20" - ], - "timestamp": "2023-06-25T06:54:28+0000" - }, "software_versions": { "content": [ "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.12.0, yaml=6.0.1}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QC_AND_NORMALISATION={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" From a7dc21603e119f3f505155c141bc2bba4768e109 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 19 Dec 2023 18:36:51 +0000 Subject: [PATCH 12/20] Template update for nf-core/tools version 2.11 --- .github/CONTRIBUTING.md | 3 + .github/PULL_REQUEST_TEMPLATE.md | 1 + .github/workflows/ci.yml | 2 +- .github/workflows/fix-linting.yml | 4 +- .github/workflows/linting.yml | 12 +- .gitpod.yml | 4 +- README.md | 17 +-- assets/multiqc_config.yml | 2 +- assets/slackreport.json | 2 +- conf/modules.config | 2 +- docs/usage.md | 4 +- lib/NfcoreTemplate.groovy | 32 ++--- modules.json | 6 +- .../dumpsoftwareversions/environment.yml | 7 ++ .../custom/dumpsoftwareversions/main.nf | 6 +- .../custom/dumpsoftwareversions/meta.yml | 7 +- .../dumpsoftwareversions/tests/main.nf.test | 38 ++++++ .../tests/main.nf.test.snap | 27 +++++ .../dumpsoftwareversions/tests/tags.yml | 2 + modules/nf-core/fastqc/environment.yml | 7 ++ modules/nf-core/fastqc/main.nf | 10 +- modules/nf-core/fastqc/meta.yml | 5 + modules/nf-core/fastqc/tests/main.nf.test | 109 ++++++++++++++++++ .../nf-core/fastqc/tests/main.nf.test.snap | 10 ++ modules/nf-core/fastqc/tests/tags.yml | 2 + modules/nf-core/multiqc/environment.yml | 7 ++ modules/nf-core/multiqc/main.nf | 8 +- modules/nf-core/multiqc/meta.yml | 11 +- modules/nf-core/multiqc/tests/main.nf.test | 63 ++++++++++ modules/nf-core/multiqc/tests/tags.yml | 2 + nextflow.config | 10 +- 31 files changed, 358 insertions(+), 64 deletions(-) create mode 100644 modules/nf-core/custom/dumpsoftwareversions/environment.yml create mode 100644 modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test create mode 100644 modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap create mode 100644 modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml create mode 100644 modules/nf-core/fastqc/environment.yml create mode 100644 modules/nf-core/fastqc/tests/main.nf.test create mode 100644 modules/nf-core/fastqc/tests/main.nf.test.snap create mode 100644 modules/nf-core/fastqc/tests/tags.yml create mode 100644 modules/nf-core/multiqc/environment.yml create mode 100644 modules/nf-core/multiqc/tests/main.nf.test create mode 100644 modules/nf-core/multiqc/tests/tags.yml diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index c178a80..6ef91e2 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -27,6 +27,9 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests +You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to +receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir `. + When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 35bc683..1213782 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -19,6 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/spat - [ ] If necessary, also make a PR on the nf-core/spatialtranscriptomics _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7df75ba..396fe22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 8b6e6d9..10e0494 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -24,7 +24,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier @prettier/plugin-php diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index b8bdd21..905c58e 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -14,9 +14,9 @@ jobs: EditorConfig: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -27,9 +27,9 @@ jobs: Prettier: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier @@ -40,7 +40,7 @@ jobs: PythonBlack: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check code lints with Black uses: psf/black@stable @@ -71,7 +71,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/.gitpod.yml b/.gitpod.yml index 25488dc..acf7269 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -4,7 +4,9 @@ tasks: command: | pre-commit install --install-hooks nextflow self-update - + - name: unset JAVA_TOOL_OPTIONS + command: | + unset JAVA_TOOL_OPTIONS vscode: extensions: # based on nf-core.nf-core-extensionpack - codezombiech.gitignore # Language support for .gitignore files diff --git a/README.md b/README.md index 761ec36..aa354fd 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,8 @@ ## Usage -:::note -If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how -to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) -with `-profile test` before running the workflow on actual data. -::: +> [!NOTE] +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data. - + diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index d50210c..c73cf3b 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,5 +1,5 @@ report_comment: > - This report has been generated by the nf-core/spatialtranscriptomics + This report has been generated by the nf-core/spatialtranscriptomics analysis pipeline. For information about how to interpret these results, please see the documentation. report_section_order: diff --git a/assets/slackreport.json b/assets/slackreport.json index d9f52a9..4bd4bfb 100644 --- a/assets/slackreport.json +++ b/assets/slackreport.json @@ -3,7 +3,7 @@ { "fallback": "Plain-text summary of the attachment.", "color": "<% if (success) { %>good<% } else { %>danger<%} %>", - "author_name": "nf-core/spatialtranscriptomics v${version} - ${runName}", + "author_name": "nf-core/spatialtranscriptomics ${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ diff --git a/conf/modules.config b/conf/modules.config index 39e8138..d91c6ab 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -39,7 +39,7 @@ process { } withName: 'MULTIQC' { - ext.args = params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' + ext.args = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' } publishDir = [ path: { "${params.outdir}/multiqc" }, mode: params.publish_dir_mode, diff --git a/docs/usage.md b/docs/usage.md index 17dc5c6..4999b8b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -20,7 +20,7 @@ You will need to create a samplesheet with information about the samples you wou The `sample` identifiers have to be the same when you have re-sequenced the same sample more than once e.g. to increase sequencing depth. The pipeline will concatenate the raw reads before performing any downstream analysis. Below is an example for the same sample sequenced across 3 lanes: -```console +```csv title="samplesheet.csv" sample,fastq_1,fastq_2 CONTROL_REP1,AEG588A1_S1_L002_R1_001.fastq.gz,AEG588A1_S1_L002_R2_001.fastq.gz CONTROL_REP1,AEG588A1_S1_L003_R1_001.fastq.gz,AEG588A1_S1_L003_R2_001.fastq.gz @@ -33,7 +33,7 @@ The pipeline will auto-detect whether a sample is single- or paired-end using th A final samplesheet file consisting of both single- and paired-end data may look something like the one below. This is for 6 samples, where `TREATMENT_REP3` has been sequenced twice. -```console +```csv title="samplesheet.csv" sample,fastq_1,fastq_2 CONTROL_REP1,AEG588A1_S1_L002_R1_001.fastq.gz,AEG588A1_S1_L002_R2_001.fastq.gz CONTROL_REP2,AEG588A2_S2_L002_R1_001.fastq.gz,AEG588A2_S2_L002_R2_001.fastq.gz diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy index 01b8653..e248e4c 100755 --- a/lib/NfcoreTemplate.groovy +++ b/lib/NfcoreTemplate.groovy @@ -4,6 +4,7 @@ import org.yaml.snakeyaml.Yaml import groovy.json.JsonOutput +import nextflow.extension.FilesEx class NfcoreTemplate { @@ -141,12 +142,14 @@ class NfcoreTemplate { try { if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } [ 'sendmail', '-t' ].execute() << sendmail_html log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" } catch (all) { // Catch failures and try with plaintext def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { + if ( mqc_report != null && mqc_report.size() <= max_multiqc_email_size.toBytes() ) { mail_cmd += [ '-A', mqc_report ] } mail_cmd.execute() << email_html @@ -155,14 +158,16 @@ class NfcoreTemplate { } // Write summary e-mail HTML to a file - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def output_hf = new File(output_d, "pipeline_report.html") + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - def output_tf = new File(output_d, "pipeline_report.txt") + FilesEx.copyTo(output_hf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.html"); + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } + FilesEx.copyTo(output_tf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.txt"); + output_tf.delete() } // @@ -227,15 +232,14 @@ class NfcoreTemplate { // Dump pipeline parameters in a json file // public static void dump_parameters(workflow, params) { - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def output_pf = new File(output_d, "params_${timestamp}.json") + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") def jsonStr = JsonOutput.toJson(params) - output_pf.text = JsonOutput.prettyPrint(jsonStr) + temp_pf.text = JsonOutput.prettyPrint(jsonStr) + + FilesEx.copyTo(temp_pf.toPath(), "${params.outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() } // diff --git a/modules.json b/modules.json index 3a83ab6..37651f9 100644 --- a/modules.json +++ b/modules.json @@ -7,17 +7,17 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "bba7e362e4afead70653f84d8700588ea28d0f9e", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "bd8092b67b5103bdd52e300f75889442275c3117", + "git_sha": "65ad3e0b9a4099592e1102e92e10455dc661cf53", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "4ab13872435962dadc239979554d13709e20bf29", "installed_by": ["modules"] } } diff --git a/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/modules/nf-core/custom/dumpsoftwareversions/environment.yml new file mode 100644 index 0000000..f0c63f6 --- /dev/null +++ b/modules/nf-core/custom/dumpsoftwareversions/environment.yml @@ -0,0 +1,7 @@ +name: custom_dumpsoftwareversions +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::multiqc=1.17 diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index ebc8727..7685b33 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.14" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.17--pyhdfd78af_0' : + 'biocontainers/multiqc:1.17--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml index c32657d..5f15a5f 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: @@ -16,7 +16,6 @@ input: type: file description: YML file containing software versions pattern: "*.yml" - output: - yml: type: file @@ -30,7 +29,9 @@ output: type: file description: File containing software versions pattern: "versions.yml" - authors: - "@drpatelh" - "@grst" +maintainers: + - "@drpatelh" + - "@grst" diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test new file mode 100644 index 0000000..eec1db1 --- /dev/null +++ b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test @@ -0,0 +1,38 @@ +nextflow_process { + + name "Test Process CUSTOM_DUMPSOFTWAREVERSIONS" + script "../main.nf" + process "CUSTOM_DUMPSOFTWAREVERSIONS" + tag "modules" + tag "modules_nfcore" + tag "custom" + tag "dumpsoftwareversions" + tag "custom/dumpsoftwareversions" + + test("Should run without failures") { + when { + process { + """ + def tool1_version = ''' + TOOL1: + tool1: 0.11.9 + '''.stripIndent() + + def tool2_version = ''' + TOOL2: + tool2: 1.9 + '''.stripIndent() + + input[0] = Channel.of(tool1_version, tool2_version).collectFile() + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap new file mode 100644 index 0000000..4274ed5 --- /dev/null +++ b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap @@ -0,0 +1,27 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + "software_versions.yml:md5,1c851188476409cda5752ce971b20b58" + ], + "1": [ + "software_versions_mqc.yml:md5,2570f4ba271ad08357b0d3d32a9cf84d" + ], + "2": [ + "versions.yml:md5,3843ac526e762117eedf8825b40683df" + ], + "mqc_yml": [ + "software_versions_mqc.yml:md5,2570f4ba271ad08357b0d3d32a9cf84d" + ], + "versions": [ + "versions.yml:md5,3843ac526e762117eedf8825b40683df" + ], + "yml": [ + "software_versions.yml:md5,1c851188476409cda5752ce971b20b58" + ] + } + ], + "timestamp": "2023-11-03T14:43:22.157011" + } +} diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml b/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml new file mode 100644 index 0000000..405aa24 --- /dev/null +++ b/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml @@ -0,0 +1,2 @@ +custom/dumpsoftwareversions: + - modules/nf-core/custom/dumpsoftwareversions/** diff --git a/modules/nf-core/fastqc/environment.yml b/modules/nf-core/fastqc/environment.yml new file mode 100644 index 0000000..1787b38 --- /dev/null +++ b/modules/nf-core/fastqc/environment.yml @@ -0,0 +1,7 @@ +name: fastqc +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::fastqc=0.12.1 diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 249f906..9e19a74 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -2,10 +2,10 @@ process FASTQC { tag "$meta.id" label 'process_medium' - conda "bioconda::fastqc=0.11.9" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : - 'biocontainers/fastqc:0.11.9--0' }" + 'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' : + 'biocontainers/fastqc:0.12.1--hdfd78af_0' }" input: tuple val(meta), path(reads) @@ -37,7 +37,7 @@ process FASTQC { cat <<-END_VERSIONS > versions.yml "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + fastqc: \$( fastqc --version | sed '/FastQC v/!d; s/.*v//' ) END_VERSIONS """ @@ -49,7 +49,7 @@ process FASTQC { cat <<-END_VERSIONS > versions.yml "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + fastqc: \$( fastqc --version | sed '/FastQC v/!d; s/.*v//' ) END_VERSIONS """ } diff --git a/modules/nf-core/fastqc/meta.yml b/modules/nf-core/fastqc/meta.yml index 4da5bb5..ee5507e 100644 --- a/modules/nf-core/fastqc/meta.yml +++ b/modules/nf-core/fastqc/meta.yml @@ -50,3 +50,8 @@ authors: - "@grst" - "@ewels" - "@FelixKrueger" +maintainers: + - "@drpatelh" + - "@grst" + - "@ewels" + - "@FelixKrueger" diff --git a/modules/nf-core/fastqc/tests/main.nf.test b/modules/nf-core/fastqc/tests/main.nf.test new file mode 100644 index 0000000..b9e8f92 --- /dev/null +++ b/modules/nf-core/fastqc/tests/main.nf.test @@ -0,0 +1,109 @@ +nextflow_process { + + name "Test Process FASTQC" + script "../main.nf" + process "FASTQC" + tag "modules" + tag "modules_nfcore" + tag "fastqc" + + test("Single-Read") { + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = [ + [ id: 'test', single_end:true ], + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. + // looks like this:
Mon 2 Oct 2023
test.gz
+ // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 + { assert process.out.html.get(0).get(1) ==~ ".*/test_fastqc.html" }, + { assert path(process.out.html.get(0).get(1)).getText().contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match("versions") }, + { assert process.out.zip.get(0).get(1) ==~ ".*/test_fastqc.zip" } + ) + } + } +// TODO +// // +// // Test with paired-end data +// // +// workflow test_fastqc_paired_end { +// input = [ +// [id: 'test', single_end: false], // meta map +// [ +// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), +// file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) +// ] +// ] + +// FASTQC ( input ) +// } + +// // +// // Test with interleaved data +// // +// workflow test_fastqc_interleaved { +// input = [ +// [id: 'test', single_end: false], // meta map +// file(params.test_data['sarscov2']['illumina']['test_interleaved_fastq_gz'], checkIfExists: true) +// ] + +// FASTQC ( input ) +// } + +// // +// // Test with bam data +// // +// workflow test_fastqc_bam { +// input = [ +// [id: 'test', single_end: false], // meta map +// file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) +// ] + +// FASTQC ( input ) +// } + +// // +// // Test with multiple samples +// // +// workflow test_fastqc_multiple { +// input = [ +// [id: 'test', single_end: false], // meta map +// [ +// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), +// file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true), +// file(params.test_data['sarscov2']['illumina']['test2_1_fastq_gz'], checkIfExists: true), +// file(params.test_data['sarscov2']['illumina']['test2_2_fastq_gz'], checkIfExists: true) +// ] +// ] + +// FASTQC ( input ) +// } + +// // +// // Test with custom prefix +// // +// workflow test_fastqc_custom_prefix { +// input = [ +// [ id:'mysample', single_end:true ], // meta map +// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) +// ] + +// FASTQC ( input ) +// } +} diff --git a/modules/nf-core/fastqc/tests/main.nf.test.snap b/modules/nf-core/fastqc/tests/main.nf.test.snap new file mode 100644 index 0000000..636a32c --- /dev/null +++ b/modules/nf-core/fastqc/tests/main.nf.test.snap @@ -0,0 +1,10 @@ +{ + "versions": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "timestamp": "2023-10-09T23:40:54+0000" + } +} \ No newline at end of file diff --git a/modules/nf-core/fastqc/tests/tags.yml b/modules/nf-core/fastqc/tests/tags.yml new file mode 100644 index 0000000..7834294 --- /dev/null +++ b/modules/nf-core/fastqc/tests/tags.yml @@ -0,0 +1,2 @@ +fastqc: + - modules/nf-core/fastqc/** diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml new file mode 100644 index 0000000..bc0bdb5 --- /dev/null +++ b/modules/nf-core/multiqc/environment.yml @@ -0,0 +1,7 @@ +name: multiqc +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::multiqc=1.18 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 1fc387b..00cc48d 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.14" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.18--pyhdfd78af_0' : + 'biocontainers/multiqc:1.18--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" @@ -25,12 +25,14 @@ process MULTIQC { def args = task.ext.args ?: '' def config = multiqc_config ? "--config $multiqc_config" : '' def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' + def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' """ multiqc \\ --force \\ $args \\ $config \\ $extra_config \\ + $logo \\ . cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index f93b5ee..f1aa660 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -1,5 +1,5 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json -name: MultiQC +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: multiqc description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: - QC @@ -13,7 +13,6 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] - input: - multiqc_files: type: file @@ -31,7 +30,6 @@ input: type: file description: Optional logo file for MultiQC pattern: "*.{png}" - output: - report: type: file @@ -54,3 +52,8 @@ authors: - "@bunop" - "@drpatelh" - "@jfy133" +maintainers: + - "@abhi18av" + - "@bunop" + - "@drpatelh" + - "@jfy133" diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test new file mode 100644 index 0000000..c2dad21 --- /dev/null +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -0,0 +1,63 @@ +nextflow_process { + + name "Test Process MULTIQC" + script "../main.nf" + process "MULTIQC" + tag "modules" + tag "modules_nfcore" + tag "multiqc" + + test("MULTIQC: FASTQC") { + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[1] = [] + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert path(process.out.report.get(0)).exists() }, + { assert path(process.out.data.get(0)).exists() }, + { assert path(process.out.versions.get(0)).getText().contains("multiqc") } + ) + } + + } + + test("MULTIQC: FASTQC and a config file") { + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert path(process.out.report.get(0)).exists() }, + { assert path(process.out.data.get(0)).exists() }, + { assert path(process.out.versions.get(0)).getText().contains("multiqc") } + ) + } + + } +} diff --git a/modules/nf-core/multiqc/tests/tags.yml b/modules/nf-core/multiqc/tests/tags.yml new file mode 100644 index 0000000..bea6c0d --- /dev/null +++ b/modules/nf-core/multiqc/tests/tags.yml @@ -0,0 +1,2 @@ +multiqc: + - modules/nf-core/multiqc/** diff --git a/nextflow.config b/nextflow.config index d732a8b..daf7f50 100644 --- a/nextflow.config +++ b/nextflow.config @@ -14,7 +14,7 @@ params { input = null // References genome = null - igenomes_base = 's3://ngi-igenomes/igenomes' + igenomes_base = 's3://ngi-igenomes/igenomes/' igenomes_ignore = false @@ -82,6 +82,7 @@ profiles { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' cleanup = false + nextflow.enable.configProcessNamesValidation = true } conda { conda.enabled = true @@ -104,13 +105,13 @@ profiles { } docker { docker.enabled = true - docker.userEmulation = true conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + runOptions = '-u $(id -u):$(id -g)' } arm { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' @@ -181,7 +182,7 @@ singularity.registry = 'quay.io' // Nextflow plugins plugins { - id 'nf-validation' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet } // Load igenomes.config if required @@ -204,6 +205,9 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Disable process selector warnings by default. Use debug profile to enable warnings. +nextflow.enable.configProcessNamesValidation = false + def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true From fabdc22bc142ee56c2d4bc62a15cf4e2a3c67e27 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Wed, 20 Dec 2023 09:50:44 +0100 Subject: [PATCH 13/20] Update Space Ranger `nf-test` pipelines --- .../pipeline/test_spaceranger_ffpe_v1.nf.test | 25 +++++++++++------ ...test_spaceranger_ffpe_v2_cytassist.nf.test | 28 +++++++++++++------ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test index 7f03e1a..2d43b9c 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test @@ -6,8 +6,8 @@ nextflow_pipeline { test("spaceranger ffpe v1") { when { params { - input = 'test-datasets/testdata/human-ovarian-cancer-1-standard_v1_ffpe/samplesheet_spaceranger.csv' - spaceranger_probeset = 'test-datasets/testdata/human-ovarian-cancer-1-standard_v1_ffpe/Visium_Human_Transcriptome_Probe_Set_v1.0_GRCh38-2020-A.csv' + input = 'https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/samplesheet_spaceranger.csv' + spaceranger_probeset = 'https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/Visium_Human_Transcriptome_Probe_Set_v1.0_GRCh38-2020-A.csv' st_preprocess_min_counts = 5 st_preprocess_min_genes = 3 outdir = "$outputDir" @@ -16,20 +16,26 @@ nextflow_pipeline { then { assertAll( + + // Workflow { assert workflow.success }, { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") }, - // data + + // Data { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/data/st_adata_processed.h5ad").exists() }, { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/data/st_adata_norm.h5ad").exists() }, { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/data/st_adata_plain.h5ad").exists() }, { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/data/st_adata_raw.h5ad").exists() }, - // reports - { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/st_clustering.html").text.contains("Saving anndata file for future use") }, - { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/st_qc_and_normalisation.html").text.contains("Distribution after filtering") }, + + // Reports + { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/st_qc_and_normalisation.html").text.contains("Distributions after filtering") }, + { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/st_clustering.html").text.contains("AnnData object is saved in the") }, { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/st_spatial_de.html").text.contains("Spatially variable genes") }, - // degs + + // DEGs { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/degs/st_spatial_de.csv").exists() }, - // spaceranger + + // Space Ranger { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/outs/web_summary.html").exists() }, { assert snapshot( path("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/outs/raw_feature_bc_matrix.h5"), @@ -37,7 +43,8 @@ nextflow_pipeline { )}, { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/outs/raw_feature_bc_matrix.h5").exists() }, { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/outs/spatial/tissue_positions.csv").exists() }, - // multiqc + + // MultiQC { assert file("$outputDir/multiqc/multiqc_report.html").text.contains("Visium_FFPE_Human_Ovarian_Cancer")} ) } diff --git a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test index 82b0d92..baad55b 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test @@ -6,27 +6,38 @@ nextflow_pipeline { test("spaceranger ffpe v2 cytassist (default `-profile test`)") { when { params { - // This is the default `test` profile, no need to specify additional parameters + input = "https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_spaceranger.csv" + spaceranger_probeset = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/outs/probe_set.csv" + spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" + st_preprocess_min_counts = 5 + st_preprocess_min_genes = 3 outdir = "$outputDir" } } then { assertAll( + + // Workflow { assert workflow.success }, { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") }, - // data + + // Data { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_processed.h5ad").exists() }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_norm.h5ad").exists() }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_plain.h5ad").exists() }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_raw.h5ad").exists() }, - // reports - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_clustering.html").text.contains("Saving anndata file for future use") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_qc_and_normalisation.html").text.contains("Distribution after filtering") }, + + // Reports + { assert + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_qc_and_normalisation.html").text.contains("Distributions after filtering") }, + file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_clustering.html").text.contains("AnnData object is saved in the") }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_spatial_de.html").text.contains("Spatially variable genes") }, - // degs + + // DEGs { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/degs/st_spatial_de.csv").exists() }, - // spaceranger + + // Space Ranger { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/outs/web_summary.html").exists() }, { assert snapshot( path("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/outs/raw_feature_bc_matrix.h5"), @@ -34,7 +45,8 @@ nextflow_pipeline { )}, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/outs/raw_feature_bc_matrix.h5").exists() }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/outs/spatial/tissue_positions.csv").exists() }, - // multiqc + + // MultiQC { assert file("$outputDir/multiqc/multiqc_report.html").text.contains("CytAssist_11mm_FFPE_Human_Glioblastoma_2_4")} ) } From 20bc7ef228a0d30dcbebf9bd1345a38713702aa3 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Wed, 20 Dec 2023 09:55:22 +0100 Subject: [PATCH 14/20] Remove misplaced newline --- tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test index baad55b..af846b9 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test @@ -29,9 +29,8 @@ nextflow_pipeline { { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_raw.h5ad").exists() }, // Reports - { assert { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_qc_and_normalisation.html").text.contains("Distributions after filtering") }, - file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_clustering.html").text.contains("AnnData object is saved in the") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_clustering.html").text.contains("AnnData object is saved in the") }, { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_spatial_de.html").text.contains("Spatially variable genes") }, // DEGs @@ -51,7 +50,4 @@ nextflow_pipeline { ) } } - - - } From 33839e9739b8f3531b76c2451347478cde049e42 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Wed, 20 Dec 2023 10:08:44 +0100 Subject: [PATCH 15/20] Update Space Ranger `nf-test` pipeline snapshots --- tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap | 4 ++-- .../pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap index d9d705d..1ea6423 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap +++ b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap @@ -8,8 +8,8 @@ }, "software_versions": { "content": [ - "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, SPACERANGER_COUNT={spaceranger=2.1.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QC_AND_NORMALISATION={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" + "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.12.0, yaml=6.0.1}, FASTQC={fastqc=0.12.1}, SPACERANGER_COUNT={spaceranger=2.1.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QC_AND_NORMALISATION={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" ], "timestamp": "2023-06-29T12:46:05+0000" } -} \ No newline at end of file +} diff --git a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap index 24f6f88..3acc223 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap +++ b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap @@ -8,8 +8,8 @@ }, "software_versions": { "content": [ - "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, SPACERANGER_COUNT={spaceranger=2.1.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QC_AND_NORMALISATION={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" + "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.12.0, yaml=6.0.1}, FASTQC={fastqc=0.12.1}, SPACERANGER_COUNT={spaceranger=2.1.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QC_AND_NORMALISATION={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" ], "timestamp": "2023-06-29T12:48:09+0000" } -} \ No newline at end of file +} From 2102629ef7a4addf4f4a3a0f7f2e9b4e05561309 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Wed, 20 Dec 2023 10:24:53 +0100 Subject: [PATCH 16/20] Clarify `nf-test` pipeline names --- tests/pipeline/test_downstream.nf.test | 5 +++-- tests/pipeline/test_spaceranger_ffpe_v1.nf.test | 7 ++++--- tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/pipeline/test_downstream.nf.test b/tests/pipeline/test_downstream.nf.test index 07dec42..0b74606 100644 --- a/tests/pipeline/test_downstream.nf.test +++ b/tests/pipeline/test_downstream.nf.test @@ -1,11 +1,12 @@ nextflow_pipeline { - name "Test workflow (only downstream part)" + name "Test downstream workflow (excl. Space Ranger)" script "main.nf" tag "pipeline" - test("CytAssist_11mm_FFPE_Human_Glioblastoma_2") { + test("Downstream FFPE v2 CytAssist") { when { params { + // This is the default `test` profile; params are not necessary outdir = "$outputDir" } } diff --git a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test index 2d43b9c..97deb41 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test @@ -1,13 +1,14 @@ nextflow_pipeline { - name "Test full workflow including spaceranger" + name "Test full workflow (incl. Space Ranger)" script "main.nf" tag "pipeline" - test("spaceranger ffpe v1") { + test("Space Ranger FFPE v1 Standard") { when { params { input = 'https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/samplesheet_spaceranger.csv' - spaceranger_probeset = 'https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/Visium_Human_Transcriptome_Probe_Set_v1.0_GRCh38-2020-A.csv' + spaceranger_probeset = 'https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/Visium_Human_Transcriptome_Probe_Set_v1.0_GRCh38-2020-A.csv' + spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" st_preprocess_min_counts = 5 st_preprocess_min_genes = 3 outdir = "$outputDir" diff --git a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test index af846b9..d259b47 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test @@ -1,9 +1,9 @@ nextflow_pipeline { - name "Test full workflow including spaceranger" + name "Test full workflow (incl. Space Ranger)" script "main.nf" tag "pipeline" - test("spaceranger ffpe v2 cytassist (default `-profile test`)") { + test("Space Ranger FFPE v2 CytAssist") { when { params { input = "https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_spaceranger.csv" From a1800010b4e5f07624a83af5a185299f5cb650a4 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Wed, 20 Dec 2023 10:27:08 +0100 Subject: [PATCH 17/20] Do not checkout test data in CI testing --- .github/workflows/ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36ef2ae..6a02675 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,14 +51,6 @@ jobs: - name: Check out pipeline code uses: actions/checkout@v3 - - name: Checkout test data - uses: actions/checkout@v3 - with: - repository: nf-core/test-datasets - ref: spatialtranscriptomics - fetch-depth: 1 - path: test-datasets - # Install Nextflow - name: Install Nextflow uses: nf-core/setup-nextflow@v1 From 0405d590703c96d5fbbc95abbef6dbbb97f74176 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Wed, 20 Dec 2023 12:01:19 +0100 Subject: [PATCH 18/20] Update FastQC and MultiQC nf-core modules --- modules.json | 4 +- modules/nf-core/fastqc/tests/main.nf.test | 68 ++++++++++++++++++++++ modules/nf-core/multiqc/tests/main.nf.test | 32 +--------- 3 files changed, 72 insertions(+), 32 deletions(-) diff --git a/modules.json b/modules.json index 6810b8b..0f162bb 100644 --- a/modules.json +++ b/modules.json @@ -12,12 +12,12 @@ }, "fastqc": { "branch": "master", - "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", + "git_sha": "65ad3e0b9a4099592e1102e92e10455dc661cf53", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "1537442a7be4a78efa3d1ff700a923c627bbda5d", + "git_sha": "4ab13872435962dadc239979554d13709e20bf29", "installed_by": ["modules"] }, "spaceranger/count": { diff --git a/modules/nf-core/fastqc/tests/main.nf.test b/modules/nf-core/fastqc/tests/main.nf.test index 6437a14..b9e8f92 100644 --- a/modules/nf-core/fastqc/tests/main.nf.test +++ b/modules/nf-core/fastqc/tests/main.nf.test @@ -38,4 +38,72 @@ nextflow_process { ) } } +// TODO +// // +// // Test with paired-end data +// // +// workflow test_fastqc_paired_end { +// input = [ +// [id: 'test', single_end: false], // meta map +// [ +// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), +// file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) +// ] +// ] + +// FASTQC ( input ) +// } + +// // +// // Test with interleaved data +// // +// workflow test_fastqc_interleaved { +// input = [ +// [id: 'test', single_end: false], // meta map +// file(params.test_data['sarscov2']['illumina']['test_interleaved_fastq_gz'], checkIfExists: true) +// ] + +// FASTQC ( input ) +// } + +// // +// // Test with bam data +// // +// workflow test_fastqc_bam { +// input = [ +// [id: 'test', single_end: false], // meta map +// file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) +// ] + +// FASTQC ( input ) +// } + +// // +// // Test with multiple samples +// // +// workflow test_fastqc_multiple { +// input = [ +// [id: 'test', single_end: false], // meta map +// [ +// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), +// file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true), +// file(params.test_data['sarscov2']['illumina']['test2_1_fastq_gz'], checkIfExists: true), +// file(params.test_data['sarscov2']['illumina']['test2_2_fastq_gz'], checkIfExists: true) +// ] +// ] + +// FASTQC ( input ) +// } + +// // +// // Test with custom prefix +// // +// workflow test_fastqc_custom_prefix { +// input = [ +// [ id:'mysample', single_end:true ], // meta map +// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) +// ] + +// FASTQC ( input ) +// } } diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test index 68fffa9..c2dad21 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -9,27 +9,13 @@ nextflow_process { test("MULTIQC: FASTQC") { - setup { - run("FASTQC") { - script "../../fastqc/main.nf" - process { - """ - input[0] = Channel.of([ - [ id: 'test', single_end: false ], - [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true)] - ]) - """ - } - } - } - when { params { outdir = "$outputDir" } process { """ - input[0] = FASTQC.out.zip.collect { it[1] } + input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) input[1] = [] input[2] = [] input[3] = [] @@ -50,27 +36,13 @@ nextflow_process { test("MULTIQC: FASTQC and a config file") { - setup { - run("FASTQC") { - script "../../fastqc/main.nf" - process { - """ - input[0] = Channel.of([ - [ id: 'test', single_end: false ], - [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true)] - ]) - """ - } - } - } - when { params { outdir = "$outputDir" } process { """ - input[0] = FASTQC.out.zip.collect { it[1] } + input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) input[2] = [] input[3] = [] From a8098435054239a599a324dd3e6b099c41e00539 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Wed, 20 Dec 2023 13:22:50 +0100 Subject: [PATCH 19/20] Update `CHANGELOG.md` --- CHANGELOG.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 521f26f..af81800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,24 +5,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -Initial release of nf-core/spatialtranscriptomics, created with the [nf-core](https://nf-co.re/) template. -This marks the point at which the pipeline development was moved to nf-core and -NBIS. The pipeline has undergone several iterations regarding its functionality -and content; there are a significant number of changes, of which not all are -listed here. In summary, the pipeline contains best-practice processing and -analyses of pre- and post-Space Ranger-processed data, including quality -controls, normalisation, dimensionality reduction, clustering, differential -expression testing as well as output files compatible with further downstream -analyses and/or exploration in _e.g._ [TissUUmaps](https://tissuumaps.github.io/) -or bespoke user code. +Initial release of nf-core/spatialtranscriptomics, created with the +[nf-core](https://nf-co.re/) template. This marks the point at which the +pipeline development was moved to nf-core and NBIS. The pipeline has undergone +several iterations regarding its functionality and content; there are a +significant number of changes, of which not all are listed here. In summary, the +pipeline contains best-practice processing and analyses of pre- and post-Space +Ranger-processed data, including quality controls, normalisation, dimensionality +reduction, clustering, differential expression testing as well as output files +compatible with further downstream analyses and/or exploration in _e.g._ +[TissUUmaps](https://tissuumaps.github.io/) or bespoke user code. ### `Added` +- Allow input directories `fastq_dir` and `spaceranger_dir` to be specified as tar archives (`.tar.gz`) +- Add a check to make sure that there are spots left after filtering [[#46](https://github.com/nf-core/spatialtranscriptomics/issues/46)] - Implement tests with nf-test [[#42](https://github.com/nf-core/spatialtranscriptomics/pull/42)] - Replace custom code to download reference with `untar` module [[#44](https://github.com/nf-core/spatialtranscriptomics/pull/44)] -- Replace custom code to download reference with `untar` module [[#44](https://github.com/nf-core/spatialtranscriptomics/pull/44)] - Embed resources in quarto reports [[#43](https://github.com/nf-core/spatialtranscriptomics/pull/43)] -- Implement tests with nf-test [[#42](https://github.com/nf-core/spatialtranscriptomics/pull/42)] - Use a samplesheet for input specification [[#30](https://github.com/nf-core/spatialtranscriptomics/pull/30), [#31](https://github.com/nf-core/spatialtranscriptomics/pull/31) and [#45](https://github.com/nf-core/spatialtranscriptomics/pull/45)] - Add Space Ranger pre-processing as an optional pipeline step using the `spaceranger` nf-core module [[#17](https://github.com/nf-core/spatialtranscriptomics/pull/17) and [#45](https://github.com/nf-core/spatialtranscriptomics/pull/45)] - Add `env/` directory with pipeline-specific container and Conda environment specifications [[#17](https://github.com/nf-core/spatialtranscriptomics/pull/17) and [#28](https://github.com/nf-core/spatialtranscriptomics/pull/28)] @@ -50,7 +50,7 @@ versions of the same tool. | ----------- | ------- | | `SpatialDE` | 1.1.3 | | `leidenalg` | 0.9.1 | -| `python` | 3.11.0 | +| `python` | 3.12.0 | | `quarto` | 1.3.302 | | `scanpy` | 1.9.3 | From 4a06e971cb467bd9c7ac460ff72f77bd965e5ab2 Mon Sep 17 00:00:00 2001 From: Erik Fasterius Date: Wed, 20 Dec 2023 13:34:02 +0100 Subject: [PATCH 20/20] Use `nf-core/test-datasets` for test data --- conf/test.config | 2 +- tests/pipeline/test_spaceranger_ffpe_v1.nf.test | 2 +- tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/test.config b/conf/test.config index 29f95c6..21e7c9d 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,7 +20,7 @@ params { max_time = '2.h' // Input and output - input = 'https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_downstream.csv' + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_downstream.csv' spaceranger_probeset = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/outs/probe_set.csv" spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" st_preprocess_min_counts = 5 diff --git a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test index 97deb41..8e4690e 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test @@ -6,7 +6,7 @@ nextflow_pipeline { test("Space Ranger FFPE v1 Standard") { when { params { - input = 'https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/samplesheet_spaceranger.csv' + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/samplesheet_spaceranger.csv' spaceranger_probeset = 'https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/Visium_Human_Transcriptome_Probe_Set_v1.0_GRCh38-2020-A.csv' spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" st_preprocess_min_counts = 5 diff --git a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test index d259b47..c67e138 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test @@ -6,7 +6,7 @@ nextflow_pipeline { test("Space Ranger FFPE v2 CytAssist") { when { params { - input = "https://raw.githubusercontent.com/fasterius/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_spaceranger.csv" + input = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/samplesheet_spaceranger.csv" spaceranger_probeset = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/outs/probe_set.csv" spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" st_preprocess_min_counts = 5