From 42ccce6d9ce6c70cd91d9808891f8368521ed4cd Mon Sep 17 00:00:00 2001 From: Sebastian Uhrig Date: Sun, 7 Apr 2024 20:27:50 +0000 Subject: [PATCH 1/6] update module sratools/prefetch --- modules.json | 4 ++-- modules/nf-core/sratools/prefetch/environment.yml | 3 ++- modules/nf-core/sratools/prefetch/main.nf | 4 ++-- .../prefetch/templates/retry_with_backoff.sh | 12 +++++++++++- modules/nf-core/sratools/prefetch/tests/main.nf.test | 4 ++++ .../sratools/prefetch/tests/main.nf.test.snap | 10 +++++----- modules/nf-core/sratools/prefetch/tests/tags.yml | 2 ++ .../tests/main.nf.test | 10 ++++++---- .../tests/main.nf.test.snap | 8 ++++---- .../tests/tags.yml | 2 ++ 10 files changed, 40 insertions(+), 19 deletions(-) create mode 100644 modules/nf-core/sratools/prefetch/tests/tags.yml create mode 100644 subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/tags.yml diff --git a/modules.json b/modules.json index 93f4553b..c6b234bb 100644 --- a/modules.json +++ b/modules.json @@ -18,7 +18,7 @@ }, "sratools/prefetch": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "1fc29f92e439d5631fdf34b8ac4687297d70f5ec", "installed_by": ["fastq_download_prefetch_fasterqdump_sratools"] }, "untar": { @@ -32,7 +32,7 @@ "nf-core": { "fastq_download_prefetch_fasterqdump_sratools": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "1fc29f92e439d5631fdf34b8ac4687297d70f5ec", "installed_by": ["subworkflows"] }, "utils_nextflow_pipeline": { diff --git a/modules/nf-core/sratools/prefetch/environment.yml b/modules/nf-core/sratools/prefetch/environment.yml index cfc7d9a8..0abad336 100644 --- a/modules/nf-core/sratools/prefetch/environment.yml +++ b/modules/nf-core/sratools/prefetch/environment.yml @@ -4,4 +4,5 @@ channels: - bioconda - defaults dependencies: - - bioconda::sra-tools=3.0.8 + - bioconda::sra-tools=3.1.0 + - conda-forge::curl=8.5.0 diff --git a/modules/nf-core/sratools/prefetch/main.nf b/modules/nf-core/sratools/prefetch/main.nf index 3c30739a..170f1753 100644 --- a/modules/nf-core/sratools/prefetch/main.nf +++ b/modules/nf-core/sratools/prefetch/main.nf @@ -4,8 +4,8 @@ process SRATOOLS_PREFETCH { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/sra-tools:3.0.8--h9f5acd7_0' : - 'biocontainers/sra-tools:3.0.8--h9f5acd7_0' }" + 'https://depot.galaxyproject.org/singularity/sra-tools:3.1.0--h9f5acd7_0' : + 'biocontainers/sra-tools:3.1.0--h9f5acd7_0' }" input: tuple val(meta), val(id) diff --git a/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh b/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh index a72a4bfb..bfee6070 100755 --- a/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh +++ b/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh @@ -47,9 +47,19 @@ retry_with_backoff !{args2} \ !{args} \ !{id} -[ -f !{id}.sralite ] && vdb-validate !{id}.sralite || vdb-validate !{id} +# check file integrity using vdb-validate or (when archive contains no checksums) md5sum +vdb-validate !{id} > vdb-validate_result.txt 2>&1 || exit 1 +if grep -q "checksums missing" vdb-validate_result.txt; then + VALID_MD5SUMS=$(curl --silent --fail --location --retry 3 --retry-delay 60 'https://locate.ncbi.nlm.nih.gov/sdl/2/retrieve?filetype=run&acc=!{id}') + LOCAL_MD5SUMS=$(md5sum !{id}/* | cut -f1 -d' ') + if ! grep -q -F -f <(echo "$LOCAL_MD5SUMS") <(echo "$VALID_MD5SUMS"); then + echo "MD5 sum check failed" 1>&2 + exit 1 + fi +fi cat <<-END_VERSIONS > versions.yml "!{task.process}": sratools: $(prefetch --version 2>&1 | grep -Eo '[0-9.]+') + curl: $(curl --version | head -n 1 | sed 's/^curl //; s/ .*$//') END_VERSIONS diff --git a/modules/nf-core/sratools/prefetch/tests/main.nf.test b/modules/nf-core/sratools/prefetch/tests/main.nf.test index c3ebd91e..f59de7cb 100644 --- a/modules/nf-core/sratools/prefetch/tests/main.nf.test +++ b/modules/nf-core/sratools/prefetch/tests/main.nf.test @@ -2,6 +2,10 @@ nextflow_process { name "Test Process SRATOOLS_PREFETCH" script "../main.nf" process "SRATOOLS_PREFETCH" + tag "modules" + tag "modules_nfcore" + tag "sratools" + tag "sratools/prefetch" test("sratools/prefetch") { diff --git a/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap b/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap index 946fc869..67795ebb 100644 --- a/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap +++ b/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap @@ -14,7 +14,7 @@ ] ], "1": [ - "versions.yml:md5,c967dea4135cb75490e1e801c4639efc" + "versions.yml:md5,83d1b23f5ff5b2ad1b96d17d7d7594ee" ], "sra": [ [ @@ -28,7 +28,7 @@ ] ], "versions": [ - "versions.yml:md5,c967dea4135cb75490e1e801c4639efc" + "versions.yml:md5,83d1b23f5ff5b2ad1b96d17d7d7594ee" ] } ], @@ -53,7 +53,7 @@ ] ], "1": [ - "versions.yml:md5,c967dea4135cb75490e1e801c4639efc" + "versions.yml:md5,83d1b23f5ff5b2ad1b96d17d7d7594ee" ], "sra": [ [ @@ -67,7 +67,7 @@ ] ], "versions": [ - "versions.yml:md5,c967dea4135cb75490e1e801c4639efc" + "versions.yml:md5,83d1b23f5ff5b2ad1b96d17d7d7594ee" ] } ], @@ -77,4 +77,4 @@ }, "timestamp": "2024-02-28T11:48:37.428307" } -} \ No newline at end of file +} diff --git a/modules/nf-core/sratools/prefetch/tests/tags.yml b/modules/nf-core/sratools/prefetch/tests/tags.yml new file mode 100644 index 00000000..52110bfd --- /dev/null +++ b/modules/nf-core/sratools/prefetch/tests/tags.yml @@ -0,0 +1,2 @@ +sratools/prefetch: + - modules/nf-core/sratools/prefetch/** diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test index 7b41ed7e..5f74ed20 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test @@ -3,10 +3,12 @@ nextflow_workflow { name "Test workflow: fastq_download_prefetch_fasterqdump_sratools/main.nf" script "../main.nf" workflow "FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS" - - tag "CUSTOM_SRATOOLSNCBISETTINGS" - tag "SRATOOLS_PREFETCH" - tag "SRATOOLS_FASTERQDUMP" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "custom/sratoolsncbisettings" + tag "sratools/prefetch" + tag "sratools/fasterqdump" + tag "subworkflows/fastq_download_prefetch_fasterqdump_sratools" test("Parameters: default") { diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap index 485411da..1a34e0b8 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap @@ -40,8 +40,8 @@ "content": [ [ "versions.yml:md5,1a2218ff913fc33408bffccb081b5048", - "versions.yml:md5,2f3b3a13b36dabf13f09327613d5558d", - "versions.yml:md5,98d78bba9f3da39a0b7db6e9c7dcc224" + "versions.yml:md5,53d6e983afde3a28add2ffc6b7eba4f3", + "versions.yml:md5,9c558ff624585a6eee82a19c8c0136db" ] ], "meta": { @@ -52,7 +52,7 @@ }, "test_pe_reads_1_size": { "content": [ - 2013376 + 2011460 ], "meta": { "nf-test": "0.8.4", @@ -94,4 +94,4 @@ }, "timestamp": "2024-02-28T12:02:56.161354" } -} \ No newline at end of file +} diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/tags.yml b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/tags.yml new file mode 100644 index 00000000..03028c32 --- /dev/null +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/fastq_download_prefetch_fasterqdump_sratools: + - subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/** From 3896fd35a4f750c75de9419559406c752bd5277b Mon Sep 17 00:00:00 2001 From: Sebastian Uhrig Date: Sun, 7 Apr 2024 20:33:29 +0000 Subject: [PATCH 2/6] update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecb962d6..dbfb81be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [PR #299](https://github.com/nf-core/fetchngs/pull/299) - Template update for nf-core/tools v2.13.1 - [PR #300](https://github.com/nf-core/fetchngs/pull/300) - Use file paths instead of tags for testing matrix, should make matrices more efficient - [PR #303](https://github.com/nf-core/fetchngs/pull/303) - Update wget container for SRA_FASTQ_FTP from 1.20.1 to 1.21.4 +- [PR #305](https://github.com/nf-core/fetchngs/pull/305) - Update module sratools/prefetch for reliable download integrity check ### Software dependencies | Dependency | Old version | New version | | ---------- | ----------- | ----------- | | `wget` | 1.20.1 | 1.21.4 | +| `sratools` | 3.0.8 | 3.1.0 | ## [[1.12.0](https://github.com/nf-core/fetchngs/releases/tag/1.12.0)] - 2024-02-29 From 577af8a7973fe085146590a1be5d1bc1d92c0d78 Mon Sep 17 00:00:00 2001 From: Sebastian Uhrig Date: Mon, 8 Apr 2024 09:32:23 +0000 Subject: [PATCH 3/6] update MD5 sum of versions.yml for nf-test --- .../tests/main.nf.test.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap index 1a34e0b8..6872d8d2 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap @@ -41,7 +41,7 @@ [ "versions.yml:md5,1a2218ff913fc33408bffccb081b5048", "versions.yml:md5,53d6e983afde3a28add2ffc6b7eba4f3", - "versions.yml:md5,9c558ff624585a6eee82a19c8c0136db" + "versions.yml:md5,2f3b3a13b36dabf13f09327613d5558d" ] ], "meta": { From 445577f72d4cff32aa2b19dbe90aa7101d8c688e Mon Sep 17 00:00:00 2001 From: Sebastian Uhrig Date: Mon, 8 Apr 2024 09:51:42 +0000 Subject: [PATCH 4/6] update MD5 sum of versions.yml for nf-test --- .../tests/main.nf.test.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap index 6872d8d2..e10ffeb3 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap @@ -40,8 +40,8 @@ "content": [ [ "versions.yml:md5,1a2218ff913fc33408bffccb081b5048", - "versions.yml:md5,53d6e983afde3a28add2ffc6b7eba4f3", - "versions.yml:md5,2f3b3a13b36dabf13f09327613d5558d" + "versions.yml:md5,2f3b3a13b36dabf13f09327613d5558d", + "versions.yml:md5,53d6e983afde3a28add2ffc6b7eba4f3" ] ], "meta": { From dd4d19cc2285825435e60c044ea9a8f57cc3734b Mon Sep 17 00:00:00 2001 From: Sebastian Uhrig Date: Mon, 8 Apr 2024 10:39:42 +0000 Subject: [PATCH 5/6] update snapshots for nf-test --- .../tests/main.nf.test.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap index e10ffeb3..2530372b 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap @@ -28,7 +28,7 @@ }, "test_pe_reads_2_size": { "content": [ - 2011460 + 2013376 ], "meta": { "nf-test": "0.8.4", @@ -52,7 +52,7 @@ }, "test_pe_reads_1_size": { "content": [ - 2011460 + 2013376 ], "meta": { "nf-test": "0.8.4", From 3056a3ad09306ce58159093c8ee013d59dc5e682 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Mon, 8 Apr 2024 14:51:27 +0200 Subject: [PATCH 6/6] Update subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap --- .../tests/main.nf.test.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap index 2530372b..0c0be2a4 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap @@ -28,7 +28,7 @@ }, "test_pe_reads_2_size": { "content": [ - 2013376 + 2011460 ], "meta": { "nf-test": "0.8.4",