From 4696cafe675a6781f99b861b2916536bab5faf77 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Thu, 18 May 2023 16:47:24 -0500 Subject: [PATCH 01/25] adds sys testing basic ver (based on https://stackoverflow.com/a/72735253/3998252) --- .github/workflows/system_testing.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/system_testing.yml diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml new file mode 100644 index 0000000..a14d9ae --- /dev/null +++ b/.github/workflows/system_testing.yml @@ -0,0 +1,40 @@ +name: system_testing +on: + push: + paths: + - 'configs/env/quac.yaml' + - '.github/workflows/system_testing.yml' + + +jobs: + create-env: + name: ubuntu-latest + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + strategy: + fail-fast: false + + steps: + - name: checkout repository + uses: actions/checkout@v2 + + - name: create environment with mamba + uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "*" + channels: conda-forge,bioconda,defaults + auto-activate-base: false + activate-environment: quac + environment-file: configs/env/quac.yaml + + - name: check solution + run: | + mamba env export + + - name: test samtools + run: | + which snakemake + snakemake --help \ No newline at end of file From be8cde35bab1da9ff2f07448172857d086eed5af Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Fri, 19 May 2023 10:03:26 -0500 Subject: [PATCH 02/25] adds singularity --- .github/workflows/system_testing.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index a14d9ae..749e97a 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -14,8 +14,8 @@ jobs: run: shell: bash -l {0} - strategy: - fail-fast: false + # strategy: + # fail-fast: false steps: - name: checkout repository @@ -34,7 +34,13 @@ jobs: run: | mamba env export - - name: test samtools + - name: test snakemake run: | which snakemake - snakemake --help \ No newline at end of file + snakemake --help + + - uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.8.3 + - name: Run a singularity container + run: singularity run docker://alpine cat /etc/os-release \ No newline at end of file From 63338795c29bea8391da2165919f6b0f3922fde7 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Fri, 19 May 2023 10:09:02 -0500 Subject: [PATCH 03/25] adds system testing task --- .github/workflows/system_testing.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 749e97a..b6791c5 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -43,4 +43,16 @@ jobs: with: singularity-version: 3.8.3 - name: Run a singularity container - run: singularity run docker://alpine cat /etc/os-release \ No newline at end of file + run: singularity run docker://alpine cat /etc/os-release + + - name: run system testing + run: | + which python + ls + python src/run_quac.py \ + --project_name test_project \ + --projects_path ".test/ngs-data/" \ + --pedigree ".test/configs/no_priorQC/project_2samples.ped" \ + --outdir "data/quac/results/test_project_2samples_wgs-no_priorQC/analysis" \ + --quac_watch_config "configs/quac_watch/wgs_quac_watch_config.yaml" \ + --workflow_config "configs/workflow.yaml" -n \ No newline at end of file From 29f3720a5c4ee126878461548d4dc8ffad12ddfb Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Fri, 19 May 2023 10:17:52 -0500 Subject: [PATCH 04/25] sets up dependencies for quac --- .github/workflows/system_testing.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index b6791c5..05cb15e 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -37,15 +37,21 @@ jobs: - name: test snakemake run: | which snakemake - snakemake --help + snakemake --version - uses: eWaterCycle/setup-singularity@v7 with: singularity-version: 3.8.3 - name: Run a singularity container - run: singularity run docker://alpine cat /etc/os-release + run: | + # singularity run docker://alpine cat /etc/os-release + singularity --version - - name: run system testing + - name: Set up dependecies for QuaC + run: | + bash src/setup_dependency_datasets.sh + + - name: Run QuaC system testing run: | which python ls From a0ff724c3363de9c7b05ab8367f078f90863e46a Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Fri, 19 May 2023 10:27:40 -0500 Subject: [PATCH 05/25] removes dryrun option; time to run sys testing --- .github/workflows/system_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 05cb15e..e7a70ab 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -61,4 +61,4 @@ jobs: --pedigree ".test/configs/no_priorQC/project_2samples.ped" \ --outdir "data/quac/results/test_project_2samples_wgs-no_priorQC/analysis" \ --quac_watch_config "configs/quac_watch/wgs_quac_watch_config.yaml" \ - --workflow_config "configs/workflow.yaml" -n \ No newline at end of file + --workflow_config "configs/workflow.yaml" \ No newline at end of file From 34846453b15232a9b71e7f54de0bc1aec6974448 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Fri, 19 May 2023 11:44:31 -0500 Subject: [PATCH 06/25] adds try-except to job submission step --- src/run_quac.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/run_quac.py b/src/run_quac.py index cb3562d..fecc917 100755 --- a/src/run_quac.py +++ b/src/run_quac.py @@ -231,7 +231,12 @@ def main(args): "resources": slurm_resources, } - submit_slurm_job(pipeline_cmd, job_dict) + try: + submit_slurm_job(pipeline_cmd, job_dict) + except Exception as e: + print ("Job submission step ran into an error:") + print(repr(e)) + raise SystemExit(1) return None From e1f5a844e6ceb6d1dc5d4f9f3df2f22db3cf5623 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Fri, 19 May 2023 13:15:02 -0500 Subject: [PATCH 07/25] removes commented code --- .github/workflows/system_testing.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index e7a70ab..771d76c 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -14,9 +14,6 @@ jobs: run: shell: bash -l {0} - # strategy: - # fail-fast: false - steps: - name: checkout repository uses: actions/checkout@v2 From bc761681c1a6b10a1596a0162d6670e8b19f9a1e Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Tue, 30 May 2023 13:11:13 -0500 Subject: [PATCH 08/25] removes try-except --- src/run_quac.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/run_quac.py b/src/run_quac.py index fecc917..cb3562d 100755 --- a/src/run_quac.py +++ b/src/run_quac.py @@ -231,12 +231,7 @@ def main(args): "resources": slurm_resources, } - try: - submit_slurm_job(pipeline_cmd, job_dict) - except Exception as e: - print ("Job submission step ran into an error:") - print(repr(e)) - raise SystemExit(1) + submit_slurm_job(pipeline_cmd, job_dict) return None From 6d3cd41e3a7efc0157acbcabfc5b9f177175e592 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Tue, 30 May 2023 13:12:48 -0500 Subject: [PATCH 09/25] removes on-path; adds more modes --- .github/workflows/system_testing.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 771d76c..370f472 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -1,10 +1,5 @@ name: system_testing -on: - push: - paths: - - 'configs/env/quac.yaml' - - '.github/workflows/system_testing.yml' - +on: [push, pull_request, workflow_dispatch] jobs: create-env: From 0a523b1128eff647daaf2cfe3230002f34cc718f Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Tue, 30 May 2023 13:19:59 -0500 Subject: [PATCH 10/25] pins OS ver; cleans up messages; cleans up commands --- .github/workflows/system_testing.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 370f472..a41e6ee 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -3,8 +3,8 @@ on: [push, pull_request, workflow_dispatch] jobs: create-env: - name: ubuntu-latest - runs-on: ubuntu-latest + name: ubuntu-20.04 + runs-on: ubuntu-20.04 defaults: run: shell: bash -l {0} @@ -13,7 +13,7 @@ jobs: - name: checkout repository uses: actions/checkout@v2 - - name: create environment with mamba + - name: Create quac environment uses: conda-incubator/setup-miniconda@v2 with: mamba-version: "*" @@ -22,11 +22,11 @@ jobs: activate-environment: quac environment-file: configs/env/quac.yaml - - name: check solution + - name: Check conda solution run: | mamba env export - - name: test snakemake + - name: Check snakemake exists in conda env run: | which snakemake snakemake --version @@ -34,19 +34,16 @@ jobs: - uses: eWaterCycle/setup-singularity@v7 with: singularity-version: 3.8.3 - - name: Run a singularity container + - name: Check singularity is working run: | - # singularity run docker://alpine cat /etc/os-release singularity --version - - name: Set up dependecies for QuaC + - name: Set up dependencies for QuaC run: | bash src/setup_dependency_datasets.sh - name: Run QuaC system testing run: | - which python - ls python src/run_quac.py \ --project_name test_project \ --projects_path ".test/ngs-data/" \ From 0e55b96b45726e3c1df206b269cc9711ea0fbd8f Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Tue, 30 May 2023 13:49:45 -0500 Subject: [PATCH 11/25] runs 4 variations of system testing serially --- .github/workflows/system_testing.yml | 64 +++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index a41e6ee..7498474 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -2,8 +2,8 @@ name: system_testing on: [push, pull_request, workflow_dispatch] jobs: - create-env: - name: ubuntu-20.04 + system-testing: + name: System testing - QuaC runs-on: ubuntu-20.04 defaults: run: @@ -42,12 +42,64 @@ jobs: run: | bash src/setup_dependency_datasets.sh - - name: Run QuaC system testing + - name: Run QuaC system testing - WGS mode AND no prior QC data run: | + PROJECT_CONFIG="project_2samples" + PRIOR_QC_STATUS="no_priorQC" + USE_SLURM="" python src/run_quac.py \ --project_name test_project \ --projects_path ".test/ngs-data/" \ - --pedigree ".test/configs/no_priorQC/project_2samples.ped" \ - --outdir "data/quac/results/test_project_2samples_wgs-no_priorQC/analysis" \ + --pedigree ".test/configs/${PRIOR_QC_STATUS}/${PROJECT_CONFIG}.ped" \ + --outdir "data/quac/results/test_${PROJECT_CONFIG}_wgs-${PRIOR_QC_STATUS}/analysis" \ --quac_watch_config "configs/quac_watch/wgs_quac_watch_config.yaml" \ - --workflow_config "configs/workflow.yaml" \ No newline at end of file + --workflow_config "configs/workflow.yaml" \ + $USE_SLURM + + - name: Run QuaC system testing - Exome mode AND no prior QC data + run: | + PROJECT_CONFIG="project_2samples" + PRIOR_QC_STATUS="no_priorQC" + USE_SLURM="" + python src/run_quac.py \ + --project_name test_project \ + --projects_path ".test/ngs-data/" \ + --pedigree ".test/configs/${PRIOR_QC_STATUS}/${PROJECT_CONFIG}.ped" \ + --outdir "data/quac/results/test_${PROJECT_CONFIG}_exome-${PRIOR_QC_STATUS}/analysis" \ + --quac_watch_config "configs/quac_watch/exome_quac_watch_config.yaml" \ + --workflow_config "configs/workflow.yaml" \ + --exome \ + $USE_SLURM + + - name: Run QuaC system testing - WGS mode AND uses prior QC data + run: | + PROJECT_CONFIG="project_2samples" + PRIOR_QC_STATUS="include_priorQC" + USE_SLURM="" + python src/run_quac.py \ + --project_name test_project \ + --projects_path ".test/ngs-data/" \ + --pedigree ".test/configs/${PRIOR_QC_STATUS}/${PROJECT_CONFIG}.ped" \ + --outdir "data/quac/results/test_${PROJECT_CONFIG}_wgs-${PRIOR_QC_STATUS}/analysis" \ + --quac_watch_config "configs/quac_watch/wgs_quac_watch_config.yaml" \ + --include_prior_qc \ + --allow_sample_renaming \ + --workflow_config "configs/workflow.yaml" \ + $USE_SLURM + + - name: Run QuaC system testing - Exome mode AND uses prior QC data + run: | + PROJECT_CONFIG="project_2samples" + PRIOR_QC_STATUS="include_priorQC" + USE_SLURM="" + python src/run_quac.py \ + --project_name test_project \ + --projects_path ".test/ngs-data/" \ + --pedigree ".test/configs/${PRIOR_QC_STATUS}/${PROJECT_CONFIG}.ped" \ + --outdir "data/quac/results/test_${PROJECT_CONFIG}_exome-${PRIOR_QC_STATUS}/analysis" \ + --quac_watch_config "configs/quac_watch/exome_quac_watch_config.yaml" \ + --exome \ + --include_prior_qc \ + --allow_sample_renaming \ + --workflow_config "configs/workflow.yaml" \ + $USE_SLURM \ No newline at end of file From e9bb0cf0d0121b77cf382b1202fb65b3b39af6ae Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Wed, 31 May 2023 10:06:27 -0500 Subject: [PATCH 12/25] specifies paths to trigger system testing --- .github/workflows/system_testing.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 7498474..fc5663c 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -1,5 +1,22 @@ name: system_testing -on: [push, pull_request, workflow_dispatch] +# on: [push, pull_request, workflow_dispatch] +on: + push: + paths: + - '.github/workflows/system_testing.yml' + - '.test/**' + - 'configs/**' + - 'src/**' + - 'workflow/**' + pull_request: + paths: + - '.github/workflows/system_testing.yml' + - '.test/**' + - 'configs/**' + - 'src/**' + - 'workflow/**' + workflow_dispatch: + jobs: system-testing: From 41d1adc63ca37d8ed065eeb1fc4313896adb8939 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Wed, 31 May 2023 10:06:59 -0500 Subject: [PATCH 13/25] specifies PR types to trigger system testing --- .github/workflows/system_testing.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index fc5663c..3fc004a 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -15,6 +15,12 @@ on: - 'configs/**' - 'src/**' - 'workflow/**' + types: + - opened + - synchronize + - reopened + - ready_for_review + - converted_to_draft workflow_dispatch: From 40c0d79bea1f1cc2c02c523d0a314a91317bece4 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Wed, 31 May 2023 10:12:49 -0500 Subject: [PATCH 14/25] removes system testing reqt from PR template --- .github/pull_request_template.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5140009..ac9ce4e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,7 +8,6 @@ Please fill in the checklist below and comment as needed. - [ ] Is this a feature addition? Briefly describe. - [ ] Did you modify QuaC-Watch config file? If so, did you modify multiqc template `configs/multiqc_config_template.jinja2` and script `src/quac_watch/create_mutliqc_configs.py` as necessary? -- [ ] Did you perform system-level testing manually as described in master readme doc? Did it pass completely? If not why? - [ ] Updated `Changelog.md` file with change logs in recommended format? From 869356cda24b430900a497d8c212829e06cefbf5 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Wed, 31 May 2023 10:14:24 -0500 Subject: [PATCH 15/25] updates changelog --- docs/Changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/Changelog.md b/docs/Changelog.md index 4e1c186..97c20a9 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -12,6 +12,10 @@ YYYY-MM-DD John Doe ``` --- +2023-05-31 Manavalan Gajapathy + +* Adds system testing as github actions workflow + 2023-05-09 Manavalan Gajapathy * Adds a verification step in the CLI wrapper script to check if the file/dirpaths to be mounted to singularity already exist as expected (#71) From 6a9c009f54d1a56322bc89ef0d5d63084401325e Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Wed, 31 May 2023 11:00:27 -0500 Subject: [PATCH 16/25] modifies just to see if system testing workflow gets triggered --- .github/workflows/system_testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 3fc004a..d5edf48 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -23,7 +23,6 @@ on: - converted_to_draft workflow_dispatch: - jobs: system-testing: name: System testing - QuaC From da3f9bbf50ac396250020ba2fe64f58ef0921a50 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Wed, 14 Jun 2023 10:28:59 -0500 Subject: [PATCH 17/25] adds slurm-based system testing back to PR template --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ac9ce4e..57d5ce6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,6 +8,7 @@ Please fill in the checklist below and comment as needed. - [ ] Is this a feature addition? Briefly describe. - [ ] Did you modify QuaC-Watch config file? If so, did you modify multiqc template `configs/multiqc_config_template.jinja2` and script `src/quac_watch/create_mutliqc_configs.py` as necessary? +- [ ] Did you perform system-level testing manually, using `--subtasks_slurm` flag, as described in the [documentation](https://quac.readthedocs.io/en/stable/system_testing/)? Did it pass completely? If not why? - [ ] Updated `Changelog.md` file with change logs in recommended format? From a986d94f94e599309af407996adb60b1963fbaac Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Wed, 14 Jun 2023 10:29:14 -0500 Subject: [PATCH 18/25] adds sys testing badge to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6049207..2f5e6ad 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Snakemake](https://img.shields.io/badge/snakemake-6.0.5-brightgreen.svg?style=flat)](https://snakemake.readthedocs.io) [![ReadTheDocs](https://readthedocs.org/projects/quac/badge/?version=latest)](https://quac.readthedocs.io/en/stable/) [![DOI](https://zenodo.org/badge/593702083.svg)](https://zenodo.org/badge/latestdoi/593702083) - +[![system_testing](https://github.com/uab-cgds-worthey/quac/actions/workflows/system_testing.yml/badge.svg)](https://github.com/uab-cgds-worthey/quac/actions/workflows/system_testing.yml) # QuaC From e7542138ece82e182844c275ae1caf1f60f6acd7 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Thu, 15 Jun 2023 17:14:50 -0500 Subject: [PATCH 19/25] adds debugging step for storage --- .github/workflows/system_testing.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index d5edf48..4f3a325 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -46,10 +46,13 @@ jobs: - name: Check conda solution run: | + df -h mamba env export + df -h - name: Check snakemake exists in conda env run: | + df -h which snakemake snakemake --version @@ -58,14 +61,17 @@ jobs: singularity-version: 3.8.3 - name: Check singularity is working run: | + df -h singularity --version - name: Set up dependencies for QuaC run: | + df -h bash src/setup_dependency_datasets.sh - name: Run QuaC system testing - WGS mode AND no prior QC data run: | + df -h PROJECT_CONFIG="project_2samples" PRIOR_QC_STATUS="no_priorQC" USE_SLURM="" @@ -77,9 +83,11 @@ jobs: --quac_watch_config "configs/quac_watch/wgs_quac_watch_config.yaml" \ --workflow_config "configs/workflow.yaml" \ $USE_SLURM + df -h - name: Run QuaC system testing - Exome mode AND no prior QC data run: | + df -h PROJECT_CONFIG="project_2samples" PRIOR_QC_STATUS="no_priorQC" USE_SLURM="" @@ -92,9 +100,11 @@ jobs: --workflow_config "configs/workflow.yaml" \ --exome \ $USE_SLURM + df -h - name: Run QuaC system testing - WGS mode AND uses prior QC data run: | + df -h PROJECT_CONFIG="project_2samples" PRIOR_QC_STATUS="include_priorQC" USE_SLURM="" @@ -108,9 +118,11 @@ jobs: --allow_sample_renaming \ --workflow_config "configs/workflow.yaml" \ $USE_SLURM + df -h - name: Run QuaC system testing - Exome mode AND uses prior QC data run: | + df -h PROJECT_CONFIG="project_2samples" PRIOR_QC_STATUS="include_priorQC" USE_SLURM="" @@ -124,4 +136,5 @@ jobs: --include_prior_qc \ --allow_sample_renaming \ --workflow_config "configs/workflow.yaml" \ - $USE_SLURM \ No newline at end of file + $USE_SLURM + df -h From fde15bb814a41d639b7608496dfd89b642b8e8ce Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Thu, 15 Jun 2023 17:22:40 -0500 Subject: [PATCH 20/25] adds df steps to actions --- .github/workflows/system_testing.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 4f3a325..776ebfe 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -32,8 +32,16 @@ jobs: shell: bash -l {0} steps: + - name: df before checking out repo + run: | + df -h + - name: checkout repository uses: actions/checkout@v2 + + - name: df before conda env creation + run: | + df -h - name: Create quac environment uses: conda-incubator/setup-miniconda@v2 @@ -43,7 +51,7 @@ jobs: auto-activate-base: false activate-environment: quac environment-file: configs/env/quac.yaml - + - name: Check conda solution run: | df -h From 8756a5242cd975f21e70f9ad95933e430f810662 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Thu, 15 Jun 2023 17:44:40 -0500 Subject: [PATCH 21/25] trying try-catch in bash --- .github/workflows/system_testing.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 776ebfe..732499e 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -90,8 +90,8 @@ jobs: --outdir "data/quac/results/test_${PROJECT_CONFIG}_wgs-${PRIOR_QC_STATUS}/analysis" \ --quac_watch_config "configs/quac_watch/wgs_quac_watch_config.yaml" \ --workflow_config "configs/workflow.yaml" \ - $USE_SLURM - df -h + $USE_SLURM || df -h + - name: Run QuaC system testing - Exome mode AND no prior QC data run: | @@ -107,8 +107,7 @@ jobs: --quac_watch_config "configs/quac_watch/exome_quac_watch_config.yaml" \ --workflow_config "configs/workflow.yaml" \ --exome \ - $USE_SLURM - df -h + $USE_SLURM || df -h - name: Run QuaC system testing - WGS mode AND uses prior QC data run: | @@ -125,8 +124,7 @@ jobs: --include_prior_qc \ --allow_sample_renaming \ --workflow_config "configs/workflow.yaml" \ - $USE_SLURM - df -h + $USE_SLURM || df -h - name: Run QuaC system testing - Exome mode AND uses prior QC data run: | @@ -144,5 +142,4 @@ jobs: --include_prior_qc \ --allow_sample_renaming \ --workflow_config "configs/workflow.yaml" \ - $USE_SLURM - df -h + $USE_SLURM || df -h From ce4a420ed3a74b048b8423ef52eafbc381d078da Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Tue, 20 Jun 2023 12:50:15 -0500 Subject: [PATCH 22/25] adds step to free disk space --- .github/workflows/system_testing.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 732499e..36d89bd 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -76,7 +76,13 @@ jobs: run: | df -h bash src/setup_dependency_datasets.sh - + + - name: Frees Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@v1.2.0 + with: + # this might remove tools that are actually needed, when set to "true" + tool-cache: true + - name: Run QuaC system testing - WGS mode AND no prior QC data run: | df -h From 126f5be3e4b00fbfa3466ac68d585cc262f0b0e9 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Tue, 20 Jun 2023 13:13:09 -0500 Subject: [PATCH 23/25] moves up task to free space --- .github/workflows/system_testing.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index 36d89bd..b6db3b7 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -36,6 +36,12 @@ jobs: run: | df -h + - name: Frees Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@v1.2.0 + with: + # this might remove tools that are actually needed, when set to "true" + tool-cache: true + - name: checkout repository uses: actions/checkout@v2 @@ -77,12 +83,6 @@ jobs: df -h bash src/setup_dependency_datasets.sh - - name: Frees Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.2.0 - with: - # this might remove tools that are actually needed, when set to "true" - tool-cache: true - - name: Run QuaC system testing - WGS mode AND no prior QC data run: | df -h From 0c23956a2129a4bfcc5074d8a52dceea7dad3791 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Tue, 20 Jun 2023 15:42:37 -0500 Subject: [PATCH 24/25] removes storage calc steps --- .github/workflows/system_testing.yml | 59 +++++++++++----------------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index b6db3b7..cc6b001 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -1,20 +1,19 @@ name: system_testing -# on: [push, pull_request, workflow_dispatch] on: push: paths: - - '.github/workflows/system_testing.yml' - - '.test/**' - - 'configs/**' - - 'src/**' - - 'workflow/**' + - ".github/workflows/system_testing.yml" + - ".test/**" + - "configs/**" + - "src/**" + - "workflow/**" pull_request: paths: - - '.github/workflows/system_testing.yml' - - '.test/**' - - 'configs/**' - - 'src/**' - - 'workflow/**' + - ".github/workflows/system_testing.yml" + - ".test/**" + - "configs/**" + - "src/**" + - "workflow/**" types: - opened - synchronize @@ -30,25 +29,18 @@ jobs: defaults: run: shell: bash -l {0} - - steps: - - name: df before checking out repo - run: | - df -h + steps: - name: Frees Disk Space (Ubuntu) + # For more info about this task, see https://github.com/uab-cgds-worthey/quac/issues/78 uses: jlumbroso/free-disk-space@v1.2.0 with: # this might remove tools that are actually needed, when set to "true" tool-cache: true - - name: checkout repository + - name: Checkout repository uses: actions/checkout@v2 - - name: df before conda env creation - run: | - df -h - - name: Create quac environment uses: conda-incubator/setup-miniconda@v2 with: @@ -60,32 +52,27 @@ jobs: - name: Check conda solution run: | - df -h mamba env export - df -h - + - name: Check snakemake exists in conda env run: | - df -h which snakemake snakemake --version - uses: eWaterCycle/setup-singularity@v7 with: singularity-version: 3.8.3 + - name: Check singularity is working run: | - df -h singularity --version - name: Set up dependencies for QuaC run: | - df -h bash src/setup_dependency_datasets.sh - name: Run QuaC system testing - WGS mode AND no prior QC data run: | - df -h PROJECT_CONFIG="project_2samples" PRIOR_QC_STATUS="no_priorQC" USE_SLURM="" @@ -96,12 +83,11 @@ jobs: --outdir "data/quac/results/test_${PROJECT_CONFIG}_wgs-${PRIOR_QC_STATUS}/analysis" \ --quac_watch_config "configs/quac_watch/wgs_quac_watch_config.yaml" \ --workflow_config "configs/workflow.yaml" \ - $USE_SLURM || df -h - + $USE_SLURM \ + || df -h - name: Run QuaC system testing - Exome mode AND no prior QC data run: | - df -h PROJECT_CONFIG="project_2samples" PRIOR_QC_STATUS="no_priorQC" USE_SLURM="" @@ -113,11 +99,11 @@ jobs: --quac_watch_config "configs/quac_watch/exome_quac_watch_config.yaml" \ --workflow_config "configs/workflow.yaml" \ --exome \ - $USE_SLURM || df -h + $USE_SLURM \ + || df -h - name: Run QuaC system testing - WGS mode AND uses prior QC data run: | - df -h PROJECT_CONFIG="project_2samples" PRIOR_QC_STATUS="include_priorQC" USE_SLURM="" @@ -130,11 +116,11 @@ jobs: --include_prior_qc \ --allow_sample_renaming \ --workflow_config "configs/workflow.yaml" \ - $USE_SLURM || df -h + $USE_SLURM \ + || df -h - name: Run QuaC system testing - Exome mode AND uses prior QC data run: | - df -h PROJECT_CONFIG="project_2samples" PRIOR_QC_STATUS="include_priorQC" USE_SLURM="" @@ -148,4 +134,5 @@ jobs: --include_prior_qc \ --allow_sample_renaming \ --workflow_config "configs/workflow.yaml" \ - $USE_SLURM || df -h + $USE_SLURM \ + || df -h From 0e0ceeccaaf80cf1c969313b3e15c29fc4cdf6b9 Mon Sep 17 00:00:00 2001 From: Manavalan Gajapathy Date: Wed, 21 Jun 2023 10:41:20 -0500 Subject: [PATCH 25/25] turns off sync PR trigger to avoid duplicate CI trigger - #79 --- .github/workflows/system_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/system_testing.yml b/.github/workflows/system_testing.yml index cc6b001..4fd80cf 100644 --- a/.github/workflows/system_testing.yml +++ b/.github/workflows/system_testing.yml @@ -16,7 +16,7 @@ on: - "workflow/**" types: - opened - - synchronize + # - synchronize - reopened - ready_for_review - converted_to_draft