migrate to pygscatalog utilities (#296)

* add correlation test * add correlation action * fix download URL * use scoring files from correlation archive * get test profile working with pygscatalog * integration updates * fix correlation scorefile wildcard * fix tests * update plink2 * gzip afreq in plink2_vcf * update custom scoring files for liftover * fix match module test * use local files in test suite * fix singularity container definition * check for environment variables with set -euxo * logs are massive, don't upload, debug locally
PGScatalog · May 23, 2024 · b2de82d · b2de82d
1 parent 4ef8794
commit b2de82d
Show file tree

Hide file tree

Showing 60 changed files with 2,153 additions and 353 deletions.
diff --git a/.github/workflows/ancestry-conda.yml b/.github/workflows/ancestry-conda.yml
@@ -61,14 +61,3 @@ jobs:
 
       - name: Run ancestry test
         run: TMPDIR=~ PROFILE=mamba pytest --kwdof --symlink --git-aware --wt 2 --tag "ancestry" --ignore tests/bin
-
-      - name: Upload logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v3
-        with:
-          name: logs-conda-ancestry
-          path: |
-            /home/runner/pytest_workflow_*/*/.nextflow.log
-            /home/runner/pytest_workflow_*/*/log.out
-            /home/runner/pytest_workflow_*/*/log.err
-            /home/runner/pytest_workflow_*/*/output/*
diff --git a/.github/workflows/ancestry-vcf.yml b/.github/workflows/ancestry-vcf.yml
@@ -147,14 +147,3 @@ jobs:
 
       - name: Run ancestry test
         run: TMPDIR=~ PROFILE=singularity pytest --kwdof --symlink --git-aware --wt 2 --tag "ancestry vcf" --ignore tests/bin
-
-      - name: Upload logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v3
-        with:
-          name: logs-singularity-ancestry
-          path: |
-            /home/runner/pytest_workflow_*/*/.nextflow.log
-            /home/runner/pytest_workflow_*/*/log.out
-            /home/runner/pytest_workflow_*/*/log.err
-            /home/runner/pytest_workflow_*/*/output/*
diff --git a/.github/workflows/ancestry.yml b/.github/workflows/ancestry.yml
@@ -137,14 +137,3 @@ jobs:
         run: TMPDIR=~ PROFILE=singularity pytest --kwdof --symlink --git-aware --wt 2 --tag "ancestry" --ignore tests/bin
         env:
           TMPDIR: ${{ runner.temp }}
-
-      - name: Upload logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v3
-        with:
-          name: logs-singularity-ancestry
-          path: |
-            /home/runner/pytest_workflow_*/*/.nextflow.log
-            /home/runner/pytest_workflow_*/*/log.out
-            /home/runner/pytest_workflow_*/*/log.err
-            /home/runner/pytest_workflow_*/*/output/*
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,7 +9,6 @@ on:
     branches:
       - dev
       - main
-      - fix_vcf
   release:
     types: [published]
 

diff --git a/.github/workflows/correlation-test.yml b/.github/workflows/correlation-test.yml
@@ -0,0 +1,35 @@
+name: Correlation test
+on:
+  push:
+    branches:
+      - correlation
+      - ci
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  preload_correlation:
+    uses: ./.github/workflows/preload-correlation.yml
+
+  preload_docker:
+    uses: ./.github/workflows/preload-docker.yml
+
+  preload_singularity:
+    uses: ./.github/workflows/preload-singularity.yml
+
+  correlation_docker:
+    needs: [preload_docker, preload_correlation]
+    uses: ./.github/workflows/correlation.yml
+    with:
+      container-cache-key: ${{ needs.preload_docker.outputs.cache-key }}
+      correlation-cache-key: ${{ needs.preload_correlation.outputs.cache-key }}
+      docker: true
+
+  correlation_singularity:
+    needs: [preload_singularity, preload_correlation]
+    uses: ./.github/workflows/correlation.yml
+    with:
+      container-cache-key: ${{ needs.preload_singularity.outputs.cache-key }}
+      correlation-cache-key: ${{ needs.preload_correlation.outputs.cache-key }}
+      singularity: true    
diff --git a/.github/workflows/correlation.yml b/.github/workflows/correlation.yml
@@ -0,0 +1,158 @@
+name: Run correlation test with singularity or docker profiles
+
+on:
+  workflow_call:
+    inputs:
+      container-cache-key:
+        type: string
+        required: true
+      correlation-cache-key:
+        type: string
+        required: true
+      docker:
+        type: boolean
+      singularity:
+        type: boolean
+
+env:
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/singularity
+  SINGULARITY_VERSION: 3.8.3
+
+jobs:
+  docker:
+    if: ${{ inputs.docker }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "CORRELATION_DIR=$RUNNER_TEMP" >> $GITHUB_ENV
+      
+      - name: Check out pipeline code
+        uses: actions/checkout@v4
+
+      - uses: nf-core/setup-nextflow@v2
+
+      - name: Restore docker images
+        id: restore-docker
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ runner.temp }}/docker
+          key: ${{ inputs.container-cache-key }}
+          fail-on-cache-miss: true
+
+      - name: Load docker images from cache
+        run: |
+          find $HOME -name '*.tar'
+          find ${{ runner.temp }}/docker/ -name '*.tar' -exec sh -c 'docker load < {}' \;
+
+      - name: Restore reference data
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ env.CORRELATION_DIR }}/correlation37.pgen
+            ${{ env.CORRELATION_DIR }}/correlation37.psam
+            ${{ env.CORRELATION_DIR }}/correlation37.pvar.zst
+            ${{ env.CORRELATION_DIR }}/PGS000018_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000027_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000137_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000727_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000728_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000729_hmPOS_GRCh37.txt.gz            
+          key: ${{ inputs.correlation-cache-key }}
+          fail-on-cache-miss: true
+
+      - name: Set up test requirements
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - run: pip install -r ${{ github.workspace }}/tests/requirements.txt
+
+      - name: Run correlation test
+        run: TMPDIR=~ PROFILE=docker pytest --kwdof --symlink --git-aware --wt 2 --tag "test score correlation"
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: logs-singularity-ancestry
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/output/*
+            
+  singularity:
+    if: ${{ inputs.singularity }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "CORRELATION_DIR=$RUNNER_TEMP" >> $GITHUB_ENV
+      
+      - name: Check out pipeline code
+        uses: actions/checkout@v4
+
+      - uses: nf-core/setup-nextflow@v2
+
+      - name: Restore singularity setup
+        id: restore-singularity-setup
+        uses: actions/cache@v4
+        with:
+          path: /opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64
+          key: ${{ runner.os }}-singularity-${{ env.SINGULARITY_VERSION }}
+          fail-on-cache-miss: true
+
+      - name: Add singularity to path
+        run: |
+          echo "/opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64/bin" >> $GITHUB_PATH
+
+      - name: Restore singularity container images
+        id: restore-singularity
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.NXF_SINGULARITY_CACHEDIR }}
+          key: ${{ inputs.container-cache-key }}
+
+      - name: Restore reference data
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ env.CORRELATION_DIR }}/correlation37.pgen
+            ${{ env.CORRELATION_DIR }}/correlation37.psam
+            ${{ env.CORRELATION_DIR }}/correlation37.pvar.zst
+            ${{ env.CORRELATION_DIR }}/PGS000018_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000027_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000137_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000727_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000728_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000729_hmPOS_GRCh37.txt.gz
+          key: ${{ inputs.correlation-cache-key }}
+          fail-on-cache-miss: true
+
+      - name: Set up test requirements
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - run: pip install -r ${{ github.workspace }}/tests/requirements.txt
+
+      - name: Run correlation test
+        run: TMPDIR=~ PROFILE=singularity pytest --kwdof --symlink --git-aware --wt 2 --tag "test score correlation"
+        env:
+          TMPDIR: ${{ runner.temp }}
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: logs-singularity-ancestry
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/output/*
diff --git a/.github/workflows/preload-correlation.yml b/.github/workflows/preload-correlation.yml
@@ -0,0 +1,38 @@
+name: Preload correlation data
+
+on:
+  workflow_call:
+    outputs:
+      cache-key:
+        value: correlation
+
+jobs:
+  preload_correlation:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "CORRELATION_DIR=$RUNNER_TEMP" >> $GITHUB_ENV
+
+      - name: Cache reference data
+        id: cache-ref
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.CORRELATION_DIR }}/correlation37.pgen
+            ${{ env.CORRELATION_DIR }}/correlation37.psam
+            ${{ env.CORRELATION_DIR }}/correlation37.pvar.zst
+            ${{ env.CORRELATION_DIR }}/PGS000018_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000027_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000137_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000727_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000728_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000729_hmPOS_GRCh37.txt.gz            
+          key: correlation
+
+      - name: Download reference data
+        if: steps.cache-ref.outputs.cache-hit != 'true'        
+        run: |
+          wget -qnc -P $CORRELATION_DIR https://ftp.ebi.ac.uk/pub/databases/spot/pgs/resources/correlation.tar.zst
+          tar -xf $CORRELATION_DIR/correlation.tar.zst -C $CORRELATION_DIR
+          
diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,4 @@ testing*
 assets/report/renv/
 assets/report/report.Rproj
 .Rprofile
+tests/.venv/
diff --git a/assets/examples/scorefiles/customgrch37.txt b/assets/examples/scorefiles/customgrch37.txt
@@ -1,4 +1,5 @@
 #pgs_name=testlift
+#pgs_id=custom
 #trait_reported=test
 #genome_build=GRCh37
 chr_name	chr_position	effect_allele	other_allele	effect_weight

diff --git a/assets/examples/scorefiles/customgrch38.txt b/assets/examples/scorefiles/customgrch38.txt
@@ -1,4 +1,5 @@
 #pgs_name=test
+#pgs_id=custom
 #trait_reported=test
 #genome_build=GRCh38
 chr_name	chr_position	effect_allele	other_allele	effect_weight
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,7 +9,6 @@ on: @@
         branches:
           - dev
           - main
-          - fix_vcf
       release:
         types: [published]
@@ Expand Down @@