Implement correlation test

.github/workflows/ci.yml

@@ -9,7 +9,6 @@ on:
     branches:
       - dev
       - main
-      - fix_vcf
   release:
     types: [published]
 

.github/workflows/correlation-test.yml

@@ -0,0 +1,35 @@
+name: Correlation test
+on:
+  push:
+    branches:
+      - correlation
+      - ci
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  preload_correlation:
+    uses: ./.github/workflows/preload-correlation.yml
+
+  preload_docker:
+    uses: ./.github/workflows/preload-docker.yml
+
+  preload_singularity:
+    uses: ./.github/workflows/preload-singularity.yml
+
+  correlation_docker:
+    needs: [preload_docker, preload_correlation]
+    uses: ./.github/workflows/correlation.yml
+    with:
+      container-cache-key: ${{ needs.preload_docker.outputs.cache-key }}
+      correlation-cache-key: ${{ needs.preload_correlation.outputs.cache-key }}
+      docker: true
+
+  correlation_singularity:
+    needs: [preload_singularity, preload_correlation]
+    uses: ./.github/workflows/correlation.yml
+    with:
+      container-cache-key: ${{ needs.preload_singularity.outputs.cache-key }}
+      correlation-cache-key: ${{ needs.preload_correlation.outputs.cache-key }}
+      singularity: true    

.github/workflows/correlation.yml

@@ -0,0 +1,158 @@
+name: Run correlation test with singularity or docker profiles
+
+on:
+  workflow_call:
+    inputs:
+      container-cache-key:
+        type: string
+        required: true
+      correlation-cache-key:
+        type: string
+        required: true
+      docker:
+        type: boolean
+      singularity:
+        type: boolean
+
+env:
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/singularity
+  SINGULARITY_VERSION: 3.8.3
+
+jobs:
+  docker:
+    if: ${{ inputs.docker }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "CORRELATION_DIR=$RUNNER_TEMP" >> $GITHUB_ENV
+
+      - name: Check out pipeline code
+        uses: actions/checkout@v4
+
+      - uses: nf-core/setup-nextflow@v2
+
+      - name: Restore docker images
+        id: restore-docker
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ runner.temp }}/docker
+          key: ${{ inputs.container-cache-key }}
+          fail-on-cache-miss: true
+
+      - name: Load docker images from cache
+        run: |
+          find $HOME -name '*.tar'
+          find ${{ runner.temp }}/docker/ -name '*.tar' -exec sh -c 'docker load < {}' \;
+
+      - name: Restore reference data
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ env.CORRELATION_DIR }}/correlation37.pgen
+            ${{ env.CORRELATION_DIR }}/correlation37.psam
+            ${{ env.CORRELATION_DIR }}/correlation37.pvar.zst
+            ${{ env.CORRELATION_DIR }}/PGS000018_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000027_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000137_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000727_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000728_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000729_hmPOS_GRCh37.txt.gz            
+          key: ${{ inputs.correlation-cache-key }}
+          fail-on-cache-miss: true
+
+      - name: Set up test requirements
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - run: pip install -r ${{ github.workspace }}/tests/requirements.txt
+
+      - name: Run correlation test
+        run: TMPDIR=~ PROFILE=docker pytest --kwdof --symlink --git-aware --wt 2 --tag "test score correlation"
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: logs-singularity-ancestry
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/output/*
+
+  singularity:
+    if: ${{ inputs.singularity }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "CORRELATION_DIR=$RUNNER_TEMP" >> $GITHUB_ENV
+
+      - name: Check out pipeline code
+        uses: actions/checkout@v4
+
+      - uses: nf-core/setup-nextflow@v2
+
+      - name: Restore singularity setup
+        id: restore-singularity-setup
+        uses: actions/cache@v4
+        with:
+          path: /opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64
+          key: ${{ runner.os }}-singularity-${{ env.SINGULARITY_VERSION }}
+          fail-on-cache-miss: true
+
+      - name: Add singularity to path
+        run: |
+          echo "/opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64/bin" >> $GITHUB_PATH
+
+      - name: Restore singularity container images
+        id: restore-singularity
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.NXF_SINGULARITY_CACHEDIR }}
+          key: ${{ inputs.container-cache-key }}
+
+      - name: Restore reference data
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ${{ env.CORRELATION_DIR }}/correlation37.pgen
+            ${{ env.CORRELATION_DIR }}/correlation37.psam
+            ${{ env.CORRELATION_DIR }}/correlation37.pvar.zst
+            ${{ env.CORRELATION_DIR }}/PGS000018_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000027_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000137_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000727_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000728_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000729_hmPOS_GRCh37.txt.gz
+          key: ${{ inputs.correlation-cache-key }}
+          fail-on-cache-miss: true
+
+      - name: Set up test requirements
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - run: pip install -r ${{ github.workspace }}/tests/requirements.txt
+
+      - name: Run correlation test
+        run: TMPDIR=~ PROFILE=singularity pytest --kwdof --symlink --git-aware --wt 2 --tag "test score correlation"
+        env:
+          TMPDIR: ${{ runner.temp }}
+
+      - name: Upload logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: logs-singularity-ancestry
+          path: |
+            /home/runner/pytest_workflow_*/*/.nextflow.log
+            /home/runner/pytest_workflow_*/*/log.out
+            /home/runner/pytest_workflow_*/*/log.err
+            /home/runner/pytest_workflow_*/*/output/*

.github/workflows/preload-correlation.yml

@@ -0,0 +1,38 @@
+name: Preload correlation data
+
+on:
+  workflow_call:
+    outputs:
+      cache-key:
+        value: correlation
+
+jobs:
+  preload_correlation:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "CORRELATION_DIR=$RUNNER_TEMP" >> $GITHUB_ENV
+
+      - name: Cache reference data
+        id: cache-ref
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.CORRELATION_DIR }}/correlation37.pgen
+            ${{ env.CORRELATION_DIR }}/correlation37.psam
+            ${{ env.CORRELATION_DIR }}/correlation37.pvar.zst
+            ${{ env.CORRELATION_DIR }}/PGS000018_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000027_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000137_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000727_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000728_hmPOS_GRCh37.txt.gz
+            ${{ env.CORRELATION_DIR }}/PGS000729_hmPOS_GRCh37.txt.gz            
+          key: correlation
+
+      - name: Download reference data
+        if: steps.cache-ref.outputs.cache-hit != 'true'        
+        run: |
+          wget -qnc -P $CORRELATION_DIR https://ftp.ebi.ac.uk/pub/databases/spot/pgs/resources/correlation.tar.zst
+          tar -xf $CORRELATION_DIR/correlation.tar.zst -C $CORRELATION_DIR
+

.gitignore

@@ -10,3 +10,4 @@ testing*
 assets/report/renv/
 assets/report/report.Rproj
 .Rprofile
+tests/.venv/

tests/config/nextflow.config

@@ -4,7 +4,7 @@ params {
     enable_conda = false
     singularity_pull_docker_container = false
     min_overlap = 0.75
-    max_memory = "4.GB"
+    max_memory = "7.GB"
     max_cpus = 2
 }
 
@@ -22,6 +22,10 @@ process {
         // prevent file name overlap when composing tests with plink2_vcf input
         ext.prefix = 'vcf_'
     }
+
+    withName: MATCH_COMBINE {
+        memory = 6.5.GB
+    }
 }
 
 def platform = "$PROFILE" == 'arm' ? '--platform linux/arm64' : '--platform linux/amd64'

tests/correlation/samplesheet.csv

@@ -0,0 +1,2 @@
+sampleset,path_prefix,chrom,format
+test,CORRELATION_DIR/correlation37,,pfile

tests/correlation/test_correlation.py

@@ -0,0 +1,29 @@
+import pathlib
+import pytest
+import pandas as pd 
+
+
+@pytest.mark.workflow('test score correlation')
+def test_correlation(workflow_dir: pathlib.Path):
+    """ This test compares calculated polygenic scores (SUM) for PGS000018, PGS000027, PGS000039, PGS000137, PGS000727, PGS000728, and PGS000729 to two reference scores:
+
+    1) Scores calculated with pgsc_calc v2.0.0-alpha.5 
+    2) An independent R script which the workflow was based on developed by Scott Ritchie (see credits in README)
+
+    If the scores don't correlate well, fail loudly.
+    """
+    calculated_score_path = workflow_dir / "output" / "test" / "score" / "aggregated_scores.txt.gz"
+    ref_score_path = workflow_dir / "tests" / "correlation" / "PGS_SUM.txt.gz"
+    pgsc_score_path = workflow_dir / "tests" / "correlation" / "PGS_SUM_alpha5.txt.gz"
+
+    columns = ["PGS", "IID", "SUM"]
+    calculated_df = pd.read_csv(calculated_score_path, sep="\t")[columns].rename(columns={"SUM": "CALC_SUM"})
+    ref_df = pd.read_csv(ref_score_path, sep="\t")[columns].rename(columns={"SUM": "REF_INDEPENDENT_SUM"})
+    pgsc_df = pd.read_csv(pgsc_score_path, sep="\t")[columns].rename(columns={"SUM": "REF_CALC_SUM"})
+    ref_scores = pd.merge(ref_df, pgsc_df, on=["PGS", "IID"], how="left")
+
+    merged_scores = pd.merge(ref_scores, calculated_df, on=["PGS", "IID"], how="left")
+    merged_scores.to_csv(workflow_dir / "output" / "correlations.csv")
+
+    for index, row in merged_scores[["CALC_SUM", "REF_INDEPENDENT_SUM", "REF_CALC_SUM"]].corr().iterrows():
+        assert all(row > 0.999), f"Bad correlation for {index}"

tests/correlation/test_correlation.yml

@@ -0,0 +1,33 @@
+# correlation test notes:
+# need environment variable CORRELATION_DIR set to a path
+# need the correlation archive extracted to $CORRELATION_DIR
+
+- name: test score correlation
+  command: >
+    bash -c " set -euxo pipefail; 
+    sed \"s|CORRELATION_DIR|$CORRELATION_DIR|\" tests/correlation/samplesheet.csv > samplesheet.csv;
+    nextflow run main.nf -c ./tests/config/nextflow.config \
+      --input samplesheet.csv \
+      --target_build GRCh37 \
+      --scorefile "${CORRELATION_DIR}/PGS000018_hmPOS_GRCh37*"
+    "
+  tags:
+    - correlation
+    - slow
+  stdout:
+    contains:
+      - "Pipeline completed successfully"
+  files:
+    - path: output/test/score/aggregated_scores.txt.gz
+      contains:
+        - "sampleset"
+        - "IID"
+        - "PGS"
+        - "SUM"
+        - "DENOM"
+        - "AVG"
+      must_not_contain:
+        - "percentile_MostSimilarPop"
+        - "Z_MostSimilarPop"
+        - "Z_norm1"
+