Oushujun master

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,19 +1,18 @@
 <!--
-# jguhlin/EDTA pull request
 
-Many thanks for contributing to jguhlin/EDTA!
+Many thanks for contributing to EDTA!
 
 Please fill in the appropriate checklist below (delete whatever is not relevant).
 These are the most common things requested on pull requests (PRs).
 
-PRs should be made against the nextflow_reboot branch.
 -->
 
 ## PR checklist
 
-- [ ] PR to `nextflow_reboot` branch
-- [ ] `conda` and `container` directives.
-- [ ] Docker container + singularity container (optional)
+For Nextflow implementation,
+
+- [ ] `conda` and `container` directives are included for each process
+- [ ] Docker container + singularity container (optional) are included for each process
 - [ ] Flow `meta.id` with each data channel
 - [ ] Use nf-core resource labels such as `process_high`
 - [ ] Used nf-core module

.github/include.yaml

@@ -0,0 +1,10 @@
+".":
+  - ./.github/workflows/**
+  - ./nf-test.config
+tests:
+  - ./assets/*
+  - ./bin/*
+  - ./conf/*
+  - ./main.nf
+  - ./nextflow_schema.json
+  - ./nextflow.config

.github/workflows/ci.yml

@@ -1,14 +1,12 @@
 name: EDTA Nextflow CI
 on:
-  push:
-    branches:
-      - nextflow_reboot
   pull_request:
-    branches:
-      - nextflow_reboot
 
 env:
   NXF_ANSI_LOG: false
+  NFT_WORKDIR: "~"
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
   NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
 
@@ -17,62 +15,93 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  nf-test-changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      nf_test_files: ${{ steps.list.outputs.components }}
+    steps:
+      - uses: actions/checkout@v4.2.1
+        with:
+          fetch-depth: 0
+
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/detect-nf-test-changes@v0.0.4
+        with:
+          head: ${{ github.sha }}
+          base: origin/${{ github.base_ref }}
+          include: .github/include.yaml
+
+      - name: print list of nf-test files
+        run: |
+          echo ${{ steps.list.outputs.components }}
+
   test:
-    name: Run pipeline with test data
-    # Only run on push if this is the jguhlin nextflow_reboot branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'jguhlin/EDTA') }}"
+    name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }}
+    needs: [nf-test-changes]
+    if: needs.nf-test-changes.outputs.nf_test_files != '[]'
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        nextflow:
-          - '24.04.4'
-        profile:
-          - docker
-          - singularity
-          - conda
+        NXF_VER:
+          - "24.04.2"
+
+        nf_test_files: ["${{ fromJson(needs.nf-test-changes.outputs.nf_test_files) }}"]
+        profile: [conda, docker, singularity]
 
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        uses: actions/checkout@v4.2.1
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
-          version: ${{ matrix.nextflow }}
+          version: "${{ matrix.NXF_VER }}"
 
-      # - name: Disk space cleanup
-      #   uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
-      # Will be needed with large data sets. Can take long to run
+      - uses: actions/setup-python@v5.2.0
+        with:
+          python-version: "3.11"
+          architecture: "x64"
 
+      - name: Install pdiff to see diff between nf-test snapshots
+        run: |
+          python -m pip install --upgrade pip
+          pip install pdiff
+
+      - uses: nf-core/setup-nf-test@v1.1.2
+        with:
+          version: 0.9.0
+
       - name: Setup apptainer
         if: matrix.profile == 'singularity'
         uses: eWaterCycle/setup-apptainer@main
-      
+
       - name: Set up Singularity
         if: matrix.profile == 'singularity'
         run: |
           mkdir -p $NXF_SINGULARITY_CACHEDIR
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
-
-      - name: Set up miniconda
-        if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
-        with:
-          miniconda-version: "latest"
-          auto-update-conda: true
-          channels: conda-forge,bioconda
 
-      - name: Conda setup
-        if: matrix.profile == 'conda'
+      - name: Run nf-test
         run: |
-          conda clean -a
-          conda install -n base conda-libmamba-solver
-          conda config --set solver libmamba
-          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
-          echo $(realpath python) >> $GITHUB_PATH
+          nf-test test --verbose ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}"
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
 
-      - name: Run pipeline with test data
+      - name: debug-print
+        if: always()
         run: |
-          nextflow run \
-            ${GITHUB_WORKSPACE} \
-            -profile ${{ matrix.profile }},test
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

.gitignore

@@ -3,3 +3,5 @@ bin/TIR-Learner*/Module3/Maize_model.sav
 work/
 .nextflow.log*
 .nextflow/*
+results/
+.nf-test*

cleanNXF.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+rm -rf .nextflow*
+echo "Cleaned .nextflow..."
+rm -rf .nextflow.pid
+echo "Cleaned .nextflow.pid..."
+for i in $(ls work | grep -v "conda");
+do
+    rm -rf "work/$i"
+done
+echo "Cleaned work..."
+
+rm -f .nf-test.log
+rm -rf .nf-test
+echo "Cleaned nf-test..."

conf/modules.config

@@ -0,0 +1,42 @@
+process {
+    withName: 'EDTA:LTRHARVEST' {
+        ext.args = '-size 1000000 -time 300'
+        ext.prefix = { "${meta.id}_ltrharvest" }
+    }
+
+    withName: 'EDTA:LTRFINDER' {
+        ext.args = '-harvest_out -size 1000000 -time 300'
+    }
+
+    withName: 'EDTA:CAT_CAT' {
+        ext.prefix = { "${meta.id}_ltrharvest_ltrfinder.tabout" }
+    }
+
+    withName: 'EDTA:ANNOSINE' {
+        ext.args = '--num_alignments 50000 -rpm 0 --copy_number 3 --shift 100 -auto 1'
+    }
+
+    withName: 'EDTA:REPEATMODELER_REPEATMODELER' {
+        ext.args = '-engine ncbi'
+    }
+
+    withName: 'EDTA:FASTA_HELITRONSCANNER_SCAN_DRAW:HELITRONSCANNER_DRAW' {
+        ext.args = '-pure_helitron'
+    }
+
+    withName: 'EDTA:FASTA_HELITRONSCANNER_SCAN_DRAW:HELITRONSCANNER_SCAN_HEAD_RC' {
+        ext.prefix = { "${meta.id}.rc" }
+        ext.args = '--rc'
+    }
+
+    withName: 'EDTA:FASTA_HELITRONSCANNER_SCAN_DRAW:HELITRONSCANNER_SCAN_TAIL_RC' {
+        ext.prefix = { "${meta.id}.rc" }
+        ext.args = '--rc'
+    }
+
+    withName: 'EDTA:FASTA_HELITRONSCANNER_SCAN_DRAW:HELITRONSCANNER_DRAW_RC' {
+        ext.prefix = { "${meta.id}.rc" }
+        ext.args = '-pure_helitron'
+    }
+
+}

conf/test.config

@@ -0,0 +1,11 @@
+process {
+    resourceLimits = [
+        cpus: 4,
+        memory: '15.GB',
+        time: '1.h'
+    ]
+}
+
+params {
+    genomes     = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/genome.fasta'
+}

main.nf

@@ -0,0 +1,27 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+params.genomes          = 'genomes/*'
+params.species          = 'others'
+params.cds              = ''
+params.curatedlib       = ''
+params.rmlib            = ''
+params.sensitive        = false
+params.anno             = false
+params.rmout            = ''
+params.maxdiv           = 40
+params.evaluate         = true
+params.exclude          = ''
+params.maxint           = 5000
+params.outdir           = 'results'
+
+include { EDTA          } from './workflows/edta.nf'
+
+// Test run: 
+// ./main.nf -profile docker,test
+// ./main.nf -profile conda,test
+
+workflow {
+    EDTA()
+}

modules.json

@@ -0,0 +1,87 @@
+{
+  "name": "",
+  "homePage": "",
+  "repos": {
+    "https://github.com/GallVp/nxf-components.git": {
+      "modules": {
+        "gallvp": {
+          "annosine": {
+            "branch": "main",
+            "git_sha": "a8939d36280e7d9037c7cf164eeede19e46546a4",
+            "installed_by": ["modules"]
+          },
+          "gunzip": {
+            "branch": "main",
+            "git_sha": "ae9714c21ede9199a3118e3c20b65484aa73e232",
+            "installed_by": ["modules"]
+          },
+          "helitronscanner/draw": {
+            "branch": "main",
+            "git_sha": "929d59d82f2e90fe79eb6f93d1ae739f22a894e1",
+            "installed_by": ["fasta_helitronscanner_scan_draw"]
+          },
+          "helitronscanner/scan": {
+            "branch": "main",
+            "git_sha": "929d59d82f2e90fe79eb6f93d1ae739f22a894e1",
+            "installed_by": ["fasta_helitronscanner_scan_draw"]
+          },
+          "tirlearner": {
+            "branch": "main",
+            "git_sha": "a8939d36280e7d9037c7cf164eeede19e46546a4",
+            "installed_by": ["modules"]
+          }
+        }
+      },
+      "subworkflows": {
+        "gallvp": {
+          "fasta_helitronscanner_scan_draw": {
+            "branch": "main",
+            "git_sha": "970e3af38229845dd38c13d29b0905651a8e61f0",
+            "installed_by": ["subworkflows"]
+          }
+        }
+      }
+    },
+    "https://github.com/nf-core/modules.git": {
+      "modules": {
+        "nf-core": {
+          "cat/cat": {
+            "branch": "master",
+            "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
+            "installed_by": ["modules"]
+          },
+          "gunzip": {
+            "branch": "master",
+            "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
+            "installed_by": ["modules"]
+          },
+          "ltrfinder": {
+            "branch": "master",
+            "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
+            "installed_by": ["modules"]
+          },
+          "ltrharvest": {
+            "branch": "master",
+            "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
+            "installed_by": ["modules"]
+          },
+          "ltrretriever/ltrretriever": {
+            "branch": "master",
+            "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
+            "installed_by": ["modules"]
+          },
+          "repeatmodeler/builddatabase": {
+            "branch": "master",
+            "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
+            "installed_by": ["modules"]
+          },
+          "repeatmodeler/repeatmodeler": {
+            "branch": "master",
+            "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
+            "installed_by": ["modules"]
+          }
+        }
+      }
+    }
+  }
+}

modules/gallvp/annosine/environment.yml

@@ -0,0 +1,6 @@
+channels:
+  - conda-forge
+  - bioconda
+
+dependencies:
+  - "bioconda::annosine2=2.0.7"