Prepare version 0.3.0

.github/workflows/devel-docs.yaml

@@ -0,0 +1,27 @@
+name: Build/Publish Develop Docs 
+on:
+  push:
+    branches:
+      - develop
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.10.13
+      - name: Install Dependencies
+        run: |
+          pip install mkdocs-material
+          pip install mike
+      - name: Setup Docs Deploy
+        run: |
+          git config --global user.name "Docs Deploy"
+          git config --global user.email "docs.deploy@example.com"
+      - name: Build Docs Website
+        run: mike deploy --push develop

.github/workflows/main.yaml

@@ -6,17 +6,21 @@ on:
       - master
     paths-ignore:
       - '**/*.md'
+      - 'docs'
+      - 'mkdocs.yml'
   pull_request:
     branches_ignore: []
     paths-ignore:
       - '**/*.md'
+      - 'docs'
+      - 'mkdocs.yml'
 
 jobs:
   formatting:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout with submodules
-      uses: actions/checkout@v3.5.2
+      uses: actions/checkout@v4
     - name: Formatting
       uses: github/super-linter@v5
       env:
@@ -27,7 +31,7 @@ jobs:
   linting:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3.5.2
+    - uses: actions/checkout@v4
     - name: Linting
       uses: snakemake/snakemake-github-action@v1.24.0
       with:
@@ -52,7 +56,7 @@ jobs:
         remove-codeql: 'true'
         remove-docker-images: 'true'
     - name: Checkout repository
-      uses: actions/checkout@v3.5.2
+      uses: actions/checkout@v4
     - name: Set up conda environment and singularity
       uses: mamba-org/setup-micromamba@v1
       with:
@@ -62,9 +66,11 @@ jobs:
     - name: Test workflow
       shell: bash -l {0}
       run: |
+        conda config --set channel_priority strict
         mkdir -p temp
         workdir=$(pwd -P)
         TMPDIR=$workdir/temp
+        snakemake --directory .test --configfile .test/config/config.yaml --config samples=config/samples_testSRA.tsv --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache --use-singularity --default-resources mem_mb=1000 --until fastp_mergedout
         snakemake --directory .test --configfile .test/config/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache --use-singularity --default-resources mem_mb=1000
     - name: Test report
       shell: bash -l {0}

.github/workflows/release-docs.yaml

@@ -0,0 +1,28 @@
+name: Build/Publish Latest Release Docs
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.10.13
+      - name: Install Dependencies
+        run: |
+          pip install mkdocs-material
+          pip install mike
+      - name: Setup Docs Deploy
+        run: |
+          git config --global user.name "Docs Deploy"
+          git config --global user.email "docs.deploy@example.com"
+      - name: Build Docs Website
+        run: mike deploy --push --update-aliases ${{ github.event.release.tag_name }} latest

.gitignore

@@ -27,4 +27,6 @@ reports
 # ignore dev containers for vscode
 .devcontainer
 # ignore benchmarks
-benchmarks
+benchmarks
+# documentation site
+site

.test/config/config.yaml

@@ -18,33 +18,38 @@ reference:
   exclude: []
   min_size: 3000
 
+ancestral: "data/ref/testref.fa"
+
 #===================== Sample Set Configuration =======================#
 
 exclude_ind: []
 
 excl_pca-admix: []
 
-#==================== Downsampling Configuration ======================#
-
-downsample_cov:
-
 #====================== Analysis Selection ============================#
 
 populations: []
 
 analyses:
-  # filtering
-  genmap: true
+  # mapping options
+  mapping:
+    historical_only_collapsed: false
+    historical_collapsed_aligner: "aln"
+  # sites file filters
+  pileup-mappability: true
   repeatmasker:
+    bed:
     local_lib:
     dfam_lib:
     build_lib: true
-  extreme_depth: [0.5, 1.5]
+  extreme_depth: true
   dataset_missing_data: 0.5
   population_missing_data: 0.5
   # quality control
   qualimap: true
+  ibs_ref_bias: true
   damageprofiler: true
+  mapdamage_rescale: false
   # population genomic analyses
   estimate_ld: true
   ld_decay: true
@@ -54,6 +59,39 @@ analyses:
     ngsrelate: true
     ibsrelate_ibs: true
     ibsrelate_sfs: true
+  1dsfs: true
+  1dsfs_boot: true
+  2dsfs: true
+  2dsfs_boot: true
+  thetas_angsd: true
+  heterozygosity_angsd: true
+  fst_angsd:
+    populations: true
+    individuals: true
+  inbreeding_ngsf-hmm: true
+  ibs_matrix: true
+
+#==================== Downsampling Configuration ======================#
+
+subsample_dp: [3, 2]
+
+subsample_by: "sitefilt" # three options: unfilt, mapqbaseq, sitefilt; sitefilt recommended (subsamples bams to `subsample_dp` within filtered regions)
+redo_depth_filts: false # has no effect when `subsample_by` == "sitefilt"
+drop_samples: ["mod1"] # These samples won't be included in the downsampled dataset
+
+subsample_analyses:
+  estimate_ld: true
+  ld_decay: true
+  pca_pcangsd: false
+  admix_ngsadmix: true
+  relatedness:
+    ngsrelate: true
+    ibsrelate_ibs: true
+    ibsrelate_sfs: true
+  1dsfs: true
+  1dsfs_boot: true
+  2dsfs: true
+  2dsfs_boot: true
   thetas_angsd: true
   heterozygosity_angsd: true
   fst_angsd:
@@ -66,7 +104,7 @@ analyses:
 
 filter_beds:
   example: "config/example-filt.bed"
-  badexample: "config/badexample-filt.bed"
+  #badexample: "config/badexample-filt.bed"
 
 only_filter_beds: false
 
@@ -76,31 +114,63 @@ mapQ: 30
 baseQ: 30
 
 params:
+  clipoverlap:
+    clip_user_provided_bams: true
   genmap:
     K: "100"
     E: "2"
     map_thresh: 1
+  extreme_depth_filt:
+    method: "median"
+    bounds: [ 0.5, 1.5 ]
+    filt-on-dataset: true
+    filt-on-depth-classes: true
   fastp:
-    extra: "-p -g --overlap_len_require 15"
+    extra: "-p -g" # don't put --merge or --overlap_len_require here, they're implicit
+    min_overlap_hist: 20
+  bwa_aln:
+    extra: "-l 16500 -n 0.01 -o 2"
+  samtools:
+    subsampling_seed: "$RANDOM"
   picard:
     MarkDuplicates: "--REMOVE_DUPLICATES true --OPTICAL_DUPLICATE_PIXEL_DISTANCE 2500"
+  damageprofiler:
+    profile_modern: false
   angsd:
     gl_model: 2 # gl_model - 1=SAMtools, 2=GATK, 3=SOAPsnp, 4=SYK
-    maxdepth: 1000
-    extra: ""
+    maxdepth: 1000 # for depth calc only
+    mindepthind: 1
+    minind_dataset: 0.75 # as a proportion of N
+    minind_pop: 0.5 # as a proportion of N
+    rmtrans: true
+    extra: "" # goes to all ANGSD runs
+    extra_saf: "" # goes to all -doSaf runs
+    extra_beagle: "" # goes to all -doGlf 2 runs
+    domajorminor: 1 # Currently only compatible with values 1, 2, 4, 5
+    domaf: 1
     snp_pval: "1e-6"
-    min_maf: 0.05
+    min_maf: -1 # Set -1 to disable
+    mindepthind_heterozygosity: 1
+  ngsrelate:
+    prune: true
   ngsld:
     max_kb_dist_est-ld: 200
-    max_kb_dist_decay: 100
-    max_kb_dist_pruning: 50
     rnd_sample_est-ld: 0.001
+    max_kb_dist_decay: 100
     rnd_sample_decay: 0.9
     fit_LDdecay_extra: "--fit_level 3"
     fit_LDdecay_n_correction: true
-    pruning_min-weight: 0.1
+    max_kb_dist_pruning_dataset: 50 # used for PCA, Admix, NGSrelate, not ngsF-HMM
+    pruning_min-weight_dataset: 0.1 # used for PCA, Admix, NGSrelate, not ngsF-HMM
+  ngsf-hmm:
+    estimate_in_pops: false
+    prune: false
+    max_kb_dist_pruning_pop: 50 # used only for ngsf-hmm
+    pruning_min-weight_pop: 0.8 # used only for ngsf-hmm
+    min_roh_length: 50000
+    roh_bins: [ 100000, 250000, 500000, 1000000, 2500000, 5000000 ] # in ascending order, bins are between adjacent values and between min_roh_length and value 1 and the last value and infinity
   realsfs:
-    fold: 1 # Should only be set to 1 for now
+    fold: 1 # Should only be set to 1, unless ancestral reference is given above
     sfsboot: 100
   fst:
     whichFst: 1

.test/config/samples_testSRA.tsv

@@ -0,0 +1,8 @@
+sample	population	time	depth
+hist1	Hjelmseryd	historical	low
+hist2	Hjelmseryd	historical	low
+#hist3	Hjelmseryd	historical	low
+mod1	Gotafors	modern	high
+mod2	Gotafors	modern	high
+#mod3	Gotafors	modern	high
+SAMN13218652	Gotafors	modern	high

.test/config/units.tsv

@@ -1,8 +1,9 @@
-sample	unit	lib	platform	fq1	fq2	realname
-hist1	BHVN22DSX2.2	hist1	ILLUMINA	data/fastq/hist1.r1.fastq.gz	data/fastq/hist1.r2.fastq.gz	MZLU153040
-hist1	BHVN22DSX2.3	hist1	ILLUMINA	data/fastq/hist1.unit2.r1.fastq.gz	data/fastq/hist1.unit2.r2.fastq.gz	MZLU153040
-hist2	BHVN22DSX2.2	hist2	ILLUMINA	data/fastq/hist2.r1.fastq.gz	data/fastq/hist2.r2.fastq.gz	MZLU153042
-hist3	BHVN22DSX2.2	hist2	ILLUMINA	data/fastq/hist3.r1.fastq.gz	data/fastq/hist3.r2.fastq.gz	MZLU153045
-mod1	AHW5NGDSX2.3	mod1	ILLUMINA	data/fastq/mod1.r1.fastq.gz	data/fastq/mod1.r2.fastq.gz	MZLU107434
-mod2	AHW5NGDSX2.3	mod2	ILLUMINA	data/fastq/mod2.r1.fastq.gz	data/fastq/mod2.r2.fastq.gz	MZLU107435
-mod3	AHW5NGDSX2.3	mod3	ILLUMINA	data/fastq/mod3.r1.fastq.gz	data/fastq/mod3.r2.fastq.gz	MZLU107436
+sample	unit	lib	platform	fq1	fq2	bam	sra	realname
+hist1	BHVN22DSX2.2	hist1	ILLUMINA	data/fastq/hist1.r1.fastq.gz	data/fastq/hist1.r2.fastq.gz			MZLU153040
+hist1	BHVN22DSX2.3	hist1a	ILLUMINA	data/fastq/hist1.unit2.r1.fastq.gz	data/fastq/hist1.unit2.r2.fastq.gz			MZLU153040
+hist2	BHVN22DSX2.2	hist2	ILLUMINA	data/fastq/hist2.r1.fastq.gz	data/fastq/hist2.r2.fastq.gz			MZLU153042
+hist3	BHVN22DSX2.2	hist2	ILLUMINA	data/fastq/hist3.r1.fastq.gz	data/fastq/hist3.r2.fastq.gz			MZLU153045
+mod1	AHW5NGDSX2.3	mod1	ILLUMINA	data/fastq/mod1.r1.fastq.gz	data/fastq/mod1.r2.fastq.gz			MZLU107434
+mod2	AHW5NGDSX2.3	mod2	ILLUMINA	data/fastq/mod2.r1.fastq.gz	data/fastq/mod2.r2.fastq.gz	data/bam/mod2.bam		MZLU107435
+mod3	AHW5NGDSX2.3	mod3	ILLUMINA	data/fastq/mod3.r1.fastq.gz	data/fastq/mod3.r2.fastq.gz			MZLU107436
+SAMN13218652	SRR10398077	SAMN13218652	ILLUMINA				SRR10398077	SAMN13218652 from NCBI, for testing NCBI downloading. Is a random, smallish paired end run fastq