Merge pull request #119 from CCBR/fix-binds

fix: set singularity bind paths

CHANGELOG.md

@@ -1,6 +1,9 @@
 ## CHARLIE development version
 
-- Fix bug where CHARLIE was falsely throwing a file permissions error for tempdir values containing bash variables. (#118, @kelly-sovacool)
+### bug fixes
+
+- CHARLIE was falsely throwing a file permissions error for tempdir values containing bash variables. (#118, @kelly-sovacool)
+- Singularity bind paths were not being set properly. (#119, @kelly-sovacool)
 
 ## CHARLIE 0.11.0
 

charlie

@@ -220,7 +220,8 @@ sed -e "s/PIPELINE_HOME/${PIPELINE_HOME//\//\\/}/g" \
     -e "s/HOST/${HOST}/g" \
     -e "s/ADDITIVES/${ADDITIVES}/g" \
     -e "s/VIRUSES/${VIRUSES}/g" \
-    ${PIPELINE_HOME}/config/$PLATFORM/config.yaml > $CONFIGFILE
+    ${PIPELINE_HOME}/config/$PLATFORM/config.yaml |\
+    cat - ${PIPELINE_HOME}/config/containers.yaml > $CONFIGFILE
 fi
 if [ ! -f $WORKDIR/nclscan.config ];then
 sed -e "s/PIPELINE_HOME/${PIPELINE_HOME//\//\\/}/g" -e "s/WORKDIR/${WORKDIR//\//\\/}/g" ${PIPELINE_HOME}/resources/NCLscan.config.template > $WORKDIR/nclscan.config
@@ -293,7 +294,9 @@ function reconfig(){
   # rebuild config file and replace the config.yaml in the WORKDIR
   # this is only for dev purposes when new key-value pairs are being added to the config file
   check_essential_files
-  sed -e "s/PIPELINE_HOME/${PIPELINE_HOME//\//\\/}/g" -e "s/WORKDIR/${WORKDIR//\//\\/}/g" ${PIPELINE_HOME}/config/config.yaml > $WORKDIR/config.yaml
+  sed -e "s/PIPELINE_HOME/${PIPELINE_HOME//\//\\/}/g" -e "s/WORKDIR/${WORKDIR//\//\\/}/g" \
+  ${PIPELINE_HOME}/config/config.yaml |\
+  cat - ${PIPELINE_HOME}/config/containers.yaml > $WORKDIR/config.yaml
   echo "$WORKDIR/config.yaml has been updated!"
 }
 
@@ -355,21 +358,10 @@ function unlock() {
 # SET SINGULARITY BINDS ... bind required singularity folders appropriately
 ##########################################################################################
 
-function set_singularity_binds(){
-  # this functions tries find what folders to bind
-  # TODO parse config file with pyyaml to determine singularity bind paths
-  echo "$PIPELINE_HOME" > ${WORKDIR}/tmp1
-  echo "$WORKDIR" >> ${WORKDIR}/tmp1
-  grep -o '\/.*' <(cat ${WORKDIR}/config.yaml ${WORKDIR}/samples.tsv)|dos2unix|tr '\t' '\n'|grep -v ' \|\/\/'|sort|uniq >> ${WORKDIR}/tmp1
-  grep gpfs ${WORKDIR}/tmp1|awk -F'/' -v OFS='/' '{print $1,$2,$3,$4,$5}'| grep "[a-zA-Z0-9]" |sort|uniq > ${WORKDIR}/tmp2
-  grep -v gpfs ${WORKDIR}/tmp1|awk -F'/' -v OFS='/' '{print $1,$2,$3}'| grep "[a-zA-Z0-9]"|sort|uniq > ${WORKDIR}/tmp3
-  while read a;do readlink -f $a;done < ${WORKDIR}/tmp3 | grep "[a-zA-Z0-9]"> ${WORKDIR}/tmp4
-  binds=$(cat ${WORKDIR}/tmp2 ${WORKDIR}/tmp3 ${WORKDIR}/tmp4|sort|uniq |tr '\n' ',')
-  rm -f ${WORKDIR}/tmp?
-  binds=$(echo $binds|awk '{print substr($1,1,length($1)-1)}')
+function set_singularity_binds() {
+  binds=$( $PIPELINE_HOME/workflow/scripts/set_singularity_bind_paths.py ${WORKDIR}/config.yaml ${WORKDIR}/samples.tsv)
   SINGULARITY_BINDS="-B $EXTRA_SINGULARITY_BINDS,$binds"
 }
-
 ##########################################################################################
 # PRINT SINGULARITY BINDS ... print bound singularity folders for debugging
 ##########################################################################################
@@ -642,7 +634,7 @@ function main(){
       ;;
   esac
   done
-  WORKDIR=$(readlink -f "$WORKDIR")
+
   echo "Working Dir: $WORKDIR"
 
   if [[ -z "$SING_CACHE_DIR" ]]; then

config/biowulf/config.yaml

@@ -4,7 +4,7 @@
 workdir: "WORKDIR"
 
 # temporary directory for intermediate files that are not saved
-tempdir: '/lscratch/$SLURM_JOB_ID'
+tempdir: "/lscratch/$SLURM_JOB_ID"
 
 # tab delimited samples file ... should have the following 3 columns
 # sampleName	path_to_R1_fastq	path_to_R2_fastq
@@ -112,20 +112,3 @@ fastas_gtfs_dir: "/data/CCBR_Pipeliner/db/PipeDB/charlie/fastas_gtfs"
 annotation_lookups:
   hg38: "PIPELINE_HOME/resources/hg38_2_hg19_lookup.txt"
   mm39: "PIPELINE_HOME/resources/mm39_circBase_annotation_lookup.txt"
-
-containers:
-  base: "docker://nciccbr/ccbr_ubuntu_base_20.04:v6"
-  bowtie1: "docker://nciccbr/charlie_bowtie1:v0.1.0"
-  circexplorer: "docker://nciccbr/ccbr_circexplorer:v1.0"
-  circRNA_finder: "docker://nciccbr/charlie_circrna_finder:v1"
-  ciri: "docker://nciccbr/charlie_ciri2:v1"
-  clear: "docker://nciccbr/ccbr_clear:2"
-  cutadapt: "docker://nciccbr/charlie_cutadapt_fqfilter:v1"
-  dcc: "docker://nciccbr/charlie_dcc:v0.2.0"
-  fastqc: "docker://nciccbr/ccrgb_qctools:v4.0"
-  mapsplice: "docker://cgrlab/mapsplice2:latest"
-  multiqc: "docker://nciccbr/ccbr_multiqc_1.15:v1"
-  picard: "docker://nciccbr/ccbr_picard_2.27.5:v1"
-  R: "docker://nciccbr/ccbr_r_4.3.0:v1"
-  star: "docker://nciccbr/ccbr_star_2.7.6a:latest"
-  star_ucsc_cufflinks: "docker://nciccbr/charlie_star_ucsc_cufflinks:v0.4.0"

config/containers.yaml

@@ -0,0 +1,16 @@
+containers:
+  base: "docker://nciccbr/ccbr_ubuntu_base_20.04:v6"
+  bowtie1: "docker://nciccbr/charlie_bowtie1:v0.1.1"
+  circexplorer: "docker://nciccbr/ccbr_circexplorer:v1.0"
+  circRNA_finder: "docker://nciccbr/charlie_circrna_finder:v1.0.1"
+  ciri: "docker://nciccbr/charlie_ciri2:v1.0.1"
+  clear: "docker://nciccbr/ccbr_clear:v2.0.1"
+  cutadapt: "docker://nciccbr/charlie_cutadapt_fqfilter:v1.0.1"
+  dcc: "docker://nciccbr/charlie_dcc:v0.2.1"
+  fastqc: "docker://nciccbr/ccrgb_qctools:v4.0"
+  mapsplice: "docker://cgrlab/mapsplice2:latest"
+  multiqc: "docker://nciccbr/ccbr_multiqc_1.15:v1"
+  picard: "docker://nciccbr/ccbr_picard_2.27.5:v1"
+  R: "docker://nciccbr/ccbr_r_4.3.0:v1"
+  star: "docker://nciccbr/ccbr_star_2.7.6a:latest"
+  star_ucsc_cufflinks: "docker://nciccbr/charlie_star_ucsc_cufflinks:v0.4.1"

config/fnlcr/config.yaml

@@ -112,20 +112,3 @@ fastas_gtfs_dir: "/mnt/projects/CCBR-Pipelines/db/charlie/fastas_gtfs"
 annotation_lookups:
   hg38: "PIPELINE_HOME/resources/hg38_2_hg19_lookup.txt"
   mm39: "PIPELINE_HOME/resources/mm39_circBase_annotation_lookup.txt"
-
-containers:
-  base: "docker://nciccbr/ccbr_ubuntu_base_20.04:v6"
-  bowtie1: "docker://nciccbr/charlie_bowtie1:v0.1.0"
-  circexplorer: "docker://nciccbr/ccbr_circexplorer:v1.0"
-  circRNA_finder: "docker://nciccbr/charlie_circrna_finder:v1"
-  ciri: "docker://nciccbr/charlie_ciri2:v1"
-  clear: "docker://nciccbr/ccbr_clear:2"
-  cutadapt: "docker://nciccbr/charlie_cutadapt_fqfilter:v1"
-  dcc: "docker://nciccbr/charlie_dcc:v0.2.0"
-  fastqc: "docker://nciccbr/ccrgb_qctools:v4.0"
-  mapsplice: "docker://cgrlab/mapsplice2:latest"
-  multiqc: "docker://nciccbr/ccbr_multiqc_1.15:v1"
-  picard: "docker://nciccbr/ccbr_picard_2.27.5:v1"
-  R: "docker://nciccbr/ccbr_r_4.3.0:v1"
-  star: "docker://nciccbr/ccbr_star_2.7.6a:latest"
-  star_ucsc_cufflinks: "docker://nciccbr/charlie_star_ucsc_cufflinks:v0.4.0"

docker/bowtie1/Dockerfile

@@ -12,6 +12,8 @@ ENV REPONAME=${REPONAME}
 COPY environment.txt /data2/
 RUN mamba install -c conda-forge -c bioconda --file /data2/environment.txt
 ENV R_LIBS_USER=/opt2/conda/lib/R/library/
+ENV PATH="/opt2/conda/bin/:$PATH"
+ENV PYTHONPATH=""
 
 # Save Dockerfile in the docker
 COPY Dockerfile /opt2/Dockerfile_${REPONAME}.${BUILD_TAG}

docker/bowtie1/meta.yml

@@ -1,4 +1,4 @@
 dockerhub_namespace: nciccbr
 image_name: charlie_bowtie1
-version: v0.1.0
+version: v0.1.1
 container: "$(dockerhub_namespace)/$(image_name):$(version)"

docker/circRNA_finder/Dockerfile

@@ -12,6 +12,8 @@ ENV REPONAME=${REPONAME}
 COPY environment.txt /data2/
 RUN mamba install -c conda-forge -c bioconda --file /data2/environment.txt
 ENV R_LIBS_USER=/opt2/conda/lib/R/library/
+ENV PATH="/opt2/conda/bin/:$PATH"
+ENV PYTHONPATH=""
 
 # install circRNA_finder
 WORKDIR /opt2

docker/circRNA_finder/meta.yml

@@ -1,4 +1,4 @@
 dockerhub_namespace: nciccbr
 image_name: charlie_circrna_finder
-version: v1
+version: v1.0.1
 container: "$(dockerhub_namespace)/$(image_name):$(version)"

docker/ciri2/Dockerfile

@@ -13,6 +13,7 @@ WORKDIR /opt2
 RUN wget -O CIRI.zip https://sourceforge.net/projects/ciri/files/CIRI2/CIRI_v2.0.6.zip/download && \
     unzip CIRI.zip
 ENV PATH="/opt2/CIRI2_v2.0.6/:$PATH"
+ENV PYTHONPATH="/usr/local/lib/python3.8/dist-packages/"
 
 # Save Dockerfile in the docker
 COPY Dockerfile /opt2/Dockerfile_${REPONAME}.${BUILD_TAG}

docker/ciri2/meta.yml

@@ -1,4 +1,4 @@
 dockerhub_namespace: nciccbr
 image_name: charlie_ciri2
-version: v1
+version: v1.0.1
 container: "$(dockerhub_namespace)/$(image_name):$(version)"

docker/clear/Dockerfile

@@ -11,6 +11,7 @@ ENV CONDA_ENV=clear
 RUN mamba env create -n ${CONDA_ENV} -f /data2/environment.yml && \
     echo "conda activate ${CONDA_ENV}" > ~/.bashrc
 ENV PATH="/opt2/conda/envs/${CONDA_ENV}/bin:$PATH"
+ENV PYTHONPATH="/opt2/conda/envs/${CONDA_ENV}/lib/${CONDA_ENV}/site-packages/"
 RUN python -m pip install git+https://github.com/YangLab/CLEAR.git
 RUN which circ_quant && circ_quant -h
 

docker/clear/meta.yml

@@ -1,4 +1,4 @@
 dockerhub_namespace: nciccbr
 image_name: ccbr_clear
-version: 2
+version: v2.0.1
 container: "$(dockerhub_namespace)/$(image_name):$(version)"

docker/cutadapt_fqfilter/Dockerfile

@@ -14,6 +14,7 @@ ENV CONDA_ENV=cutadapt
 RUN mamba env create -n ${CONDA_ENV} -f /data2/environment.yml && \
     echo "conda activate ${CONDA_ENV}" > ~/.bashrc
 ENV PATH="/opt2/conda/envs/${CONDA_ENV}/bin:$PATH"
+ENV PYTHONPATH="/opt2/conda/envs/${CONDA_ENV}/lib/${CONDA_ENV}/site-packages/"
 ENV R_LIBS_USER=/opt2/conda/lib/R/library/
 
 # Save Dockerfile in the docker

docker/cutadapt_fqfilter/meta.yml

@@ -1,4 +1,4 @@
 dockerhub_namespace: nciccbr
 image_name: charlie_cutadapt_fqfilter
-version: v1
+version: v1.0.1
 container: "$(dockerhub_namespace)/$(image_name):$(version)"

docker/dcc/Dockerfile

@@ -14,7 +14,9 @@ WORKDIR /opt2
 RUN wget https://github.com/dieterich-lab/DCC/archive/${DCC_VERSION}.zip -O dcc.zip && \
     unzip dcc.zip && \
     cd DCC-${DCC_VERSION} && \
+    pip install numpy==1.21.0 && \
     python setup.py install
+ENV PYTHONPATH="/usr/local/lib/python3.8/dist-packages/"
 RUN which DCC && DCC -h
 
 # Save Dockerfile in the docker

docker/dcc/meta.yml

@@ -1,4 +1,4 @@
 dockerhub_namespace: nciccbr
 image_name: charlie_dcc
-version: v0.2.0
+version: v0.2.1
 container: "$(dockerhub_namespace)/$(image_name):$(version)"

docker/star_ucsc_cufflinks/Dockerfile

@@ -14,6 +14,7 @@ ENV CONDA_ENV=py3.6
 RUN mamba env create -n ${CONDA_ENV} -f /data2/environment.yml && \
     echo "conda activate ${CONDA_ENV}" > ~/.bashrc
 ENV PATH="/opt2/conda/envs/${CONDA_ENV}/bin:$PATH"
+ENV PYTHONPATH="/opt2/conda/envs/${CONDA_ENV}/lib/${CONDA_ENV}/site-packages/"
 ENV R_LIBS_USER=/opt2/conda/lib/R/library/
 
 # install find_circ

docker/star_ucsc_cufflinks/meta.yml

@@ -1,4 +1,4 @@
 dockerhub_namespace: nciccbr
 image_name: charlie_star_ucsc_cufflinks
-version: v0.4.0
+version: v0.4.1
 container: "$(dockerhub_namespace)/$(image_name):$(version)"

workflow/Snakefile

@@ -305,7 +305,7 @@ include: "rules/post_findcircrna_processing.smk"
 
 on_finish_cmd = f"""
 sleep 10
-mkdir -p {WORKDIR}/log && run_jobby_on_snakemake_log {WORKDIR}/snakemake.log | tee {WORKDIR}/log/snakemake.log.jobby | cut -f2,3,18 > {WORKDIR}/log/snakemake.log.jobby.short
+mkdir -p {WORKDIR}/logs/ && run_jobby_on_snakemake_log {WORKDIR}/snakemake.log | tee {WORKDIR}/logs/snakemake.log.jobby | cut -f2,3,18 > {WORKDIR}/logs/snakemake.log.jobby.short
 if command -v spooker &> /dev/null; then
     spooker {WORKDIR} CHARLIE
 else

workflow/rules/align.smk

@@ -244,7 +244,7 @@ rule merge_SJ_tabs:
         """
 set -exo pipefail
 cat {input} | \\
-python {params.script1} \\
+python -E {params.script1} \\
     --regions {params.regions} \\
     --filter1regions {params.filter1regions} \\
     --filter1_noncanonical {params.filter1_noncanonical} \\

workflow/rules/create_index.smk

@@ -33,13 +33,13 @@ samtools faidx {params.reffa} && \\
 # bwa index -p ref {params.reffa} > bwa_index.log ... created in a separate rule
 
 # NCLscan files
-python {params.script3} --ingtf {params.refgtf} --outgtf {output.fixed_gtf}
+python -E {params.script3} --ingtf {params.refgtf} --outgtf {output.fixed_gtf}
 gffread -w {output.transcripts_fa} -g {params.reffa} {output.fixed_gtf}
 touch {output.lncRNA_transcripts_fa}
 create_reference.py -c {params.nclscan_config}
 
 gtfToGenePred -ignoreGroupsWithoutExons {output.fixed_gtf} ref.genes.genepred && \\
-    python {params.script1} {output.fixed_gtf} ref.genes.genepred > {output.genepred_w_geneid}
+    python -E {params.script1} {output.fixed_gtf} ref.genes.genepred > {output.genepred_w_geneid}
 
 stardir=$(dirname {output.sa})
 mkdir -p $stardir && \\
@@ -125,4 +125,4 @@ set -exo pipefail
 refdir=$(dirname {params.reffa})
 cd $refdir
 bowtie-build {params.reffa} ref
-"""  
+"""

workflow/rules/findcircrna.smk

@@ -141,7 +141,7 @@ def get_per_sample_files_to_merge(wildcards):
 # 2. parse the back_spliced_junction BED from above along with known splicing annotations to CircExplorer2 'parse' to create
 #       a. circularRNA_known.txt ... circRNAs around known gene exons
 #       b. low_conf_circularRNA_known.txt .... circRNAs with low confidence
-# 3. parse back_spliced_junction BED along with circularRNA_known.txt and low_conf_circularRNA_known.txt to custom python script
+# 3. parse back_spliced_junction BED along with circularRNA_known.txt and low_conf_circularRNA_known.txt to custom python -E script
 # to create an aggregated list of BSJs with following columns:
 # | # | ColName     |
 # |---|-------------|
@@ -305,7 +305,7 @@ perl {params.ciripl} \\
 # samtools view -@{threads} -T {params.reffa} -CS {params.sample}.bwa.sam | samtools sort -l 9 -T {params.tmpdir} --write-index -@{threads} -O CRAM -o {output.ciribam} -
 samtools view -@{threads} -bS {params.sample}.bwa.sam | samtools sort -l 9 -T {params.tmpdir} --write-index -@{threads} -O BAM -o {output.ciribam} -
 rm -rf {params.sample}.bwa.sam
-python {params.script} \\
+python -E {params.script} \\
     --ciriout {output.ciriout} \\
     --back_spliced_min_reads {params.bsj_min_nreads} \\
     --host "{params.host}" \\
@@ -411,7 +411,7 @@ rule create_ciri_count_matrix:
         """
 set -exo pipefail
 cd {params.outdir}
-python {params.script} {params.lookup} {params.hostID}
+python -E {params.script} {params.lookup} {params.hostID}
 """
 
 
@@ -441,8 +441,8 @@ rule create_circexplorer_count_matrix:
     shell:
         """
 cd {params.outdir}
-python {params.script} {params.lookup} {params.hostID}
-python {params.script2} {params.lookup} {params.hostID}
+python -E {params.script} {params.lookup} {params.hostID}
+python -E {params.script2} {params.lookup} {params.hostID}
 """
 
 
@@ -563,7 +563,7 @@ set -exo pipefail
 find {params.cleardir} -maxdepth 1 -type d -name "quant.txt*" -exec rm -rf {{}} \;
 if [[ "$(cat {input.quantfile} | wc -l)" != "0" ]]
 then
-python {params.script} {params.lookup} {input.quantfile} {params.hostID}
+python -E {params.script} {params.lookup} {input.quantfile} {params.hostID}
 else
 touch {output.annotatedquantfile}
 fi
@@ -722,10 +722,10 @@ ls -alrth {params.tmpdir}
 
 paste {output.cr} {output.linear} | cut -f1-5,9 > {params.tmpdir}/CircRNALinearCount
 
-python {params.script} \\
+python -E {params.script} \\
   --CircCoordinates {output.cc} --CircRNALinearCount {params.tmpdir}/CircRNALinearCount -o {output.ct}
 
-python {params.script2} \\
+python -E {params.script2} \\
     --in_dcc_counts_table {output.ct} \\
     --out_dcc_filtered_counts_table {output.ctf} \\
     --back_spliced_min_reads {params.bsj_min_nreads} \\
@@ -843,7 +843,7 @@ R2fn=$(basename {input.R2})
 zcat {input.R1} > {params.tmpdir}/${{R1fn%.*}}
 zcat {input.R2} > {params.tmpdir}/${{R2fn%.*}}
 
-python $MSHOME/mapsplice.py \\
+python -E $MSHOME/mapsplice.py \\
  -1 {params.tmpdir}/${{R1fn%.*}} \\
  -2 {params.tmpdir}/${{R2fn%.*}} \\
  -c {params.separate_fastas} \\
@@ -862,7 +862,7 @@ else
 R1fn=$(basename {input.R1})
 zcat {input.R1} > {params.tmpdir}/${{R1fn%.*}}
 
-python $MSHOME/mapsplice.py \
+python -E $MSHOME/mapsplice.py \
  -1 {params.tmpdir}/${{R1fn%.*}} \
  -c {params.separate_fastas} \
  -p {threads} \
@@ -934,7 +934,7 @@ rule mapsplice_postprocess:
         """
 set -exo pipefail
 mkdir -p {params.tmpdir}
-python {params.script} \\
+python -E {params.script} \\
     --circularRNAstxt {input.circRNAs} \\
     -o {output.ct} \\
     -fo {output.ctf} \\
@@ -1027,7 +1027,7 @@ results_bn=$(basename {output.result})
 if [ "{params.peorse}" == "PE" ];then
 NCLscan.py -c {params.nclscan_config} -pj {params.sample} -o {params.tmpdir} --fq1 {input.R1} --fq2 {input.R2}
 rsync -az --progress {params.tmpdir}/${{results_bn}} {output.result}
-python {params.script} \\
+python -E {params.script} \\
     --result {output.result} \\
     -o {output.ct} \\
     -fo {output.ctf} \\
@@ -1163,7 +1163,7 @@ rule find_circ:
     shell:
         """
 set -exo pipefail
-python --version
+python -E --version
 which python
 mkdir -p {params.tmpdir}
 cd {params.tmpdir}
@@ -1223,7 +1223,7 @@ grep CIRCULAR {params.tmpdir}/{params.sample}.splice_sites.bed | \\
     > {output.find_circ_bsj_bed}
 
 echo -ne "chrom\\tstart\\tend\\tname\\tn_reads\\tstrand\\tn_uniq\\tuniq_bridges\\tbest_qual_left\\tbest_qual_right\\ttissues\\ttiss_counts\\tedits\\tanchor_overlap\\tbreakpoints\\tsignal\\tstrandmatch\\tcategory\\n" > {output.find_circ_bsj_bed_filtered}
-cat {output.find_circ_bsj_bed} | python {params.collapse_script} | awk -F"\\t" -v m={params.min_reads} -v OFS="\\t" '{{if ($5>=m) {{print}}}}'  \\
+cat {output.find_circ_bsj_bed} | python -E {params.collapse_script} | awk -F"\\t" -v m={params.min_reads} -v OFS="\\t" '{{if ($5>=m) {{print}}}}'  \\
     >> {output.find_circ_bsj_bed_filtered}
 """
 
@@ -1332,7 +1332,7 @@ for f in {input};do
     fi
 done
 
-python {params.script} \\
+python -E {params.script} \\
     --counttablelist $infiles \\
     -o {output.matrix} \\
     --minreads {params.bsj_min_nreads}